diff --git a/.github/scripts/ci-affected-tasks.sh b/.github/scripts/ci-affected-tasks.sh new file mode 100755 index 00000000..dc42fe49 --- /dev/null +++ b/.github/scripts/ci-affected-tasks.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +event_name="${GITHUB_EVENT_NAME:-push}" +base_ref="${GITHUB_BASE_REF:-}" +before="${GITHUB_EVENT_BEFORE:-}" +zero_sha="0000000000000000000000000000000000000000" +sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" +output_file="${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +task_name="${AFFECTED_GRADLE_TASK:-build}" +case "$task_name" in + build|test) ;; + *) echo "Unsupported affected Gradle task: $task_name" >&2; exit 2 ;; +esac + +emit_tasks() { + if [[ -z "$1" ]]; then + return + fi + + { + echo "tasks<>"$output_file" +} + +if [[ "$event_name" != "push" && "$event_name" != "pull_request" ]]; then + emit_tasks "$task_name" + exit 0 +fi + +if [[ "$event_name" == "pull_request" ]]; then + if [[ -z "$base_ref" ]]; then + echo "Missing base ref for pull request" >&2 + exit 1 + fi + + git fetch --no-tags origin "$base_ref" + base_commit="$(git rev-parse "origin/$base_ref")" + mapfile -d '' -t changed_files < <( + git diff --name-only -z "$base_commit...$sha" + ) +elif [[ -n "$before" && "$before" != "$zero_sha" ]] && + git cat-file -e "$before^{commit}" 2>/dev/null; then + mapfile -d '' -t changed_files < <( + git diff --name-only -z "$before...$sha" + ) +elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then + mapfile -d '' -t changed_files < <( + git diff --name-only -z "HEAD^...$sha" + ) +else + emit_tasks "$task_name" + exit 0 +fi + +is_shared_change() { + case "$1" in + .github/scripts/* | .github/workflows/ci.yml | .github/workflows/coverage.yml) + return 0 + ;; + .github/workflows/openapi.yml) + return 0 + ;; + build.gradle.kts | settings.gradle.kts | gradle.properties | gradlew | gradle/*) + return 0 + ;; + openapi.yaml | openapi/* | src/*) + return 0 + ;; + Dockerfile) + return 0 + ;; + esac + + return 1 +} + +is_ignored_change() { + case "$1" in + *.md | *.png | *.svg | .gitignore | .editorconfig | LICENSE*) + return 0 + ;; + docs/* | .github/ISSUE_TEMPLATE/*) + return 0 + ;; + esac + + return 1 +} + +full_build=false +declare -A changed_modules=() + +for file in "${changed_files[@]}"; do + if is_ignored_change "$file"; then + continue + fi + + if is_shared_change "$file"; then + full_build=true + continue + fi + + if [[ "$file" == server-*/* ]]; then + module="${file%%/*}" + changed_modules["$module"]=1 + else + full_build=true + fi +done + +if [[ "$full_build" == true ]]; then + emit_tasks "$task_name" + exit 0 +fi + +if [[ ${#changed_modules[@]} -eq 0 ]]; then + exit 0 +fi + +declare -A dependents_by_module=() +for build_file in server-*/build.gradle.kts; do + module="${build_file%/*}" + while IFS= read -r line; do + if [[ "$line" =~ project\(\":([A-Za-z0-9_-]+)\"\) ]]; then + dependency="${BASH_REMATCH[1]}" + dependents_by_module["$dependency"]+="$module " + fi + done <"$build_file" +done + +declare -A affected_modules=() + +mark_affected() { + local module="$1" + + if [[ -n "${affected_modules[$module]:-}" ]]; then + return + fi + + affected_modules["$module"]=1 + + local dependent + for dependent in ${dependents_by_module["$module"]:-}; do + mark_affected "$dependent" + done +} + +for module in "${!changed_modules[@]}"; do + if [[ ! -f "$module/build.gradle.kts" ]]; then + echo "Changed module no longer exists: $module" >&2 + emit_tasks "$task_name" + exit 0 + fi + + mark_affected "$module" +done + +module_tasks="$( + for module in "${!affected_modules[@]}"; do + printf ':%s:%s\n' "$module" "$task_name" + done | sort +)" + +emit_tasks "$module_tasks" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c62a2a2..229ca0fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","ci-stack"]') }} steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" @@ -29,5 +31,19 @@ jobs: - uses: gradle/actions/setup-gradle@v6 - - name: Build - run: ./gradlew build --max-workers=8 + - name: Detect affected Gradle modules + id: affected + run: bash .github/scripts/ci-affected-tasks.sh + + - name: Build affected modules + env: + TASKS: ${{ steps.affected.outputs.tasks }} + run: | + if [[ -z "$TASKS" ]]; then + echo "No Gradle compilation or tests are required." + exit 0 + fi + + readarray -t task_args <<<"$TASKS" + printf 'Running:\n%s\n' "$TASKS" + ./gradlew "${task_args[@]}" --parallel --max-workers=32 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0d3bc5a9..c34c49e6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,6 +16,8 @@ jobs: runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","ci-stack"]') }} steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" @@ -28,18 +30,35 @@ jobs: - uses: gradle/actions/setup-gradle@v6 - - name: Run tests and coverage report - run: ./gradlew test jacocoTestReport jacocoTestCoverageVerification + - name: Detect affected test modules + id: affected + env: + AFFECTED_GRADLE_TASK: test + run: bash .github/scripts/ci-affected-tasks.sh + + - name: Run affected tests and coverage report + if: steps.affected.outputs.tasks != '' + env: + TASKS: ${{ steps.affected.outputs.tasks }} + run: | + readarray -t task_args <<<"$TASKS" + printf 'Running affected tests:\n%s\n' "$TASKS" + ./gradlew "${task_args[@]}" --parallel --max-workers=32 + ./gradlew jacocoTestReport jacocoTestCoverageVerification --parallel --max-workers=32 + + - name: No coverage required + if: steps.affected.outputs.tasks == '' + run: echo "No source or build changes require coverage." - name: Upload coverage HTML report - if: always() + if: always() && steps.affected.outputs.tasks != '' uses: actions/upload-artifact@v7 with: name: jacoco-html-report path: build/reports/jacoco/test/html - name: Upload coverage XML report - if: always() + if: always() && steps.affected.outputs.tasks != '' uses: actions/upload-artifact@v7 with: name: jacoco-xml-report diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a2f9b90c..5c9f44b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,10 @@ on: branches: - main - dev + paths-ignore: + - "**.md" + - "docs/**" + - ".github/ISSUE_TEMPLATE/**" tags: - "v*" workflow_dispatch: @@ -29,6 +33,7 @@ jobs: image: ${{ steps.build-info.outputs.image }} labels: ${{ steps.meta.outputs.labels }} metadata-json: ${{ steps.meta.outputs.json }} + platform-matrix: ${{ steps.build-info.outputs.platform-matrix }} version: ${{ steps.build-info.outputs.version }} steps: @@ -50,10 +55,16 @@ jobs: else image="${REGISTRY}/${GITHUB_REPOSITORY,,}" fi + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + platform_matrix='{"include":[{"platform":"linux/amd64","arch":"amd64"}]}' + else + platform_matrix='{"include":[{"platform":"linux/amd64","arch":"amd64"},{"platform":"linux/arm64","arch":"arm64"}]}' + fi { echo "image=$image" echo "version=$version" echo "build-time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + echo "platform-matrix=$platform_matrix" } >> "$GITHUB_OUTPUT" - name: Extract metadata @@ -83,12 +94,7 @@ jobs: packages: write strategy: fail-fast: false - matrix: - include: - - platform: linux/amd64 - arch: amd64 - - platform: linux/arm64 - arch: arm64 + matrix: ${{ fromJSON(needs.prepare-image.outputs.platform-matrix) }} steps: - name: Checkout @@ -191,8 +197,12 @@ jobs: run: | cd "$DIGEST_DIR" mapfile -t digests < <(find . -maxdepth 1 -type f -printf '%f\n' | sort) - if [[ "${#digests[@]}" -ne 2 ]]; then - echo "Expected two platform digests, found ${#digests[@]}" + expected_digest_count=2 + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + expected_digest_count=1 + fi + if [[ "${#digests[@]}" -ne "$expected_digest_count" ]]; then + echo "Expected $expected_digest_count platform digest(s), found ${#digests[@]}" exit 1 fi @@ -227,8 +237,13 @@ jobs: VERSION: ${{ needs.prepare-image.outputs.version }} run: | platforms="$(docker buildx imagetools inspect "$IMAGE:$VERSION" --format '{{json .Manifest}}' | jq -r '.manifests[].platform | "\(.os)/\(.architecture)"' | sort -u)" - if [[ "$platforms" != $'linux/amd64\nlinux/arm64' ]]; then - printf 'Unexpected manifest platforms:\n%s\n' "$platforms" + expected_platforms=$'linux/amd64\nlinux/arm64' + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + expected_platforms='linux/amd64' + fi + if [[ "$platforms" != "$expected_platforms" ]]; then + printf 'Unexpected manifest platforms:\n%s\nExpected:\n%s\n' \ + "$platforms" "$expected_platforms" exit 1 fi diff --git a/Dockerfile b/Dockerfile index db8e4f19..ece4c2c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM --platform=$BUILDPLATFORM eclipse-temurin:25-jdk-alpine AS builder ARG APP_VERSION=0.1.0 ARG GITHUB_SHA=unknown ARG BUILD_TIME=unknown +ARG GRADLE_MAX_WORKERS=32 ENV GITHUB_SHA=$GITHUB_SHA ENV BUILD_TIME=$BUILD_TIME WORKDIR /app @@ -9,9 +10,23 @@ COPY gradlew ./ COPY gradle/ ./gradle/ RUN ./gradlew --version --no-daemon -q COPY build.gradle.kts gradle.properties settings.gradle.kts* ./ -RUN ./gradlew dependencies --no-daemon -q || true COPY src/ ./src/ -RUN ./gradlew shadowJar --no-daemon -q -PappVersion="$APP_VERSION" +COPY server-admin/ ./server-admin/ +COPY server-auth/ ./server-auth/ +COPY server-cache/ ./server-cache/ +COPY server-core/ ./server-core/ +COPY server-db/ ./server-db/ +COPY server-domain/ ./server-domain/ +COPY server-downloader/ ./server-downloader/ +COPY server-http/ ./server-http/ +COPY server-playback/ ./server-playback/ +COPY server-portability/ ./server-portability/ +COPY server-sabr/ ./server-sabr/ +COPY server-test-support/ ./server-test-support/ +COPY server-services/ ./server-services/ +COPY server-token-gateway/ ./server-token-gateway/ +RUN ./gradlew dependencies --no-daemon --parallel --max-workers="$GRADLE_MAX_WORKERS" -q || true +RUN ./gradlew shadowJar --no-daemon --parallel --max-workers="$GRADLE_MAX_WORKERS" -q -PappVersion="$APP_VERSION" FROM eclipse-temurin:25-jre-alpine AS runner RUN apk upgrade --no-cache \ diff --git a/PERF.md b/PERF.md new file mode 100644 index 00000000..943baef6 --- /dev/null +++ b/PERF.md @@ -0,0 +1,81 @@ +# Playback Startup Performance + +## Target + +The target is less than 3 seconds from selecting a video on `beta.typetype.video` to its first rendered video frame. The browser trace's `playback.first_frame` event measures this interval from the click trace; a direct watch-page load starts at route entry instead. + +Two prewarmed Beta browser samples below are under 3 seconds. This does not establish cold-start behavior or a percentile target. Do not infer a pass from a cached thumbnail or `loadedmetadata`; require `first_frame`. + +## PipePipe Reference + +The checked upstreams are the official `InfinityLoop1308` repositories: [PipePipe Client `c2a166f7`](https://github.com/InfinityLoop1308/PipePipeClient/commit/c2a166f7df05e5ace4c80fb3e3fd27702cfc3e56), declared version `5.4.0`, and [PipePipeExtractor `c68e10e2`](https://github.com/InfinityLoop1308/PipePipeExtractor/commit/c68e10e2e97495877832d8df6cbac55478083019), version `5.4.0`. In Client, `SabrDashMediaSource` calls `SabrMediaBridge.prepareTimelines`; the bridge requests initial audio and video timelines together and retains returned media. Its `PlaybackStartupTrace` marks production player phases, while `SabrPlaybackSmokeTest` and `YoutubePlaybackBenchmarkTest` provide Android test instrumentation. Those harnesses do not measure TypeType's browser or Beta backend. Extractor owns the UMP/SABR session and request logic; its [SABR research log](https://github.com/InfinityLoop1308/PipePipeExtractor/issues/66) is issue-based rather than a standalone current guide. + +TypeType's relevant difference is its separate HTTP orchestration: browser requests for stream metadata/bootstrap, Server-side Token resolution, a prewarm-to-consumer session handoff, then browser MSE requests for init and segments. The PipePipe sources are a protocol and instrumentation reference, not evidence that TypeType has the same startup profile or that an upstream change is needed. + +## Enable Browser Trace + +Before starting a video, open the browser developer console on Beta and run: + +```js +localStorage.setItem("typetype-debug-console", "1"); +location.reload(); +``` + +Start a video from its card. In the console, collect all `[typetype] playback.*` records for the `traceId` emitted by `playback.trace_start`, through `playback.first_frame`. Remove the setting with `localStorage.removeItem("typetype-debug-console")` to stop tracing. Browser records are kept in session storage for up to 30 minutes, capped at 800 playback entries. + +## Correlated Logs + +The same trace ID is sent on selected stream, comments, subtitle, SABR prewarm, and SABR MSE requests. Server echoes it and forwards it, with the request ID, to Token. Filter Server and Token runtime logs for `[playback_trace]` and the same `traceId`. + +Browser events include API start/end/status/duration; video readiness/playback and first-frame callbacks; same-origin SABR resource timing/transfer sizes; and supported long-task observations. MSE events include `mse_state`, `mse_manifest` (generation and segment count), `mse_quality`, `mse_segment_appended` (track, init/media phase, start and duration), `mse_buffer`, `mse_seek`, and `mse_error`. Segment URLs are omitted. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation, and `sabr_init_preload_start`/`sabr_init_preload_complete` with per-track duration, result and byte count. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. + +Compare the browser's full API duration with the Server request duration and the nested Token phases. This separates browser/MSE waiting, Server work, and Token work without treating a fast response header as a rendered frame. + +Detailed records are emitted only for requests carrying a valid trace ID. They do not include bearer tokens, cookies, PO-token values, session bindings, request bodies, or signed query strings. Subtitle timing is observational; this harness does not make subtitle loading a prerequisite for video playback. + +## Measurements (2026-09-25) + +The public Beta instance reported Server revision `d596c984`. With the same video and fresh browser contexts, both runs used the SABR prewarm path: + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,731 ms | 1,759 ms | 3.94 s | Pass | +| Firefox | 2,114 ms | 2,159 ms | 4.03 s | Pass | + +Both rendered a 1920x1080 frame, continued playing without a media error, and had about 5.34 seconds buffered at `loadeddata`, growing to about 10.68 seconds. Two text tracks were attached and zero were visible at first frame in both runs. This demonstrates that subtitles did not block these starts; it does not establish behavior for every subtitle format or cold start. + +These are two prewarmed samples, not cold-start measurements or a percentile. Browser SABR stream-info calls took 369 ms in Chromium and 50 ms in Firefox; the measured SABR create requests took 94/10 ms and 36/26 ms respectively (prewarm/consumer). Segment requests completed in 25-38 ms in Chromium and 26-66 ms in Firefox. + +A later card-click sample on video `ynKvUYpu1Qw` measured 1,892 ms to first frame in Chromium and 1,913 ms in Firefox. Playback started at 1,908/1,941 ms, continued for 3.94/4.01 seconds, and rendered 1920x1080 without a media error. Chromium's stream-info call took 1,291 ms and bootstrap took 172 ms; session creation took 318 ms for the consumer and 139 ms for prewarm. Firefox ran after Chromium had used the same video: stream-info took 43 ms, session calls 29-30 ms, and init calls 26-27 ms. No text tracks were attached at first frame. Firefox was cache-warm, and Chromium's origin-side cache state was not measured, so these results do not prove a cold-start pass. + +The browser MSE trace was also captured for `GlosP5N7DkA` in fresh Chromium and Firefox contexts. A separate `stream_resolve` request was active around navigation from search results, and the click path used SABR prewarm plus session handoff; these are not cold-start measurements. + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,678 ms | 1,713 ms | 3.96 s | Pass | +| Firefox | 1,539 ms | 1,549 ms | 4.04 s | Pass | + +Both rendered a 1920x1080 frame with no media error or intercepted pause call. Firefox's click trace recorded the MSE manifest ready at 1,436 ms, `loadeddata` at 1,512 ms, and `first_frame` at 1,541 ms; its bootstrap trace duration was 284 ms and the prewarm/consumer session requests took 26/25 ms. Both traces showed two text tracks and zero visible tracks at first frame. This confirms a sub-3-second start for these prewarmed samples, not every subtitle format or a cold start. + +The TypeType SABR extractor configuration disables supplemental subtitle lookup. This shows subtitles did not gate this start, not that every subtitle path is non-blocking. + +A separate cold direct Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. + +The direct Beta request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Beta Server or Token logs, so public-runtime log correlation is not yet verified. The new trace-gated Server events split provider setup, page fetch, stream extraction, SponsorBlock, supplemental subtitles, authenticated resolution, SABR bootstrap metadata, format preparation, and per-track initialization preload. Verify they appear with the browser trace ID after the change reaches Beta before attributing public latency to a backend phase. + +## Latest Beta Spot Check (2026-09-25) + +Fresh browser contexts on `jNQXAC9IVRw` used card-click SABR prewarm and rendered 320x240 frames: + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,464 ms | 1,464 ms | 3.93 s of 4 s | Pass | +| Firefox | 1,494 ms | 1,529 ms | 3.99 s of 4 s | Pass | + +Neither browser reported a media error. In Firefox, stream/bootstrap resolution took 181 ms, the full stream-info path 318 ms (including a concurrent 163 ms `stream_resolve`), session creation took 26 ms per request, init responses took 24-25 ms, and the first video/audio segment responses took 25/37 ms. Comments took about 941 ms and completed before the first frame, so they did not gate this sample. Two text tracks were attached, neither visible; no subtitle request was observed on the first-frame critical path. + +These were prewarmed card-click samples; fresh browser contexts do not establish a cold Server/cache state. The observed 240p output is not evidence of high-resolution startup or a general latency distribution. The earlier network logs include SABR create calls taking about 5.0-11.3 seconds, including a request selecting itag 299; without matching trace IDs and Server/Token phase records, those outliers cannot be attributed to Token or Server. The new per-track initialization events should show whether the synchronous init preloader is on the delay path once deployed and correlated. The under-3-second cold-start goal remains unverified. + +## Acceptance + +A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The earlier samples and latest spot checks pass individually. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. diff --git a/README.md b/README.md index 0886b3b9..f3c9420e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@

Extraction, API, and private user data backend for TypeType.

+You want to know the current position of TypeType about AI ? Go check [this](https://github.com/TypeType-Video/TypeType/blob/dev/AI_TRANSPARENCY.md). + TypeType-Server is the Kotlin/Ktor HTTP API behind TypeType. It wraps [PipePipeExtractor](https://github.com/InfinityLoop1308/PipePipeExtractor) for supported media services and stores instance and user data in PostgreSQL. If you want to run a complete TypeType instance, use the [central stack](https://github.com/TypeType-Video/TypeType) rather than deploying this service alone. diff --git a/build.gradle.kts b/build.gradle.kts index fe90f2be..992461b4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,6 +25,19 @@ repositories { } dependencies { + implementation(project(":server-admin")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-core")) + implementation(project(":server-db")) + implementation(project(":server-downloader")) + implementation(project(":server-domain")) + implementation(project(":server-portability")) + implementation(project(":server-playback")) + implementation(project(":server-services")) + implementation(project(":server-http")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) @@ -44,19 +57,22 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") - implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.4.0") - implementation("com.password4j:password4j:1.8.4") - implementation("com.auth0:java-jwt:4.6.1") + testImplementation(project(":server-test-support")) + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("com.password4j:password4j:1.8.4") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") @@ -66,46 +82,6 @@ dependencies { testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") testImplementation("org.testcontainers:testcontainers-junit-jupiter:2.0.5") } -val buildInfoVersion = version.toString().trim().takeUnless { it.isBlank() || it == "unspecified" } ?: "0.0.0-dev" -fun gitRevisionOrUnknown(): String = runCatching { - providers.exec { commandLine("git", "rev-parse", "HEAD") } - .standardOutput - .asText - .get() - .trim() - .ifBlank { "unknown" } -}.getOrElse { "unknown" } - -val buildInfoRevision = providers.environmentVariable("GITHUB_SHA") - .map { it.trim().ifBlank { "unknown" } } - .getOrElse(gitRevisionOrUnknown()) -val buildInfoShortRevision = buildInfoRevision.takeIf { it != "unknown" }?.take(12) ?: "unknown" -val buildInfoBuildTime = providers.environmentVariable("BUILD_TIME") - .orElse(providers.provider { Instant.now().toString() }) - .get() -val generatedBuildInfoDir = layout.buildDirectory.dir("generated/sources/buildInfo/main") -val generateBuildInfo = tasks.register("generateBuildInfo") { - inputs.property("version", buildInfoVersion) - inputs.property("revision", buildInfoRevision) - inputs.property("shortRevision", buildInfoShortRevision) - inputs.property("buildTime", buildInfoBuildTime) - outputs.dir(generatedBuildInfoDir) - doLast { - val output = generatedBuildInfoDir.get().file("dev/typetype/server/BuildInfo.kt").asFile - output.parentFile.mkdirs() - output.writeText(""" - package dev.typetype.server - - object BuildInfo { - const val VERSION: String = "${buildInfoVersion.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val REVISION: String = "${buildInfoRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val SHORT_REVISION: String = "${buildInfoShortRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val BUILD_TIME: String = "${buildInfoBuildTime.replace("\\", "\\\\").replace("\"", "\\\"")}" - } - """.trimIndent()) - } -} - tasks.test { useJUnitPlatform { // network-tagged tests are off by default. Flip on for the live SABR probe: @@ -132,19 +108,6 @@ tasks.jacocoTestReport { } } -tasks.jacocoTestCoverageVerification { - dependsOn(tasks.test) - violationRules { - rule { - limit { - counter = "LINE" - value = "COVEREDRATIO" - minimum = "0.20".toBigDecimal() - } - } - } -} - val verifySabrBoundary = tasks.register("verifySabrBoundary") { doLast { val adapterRoot = file("src/main/kotlin/dev/typetype/server/sabr") @@ -166,17 +129,13 @@ val verifySabrBoundary = tasks.register("verifySabrBoundary") { } tasks.check { - dependsOn(tasks.jacocoTestCoverageVerification) dependsOn(verifySabrBoundary) } kotlin { jvmToolchain(25) - sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } } -tasks.named("compileKotlin") { dependsOn(generateBuildInfo) } - tasks.withType().configureEach { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } diff --git a/gradle.properties b/gradle.properties index 3e32a72b..3daee93a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.8.1 +appVersion=1.9.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index 72d2cc37..71741dd6 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -22,6 +22,7 @@ paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } /streams/youtube/sabr: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrStreams } + /streams/youtube/live: { $ref: ./openapi/paths/streams.yaml#/YoutubeLiveHls } /streams/youtube/sabr/bootstrap: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrBootstrap } /streams/niconico: { $ref: ./openapi/paths/streams.yaml#/NicoNicoStreams } /streams/bilibili: { $ref: ./openapi/paths/streams.yaml#/BiliBiliStreams } @@ -31,6 +32,7 @@ paths: /proxy: { $ref: ./openapi/paths/proxy.yaml#/Proxy } /media/{handle}: { $ref: ./openapi/paths/media.yaml#/ProviderMedia } /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } + /sabr/playback/{videoId}: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackCreate } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } /sabr/playback/{sessionId}/prefetch: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPrefetch } @@ -47,6 +49,8 @@ paths: /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } /subscriptions/group-memberships: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupMemberships } + /subscriptions/group-memberships/page: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Page } + /subscriptions/group-memberships/lookup: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Lookup } /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } @@ -69,6 +73,9 @@ paths: /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /progress/batch: { $ref: ./openapi/paths/progress.yaml#/ProgressBatch } + /presence/keys: { $ref: ./openapi/paths/presence.yaml#/PresenceKeys } + /presence/keys/{id}: { $ref: ./openapi/paths/presence.yaml#/PresenceKeyRoute } + /presence/now-playing: { $ref: ./openapi/paths/presence.yaml#/PresenceNowPlaying } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats } diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 5e3d7d25..94554bdc 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -10,7 +10,18 @@ SettingsItem: maximum: 4 default: 1 defaultLandingPage: { type: string, default: home } + videoGridColumns: + type: integer + enum: [0, 4, 5, 6] + default: 0 + relatedVideoSize: { type: string, enum: [default, large], default: default } autoplay: { type: boolean, default: true } + autoplayOnOpen: { type: boolean, default: true } + autoplayCountdownSeconds: + type: integer + minimum: 0 + maximum: 60 + default: 10 volume: { type: number, format: double, default: 1.0 } muted: { type: boolean, default: false } notificationPopupsEnabled: { type: boolean, default: true } diff --git a/openapi/components/common.yaml b/openapi/components/common.yaml index d40be441..e6193b47 100644 --- a/openapi/components/common.yaml +++ b/openapi/components/common.yaml @@ -8,7 +8,16 @@ ErrorResponse: required: [error] properties: error: { type: string } - code: { type: string, default: error } + code: + type: string + default: error + description: >- + Stable machine-readable error code when available. Playback extraction + routes may return geographic_restriction, provider_access_blocked, + private_content, no_playable_streams, age_restricted, scheduled_premiere, + live_not_started, content_unavailable, upstream_failure, + youtube_session_rejected, youtube_session_needs_reconnect, paid_content, + members_only, or youtube_session_required. requestId: { type: string, nullable: true } RequestIdHeader: description: Request correlation id returned by TypeType-Server. diff --git a/openapi/components/portability.yaml b/openapi/components/portability.yaml index 05302336..04351601 100644 --- a/openapi/components/portability.yaml +++ b/openapi/components/portability.yaml @@ -87,6 +87,11 @@ PortabilityJobProgress: unit: { type: string, enum: [records, categories, bytes] } processed: { type: integer, format: int64, minimum: 0 } total: { type: integer, format: int64, minimum: 0, nullable: true } + category: { allOf: [{ $ref: '#/PortabilityCategory' }], nullable: true } + stage: { type: string, enum: [subscriptions, recentHistory, playlists, history, remaining], nullable: true } + stageProcessed: { type: integer, format: int64, minimum: 0 } + stageTotal: { type: integer, format: int64, minimum: 0, nullable: true } + checkpoint: { type: integer, format: int64, minimum: 0 } PortabilityJobReport: type: object required: [id, state] diff --git a/openapi/components/sabr-playback.yaml b/openapi/components/sabr-playback.yaml index 6a46b981..382e84ee 100644 --- a/openapi/components/sabr-playback.yaml +++ b/openapi/components/sabr-playback.yaml @@ -1,3 +1,20 @@ +SabrPlaybackCreateResponse: + type: object + required: [sessionId, videoId, videoItag, audioItag, startTimeMs, generation, ready, status] + additionalProperties: true + properties: + sessionId: { type: string } + videoId: { type: string } + manifestUrl: { type: string, nullable: true } + videoItag: { type: integer, minimum: 1 } + audioItag: { type: integer, minimum: 1 } + audioTrackId: { type: string, nullable: true } + startTimeMs: { type: integer, format: int64, minimum: 0 } + generation: { type: integer, format: int64, minimum: 0 } + ready: { type: boolean } + status: { type: string } + retryAfterMs: { type: integer, format: int64, nullable: true } + live: { type: object, nullable: true, additionalProperties: true } SabrPlaybackBufferedRange: type: object required: [itag, startMs, endMs] diff --git a/openapi/components/streams.yaml b/openapi/components/streams.yaml index 4ef4c5e2..1740e02b 100644 --- a/openapi/components/streams.yaml +++ b/openapi/components/streams.yaml @@ -59,7 +59,7 @@ SubtitleItem: isAutoGenerated: { type: boolean } StreamResponse: type: object - required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, isLive, isPostLive, isLiveContent, hasLiveManifest, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] + required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, isLive, isPostLive, isLiveContent, hasLiveManifest, hlsUrl, dashMpdUrl, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] properties: id: { type: string } title: { type: string } @@ -80,6 +80,8 @@ StreamResponse: isPostLive: { type: boolean } isLiveContent: { type: boolean } hasLiveManifest: { type: boolean } + hlsUrl: { type: string, description: HLS manifest URL, signed for server proxying when needed } + dashMpdUrl: { type: string, description: DASH manifest URL } isShortFormContent: { type: boolean } originalAudioTrackId: { type: string, nullable: true } preferredDefaultAudioTrackId: { type: string, nullable: true } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index d0af4c01..9117764b 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -18,6 +18,19 @@ SubscriptionGroupMembershipItem: type: array uniqueItems: true items: { type: string, format: uuid } +SubscriptionMembershipPage: + type: object + required: [items, total, totalSubscriptions, ungroupedCount, page, limit] + properties: + items: + type: array + maxItems: 100 + items: { $ref: '#/SubscriptionGroupMembershipItem' } + total: { type: integer, format: int64, minimum: 0, description: Number of subscriptions matching all filters. } + totalSubscriptions: { type: integer, format: int64, minimum: 0, description: Unfiltered account subscription count. } + ungroupedCount: { type: integer, format: int64, minimum: 0, description: Unfiltered account count with no memberships. } + page: { type: integer, minimum: 0, maximum: 1000000 } + limit: { type: integer, minimum: 1, maximum: 100 } SubscriptionCreateRequest: type: object required: [channelUrl, name, avatarUrl] diff --git a/openapi/paths/presence.yaml b/openapi/paths/presence.yaml new file mode 100644 index 00000000..a22187b2 --- /dev/null +++ b/openapi/paths/presence.yaml @@ -0,0 +1,111 @@ +PresenceKeys: + post: + tags: [user-data] + summary: Create a presence-only access key + description: The returned token is displayed once and grants only the presence:read scope. + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/PresenceKeyName' } + responses: + '201': + description: Created key metadata and the one-time token. + content: + application/json: + schema: { $ref: '#/PresenceKeyCreated' } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + get: + tags: [user-data] + summary: List presence access keys + security: [{ bearerAuth: [] }] + responses: + '200': + description: Key metadata without token material. + content: + application/json: + schema: + type: array + items: { $ref: '#/PresenceKey' } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + +PresenceKeyRoute: + delete: + tags: [user-data] + summary: Revoke a presence access key + security: [{ bearerAuth: [] }] + parameters: + - name: id + in: path + required: true + schema: { type: string, format: uuid } + responses: + '204': { description: Key revoked. } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + +PresenceNowPlaying: + get: + tags: [user-data] + summary: Read the user's current playback presence + description: Requires a dedicated presence key. Clients normally poll every 15 to 30 seconds. + security: [{ bearerAuth: [] }] + responses: + '200': + description: Current presence, including an explicit inactive state. + content: + application/json: + schema: { $ref: '#/PresenceResponse' } + '401': { $ref: ../components/common.yaml#/JsonError } + +PresenceKeyName: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 50 } + +PresenceKeyCreated: + type: object + required: [key, token] + properties: + key: { $ref: '#/PresenceKey' } + token: { type: string, description: One-time presence token in ttp1_ form. } + +PresenceKey: + type: object + required: [id, name, tokenPrefix, scope, createdAt] + properties: + id: { type: string, format: uuid } + name: { type: string } + tokenPrefix: { type: string } + scope: { type: string, enum: [presence:read] } + createdAt: { type: integer, format: int64 } + lastUsedAt: { type: integer, format: int64, nullable: true } + +PresenceMedia: + type: object + required: [videoUrl, title, positionMs, paused, updatedAt] + properties: + videoUrl: { type: string } + title: { type: string } + thumbnail: { type: string, nullable: true } + channelName: { type: string, nullable: true } + positionMs: { type: integer, format: int64, minimum: 0 } + durationMs: { type: integer, format: int64, minimum: 0, nullable: true } + paused: { type: boolean } + updatedAt: { type: integer, format: int64 } + +PresenceResponse: + type: object + required: [active, retryAfterMs, serverTimeMs] + properties: + active: { type: boolean } + nowPlaying: { $ref: '#/PresenceMedia' } + retryAfterMs: { type: integer, format: int64, minimum: 0 } + serverTimeMs: { type: integer, format: int64 } diff --git a/openapi/paths/sabr-playback.yaml b/openapi/paths/sabr-playback.yaml index aa281000..00e21ceb 100644 --- a/openapi/paths/sabr-playback.yaml +++ b/openapi/paths/sabr-playback.yaml @@ -1,3 +1,49 @@ +SabrPlaybackCreate: + post: + tags: [playback] + summary: Create a SABR session or transfer a matching prewarmed session + parameters: + - name: videoId + in: path + required: true + schema: { type: string } + - name: videoItag + in: query + schema: { type: integer, minimum: 1 } + - name: audioItag + in: query + schema: { type: integer, minimum: 1 } + - name: audioTrackId + in: query + schema: { type: string } + - name: startTimeMs + in: query + schema: { type: integer, format: int64, minimum: 0, default: 0 } + - name: audioOnly + in: query + schema: { type: boolean, default: false } + - name: isLive + in: query + schema: { type: boolean, default: false } + - name: prewarm + in: query + description: Keep this session briefly for the next matching creation by the same account. + schema: { type: boolean, default: false } + responses: + '200': + description: Playback session created or transferred. + content: + application/json: + schema: + $ref: ../components/sabr-playback.yaml#/SabrPlaybackCreateResponse + '202': + description: Playback session is still preparing. + content: + application/json: + schema: + $ref: ../components/sabr-playback.yaml#/SabrPlaybackCreateResponse + '422': + $ref: ../components/common.yaml#/JsonError SabrPlaybackPosition: post: tags: [playback] diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index a0bab142..dc3d2aff 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -1,7 +1,7 @@ YoutubeSabrStreams: get: tags: [extraction] - summary: Extract YouTube SABR stream metadata + summary: Resolve YouTube playback metadata with HLS for live streams and SABR for VOD parameters: - name: url in: query @@ -9,7 +9,7 @@ YoutubeSabrStreams: schema: { type: string } responses: '200': - description: Stream metadata containing only SABR sources. + description: Live streams contain only an HLS manifest; VOD responses contain only SABR sources. headers: X-Request-ID: $ref: ../components/common.yaml#/RequestIdHeader @@ -27,12 +27,107 @@ YoutubeSabrStreams: $ref: ../components/common.yaml#/JsonError '403': $ref: ../components/common.yaml#/JsonError - '422': + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse +YoutubeLiveHls: + get: + tags: [extraction] + summary: Resolve a known YouTube live stream directly to HLS + parameters: + - name: url + in: query + required: true + schema: { type: string } + responses: + '200': + description: Active live streams contain only an HLS manifest and no SABR or DASH sources. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/streams.yaml#/StreamResponse + '400': + $ref: ../components/common.yaml#/JsonError + '401': $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse YoutubeSabrBootstrap: get: tags: [extraction] - summary: Resolve playable SABR audio and video formats before full metadata + summary: Resolve early playback metadata with HLS for live streams and SABR formats for VOD parameters: - name: url in: query @@ -40,7 +135,7 @@ YoutubeSabrBootstrap: schema: { type: string } responses: '200': - description: Minimal stream metadata containing only SABR sources. + description: Live streams contain only an HLS manifest; VOD responses contain only SABR sources. headers: X-Request-ID: $ref: ../components/common.yaml#/RequestIdHeader @@ -58,8 +153,42 @@ YoutubeSabrBootstrap: $ref: ../components/common.yaml#/JsonError '403': $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse NicoNicoStreams: get: tags: [extraction] diff --git a/openapi/paths/subscription-membership-pages.yaml b/openapi/paths/subscription-membership-pages.yaml new file mode 100644 index 00000000..18025f48 --- /dev/null +++ b/openapi/paths/subscription-membership-pages.yaml @@ -0,0 +1,72 @@ +Page: + get: + tags: [user-data] + summary: Read a bounded page of subscriptions with group memberships + description: | + Filtering, counting and pagination run in the database before channel rows and their + memberships are loaded. Rows are ordered by lowercase name and then canonical channel URL. + The zero-based page is an offset into current results, not a snapshot; changes between + requests may move rows. A page beyond the last returns empty items with the current total. + Group IDs and results are scoped to the authenticated profile. Unknown or inaccessible + groupId returns subscription_group_not_found. Omit groupId and ungrouped for all subscriptions. + parameters: + - name: page + in: query + schema: { type: integer, minimum: 0, maximum: 1000000, default: 0 } + - name: limit + in: query + schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + - name: search + in: query + description: Case-insensitive literal substring of channel name or URL. Whitespace is trimmed; SQL wildcard characters are literal. + schema: { type: string, maxLength: 200 } + - name: groupId + in: query + schema: { type: string, format: uuid } + - name: ungrouped + in: query + description: Select channels in no group. Cannot be combined with groupId. + schema: { type: boolean, default: false } + - name: excluded + in: query + description: Select channels outside groupId, including channels in other groups. Requires groupId. + schema: { type: boolean, default: false } + responses: + '200': + description: Requested page and counts, including all memberships for each returned channel. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionMembershipPage } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +Lookup: + post: + tags: [user-data] + summary: Refresh memberships for explicitly selected subscriptions + description: | + Read-only lookup for selections retained across pages. Accepts one channelUrl or up to + 500 channelUrls, with a 2048-character URL limit and a 1 MiB request-body limit. + URLs are canonicalized and deduplicated. Returns only existing subscriptions owned by + the authenticated profile, with all their memberships; missing channels are omitted. + Results have no guaranteed order. This endpoint does not change subscriptions or groups. + requestBody: + required: true + content: + application/json: + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } + responses: + '200': + description: Current memberships for the selected channels still subscribed to. + content: + application/json: + schema: + type: array + maxItems: 500 + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts new file mode 100644 index 00000000..24207a8c --- /dev/null +++ b/server-admin/build.gradle.kts @@ -0,0 +1,45 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-services")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-downloader")) + implementation(project(":server-playback")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("org.slf4j:slf4j-api:2.0.16") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt index 5fdc23b5..2c26ac21 100644 --- a/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt @@ -4,7 +4,7 @@ import io.ktor.http.HttpHeaders import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond -internal suspend fun ApplicationCall.respondNoStore(value: Any): Unit { +suspend fun ApplicationCall.respondNoStore(value: Any): Unit { response.headers.append(HttpHeaders.CacheControl, "no-store, no-cache, must-revalidate, max-age=0", safeOnly = false) respond(value) } diff --git a/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt index 92c4dee1..dcbf95f2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.routes import io.ktor.server.application.ApplicationCall import io.ktor.server.request.queryString -internal fun ApplicationCall.urlTailParameter(name: String): String? { +fun ApplicationCall.urlTailParameter(name: String): String? { val pathUrl = parameters.getAll(name)?.joinToString("/") ?: return null val normalizedPathUrl = pathUrl.withUrlSchemeSlashes() val queryString = request.queryString() diff --git a/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt new file mode 100644 index 00000000..30d3052e --- /dev/null +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.RssFeedException +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +suspend fun ApplicationCall.respondRssError(error: RssFeedException) { + val status = when (error.code) { + "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound + "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden + "rss_feed_limit_reached" -> HttpStatusCode.Conflict + else -> HttpStatusCode.BadRequest + } + respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) +} diff --git a/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt diff --git a/server-auth/build.gradle.kts b/server-auth/build.gradle.kts new file mode 100644 index 00000000..72c21688 --- /dev/null +++ b/server-auth/build.gradle.kts @@ -0,0 +1,29 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-db")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + api("com.auth0:java-jwt:4.6.1") + implementation("com.password4j:password4j:1.8.4") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AvatarService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AvatarService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AvatarService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AvatarService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcConfig.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcState.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcState.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcUserService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt diff --git a/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt diff --git a/server-cache/build.gradle.kts b/server-cache/build.gradle.kts new file mode 100644 index 00000000..8b432e73 --- /dev/null +++ b/server-cache/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + `java-test-fixtures` + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api("io.lettuce:lettuce-core:7.7.0.RELEASE") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + testImplementation(testFixtures(project(":server-cache"))) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt similarity index 53% rename from src/main/kotlin/dev/typetype/server/cache/CacheJson.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt index 4af329b2..b864ae08 100644 --- a/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt +++ b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt @@ -2,4 +2,4 @@ package dev.typetype.server.cache import kotlinx.serialization.json.Json -internal val CacheJson: Json = Json { ignoreUnknownKeys = true } +val CacheJson: Json = Json { ignoreUnknownKeys = true } diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/cache/CacheService.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/CacheService.kt diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt diff --git a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt b/server-cache/src/testFixtures/kotlin/dev/typetype/server/FakeCacheService.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeCacheService.kt rename to server-cache/src/testFixtures/kotlin/dev/typetype/server/FakeCacheService.kt diff --git a/server-core/build.gradle.kts b/server-core/build.gradle.kts new file mode 100644 index 00000000..4df09914 --- /dev/null +++ b/server-core/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + `java-test-fixtures` + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.slf4j:slf4j-api:2.0.18") + testImplementation(testFixtures(project(":server-core"))) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt b/server-core/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/InstanceDefaults.kt rename to server-core/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt new file mode 100644 index 00000000..de9dd35d --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt @@ -0,0 +1,16 @@ +package dev.typetype.server + +import org.slf4j.LoggerFactory + +object PlaybackTraceLog { + private val logger = LoggerFactory.getLogger(PlaybackTraceLog::class.java) + + fun record(event: String, fields: String = "") { + record(currentPlaybackTraceId(), currentRequestId(), event, fields) + } + + fun record(traceId: String?, requestId: String?, event: String, fields: String = "") { + if (traceId == null) return + logger.info("playback_trace traceId={} requestId={} event={} {}", traceId, requestId ?: "unknown", event, fields) + } +} diff --git a/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt new file mode 100644 index 00000000..528824ed --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt @@ -0,0 +1,16 @@ +package dev.typetype.server + +import kotlinx.coroutines.asContextElement +import kotlin.coroutines.CoroutineContext + +private val requestIdContext = ThreadLocal() +private val playbackTraceIdContext = ThreadLocal() + +const val REQUEST_ID_HEADER = "X-Request-ID" +const val PLAYBACK_TRACE_ID_HEADER = "X-Playback-Trace-ID" + +fun currentRequestId(): String? = requestIdContext.get() +fun currentPlaybackTraceId(): String? = playbackTraceIdContext.get() + +fun requestContextElement(requestId: String?, playbackTraceId: String?): CoroutineContext = + requestIdContext.asContextElement(requestId) + playbackTraceIdContext.asContextElement(playbackTraceId) diff --git a/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt new file mode 100644 index 00000000..675d710b --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt @@ -0,0 +1,35 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class BiliBiliSessionStatusResponse( + val status: String, + val updatedAt: Long, + val lastUsedAt: Long, + val expiresAt: Long = 0, +) + +@Serializable +data class BiliBiliQrLoginResponse( + val qrUrl: String, + val qrcodeKey: String, + val expiresAt: Long, +) + +@Serializable +data class BiliBiliQrPollRequest( + val qrcodeKey: String, +) + +@Serializable +data class BiliBiliHealthResponse( + val status: String, + val message: String = "", +) + +@Serializable +data class BiliBiliQrPollResponse( + val status: String, + val message: String = "", +) diff --git a/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BlockedItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportContext.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CommentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CommentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CommentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CommentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt index 01313a9a..2e9b2d8e 100644 --- a/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt @@ -13,4 +13,8 @@ sealed class ExtractionResult { enum class ExtractionFailureKind { Unknown, YoutubeSessionRejected, + LiveEventNotStarted, + ScheduledPremiere, + ContentUnavailable, + ProviderAccessBlocked, } diff --git a/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HealthResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HistoryItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt b/server-core/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/NotificationItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt new file mode 100644 index 00000000..a4b8ea1b --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt @@ -0,0 +1,37 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PresenceKeyCreateRequest( + val name: String? = null, +) + +@Serializable +data class PresenceKeyItem( + val id: String, + val name: String, + val tokenPrefix: String, + val scope: String = PresenceScopes.READ, + val createdAt: Long, + val lastUsedAt: Long? = null, +) + +@Serializable +data class PresenceKeyCreatedResponse( + val key: PresenceKeyItem, + val token: String, +) + +@Serializable +data class PresenceResponse( + val active: Boolean, + val nowPlaying: ActiveSessionNowPlayingItem? = null, + val retryAfterMs: Long = 15_000L, + val serverTimeMs: Long, +) + +object PresenceScopes { + const val READ = "presence:read" + const val PRESENCE_RETRY_MS = 15_000L +} diff --git a/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProgressItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/models/SettingsItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 5d5d7447..e75da1f9 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -8,7 +8,11 @@ data class SettingsItem( val defaultQuality: String = "1080p", val defaultPlaybackSpeed: Double = 1.0, val defaultLandingPage: String = "home", + val videoGridColumns: Int = 0, + val relatedVideoSize: String = "default", val autoplay: Boolean = true, + val autoplayOnOpen: Boolean = true, + val autoplayCountdownSeconds: Int = 10, val skipPlaylistAutoplayScreen: Boolean = false, val volume: Double = 1.0, val muted: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/StreamResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt new file mode 100644 index 00000000..d4968a27 --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt @@ -0,0 +1,13 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionMembershipPage( + val items: List, + val total: Long, + val totalSubscriptions: Long, + val ungroupedCount: Long, + val page: Int, + val limit: Int, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VersionResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VideoItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VideoItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VideoItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VideoItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt b/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt new file mode 100644 index 00000000..e8aa885c --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AdminSettingsItem + +object AdminSettingsCache { + @Volatile + private var cachedSettings: AdminSettingsItem? = null + + fun get(): AdminSettingsItem? = cachedSettings + + fun set(value: AdminSettingsItem?) { + cachedSettings = value + } + + fun clear() { + cachedSettings = null + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt b/server-core/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt rename to server-core/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt diff --git a/src/test/kotlin/dev/typetype/server/TestFixtures.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestFixtures.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/TestStreamResponseFixture.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestStreamResponseFixture.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestStreamResponseFixture.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestStreamResponseFixture.kt diff --git a/src/test/kotlin/dev/typetype/server/TestVideoItemFixture.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestVideoItemFixture.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestVideoItemFixture.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestVideoItemFixture.kt diff --git a/server-db/build.gradle.kts b/server-db/build.gradle.kts new file mode 100644 index 00000000..8b78c3c4 --- /dev/null +++ b/server-db/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + `java-test-fixtures` + kotlin("jvm") +} + +dependencies { + api(project(":server-core")) + api("org.jetbrains.exposed:exposed-core:1.5.0") + api("org.jetbrains.exposed:exposed-jdbc:1.5.0") + api("com.zaxxer:HikariCP:7.1.0") + implementation("org.postgresql:postgresql:42.7.13") + implementation("org.xerial:sqlite-jdbc:3.53.4.0") + testImplementation("org.testcontainers:testcontainers:2.0.5") + testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation(testFixtures(project(":server-db"))) + "testFixturesImplementation"("org.testcontainers:testcontainers:2.0.5") + "testFixturesImplementation"("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 1368eb0f..d24371b9 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -33,6 +33,7 @@ import dev.typetype.server.db.tables.PushNotificationSeenVideosTable import dev.typetype.server.db.tables.PushNotificationEventsTable import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.PresenceKeysTable import dev.typetype.server.db.tables.RecommendationEventsTable import dev.typetype.server.db.tables.RecommendationFeedHistoryTable import dev.typetype.server.db.tables.RecommendationFeedbackTable @@ -47,6 +48,7 @@ import dev.typetype.server.db.tables.UserTopicInterestTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable +import dev.typetype.server.db.tables.BiliBiliSessionsTable import dev.typetype.server.db.tables.YoutubeSessionsTable import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -73,6 +75,7 @@ object DatabaseFactory { SchemaUtils.create( UsersTable, ProfileAccountsTable, + PresenceKeysTable, UserAvatarsTable, SessionsTable, AdminSettingsTable, @@ -98,6 +101,7 @@ object DatabaseFactory { YoutubeTakeoutPlaylistKeysTable, YoutubeSessionsTable, YoutubeSessionPairingsTable, + BiliBiliSessionsTable, BugReportsTable, NotificationStatesTable, NotificationReadItemsTable, diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt index 647fb241..2cf6a811 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt @@ -8,6 +8,8 @@ object DatabaseIndexMigrations { exec("CREATE INDEX IF NOT EXISTS idx_history_user_watched_id ON history (user_id, watched_at DESC, id DESC)") exec("CREATE INDEX IF NOT EXISTS idx_playlist_videos_user_playlist_position ON playlist_videos (user_id, playlist_id, position)") exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_user_subscribed_at ON subscriptions (user_id, subscribed_at DESC)") + exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_name_trgm ON subscriptions USING gin (lower(name) gin_trgm_ops)") + exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_channel_url_trgm ON subscriptions USING gin (lower(channel_url) gin_trgm_ops)") exec("CREATE INDEX IF NOT EXISTS idx_favorites_user_favorited_at ON favorites (user_id, favorited_at DESC)") exec("CREATE INDEX IF NOT EXISTS idx_watch_later_user_added_at ON watch_later (user_id, added_at DESC)") exec("CREATE INDEX IF NOT EXISTS idx_search_history_user_searched_at ON search_history (user_id, searched_at DESC)") diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index e49baf63..d8d83fb1 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -29,8 +29,13 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_members_only_content BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_on_open BOOLEAN NOT NULL DEFAULT true") + exec("UPDATE settings SET autoplay_on_open = autoplay") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_countdown_seconds INTEGER NOT NULL DEFAULT 10") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_landing_page TEXT NOT NULL DEFAULT 'home'") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS video_grid_columns INTEGER NOT NULL DEFAULT 0") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS related_video_size TEXT NOT NULL DEFAULT 'default'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode_admin_managed BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode_admin_managed_at BIGINT NOT NULL DEFAULT 0") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt diff --git a/server-db/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt new file mode 100644 index 00000000..0d9b51bc --- /dev/null +++ b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object BiliBiliSessionsTable : Table("bilibili_sessions") { + val userId = text("user_id") + val encryptedCookies = text("encrypted_cookies") + val status = text("status") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + val lastUsedAt = long("last_used_at").default(0) + val expiresAt = long("expires_at").default(0) + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt diff --git a/server-db/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt new file mode 100644 index 00000000..a84c9509 --- /dev/null +++ b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PresenceKeysTable : Table("presence_keys") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val tokenPrefix = text("token_prefix") + val tokenHash = text("token_hash").uniqueIndex() + val createdAt = long("created_at") + val lastUsedAt = long("last_used_at").nullable() + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 253b348f..7a51616c 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -8,7 +8,11 @@ object SettingsTable : Table("settings") { val defaultQuality = text("default_quality").default("1080p") val defaultPlaybackSpeed = double("default_playback_speed").default(1.0) val defaultLandingPage = text("default_landing_page").default("home") + val videoGridColumns = integer("video_grid_columns").default(0) + val relatedVideoSize = text("related_video_size").default("default") val autoplay = bool("autoplay").default(true) + val autoplayOnOpen = bool("autoplay_on_open").default(true) + val autoplayCountdownSeconds = integer("autoplay_countdown_seconds").default(10) val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) val volume = double("volume").default(1.0) val muted = bool("muted").default(false) diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt diff --git a/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt b/server-db/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt rename to server-db/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt b/server-db/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt rename to server-db/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt similarity index 90% rename from src/test/kotlin/dev/typetype/server/TestDatabase.kt rename to server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt index de24cde7..9624f526 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.AdminSettingsCache import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.AllowedPlaylistsTable import dev.typetype.server.db.tables.BlockedChannelsTable @@ -48,7 +49,7 @@ import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.db.tables.ProfileAccountsTable -import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.db.tables.PresenceKeysTable import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.testcontainers.containers.ContainerLaunchException @@ -70,11 +71,18 @@ object TestDatabase { } fun setup() { - AdminSettingsService.clearCache() + AdminSettingsCache.clear() if (initialized) return synchronized(this) { if (initialized) return - val (baseUrl, user, password) = runCatching { + val externalUrl = System.getenv("TEST_DATABASE_URL")?.takeIf(String::isNotBlank) + val (baseUrl, user, password) = if (externalUrl != null) { + Triple( + externalUrl, + firstNonBlank(System.getenv("TEST_DATABASE_USER"), "typetype"), + firstNonBlank(System.getenv("TEST_DATABASE_PASSWORD"), "typetype"), + ) + } else runCatching { val c = container Triple(c.jdbcUrl, c.username, c.password) }.getOrElse { @@ -95,6 +103,7 @@ object TestDatabase { private fun ensureSchemaExists(baseUrl: String, user: String, password: String, schema: String): Unit { DriverManager.getConnection(baseUrl, user, password).use { connection -> connection.createStatement().use { statement -> + statement.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public") statement.execute("CREATE SCHEMA IF NOT EXISTS \"$schema\"") } } @@ -156,6 +165,7 @@ object TestDatabase { RecommendationOnboardingPreferencesTable.deleteAll() RecommendationOnboardingStateTable.deleteAll() ProfileAccountsTable.deleteAll() - AdminSettingsService.clearCache() + PresenceKeysTable.deleteAll() + AdminSettingsCache.clear() } } diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts new file mode 100644 index 00000000..dc133b34 --- /dev/null +++ b/server-domain/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-downloader")) + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt new file mode 100644 index 00000000..30711273 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.services + +const val ACCESS_MODE_UNRESTRICTED = "unrestricted" +const val ACCESS_MODE_ALLOW_LIST = "allow_list" +const val ALLOW_SCOPE_USER = "user" +const val ALLOW_SCOPE_GLOBAL = "global" + +fun String.toAccessMode(): String = when (this) { + ACCESS_MODE_ALLOW_LIST -> ACCESS_MODE_ALLOW_LIST + else -> ACCESS_MODE_UNRESTRICTED +} diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt new file mode 100644 index 00000000..d65424f5 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt @@ -0,0 +1,42 @@ +package dev.typetype.server.services + +import java.text.Normalizer +import java.net.URI + +fun normalizeChannelKey(value: String): String = value.trim() + .substringBefore('#') + .substringBefore('?') + .removeSuffix("/") + .replace("http://", "https://") + .replace( + Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), + "https://youtube.com", + ) + .withoutYoutubeTab() + +private fun String.withoutYoutubeTab(): String { + val uri = runCatching { URI(this) }.getOrNull() ?: return this + if (!uri.host.equals("youtube.com", ignoreCase = true)) return this + val segments = uri.path.split('/').filter(String::isNotBlank) + if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this + val path = "/${segments.dropLast(1).joinToString("/")}" + return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() +} + +private val YOUTUBE_CHANNEL_TABS = setOf( + "featured", + "videos", + "shorts", + "streams", + "playlists", + "community", + "about", +) + +fun normalizePlaylistKey(value: String): String = value.trim() + .substringBefore('#') + .removeSuffix("/") + .replace("http://", "https://") + +fun normalizeBlockedKeyword(value: String): String = + Normalizer.normalize(value, Normalizer.Form.NFKC).trim().lowercase() diff --git a/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt index 9b4fe439..75f1d981 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.security.MessageDigest -internal object PublicCacheKey { +object PublicCacheKey { private val hex = "0123456789abcdef".toCharArray() fun of(area: String, vararg parts: String?): String = "$area:v2:${digest(parts)}" diff --git a/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt index f90a3e37..46d36da8 100644 --- a/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException -internal inline fun runCatchingNonCancellation(block: () -> T): Result = try { +inline fun runCatchingNonCancellation(block: () -> T): Result = try { Result.success(block()) } catch (error: CancellationException) { throw error diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index e2087004..868eaa64 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -16,12 +16,16 @@ import org.jetbrains.exposed.v1.core.statements.UpdateBuilder private val SPONSOR_BLOCK_CATEGORY_ACTIONS_SERIALIZER = MapSerializer(String.serializer(), SponsorBlockMode.serializer()) private val CAPTION_STYLES_SERIALIZER = CaptionStylesItem.serializer() -internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( +fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultService = this[SettingsTable.defaultService], defaultQuality = this[SettingsTable.defaultQuality], defaultPlaybackSpeed = this[SettingsTable.defaultPlaybackSpeed], defaultLandingPage = this[SettingsTable.defaultLandingPage], + videoGridColumns = this[SettingsTable.videoGridColumns], + relatedVideoSize = this[SettingsTable.relatedVideoSize], autoplay = this[SettingsTable.autoplay], + autoplayOnOpen = this[SettingsTable.autoplayOnOpen], + autoplayCountdownSeconds = this[SettingsTable.autoplayCountdownSeconds], skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], volume = this[SettingsTable.volume], muted = this[SettingsTable.muted], @@ -56,12 +60,16 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( accessMode = this[SettingsTable.accessMode].toAccessMode(), ) -internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { +fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultService] = settings.defaultService this[SettingsTable.defaultQuality] = settings.defaultQuality this[SettingsTable.defaultPlaybackSpeed] = settings.defaultPlaybackSpeed this[SettingsTable.defaultLandingPage] = settings.defaultLandingPage + this[SettingsTable.videoGridColumns] = settings.videoGridColumns + this[SettingsTable.relatedVideoSize] = settings.relatedVideoSize this[SettingsTable.autoplay] = settings.autoplay + this[SettingsTable.autoplayOnOpen] = settings.autoplayOnOpen + this[SettingsTable.autoplayCountdownSeconds] = settings.autoplayCountdownSeconds this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen this[SettingsTable.volume] = settings.volume this[SettingsTable.muted] = settings.muted @@ -98,11 +106,14 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.accessModeAdminManagedAt] = 0 } -internal fun SettingsItem.normalized(): SettingsItem = copy( +fun SettingsItem.normalized(): SettingsItem = copy( defaultLandingPage = defaultLandingPage.ifBlank { "home" }, + videoGridColumns = videoGridColumns.takeIf { it in VIDEO_GRID_COLUMNS } ?: 0, + relatedVideoSize = relatedVideoSize.takeIf { it in RELATED_VIDEO_SIZES } ?: "default", defaultPlaybackSpeed = defaultPlaybackSpeed.takeIf { it.isFinite() } ?.coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED) ?: 1.0, accessMode = accessMode.toAccessMode(), + autoplayCountdownSeconds = autoplayCountdownSeconds.coerceIn(0, 60), sponsorBlockCategoryActions = sponsorBlockCategoryActions.withDefaultSponsorBlockCategoryActions(), sponsorBlockMinimumDuration = sponsorBlockMinimumDuration.coerceAtLeast(0), deArrowTitleMode = deArrowTitleMode.takeIf { it in DEARROW_TITLE_MODES } ?: "dearrow", @@ -111,6 +122,8 @@ internal fun SettingsItem.normalized(): SettingsItem = copy( ) private val DEARROW_TITLE_MODES = setOf("original", "dearrow") +private val VIDEO_GRID_COLUMNS = setOf(0, 4, 5, 6) +private val RELATED_VIDEO_SIZES = setOf("default", "large") private const val MIN_PLAYBACK_SPEED = 0.25 private const val MAX_PLAYBACK_SPEED = 4.0 private val DEARROW_THUMBNAIL_MODES = setOf("original", "dearrow", "random", "dearrow_or_random") diff --git a/src/main/kotlin/dev/typetype/server/services/StreamService.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/StreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StreamService.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/StreamService.kt diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt new file mode 100644 index 00000000..b2c3d585 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt @@ -0,0 +1,184 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.models.SubscriptionItem +import org.jetbrains.exposed.v1.core.Column +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.neq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +object SubscriptionAvatarRepairer { + fun repair(userId: String, items: List): List { + val avatars = recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST, + maxSourceRows = MAX_AVATAR_SOURCE_ROWS, + ) + avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } + return items.withAvatars(avatars) + } + + fun repairImported(userId: String, items: List): List { + val avatars = recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_IMPORTED_AVATAR_REPAIR, + maxSourceRows = MAX_IMPORTED_AVATAR_SOURCE_ROWS, + ) + avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } + return items.withAvatars(avatars) + } + + fun resolve(userId: String, items: List): List = + items.withAvatars( + recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST, + maxSourceRows = MAX_AVATAR_SOURCE_ROWS, + ), + ) + + private fun recoverableAvatars( + userId: String, + items: List, + maxRepairs: Int, + maxSourceRows: Int, + ): Map { + val candidateUrls = items.filter { it.avatarUrl.isBlank() } + .map { it.channelUrl } + .distinct() + if (candidateUrls.isEmpty()) return emptyMap() + val avatars = knownAvatars(userId, candidateUrls, maxSourceRows) + .entries + .take(maxRepairs) + .associate { it.toPair() } + return avatars + } + + private fun List.withAvatars(avatars: Map): List = + if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } + + private fun knownAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map { + val avatars = linkedMapOf() + historyAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + playlistAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + watchLaterAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + favoriteAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + return avatars + } + + private fun historyAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = HistoryTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + HistoryTable.userId, + HistoryTable.channelUrl, + HistoryTable.channelAvatar, + ) + } + .orderBy(HistoryTable.watchedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(HistoryTable.channelUrl, HistoryTable.channelAvatar) + + private fun playlistAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = PlaylistVideosTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + PlaylistVideosTable.userId, + PlaylistVideosTable.channelUrl, + PlaylistVideosTable.channelAvatar, + ) + } + .limit(maxSourceRows) + .associateAvatarRows(PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) + + private fun watchLaterAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = WatchLaterTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + WatchLaterTable.userId, + WatchLaterTable.channelUrl, + WatchLaterTable.channelAvatar, + ) + } + .orderBy(WatchLaterTable.addedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) + + private fun favoriteAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = FavoritesTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + FavoritesTable.userId, + FavoritesTable.channelUrl, + FavoritesTable.channelAvatar, + ) + } + .orderBy(FavoritesTable.favoritedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(FavoritesTable.channelUrl, FavoritesTable.channelAvatar) + + private fun updateAvatar(userId: String, channelUrl: String, avatarUrl: String): Int = SubscriptionsTable.update({ + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) and (SubscriptionsTable.avatarUrl eq "") + }) { + it[SubscriptionsTable.avatarUrl] = avatarUrl + } + + private fun avatarSourceFilter( + userId: String, + channelUrls: List, + userColumn: Column, + urlColumn: Column, + avatarColumn: Column, + ) = (userColumn eq userId) and (urlColumn inList channelUrls) and (avatarColumn neq "") + + private fun Iterable.associateAvatarRows( + urlColumn: Column, + avatarColumn: Column, + ): Map = mapNotNull { row -> + val avatar = row[avatarColumn].trim() + if (avatar.isProxyableAvatar()) ChannelUrlCanonicalizer.canonicalize(row[urlColumn]) to avatar else null + }.distinctBy { it.first }.toMap() + + private fun String.isProxyableAvatar(): Boolean = startsWith("https://") || startsWith("http://") + + private const val MAX_AVATAR_REPAIR_PER_REQUEST = 25 + private const val MAX_AVATAR_SOURCE_ROWS = 100 + private const val MAX_IMPORTED_AVATAR_REPAIR = 5_000 + private const val MAX_IMPORTED_AVATAR_SOURCE_ROWS = 20_000 +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt index 410755b6..44fc3ff2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt @@ -1,5 +1,10 @@ package dev.typetype.server.services +interface SubscriptionFeedCacheInvalidator { + suspend fun invalidate(userId: String) + suspend fun awaitRefresh(userId: String) +} + object SubscriptionFeedCacheInvalidation { @Volatile private var invalidator: SubscriptionFeedCacheInvalidator? = null diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt index d78f7833..55a1368f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt @@ -6,7 +6,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.jdbc.deleteWhere -internal object SubscriptionGroupMembershipCleaner { +object SubscriptionGroupMembershipCleaner { fun retain(userId: String, channelUrls: Collection) { val retained = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) SubscriptionGroupMembershipsTable.deleteWhere { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt similarity index 62% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt index 8320b1ab..cb39a255 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager -internal object SubscriptionMutationLock { +object SubscriptionMutationLock { fun acquire(userId: String) { val userKey = userId.hashCode() and Int.MAX_VALUE TransactionManager.current().exec( @@ -10,6 +10,13 @@ internal object SubscriptionMutationLock { ) } + fun acquireRead(userId: String) { + val userKey = userId.hashCode() and Int.MAX_VALUE + TransactionManager.current().exec( + "SELECT pg_advisory_xact_lock_shared($LOCK_NAMESPACE, $userKey)", + ) + } + // Precomputed PostgreSQL hashtext('subscriptions'). private const val LOCK_NAMESPACE = 720_815_616 } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt index e2082b0a..ca74ad09 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt @@ -33,6 +33,7 @@ object YoutubeTypeTypeMapper { val trimmed = url.trim() WATCH_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } SHORTS_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } + LIVE_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } SHORT_URL_REGEX.find(trimmed)?.let { return it.groupValues[1] } return trimmed.takeIf { RAW_ID_REGEX.matches(it) } } @@ -43,6 +44,7 @@ object YoutubeTypeTypeMapper { private val WATCH_ID_REGEX = Regex("""[?&]v=([A-Za-z0-9_-]{6,})""") private val SHORTS_ID_REGEX = Regex("""youtube\.com/shorts/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) + private val LIVE_ID_REGEX = Regex("""youtube\.com/live/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) private val SHORT_URL_REGEX = Regex("""youtu\.be/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) private val RAW_ID_REGEX = Regex("""^[A-Za-z0-9_-]{6,}$""") } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt index f086ba35..da830793 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.net.URI -internal fun isYoutubeUrl(url: String): Boolean = runCatching { +fun isYoutubeUrl(url: String): Boolean = runCatching { val host = URI(url).host?.lowercase() ?: return false host == "youtu.be" || host == "youtube.com" || host.endsWith(".youtube.com") }.getOrDefault(false) diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts new file mode 100644 index 00000000..2cedb894 --- /dev/null +++ b/server-downloader/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + kotlin("jvm") +} + +dependencies { + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt new file mode 100644 index 00000000..193493f5 --- /dev/null +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.downloader + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +object BilibiliCookieContext { + @Volatile private var cookieHeader: String? = null + + fun set(value: String?) { + cookieHeader = value?.trim()?.takeIf { it.isNotEmpty() } + } + + fun headerFor(url: String): String? { + val parsed = url.toHttpUrlOrNull() ?: return null + val host = parsed.host.lowercase() + val isBilibiliApi = host == "bilibili.com" || host.endsWith(".bilibili.com") + return cookieHeader?.takeIf { parsed.isHttps && isBilibiliApi } + } +} diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt index 1e9072ca..3e55b5f2 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt @@ -113,6 +113,7 @@ class OkHttpDownloader private constructor( request.headers().forEach { (name, values) -> values.forEach { value -> builder.addHeader(name, value) } } + BilibiliCookieContext.headerFor(normalizedUrl)?.let { builder.header("Cookie", it) } YoutubeAuthUserContext.headerFor(normalizedUrl)?.let { builder.header(YOUTUBE_AUTH_USER_HEADER, it) } diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt similarity index 71% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt index ff13f574..0e770616 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt @@ -2,10 +2,11 @@ package dev.typetype.server.downloader import org.schabi.newpipe.extractor.downloader.Response -internal object OkHttpExtractorResponseMapper { +object OkHttpExtractorResponseMapper { fun toExtractorResponse(httpResponse: okhttp3.Response): Response { + val contentType = httpResponse.body.contentType() val responseBodyBytes = httpResponse.body.bytes() - val responseBody = responseBodyBytes.toString(Charsets.UTF_8) + val responseBody = responseBodyBytes.toString(contentType?.charset() ?: Charsets.UTF_8) return Response( httpResponse.code, httpResponse.message, diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt index 6682dd9f..4c94046b 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt @@ -2,7 +2,7 @@ package dev.typetype.server.downloader import java.io.FilterInputStream -internal class OkHttpStreamingBodyStream( +class OkHttpStreamingBodyStream( private val response: okhttp3.Response, ) : FilterInputStream(response.body.byteStream()) { override fun close() { diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt index ad03ff60..2bb6f80e 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt @@ -2,14 +2,14 @@ package dev.typetype.server.downloader import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -internal object YoutubeAuthUserContext { +object YoutubeAuthUserContext { @Volatile private var value: Int? = null fun set(authUser: Int?): Unit { value = authUser } - internal fun headerFor(url: String): String? { + fun headerFor(url: String): String? { val parsed = url.toHttpUrlOrNull() ?: return null val host = parsed.host.lowercase() val isYoutube = host == "youtube.com" || host.endsWith(".youtube.com") diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt index d9016d4b..d80706f4 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt @@ -7,7 +7,7 @@ import java.net.ProxySelector import java.net.SocketAddress import java.net.URI -internal class YoutubeProxySelector private constructor( +class YoutubeProxySelector private constructor( private val proxy: Proxy, ) : ProxySelector() { override fun select(uri: URI): List = diff --git a/server-downloader/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt new file mode 100644 index 00000000..a1805f74 --- /dev/null +++ b/server-downloader/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt @@ -0,0 +1,28 @@ +package dev.typetype.server + +import dev.typetype.server.downloader.BilibiliCookieContext +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class BilibiliCookieContextTest { + @Test + fun `limits cookie injection to bilibili api hosts`() { + BilibiliCookieContext.set("SESSDATA=session; bili_jct=csrf") + try { + assertEquals( + "SESSDATA=session; bili_jct=csrf", + BilibiliCookieContext.headerFor("https://api.bilibili.com/x/player"), + ) + assertEquals( + "SESSDATA=session; bili_jct=csrf", + BilibiliCookieContext.headerFor("https://www.bilibili.com/video/BV1"), + ) + assertNull(BilibiliCookieContext.headerFor("http://api.bilibili.com/x/player")) + assertNull(BilibiliCookieContext.headerFor("https://upos-sz-mirrorali.bilivideo.com/video")) + assertNull(BilibiliCookieContext.headerFor("https://example.com/video")) + } finally { + BilibiliCookieContext.set(null) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt diff --git a/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt new file mode 100644 index 00000000..b1bb99c1 --- /dev/null +++ b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt @@ -0,0 +1,81 @@ +package dev.typetype.server.downloader + +import java.nio.charset.Charset +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class OkHttpExtractorResponseMapperTest { + + private fun httpResponse(contentType: String?, bytes: ByteArray): okhttp3.Response = + okhttp3.Response.Builder() + .request(Request.Builder().url("https://example.org/complete/search").build()) + .protocol(Protocol.HTTP_2) + .code(200) + .message("OK") + .body(bytes.toResponseBody(contentType?.toMediaType())) + .build() + + @Test + fun `decodes iso-8859-1 accented characters`() { + val expected = "ä ö ü ß ë Ë é è ê ç à ñ" + val bytes = expected.toByteArray(Charsets.ISO_8859_1) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=ISO-8859-1", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + assertArrayEquals(bytes, mapped.rawResponseBody()) + } + + @Test + fun `decodes utf-8 body when charset is declared`() { + val expected = "ä ö ü ß 🎵" + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("application/json; charset=utf-8", expected.toByteArray(Charsets.UTF_8)), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `defaults to utf-8 without content type`() { + val expected = "ä ö ü ß 🎵" + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse(null, expected.toByteArray(Charsets.UTF_8)), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `decodes shift_jis body used for japanese locales`() { + val expected = "あいう" + val bytes = expected.toByteArray(Charset.forName("Shift_JIS")) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=Shift_JIS", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `decodes windows-1251 body used for cyrillic locales`() { + val expected = "А Б В" + val bytes = expected.toByteArray(Charset.forName("windows-1251")) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=windows-1251", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + } +} diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts new file mode 100644 index 00000000..bd57eee7 --- /dev/null +++ b/server-http/build.gradle.kts @@ -0,0 +1,52 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-services")) + api(project(":server-admin")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-downloader")) + implementation(project(":server-playback")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-server-rate-limit-jvm:3.5.2") + implementation("io.ktor:ktor-server-status-pages-jvm:3.5.2") + implementation("io.ktor:ktor-server-compression-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-compression-jvm:3.5.2") + implementation("io.ktor:ktor-server-call-logging-jvm:3.5.2") + implementation("io.ktor:ktor-utils-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 2e52330d..29590bfa 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -16,6 +16,7 @@ import dev.typetype.server.routes.downloaderGatewayRoutes import dev.typetype.server.routes.deArrowRoutes import dev.typetype.server.routes.internalObservabilityRoutes import dev.typetype.server.routes.oidcAuthRoutes +import dev.typetype.server.routes.presenceRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes @@ -43,6 +44,8 @@ import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.portability.PortabilityEngine @@ -50,12 +53,14 @@ import io.ktor.server.application.Application import io.ktor.server.plugins.ratelimit.rateLimit import io.ktor.server.routing.routing -internal fun Application.installApplicationRoutes( +fun Application.installApplicationRoutes( svc: ServiceRegistry, authService: AuthService, authSessionConfig: AuthSessionConfig, adminSettingsService: AdminSettingsService, activeSessionService: ActiveSessionService, + presenceKeyService: PresenceKeyService, + presenceService: PresenceService, downloaderGatewayService: DownloaderGatewayService, gitHubIssueService: GitHubIssueService, instanceService: InstanceService, @@ -81,12 +86,24 @@ internal fun Application.installApplicationRoutes( searchRoutes(svc.searchService, authService, svc.accessControlService, adminSettingsService, svc.blockedService) suggestionRoutes(svc.suggestionService, authService, adminSettingsService) trendingRoutes(svc.trendingService, authService, svc.accessControlService, adminSettingsService) - publicPlaylistRoutes(svc.publicPlaylistService, authService, svc.accessControlService, adminSettingsService) + publicPlaylistRoutes( + svc.publicPlaylistService, + authService, + svc.accessControlService, + adminSettingsService, + svc.blockedService, + ) commentRoutes(svc.commentService, authService, adminSettingsService) bulletCommentRoutes(svc.bulletCommentService, authService, adminSettingsService) } rateLimit(CHANNEL_ZONE) { - channelRoutes(svc.channelService, authService, svc.accessControlService, adminSettingsService) + channelRoutes( + svc.channelService, + authService, + svc.accessControlService, + adminSettingsService, + svc.blockedService, + ) podcastRoutes(svc.podcastService, authService, adminSettingsService) } installProxyRoutes(svc) @@ -120,7 +137,8 @@ internal fun Application.installApplicationRoutes( adminIdentityRoutes(svc.accountIdentityService, authService) adminAllowListRoutes(authService, userAdminService, svc.adminManagedAccessService, svc.adminUserLookupService, svc.allowedChannelsService, svc.allowedPlaylistsService) adminSessionRoutes(authService, activeSessionService) - sessionActivityRoutes(authService, activeSessionService) + sessionActivityRoutes(authService, activeSessionService, presenceService) + rateLimit(USER_DATA_ZONE) { presenceRoutes(authService, presenceKeyService, presenceService) } adminBugReportRoutes(authService, svc.bugReportService, gitHubIssueService) avatarRoutes(avatarService, openMojiProxyService, svc.customAvatarService) rateLimit(USER_DATA_ZONE) { youtubeRemoteBrowserRoutes(youtubeRemoteBrowserService, authService) } diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 2e729658..e6fd90b9 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -25,6 +25,7 @@ internal fun Route.installStreamRoutes( streamService = svc.youtubeSabrStreamService, nicoNicoStreamService = svc.nicoNicoStreamService, bilibiliStreamService = svc.bilibiliStreamService, + youtubeLiveHlsStreamService = svc.youtubeLiveHlsStreamService, sabrBootstrapStreamService = svc.youtubeSabrBootstrapStreamService, authService = authService, accessControlService = svc.accessControlService, @@ -33,6 +34,7 @@ internal fun Route.installStreamRoutes( publicHlsManifestTokenService = svc.publicHlsManifestTokenService, providerMediaHandleService = svc.providerMediaHandleService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, + bilibiliSessionStreamInfo = { userId, url -> svc.bilibiliSessionStreamService.getStreamInfo(userId, url) }, youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> { userId, url -> service.getStreamInfo(userId, url) } }, diff --git a/src/main/kotlin/dev/typetype/server/CompressionConfig.kt b/server-http/src/main/kotlin/dev/typetype/server/CompressionConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/CompressionConfig.kt rename to server-http/src/main/kotlin/dev/typetype/server/CompressionConfig.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt b/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt new file mode 100644 index 00000000..173179ad --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt @@ -0,0 +1,11 @@ +package dev.typetype.server + +import io.ktor.server.plugins.ratelimit.RateLimitName + +val EXTRACTION_ZONE = RateLimitName("extraction") +val DEARROW_ZONE = RateLimitName("dearrow") +val STREAMS_ZONE = RateLimitName("streams") +val CHANNEL_ZONE = RateLimitName("channel") +val PROXY_ZONE = RateLimitName("proxy") +val PROXY_STORYBOARD_ZONE = RateLimitName("proxy-storyboard") +val USER_DATA_ZONE = RateLimitName("user-data") diff --git a/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt new file mode 100644 index 00000000..9e102365 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt @@ -0,0 +1,42 @@ +package dev.typetype.server + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.routes.TooManyRequestsBodyAttribute +import dev.typetype.server.routes.isMultipartSizeLimit +import dev.typetype.server.routes.respondPortabilityError +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.install +import io.ktor.server.plugins.statuspages.StatusPages +import io.ktor.server.request.path +import io.ktor.server.response.respond +import kotlinx.coroutines.CancellationException +import org.slf4j.LoggerFactory + +fun Application.configureStatusPages() { + val log = LoggerFactory.getLogger("RequestLogger") + install(StatusPages) { + status(HttpStatusCode.TooManyRequests) { call, status -> + if (call.attributes.contains(TooManyRequestsBodyAttribute)) return@status + if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") + call.respond(status, ErrorResponse("Too many requests", "rate_limited")) + } + exception { call, cause -> + log.warn("Bad request: ${cause.message}") + call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request", "bad_request")) + } + exception { call, cause -> + if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception + if (cause is kotlinx.coroutines.CancellationException) throw cause + // Ktor's multipart producer can fail outside the route's receive block. + if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) { + call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException()) + return@exception + } + log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) + call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt index 44f77935..936f2f8b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt @@ -38,7 +38,7 @@ internal suspend fun ApplicationCall.respondAudioOnlyHead( respondOutputStream(containerMime(result.data.stream.mimeType), HttpStatusCode.OK, length ?: 0L) {} } } - is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt index 6162efd2..6e4040e8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt @@ -21,11 +21,11 @@ internal fun AudioOnlyStreamSelection.toResponse( val src = when (kind) { AudioOnlyStreamKind.Progressive -> "/streams/audio-only/source?token=${encode(token)}" AudioOnlyStreamKind.Hls -> hlsTokenService?.createPath(stream.url) - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") AudioOnlyStreamKind.Dash -> stream.manifestUrl?.let { audioOnlyDashManifest(it) } - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") AudioOnlyStreamKind.SabrHls -> stream.manifestUrl?.let { audioOnlySabrHlsManifest(it, token) } - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") } return ExtractionResult.Success(AudioOnlyStreamResponse( src = src, diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt index 761e5f02..7ee873c8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt @@ -61,12 +61,12 @@ fun Route.audioOnlyContractRoutes( preferredLocale, )) { is ExtractionResult.Success -> call.respond(response.data) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(response.message)) - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(response.message)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(response.message, response.code)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(response.message, response.code)) } } - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } } @@ -99,8 +99,8 @@ internal fun Route.audioOnlySourceRoutes( .ensureProgressiveAudio(), contentTypeOverride = result.data.stream.mimeType, ) - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt new file mode 100644 index 00000000..2c465755 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt @@ -0,0 +1,69 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.BiliBiliQrPollRequest +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BiliBiliQrLoginResult +import dev.typetype.server.services.BiliBiliSessionService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +fun Route.biliBiliSessionRoutes(service: BiliBiliSessionService, authService: AuthService): Unit { + post("/bilibili-session/qr") { + call.withJwtAuth(authService) { _ -> + when (val result = service.startQrLogin()) { + is BiliBiliQrLoginResult.Success -> call.respond(HttpStatusCode.Created, result.response) + is BiliBiliQrLoginResult.Unavailable -> call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("BiliBili Session is unavailable", "bilibili_session_unavailable"), + ) + is BiliBiliQrLoginResult.Error -> call.respond( + HttpStatusCode.BadGateway, + ErrorResponse(result.message, "bilibili_qr_error"), + ) + } + } + } + post("/bilibili-session/qr/poll") { + call.withJwtAuth(authService) { userId -> + val request = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (request.qrcodeKey.isBlank()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing qrcodeKey")) + } + call.respond(service.pollQrLogin(userId, request.qrcodeKey)) + } + } + get("/bilibili-session/health") { + call.withJwtAuth(authService) { userId -> + val result = service.healthCheck(userId) + val status = when (result) { + is dev.typetype.server.services.BiliBiliHealthResult.Healthy -> "healthy" + is dev.typetype.server.services.BiliBiliHealthResult.Expired -> "expired" + is dev.typetype.server.services.BiliBiliHealthResult.RateLimited -> "rate_limited" + is dev.typetype.server.services.BiliBiliHealthResult.Disconnected -> "disconnected" + is dev.typetype.server.services.BiliBiliHealthResult.Unconfigured -> "unconfigured" + is dev.typetype.server.services.BiliBiliHealthResult.Error -> "error" + } + call.respond(dev.typetype.server.models.BiliBiliHealthResponse(status = status)) + } + } + + get("/bilibili-session/status") { + call.withJwtAuth(authService) { userId -> + call.respond(service.status(userId)) + } + } + delete("/bilibili-session") { + call.withJwtAuth(authService) { userId -> + service.delete(userId) + call.respond(HttpStatusCode.NoContent) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt similarity index 65% rename from src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt index 75c25b87..ce89b38a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt @@ -9,8 +9,11 @@ import dev.typetype.server.services.AccessControlProfile import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedContentProfile +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.ChannelService import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -24,20 +27,32 @@ fun Route.channelRoutes( authService: AuthService? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, + blockedService: BlockedService? = null, ) { get("/channel") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] val sort = call.request.queryParameters["sort"]?.takeIf { it.isNotBlank() } - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty when (val result = channelService.getChannel(url = url, nextpage = nextpage, sort = sort)) { is ExtractionResult.Success -> { - if (!profile.allowsChannel(url = url, name = result.data.name)) { + if (!access.profile.allowsChannel(url = url, name = result.data.name)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - call.respond(result.data.filterAllowed(profile)) + if (!blocked.allowsChannel(url = url, name = result.data.name)) { + return@get call.respond( + HttpStatusCode.Forbidden, + ErrorResponse("Channel is blocked", "content_blocked"), + ) + } + call.respond(result.data.filterAllowed(access.profile).filterBlocked(blocked)) } is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) @@ -47,7 +62,12 @@ fun Route.channelRoutes( val request = call.receive() val url = request.url?.takeIf { it.isNotBlank() } ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@post + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@post + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty call.respondChannelResult( channelService.getChannel( @@ -56,19 +76,29 @@ fun Route.channelRoutes( sort = request.sort?.takeIf { it.isNotBlank() }, ), url, - profile, + access.profile, + blocked, ) } get("/channel/playlists") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get - if (!profile.allowsChannel(url)) { + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty + if (!access.profile.allowsChannel(url)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - call.respondChannelPlaylistsResult(channelService.getPlaylists(url = url, nextpage = nextpage), profile) + call.respondChannelPlaylistsResult( + channelService.getPlaylists(url = url, nextpage = nextpage), + access.profile, + blocked, + ) } } @@ -76,13 +106,17 @@ private suspend fun ApplicationCall.respondChannelResult( result: ExtractionResult, url: String, profile: AccessControlProfile, + blocked: BlockedContentProfile, ) { when (result) { is ExtractionResult.Success -> { if (!profile.allowsChannel(url = url, name = result.data.name)) { return respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - respond(result.data.filterAllowed(profile)) + if (!blocked.allowsChannel(url = url, name = result.data.name)) { + return respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is blocked", "content_blocked")) + } + respond(result.data.filterAllowed(profile).filterBlocked(blocked)) } is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) @@ -92,9 +126,10 @@ private suspend fun ApplicationCall.respondChannelResult( private suspend fun ApplicationCall.respondChannelPlaylistsResult( result: ExtractionResult, profile: AccessControlProfile, + blocked: BlockedContentProfile, ) { when (result) { - is ExtractionResult.Success -> respond(result.data.filterAllowed(profile)) + is ExtractionResult.Success -> respond(result.data.filterAllowed(profile).filterBlocked(blocked)) is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) } diff --git a/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt similarity index 63% rename from src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt index a85979d3..fed86656 100644 --- a/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt @@ -1,6 +1,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.DeArrowUnavailableException import dev.typetype.server.services.DeArrowService import io.ktor.http.ContentType import io.ktor.http.HttpHeaders @@ -13,8 +14,16 @@ import io.ktor.server.routing.get fun Route.deArrowRoutes(service: DeArrowService) { get("/dearrow") { val videoId = call.request.queryParameters["videoId"].orEmpty() - val item = service.get(videoId) - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoId")) + val item = try { + service.get(videoId) + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoId")) + } catch (_: DeArrowUnavailableException) { + call.response.headers.append(HttpHeaders.RetryAfter, "30") + return@get call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("DeArrow is temporarily unavailable", "dearrow_unavailable"), + ) + } call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=3600") call.respond(item) } @@ -22,10 +31,11 @@ fun Route.deArrowRoutes(service: DeArrowService) { val videoId = call.request.queryParameters["videoId"].orEmpty() val timestamp = call.request.queryParameters["time"]?.toDoubleOrNull() ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid time")) - val bytes = service.thumbnail(videoId, timestamp) + val thumbnail = service.thumbnail(videoId, timestamp) ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Thumbnail not found")) - call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=604800") - call.respondBytes(bytes, deArrowThumbnailContentType(bytes)) + val maxAge = if (thumbnail.fallback) 900 else 604800 + call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=$maxAge") + call.respondBytes(thumbnail.bytes, deArrowThumbnailContentType(thumbnail.bytes)) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt index 29222c84..f5aa5077 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt @@ -38,9 +38,9 @@ fun Route.manifestRoutes( is ExtractionResult.Success -> call.respondText(result.data, ContentType.parse("application/dash+xml")) is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } @@ -53,9 +53,9 @@ fun Route.manifestRoutes( is ExtractionResult.Success -> call.respondText(result.data, ContentType.parse("application/dash+xml")) is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } @@ -105,9 +105,9 @@ private suspend fun ApplicationCall.respondHlsResult( is ExtractionResult.Success -> respondText(result.data, ContentType.parse("application/vnd.apple.mpegurl")) is ExtractionResult.BadRequest -> - respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt index a9751289..98a6a4a5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt @@ -2,7 +2,7 @@ package dev.typetype.server.routes import java.io.IOException -internal fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && ( +fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && ( message?.let { (it.startsWith("Multipart content length exceeds limit ") && "formFieldLimit" in it) || (it.startsWith("Limit of ") && " bytes exceeded while searching for " in it) diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt index 11c9bd47..73cf0fc6 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt @@ -16,7 +16,7 @@ internal fun parsePortabilityFormat(value: String?): PortabilityFormat? { ?: throw IllegalArgumentException("Unsupported portability format") } -internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) { +suspend fun ApplicationCall.respondPortabilityError(error: Exception) { if (error is kotlinx.coroutines.CancellationException) throw error if (error.isMultipartSizeLimit()) { respondPortabilityError(PortabilityUploadTooLargeException()) diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt new file mode 100644 index 00000000..edd8c04e --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.PresenceKeyCreateRequest +import dev.typetype.server.models.PresenceResponse +import dev.typetype.server.models.PresenceScopes +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceKeyLimitException +import dev.typetype.server.services.PresenceService +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +fun Route.presenceRoutes( + authService: dev.typetype.server.services.AuthService, + presenceKeyService: PresenceKeyService, + presenceService: PresenceService, +): Unit { + post("/presence/keys") { + call.withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Guests cannot create presence keys")) + } + val body = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + val created = try { + presenceKeyService.create(userId, body.name ?: "") + } catch (error: PresenceKeyLimitException) { + return@withJwtAuth call.respond( + HttpStatusCode.Conflict, + ErrorResponse(error.message ?: "Presence key limit reached", "presence_key_limit"), + ) + } catch (error: IllegalArgumentException) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse(error.message ?: "Invalid request")) + } + call.respond(HttpStatusCode.Created, created) + } + } + get("/presence/keys") { + call.withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Guests cannot own presence keys")) + } + call.respond(presenceKeyService.list(userId)) + } + } + delete("/presence/keys/{id}") { + call.withJwtAuth(authService) { userId -> + val id = call.parameters["id"].orEmpty() + val revoked = presenceKeyService.revoke(userId, id) + if (revoked) call.respond(HttpStatusCode.NoContent) + else call.respond(HttpStatusCode.NotFound, ErrorResponse("Presence key not found")) + } + } + get("/presence/now-playing") { + val principal = call.presencePrincipal(presenceKeyService) ?: return@get + val nowPlaying = presenceService.current(principal.userId) + call.response.headers.append(HttpHeaders.CacheControl, "no-store") + call.respond( + PresenceResponse( + active = nowPlaying != null, + nowPlaying = nowPlaying, + retryAfterMs = PresenceScopes.PRESENCE_RETRY_MS, + serverTimeMs = System.currentTimeMillis(), + ), + ) + } +} + +private suspend fun io.ktor.server.application.ApplicationCall.presencePrincipal( + presenceKeyService: PresenceKeyService, +): dev.typetype.server.services.PresenceTokenPrincipal? { + val authorization = request.headers["Authorization"] + val token = authorization?.takeIf { it.startsWith("Bearer ") }?.substringAfter("Bearer ")?.trim() + val principal = token?.let { presenceKeyService.resolve(it) } + if (principal == null) { + respond(HttpStatusCode.Unauthorized, ErrorResponse("Invalid presence token")) + } + return principal +} diff --git a/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt index 4f5676a2..a82a3585 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt @@ -16,8 +16,8 @@ internal suspend fun ApplicationCall.respondProxyResult( contentTypeOverride: String? = null, ): Unit = when (result) { is ExtractionResult.Success -> respondProxy(result.data, contentTypeOverride) - is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } private suspend fun ApplicationCall.respondProxy(proxy: ProxyResponse, contentTypeOverride: String?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt similarity index 68% rename from src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt index 79801e13..891ad657 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt @@ -5,8 +5,11 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedContentProfile +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicPlaylistService import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route @@ -17,15 +20,23 @@ fun Route.publicPlaylistRoutes( authService: AuthService? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, + blockedService: BlockedService? = null, ) { get("/playlist") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty when (val result = playlistService.getPlaylist(url = url, nextpage = nextpage)) { - is ExtractionResult.Success -> call.respond(result.data.filterAllowed(profile)) + is ExtractionResult.Success -> call.respond( + result.data.filterAllowed(access.profile).filterBlocked(blocked), + ) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) } diff --git a/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt index abd8ac9d..ebde90f4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt @@ -22,13 +22,11 @@ internal fun Route.pushNotificationRoutes( ) { get("/notifications/channel-preferences") { call.withJwtAuth(authService) { userId -> - if (!call.requirePushEnabled(service)) return@withJwtAuth call.respond(service.listPreferences(userId)) } } put("/notifications/channel-preferences") { call.withJwtAuth(authService) { userId -> - if (!call.requirePushEnabled(service)) return@withJwtAuth val request = runCatching { call.receive() }.getOrNull() ?: return@withJwtAuth call.respond( HttpStatusCode.BadRequest, diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt index 14abe962..ff8bbc89 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt @@ -31,13 +31,16 @@ fun Route.recommendationEventsRoutes(eventService: RecommendationEventService, a if ((request.eventType == "click" || request.eventType == "watch") && request.videoUrl.isNullOrBlank()) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoUrl")) } - if (request.watchRatio != null && (request.watchRatio < 0.0 || request.watchRatio > 1.0)) { + val watchRatio = request.watchRatio + if (watchRatio != null && (watchRatio < 0.0 || watchRatio > 1.0)) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid watchRatio")) } - if (request.watchDurationMs != null && request.watchDurationMs < 0) { + val watchDurationMs = request.watchDurationMs + if (watchDurationMs != null && watchDurationMs < 0) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid watchDurationMs")) } - if (request.contextKey != null && request.contextKey.length > 120) { + val contextKeyValue = request.contextKey + if (contextKeyValue != null && contextKeyValue.length > 120) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid contextKey")) } val contextKey = request.contextKey ?: HomeRecommendationContextualBandit.contextKey( diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt index b36b0faa..3fe90e12 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt @@ -82,15 +82,6 @@ fun Route.rssPublicRoutes(service: RssFeedReaderService) { } } -internal suspend fun ApplicationCall.respondRssError(error: RssFeedException) { - val status = when (error.code) { - "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound - "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden - "rss_feed_limit_reached" -> HttpStatusCode.Conflict - else -> HttpStatusCode.BadRequest - } - respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) -} private suspend inline fun ApplicationCall.withRssUser( authService: AuthService, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt index b98707ce..e10ac666 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt @@ -69,9 +69,9 @@ internal class SabrManifestHandler( } } is ExtractionResult.Failure -> - return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) is ExtractionResult.BadRequest -> - return call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + return call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) } } val requestedStartTimeMs = call.request.queryParameters["playerTimeMs"]?.toLongOrNull()?.coerceAtLeast(0L) @@ -89,11 +89,11 @@ internal class SabrManifestHandler( prepared.info, audioToken?.selectedItag ?: call.request.queryParameters["audioItag"]?.toIntOrNull(), audioToken?.selectedAudioTrackId ?: call.request.queryParameters["audioTrackId"], - requireAac = true, + requireAac = false, ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR audio for this video"), + ErrorResponse("No SABR audio for this video", "no_playable_streams"), ) val video = SabrFormatSelector.video( prepared.info, @@ -101,7 +101,7 @@ internal class SabrManifestHandler( ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR video for this video"), + ErrorResponse("No SABR video for this video", "no_playable_streams"), ) val startTimeMs = downloadRange?.startTimeMs(audio, video, audioOnly) ?: requestedStartTimeMs val userId = audioToken?.userId ?: access?.userId ?: videoId @@ -149,7 +149,7 @@ internal class SabrManifestHandler( if (preflight(holder, startTimeMs)) return holder sabrSessionStore.release(holder) val refreshed = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = false) ?: prepared - val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = true) + val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = false) ?: return null val refreshedVideo = SabrFormatSelector.video(refreshed.info, video.itag) ?: return null val fresh = createHolder(videoId, userId, refreshed, refreshedAudio, refreshedVideo, startTimeMs, purpose) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 7f7f26dc..87ec3e89 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -12,12 +12,12 @@ import dev.typetype.server.services.SabrPlaybackSegmentResult import dev.typetype.server.services.SabrPlaybackSessionService import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import dev.typetype.server.services.markServed import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall -import io.ktor.server.request.receive import io.ktor.server.response.respond import dev.typetype.server.sabr.YoutubeSabrFormat @@ -33,16 +33,30 @@ internal class SabrPlaybackHandler( private val playbackService = SabrPlaybackSessionService(sabrSessionStore) private val infoResolver = SabrPlaybackInfoResolver(sabrSessionStore, authenticatedSabrInfoService) private val accessValidator = SabrPlaybackAccessValidator(streamService, youtubeSessionStreamInfo) + private val prewarmHandoffs = SabrPlaybackPrewarmHandoffs() suspend fun create(call: ApplicationCall, videoId: String) { val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return if (!validateAccess(call, videoId, access)) return val request = call.playbackRequest() val startTimeMs = request.effectiveStartTimeMs() + val isPrewarm = call.request.queryParameters["prewarm"]?.toBooleanStrictOrNull() == true + val handoffKey = prewarmHandoffs.key(access.userId, videoId, request, startTimeMs) + if (!isPrewarm) { + val handoff = handoffKey?.let(prewarmHandoffs::take) + val existing = handoff?.let { playbackService.lookup(it.sessionToken) } + if (existing != null && existing.terminalFailure() == null) { + PlaybackTraceLog.record("sabr_prewarm_handoff", "result=hit videoItag=${existing.videoFormat.itag} audioItag=${existing.audioFormat.itag} audioTrackId=${existing.audioFormat.audioTrackId} startTimeMs=${handoff.startTimeMs} ready=${handoff.ready}") + return respondPrepared(call, existing, videoId, handoff.startTimeMs, handoff.ready) + } + } + val infoStartedAt = System.nanoTime() val prepared = infoResolver.initial(access.userId, videoId, startTimeMs) - ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) + PlaybackTraceLog.record("sabr_info", "durationMs=${(System.nanoTime() - infoStartedAt) / 1_000_000} result=${if (prepared == null) "miss" else "ready"} startTimeMs=$startTimeMs") + prepared ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = selectAudio(call, prepared, request) ?: return val video = selectVideo(call, prepared, request) ?: return + val prepareStartedAt = System.nanoTime() val preparation = playbackService.prepare( videoId = videoId, userId = access.userId ?: videoId, @@ -53,7 +67,12 @@ internal class SabrPlaybackHandler( audioOnly = request.audioOnly, isLive = request.isLive, ) + PlaybackTraceLog.record("sabr_session_prepare", "durationMs=${(System.nanoTime() - prepareStartedAt) / 1_000_000} ready=${preparation.ready} videoItag=${video.itag} audioItag=${audio.itag} audioTrackId=${audio.audioTrackId} audioOnly=${request.audioOnly} isLive=${request.isLive} startTimeMs=${preparation.startTimeMs}") preparation.holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) + if (isPrewarm && handoffKey != null && preparation.holder.terminalFailure() == null) { + prewarmHandoffs.remember(handoffKey, SabrPlaybackPrewarmSession(preparation.holder.sessionToken, preparation.startTimeMs, preparation.ready)) + PlaybackTraceLog.record("sabr_prewarm_handoff", "result=stored videoItag=${video.itag} audioItag=${audio.itag} audioTrackId=${audio.audioTrackId} startTimeMs=$startTimeMs") + } respondPrepared(call, preparation.holder, videoId, preparation.startTimeMs, preparation.ready) } @@ -72,7 +91,7 @@ internal class SabrPlaybackHandler( prepared.info, request.audioItag ?: holder.audioFormat.itag, request.audioTrackId ?: holder.audioFormat.audioTrackId, - requireAac = true, + requireAac = false, ) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) val video = SabrFormatSelector.video(prepared.info, request.videoItag ?: holder.videoFormat.itag) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR video for this video")) @@ -155,26 +174,6 @@ internal class SabrPlaybackHandler( } } - private suspend fun ApplicationCall.playbackRequest(): SabrPlaybackRequest { - val body = runCatching { receive() }.getOrNull() - return SabrPlaybackRequest( - videoItag = body?.videoItag ?: request.queryParameters["videoItag"]?.toIntOrNull(), - audioItag = body?.audioItag ?: request.queryParameters["audioItag"]?.toIntOrNull(), - audioTrackId = body?.audioTrackId ?: request.queryParameters["audioTrackId"], - startTimeMs = body?.startTimeMs ?: request.queryParameters["startTimeMs"]?.toLongOrNull(), - playerTimeMs = body?.playerTimeMs ?: request.queryParameters["playerTimeMs"]?.toLongOrNull(), - audioOnly = body?.audioOnly ?: request.queryParameters["audioOnly"]?.toBooleanStrictOrNull() ?: false, - isLive = body?.isLive ?: request.queryParameters["isLive"]?.toBooleanStrictOrNull() ?: false, - ) - } - - private fun SabrPlaybackRequest.effectiveStartTimeMs(): Long = - (playerTimeMs ?: startTimeMs ?: 0L).coerceAtLeast(0L) - - private fun SabrPlaybackRequest.keepsCurrentFormats(holder: SabrSessionHolder): Boolean = - (videoItag == null || videoItag == holder.videoFormat.itag) && - (audioItag == null || audioItag == holder.audioFormat.itag) && - (audioTrackId.isNullOrBlank() || audioTrackId == holder.audioFormat.audioTrackId) private suspend fun selectAudio( call: ApplicationCall, @@ -184,7 +183,7 @@ internal class SabrPlaybackHandler( prepared.info, request.audioItag, request.audioTrackId, - requireAac = true, + requireAac = false, ) ?: run { call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) null diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt new file mode 100644 index 00000000..97fea167 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.routes + +import java.util.LinkedHashMap + +internal data class SabrPlaybackPrewarmKey( + val userId: String, + val videoId: String, + val videoItag: Int, + val audioItag: Int, + val audioTrackId: String?, + val startTimeMs: Long, + val audioOnly: Boolean, + val isLive: Boolean, +) + +internal data class SabrPlaybackPrewarmSession( + val sessionToken: String, + val startTimeMs: Long, + val ready: Boolean, +) + +internal class SabrPlaybackPrewarmHandoffs( + private val maxEntries: Int = MAX_ENTRIES, + private val ttlNanos: Long = TTL_NANOS, +) { + private data class Entry(val session: SabrPlaybackPrewarmSession, val expiresAtNanos: Long) + + private val entries = LinkedHashMap() + + fun key( + userId: String?, + videoId: String, + request: SabrPlaybackRequest, + startTimeMs: Long, + ): SabrPlaybackPrewarmKey? { + val scope = userId?.takeIf { it.isNotBlank() } ?: return null + val videoItag = request.videoItag ?: return null + val audioItag = request.audioItag ?: return null + return SabrPlaybackPrewarmKey( + userId = scope, + videoId = videoId, + videoItag = videoItag, + audioItag = audioItag, + audioTrackId = request.audioTrackId, + startTimeMs = startTimeMs, + audioOnly = request.audioOnly, + isLive = request.isLive, + ) + } + + @Synchronized + fun remember(key: SabrPlaybackPrewarmKey, session: SabrPlaybackPrewarmSession) { + val now = System.nanoTime() + evictExpired(now) + entries[key] = Entry(session, now + ttlNanos) + while (entries.size > maxEntries) { + entries.remove(entries.keys.first()) + } + } + + @Synchronized + fun take(key: SabrPlaybackPrewarmKey): SabrPlaybackPrewarmSession? { + val now = System.nanoTime() + val entry = entries.remove(key) ?: return null + return entry.session.takeUnless { now - entry.expiresAtNanos >= 0L } + } + + private fun evictExpired(now: Long) { + val iterator = entries.values.iterator() + while (iterator.hasNext()) { + if (now - iterator.next().expiresAtNanos >= 0L) iterator.remove() + } + } + + private companion object { + const val MAX_ENTRIES = 64 + const val TTL_NANOS = 60_000_000_000L + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt new file mode 100644 index 00000000..3c3b658b --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt @@ -0,0 +1,26 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive + +internal suspend fun ApplicationCall.playbackRequest(): SabrPlaybackRequest { + val body = runCatching { receive() }.getOrNull() + return SabrPlaybackRequest( + videoItag = body?.videoItag ?: request.queryParameters["videoItag"]?.toIntOrNull(), + audioItag = body?.audioItag ?: request.queryParameters["audioItag"]?.toIntOrNull(), + audioTrackId = body?.audioTrackId ?: request.queryParameters["audioTrackId"], + startTimeMs = body?.startTimeMs ?: request.queryParameters["startTimeMs"]?.toLongOrNull(), + playerTimeMs = body?.playerTimeMs ?: request.queryParameters["playerTimeMs"]?.toLongOrNull(), + audioOnly = body?.audioOnly ?: request.queryParameters["audioOnly"]?.toBooleanStrictOrNull() ?: false, + isLive = body?.isLive ?: request.queryParameters["isLive"]?.toBooleanStrictOrNull() ?: false, + ) +} + +internal fun SabrPlaybackRequest.effectiveStartTimeMs(): Long = + (playerTimeMs ?: startTimeMs ?: 0L).coerceAtLeast(0L) + +internal fun SabrPlaybackRequest.keepsCurrentFormats(holder: SabrSessionHolder): Boolean = + (videoItag == null || videoItag == holder.videoFormat.itag) && + (audioItag == null || audioItag == holder.audioFormat.itag) && + (audioTrackId.isNullOrBlank() || audioTrackId == holder.audioFormat.audioTrackId) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt new file mode 100644 index 00000000..f31e90e7 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -0,0 +1,107 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrInitializationData +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.livePlaybackSnapshot +import dev.typetype.server.services.resolvePlaybackStartMs + +internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessionStore) { + private val trackBuilder = SabrPlaybackWindowTrackBuilder(sabrSessionStore) + + suspend fun build( + holder: SabrSessionHolder, + request: SabrPlaybackWindowRequest, + ): SabrPlaybackWindowBuildResult { + val startTimeMs = holder.resolvePlaybackStartMs(request.playerTimeMs) + val effectiveRequest = if (startTimeMs == request.playerTimeMs) request else request.copy(playerTimeMs = startTimeMs) + val live = holder.livePlaybackSnapshot() + SabrInitializationData.ingestRemembered(holder.audioFormat, holder) + if (!effectiveRequest.audioOnly) SabrInitializationData.ingestRemembered(holder.videoFormat, holder) + if (effectiveRequest.audioOnly) return buildAudioOnly(holder, effectiveRequest, live?.toResponse()) + val activeLive = live?.active == true + var video = trackBuilder.buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, activeLive) + val decodeStartMs = video.track.segments.firstOrNull()?.startMs ?: effectiveRequest.playerTimeMs + val audioStartMs = if (activeLive) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs + var audio = trackBuilder.buildTrack( + holder, + holder.audioFormat, + effectiveRequest, + audioStartMs, + activeLive, + ) + var playbackStartMs = resolvedPlaybackStartMs( + effectiveRequest.playerTimeMs, + activeLive, + video.track, + audio.track, + ) + val readyAheadMs = readyAheadMs(effectiveRequest, activeLive) + var requestedReadyEndMs = playbackStartMs + readyAheadMs + if (activeLive) { + if (!video.covers(requestedReadyEndMs) && video.blockedRequest == null) { + video = trackBuilder.buildTrack( + holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, activeLive, + requestedReadyEndMs, + ) + } + if (!audio.covers(requestedReadyEndMs) && audio.blockedRequest == null) { + audio = trackBuilder.buildTrack( + holder, holder.audioFormat, effectiveRequest, audioStartMs, activeLive, requestedReadyEndMs, + ) + } + playbackStartMs = resolvedPlaybackStartMs( + effectiveRequest.playerTimeMs, activeLive, video.track, audio.track, + ) + requestedReadyEndMs = playbackStartMs + readyAheadMs + } + val blocked = blockedTrack(audio, video) + return SabrPlaybackWindowBuildResult( + response = SabrPlaybackWindowReadyResponse( + sessionId = holder.sessionToken, + generation = holder.activeGeneration(), + ready = true, + retryAfterMs = null, + durationMs = holder.durationMs(), + endOfStream = live?.active != true && audio.atEnd && video.atEnd, + audio = audio.track, + video = video.track, + startTimeMs = playbackStartMs, + live = live?.toResponse(), + ), + blockedBy = blocked?.blockedBy, + blockedRequests = listOfNotNull(video.blockedRequest, audio.blockedRequest), + isReady = audio.covers(holder.readyEndMs(holder.audioFormat, requestedReadyEndMs)) && + video.covers(holder.readyEndMs(holder.videoFormat, requestedReadyEndMs)), + ) + } + private suspend fun buildAudioOnly( + holder: SabrSessionHolder, + request: SabrPlaybackWindowRequest, + live: SabrLivePlaybackResponse?, + ): SabrPlaybackWindowBuildResult { + val audio = trackBuilder.buildTrack(holder, holder.audioFormat, request, request.playerTimeMs, live?.active == true) + val playbackStartMs = resolvedPlaybackStartMs(request.playerTimeMs, live?.active == true, audio.track) + val readyEndMs = playbackStartMs + readyAheadMs(request, live?.active == true) + return SabrPlaybackWindowBuildResult( + response = SabrPlaybackWindowReadyResponse( + sessionId = holder.sessionToken, + generation = holder.activeGeneration(), + ready = true, + retryAfterMs = null, + durationMs = holder.durationMs(), + endOfStream = live?.active != true && audio.atEnd, + audio = audio.track, + startTimeMs = playbackStartMs, + live = live, + ), + blockedBy = audio.blockedBy, + blockedRequests = listOfNotNull(audio.blockedRequest), + isReady = audio.covers(holder.readyEndMs(holder.audioFormat, readyEndMs)), + ) + } + private fun blockedTrack(audio: TrackBuildResult, video: TrackBuildResult): TrackBuildResult? = + sequenceOf(video, audio) + .filter { it.blockedRequest != null } + .minByOrNull { it.coveredEndMs } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt index 9d0aee52..3f21c92f 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt @@ -12,7 +12,7 @@ internal fun SabrSessionHolder.durationMs(): Long { } val audioEndMs = indexedEndMs(audioFormat) val videoEndMs = indexedEndMs(videoFormat) - if (audioEndMs > 0L && videoEndMs > 0L) return maxOf(audioEndMs, videoEndMs) + if (audioEndMs > 0L && videoEndMs > 0L) return minOf(audioEndMs, videoEndMs) return maxOf(audioFormat.approxDurationMs, videoFormat.approxDurationMs, 0L) } @@ -35,7 +35,12 @@ internal fun readyAheadMs(request: SabrPlaybackWindowRequest, activeLive: Boolea } else { MIN_READY_AHEAD_MS } - return minOf(request.bufferGoalMs.coerceAtLeast(1L), minimum) + return minOf(liveWindowBufferGoalMs(request.bufferGoalMs, activeLive), minimum) +} + +internal fun liveWindowBufferGoalMs(requestedMs: Long, activeLive: Boolean): Long { + val requested = requestedMs.coerceAtLeast(1L) + return if (activeLive) minOf(requested, LIVE_STARTUP_READY_AHEAD_MS) else requested } internal fun YoutubeSabrFormat.trackName(): String = if (isAudio) "audio" else "video" @@ -78,4 +83,4 @@ internal fun SabrSessionHolder.previousPlaybackSequence( } private const val MIN_READY_AHEAD_MS = 1_000L -private const val LIVE_STARTUP_READY_AHEAD_MS = 8_000L +private const val LIVE_STARTUP_READY_AHEAD_MS = 2_000L diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt similarity index 52% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt index 183a4a76..74b2eaab 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -1,120 +1,47 @@ package dev.typetype.server.routes import dev.typetype.server.services.CachedSabrSegment -import dev.typetype.server.services.SabrInitializationData import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.coversPlaybackTime +import dev.typetype.server.services.failLivePlaybackDiscontinuity import dev.typetype.server.services.findCachedPlaybackMediaAt import dev.typetype.server.services.isAcceptableLiveFollowingSegment -import dev.typetype.server.services.failLivePlaybackDiscontinuity -import dev.typetype.server.services.livePlaybackSnapshot import dev.typetype.server.services.playbackContinuationSequence import dev.typetype.server.services.playbackSegmentStartMs -import dev.typetype.server.services.resolvePlaybackStartMs import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat private const val MAX_SEGMENTS_PER_TRACK = 12 -internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessionStore) { - suspend fun build( - holder: SabrSessionHolder, - request: SabrPlaybackWindowRequest, - ): SabrPlaybackWindowBuildResult { - val startTimeMs = holder.resolvePlaybackStartMs(request.playerTimeMs) - val effectiveRequest = if (startTimeMs == request.playerTimeMs) request else request.copy(playerTimeMs = startTimeMs) - val live = holder.livePlaybackSnapshot() - SabrInitializationData.ingestRemembered(holder.audioFormat, holder) - if (!effectiveRequest.audioOnly) SabrInitializationData.ingestRemembered(holder.videoFormat, holder) - if (effectiveRequest.audioOnly) return buildAudioOnly(holder, effectiveRequest, live?.toResponse()) - val video = buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, live?.active == true) - val decodeStartMs = video.track.segments.firstOrNull()?.startMs ?: effectiveRequest.playerTimeMs - val audioStartMs = if (live?.active == true) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs - val audio = buildTrack( - holder, - holder.audioFormat, - effectiveRequest, - audioStartMs, - live?.active == true, - ) - val playbackStartMs = resolvedPlaybackStartMs( - effectiveRequest.playerTimeMs, - live?.active == true, - video.track, - audio.track, - ) - val blocked = blockedTrack(audio, video) - val readyAheadMs = readyAheadMs(effectiveRequest, live?.active == true) - val requestedReadyEndMs = playbackStartMs + readyAheadMs - return SabrPlaybackWindowBuildResult( - response = SabrPlaybackWindowReadyResponse( - sessionId = holder.sessionToken, - generation = holder.activeGeneration(), - ready = true, - retryAfterMs = null, - durationMs = holder.durationMs(), - endOfStream = live?.active != true && audio.atEnd && video.atEnd, - audio = audio.track, - video = video.track, - startTimeMs = playbackStartMs, - live = live?.toResponse(), - ), - blockedBy = blocked?.blockedBy, - blockedRequests = listOfNotNull(video.blockedRequest, audio.blockedRequest), - isReady = audio.covers(holder.readyEndMs(holder.audioFormat, requestedReadyEndMs)) && - video.covers(holder.readyEndMs(holder.videoFormat, requestedReadyEndMs)), - ) - } - private suspend fun buildAudioOnly( - holder: SabrSessionHolder, - request: SabrPlaybackWindowRequest, - live: SabrLivePlaybackResponse?, - ): SabrPlaybackWindowBuildResult { - val audio = buildTrack(holder, holder.audioFormat, request, request.playerTimeMs, live?.active == true) - val playbackStartMs = resolvedPlaybackStartMs(request.playerTimeMs, live?.active == true, audio.track) - val readyEndMs = playbackStartMs + readyAheadMs(request, live?.active == true) - return SabrPlaybackWindowBuildResult( - response = SabrPlaybackWindowReadyResponse( - sessionId = holder.sessionToken, - generation = holder.activeGeneration(), - ready = true, - retryAfterMs = null, - durationMs = holder.durationMs(), - endOfStream = live?.active != true && audio.atEnd, - audio = audio.track, - startTimeMs = playbackStartMs, - live = live, - ), - blockedBy = audio.blockedBy, - blockedRequests = listOfNotNull(audio.blockedRequest), - isReady = audio.covers(holder.readyEndMs(holder.audioFormat, readyEndMs)), - ) - } - private fun blockedTrack(audio: TrackBuildResult, video: TrackBuildResult): TrackBuildResult? = - sequenceOf(video, audio) - .filter { it.blockedRequest != null } - .minByOrNull { it.coveredEndMs } - private suspend fun buildTrack( +internal class SabrPlaybackWindowTrackBuilder(private val sabrSessionStore: SabrSessionStore) { + suspend fun buildTrack( holder: SabrSessionHolder, format: YoutubeSabrFormat, request: SabrPlaybackWindowRequest, requestedStartMs: Long, activeLive: Boolean, + requiredEndMs: Long? = null, ): TrackBuildResult { val continuesServedTrack = !activeLive || holder.lastServedSequence(format) != null val targetMs = if (continuesServedTrack) request.bufferedEndFor(format, requestedStartMs) else requestedStartMs.coerceAtLeast(0L) val goalStartMs = if (activeLive) targetMs else request.playerTimeMs.coerceAtLeast(0L) - var goalEndMs = maxOf(goalStartMs, request.playerTimeMs.coerceAtLeast(0L)) + - request.bufferGoalMs.coerceAtLeast(1L) + val liveBufferGoalMs = liveWindowBufferGoalMs(request.bufferGoalMs, activeLive) + var goalEndMs = maxOf(goalStartMs, request.playerTimeMs.coerceAtLeast(0L)) + liveBufferGoalMs + if (activeLive && requiredEndMs != null) goalEndMs = maxOf(goalEndMs, requiredEndMs) val segments = mutableListOf() var blockedBy: String? = null var blockedRequest: SabrSegmentRequest? = null - var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) - var coveredEndMs = targetMs val endSequence = if (activeLive) 0 else holder.session.streamState.getEndSegment(format).toInt() - var atEnd = false - while (segments.size < MAX_SEGMENTS_PER_TRACK) { + val endTimeMs = if (endSequence > 0) { + holder.session.streamState.getSegmentEndMs(format, endSequence).takeIf { it > 0L } ?: 0L + } else { + 0L + } + var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) + var coveredEndMs = if (endTimeMs > 0L && targetMs >= endTimeMs) endTimeMs else targetMs + var atEnd = endTimeMs > 0L && targetMs >= endTimeMs + while (!atEnd && segments.size < MAX_SEGMENTS_PER_TRACK) { if (endSequence > 0 && seq > endSequence) { atEnd = true break @@ -139,13 +66,19 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi holder, format, seq, expectedStartMs, ) if (progressive != null) { + if (!progressive.hasReadableMedia) { + blockedRequest = mediaRequest + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + break + } seq = progressive.nextSequence coveredEndMs = progressive.coveredEndMs + if (endSequence > 0 && seq > endSequence) { + atEnd = true + break + } if (coveredEndMs >= goalEndMs) break - if (progressive.hasReadableMedia) continue - blockedRequest = SabrSegmentRequest.media(format, seq) - blockedBy = "${format.trackName()}:${format.itag}:$seq pending" - break + continue } blockedBy = "${format.trackName()}:${format.itag}:$seq pending" blockedRequest = mediaRequest @@ -169,7 +102,7 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi ) segments += windowSegment if (activeLive && segments.size == 1) { - goalEndMs = maxOf(goalEndMs, windowSegment.startMs + request.bufferGoalMs.coerceAtLeast(1L)) + goalEndMs = maxOf(goalEndMs, windowSegment.startMs + liveBufferGoalMs) } coveredEndMs = windowSegment.startMs + windowSegment.durationMs if (endSequence > 0 && seq >= endSequence) { @@ -195,6 +128,7 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi atEnd = atEnd, ) } + private fun CachedSabrSegment.toWindowSegment( holder: SabrSessionHolder, format: YoutubeSabrFormat, @@ -208,13 +142,15 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi durationMs = resolvedDurationMs, ) } - private data class TrackBuildResult( - val track: SabrPlaybackWindowTrack, - val blockedBy: String?, - val blockedRequest: SabrSegmentRequest?, - val coveredEndMs: Long, - val atEnd: Boolean, - ) { - fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs - } +} + +internal data class TrackBuildResult( + val track: SabrPlaybackWindowTrack, + val blockedBy: String?, + val blockedRequest: SabrSegmentRequest?, + val coveredEndMs: Long, + val atEnd: Boolean, +) { + fun covers(requiredEndMs: Long): Boolean = + (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt index acd908d5..d292c9ec 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt @@ -39,11 +39,11 @@ internal class SabrSessionDescriptorHandler( } is ExtractionResult.Failure -> return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse(result.message), + ErrorResponse(result.message, result.code), ) is ExtractionResult.BadRequest -> return call.respond( HttpStatusCode.BadRequest, - ErrorResponse(result.message), + ErrorResponse(result.message, result.code), ) } } @@ -56,10 +56,10 @@ internal class SabrSessionDescriptorHandler( prepared.info, call.request.queryParameters["audioItag"]?.toIntOrNull(), call.request.queryParameters["audioTrackId"], - requireAac = true, + requireAac = false, ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR audio for this video"), + ErrorResponse("No SABR audio for this video", "no_playable_streams"), ) val video = SabrFormatSelector.video( prepared.info, @@ -67,7 +67,7 @@ internal class SabrSessionDescriptorHandler( ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR video for this video"), + ErrorResponse("No SABR video for this video", "no_playable_streams"), ) val userId = access.userId ?: videoId val holder = createHolder(videoId, userId, prepared, audio, video, startTimeMs) @@ -109,7 +109,7 @@ internal class SabrSessionDescriptorHandler( if (preflight(holder, startTimeMs)) return holder sabrSessionStore.release(holder) val refreshed = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = false) ?: prepared - val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = true) + val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = false) ?: return null val refreshedVideo = SabrFormatSelector.video(refreshed.info, video.itag) ?: return null val fresh = createHolder(videoId, userId, refreshed, refreshedAudio, refreshedVideo, startTimeMs) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt similarity index 65% rename from src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt index f966d9a4..19135777 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt @@ -4,8 +4,11 @@ import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.withSabrManifestUrls import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo +import java.net.URLEncoder +import java.nio.charset.StandardCharsets internal suspend fun StreamResponse.withPlayableSabrStreams( url: String, @@ -15,13 +18,14 @@ internal suspend fun StreamResponse.withPlayableSabrStreams( val videoId = url.youtubeVideoId() ?: return this val prepared = sabrSessionStore.fetchInfo(videoId, cachedFirst = true) ?: return withoutSabrStreams() - val hasAudio = SabrFormatSelector.audio(prepared.info, null, null, requireAac = true) != null + val hasAudio = SabrFormatSelector.audio(prepared.info, null, null, requireAac = false) != null if (!hasAudio) return withoutSabrStreams() val enriched = withMissingSabrVideoStreams(videoId, prepared.info) + .withMissingSabrAudioStreams(videoId, prepared.info) return enriched.copy( videoStreams = enriched.videoStreams.filter { it.isPlayableSabrVideo(prepared.info) }, videoOnlyStreams = enriched.videoOnlyStreams.filter { it.isPlayableSabrVideo(prepared.info) }, - audioStreams = audioStreams.filter { it.isPlayableSabrAudio(prepared.info) }, + audioStreams = enriched.audioStreams.filter { it.isPlayableSabrAudio(prepared.info) }, ) } @@ -35,6 +39,17 @@ private fun StreamResponse.withMissingSabrVideoStreams(videoId: String, info: Yo return copy(videoOnlyStreams = videoOnlyStreams + missing) } +private fun StreamResponse.withMissingSabrAudioStreams(videoId: String, info: YoutubeSabrInfo): StreamResponse { + val existing = audioStreams.map { it.itag to it.audioTrackId }.toSet() + val missing = info.formats.asSequence() + .filter { it.isAudio && (it.itag to it.audioTrackId) !in existing } + .filter { SabrFormatSelector.audio(info, it.itag, it.audioTrackId, requireAac = false) != null } + .mapNotNull { it.toAudioStreamItem(videoId) } + .toList() + if (missing.isEmpty()) return this + return copy(audioStreams = audioStreams + missing) +} + internal fun StreamResponse.hasSabrStreams(): Boolean = videoStreams.any { it.deliveryMethod == SABR_DELIVERY_METHOD } || videoOnlyStreams.any { it.deliveryMethod == SABR_DELIVERY_METHOD } || @@ -46,6 +61,14 @@ internal fun StreamResponse.withoutSabrStreams(): StreamResponse = copy( audioStreams = audioStreams.filterNot { it.deliveryMethod == SABR_DELIVERY_METHOD }, ) +internal fun StreamResponse.onlyLiveHls(): StreamResponse = copy( + dashMpdUrl = "", + videoStreams = emptyList(), + videoOnlyStreams = emptyList(), + audioStreams = emptyList(), + hasLiveManifest = hlsUrl.isNotBlank(), +) + internal fun StreamResponse.onlySabrStreams(): StreamResponse = copy( hlsUrl = "", dashMpdUrl = "", @@ -59,7 +82,7 @@ private fun VideoStreamItem.isPlayableSabrVideo(info: YoutubeSabrInfo): Boolean private fun AudioStreamItem.isPlayableSabrAudio(info: YoutubeSabrInfo): Boolean = deliveryMethod != SABR_DELIVERY_METHOD || - SabrFormatSelector.audio(info, itag, audioTrackId, requireAac = true) != null + SabrFormatSelector.audio(info, itag, audioTrackId, requireAac = false) != null private fun YoutubeSabrFormat.toVideoStreamItem(videoId: String, info: YoutubeSabrInfo): VideoStreamItem? { if (SabrFormatSelector.video(info, itag) == null) return null @@ -88,6 +111,35 @@ private fun YoutubeSabrFormat.toVideoStreamItem(videoId: String, info: YoutubeSa ) } +private fun YoutubeSabrFormat.toAudioStreamItem(videoId: String): AudioStreamItem? { + val mime = mimeType?.takeIf { it.isNotBlank() } ?: return null + val container = mime.substringBefore(';').trim() + val track = audioTrackId?.takeIf { it.isNotBlank() } + ?.let { "&audioTrackId=${URLEncoder.encode(it, StandardCharsets.UTF_8)}" } + .orEmpty() + return AudioStreamItem( + url = "", + mimeType = container, + format = container.substringAfter('/').uppercase(), + bitrate = bitrate.takeIf { it > 0 }, + codec = mime.codec(), + quality = audioQuality, + itag = itag, + contentLength = contentLength.coerceAtLeast(0L), + initStart = 0L, + initEnd = 0L, + indexStart = 0L, + indexEnd = 0L, + audioTrackId = audioTrackId, + audioTrackName = audioTrackDisplayName, + audioLocale = audioTrackId?.substringBefore('.'), + isOriginal = isOriginalAudio, + deliveryMethod = SABR_DELIVERY_METHOD, + manifestUrl = "/sabr/manifest/$videoId", + sabrSessionUrl = "/sabr/session/$videoId?audioItag=$itag$track", + ) +} + private fun YoutubeSabrFormat.heightLabel(): String = height.takeIf { it > 0 }?.let { "${it}p" } ?: "" private fun String.formatName(): String = when (lowercase()) { @@ -108,3 +160,6 @@ private fun String.codec(): String? { } private const val SABR_DELIVERY_METHOD = "sabr" + +internal fun StreamResponse.forSabrPlayback(): StreamResponse = + if (isLive) onlyLiveHls() else withSabrManifestUrls().onlySabrStreams() diff --git a/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt index cfeaec91..11d3d3f2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.models.SessionPlaybackStartRequest import dev.typetype.server.models.SessionPlaybackStopRequest import dev.typetype.server.services.ActiveSessionService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.PresenceService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.request.receive @@ -14,7 +15,11 @@ import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.post -fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: ActiveSessionService): Unit { +fun Route.sessionActivityRoutes( + authService: AuthService, + activeSessionService: ActiveSessionService, + presenceService: PresenceService = PresenceService(), +): Unit { post("/sessions/activity") { call.withJwtAuth(authService) { userId -> val body = runCatching { call.receive() }.getOrElse { @@ -34,6 +39,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing now playing fields")) } activeSessionService.reportPlaybackStart(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackStart(userId, body) call.respond(HttpStatusCode.NoContent) } } @@ -44,6 +50,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } activeSessionService.reportPlaybackProgress(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackProgress(userId, body) call.respond(HttpStatusCode.NoContent) } } @@ -54,6 +61,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } activeSessionService.reportPlaybackStop(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackStop(userId) call.respond(HttpStatusCode.NoContent) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt index 6ab72c9e..94beff2c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt @@ -4,6 +4,7 @@ import java.net.URI internal enum class StreamDeliveryMode { YoutubeSabr, + YoutubeLiveHls, NicoNico, BiliBili; @@ -11,10 +12,13 @@ internal enum class StreamDeliveryMode { val host = runCatching { URI(url).host?.lowercase() }.getOrNull() ?: return false return when (this) { YoutubeSabr -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" + YoutubeLiveHls -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" NicoNico -> host == "nico.ms" || host.endsWith(".nicovideo.jp") || host == "nicovideo.jp" BiliBili -> host == "b23.tv" || host.endsWith(".bilibili.com") || host == "bilibili.com" } } fun isSabr(): Boolean = this == YoutubeSabr + + fun isYoutube(): Boolean = this == YoutubeSabr || this == YoutubeLiveHls } diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 24addc64..b4bde0f4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -18,4 +18,5 @@ internal data class StreamRouteDependencies( val providerMediaHandleService: ProviderMediaHandleService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, + val bilibiliSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt new file mode 100644 index 00000000..998045fb --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaType +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR +import dev.typetype.server.services.requiresYoutubeSession + +private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" +private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" +private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" + +internal fun streamCacheControl( + deliveryMode: StreamDeliveryMode, + isLive: Boolean, + userId: String?, +): String = when { + deliveryMode.isSabr() && isLive -> PROVIDER_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.YoutubeLiveHls -> PROVIDER_STREAMS_CACHE_CONTROL + userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.NicoNico || + deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL + else -> STREAMS_CACHE_CONTROL +} + +internal fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { + StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO + StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI + StreamDeliveryMode.YoutubeSabr -> null + StreamDeliveryMode.YoutubeLiveHls -> null +} + +internal data class StreamResolution( + val result: ExtractionResult, + val authenticated: Boolean = false, +) + +internal suspend fun resolveStreamInfo( + url: String, + deliveryMode: StreamDeliveryMode, + userId: String?, + publicResult: ExtractionResult, + dependencies: StreamRouteDependencies, +): StreamResolution { + if (deliveryMode == StreamDeliveryMode.BiliBili) { + val bilibiliInfo = dependencies.bilibiliSessionStreamInfo + if (bilibiliInfo != null && userId != null) { + val bilibiliResult = bilibiliInfo(userId, url) + if (bilibiliResult != null) return StreamResolution(bilibiliResult, authenticated = true) + } + return StreamResolution(publicResult) + } + val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo + if (!deliveryMode.isSabr() || authenticatedInfo == null) { + return StreamResolution(publicResult) + } + val authenticatedResult = userId?.let { authenticatedInfo(it, url) } + if (authenticatedResult != null) { + val authenticatedLive = (authenticatedResult as? ExtractionResult.Success)?.data?.isLive == true + val publicLive = (publicResult as? ExtractionResult.Success)?.data?.isLive == true + if (!authenticatedLive || !publicLive) { + return StreamResolution(authenticatedResult, authenticated = true) + } + } + return if (publicResult.requiresYoutubeSession()) { + StreamResolution( + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), + ) + } else { + StreamResolution(publicResult) + } +} + +internal fun StreamResponse.hasPlayableSource(): Boolean = + videoStreams.isNotEmpty() || + videoOnlyStreams.isNotEmpty() || + audioStreams.isNotEmpty() || + hlsUrl.isNotBlank() || + dashMpdUrl.isNotBlank() + +internal fun StreamResponse.withSignedPublicHlsUrl( + shouldSign: Boolean, + tokenService: PublicHlsManifestTokenService?, +): StreamResponse { + if (!shouldSign || tokenService == null || hlsUrl.isBlank()) return this + if (hlsUrl.startsWith("/streams/hls-manifest?token=")) return this + return copy(hlsUrl = tokenService.createPath(hlsUrl)) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index cc1070f3..7c472519 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -1,5 +1,6 @@ package dev.typetype.server.routes +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse @@ -10,14 +11,9 @@ import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.ProviderMediaHandleService -import dev.typetype.server.services.ProviderMediaType import dev.typetype.server.services.StreamService -import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE -import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR import dev.typetype.server.services.filterAllowed import dev.typetype.server.services.filterBlocked -import dev.typetype.server.services.requiresYoutubeSession -import dev.typetype.server.services.withSabrManifestUrls import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond @@ -25,9 +21,6 @@ import io.ktor.server.routing.Route import io.ktor.server.routing.get import kotlinx.coroutines.CancellationException -private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" -private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" -private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" fun Route.streamRoutes( streamService: StreamService, @@ -40,7 +33,9 @@ fun Route.streamRoutes( nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, + youtubeLiveHlsStreamService: StreamService = streamService, youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, + bilibiliSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, ) { val dependencies = StreamRouteDependencies( @@ -52,8 +47,10 @@ fun Route.streamRoutes( providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrStreamContractFilter, youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, + bilibiliSessionStreamInfo = bilibiliSessionStreamInfo, ) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) + streamRoute("/streams/youtube/live", StreamDeliveryMode.YoutubeLiveHls, youtubeLiveHlsStreamService, dependencies) streamRoute( "/streams/youtube/sabr/bootstrap", StreamDeliveryMode.YoutubeSabr, @@ -79,6 +76,8 @@ private fun Route.streamRoute( ErrorResponse("URL does not match stream endpoint", "provider_mismatch"), ) } + PlaybackTraceLog.record("stream_route_start", "deliveryMode=$deliveryMode") + val accessStartedAt = System.nanoTime() val access = call.accessProfileOrRespond( dependencies.authService, dependencies.accessControlService, @@ -88,14 +87,38 @@ private fun Route.streamRoute( val blockedProfile = access.userId ?.let { dependencies.blockedService?.profileFor(it) } ?: BlockedContentProfile.empty + PlaybackTraceLog.record( + "stream_route_access", + "durationMs=${(System.nanoTime() - accessStartedAt) / 1_000_000} guest=${access.userId == null}", + ) if (blockedProfile.blocksVideo(url)) { return@get call.respond( HttpStatusCode.Forbidden, ErrorResponse("Video is blocked", "content_blocked"), ) } + val providerStartedAt = System.nanoTime() val publicResult = streamService.getStreamInfo(url) + val providerResult = when (publicResult) { + is ExtractionResult.Success -> "success" + is ExtractionResult.Failure -> "failure" + is ExtractionResult.BadRequest -> "bad_request" + } + PlaybackTraceLog.record( + "stream_provider_info", + "durationMs=${(System.nanoTime() - providerStartedAt) / 1_000_000} result=$providerResult", + ) + val resolutionStartedAt = System.nanoTime() val resolution = resolveStreamInfo(url, deliveryMode, access.userId, publicResult, dependencies) + val resolutionResult = when (resolution.result) { + is ExtractionResult.Success -> "success" + is ExtractionResult.Failure -> "failure" + is ExtractionResult.BadRequest -> "bad_request" + } + PlaybackTraceLog.record( + "stream_resolution", + "durationMs=${(System.nanoTime() - resolutionStartedAt) / 1_000_000} result=$resolutionResult authenticated=${resolution.authenticated}", + ) when (val result = resolution.result) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { @@ -108,7 +131,9 @@ private fun Route.streamRoute( ) } val selected = if (deliveryMode.isSabr()) { - result.data.withSabrManifestUrls().onlySabrStreams() + result.data.forSabrPlayback() + } else if (deliveryMode == StreamDeliveryMode.YoutubeLiveHls) { + result.data.onlyLiveHls() } else { result.data.withoutSabrStreams() } @@ -116,7 +141,9 @@ private fun Route.streamRoute( .filterAllowed(accessProfile) .filterBlocked(blockedProfile) .withSignedPublicHlsUrl( - deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, + deliveryMode.isSabr() && selected.isLive || + deliveryMode == StreamDeliveryMode.YoutubeLiveHls && selected.isLive || + access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, ) val data = if ( @@ -130,8 +157,13 @@ private fun Route.streamRoute( } if (!data.hasPlayableSource()) { return@get call.respond( - HttpStatusCode.UnprocessableEntity, - ErrorResponse("No playable streams available", "no_playable_streams"), + if (deliveryMode.isYoutube()) { + HttpStatusCode.BadGateway + } else HttpStatusCode.UnprocessableEntity, + ErrorResponse( + "No compatible stream is available for this video", + "no_playable_streams", + ), ) } val publicData = try { @@ -151,68 +183,20 @@ private fun Route.streamRoute( } call.response.headers.append( HttpHeaders.CacheControl, - when { - access.userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL - deliveryMode == StreamDeliveryMode.NicoNico || - deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL - else -> STREAMS_CACHE_CONTROL - }, + streamCacheControl(deliveryMode, selected.isLive, access.userId), ) call.respond(publicData) } - is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) - is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) + is ExtractionResult.BadRequest -> call.respond( + if (deliveryMode.isYoutube()) youtubeBadRequestStatus(result.code) else HttpStatusCode.BadRequest, + ErrorResponse(result.message, result.code), + ) + is ExtractionResult.Failure -> call.respond( + if (deliveryMode.isYoutube()) { + youtubeExtractionFailureStatus(result) + } else HttpStatusCode.UnprocessableEntity, + ErrorResponse(result.message, result.code), + ) } } } - -private fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { - StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO - StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI - StreamDeliveryMode.YoutubeSabr -> null -} - -private data class StreamResolution( - val result: ExtractionResult, - val authenticated: Boolean = false, -) - -private suspend fun resolveStreamInfo( - url: String, - deliveryMode: StreamDeliveryMode, - userId: String?, - publicResult: ExtractionResult, - dependencies: StreamRouteDependencies, -): StreamResolution { - val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo - if (!deliveryMode.isSabr() || authenticatedInfo == null) { - return StreamResolution(publicResult) - } - val authenticatedResult = userId?.let { authenticatedInfo(it, url) } - if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) - return if (publicResult.requiresYoutubeSession()) { - StreamResolution( - ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), - ) - } else { - StreamResolution(publicResult) - } -} - -private fun StreamResponse.hasPlayableSource(): Boolean = - videoStreams.isNotEmpty() || - videoOnlyStreams.isNotEmpty() || - audioStreams.isNotEmpty() || - hlsUrl.isNotBlank() || - dashMpdUrl.isNotBlank() - -private fun StreamResponse.withSignedPublicHlsUrl( - shouldSign: Boolean, - tokenService: PublicHlsManifestTokenService?, -): StreamResponse { - if (!shouldSign || tokenService == null || hlsUrl.isBlank()) return this - if (hlsUrl.startsWith("/streams/hls-manifest?token=")) return this - return copy(hlsUrl = tokenService.createPath(hlsUrl)) -} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index abdfb136..e4cd3338 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.SubscriptionFeedPreparingResponse -import dev.typetype.server.preserveTooManyRequestsBody +import dev.typetype.server.routes.preserveTooManyRequestsBody import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt new file mode 100644 index 00000000..90cac507 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +internal fun Route.subscriptionMembershipPageRoutes(auth: AuthService, groups: SubscriptionGroupsService) { + val service = SubscriptionMembershipPageService() + get("/subscriptions/group-memberships/page") { + call.withJwtAuth(auth) { userId -> + val filter = call.membershipFilter() ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, ErrorResponse("Invalid membership page filter", "subscription_group_invalid_filter"), + ) + val groupId = filter.groupId + if (groupId != null && !groups.exists(userId, groupId)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } + call.respond(service.getPage(userId, filter)) + } + } + post("/subscriptions/group-memberships/lookup") { + call.withJwtAuth(auth) { userId -> + val body = call.receiveMembershipBody() ?: return@withJwtAuth + val urls = when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { + is MembershipChannels.Single -> listOf(request.channelUrl) + is MembershipChannels.Batch -> request.channelUrls + } + call.respond(service.lookup(userId, urls)) + } + } +} + +private fun ApplicationCall.membershipFilter(): SubscriptionMembershipFilter? { + val params = request.queryParameters + val page = params["page"]?.toIntOrNull() ?: if (params["page"] == null) 0 else return null + val limit = params["limit"]?.toIntOrNull() ?: if (params["limit"] == null) 20 else return null + val search = params["search"]?.trim().orEmpty() + val group = params["groupId"] + val ungrouped = params["ungrouped"]?.toBooleanStrictOrNull() + ?: if (params["ungrouped"] == null) false else return null + val excluded = params["excluded"]?.toBooleanStrictOrNull() + ?: if (params["excluded"] == null) false else return null + if (page !in 0..1_000_000 || limit !in 1..100 || search.length > 200) return null + if (group != null && group.isBlank() || group != null && ungrouped || excluded && group == null) return null + return SubscriptionMembershipFilter(page, limit, search, group, ungrouped, excluded) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 1903cc99..47627f1c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionAvatarWarmupService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.PushNotificationService @@ -28,7 +29,9 @@ internal fun Route.subscriptionsRoutes( warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), pushNotificationService: PushNotificationService? = null, + avatarWarmupService: SubscriptionAvatarWarmupService? = null, ) { + subscriptionMembershipPageRoutes(authService, groupsService) get("/subscriptions/group-memberships") { call.withJwtAuth(authService) { userId -> call.respond(subscriptionsService.getAllWithGroupMemberships(userId)) @@ -47,7 +50,9 @@ internal fun Route.subscriptionsRoutes( ErrorResponse("Subscription group not found", "subscription_group_not_found"), ) } - call.respond(subscriptionsService.getAll(userId, selection)) + val subscriptions = subscriptionsService.getAll(userId, selection) + avatarWarmupService?.schedule(call.application, userId, subscriptions) + call.respond(subscriptions) } } post("/subscriptions") { diff --git a/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt new file mode 100644 index 00000000..bddccde8 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.routes + +import io.ktor.server.application.ApplicationCall +import io.ktor.util.AttributeKey + +val TooManyRequestsBodyAttribute = AttributeKey("preserveTooManyRequestsBody") + +fun ApplicationCall.preserveTooManyRequestsBody() { + attributes.put(TooManyRequestsBodyAttribute, Unit) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/UserAuth.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index e5c98696..5b40ebfd 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -26,6 +26,7 @@ internal fun Route.userDataRoutes( svc.homeRecommendationWarmupService, svc.subscriptionGroupsService, svc.pushNotificationService, + svc.subscriptionAvatarWarmupService, ) subscriptionFeedRoutes( svc.subscriptionFeedService, @@ -47,6 +48,7 @@ internal fun Route.userDataRoutes( notificationsRoutes(svc.notificationsService, authService) pushNotificationRoutes(svc.pushNotificationService, authService) youtubeSessionRoutes(svc.youtubeSessionService, authService) + biliBiliSessionRoutes(svc.bilibiliSessionService, authService) youtubeTakeoutImportRoutes(svc.youtubeTakeoutImportService, authService) profileRoutes(profileService, avatarService, svc.customAvatarService, authService) customAvatarRoutes(svc.customAvatarService, authService) diff --git a/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt index ddd76ea1..60201afb 100644 --- a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt @@ -1,7 +1,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.preserveTooManyRequestsBody +import dev.typetype.server.routes.preserveTooManyRequestsBody import dev.typetype.server.services.YouTubeSubtitleContentResult import io.ktor.http.ContentType import io.ktor.http.HttpHeaders diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt new file mode 100644 index 00000000..9e2a92af --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionFailureKind +import dev.typetype.server.models.ExtractionResult +import io.ktor.http.HttpStatusCode + +internal fun youtubeExtractionFailureStatus(failure: ExtractionResult.Failure): HttpStatusCode = + when (failure.kind) { + ExtractionFailureKind.LiveEventNotStarted, + ExtractionFailureKind.ScheduledPremiere -> HttpStatusCode.Conflict + ExtractionFailureKind.ContentUnavailable -> HttpStatusCode.NotFound + ExtractionFailureKind.ProviderAccessBlocked -> HttpStatusCode.ServiceUnavailable + ExtractionFailureKind.Unknown, + ExtractionFailureKind.YoutubeSessionRejected -> HttpStatusCode.BadGateway + } + +internal fun youtubeBadRequestStatus(code: String): HttpStatusCode = when (code) { + "age_restricted", + "geographic_restriction", + "members_only", + "paid_content", + "private_content" -> HttpStatusCode.Forbidden + else -> HttpStatusCode.BadRequest +} diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt diff --git a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt b/server-http/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt rename to server-http/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt diff --git a/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt index ef31b29a..1a623853 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt @@ -69,6 +69,7 @@ class ManifestRoutesTest { } val response = client.get("/streams/manifest?url=https://youtube.com/watch?v=empty") assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } @Test diff --git a/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt index 56ff39a1..45b69bad 100644 --- a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt @@ -37,6 +37,7 @@ class PortabilityArchiveReproductionTest { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map = error("Account writes are forbidden in this probe") override suspend fun export( diff --git a/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt similarity index 96% rename from src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt index babd1b5a..11ce8800 100644 --- a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt @@ -152,8 +152,13 @@ private object RouteDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ) = source.counts().mapKeys { it.key.wireName }.also { result -> - request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + request.categories.forEach { category -> + val count = result[category.wireName] ?: 0L + onCategoryProgress(category, count) + onCategoryComplete(category, count) + } } override suspend fun export( diff --git a/server-http/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt new file mode 100644 index 00000000..7dbbb5fc --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt @@ -0,0 +1,171 @@ +package dev.typetype.server + +import dev.typetype.server.routes.adminSessionRoutes +import dev.typetype.server.routes.presenceRoutes +import dev.typetype.server.routes.sessionActivityRoutes +import dev.typetype.server.services.ActiveSessionService +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService +import io.ktor.client.request.bearerAuth +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonPrimitive +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class PresenceRoutesTest { + private val presenceKeys = PresenceKeyService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `presence key can read now playing and is revoked independently`() = runBlocking { + insertActiveSessionUser() + val presenceService = PresenceService(hasActiveKey = presenceKeys::hasActiveKey) + withRoutes(presenceService) { + val created = client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Discord RPC"}""") + } + assertEquals(HttpStatusCode.Created, created.status) + val createdBody = Json.parseToJsonElement(created.bodyAsText()).jsonObject + val token = createdBody["token"]?.jsonPrimitive?.contentOrNull ?: error("missing token") + val keyId = createdBody["key"]?.jsonObject?.get("id")?.jsonPrimitive?.contentOrNull + assertTrue(token.startsWith("ttp1_")) + + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + }.let { assertEquals(HttpStatusCode.NoContent, it.status) } + + val presence = client.get("/presence/now-playing") { bearerAuth(token) } + assertEquals(HttpStatusCode.OK, presence.status) + val presenceBody = Json.parseToJsonElement(presence.bodyAsText()).jsonObject + assertEquals(true, presenceBody["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + assertEquals("Video", presenceBody["nowPlaying"]?.jsonObject?.get("title")?.jsonPrimitive?.contentOrNull) + assertEquals("Channel", presenceBody["nowPlaying"]?.jsonObject?.get("channelName")?.jsonPrimitive?.contentOrNull) + + val unauthorized = client.get("/presence/now-playing") { bearerAuth("test-jwt") } + assertEquals(HttpStatusCode.Unauthorized, unauthorized.status) + + val listed = Json.parseToJsonElement( + client.get("/presence/keys") { bearerAuth("test-jwt") }.bodyAsText(), + ) + assertTrue(listed.toString().contains(""""id":"$keyId"""")) + assertFalse(listed.toString().contains(""""token":"$token"""")) + + val revoked = client.delete("/presence/keys/$keyId") { bearerAuth("test-jwt") } + assertEquals(HttpStatusCode.NoContent, revoked.status) + assertEquals(HttpStatusCode.Unauthorized, client.get("/presence/now-playing") { bearerAuth(token) }.status) + } + } + + @Test + fun `presence works without admin active session tracking`() = runBlocking { + insertActiveSessionUser() + val presenceService = PresenceService(hasActiveKey = presenceKeys::hasActiveKey) + withRoutes(presenceService) { + val created = client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Local monitor"}""") + } + val token = Json.parseToJsonElement(created.bodyAsText()).jsonObject["token"] + ?.jsonPrimitive?.contentOrNull ?: error("missing token") + val adminSessions = client.get("/admin/sessions") { bearerAuth("test-jwt") } + assertEquals("[]", adminSessions.bodyAsText()) + + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + } + val presence = Json.parseToJsonElement( + client.get("/presence/now-playing") { bearerAuth(token) }.bodyAsText(), + ).jsonObject + assertEquals(true, presence["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + } + } + + @Test + fun `stale presence reports inactive`() = runBlocking { + var now = 1_000L + insertActiveSessionUser() + val keyService = PresenceKeyService { now } + val presenceService = PresenceService({ now }, keyService::hasActiveKey) + withRoutes(presenceService, keyService) { + val token = Json.parseToJsonElement( + client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Clock"}""") + }.bodyAsText(), + ).jsonObject["token"]?.jsonPrimitive?.contentOrNull ?: error("missing token") + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + } + now += PresenceService.ACTIVITY_TTL_MS + 1 + val presence = Json.parseToJsonElement( + client.get("/presence/now-playing") { bearerAuth(token) }.bodyAsText(), + ).jsonObject + assertEquals(false, presence["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + } + } + + private fun withRoutes( + presenceService: PresenceService, + keyService: PresenceKeyService = presenceKeys, + block: suspend ApplicationTestBuilder.() -> Unit, + ): Unit = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + val auth = AuthService.fixed(TEST_USER_ID) + adminSessionRoutes(auth, ActiveSessionService(AdminSettingsService())) + presenceRoutes(auth, keyService, presenceService) + sessionActivityRoutes(auth, ActiveSessionService(AdminSettingsService()), presenceService) + } + } + block() + } + + private fun playbackBody(): String = """ + {"clientName":"web","deviceId":"device-1","videoUrl":"https://example.test/watch?v=1", + "title":"Video","channelName":"Channel","positionMs":1000,"durationMs":60000,"paused":false} + """.trimIndent() +} diff --git a/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt similarity index 83% rename from src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index f2fc67f3..7c22fc8c 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -62,6 +62,8 @@ class SettingsRoutesTest { assertTrue(body.contains("\"notificationPopupsEnabled\":true")) assertTrue(body.contains("\"defaultLandingPage\":\"home\"")) assertTrue(body.contains("\"defaultPlaybackSpeed\":1.0")) + assertTrue(body.contains("\"videoGridColumns\":0")) + assertTrue(body.contains("\"relatedVideoSize\":\"default\"")) } @Test @@ -96,9 +98,29 @@ class SettingsRoutesTest { fun `GET settings returns defaults for new fields when no row exists`() = withApp { val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() assertContainsAll(body, listOf("\"subtitlesEnabled\":false", "\"defaultSubtitleLanguage\":\"\"", "\"defaultAudioLanguage\":\"\"", "\"preferOriginalLanguage\":false", "\"enableHighQualityPlayback\":false")) + assertContainsAll(body, listOf("\"videoGridColumns\":0", "\"relatedVideoSize\":\"default\"")) assertContainsNone(body, listOf("recommendationPersonalizationEnabled", "subscriptionSyncInterval")) } + @Test + fun `PUT settings persists and bounds layout preferences`() = withApp { + client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoGridColumns":5,"relatedVideoSize":"large"}""") + } + var body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() + assertContainsAll(body, listOf("\"videoGridColumns\":5", "\"relatedVideoSize\":\"large\"")) + + client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoGridColumns":9,"relatedVideoSize":"huge"}""") + } + body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() + assertContainsAll(body, listOf("\"videoGridColumns\":0", "\"relatedVideoSize\":\"default\"")) + } + @Test fun `PUT settings persists new fields and GET returns them`() = withApp { client.put("/settings") { diff --git a/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt diff --git a/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt new file mode 100644 index 00000000..f5d160b6 --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -0,0 +1,272 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind +import dev.typetype.server.routes.streamRoutes +import dev.typetype.server.services.StreamService +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class StreamRoutesTest { + + private val streamService: StreamService = mockk() + + @Test + fun `GET streams without url returns 400`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr") + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + @Test + fun `GET streams with valid url returns 200 on Success`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse()) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) + } + + @Test + fun `GET live YouTube streams exposes HLS only`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( + sabrResponse().copy(isLive = true, hlsUrl = "/streams/hls-manifest?token=live"), + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=live") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?token=live\"")) + assertTrue(body.contains("\"dashMpdUrl\":\"\"")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) + } + + @Test + fun `GET live YouTube streams fails when HLS is unavailable instead of using SABR`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse().copy(isLive = true, hlsUrl = "")) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=live") + + assertEquals(HttpStatusCode.BadGateway, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) + } + + @Test + fun `GET scheduled live events return a typed conflict`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "This live event will begin in 20 minutes.", + "live_not_started", + ExtractionFailureKind.LiveEventNotStarted, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") + assertEquals(HttpStatusCode.Conflict, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"live_not_started\"")) + } + + @Test + fun `GET scheduled premieres return a typed conflict`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "Premieres in 200 days", + "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=premiere") + + assertEquals(HttpStatusCode.Conflict, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"scheduled_premiere\"")) + } + + @Test + fun `GET unavailable YouTube content returns not found`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "This video is not available", + "content_unavailable", + ExtractionFailureKind.ContentUnavailable, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=missing") + + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"content_unavailable\"")) + } + + @Test + fun `GET YouTube extraction failures return gateway errors`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure("Temporary provider error", "upstream_failure") + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") + assertEquals(HttpStatusCode.BadGateway, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"upstream_failure\"")) + } + + @Test + fun `GET provider blocks return service unavailable`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "The provider is blocking requests", + "provider_access_blocked", + ExtractionFailureKind.ProviderAccessBlocked, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=blocked") + + assertEquals(HttpStatusCode.ServiceUnavailable, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"provider_access_blocked\"")) + } + + @Test + fun `GET stream failure preserves non-YouTube status behavior`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure("Provider failure", "upstream_failure") + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/niconico?url=https://www.nicovideo.jp/watch/sm9") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + } + + @Test + fun `GET streams returns 400 on BadRequest`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.BadRequest("This video is a paid video", "paid_content") + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=paid") + assertEquals(HttpStatusCode.Forbidden, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) + } + + @Test + fun `GET restricted YouTube streams asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.BadRequest("Sign in to confirm your age", "age_restricted") + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=restricted") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + + @Test + fun `GET members-only metadata asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse().copy(requiresMembership = true)) + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=members") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + + @Test + fun `GET sabr streams returns 422 when final response has no playable source`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(testStreamResponse()) + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes(streamService) { _, data -> + data.copy( + videoStreams = emptyList(), + videoOnlyStreams = emptyList(), + audioStreams = emptyList(), + hlsUrl = "", + dashMpdUrl = "", + ) + } + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.BadGateway, response.status) + assertTrue(body.contains("\"code\":\"no_playable_streams\"")) + } + + private fun sabrResponse() = testStreamResponse( + videoOnlyStreams = listOf(testVideoStream().copy(deliveryMethod = "sabr")), + audioStreams = listOf(testAudioStream(deliveryMethod = "sabr")), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt diff --git a/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt new file mode 100644 index 00000000..b9ab3243 --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt @@ -0,0 +1,109 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.SubscriptionMembershipPage +import dev.typetype.server.routes.subscriptionsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionMembershipPageRoutesTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + private val path = "/subscriptions/group-memberships" + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit): Unit = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionsRoutes(subscriptions, AuthService.fixed(TEST_USER_ID), groupsService = groups) } + } + block() + } + + @Test + fun `page and lookup require authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("$path/page").status) + assertEquals(HttpStatusCode.Unauthorized, client.post("$path/lookup").status) + } + + @Test + fun `page validates limits filters and group ownership`() = withApp { + for (query in listOf("page=-1", "page=x", "page=1000001", "limit=0", "limit=101", "limit=x", + "groupId=", "excluded=true", "ungrouped=maybe", "excluded=maybe", "groupId=x&ungrouped=true", + "search=${"a".repeat(201)}")) { + assertEquals(HttpStatusCode.BadRequest, get("$path/page?$query").status, query) + } + val foreign = (groups.create("foreign", "Hidden") as SubscriptionGroupWriteResult.Success).group + for (id in listOf("missing", foreign.id)) { + val response = get("$path/page?groupId=$id") + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("subscription_group_not_found")) + } + } + + @Test + fun `page contract returns requested rows and accurate counts without changing legacy reads`() = withApp { + for (index in 0..2) subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/$index", "Name $index", "avatar")) + val response = get("$path/page?page=1&limit=2&search=NAME") + assertEquals(HttpStatusCode.OK, response.status) + val page = Json.decodeFromString(response.bodyAsText()) + assertEquals(1, page.page) + assertEquals(2, page.limit) + assertEquals(3L, page.total) + assertEquals(3L, page.totalSubscriptions) + assertEquals(3L, page.ungroupedCount) + assertEquals("Name 2", page.items.single().name) + assertTrue(get(path).bodyAsText().startsWith("[")) + } + + @Test + fun `lookup enforces bounded bodies and returns only current owned subscriptions`() = withApp { + subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/one", "One", "avatar")) + val valid = lookup("""{"channelUrls":["https://example.com/channel/one","missing"]}""") + assertEquals(HttpStatusCode.OK, valid.status) + assertTrue(valid.bodyAsText().contains("\"name\":\"One\"")) + assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":[]}""").status) + assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":["${"x".repeat(2049)}"]}""").status) + assertEquals(HttpStatusCode.BadRequest, lookup("{\"channelUrls\":[${List(501) { "\"x\"" }.joinToString(",")}]}").status) + assertEquals(HttpStatusCode.PayloadTooLarge, lookup("x".repeat(1024 * 1024 + 1)).status) + } + + private suspend fun ApplicationTestBuilder.get(url: String): io.ktor.client.statement.HttpResponse = client.get(url) { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private suspend fun ApplicationTestBuilder.lookup(body: String): io.ktor.client.statement.HttpResponse = client.post("$path/lookup") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + header(HttpHeaders.ContentType, "application/json") + setBody(body) + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt similarity index 97% rename from src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt index 00e21bb7..d3152964 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt @@ -22,7 +22,7 @@ import java.time.Instant class SabrLiveGapWindowTest { @Test - fun `small live gap extends the current playback window`() = runTest { + fun `small live gap requests only the first segment covering the low latency runway`() = runTest { val audio = format(140, isAudio = true) val video = format(299, isAudio = false) val session = mockk(relaxed = true) @@ -58,10 +58,7 @@ class SabrLiveGapWindowTest { assertTrue(result.isReady) assertEquals(480_000L, result.response.startTimeMs) assertEquals( - listOf( - "/api/sabr/playback/session/299/segment/93?generation=0", - "/api/sabr/playback/session/299/segment/94?generation=0", - ), + listOf("/api/sabr/playback/session/299/segment/93?generation=0"), requireNotNull(result.response.video).segments.map { it.url }, ) assertNull(holder.terminalFailure()) diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt similarity index 91% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt index 5d9d1d6b..e6af9561 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt @@ -20,7 +20,7 @@ import java.time.Instant class SabrLivePlaybackOverlapTest { @Test - fun `audio overlap still covers the ready horizon after player time`() = runTest { + fun `audio overlap covers the minimal live ready horizon after player time`() = runTest { val audio = format(140, true) val video = format(137, false) val session = mockk(relaxed = true) @@ -54,8 +54,10 @@ class SabrLivePlaybackOverlapTest { ) assertTrue(result.isReady) - assertEquals(4, result.response.audio.segments.size) - assertTrue(result.response.audio.segments.last().let { it.startMs + it.durationMs } >= 176_925_494L) + assertTrue(result.response.audio.segments.size <= 3) + val endMs = result.response.audio.segments.last().let { it.startMs + it.durationMs } + assertTrue(endMs >= 176_919_494L) + assertTrue(endMs <= 176_923_494L) } private fun holder( diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt new file mode 100644 index 00000000..155a5a4d --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.CachedSabrSegment +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class SabrLivePlaybackStartupTest { + @Test + fun `active live startup responds after the first media pair`() = runTest { + val audio = format(itag = 140, isAudio = true) + val video = format(itag = 299, isAudio = false) + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + every { session.streamState } returns streamState + every { session.isLive } returns true + every { streamState.isLive } returns true + every { streamState.liveHeadTimeMs } returns 120_000L + every { streamState.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 50 + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } answers { + val request = secondArg() + if (request.sequenceNumber in 50..52) { + cached(request.format.itag, request.sequenceNumber, 100_000L + (request.sequenceNumber - 50) * 2_000L) + } else { + null + } + } + + val builder = SabrPlaybackWindowBuilder(store) + val request = SabrPlaybackWindowRequest(0L, 100_000L, 299, 140, bufferGoalMs = 8_000L) + val startup = builder.build(holder, request) + val continuation = builder.build( + holder, + request.copy( + bufferedRanges = listOf( + SabrPlaybackBufferedRange(140, 0L, 100_000L), + SabrPlaybackBufferedRange(299, 0L, 100_000L), + ), + ), + ) + + assertTrue(startup.isReady) + assertTrue(continuation.isReady) + assertEquals(1, startup.response.audio.segments.size) + assertEquals(1, requireNotNull(startup.response.video).segments.size) + assertEquals(1, continuation.response.audio.segments.size) + assertEquals(1, requireNotNull(continuation.response.video).segments.size) + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ) = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { this@mockk.isAudio } returns isAudio + every { mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { approxDurationMs } returns 900_000L + } + + private fun cached(itag: Int, sequence: Int, startMs: Long): CachedSabrSegment = CachedSabrSegment( + itag = itag, + sequence = sequence, + init = false, + startMs = startMs, + durationMs = 2_000L, + mimeType = if (itag == 140) "audio/mp4" else "video/mp4", + bytesBase64 = "AA==", + byteLength = 1, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt similarity index 83% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt index 21613725..e59955df 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt @@ -64,7 +64,7 @@ class SabrLivePlaybackWindowBuilderTest { } @Test - fun `live window requests every track missing from the target window`() = runTest { + fun `live window only requires the first segment pair even for a large requested buffer`() = runTest { val audio = format(itag = 140, isAudio = true) val video = format(itag = 248, isAudio = false) val session = mockk(relaxed = true) @@ -93,8 +93,9 @@ class SabrLivePlaybackWindowBuilderTest { SabrPlaybackWindowRequest(0L, 100_000L, 248, 140, bufferGoalMs = 30_000L), ) - assertEquals(listOf(140, 248), result.blockedRequests.map { it.format.itag }.sorted()) - assertEquals(listOf(101, 106), result.blockedRequests.map { it.sequenceNumber }) + assertTrue(result.isReady) + assertEquals(1, result.response.audio.segments.size) + assertEquals(1, requireNotNull(result.response.video).segments.size) } @Test @@ -130,46 +131,6 @@ class SabrLivePlaybackWindowBuilderTest { assertEquals(102_010L, result.response.audio.segments.single().startMs) } - @Test - fun `active live startup waits for the full media cushion`() = runTest { - val audio = format(itag = 140, isAudio = true) - val video = format(itag = 299, isAudio = false) - val session = mockk(relaxed = true) - val streamState = mockk(relaxed = true) - every { session.streamState } returns streamState - every { session.isLive } returns true - every { streamState.isLive } returns true - every { streamState.liveHeadTimeMs } returns 120_000L - every { streamState.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 50 - val holder = holder(session, audio, video) - val store = mockk() - coEvery { store.cachedSegment(holder, any()) } answers { - val request = secondArg() - if (request.sequenceNumber in 50..52) { - cached(request.format.itag, request.sequenceNumber, 100_000L + (request.sequenceNumber - 50) * 2_000L, 2_000L) - } else { - null - } - } - - val builder = SabrPlaybackWindowBuilder(store) - val request = SabrPlaybackWindowRequest(0L, 100_000L, 299, 140, bufferGoalMs = 8_000L) - val startup = builder.build(holder, request) - val continuation = builder.build( - holder, - request.copy( - bufferedRanges = listOf( - SabrPlaybackBufferedRange(140, 0L, 100_000L), - SabrPlaybackBufferedRange(299, 0L, 100_000L), - ), - ), - ) - - assertFalse(startup.isReady) - assertTrue(continuation.isReady) - assertEquals(listOf(53, 53), continuation.blockedRequests.map { it.sequenceNumber }) - } - @Test fun `active live startup derives timing from adjacent cached segments`() = runTest { val audio = format(itag = 140, isAudio = true) @@ -206,12 +167,12 @@ class SabrLivePlaybackWindowBuilderTest { ) assertTrue(result.isReady) assertEquals(100_000L, result.response.startTimeMs) - assertEquals(4, result.response.audio.segments.size) - assertEquals(List(4) { 2_000L }, result.response.audio.segments.map { it.durationMs }) + assertEquals(1, result.response.audio.segments.size) + assertEquals(List(1) { 2_000L }, result.response.audio.segments.map { it.durationMs }) } @Test - fun `asymmetric live buffers keep requesting the shorter track`() = runTest { + fun `asymmetric live buffers request the shorter track at the shared playback start`() = runTest { val audio = format(itag = 140, isAudio = true) val video = format(itag = 299, isAudio = false) val session = mockk(relaxed = true) diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt similarity index 99% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt index 19ddf9d7..83b00110 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt @@ -232,7 +232,7 @@ class SabrPlaybackWindowBuilderTest { ) assertTrue(result.isReady) assertTrue(result.response.endOfStream) - assertEquals(898_000L, result.response.durationMs) + assertEquals(897_000L, result.response.durationMs) coVerify(exactly = 0) { store.cachedSegment(holder, match { it.sequenceNumber > 100 }) } } diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt similarity index 67% rename from src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index 43989f9d..cbbbf6c9 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -121,6 +121,90 @@ class SabrProgressivePlaybackWindowTest { assertEquals(4, result.response.audio.segments.size) } + @Test + fun `progressive window marks the final segment as end of stream`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 1 + every { state.getEndSegment(audio) } returns 2L + every { state.getEndSegment(video) } returns 2L + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } answers { + val request = firstArg() + val durationMs = when { + request.sequenceNumber == 1 -> 4_000L + else -> 6_000L + } + readableSegment( + request.format, + sequence = request.sequenceNumber, + startMs = if (request.sequenceNumber == 1) 0L else 4_000L, + durationMs = durationMs, + ) + } + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest( + generation = 0L, + playerTimeMs = 0L, + videoItag = video.itag, + audioItag = audio.itag, + bufferGoalMs = 9_000L, + ), + ) + + assertTrue(result.isReady) + assertTrue(result.response.endOfStream) + assertEquals(2, result.response.audio.segments.size) + assertEquals(2, requireNotNull(result.response.video).segments.size) + } + + @Test + fun `window at the indexed end closes without requesting past the timeline`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val state = mockk(relaxed = true) + every { state.getEndSegment(audio) } returns 2L + every { state.getEndSegment(video) } returns 2L + every { state.getSegmentEndMs(audio, 2) } returns 20_000L + every { state.getSegmentEndMs(video, 2) } returns 10_000L + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + val ranges = listOf( + SabrPlaybackBufferedRange(audio.itag, 0L, 20_000L), + SabrPlaybackBufferedRange(video.itag, 0L, 10_000L), + ) + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest( + generation = 0L, + playerTimeMs = 9_000L, + videoItag = video.itag, + audioItag = audio.itag, + bufferGoalMs = 10_000L, + bufferedRanges = ranges, + ), + ) + + assertTrue(result.isReady) + assertTrue(result.response.endOfStream) + assertTrue(result.blockedRequests.isEmpty()) + assertTrue(result.response.audio.segments.isEmpty()) + assertTrue(requireNotNull(result.response.video).segments.isEmpty()) + } + private fun readableSegment( format: YoutubeSabrFormat, sequence: Int = 1, diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt similarity index 62% rename from src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt index 5672234b..40a67311 100644 --- a/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt @@ -87,6 +87,46 @@ class SabrStreamContractFilterTest { assertEquals(setOf("avc1.4d4028", "vp9", "av01.0.08M.08"), sabr.videoOnlyStreams.map { it.codec }.toSet()) } + @Test + fun `sabr contract exposes Opus audio for browsers without AAC MSE`() = runTest { + val aac = audioFormat(140, "audio/mp4; codecs=\"mp4a.40.2\"") + val opus = audioFormat(249, "audio/webm; codecs=\"opus\"") + val video = videoFormat(137, "video/mp4; codecs=\"avc1.4d4028\"") + val info = mockk { + every { formats } returns listOf(aac, opus, video) + every { findFormatByItag(any()) } answers { + formats.firstOrNull { it.itag == firstArg() } + } + } + val store = mockk() + coEvery { store.fetchInfo(VIDEO_ID, cachedFirst = true) } returns SabrPreparedInfo(info, null) + val aacStream = testAudioStream( + itag = 140, + deliveryMethod = "sabr", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?audioItag=140", + ) + val opusStream = aacStream.copy( + itag = 249, + codec = "opus", + mimeType = "audio/webm", + format = "WEBM", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?audioItag=249", + ) + val filtered = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream(itag = 137).copy( + deliveryMethod = "sabr", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?videoItag=137", + ), + ), + audioStreams = listOf(aacStream), + ).withPlayableSabrStreams(YOUTUBE_URL, store).onlySabrStreams() + + assertEquals(setOf(140, 249), filtered.audioStreams.map { it.itag }.toSet()) + assertEquals("audio/webm", filtered.audioStreams.first { it.itag == 249 }.mimeType) + assertEquals("opus", filtered.audioStreams.first { it.itag == 249 }.codec) + } + private fun videoFormat(itag: Int, mime: String): YoutubeSabrFormat = mockk(relaxed = true) { every { isAudio } returns false @@ -98,6 +138,23 @@ class SabrStreamContractFilterTest { every { qualityLabel } returns "1080p" } + private fun audioFormat(itag: Int, mime: String): YoutubeSabrFormat = + mockk(relaxed = true) { + every { isAudio } returns true + every { isVideo } returns false + every { this@mockk.itag } returns itag + every { mimeType } returns mime + every { audioTrackId } returns null + every { isOriginalAudio } returns true + every { isDrc } returns false + every { xtags } returns null + every { bitrate } returns 128000 + every { width } returns 0 + every { height } returns 0 + every { qualityLabel } returns null + every { contentLength } returns 1_000_000L + } + private companion object { const val VIDEO_ID = "X4VbdwhkE10" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" diff --git a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt index 29e31f97..97f662ad 100644 --- a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt @@ -130,8 +130,8 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody()) } - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("segment/1")) + assertEquals(HttpStatusCode.Accepted, response.status) + assertTrue(response.bodyAsText().contains("video:136:1 pending")) verify(atLeast = 1) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts new file mode 100644 index 00000000..148855f3 --- /dev/null +++ b/server-playback/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + `java-test-fixtures` + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-sabr")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-token-gateway")) + implementation(project(":server-downloader")) + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + implementation("org.slf4j:slf4j-api:2.0.16") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt index da44b509..727c13fc 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.ActiveSessionNowPlayingItem import dev.typetype.server.models.SessionPlaybackProgressRequest import dev.typetype.server.models.SessionPlaybackStartRequest -internal object ActiveSessionNowPlayingMapper { +object ActiveSessionNowPlayingMapper { fun fromStart(request: SessionPlaybackStartRequest, now: Long): ActiveSessionNowPlayingItem = ActiveSessionNowPlayingItem( videoUrl = request.videoUrl.trim(), title = request.title.trim(), diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt index 7ebbe0c2..4abe3612 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ActiveSessionNowPlayingItem -internal data class ActiveSessionRecord( +data class ActiveSessionRecord( val id: String, val userId: String, val username: String?, diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt index eb52953a..5d9b5d8d 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object ActiveSessionStrings { +object ActiveSessionStrings { fun text(value: String?): String? = value?.trim()?.take(MAX_TEXT_LENGTH)?.takeIf { it.isNotEmpty() } fun userAgent(value: String?): String? = value?.trim()?.take(MAX_USER_AGENT_LENGTH)?.takeIf { it.isNotEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt index 1772414c..1556f569 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.withTimeout import java.time.Duration import java.util.concurrent.ConcurrentHashMap -internal class AuthenticatedSabrInfoCache( +class AuthenticatedSabrInfoCache( ttl: Duration = Duration.ofMinutes(5), maxEntries: Int = 256, private val timeoutMs: Long = AuthenticatedSabrPolicy.INFO_TIMEOUT_MS, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index 1768eaec..40dd2abf 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -13,7 +13,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.SabrAdapter import org.slf4j.LoggerFactory -internal class AuthenticatedSabrInfoService( +class AuthenticatedSabrInfoService( private val youtubeSessionService: YoutubeSessionService, private val tokenClient: TypetypeTokenSabrTokenClient, private val visitorDataFetcher: () -> String = AuthenticatedYoutubeVisitorData::fetch, @@ -71,14 +71,14 @@ internal class AuthenticatedSabrInfoService( } } -internal sealed interface AuthenticatedSabrInfoResult { +sealed interface AuthenticatedSabrInfoResult { data object NotConnected : AuthenticatedSabrInfoResult data object Failed : AuthenticatedSabrInfoResult data object TimedOut : AuthenticatedSabrInfoResult data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult } -internal fun interface AuthenticatedSabrProbe { +fun interface AuthenticatedSabrProbe { fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt index a0a67f70..d67a3ea7 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object AuthenticatedSabrPolicy { +object AuthenticatedSabrPolicy { const val INFO_TIMEOUT_MS = 15_000L const val STREAM_TIMEOUT_MS = 20_000L const val TIMEOUT_CODE = "authenticated_sabr_timeout" diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt index 443c6b6d..3a5c7602 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import java.util.Base64 -internal class CachedSabrSegment( +class CachedSabrSegment( val itag: Int, val sequence: Int, val init: Boolean, @@ -36,7 +36,7 @@ internal class CachedSabrSegment( val length: Int get() = bytes.size } -internal fun SabrMediaSegment.toCachedSabrSegment( +fun SabrMediaSegment.toCachedSabrSegment( mimeType: String, bytes: ByteArray = data, ): CachedSabrSegment = CachedSabrSegment( diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt index 70b2e088..cf5afd08 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt @@ -6,7 +6,7 @@ import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.Localization import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrAdaptiveInitialization { +object SabrAdaptiveInitialization { private val localization = Localization("en", "US") suspend fun fetch( diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt new file mode 100644 index 00000000..230a8e4f --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -0,0 +1,39 @@ +package dev.typetype.server.services + +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class SabrBootstrapStreamService( + private val sessionStore: SabrSessionStore, + private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, + private val liveHlsStreamService: StreamService, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + val videoId = youtubeVideoId(url) + ?: return ExtractionResult.BadRequest("Invalid YouTube URL") + val metadataStartedAt = System.nanoTime() + val metadataResult = tokenSessionClient.fetchPlaybackSession(videoId) + PlaybackTraceLog.record( + "sabr_bootstrap_metadata", + "durationMs=${(System.nanoTime() - metadataStartedAt) / 1_000_000} result=${if (metadataResult == null) "miss" else "ready"}", + ) + val metadata = metadataResult + ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") + if (metadata.isLive) return liveHlsStreamService.getStreamInfo(url) + val formatsStartedAt = System.nanoTime() + val fromSession = metadata.preparedSabrInfo() + val formatsResult = fromSession ?: sessionStore.fetchInfo(videoId, cachedFirst = true) + PlaybackTraceLog.record( + "sabr_bootstrap_formats", + "durationMs=${(System.nanoTime() - formatsStartedAt) / 1_000_000} source=${if (fromSession != null) "session" else "cache_or_probe"} result=${if (formatsResult == null) "miss" else "ready"}", + ) + val prepared = formatsResult + ?: return ExtractionResult.Failure("SABR playback formats unavailable") + sessionStore.rememberPreparedInfo(videoId, prepared) + return ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) + } +} + +fun youtubeVideoId(url: String): String? = + Regex("(?:[?&]v=|/shorts/|youtu\\.be/)([A-Za-z0-9_-]{6,})").find(url)?.groupValues?.get(1) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt index a532cdfd..8b1baa41 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrSession -internal fun YoutubeSabrSession.findCachedMediaAt( +fun YoutubeSabrSession.findCachedMediaAt( format: YoutubeSabrFormat, targetMs: Long, predictedSequence: Int, @@ -53,7 +53,7 @@ private fun SabrMediaSegment.startsWithinFollowingRange( return header.startMs >= 0L && leadMs in -TIMING_TOLERANCE_MS..maximumLeadMs } -internal fun maximumFollowingLeadMs(durationMs: Long, sequenceDistance: Int): Long { +fun maximumFollowingLeadMs(durationMs: Long, sequenceDistance: Int): Long { val distance = sequenceDistance.coerceAtLeast(1).toLong() val toleranceMs = LIVE_FOLLOWING_TIMING_TOLERANCE_MS val boundedDurationMs = durationMs.coerceAtMost((Long.MAX_VALUE - toleranceMs) / distance) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt index 6c443d75..62613783 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrDashManifestBuilder { +object SabrDashManifestBuilder { fun build( videoId: String, audio: YoutubeSabrFormat, diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt new file mode 100644 index 00000000..1e7e109f --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrSession + +internal suspend fun runSabrDemandAttempt( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + wasFutureLiveRequest: Boolean, + runtime: SabrPumpRuntime, + pump: suspend () -> YoutubeSabrSession.DemandResponseResult, + onResolved: (SabrMediaSegment) -> Unit, +): Boolean { + if (!holder.beginInFlightSegmentDemand(request, identity, wasFutureLiveRequest)) return true + try { + SabrPumpLogger.start(holder, "demand", request) + runtime.beginDemand(identity) + return coroutineScope { + val attempt = async(start = CoroutineStart.UNDISPATCHED) { + holder.recordInFlightAttemptStarted() + try { + pump() + } finally { + holder.recordInFlightAttemptFinished() + } + } + holder.registerInFlightDemandCancellation(identity) { + attempt.cancel(CancellationException("SABR demand interrupted")) + } + try { + val result = attempt.await() + val interruption = holder.consumeInFlightDemandInterruption(identity) + if (interruption != null) { + return@coroutineScope finishInterruptedDemand(holder, request, identity, runtime, interruption) + } + SabrDemandAttemptFinisher.finish( + holder, + request, + identity, + result, + runtime, + wasFutureLiveRequest, + onResolved = { onResolved(it) }, + ) + } catch (error: CancellationException) { + val interruption = holder.consumeInFlightDemandInterruption(identity) + ?: throw error + finishInterruptedDemand(holder, request, identity, runtime, interruption) + } finally { + holder.clearInFlightDemandCancellation(identity) + } + } + } finally { + holder.finishInFlightSegmentDemand(identity) + } +} + +private fun finishInterruptedDemand( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + runtime: SabrPumpRuntime, + interruption: SabrDemandInterruption, +): Boolean = synchronized(holder) { + runtime.finishDemand(identity) + when (interruption.action) { + SabrDemandInterruptionAction.WAIT_FOR_LIVE -> { + if (holder.isSegmentDemandActive(request, identity)) { + holder.requeueSegmentDemand(request, identity, interruption.atMs) + } + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + false + } + + SabrDemandInterruptionAction.RESTART -> { + if (holder.playbackState() != SabrPlaybackState.TERMINAL && + holder.playbackState() != SabrPlaybackState.NETWORK_FAILED + ) { + holder.setPlaybackState(SabrPlaybackState.IDLE) + } + true + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index e4eccca5..2958e1a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -4,21 +4,68 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession -internal object SabrDemandAttemptFinisher { +object SabrDemandAttemptFinisher { fun interruptCompletedInFlightDemand(holder: SabrSessionHolder, demand: SabrInFlightDemand): Boolean = synchronized(holder) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false if (holder.session.getCachedSegment(demand.request) == null) return@synchronized false + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.RESTART, + atMs = System.currentTimeMillis(), + reason = "completed_demand", + ), + ) holder.setPlaybackState(SabrPlaybackState.IDLE) true } + fun interruptStalledInFlightDemand( + holder: SabrSessionHolder, + demand: SabrInFlightDemand, + nowMs: Long, + expectedDelayMs: Long, + reason: String, + ): Boolean = synchronized(holder) { + val state = holder.playbackState() + if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false + if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false + if (holder.session.getCachedSegment(demand.request) != null) return@synchronized false + if (!holder.requeueSegmentDemand(demand.request, demand.identity, nowMs)) return@synchronized false + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.WAIT_FOR_LIVE, + atMs = nowMs, + reason = "future_live_not_published", + ), + ) + SabrPumpLogger.expired( + holder, + demand.request, + recoverable = true, + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = reason, + attempts = demand.attempts(), + lastAttemptDurationMs = demand.lastAttemptDurationMs(), + event = "demand_requeued", + ) + true + } + fun expireStalledInFlightDemand( holder: SabrSessionHolder, demand: SabrInFlightDemand, recoverable: Boolean = demand.futureLiveRequest, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason: String = "terminal_deadline", ): Boolean = synchronized(holder) { val state = holder.playbackState() @@ -26,7 +73,25 @@ internal object SabrDemandAttemptFinisher { if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false holder.clearSegmentDemands() val message = "SABR demand stalled for ${demand.request.summary()}" - SabrPumpLogger.expired(holder, demand.request, recoverable) + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.RESTART, + atMs = nowMs, + reason = "terminal_deadline", + ), + ) + SabrPumpLogger.expired( + holder, + demand.request, + recoverable, + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = reason, + attempts = demand.attempts(), + lastAttemptDurationMs = demand.lastAttemptDurationMs(), + ) holder.failTerminal(if (recoverable) sabrRecoverableFailureMessage(message) else message) true } @@ -36,12 +101,23 @@ internal object SabrDemandAttemptFinisher { request: SabrSegmentRequest, identity: String, recoverable: Boolean = false, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason: String = "terminal_deadline", ): Boolean = synchronized(holder) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false val current = holder.nextSegmentDemand() ?: return@synchronized false if (!current.matches(request) || holder.segmentDemandIdentity(current) != identity) return@synchronized false - SabrPumpLogger.expired(holder, request, recoverable) + SabrPumpLogger.expired( + holder, + request, + recoverable, + registeredAtMs = holder.segmentDemandRegisteredAtMs(request, identity), + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = reason, + ) fail(holder, request, identity, recoverable) } @@ -68,7 +144,7 @@ internal object SabrDemandAttemptFinisher { } val action = runtime.demandRecoveryAction( requestKey = identity, - targetTrackSegmentCount = result.targetTrackSegmentCount, + requestPerformed = result.requestPerformed, resolved = resolved, ) val recovering = recover(holder, request, identity, action, runtime) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt index 39aa90b6..beb01df8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrDemandDeadline( +class SabrDemandDeadline( private val timeoutMs: Long, private val maximumBackoffExtensionMs: Long = SabrPumpPolicy.MAX_DEMAND_BACKOFF_EXTENSION_MS, ) { @@ -15,12 +15,19 @@ internal class SabrDemandDeadline( registeredAtMs: Long, nowMs: Long, backoffRemainingMs: Long, - ): Boolean { + ): Boolean = nowMs >= deadlineAtMs(identity, registeredAtMs, nowMs, backoffRemainingMs) + + fun deadlineAtMs( + identity: String, + registeredAtMs: Long, + nowMs: Long, + backoffRemainingMs: Long, + ): Long { if (this.identity != identity || this.registeredAtMs != registeredAtMs) { reset(identity, registeredAtMs) } extendForBackoff(nowMs, backoffRemainingMs) - return nowMs >= expiresAtMs + return expiresAtMs } private fun reset(identity: String, registeredAtMs: Long) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt index c7525c26..f1834a49 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrDemandRecoveryAction { +enum class SabrDemandRecoveryAction { WAIT, READVERTISE_TRACK, } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt new file mode 100644 index 00000000..62f3c641 --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -0,0 +1,141 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.delay + +class SabrDemandWatchdog( + private val clock: () -> Long = System::currentTimeMillis, + private val intervalMs: Long = SabrPumpPolicy.IDLE_POLL_MS, +) { + suspend fun monitor(isAlive: () -> Boolean, holder: SabrSessionHolder): Boolean { + val deadline = SabrDemandDeadline(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS) + while (isAlive()) { + val state = holder.playbackState() + if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return false + val inFlightDemand = holder.inFlightSegmentDemand() + if (inFlightDemand != null) { + val nowMs = clock() + val backoffRemainingMs = holder.session.demandBackoffRemainingMs + if (holder.isLiveDemandOutsideRecoverableWindow(inFlightDemand.request) && + SabrDemandAttemptFinisher.expireStalledInFlightDemand( + holder, + inFlightDemand, + recoverable = true, + nowMs = nowMs, + reason = "outside_recoverable_window", + ) + ) { + return true + } + if (inFlightDemand.futureLiveRequest) holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + val lastProgressAtMs = inFlightDemand.observeProgress(holder.session.mediaProgressVersion, nowMs) + val completedIdle = holder.session.getCachedSegment(inFlightDemand.request) != null && + nowMs - lastProgressAtMs >= SabrPumpPolicy.COMPLETED_DEMAND_IDLE_MS + if (completedIdle && SabrDemandAttemptFinisher.interruptCompletedInFlightDemand(holder, inFlightDemand)) { + return true + } + val deadlineAtMs = deadline.deadlineAtMs( + inFlightDemand.identity, + inFlightDemand.registeredAtMs, + nowMs, + backoffRemainingMs, + ) + if (nowMs >= deadlineAtMs) { + val recoverableLiveDemand = + holder.livePlaybackSnapshot()?.active == true && + !holder.isLiveDemandOutsideRecoverableWindow(inFlightDemand.request) + if (recoverableLiveDemand) { + SabrDemandAttemptFinisher.interruptStalledInFlightDemand( + holder, + inFlightDemand, + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + reason = if (inFlightDemand.futureLiveRequest) { + "future_live_not_published" + } else { + "network_slow" + }, + ) + } else if (SabrDemandAttemptFinisher.expireStalledInFlightDemand( + holder, + inFlightDemand, + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + reason = "terminal_deadline", + ) + ) { + return true + } + } + delay(nextCheckDelayMs(backoffRemainingMs, inFlightDemand.futureLiveRequest)) + continue + } + val request = holder.nextSegmentDemand() + if (request == null) { + delay(intervalMs) + continue + } + val outsideLiveWindow = holder.isLiveDemandOutsideRecoverableWindow(request) + val futureLiveRequest = holder.isFutureLiveRequest(request) + if (outsideLiveWindow) { + val identity = holder.segmentDemandIdentity(request) + if (identity != null && + SabrDemandAttemptFinisher.expireStalledDemand( + holder, + request, + identity, + recoverable = true, + reason = "outside_recoverable_window", + ) + ) { + return true + } + } + if (futureLiveRequest) { + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + delay(nextCheckDelayMs(holder.session.demandBackoffRemainingMs, futureLiveRequest)) + continue + } + val recoverableLiveRequest = + holder.livePlaybackSnapshot()?.active == true && !outsideLiveWindow + if (recoverableLiveRequest && holder.inFlightSegmentDemand() == null) { + val identity = holder.segmentDemandIdentity(request) + val requeued = synchronized(holder) { + holder.inFlightSegmentDemand() == null && + identity != null && + holder.requeueSegmentDemand(request, identity, clock()) + } + if (requeued) { + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + delay(nextCheckDelayMs(holder.session.demandBackoffRemainingMs, true)) + continue + } + } + val identity = holder.segmentDemandIdentity(request) + val registeredAtMs = identity?.let { holder.segmentDemandRegisteredAtMs(request, it) } + val nowMs = clock() + val backoffRemainingMs = holder.session.demandBackoffRemainingMs + val deadlineAtMs = identity?.let { demandIdentity -> + registeredAtMs?.let { registered -> + deadline.deadlineAtMs(demandIdentity, registered, nowMs, backoffRemainingMs) + } + } + if (identity != null && registeredAtMs != null && deadlineAtMs != null && nowMs >= deadlineAtMs && + SabrDemandAttemptFinisher.expireStalledDemand( + holder, + request, + identity, + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - registeredAtMs).coerceAtLeast(0L), + reason = "terminal_deadline", + ) + ) { + return true + } + delay(nextCheckDelayMs(backoffRemainingMs, futureLiveRequest)) + } + return false + } + + private fun nextCheckDelayMs(backoffRemainingMs: Long, futureLiveRequest: Boolean): Long = + maxOf(intervalMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt index 5d2eeee6..80809b00 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.io.InputStream import java.io.OutputStream -internal class SabrDownloadFrameWriter(private val output: OutputStream) { +class SabrDownloadFrameWriter(private val output: OutputStream) { private val header = ByteArray(FRAME_HEADER_SIZE) private val copyBuffer = ByteArray(COPY_BUFFER_SIZE) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt index f4ed0722..069aa5c2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrDownloadInitialization { +object SabrDownloadInitialization { suspend fun fetch( store: SabrSessionStore, holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt index f5f80626..431e0ceb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal data class SabrDownloadRange( +data class SabrDownloadRange( val part: Int = 0, val parts: Int = 1, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt index 16fdcc33..538eb082 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory import java.io.IOException import java.io.OutputStream -internal class SabrDownloadStreamer( +class SabrDownloadStreamer( private val store: SabrSessionStore, private val pumpTimeoutMs: Long = PUMP_TIMEOUT_MS, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt index 6a1b30c9..10557228 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -internal const val SABR_RECOVERABLE_FAILURE_PREFIX = "SABR recoverable failure:" +const val SABR_RECOVERABLE_FAILURE_PREFIX = "SABR recoverable failure:" -internal fun sabrRecoverableFailureMessage(message: String?): String { +fun sabrRecoverableFailureMessage(message: String?): String { val failure = message ?: "Unknown SABR recoverable failure" if (failure.contains("SABR spool", ignoreCase = true)) return failure return "$SABR_RECOVERABLE_FAILURE_PREFIX $failure" diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt index 305edca5..38b84e5d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt @@ -8,7 +8,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrInfo): StreamResponse { +fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrInfo): StreamResponse { val video = info.formats.filter { it.isVideo }.mapNotNull { it.toFallbackVideo(videoId) } val audio = info.formats.filter { it.isAudio }.mapNotNull { it.toFallbackAudio(videoId) } return copy( @@ -19,7 +19,7 @@ internal fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrI ) } -internal fun TokenYoutubeSession.toFallbackStreamResponse(videoId: String): StreamResponse = StreamResponse( +fun TokenYoutubeSession.toFallbackStreamResponse(videoId: String): StreamResponse = StreamResponse( id = videoId, title = title, uploaderName = author, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt similarity index 56% rename from src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt index 5be6711a..6976a1bb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt @@ -2,29 +2,28 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -internal class SabrFallbackStreamService( +class SabrFallbackStreamService( private val delegate: StreamService, private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, ) : StreamService { - override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { - if (!isYoutubeUrl(url)) return@coroutineScope delegate.getStreamInfo(url) + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return delegate.getStreamInfo(url) val videoId = youtubeVideoId(url) - val prepared = videoId?.let { async { sessionStore.fetchInfo(it, cachedFirst = true) } } val result = delegate.getStreamInfo(url) val response = (result as? ExtractionResult.Success)?.data + if (response?.isLive == true || response?.hasSabrStreams() == true) { + return result + } if (response == null) { - if (result !is ExtractionResult.Failure || videoId == null) return@coroutineScope result - prepared?.await() - val session = tokenSessionClient.fetchPlaybackSession(videoId) ?: return@coroutineScope result - return@coroutineScope ExtractionResult.Success(session.toFallbackStreamResponse(videoId)) + if (result !is ExtractionResult.Failure || videoId == null) return result + val session = tokenSessionClient.fetchPlaybackSession(videoId) ?: return result + return ExtractionResult.Success(session.toFallbackStreamResponse(videoId)) } - val playable = prepared?.await() - if (response.hasSabrStreams() || videoId == null || playable == null) return@coroutineScope result - ExtractionResult.Success(response.withSabrFallback(videoId, playable.info)) + if (videoId == null) return result + val playable = sessionStore.fetchInfo(videoId, cachedFirst = true) ?: return result + return ExtractionResult.Success(response.withSabrFallback(videoId, playable.info)) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt index 2ef01b4d..069b34d6 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState import java.util.Locale -internal object SabrHlsManifestBuilder { +object SabrHlsManifestBuilder { fun buildMaster( videoId: String, audio: YoutubeSabrFormat, diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt new file mode 100644 index 00000000..d12ea1f0 --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt @@ -0,0 +1,170 @@ +package dev.typetype.server.services + +import dev.typetype.server.sabr.SabrSegmentRequest +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +internal enum class SabrDemandInterruptionAction { + WAIT_FOR_LIVE, + RESTART, +} + +internal data class SabrDemandInterruption( + val action: SabrDemandInterruptionAction, + val atMs: Long, + val reason: String, +) + +class SabrInFlightDemand( + val request: SabrSegmentRequest, + val identity: String, + val registeredAtMs: Long, + val futureLiveRequest: Boolean, +) { + private var lastProgressVersion = Long.MIN_VALUE + private var lastProgressAtMs = registeredAtMs + private var attemptCancellation: (() -> Unit)? = null + private var interruption: SabrDemandInterruption? = null + private val attemptCount = AtomicInteger(0) + @Volatile private var attemptStartedAtNanos = 0L + @Volatile private var lastAttemptDurationMs = -1L + + fun observeProgress(version: Long, observedAtMs: Long): Long { + if (version != lastProgressVersion) { + lastProgressVersion = version + lastProgressAtMs = observedAtMs + } + return lastProgressAtMs + } + + fun registerAttemptCancellation(cancel: () -> Unit): Unit { + val cancelNow = synchronized(this) { + if (interruption != null) true else { + attemptCancellation = cancel + false + } + } + if (cancelNow) cancel() + } + + internal fun requestInterruption(next: SabrDemandInterruption): Boolean { + val callback = synchronized(this) { + if (interruption != null) return@synchronized null + interruption = next + val pending = attemptCancellation + attemptCancellation = null + pending + } + callback?.invoke() + return true + } + + internal fun consumeInterruption(): SabrDemandInterruption? = synchronized(this) { + interruption.also { interruption = null } + } + + fun clearAttemptCancellation(): Unit = synchronized(this) { + attemptCancellation = null + } + + fun recordAttemptStarted(): Unit { + attemptCount.incrementAndGet() + attemptStartedAtNanos = System.nanoTime() + } + + fun recordAttemptFinished(): Unit { + val startedAt = attemptStartedAtNanos + if (startedAt > 0L) { + lastAttemptDurationMs = ((System.nanoTime() - startedAt) / 1_000_000L).coerceAtLeast(0L) + } + attemptStartedAtNanos = 0L + } + + fun attempts(): Int = attemptCount.get() + + fun lastAttemptDurationMs(): Long = lastAttemptDurationMs +} + +object SabrInFlightDemandTracker { + private val demands = ConcurrentHashMap() + + fun begin( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + futureLiveRequest: Boolean, + ): Boolean { + val registeredAtMs = holder.segmentDemandRegisteredAtMs(request, identity) ?: return false + val demand = SabrInFlightDemand( + request, + identity, + registeredAtMs, + futureLiveRequest, + ) + return demands.putIfAbsent(holder.sessionToken, demand) == null + } + + fun current(holder: SabrSessionHolder): SabrInFlightDemand? = demands[holder.sessionToken] + + fun finish(holder: SabrSessionHolder, identity: String): Boolean { + val demand = demands[holder.sessionToken] ?: return false + if (demand.identity != identity) return false + return demands.remove(holder.sessionToken, demand) + } + + fun clear(holder: SabrSessionHolder): Unit { + demands.remove(holder.sessionToken) + } + + fun clearAll(): Unit = demands.clear() +} + +fun SabrSessionHolder.beginInFlightSegmentDemand( + request: SabrSegmentRequest, + identity: String, + futureLiveRequest: Boolean, +): Boolean = SabrInFlightDemandTracker.begin(this, request, identity, futureLiveRequest) + +fun SabrSessionHolder.inFlightSegmentDemand(): SabrInFlightDemand? = + SabrInFlightDemandTracker.current(this) + +fun SabrSessionHolder.finishInFlightSegmentDemand(identity: String): Boolean = + SabrInFlightDemandTracker.finish(this, identity) + +fun SabrSessionHolder.clearInFlightSegmentDemand(): Unit = + SabrInFlightDemandTracker.clear(this) + +internal fun SabrSessionHolder.registerInFlightDemandCancellation( + identity: String, + cancel: () -> Unit, +): Boolean = SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.let { it.registerAttemptCancellation(cancel); true } + ?: false + +internal fun SabrSessionHolder.requestInFlightDemandInterruption( + identity: String, + interruption: SabrDemandInterruption, +): Boolean = SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.requestInterruption(interruption) + ?: false + +internal fun SabrSessionHolder.consumeInFlightDemandInterruption(identity: String): SabrDemandInterruption? = + SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.consumeInterruption() + +internal fun SabrSessionHolder.clearInFlightDemandCancellation(identity: String): Unit { + SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.clearAttemptCancellation() +} + +internal fun SabrSessionHolder.recordInFlightAttemptStarted(): Unit { + SabrInFlightDemandTracker.current(this)?.recordAttemptStarted() +} + +internal fun SabrSessionHolder.recordInFlightAttemptFinished(): Unit { + SabrInFlightDemandTracker.current(this)?.recordAttemptFinished() +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index a1e9b15c..81543c2d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.cache.CacheService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -9,7 +10,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import org.slf4j.LoggerFactory -internal class SabrInfoFetcher( +class SabrInfoFetcher( private val tokenClient: TypetypeTokenSabrTokenClient, private val sessionClient: TypetypeTokenYoutubeSessionClient? = null, private val infoCache: SabrPreparedInfoCache = SabrPreparedInfoCache(), @@ -48,12 +49,17 @@ internal class SabrInfoFetcher( } private fun logFetch(videoId: String, startTimeMs: Long, startedAt: Long, source: String): Unit { + val durationMs = System.currentTimeMillis() - startedAt logger.info( "sabr_info_fetch videoId={} startTimeMs={} source={} elapsedMs={}", videoId, startTimeMs, source, - System.currentTimeMillis() - startedAt, + durationMs, + ) + PlaybackTraceLog.record( + "sabr_info_fetch", + "videoId=$videoId startTimeMs=$startTimeMs source=$source durationMs=$durationMs", ) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt index f0fbd251..a84b0f74 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration -internal class SabrInfoRepository( +class SabrInfoRepository( infoCache: SabrPreparedInfoCache, sharedCache: CacheService?, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt index eabccc84..d0b61d4f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt @@ -8,7 +8,7 @@ import java.io.ObjectInputStream import java.io.ObjectOutputStream import java.util.Base64 -internal class SabrInfoSharedCache(private val cache: CacheService?) { +class SabrInfoSharedCache(private val cache: CacheService?) { suspend fun getPlayback(videoId: String): YoutubeSabrInfo? = get("sabr:info:v2:$videoId") suspend fun getInitialization(videoId: String): YoutubeSabrInfo? = get("sabr:init-info:v1:$videoId") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt index 0b9788ca..d4511d47 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt @@ -11,7 +11,7 @@ import java.util.Base64 import java.util.Collections import java.util.WeakHashMap -internal object SabrInitializationData { +object SabrInitializationData { private const val CACHE_TTL_SECONDS = 21_600L private val memoryCache = BoundedExpiringCache( maxEntries = 512, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt index 6f85a8a0..5c5062a3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrInitializationPolicy { +object SabrInitializationPolicy { fun warmFormats( audioOnly: Boolean, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt index 1ccab335..3c4b7395 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt @@ -24,7 +24,7 @@ internal suspend fun fetchSabrInitializationSegment( holder.withPlayerContext { fetchSegment(request, localization) } } result.onFailure { error -> - logger.warn( + initFetcherLogger.warn( "sabr_init event=fetch_failed videoId={} itag={} errorType={} error={}", holder.key.videoId, request.format.itag, @@ -41,4 +41,4 @@ internal suspend fun fetchSabrInitializationSegment( } } -private val logger = LoggerFactory.getLogger("SabrInitializationSegmentFetcher") +private val initFetcherLogger = LoggerFactory.getLogger("SabrInitializationSegmentFetcher") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt index d89e3736..d522aa9c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrBufferedRange import dev.typetype.server.sabr.YoutubeSabrFormat -internal inline fun withLiveContinuationRequestShape( +inline fun withLiveContinuationRequestShape( holder: SabrSessionHolder, block: () -> T, ): T { @@ -24,7 +24,7 @@ internal inline fun withLiveContinuationRequestShape( } } -private fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { +@PublishedApi internal fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { observedMediaSegment(format) ?: return null val sequence = lastServedSequence(format) ?: (playbackStartSequence(format, requestedSeekTimeMs() ?: playerTimeMs()) - 1).coerceAtLeast(0) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt index 580dcfba..75380261 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services -internal data class SabrLiveMediaParts( +data class SabrLiveMediaParts( val initialization: ByteArray, val media: ByteArray, ) -internal object SabrLiveMediaNormalizer { +object SabrLiveMediaNormalizer { fun split(mimeType: String, data: ByteArray): SabrLiveMediaParts? = when { mimeType.substringBefore(';').trim().lowercase().endsWith("/mp4") -> splitMp4(data) mimeType.substringBefore(';').trim().lowercase().endsWith("/webm") -> splitWebM(data) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index 0507aa72..7dd09fb4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal data class SabrLivePlaybackSnapshot( +data class SabrLivePlaybackSnapshot( val active: Boolean, val postLiveDvr: Boolean, val headSequence: Long, @@ -14,7 +14,7 @@ internal data class SabrLivePlaybackSnapshot( val targetLatencyMs: Long, ) -internal fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? { +fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? { val state = session.streamState val postLiveDvr = runCatching { state.isPostLiveDvr }.getOrDefault(false) val sessionLive = runCatching { session.isLive }.getOrDefault(false) @@ -57,7 +57,7 @@ internal fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? ) } -internal fun SabrSessionHolder.resolvePlaybackStartMs(requestedStartMs: Long): Long { +fun SabrSessionHolder.resolvePlaybackStartMs(requestedStartMs: Long): Long { val requested = requestedStartMs.coerceAtLeast(0L) val live = livePlaybackSnapshot() ?: return requested if (!live.active) return requested.coerceAtMost(live.seekableEndMs.takeIf { it > 0L } ?: requested) @@ -73,7 +73,7 @@ private fun SabrSessionHolder.availableLiveMediaStartMs(): Long? { return maxOf(audioStartMs, videoStartMs) } -internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false if (session.getCachedSegment(request) != null) return false @@ -105,7 +105,7 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): return requestStartMs > 0L && requestStartMs >= completeEndMs && !isHistoricalLiveRequest(request) } -internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false val observed = observedMediaSegment(request.format) ?: return false @@ -117,7 +117,7 @@ internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentReque return request.sequenceNumber < observed.header.sequenceNumber } -internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = +fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = DEFAULT_PLAYBACK_RETRY_MS private fun dev.typetype.server.sabr.YoutubeSabrStreamState.observedEndMs( @@ -129,8 +129,8 @@ private fun dev.typetype.server.sabr.YoutubeSabrStreamState.observedEndMs( internal const val LIVE_EDGE_POLL_MS = 2_000L internal const val DEFAULT_PLAYBACK_RETRY_MS = 500L -private const val LIVE_TARGET_LATENCY_MS = 20_000L +private const val LIVE_TARGET_LATENCY_MS = 2_000L private const val LIVE_EDGE_TOLERANCE_MS = 15_000L -private const val LIVE_HISTORICAL_REQUEST_TOLERANCE_MS = LIVE_TARGET_LATENCY_MS + LIVE_EDGE_TOLERANCE_MS +private const val LIVE_HISTORICAL_REQUEST_TOLERANCE_MS = 35_000L private const val LIVE_DVR_WINDOW_MS = 12L * 60L * 60L * 1_000L internal const val LIVE_FUTURE_SEGMENT_TOLERANCE = 2 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt index 91ed693c..84140e8a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt @@ -3,14 +3,14 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.isLiveDemandOutsideRecoverableWindow(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isLiveDemandOutsideRecoverableWindow(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) return requestEndMs > 0L && live.headTimeMs - requestEndMs > MAX_RECOVERABLE_LIVE_GAP_MS } -internal fun SabrSessionHolder.failLivePlaybackDiscontinuity( +fun SabrSessionHolder.failLivePlaybackDiscontinuity( format: YoutubeSabrFormat, targetMs: Long, segment: CachedSabrSegment, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt index ac33f9f7..3657d332 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt @@ -18,11 +18,12 @@ internal suspend fun pumpLiveReadAhead( holder.setPlaybackState(SabrPlaybackState.REQUESTING) withLiveContinuationRequestShape(holder) { pump() } val cached = holder.cacheObservedLiveContinuation(onResolved) - holder.setPlaybackState(SabrPlaybackState.IDLE) + val waitingForLive = cached == 0 && holder.nextSegmentDemand()?.let(holder::isFutureLiveRequest) == true + holder.setPlaybackState(if (waitingForLive) SabrPlaybackState.WAITING_FOR_LIVE else SabrPlaybackState.IDLE) return cached > 0 && holder.observedLiveAheadMs() < targetAheadMs } -internal fun SabrSessionHolder.observedLiveAheadMs(): Long { +fun SabrSessionHolder.observedLiveAheadMs(): Long { val audioEndMs = observedMediaEndMs(audioFormat) ?: return 0L val videoEndMs = if (isVideoActive()) observedMediaEndMs(videoFormat) ?: return 0L else Long.MAX_VALUE return (minOf(audioEndMs, videoEndMs) - playerTimeMs()).coerceAtLeast(0L) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt index 189488ab..e72039ee 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt @@ -4,13 +4,13 @@ import dev.typetype.server.sabr.SabrBufferedRange import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -internal data class SabrLiveWarmupTarget( +data class SabrLiveWarmupTarget( val sequence: Int, val timeMs: Long, val segmentDurationMs: Long, ) -internal fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { +fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { val live = livePlaybackSnapshot() ?.takeIf { it.active && it.headSequence > 1L && it.headTimeMs > 0L } ?: return null @@ -27,7 +27,7 @@ internal fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { return SabrLiveWarmupTarget(targetSequence, targetTimeMs, segmentDurationMs) } -internal fun SabrSessionHolder.advanceLiveWarmupTarget( +fun SabrSessionHolder.advanceLiveWarmupTarget( target: SabrLiveWarmupTarget, segments: List, ): SabrLiveWarmupTarget { @@ -52,7 +52,7 @@ internal fun SabrSessionHolder.advanceLiveWarmupTarget( ) } -internal inline fun withLiveWarmupRequestShape( +inline fun withLiveWarmupRequestShape( holder: SabrSessionHolder, target: SabrLiveWarmupTarget?, block: () -> T, @@ -74,7 +74,7 @@ internal inline fun withLiveWarmupRequestShape( } } -private fun YoutubeSabrFormat.liveWarmupRange(targetSequence: Int, targetTimeMs: Long): SabrBufferedRange { +@PublishedApi internal fun YoutubeSabrFormat.liveWarmupRange(targetSequence: Int, targetTimeMs: Long): SabrBufferedRange { val bufferedSequence = (targetSequence - 1).coerceAtLeast(0) return SabrBufferedRange( itag, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt index 33b45d76..16cb10e5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrManifestBuilder { +object SabrManifestBuilder { fun build( videoId: String, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt index 433b3aec..5d28f032 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrManifestTiming { +object SabrManifestTiming { fun videoDurationSec( audio: YoutubeSabrFormat, video: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt index 101ea2a6..e100171b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem -internal fun StreamResponse.withSabrManifestUrls(): StreamResponse = copy( +fun StreamResponse.withSabrManifestUrls(): StreamResponse = copy( videoStreams = videoStreams.map { it.withSabrManifestUrl(id) }, videoOnlyStreams = videoOnlyStreams.map { it.withSabrManifestUrl(id) }, audioStreams = audioStreams.map { it.withSabrManifestUrl(id) }, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt index e0c827de..ac4d01fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrMemorySegmentCache(private val maxBytes: Long) { +class SabrMemorySegmentCache(private val maxBytes: Long) { private val segments = LinkedHashMap(64, 0.75f, true) private var bytes = 0L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt index 4b99aaca..39346c8c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun splitMime(mime: String): Pair { +fun splitMime(mime: String): Pair { val parts = mime.split(";", limit = 2) val container = parts[0].trim() val codecs = if (parts.size > 1) { @@ -13,7 +13,7 @@ internal fun splitMime(mime: String): Pair { return container to codecs } -internal fun videoSizeAttr(video: YoutubeSabrFormat): String { +fun videoSizeAttr(video: YoutubeSabrFormat): String { val width = video.width.takeIf { it > 0 } val height = video.height.takeIf { it > 0 } return if (width != null && height != null) " width=\"$width\" height=\"$height\"" else "" diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt index 43170eca..1e478709 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest -internal fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): Boolean { pendingRefetchRequest()?.takeIf { it.matches(request) }?.let { consumeRefetch() setPlaybackState(SabrPlaybackState.REPOSITIONING) @@ -18,13 +18,13 @@ internal fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): return false } -internal fun SabrSessionHolder.prepareForExplicitRewind(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.prepareForExplicitRewind(request: SabrSegmentRequest): Unit = session.prepareForRewind(request, explicitSeekPositionMs()) -internal fun SabrSessionHolder.prepareForExplicitForwardJump(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.prepareForExplicitForwardJump(request: SabrSegmentRequest): Unit = session.prepareForForwardJump(request, explicitSeekPositionMs()) -internal fun SabrSessionHolder.prepareForHistoricalLiveRewind(request: SabrSegmentRequest): Unit { +fun SabrSessionHolder.prepareForHistoricalLiveRewind(request: SabrSegmentRequest): Unit { val seekPositionMs = requestedSeekTimeMs()?.takeIf { request.contains(this, it) } if (seekPositionMs == null) session.prepareForRewind(request) else session.prepareForRewind(request, seekPositionMs) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt index 1eee1935..c72c0837 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal suspend fun SabrSessionStore.findCachedPlaybackMediaAt( +suspend fun SabrSessionStore.findCachedPlaybackMediaAt( holder: SabrSessionHolder, format: YoutubeSabrFormat, targetMs: Long, @@ -41,7 +41,7 @@ private suspend fun SabrSessionStore.cachedMedia( return cachedSegment(holder, SabrSegmentRequest.media(format, sequence)) } -internal fun CachedSabrSegment.coversPlaybackTime( +fun CachedSabrSegment.coversPlaybackTime( holder: SabrSessionHolder, format: YoutubeSabrFormat, targetMs: Long, @@ -51,7 +51,7 @@ internal fun CachedSabrSegment.coversPlaybackTime( return targetMs >= effectiveStartMs - TIMING_TOLERANCE_MS && targetMs < effectiveStartMs + effectiveDurationMs } -internal fun CachedSabrSegment.isAcceptableLiveFollowingSegment( +fun CachedSabrSegment.isAcceptableLiveFollowingSegment( expectedSequence: Int, targetMs: Long, continuesServedTrack: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt index 2c003d40..3001f288 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap -internal object SabrPlaybackDiagnostics { +object SabrPlaybackDiagnostics { private val blockers = ConcurrentHashMap() fun record(holder: SabrSessionHolder, request: SabrSegmentRequest, message: String?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt similarity index 62% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt index ddbd5804..a0d66257 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt @@ -1,4 +1,4 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.nextReplacementGeneration(): Long = +fun SabrSessionHolder.nextReplacementGeneration(): Long = activeGeneration().let { if (it == Long.MAX_VALUE) it else it + 1L } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt index 0df44214..b9d2f033 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrPlaybackInfoResolver( +class SabrPlaybackInfoResolver( private val sessionStore: SabrSessionStore, private val authenticatedInfoService: AuthenticatedSabrInfoService?, ) { diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt new file mode 100644 index 00000000..f8c6d14d --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt @@ -0,0 +1,72 @@ +package dev.typetype.server.services + +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.sabr.YoutubeSabrFormat +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withTimeoutOrNull + +data class SabrPlaybackInitializationPreloadResult( + val video: ByteArray?, + val audio: ByteArray?, +) { + fun isComplete(audioOnly: Boolean): Boolean = + audio != null && (audioOnly || video != null) + + fun missingTracks(audioOnly: Boolean, videoItag: Int, audioItag: Int): String = + buildList { + if (!audioOnly && video == null) add("video:$videoItag") + if (audio == null) add("audio:$audioItag") + }.joinToString() +} + +object SabrPlaybackInitializationPreloader { + suspend fun preload( + sessionStore: SabrSessionStore, + holder: SabrSessionHolder, + audioOnly: Boolean, + timeoutMs: Long, + ): SabrPlaybackInitializationPreloadResult = withTimeoutOrNull(timeoutMs) { + coroutineScope { + val video = holder.videoFormat + .takeUnless { audioOnly } + ?.let { format -> async { fetchTrack(sessionStore, holder, format, "video") } } + val audio = async { fetchTrack(sessionStore, holder, holder.audioFormat, "audio") } + SabrPlaybackInitializationPreloadResult(video?.await(), audio.await()) + } + } ?: SabrPlaybackInitializationPreloadResult(null, null) + + private suspend fun fetchTrack( + sessionStore: SabrSessionStore, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + track: String, + ): ByteArray? { + val startedAt = System.nanoTime() + val fields = "videoId=${holder.key.videoId} track=$track itag=${format.itag}" + PlaybackTraceLog.record("sabr_init_preload_start", fields) + val bytes = try { + sessionStore.fetchInitializationData(holder, format) + } catch (error: CancellationException) { + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=cancelled", + ) + throw error + } catch (error: Exception) { + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=failed errorType=${error.javaClass.simpleName}", + ) + throw error + } + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=${if (bytes == null) "miss" else "hit"} bytes=${bytes?.size ?: 0}", + ) + return bytes + } + + private fun elapsedMs(startedAt: Long): Long = (System.nanoTime() - startedAt) / 1_000_000 +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt index c713e7a0..61329705 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrPlaybackManifestService { +class SabrPlaybackManifestService { fun build(holder: SabrSessionHolder, mediaBasePath: String): SabrPlaybackManifestResult { val state = holder.session.streamState val knownAudio = maxOf(state.getEndSegment(holder.audioFormat), state.getMaxSegment(holder.audioFormat).toLong()) @@ -53,7 +53,7 @@ internal class SabrPlaybackManifestService { } } -internal sealed class SabrPlaybackManifestResult { +sealed class SabrPlaybackManifestResult { data class Ready(val manifest: String) : SabrPlaybackManifestResult() data class Retry(val status: String) : SabrPlaybackManifestResult() } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt index bb1bd6c9..86e1b330 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt @@ -7,7 +7,7 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { +class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { suspend fun fetch( holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt index edc38946..05156fc0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal data class SabrPlaybackPreparation( +data class SabrPlaybackPreparation( val holder: SabrSessionHolder, val startTimeMs: Long, val ready: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt index 3f2cfc7c..e994b04e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment -internal sealed class SabrPlaybackSegmentResult { +sealed class SabrPlaybackSegmentResult { data class Ready(val mimeType: String, val bytes: ByteArray) : SabrPlaybackSegmentResult() data class Stream( val mimeType: String, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt similarity index 82% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt index 5fa0feb9..35c2e614 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt @@ -2,13 +2,16 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { +fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { liveSequenceAt(format, playerTimeMs)?.let { return it } - return session.streamState.getSegmentNumberAtOrAfterTimeMs(format, playerTimeMs.coerceAtLeast(0L)) + val mappedSequence = session.streamState + .getSegmentNumberAtOrAfterTimeMs(format, playerTimeMs.coerceAtLeast(0L)) .coerceAtLeast(1) + val endSequence = session.streamState.getEndSegment(format).toInt() + return mappedSequence.coerceAtMost(endSequence.takeIf { it > 0 } ?: mappedSequence) } -internal fun SabrSessionHolder.playbackContinuationSequence( +fun SabrSessionHolder.playbackContinuationSequence( format: YoutubeSabrFormat, playerTimeMs: Long, continueAfterLastServed: Boolean, @@ -17,7 +20,7 @@ internal fun SabrSessionHolder.playbackContinuationSequence( ?.plus(1) ?: playbackStartSequence(format, playerTimeMs) -internal fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, sequence: Int): Long { +fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, sequence: Int): Long { val observed = observedMediaSegment(format) val observedStartMs = observed?.header?.startMs?.takeIf { it >= 0L } if (observed != null && observedStartMs != null) { @@ -28,7 +31,7 @@ internal fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, return session.streamState.getSegmentStartMs(format, sequence).coerceAtLeast(0L) } -internal fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrFormat, sequence: Int): Long { +fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrFormat, sequence: Int): Long { val observed = observedMediaSegment(format) val observedStartMs = observed?.header?.startMs?.takeIf { it >= 0L } val observedDurationMs = observed?.header?.durationMs?.takeIf { it > 0L } @@ -44,7 +47,7 @@ internal fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrForm return (session.streamState.getSegmentEndMs(format, sequence) - startMs).coerceAtLeast(1L) } -internal fun SabrSessionHolder.playbackSegmentEndMs(format: YoutubeSabrFormat, sequence: Int): Long = +fun SabrSessionHolder.playbackSegmentEndMs(format: YoutubeSabrFormat, sequence: Int): Long = playbackSegmentStartMs(format, sequence) + playbackSegmentDurationMs(format, sequence) private fun SabrSessionHolder.liveSequenceAt(format: YoutubeSabrFormat, playerTimeMs: Long): Int? { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index b19fc4dd..160775e5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { +class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { private val mediaFetcher = SabrPlaybackMediaFetcher(sessionStore) suspend fun prepare( videoId: String, @@ -63,14 +63,13 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS ) } - private suspend fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { + private fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) if (startTimeMs == 0L) { holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) } - sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) } suspend fun seek( @@ -159,7 +158,6 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS private companion object { const val PREPARING = "preparing" const val INITIALIZATION_PRELOAD_TIMEOUT_MS = 6_000L - const val LIVE_INITIAL_PUMPS = 8 const val OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS = 9_007_199_254_740_991L } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt index e2121ed0..644e25a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrPlaybackStarter { +object SabrPlaybackStarter { fun start( store: SabrSessionStore, holder: SabrSessionHolder, @@ -33,7 +33,7 @@ internal object SabrPlaybackStarter { private val logger = LoggerFactory.getLogger(SabrPlaybackStarter::class.java) } -internal fun SabrSessionHolder.requestPlaybackReposition(playerTimeMs: Long, generation: Long): Unit { +fun SabrSessionHolder.requestPlaybackReposition(playerTimeMs: Long, generation: Long): Unit { val targetFormat = if (isVideoActive()) videoFormat else audioFormat val request = SabrSegmentRequest.media(targetFormat, playbackStartSequence(targetFormat, playerTimeMs)) val companion = audioFormat.takeIf { isVideoActive() }?.let { format -> diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt similarity index 82% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt index 3739acba..8e7c3d67 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrPlaybackState { +enum class SabrPlaybackState { IDLE, PREPARING, REQUESTING, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt index 4809cf53..967c12a1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.util.concurrent.atomic.AtomicReference -internal class SabrPlaybackStatus { +class SabrPlaybackStatus { private val state = AtomicReference(SabrPlaybackState.IDLE) private val terminalError = AtomicReference() private val networkError = AtomicReference() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt index 45c37f49..cd10cf32 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import org.slf4j.LoggerFactory -internal class SabrPlaybackWarmer { +class SabrPlaybackWarmer { suspend fun preflight(store: SabrSessionStore, holder: SabrSessionHolder, playerTimeMs: Long): Boolean { holder.setActiveTracks(videoActive = true, audioActive = true) store.ensureWarmed(holder) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt index f28cfbbc..506d5f22 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrProtocolException import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo -internal class SabrPlayerContextRecovery( +class SabrPlayerContextRecovery( private val videoId: String, initialToken: SabrTokenBundle, private val tokenClient: TypetypeTokenSabrTokenClient, @@ -46,7 +46,7 @@ internal class SabrPlayerContextRecovery( this is SabrProtocolException && message == "Player response has no streamingData for $profile" } -internal sealed interface SabrPlayerProbeResult { +sealed interface SabrPlayerProbeResult { data class Success( val info: YoutubeSabrInfo, val token: SabrTokenBundle, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt index c5ba312b..5874ae01 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.SabrAdapter -internal fun interface SabrPlayerInfoProbe { +fun interface SabrPlayerInfoProbe { fun fetch( videoId: String, profile: YoutubeSabrClientProfile, @@ -14,7 +14,7 @@ internal fun interface SabrPlayerInfoProbe { ): YoutubeSabrInfo } -internal object PipePipeSabrPlayerInfoProbe : SabrPlayerInfoProbe { +object PipePipeSabrPlayerInfoProbe : SabrPlayerInfoProbe { private val localization = Localization("en", "US") private val contentCountry = ContentCountry("US") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt index b7f160ff..336628d3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.time.Duration -internal class SabrPreparedInfoCache( +class SabrPreparedInfoCache( private val ttl: Duration = Duration.ofMinutes(10), maxEntries: Int = 256, clock: () -> Long = System::currentTimeMillis, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt index 47058543..c5cea9ef 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrProtectedContextRecovery( +class SabrProtectedContextRecovery( private val tokenClient: TypetypeTokenSabrTokenClient, ) { fun refreshIfRejected(videoId: String, rejectedVisitorData: String?) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt index 05d7f326..63f6f0b5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrProtectedResponseGuard( +class SabrProtectedResponseGuard( private val maximumResponses: Int = SabrPumpPolicy.MAX_PROTECTED_NO_MEDIA_RESPONSES, ) { private var lastResponseNumber: Int? = null @@ -42,4 +42,4 @@ internal class SabrProtectedResponseGuard( } } -internal class SabrProtectedNoMediaException(message: String) : RuntimeException(message) +class SabrProtectedNoMediaException(message: String) : RuntimeException(message) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt index 0344333d..95695438 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.withTimeoutOrNull import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong -internal class SabrPumpCoordinator { +class SabrPumpCoordinator { private val started = AtomicBoolean(false) private val wakeVersion = AtomicLong() private val wakeups = Channel(Channel.CONFLATED) @@ -30,13 +30,13 @@ internal class SabrPumpCoordinator { } } -internal fun SabrSessionHolder.markPumpStarted(): Boolean = pumpCoordinator.markStarted() +fun SabrSessionHolder.markPumpStarted(): Boolean = pumpCoordinator.markStarted() -internal fun SabrSessionHolder.markPumpStopped(): Unit = pumpCoordinator.markStopped() +fun SabrSessionHolder.markPumpStopped(): Unit = pumpCoordinator.markStopped() -internal fun SabrSessionHolder.wakePump(): Unit = pumpCoordinator.wake() +fun SabrSessionHolder.wakePump(): Unit = pumpCoordinator.wake() -internal fun SabrSessionHolder.pumpWakeVersion(): Long = pumpCoordinator.wakeVersion() +fun SabrSessionHolder.pumpWakeVersion(): Long = pumpCoordinator.wakeVersion() internal suspend fun SabrSessionHolder.awaitPumpWake(observedVersion: Long, timeoutMs: Long): Unit = pumpCoordinator.awaitWake(observedVersion, timeoutMs) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt index 645968ef..351b17fe 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.job import kotlinx.coroutines.launch -internal fun CoroutineScope.launchSabrPump( +fun CoroutineScope.launchSabrPump( pump: SabrSessionPump, registry: SabrSessionRegistry, holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt similarity index 67% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index 849a5068..532b1876 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrPumpLogger { +object SabrPumpLogger { private val logger = LoggerFactory.getLogger(SabrSessionPumpLoop::class.java) fun start(holder: SabrSessionHolder, event: String, request: SabrSegmentRequest?): Unit { @@ -52,15 +52,42 @@ internal object SabrPumpLogger { ) } - fun expired(holder: SabrSessionHolder, request: SabrSegmentRequest, recoverable: Boolean): Unit { + fun expired( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + recoverable: Boolean, + registeredAtMs: Long? = null, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long? = null, + reason: String = "deadline", + attempts: Int = 0, + lastAttemptDurationMs: Long = -1L, + event: String = "demand_expired", + ): Unit { + val liveHeadTimeMs = holder.livePlaybackSnapshot()?.headTimeMs ?: -1L + val segmentEndMs = runCatching { holder.playbackSegmentEndMs(request.format, request.sequenceNumber) } + .getOrDefault(-1L) + val bufferedEdgeMs = holder.session.streamState.getMinBufferedEndMs() + val ageMs = registeredAtMs?.let { (nowMs - it).coerceAtLeast(0L) } ?: -1L logger.warn( - "sabr_pump event=demand_expired videoId={} request={} recoverable={} state={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", + "sabr_pump event={} videoId={} request={} track={} sequence={} recoverable={} ageMs={} registeredAtMs={} expectedDelayMs={} liveHeadTimeMs={} segmentEndMs={} bufferedEdgeMs={} lastAttemptDurationMs={} attempts={} reason={} state={} requestNumber={} readerHeadMs={} readerTailMs={} cachedBytes={}", + event, holder.key.videoId, request.summary(), + if (request.format.isAudio) "audio" else "video", + request.sequenceNumber, recoverable, + ageMs, + registeredAtMs ?: -1L, + expectedDelayMs ?: -1L, + liveHeadTimeMs, + segmentEndMs, + bufferedEdgeMs, + lastAttemptDurationMs, + attempts, + reason, holder.playbackState(), holder.session.requestNumber, - holder.session.streamState.getMinBufferedEndMs(), holder.readerHeadMs(), holder.readerTailMs(), holder.session.cachedBytes, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt index 15002930..8076b077 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt @@ -1,13 +1,13 @@ package dev.typetype.server.services -internal fun preparePumpEviction(holder: SabrSessionHolder) { +fun preparePumpEviction(holder: SabrSessionHolder) { val retainedPlayHeadMs = (holder.readerTailMs() - SabrPumpPolicy.backBufferMs(holder)).coerceAtLeast(0L) holder.session.setPlayHeadMs(retainedPlayHeadMs) holder.session.evictPlayed() } -internal fun pumpDemandDelayMs(holder: SabrSessionHolder, intervalMs: Long): Long { +fun pumpDemandDelayMs(holder: SabrSessionHolder, intervalMs: Long): Long { val demand = holder.nextSegmentDemand() return SabrPumpPolicy.demandDelayMs( intervalMs, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt index 8a5c6eae..34685d52 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object SabrPumpPolicy { +object SabrPumpPolicy { const val IDLE_POLL_MS = 100L const val ERROR_RETRY_MS = 1_000L const val MAX_CONSECUTIVE_IO_ERRORS = 5 @@ -12,6 +12,7 @@ internal object SabrPumpPolicy { const val SEEK_MODE_MS = 8_000L const val MIN_SERVER_READAHEAD_CUSHION_MS = 3_000L const val SERVER_AHEAD_MARGIN_MS = 16_000L + const val LIVE_READAHEAD_CUSHION_MS = 2_000L const val DEMAND_TARGET_DEADLINE_MS = 15_000L const val MAX_DEMAND_BACKOFF_EXTENSION_MS = 30_000L const val MAX_PROTECTED_NO_MEDIA_RESPONSES = 5 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt index da984e00..1ca188eb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import kotlin.math.roundToLong -internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) { +class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) { private val startedAtMs = clock() private val protectedResponseGuard = SabrProtectedResponseGuard() private var lastRequestMs = 0L @@ -37,22 +37,22 @@ internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTi if (demandKey == requestKey) resetDemandRecovery() } + fun demandNeedsTargetedShape(requestKey: String): Boolean = + demandKey == requestKey && demandTrackReadvertised + fun demandRecoveryAction( requestKey: String, - targetTrackSegmentCount: Int, + requestPerformed: Boolean, resolved: Boolean, ): SabrDemandRecoveryAction { - if (resolved) { + if (resolved || !requestPerformed) { resetDemandRecovery() return SabrDemandRecoveryAction.WAIT } ensureDemand(requestKey) - if (targetTrackSegmentCount > 0) { - if (!demandTrackReadvertised) { - demandTrackReadvertised = true - return SabrDemandRecoveryAction.READVERTISE_TRACK - } - return SabrDemandRecoveryAction.WAIT + if (!demandTrackReadvertised) { + demandTrackReadvertised = true + return SabrDemandRecoveryAction.READVERTISE_TRACK } return SabrDemandRecoveryAction.WAIT } @@ -64,15 +64,32 @@ internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTi holder.session.cachedBytes > SabrPumpPolicy.MAX_AHEAD_BYTES } - internal fun targetReadaheadCushionMs(holder: SabrSessionHolder): Long { + fun targetReadaheadCushionMs(holder: SabrSessionHolder): Long { + var usesLiveReadAhead = false val baseCushionMs = when { isSeekMode() -> SabrPumpPolicy.SEEK_READAHEAD_CUSHION_MS isStartupBurst() -> SabrPumpPolicy.STARTUP_BURST_READAHEAD_CUSHION_MS holder.playerTimeMs() == 0L && holder.readerTailMs() == 0L -> SabrPumpPolicy.STARTUP_READAHEAD_CUSHION_MS + holder.livePlaybackSnapshot()?.active == true -> { + usesLiveReadAhead = true + liveReadaheadCushionMs(holder) + } else -> serverReadaheadCushionMs(holder) } - return rateAwareCushionMs(baseCushionMs, holder.playbackRate()) + val maximumCushionMs = if (usesLiveReadAhead) baseCushionMs else MAX_RATE_AWARE_CUSHION_MS + return rateAwareCushionMs(baseCushionMs, holder.playbackRate(), maximumCushionMs) + } + + private fun liveReadaheadCushionMs(holder: SabrSessionHolder): Long { + val bitsPerSecond = holder.videoFormat.bitrate.toLong().coerceAtLeast(0L) + + holder.audioFormat.bitrate.toLong().coerceAtLeast(0L) + if (bitsPerSecond <= 0L) return SabrPumpPolicy.LIVE_READAHEAD_CUSHION_MS + val bytesPerMs = (bitsPerSecond / 8_000L).coerceAtLeast(1L) + val memoryBoundMs = SabrPumpPolicy.MAX_AHEAD_BYTES / bytesPerMs + return SabrPumpPolicy.LIVE_READAHEAD_CUSHION_MS.coerceAtMost( + memoryBoundMs.coerceAtLeast(SabrPumpPolicy.MIN_SERVER_READAHEAD_CUSHION_MS), + ) } private fun serverReadaheadCushionMs(holder: SabrSessionHolder): Long { @@ -85,8 +102,8 @@ internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTi ) } - private fun rateAwareCushionMs(baseMs: Long, playbackRate: Float): Long = - (baseMs * playbackRate.coerceIn(0.25f, 4.0f)).roundToLong().coerceAtMost(MAX_RATE_AWARE_CUSHION_MS) + private fun rateAwareCushionMs(baseMs: Long, playbackRate: Float, maximumMs: Long): Long = + (baseMs * playbackRate.coerceIn(0.25f, 4.0f)).roundToLong().coerceAtMost(maximumMs) private fun cappedServerAheadPlayerTimeMs(holder: SabrSessionHolder, edgeMs: Long): Long = maxOf(holder.playerTimeMs(), edgeMs - SabrPumpPolicy.SERVER_AHEAD_MARGIN_MS) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt index 3e262d83..1dda4386 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrSegmentCache { +class SabrSegmentCache { fun get(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? = holder.cachedSegment(key(holder, request)) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt index 8d8ee50b..dabd7641 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal fun SabrSessionHolder.resolveSegmentDemand( +fun SabrSessionHolder.resolveSegmentDemand( request: SabrSegmentRequest, identity: String, onResolved: (SabrMediaSegment) -> Unit = {}, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt similarity index 75% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt index f68e0adc..864bc979 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong -internal object SabrSegmentDemandTracker { +object SabrSegmentDemandTracker { private val demands = ConcurrentHashMap() private val order = AtomicLong() @@ -72,6 +72,19 @@ internal object SabrSegmentDemandTracker { return demand.registeredAtMs.takeIf { identity(requestKey, demand) == identity } } + fun requeue( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + registeredAtMs: Long, + ): Boolean { + val requestKey = key(holder, request) + val demand = demands[requestKey] ?: return false + if (identity(requestKey, demand) != identity) return false + val replacement = SegmentDemand(request, order.incrementAndGet(), registeredAtMs) + return demands.replace(requestKey, demand, replacement) + } + fun clear(holder: SabrSessionHolder, request: SabrSegmentRequest, identity: String): Boolean { val requestKey = key(holder, request) val demand = demands[requestKey] ?: return false @@ -100,34 +113,41 @@ internal object SabrSegmentDemandTracker { ) } -internal fun SabrSessionHolder.requestSegmentDemand( +fun SabrSessionHolder.requestSegmentDemand( request: SabrSegmentRequest, generation: Long = activeGeneration(), registeredAtMs: Long = System.currentTimeMillis(), ): Unit = synchronized(this) { + touch() val state = playbackState() if (generation == activeGeneration() && state != SabrPlaybackState.TERMINAL && state != SabrPlaybackState.NETWORK_FAILED) { SabrSegmentDemandTracker.request(this, request, registeredAtMs) } } -internal fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest): Unit = SabrSegmentDemandTracker.clear(this, request) -internal fun SabrSessionHolder.clearSegmentDemands(): Unit = SabrSegmentDemandTracker.clear(this) +fun SabrSessionHolder.clearSegmentDemands(): Unit = SabrSegmentDemandTracker.clear(this) -internal fun SabrSessionHolder.nextSegmentDemand(): SabrSegmentRequest? = SabrSegmentDemandTracker.next(this) +fun SabrSessionHolder.nextSegmentDemand(): SabrSegmentRequest? = SabrSegmentDemandTracker.next(this) -internal fun SabrSessionHolder.segmentDemandIdentity(request: SabrSegmentRequest): String? = +fun SabrSessionHolder.segmentDemandIdentity(request: SabrSegmentRequest): String? = SabrSegmentDemandTracker.identity(this, request) -internal fun SabrSessionHolder.isSegmentDemandActive(request: SabrSegmentRequest, identity: String): Boolean = +fun SabrSessionHolder.isSegmentDemandActive(request: SabrSegmentRequest, identity: String): Boolean = SabrSegmentDemandTracker.isActive(this, request, identity) -internal fun SabrSessionHolder.segmentDemandRegisteredAtMs(request: SabrSegmentRequest, identity: String): Long? = +fun SabrSessionHolder.segmentDemandRegisteredAtMs(request: SabrSegmentRequest, identity: String): Long? = SabrSegmentDemandTracker.registeredAtMs(this, request, identity) -internal fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean = +internal fun SabrSessionHolder.requeueSegmentDemand( + request: SabrSegmentRequest, + identity: String, + registeredAtMs: Long, +): Boolean = SabrSegmentDemandTracker.requeue(this, request, identity, registeredAtMs) + +fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean = SabrSegmentDemandTracker.clear(this, request, identity) -internal fun SabrSessionHolder.pendingSegmentDemandSummary(): String? = SabrSegmentDemandTracker.pendingSummary(this) +fun SabrSessionHolder.pendingSegmentDemandSummary(): String? = SabrSegmentDemandTracker.pendingSummary(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt index 19dbecf3..7da9bde4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant -internal class SabrSessionFactory( +class SabrSessionFactory( private val tokenClient: TypetypeTokenSabrTokenClient, ) { fun create( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index 8bb0eb72..dbd9c57b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -12,7 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference -internal class SabrSessionHolder( +class SabrSessionHolder( val session: YoutubeSabrSession, val info: YoutubeSabrInfo, val audioFormat: YoutubeSabrFormat, @@ -33,7 +33,7 @@ internal class SabrSessionHolder( private val liveInitializationData = ConcurrentHashMap() private val pendingRefetch = AtomicReference() private val pendingForwardSeek = AtomicReference() - internal val pumpCoordinator = SabrPumpCoordinator() + val pumpCoordinator = SabrPumpCoordinator() private val unauthorizedRefreshAttempted = AtomicBoolean(false) private val expectedLive = AtomicBoolean(false) private val activeGeneration = AtomicLong(initialGeneration.coerceAtLeast(0L)) @@ -49,6 +49,7 @@ internal class SabrSessionHolder( setPlayerTimeMs(ms) val generation = activeGeneration.incrementAndGet() clearReaderStateBefore(generation) + SabrPlaybackDiagnostics.clear(this) pendingRefetch.set(null) pendingForwardSeek.set(null) for (itag in activeItags) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt index a11a99fd..6851d04f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import dev.typetype.server.sabr.SabrAdapter import dev.typetype.server.sabr.YoutubeSabrInfo -internal object SabrSessionIdentity { +object SabrSessionIdentity { fun fresh(info: YoutubeSabrInfo): YoutubeSabrInfo { val cpn = YoutubeParsingHelper.generateContentPlaybackNonce() val url = requireNotNull(info.serverAbrStreamingUrl) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt index aaf6b375..bdf62111 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal data class SabrSessionKey( +data class SabrSessionKey( val videoId: String, val userId: String, val audioItag: Int, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt index f0af4faf..54743604 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory import java.time.Instant -internal object SabrSessionMediaFetcher { +object SabrSessionMediaFetcher { private val logger = LoggerFactory.getLogger(SabrSessionMediaFetcher::class.java) suspend fun fetch( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt similarity index 68% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt index db7e1bf3..75377809 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrSession -internal inline fun SabrSessionHolder.withPlayerContext(crossinline block: YoutubeSabrSession.() -> T): T { +inline fun SabrSessionHolder.withPlayerContext(crossinline block: YoutubeSabrSession.() -> T): T { val token = playerContextToken ?: return session.block() return TypetypeYoutubeSessionPoTokenProvider.withToken(token) { session.block() } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt index f68876c8..7d84aa85 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt @@ -3,11 +3,11 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment): Unit { +fun SabrSessionHolder.markServed(segment: SabrMediaSegment): Unit { markServed(segment, activeGeneration()) } -internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: Long): Unit { +fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: Long): Unit { if (!segment.header.isInitSegment) { observeMediaSegment(segment) val format = if (audioFormat.itag == segment.header.itag) audioFormat else videoFormat @@ -17,7 +17,7 @@ internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: } } -internal fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { +fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { if (!segment.header.isInitSegment) { observeMediaSegment(segment) val format = if (audioFormat.itag == segment.header.itag) audioFormat else videoFormat @@ -25,11 +25,11 @@ internal fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { } } -internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment): Unit { +fun SabrSessionHolder.markServed(segment: CachedSabrSegment): Unit { markServed(segment, activeGeneration()) } -internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation: Long): Unit { +fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation: Long): Unit { if (!segment.init) { val format = if (audioFormat.itag == segment.itag) audioFormat else videoFormat setReaderPosition(format, segment.startMs + segment.durationMs, generation) @@ -38,13 +38,13 @@ internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation } } -internal fun SabrSessionHolder.shouldSend(segment: SabrMediaSegment): Boolean { +fun SabrSessionHolder.shouldSend(segment: SabrMediaSegment): Boolean { if (segment.header.isInitSegment) return true val lastSequence = lastServedSequence(formatForItag(segment.header.itag)) ?: return true return segment.header.sequenceNumber > lastSequence } -internal fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { +fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { if (segment.init) return true val lastSequence = lastServedSequence(formatForItag(segment.itag)) ?: return true return segment.sequence > lastSequence @@ -53,7 +53,7 @@ internal fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { private fun SabrSessionHolder.formatForItag(itag: Int): YoutubeSabrFormat = if (audioFormat.itag == itag) audioFormat else videoFormat -internal fun bothFormatsKnown(holder: SabrSessionHolder): Boolean { +fun bothFormatsKnown(holder: SabrSessionHolder): Boolean { val state = holder.session.streamState return state.getEndSegment(holder.audioFormat) > 0L && state.getEndSegment(holder.videoFormat) > 0L && diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 052e08d3..7f3696a9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -7,7 +7,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import java.time.Instant -internal class SabrSessionPump( +class SabrSessionPump( private val segmentCache: SabrSegmentCache? = null, refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle? = { null }, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 8f541214..676260f5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -13,7 +13,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession import java.io.IOException -internal class SabrSessionPumpLoop( +class SabrSessionPumpLoop( private val unauthorizedRecovery: SabrUnauthorizedResponseRecovery = SabrUnauthorizedResponseRecovery { null }, private val runtimeFactory: () -> SabrPumpRuntime = { SabrPumpRuntime() }, private val onResolved: (SabrSessionHolder, SabrMediaSegment) -> Unit = { _, _ -> }, @@ -106,23 +106,15 @@ internal class SabrSessionPumpLoop( holder.nextSegmentDemand()?.let { request -> val demandIdentity = holder.segmentDemandIdentity(request) ?: return true val wasFutureLiveRequest = holder.isFutureLiveRequest(request) - if (!holder.beginInFlightSegmentDemand(request, demandIdentity, wasFutureLiveRequest)) return true - try { - SabrPumpLogger.start(holder, "demand", request) - runtime.beginDemand(demandIdentity) - val result = pumpDemand(holder, localization, request, runtime) - return SabrDemandAttemptFinisher.finish( - holder, - request, - demandIdentity, - result, - runtime, - wasFutureLiveRequest, - onResolved = { onResolved(holder, it) }, - ) - } finally { - holder.finishInFlightSegmentDemand(demandIdentity) - } + return runSabrDemandAttempt( + holder = holder, + request = request, + identity = demandIdentity, + wasFutureLiveRequest = wasFutureLiveRequest, + runtime = runtime, + pump = { pumpDemand(holder, localization, request, demandIdentity, runtime) }, + onResolved = { onResolved(holder, it) }, + ) } if (holder.livePlaybackSnapshot()?.active == true) { return pumpLiveReadAhead( @@ -174,6 +166,7 @@ internal class SabrSessionPumpLoop( holder: SabrSessionHolder, localization: Localization, request: SabrSegmentRequest, + identity: String, runtime: SabrPumpRuntime, ): YoutubeSabrSession.DemandResponseResult { val edgeMs = holder.session.streamState.getMinBufferedEndMs() @@ -187,6 +180,12 @@ internal class SabrSessionPumpLoop( pumpUntilCached(holder, localization, request, runtime) } } + if (runtime.demandNeedsTargetedShape(identity)) { + holder.setPlaybackState(SabrPlaybackState.REPOSITIONING) + return withTargetedRequestShape(holder, request, prepareSession = false) { + pumpUntilCached(holder, localization, request, runtime) + } + } holder.setPlaybackState(SabrPlaybackState.REQUESTING) return withLiveContinuationRequestShape(holder) { pumpUntilCached(holder, localization, request, runtime) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt index 83f64b5e..95ff491c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrSessionPurpose { +enum class SabrSessionPurpose { MANIFEST, PLAYBACK, DOWNLOAD, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt index 34ef8db2..beae5244 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt @@ -2,11 +2,13 @@ package dev.typetype.server.services import java.time.Instant import java.util.concurrent.ConcurrentHashMap +import org.slf4j.LoggerFactory -internal class SabrSessionRegistry { +class SabrSessionRegistry { private val sessions = ConcurrentHashMap() private val sessionsByToken = ConcurrentHashMap() private val mutationLock = Any() + private val logger = LoggerFactory.getLogger(SabrSessionRegistry::class.java) fun get(key: SabrSessionKey): SabrSessionHolder? { val holder = sessions[key] @@ -76,31 +78,47 @@ internal class SabrSessionRegistry { return holder } - fun ensureCapacity(maxSessions: Int) { + fun ensureCapacity( + maxSessions: Int, + activeCutoff: Instant = Instant.now().minus(SabrSessionStoreDefaults.idleEviction()), + ) { while (sessions.size >= maxSessions) { val oldest = sessions.entries + .asSequence() + .filterNot { isActiveLive(it.value, activeCutoff) } .minByOrNull { it.value.lastRequestAt } - ?: return - remove(oldest.key) + if (oldest == null) { + logCapacitySaturated(maxSessions) + return + } + evict(oldest.key, "capacity") } } - fun trimToCapacity(maxSessions: Int, protected: SabrSessionHolder) { + fun trimToCapacity( + maxSessions: Int, + protected: SabrSessionHolder, + activeCutoff: Instant = Instant.now().minus(SabrSessionStoreDefaults.idleEviction()), + ) { while (sessions.size > maxSessions) { val oldest = sessions.entries .asSequence() .filterNot { it.value === protected } + .filterNot { isActiveLive(it.value, activeCutoff) } .minByOrNull { it.value.lastRequestAt } - ?: return - remove(oldest.key) + if (oldest == null) { + logCapacitySaturated(maxSessions) + return + } + evict(oldest.key, "capacity") } } fun evictIdle(cutoff: Instant) { val stale = sessions.entries - .filter { it.value.lastRequestAt.isBefore(cutoff) } + .filter { it.value.lastRequestAt.isBefore(cutoff) && !isActiveLive(it.value, cutoff) } .map { it.key } - stale.forEach(::remove) + stale.forEach { evict(it, "idle") } } fun clear() { @@ -122,4 +140,40 @@ internal class SabrSessionRegistry { } holder?.releaseResources() } + + private fun isActiveLive(holder: SabrSessionHolder, cutoff: Instant): Boolean { + if (!holder.expectsLive() || holder.lastRequestAt.isBefore(cutoff)) return false + return holder.playbackState() !in setOf( + SabrPlaybackState.NETWORK_FAILED, + SabrPlaybackState.TERMINAL, + SabrPlaybackState.STOPPED, + ) + } + + private fun evict(key: SabrSessionKey, reason: String) { + val now = Instant.now() + val holder = synchronized(mutationLock) { + sessions.remove(key)?.also { sessionsByToken.remove(it.sessionToken, it) } + } ?: return + val ageMs = java.time.Duration.between(holder.lastRequestAt, now).toMillis().coerceAtLeast(0L) + holder.releaseResources() + logger.info( + "sabr_session event=evicted reason={} videoId={} ageMs={} registrySize={} liveSessions={} state={}", + reason, + holder.key.videoId, + ageMs, + sessions.size, + sessions.values.count { it.expectsLive() }, + holder.playbackState(), + ) + } + + private fun logCapacitySaturated(maxSessions: Int) { + logger.warn( + "sabr_session event=capacity_saturated maxSessions={} registrySize={} liveSessions={}", + maxSessions, + sessions.size, + sessions.values.count { it.expectsLive() }, + ) + } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt similarity index 75% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt index 30cf98c8..b8513a1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.releaseResources() { +fun SabrSessionHolder.releaseResources() { clearSegmentDemands() clearInFlightSegmentDemand() SabrPlaybackDiagnostics.clear(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt similarity index 59% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt index 77277499..6fddb14f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt @@ -2,4 +2,4 @@ package dev.typetype.server.services internal const val SABR_MAX_SEGMENT_MEMORY_BYTES = 64L * 1024L * 1024L -internal data class SabrReaderTrackKey(val generation: Long, val itag: Int) +data class SabrReaderTrackKey(val generation: Long, val itag: Int) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 56ae348a..702c3d1e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -16,7 +16,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration import java.time.Instant -internal class SabrSessionStore( +class SabrSessionStore( tokenServiceUrl: String, private val maxSessions: Int = SabrSessionStoreDefaults.maxSessions(), private val idleEviction: Duration = SabrSessionStoreDefaults.idleEviction(), @@ -41,7 +41,7 @@ internal class SabrSessionStore( init { runCatching { NewPipeInitializer.init() } } - internal fun getOrCreate( + fun getOrCreate( videoId: String, userId: String, info: YoutubeSabrInfo, @@ -71,7 +71,8 @@ internal class SabrSessionStore( isolatedSourceId, ) registry.getReusable(key)?.let { return it } - registry.ensureCapacity(maxSessions) + val activeCutoff = Instant.now().minus(idleEviction) + registry.ensureCapacity(maxSessions, activeCutoff) val holder = sessionFactory.create( key, info, @@ -84,47 +85,45 @@ internal class SabrSessionStore( ) val active = registry.put(key, holder) if (active !== holder) return active - registry.trimToCapacity(maxSessions, holder) + registry.trimToCapacity(maxSessions, holder, activeCutoff) if (startPump) startPump(holder) return holder } - internal fun startPump(holder: SabrSessionHolder) { + fun startPump(holder: SabrSessionHolder) { scope.launchSabrPump(pump, registry, holder, pumpLoopIntervalMs) } - - internal fun warmPlaybackAsync(holder: SabrSessionHolder) { + fun warmPlaybackAsync(holder: SabrSessionHolder) { if (holder.playerTimeMs() > 0L) return if (holder.playbackState() == SabrPlaybackState.REQUESTING || holder.playbackState() == SabrPlaybackState.REPOSITIONING) return scope.launch { fetchMediaAt(holder, holder.playerTimeMs()) } } - internal fun warmInitializationAsync(holder: SabrSessionHolder): Unit = + fun warmInitializationAsync(holder: SabrSessionHolder): Unit = SabrInitializationPolicy.warmFormats(holder.key.audioOnly, holder.audioFormat, holder.videoFormat).forEach { format -> scope.launch { fetchDirectInitialization(holder, format) } } - internal fun lookup(videoId: String, userId: String, audioItag: Int, videoItag: Int): SabrSessionHolder? = + fun lookup(videoId: String, userId: String, audioItag: Int, videoItag: Int): SabrSessionHolder? = registry.get(SabrSessionKey(videoId, userId, audioItag, null, videoItag, 0L)) - internal fun lookupByItag(videoId: String, userId: String, itag: Int): SabrSessionHolder? = + fun lookupByItag(videoId: String, userId: String, itag: Int): SabrSessionHolder? = registry.lookupByItag(videoId, userId, itag) - internal fun lookupByToken(videoId: String, token: String, itag: Int): SabrSessionHolder? = registry.lookupByToken(videoId, token, itag) - - internal fun lookupByToken(videoId: String, token: String): SabrSessionHolder? = registry.lookupByToken(videoId, token) - - internal fun lookupByToken(token: String): SabrSessionHolder? = registry.lookupByToken(token) + fun lookupByToken(videoId: String, token: String, itag: Int): SabrSessionHolder? = registry.lookupByToken(videoId, token, itag) + fun lookupByToken(videoId: String, token: String): SabrSessionHolder? = registry.lookupByToken(videoId, token) + fun lookupByToken(token: String): SabrSessionHolder? = registry.lookupByToken(token) - internal suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int = 8): Unit = pump.ensureWarmed(holder, maxPumps) + suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int = 8): Unit = pump.ensureWarmed(holder, maxPumps) - internal suspend fun preflightPlayback(holder: SabrSessionHolder, playerTimeMs: Long): Boolean = + suspend fun preflightPlayback(holder: SabrSessionHolder, playerTimeMs: Long): Boolean = warmer.preflight(this, holder, playerTimeMs) - internal suspend fun cachedMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = + suspend fun cachedMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = holder.mediaRequestsAt(playerTimeMs).map { segmentCache.get(holder, it) } .takeIf { cached -> cached.all { it != null } }?.filterNotNull() - internal suspend fun cachedSegment(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? { + suspend fun cachedSegment(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? { + holder.touch() holder.session.getCachedSegment(request)?.let { segmentCache.put(holder, it) holder.clearSegmentDemand(request) @@ -133,7 +132,7 @@ internal class SabrSessionStore( return segmentCache.get(holder, request)?.also { holder.clearSegmentDemand(request) } } - internal fun requestSegmentDemand( + fun requestSegmentDemand( holder: SabrSessionHolder, request: SabrSegmentRequest, generation: Long, @@ -142,36 +141,37 @@ internal class SabrSessionStore( startPump(holder) } - internal suspend fun fetchInfo( + suspend fun fetchInfo( videoId: String, startTimeMs: Long = 0L, cachedFirst: Boolean = false, isolatedPlayback: Boolean = false, ): SabrPreparedInfo? = infoFetcher.fetchInfo(videoId, startTimeMs, cachedFirst, isolatedPlayback) - - internal suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = + fun rememberExtractedInfoAsync(videoId: String, info: YoutubeSabrInfo): Unit { + scope.launch { infoFetcher.rememberExtractedInfo(videoId, info) } + } + suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) - internal suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) - internal suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) + suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) + suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) - internal suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = + suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = infoFetcher.recoverProtectedPlayback( holder.key.videoId, holder.playerContextToken?.visitorData ?: holder.info.visitorData, ) - internal fun refreshVideoPoToken(videoId: String): SabrTokenBundle? = + fun refreshVideoPoToken(videoId: String): SabrTokenBundle? = tokenClient.fetch(videoId, refreshVideo = true) - internal suspend fun fetchSegment( + suspend fun fetchSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, - ): SabrMediaSegment? = pump.fetchSegment(holder, request) - - internal suspend fun fetchMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = - pump.fetchMediaAt(holder, playerTimeMs) + ): SabrMediaSegment? = holder.also { it.touch() }.let { pump.fetchSegment(it, request) } - internal suspend fun fetchInitializationData( + suspend fun fetchMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = + holder.also { it.touch() }.let { pump.fetchMediaAt(it, playerTimeMs) } + suspend fun fetchInitializationData( holder: SabrSessionHolder, format: YoutubeSabrFormat, ): ByteArray? { @@ -207,7 +207,7 @@ internal class SabrSessionStore( registry.clear() } - internal fun release(holder: SabrSessionHolder): Unit = registry.remove(holder) + fun release(holder: SabrSessionHolder): Unit = registry.remove(holder) private suspend fun idleEvictionLoop() { while (true) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt index 553437b9..ed6291a2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.time.Duration -internal object SabrSessionStoreDefaults { +object SabrSessionStoreDefaults { const val INFO_TIMEOUT_MS = 8_000L const val INFO_ATTEMPTS = 1 const val INFO_RETRY_DELAY_MS = 250L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt index 3d4c0b1a..0b256238 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt @@ -3,10 +3,10 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long): List = +fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long): List = mediaRequestsAt(playerTimeMs, activeGeneration()) -internal fun SabrSessionHolder.repositionTargets( +fun SabrSessionHolder.repositionTargets( targets: List, playerTimeMs: Long, generation: Long, @@ -39,7 +39,7 @@ private fun SabrSessionHolder.adjacentWarmedLiveMediaStartMs( } } -internal fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long, generation: Long): List = buildList { +fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long, generation: Long): List = buildList { if (isVideoActive() && needsMediaAt(videoFormat, playerTimeMs, generation)) add(mediaRequestAt(videoFormat, playerTimeMs, generation)) if (isAudioActive() && needsMediaAt(audioFormat, playerTimeMs, generation)) add(mediaRequestAt(audioFormat, playerTimeMs, generation)) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt index 755d4dbd..0a6c52b5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.security.SecureRandom import java.util.Base64 -internal object SabrSessionTokenGenerator { +object SabrSessionTokenGenerator { private val random = SecureRandom() fun newToken(): String { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt index 47797f98..df118c68 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.prepareStartupBootstrapPump(): Boolean { +fun SabrSessionHolder.prepareStartupBootstrapPump(): Boolean { if (expectsLive() || !hasPendingSeek()) return false if (session.requestNumber == 0) { session.streamState.setPlayerTimeMs(0L) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt index 8465ed22..21d8a181 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession import org.slf4j.LoggerFactory -internal fun YoutubeSabrSession.fetchTargetedSegment( +fun YoutubeSabrSession.fetchTargetedSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, localization: Localization, @@ -24,7 +24,7 @@ internal fun YoutubeSabrSession.fetchTargetedSegment( } result.onFailure { error -> SabrPlaybackDiagnostics.record(holder, request, error.message) - logger.warn( + targetRequestLogger.warn( "sabr_target event=fetch_failed videoId={} itag={} seq={} init={} targetMs={} errorType={} error={}", holder.key.videoId, request.format.itag, @@ -62,6 +62,6 @@ private fun SabrSessionHolder.targetTimeInsideSegment(request: SabrSegmentReques return minOf(startMs + TARGET_SEGMENT_OFFSET_MS, nextStartMs - 1L) } -private val logger = LoggerFactory.getLogger("SabrTargetRequest") +private val targetRequestLogger = LoggerFactory.getLogger("SabrTargetRequest") private const val TARGET_SEGMENT_OFFSET_MS = 1_000L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt index 1fef82eb..0aeae523 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState import org.slf4j.LoggerFactory -internal inline fun withTargetedRequestShape( +inline fun withTargetedRequestShape( holder: SabrSessionHolder, request: SabrSegmentRequest, prepareSession: Boolean = true, @@ -40,19 +40,19 @@ internal inline fun withTargetedRequestShape( } } -private fun SabrSessionHolder.companionFormat(format: YoutubeSabrFormat): YoutubeSabrFormat = +@PublishedApi internal fun SabrSessionHolder.companionFormat(format: YoutubeSabrFormat): YoutubeSabrFormat = if (format.isAudio) videoFormat else audioFormat -private fun SabrSegmentRequest.trackMode(): Int = +@PublishedApi internal fun SabrSegmentRequest.trackMode(): Int = if (format.isAudio) YoutubeSabrStreamState.TRACK_MODE_AUDIO_ONLY else YoutubeSabrStreamState.TRACK_MODE_VIDEO_ONLY -private fun SabrSegmentRequest.targetRange(holder: SabrSessionHolder): SabrBufferedRange = +@PublishedApi internal fun SabrSegmentRequest.targetRange(holder: SabrSessionHolder): SabrBufferedRange = format.bufferedRange(holder, (sequenceNumber - 1).coerceAtLeast(0)) -private fun YoutubeSabrFormat.targetCompanionRange(holder: SabrSessionHolder, targetPlayerTimeMs: Long): SabrBufferedRange = +@PublishedApi internal fun YoutubeSabrFormat.targetCompanionRange(holder: SabrSessionHolder, targetPlayerTimeMs: Long): SabrBufferedRange = bufferedRange(holder, (holder.playbackStartSequence(this, targetPlayerTimeMs) - 1).coerceAtLeast(0)) -private fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, startMs: Long): Long { +@PublishedApi internal fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, startMs: Long): Long { val playerTimeMs = holder.playerTimeMs() val endMs = holder.playbackSegmentEndMs(format, sequenceNumber) if (endMs > startMs && playerTimeMs >= startMs && playerTimeMs < endMs) return playerTimeMs @@ -60,7 +60,7 @@ private fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, sta return startMs } -private fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedSequence: Int): SabrBufferedRange { +@PublishedApi internal fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedSequence: Int): SabrBufferedRange { val endMs = holder.playbackSegmentEndMs(this, bufferedSequence) val durationMs = endMs.takeIf { it > 0L } ?: 1L return SabrBufferedRange( @@ -75,8 +75,8 @@ private fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedS ) } -private const val SEEK_FORMAT_ORDER_MS = 1_000L +@PublishedApi internal const val SEEK_FORMAT_ORDER_MS = 1_000L private const val TARGET_SEGMENT_OFFSET_MS = 1_000L private const val TIMESCALE = 1_000 -private val logger = LoggerFactory.getLogger("SabrTargetRequestShape") +@PublishedApi internal val logger = LoggerFactory.getLogger("SabrTargetRequestShape") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index db9afa70..d61b5733 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrRecoverableException -internal class SabrUnauthorizedResponseRecovery( +class SabrUnauthorizedResponseRecovery( private val refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle?, ) { fun verify(holder: SabrSessionHolder): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt index a6041877..f82ce8bd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrWindowSegmentFetcher { +object SabrWindowSegmentFetcher { suspend fun fetch( holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt index ec844eb2..b29b7948 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrRecoverableException import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrStreamState -internal class TypetypeTokenSabrPoTokenProvider( +class TypetypeTokenSabrPoTokenProvider( private val tokenClient: TypetypeTokenSabrTokenClient, private val initialToken: SabrTokenBundle? = null, ) : SabrPoTokenProvider { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt new file mode 100644 index 00000000..8839201d --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt @@ -0,0 +1,77 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AudioStreamItem +import dev.typetype.server.models.VideoItem +import dev.typetype.server.models.VideoStreamItem + +fun playbackTestVideoStream( + url: String = "https://example.googlevideo.com/videoplayback?id=1", + codec: String? = "avc1.42c01e", + format: String = "MPEG_4", + resolution: String = "720p", + bitrate: Int = 2000000, + width: Int = 1280, + height: Int = 720, + indexStart: Long = 221, + indexEnd: Long = 893, + initStart: Long = 0, + initEnd: Long = 220, + itag: Int = 137, +): VideoStreamItem = VideoStreamItem( + url = url, + mimeType = "video/mp4", + format = format, + resolution = resolution, + bitrate = bitrate, + codec = codec, + isVideoOnly = true, + itag = itag, + width = width, + height = height, + fps = 30, + contentLength = 10_000_000, + initStart = initStart, + initEnd = initEnd, + indexStart = indexStart, + indexEnd = indexEnd, +) + +fun playbackTestAudioStream( + url: String = "https://example.googlevideo.com/audioplayback?id=1", + codec: String? = "mp4a.40.2", + format: String = "M4A", + bitrate: Int = 128, + indexStart: Long = 100, + indexEnd: Long = 500, + initStart: Long = 0, + initEnd: Long = 99, + audioTrackId: String? = null, + audioTrackName: String? = null, + audioLocale: String? = null, + isOriginal: Boolean = false, + itag: Int = 140, + deliveryMethod: String = "progressive", + manifestUrl: String? = null, + sabrSessionUrl: String? = null, + contentLength: Long = 5_000_000, +): AudioStreamItem = AudioStreamItem( + url = url, + mimeType = "audio/mp4", + format = format, + bitrate = bitrate, + codec = codec, + quality = "medium", + itag = itag, + contentLength = contentLength, + initStart = initStart, + initEnd = initEnd, + indexStart = indexStart, + indexEnd = indexEnd, + audioTrackId = audioTrackId, + audioTrackName = audioTrackName, + audioLocale = audioLocale, + isOriginal = isOriginal, + deliveryMethod = deliveryMethod, + manifestUrl = manifestUrl, + sabrSessionUrl = sabrSessionUrl, +) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt new file mode 100644 index 00000000..cd091f31 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class SabrBootstrapStreamLiveHlsTest { + @Test + fun usesLiveHlsExtractionWithoutPreparingSabr() = runTest { + val sessionStore = mockk() + val tokenClient = mockk() + val liveHlsService = mockk() + val liveMetadata = mockk() + every { liveMetadata.isLive } returns true + val expected = ExtractionResult.Success(mockk()) + coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns liveMetadata + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = SabrBootstrapStreamService(sessionStore, tokenClient, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + coVerify(exactly = 0) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt index feea11e9..a2a04d39 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt @@ -20,7 +20,7 @@ class SabrBootstrapStreamServiceTest { val session = tokenSession(prepared.info, tokenBundle()) coEvery { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } returns Unit coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns session - val service = SabrBootstrapStreamService(sessionStore, tokenClient) + val service = SabrBootstrapStreamService(sessionStore, tokenClient, mockk()) val result = service.getStreamInfo(YOUTUBE_URL) @@ -40,7 +40,7 @@ class SabrBootstrapStreamServiceTest { val prepared = preparedInfo() coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns null coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession(prepared.info) - val service = SabrBootstrapStreamService(sessionStore, tokenClient) + val service = SabrBootstrapStreamService(sessionStore, tokenClient, mockk()) val result = service.getStreamInfo(YOUTUBE_URL) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt index 880dcb81..f131a4fb 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt @@ -138,6 +138,7 @@ class SabrDemandFailurePumpTest { val result = mockk() every { result.segmentCount } returns 7 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true return result } @@ -145,6 +146,7 @@ class SabrDemandFailurePumpTest { val result = mockk() every { result.segmentCount } returns 0 every { result.targetTrackSegmentCount } returns 0 + every { result.requestPerformed } returns true return result } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt similarity index 55% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt index 0541b1e8..28556fb2 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt @@ -9,6 +9,7 @@ import kotlinx.coroutines.test.runCurrent import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.SabrSegmentRequest @@ -100,6 +101,98 @@ class SabrDemandWatchdogBackoffTest { } } + @Test + fun `future live demand waits past deadline in the same generation`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val job = launch { + SabrDemandWatchdog( + clock = { testScheduler.currentTime }, + intervalMs = 100L, + ).monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS) + runCurrent() + advanceTimeBy(LIVE_EDGE_POLL_MS) + runCurrent() + + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + job.cancel() + } + } + + @Test + fun `stalled live attempt is cancelled and requeued`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val identity = requireNotNull(holder.segmentDemandIdentity(request)) + assertTrue(holder.beginInFlightSegmentDemand(request, identity, futureLiveRequest = true)) + var cancelled = false + holder.registerInFlightDemandCancellation(identity) { cancelled = true } + val demand = requireNotNull(holder.inFlightSegmentDemand()) + + assertTrue( + SabrDemandAttemptFinisher.interruptStalledInFlightDemand( + holder, + demand, + nowMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + expectedDelayMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason = "future_live_not_published", + ), + ) + + assertTrue(cancelled) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertNotEquals(identity, holder.segmentDemandIdentity(request)) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + } + } + + @Test + fun `recoverable late live demand waits past deadline`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.liveHeadSequenceNumber } returns 52L + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.liveHeadSequenceNumber } returns 52L + every { holder.session.streamState.liveHeadTimeMs } returns 1_000L + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 52 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val job = launch { + SabrDemandWatchdog( + clock = { testScheduler.currentTime }, + intervalMs = 100L, + ).monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS + LIVE_EDGE_POLL_MS) + runCurrent() + + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(null, holder.terminalFailure()) + job.cancel() + } + } + private fun watchdog(clock: () -> Long): SabrDemandWatchdog = SabrDemandWatchdog( clock = clock, intervalMs = 100L, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt similarity index 95% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt index 4d183d57..30f75724 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt @@ -84,16 +84,15 @@ class SabrDemandWatchdogLifecycleTest { } @Test - fun `future live demand expires as recoverable`() = runTest { + fun `future live demand waits without terminal failure`() = runTest { withTracker { holder -> val request = SabrSegmentRequest.media(holder.videoFormat, 50) every { holder.session.isLive } returns true every { holder.session.streamState.isLive } returns true every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 holder.requestSegmentDemand(request, registeredAtMs = 0L) - var expired = false val job = launch { - expired = SabrDemandWatchdog( + SabrDemandWatchdog( clock = { testScheduler.currentTime }, intervalMs = 100L, ).monitor({ true }, holder) @@ -105,12 +104,12 @@ class SabrDemandWatchdogLifecycleTest { advanceTimeBy(LIVE_EDGE_POLL_MS) runCurrent() - assertTrue(job.isCompleted) - assertTrue(expired) - assertEquals( - "$SABR_RECOVERABLE_FAILURE_PREFIX SABR demand stalled for 299:50", - holder.terminalFailure(), - ) + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + job.cancel() } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt new file mode 100644 index 00000000..2ce89bfa --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.services + +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.YoutubeSabrFormat + +class SabrInitializationPolicyTest { + @Test + fun `audio only warmup excludes video initialization`() { + val audio = mockk() + val video = mockk() + + assertEquals(listOf(audio), SabrInitializationPolicy.warmFormats(true, audio, video)) + assertEquals(listOf(video, audio), SabrInitializationPolicy.warmFormats(false, audio, video)) + } + + @Test + fun `audio only fetch never waits for video initialization`() { + assertFalse(SabrInitializationPolicy.requiresVideoFirst(true, true, 60_000L)) + assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, true, 0L)) + assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, false, 60_000L)) + assertTrue(SabrInitializationPolicy.requiresVideoFirst(false, true, 60_000L)) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt similarity index 91% rename from src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt index e1b3d56c..7134c8ad 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger class SabrLivePlaybackSessionServiceTest { @Test - fun `live start uses the warmed media pair when it is closer to the head`() { + fun `live start uses the target when available media is behind it`() { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) @@ -36,11 +36,11 @@ class SabrLivePlaybackSessionServiceTest { holder.observeMediaSegment(mediaSegment(audio.itag, 995_002L)) holder.observeMediaSegment(mediaSegment(video.itag, 995_000L)) - assertEquals(995_002L, holder.resolvePlaybackStartMs(0L)) + assertEquals(1_003_000L, holder.resolvePlaybackStartMs(0L)) } @Test - fun `live prepare warms metadata and starts behind the live head`() = runTest { + fun `live prepare starts near the live head without waiting for warmup`() = runTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val info = mockk() @@ -60,10 +60,10 @@ class SabrLivePlaybackSessionServiceTest { every { state.getBufferedEndMs(audio) } returns 1_000_000L every { state.getBufferedEndMs(video) } returns 1_002_000L every { state.getMinBufferedEndMs() } returns 1_000_000L - every { state.getSegmentNumberAtOrAfterTimeMs(video, 985_000L) } returns 198 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 985_000L) } returns 99 - every { state.getSegmentStartMs(video, 198) } returns 990_000L - every { state.getSegmentStartMs(audio, 99) } returns 990_000L + every { state.getSegmentNumberAtOrAfterTimeMs(video, 1_003_000L) } returns 200 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 1_003_000L) } returns 100 + every { state.getSegmentStartMs(video, 200) } returns 1_002_000L + every { state.getSegmentStartMs(audio, 100) } returns 1_000_000L val store = mockk() every { store.getOrCreate( @@ -80,17 +80,16 @@ class SabrLivePlaybackSessionServiceTest { 0L, ) } returns holder - coEvery { store.ensureWarmed(holder, 8) } returns Unit every { store.startPump(holder) } returns Unit val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) - assertEquals(985_000L, result.startTimeMs) - assertEquals(985_000L, holder.playerTimeMs()) + assertEquals(1_003_000L, result.startTimeMs) + assertEquals(1_003_000L, holder.playerTimeMs()) assertTrue(holder.expectsLive()) verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 1) { state.setWriteTopLevelPlayerTimeMs(false) } - coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } coVerify(exactly = 0) { store.fetchInitializationData(any(), any()) } verify(exactly = 1) { store.startPump(holder) } } @@ -161,7 +160,7 @@ class SabrLivePlaybackSessionServiceTest { } @Test - fun `live format change starts the replacement session from warmed track boundaries`() = runTest { + fun `live format change starts the replacement session without synchronous warmup`() = runTest { val audio = format(140, isAudio = true) val source = holder(audio, format(137, isAudio = false), initialGeneration = 4L) val video = format(248, isAudio = false) @@ -201,7 +200,6 @@ class SabrLivePlaybackSessionServiceTest { 5L, ) } returns replacement - coEvery { store.ensureWarmed(replacement, 8) } returns Unit every { store.startPump(replacement) } returns Unit val result = SabrPlaybackSessionService(store).seek(source, prepared, audio, video, 995_000L) @@ -217,7 +215,7 @@ class SabrLivePlaybackSessionServiceTest { assertFalse(replacement.mediaRequestsAt(995_000L).any { it.format.itag == video.itag }) verify(exactly = 0) { replacementState.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 0) { replacementState.setWriteTopLevelPlayerTimeMs(false) } - coVerify(exactly = 1) { store.ensureWarmed(replacement, 8) } + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } verify(exactly = 1) { store.startPump(replacement) } } diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt new file mode 100644 index 00000000..b8ad0f09 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import java.time.Instant + +class SabrLivePumpStepTest { + @Test + fun `live pump fills below read ahead cushion`() = runTest { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 101_000L) + var pumps = 0 + + val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + + assertTrue(pumps == 1) + assertFalse(immediate) + } + + @Test + fun `live pump idles above read ahead cushion`() = runTest { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 130_000L) + var pumps = 0 + + val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + + assertTrue(pumps == 0) + assertFalse(immediate) + } + + @Test + fun `live pump waits when the next demand is not published yet`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 101_000L) + every { fixture.holder.session.isLive } returns true + every { fixture.holder.session.streamState.isLive } returns true + every { fixture.holder.session.streamState.getMaxSegment(fixture.holder.videoFormat) } returns 100 + fixture.holder.requestSegmentDemand(SabrSegmentRequest.media(fixture.holder.videoFormat, 101)) + + pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { 0 }) { _, _ -> } + + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + + private fun fixture(playerTimeMs: Long, observedEndMs: Long): Fixture { + val audio = format(140, true) + val video = format(299, false) + val session = mockk(relaxed = true) + every { session.streamState } returns mockk(relaxed = true) + every { session.getCachedSegment(any()) } returns null + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.setPlayerTimeMs(playerTimeMs) + holder.observeMediaSegment(segment(audio.itag, observedEndMs - SEGMENT_DURATION_MS)) + holder.observeMediaSegment(segment(video.itag, observedEndMs - SEGMENT_DURATION_MS)) + return Fixture(holder) + } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { isAudio } returns audio + every { bitrate } returns if (audio) 128_000 else 2_000_000 + every { lastModified } returns 1L + every { xtags } returns null + } + + private fun segment(itag: Int, startMs: Long): SabrMediaSegment { + val header = mockk { + every { this@mockk.itag } returns itag + every { sequenceNumber } returns 100 + every { this@mockk.startMs } returns startMs + every { durationMs } returns SEGMENT_DURATION_MS + every { isInitSegment } returns false + } + return mockk { every { this@mockk.header } returns header } + } + + private data class Fixture(val holder: SabrSessionHolder) + + private companion object { + const val SEGMENT_DURATION_MS = 2_000L + } +} diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt new file mode 100644 index 00000000..e78a6b11 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class SabrMemorySegmentCacheTest { + @Test + fun `eviction behind reader tail keeps the boundary and newer media`() { + val cache = SabrMemorySegmentCache(maxBytes = 1_000) + cache.put("before-tail", segment(startMs = 0L, durationMs = 5L)) + cache.put("at-tail", segment(startMs = 5L, durationMs = 5L)) + cache.put("ahead", segment(startMs = 10L, durationMs = 5L)) + + cache.evictBefore(6L) + + assertNull(cache.get("before-tail")) + assertNotNull(cache.get("at-tail")) + assertNotNull(cache.get("ahead")) + } + + @Test + fun `least recently used media is removed when the byte limit is reached`() { + val cache = SabrMemorySegmentCache(maxBytes = 10L) + cache.put("oldest", segment(bytes = ByteArray(6))) + cache.put("newest", segment(bytes = ByteArray(6))) + + assertNull(cache.get("oldest")) + assertNotNull(cache.get("newest")) + } + + private fun segment( + startMs: Long = 0L, + durationMs: Long = 1L, + bytes: ByteArray = ByteArray(1), + ): CachedSabrSegment = CachedSabrSegment( + itag = 137, + sequence = 1, + init = false, + startMs = startMs, + durationMs = durationMs, + mimeType = "video/webm", + bytes = bytes, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt similarity index 93% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt index 53ae9e5b..a48e6860 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt @@ -18,7 +18,7 @@ import java.time.Instant class SabrLiveSessionWarmupTest { @Test - fun `warmup continues when the first media pair is too close to the live head`() = runTest { + fun `warmup accepts the first media pair at the low latency live edge`() = runTest { val audio = format(140, audio = true, "audio/mp4") val video = format(299, audio = false, "video/mp4") val streamState = mockk(relaxed = true) @@ -49,10 +49,10 @@ class SabrLiveSessionWarmupTest { SabrSessionPump(SabrSegmentCache()).ensureWarmed(holder, maxPumps = 8) - assertEquals(2, pumps) - assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(audio)) - assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(video)) - assertEquals(1_980_000L, holder.resolvePlaybackStartMs(0L)) + assertEquals(1, pumps) + assertEquals(2_000_000L, holder.earliestObservedMediaStartMs(audio)) + assertEquals(2_000_000L, holder.earliestObservedMediaStartMs(video)) + assertEquals(2_000_000L, holder.resolvePlaybackStartMs(0L)) } @Test @@ -120,11 +120,11 @@ class SabrLiveSessionWarmupTest { assertEquals(TARGET_SEQUENCE, holder.observedMediaSegment(video)?.header?.sequenceNumber) assertArrayEquals(audioInit, holder.liveInitialization(audio)) assertArrayEquals(videoInit, holder.liveInitialization(video)) - assertEquals(liveHeadTimeMs(pumps) - 20_000L, holder.resolvePlaybackStartMs(0L)) + assertEquals(liveHeadTimeMs(pumps) - 2_000L, holder.resolvePlaybackStartMs(0L)) val targetedRanges = rangeOverrides.filterNotNull().map { ranges -> ranges.map(SabrBufferedRange::summarize) } val expectedRanges = listOf( - "itag=140:seq=1-3319:time=0+6640000:timescale=1000", - "itag=299:seq=1-3319:time=0+6640000:timescale=1000", + "itag=140:seq=1-3328:time=0+6658000:timescale=1000", + "itag=299:seq=1-3328:time=0+6658000:timescale=1000", ) assertEquals(2, targetedRanges.size) assertEquals(expectedRanges, targetedRanges.first()) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt similarity index 65% rename from src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt index 84a84099..0eab1b8e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt @@ -33,6 +33,7 @@ class SabrMissingDemandRecoveryTest { every { streamState.getSegmentStartMs(audio, 44) } returns 429_337L every { result.segmentCount } returns 2 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true every { session.pumpOnceStreamingForDemand(any(), request) } returns result val holder = holder(session, audio, video) holder.requestSegmentDemand(request) @@ -48,6 +49,38 @@ class SabrMissingDemandRecoveryTest { } } + @Test + fun `companion-only response readvertises the missing target track`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val request = SabrSegmentRequest.media(audio, 44) + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + val result = mockk() + every { session.streamState } returns streamState + every { session.getCachedSegment(any()) } returns null + every { session.requestNumber } returns 25 + every { streamState.getMinBufferedEndMs() } returns 416_100L + every { streamState.getSegmentStartMs(audio, 44) } returns 429_337L + every { result.segmentCount } returns 1 + every { result.targetTrackSegmentCount } returns 0 + every { result.requestPerformed } returns true + every { session.pumpOnceStreamingForDemand(any(), request) } returns result + val holder = holder(session, audio, video) + holder.requestSegmentDemand(request) + var rounds = 0 + + SabrSessionPump().pumpLoop({ rounds++ == 0 }, holder, intervalMs = 100L) + + verify(exactly = 1) { session.prepareForMissingSegment(request) } + assertEquals("140:44", holder.pendingSegmentDemandSummary()) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + private fun holder( session: YoutubeSabrSession, audio: YoutubeSabrFormat, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt similarity index 97% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt index 83e37887..0987a39a 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt @@ -33,6 +33,7 @@ class SabrPlaybackSessionServiceTest { val prepared = SabrPreparedInfo(info, token()) val holder = holder(audio, video) val initializationFetches = AtomicInteger() + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(video, 88_168L) } answers { @@ -169,6 +170,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(any(), 90_000L) } returns 9 @@ -196,6 +198,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(video, 120_000L) } returns 24 @@ -220,6 +223,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.setSelectVideoFormatBeforeAudio(false) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(audio, 299L) } returns 1 @@ -248,6 +252,7 @@ class SabrPlaybackSessionServiceTest { every { session.awaitReadableSegment(any(), any()) } returns null every { session.isBeyondEnd(any()) } returns false every { session.prepareForInitialization(any()) } returns Unit + every { state.getEndSegment(any()) } returns 0L every { state.setActiveTrackTypes(any(), any()) } returns Unit every { state.setSelectVideoFormatBeforeAudio(any()) } returns Unit return SabrSessionHolder( diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt new file mode 100644 index 00000000..0552eddf --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Test + +class SabrProtectedContextRecoveryTest { + @Test + fun `refreshes the active rejected context`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns token("rejected") + every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + } + + @Test + fun `keeps a context already refreshed by another recovery`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 0) { tokenClient.fetch("video", forceRefresh = true) } + } + + @Test + fun `refreshes when the current context cannot be read`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns null + every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + } + + private fun token(visitorData: String): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "player-$visitorData", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = visitorData, + videoBoundPoToken = "video-$visitorData", + videoBoundPoTokenBytes = byteArrayOf(2), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt similarity index 88% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt index 9f021795..0a32c144 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt @@ -19,10 +19,38 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) class SabrPumpLauncherTest { + @Test + fun `active live playback keeps read ahead to the startup target`() { + val audio = format(140, isAudio = true) + val video = format(136, isAudio = false) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { state.isLive } returns true + every { state.isPostLiveDvr } returns false + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.setPlayerTimeMs(100_000L) + var now = 0L + val runtime = SabrPumpRuntime { now } + now = 30_000L + + assertEquals(2_000L, runtime.targetReadaheadCushionMs(holder)) + } + @Test fun `watchdog ignores companion progress for missing demand`() = runTest { SabrSegmentDemandTracker.clearAll() @@ -226,6 +254,7 @@ class SabrPumpLauncherTest { val format = mockk() every { format.itag } returns itag every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 return format } } diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt new file mode 100644 index 00000000..e8045bea --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SabrPumpRuntimeTargetedShapeTest { + @Test + fun `unresolved demand switches to targeted requests after readvertisement`() { + val runtime = SabrPumpRuntime() + + assertFalse(runtime.demandNeedsTargetedShape("demand")) + runtime.beginDemand("demand") + assertFalse(runtime.demandNeedsTargetedShape("demand")) + + assertEquals( + SabrDemandRecoveryAction.READVERTISE_TRACK, + runtime.demandRecoveryAction("demand", requestPerformed = true, resolved = false), + ) + assertTrue(runtime.demandNeedsTargetedShape("demand")) + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("demand", requestPerformed = true, resolved = false), + ) + assertTrue(runtime.demandNeedsTargetedShape("demand")) + + runtime.finishDemand("demand") + assertFalse(runtime.demandNeedsTargetedShape("demand")) + } + + @Test + fun `a new demand starts with continuation requests`() { + val runtime = SabrPumpRuntime() + + runtime.beginDemand("first") + runtime.demandRecoveryAction("first", requestPerformed = true, resolved = false) + runtime.beginDemand("second") + + assertFalse(runtime.demandNeedsTargetedShape("first")) + assertFalse(runtime.demandNeedsTargetedShape("second")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt new file mode 100644 index 00000000..a477f8dc --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class SabrRecoverablePumpFailureTest { + @Test + fun `recoverable media failure requests a fresh session`() = runTest { + val holder = pumpFailure("Unexpected EOF") + + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertTrue(holder.terminalFailure().orEmpty().startsWith(SABR_RECOVERABLE_FAILURE_PREFIX)) + } + + @Test + fun `local spool failure remains terminal without fresh session recovery`() = runTest { + val holder = pumpFailure("Could not write SABR spool file") + + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertEquals("Could not write SABR spool file", holder.terminalFailure()) + } + + private suspend fun pumpFailure(message: String): SabrSessionHolder { + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + every { session.streamState } returns streamState + every { session.requestNumber } returns 1 + every { session.isComplete } returns false + every { session.cachedBytes } returns 0L + every { streamState.nextRequestPolicy } returns null + every { session.pumpOnceStreaming(any()) } throws SabrRecoverableException(message) + val holder = holder(session) + + SabrSessionPump().pumpLoop({ true }, holder, intervalMs = 0L) + return holder + } + + private fun holder(session: YoutubeSabrSession): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = format(140, true), + videoFormat = format(137, false), + sessionToken = "session-token", + key = SabrSessionKey("video", "user", 140, null, 137, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + return format + } +} diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt new file mode 100644 index 00000000..d4198746 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrSeekDiagnosticsTest { + @Test + fun `seek generation clears diagnostics from the previous position`() { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, true) + val video = format(137, false) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(any(), 120_000L) } returns 1 + every { state.getMinBufferedEndMs() } returns 0L + val holder = holder(session, audio, video) + SabrPlaybackDiagnostics.record(holder, SabrSegmentRequest.media(audio, 1), "status=2 policy=true") + + SabrPlaybackSessionService(mockk(relaxed = true)).seekExisting(holder, 120_000L) + + assertNull(SabrPlaybackDiagnostics.blocker(holder)) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.audioTrackId } returns null + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + every { format.lastModified } returns 0L + every { format.xtags } returns "" + return format + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt similarity index 86% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt index d718d8df..e990af50 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt @@ -6,6 +6,7 @@ import io.mockk.verify import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.SabrMediaHeader import dev.typetype.server.sabr.SabrMediaSegment @@ -17,6 +18,20 @@ import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSegmentCacheTest { + @Test + fun `segment demand refreshes session idle timestamp`() { + val audio = format(140, isAudio = true) + val video = format(137, isAudio = false) + val holder = holder(audio, video) + val stale = holder.lastRequestAt + val request = SabrSegmentRequest.media(audio, 1) + + holder.requestSegmentDemand(request) + + assertTrue(holder.lastRequestAt.isAfter(stale)) + SabrSegmentDemandTracker.clearAll() + } + @Test fun `vod cache observes media without copying segment bytes`() { val segmentCache = SabrSegmentCache() @@ -57,6 +72,8 @@ class SabrSegmentCacheTest { val session = mockk() val state = mockk() every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.isBeyondEnd(any()) } returns false every { state.setActiveTrackTypes(any(), any()) } returns Unit return SabrSessionHolder( session = session, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt similarity index 99% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt index d6e708ec..8394c93f 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt @@ -270,6 +270,7 @@ class SabrSessionPumpLoopTest { val result = mockk() every { result.segmentCount } returns segmentCount every { result.targetTrackSegmentCount } returns targetTrackSegmentCount + every { result.requestPerformed } returns true return result } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt index e1209e0f..503e7361 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt @@ -145,6 +145,7 @@ class SabrSessionPumpTest { every { streamState.setActiveTrackTypes(true, true) } returns Unit every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 60_862L) } returns 7 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 50_876L, 9_985L, sequence = 7)) @@ -162,6 +163,7 @@ class SabrSessionPumpTest { val session = mockk() val streamState = mockk() every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 59_907L, 9_985L, sequence = 7)) @@ -180,6 +182,7 @@ class SabrSessionPumpTest { every { session.streamState } returns streamState every { streamState.setActiveTrackTypes(true, true) } returns Unit every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 59_907L, 9_985L, sequence = 7)) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt similarity index 84% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt index 7bc34a9b..aae658f8 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt @@ -31,6 +31,20 @@ class SabrSessionRegistryTest { assertSame(idle, registry.get(idleKey)) } + @Test + fun `capacity eviction keeps a recently active live session`() { + val registry = SabrSessionRegistry() + val live = holder("live", Instant.now()).also { it.markExpectedLive() } + val idle = holder("idle", Instant.EPOCH) + registry.put(live.key, live) + registry.put(idle.key, idle) + + registry.ensureCapacity(2, Instant.now().minusSeconds(60)) + + assertSame(live, registry.get(live.key)) + assertNull(registry.get(idle.key)) + } + @Test fun `idle eviction removes stale sessions`() { val registry = SabrSessionRegistry() @@ -54,6 +68,22 @@ class SabrSessionRegistryTest { assertSame(holder, registry.lookupByToken("token-playback")) } + @Test + fun `active live playback requests refresh idle eviction timestamp`() { + val registry = SabrSessionRegistry() + val holder = holder("live", Instant.EPOCH) + holder.markExpectedLive() + registry.put(holder.key, holder) + + assertSame(holder, registry.lookupByToken("live", holder.sessionToken)) + assertSame(holder, registry.lookupByToken("live", holder.sessionToken, holder.audioFormat.itag)) + assertSame(holder, registry.lookupByItag("live", "user", holder.audioFormat.itag)) + + registry.evictIdle(Instant.now().minusSeconds(1)) + + assertSame(holder, registry.lookupByToken(holder.sessionToken)) + } + @Test fun `healthy session remains reusable`() { val registry = SabrSessionRegistry() diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt new file mode 100644 index 00000000..a8b9a821 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt @@ -0,0 +1,189 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrSessionTimeRequestsTest { + @Test + fun `playback start sequence is clamped to the known final segment`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 91 + every { state.getEndSegment(audio) } returns 90L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(90, holder.playbackStartSequence(audio, 88_168L)) + } + + @Test + fun `playback start sequence stays valid without an indexed end`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 33 + every { state.getEndSegment(audio) } returns 0L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(33, holder.playbackStartSequence(audio, 88_168L)) + } + + @Test + fun `mediaRequestsAt returns active audio and video requests for player time`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 321_601L) } returns 33 + val holder = holder(session, audio, video) + + val requests = holder.mediaRequestsAt(321_601L) + + assertEquals(listOf(137, 140), requests.map { it.format.itag }) + assertEquals(listOf(64, 33), requests.map { it.sequenceNumber }) + } + + @Test + fun `mediaRequestsAt uses mapped video and audio sequences`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 247, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 340_000L) } returns 64 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 340_000L) } returns 35 + val holder = holder(session, audio, video) + + val requests = holder.mediaRequestsAt(340_000L) + + assertEquals(listOf(247, 140), requests.map { it.format.itag }) + assertEquals(listOf(64, 35), requests.map { it.sequenceNumber }) + } + + @Test + fun `mediaRequestsAt excludes inactive tracks`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 + val holder = holder(session, audio, video) + holder.setActiveTracks(videoActive = true, audioActive = false) + + val requests = holder.mediaRequestsAt(321_601L) + + assertEquals(listOf(137), requests.map { it.format.itag }) + assertEquals(listOf(64), requests.map { it.sequenceNumber }) + } + + @Test + fun `reposition keeps a non adjacent live boundary request pending`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentStartMs(audio, 100) } returns 995_000L + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 102, startMs = 995_010L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 102 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 995_000L, generation = 0L) + + assertEquals(listOf(request), missing) + } + + @Test + fun `reposition advances to the next warmed live segment from inside a boundary`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 101, startMs = 1_000_000L, durationMs = 5_000L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 101 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 996_200L, generation = 0L) + + assertEquals(emptyList(), missing) + assertEquals(1_000_000L, holder.readerPosition(audio)) + } + + private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.isVideo } returns !isAudio + return format + } + + private fun mediaSegment( + itag: Int, + sequence: Int, + startMs: Long, + durationMs: Long = 0L, + ): SabrMediaSegment { + val header = mockk(relaxed = true) + every { header.itag } returns itag + every { header.sequenceNumber } returns sequence + every { header.startMs } returns startMs + every { header.durationMs } returns durationMs + every { header.isInitSegment } returns false + return mockk { every { this@mockk.header } returns header } + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session, + mockk(), + audio, + video, + "session", + SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + Instant.now(), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt similarity index 99% rename from src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt index cce09dc0..b4e4a134 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt @@ -77,6 +77,7 @@ class SabrTransientDemandFailureTest { every { streamState.getMinBufferedEndMs() } returns 379_233L every { result.segmentCount } returns 1 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true every { session.pumpOnceStreamingForDemand(any(), request) } answers { attempts++ if (attempts == 1) throw IOException("timeout") @@ -236,6 +237,7 @@ class SabrTransientDemandFailureTest { val result = mockk() every { result.segmentCount } returns segmentCount every { result.targetTrackSegmentCount } returns targetTrackSegmentCount + every { result.requestPerformed } returns true return result } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt similarity index 95% rename from src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt index d7e90ce1..400bb8cd 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt @@ -59,7 +59,6 @@ class SabrTransitioningLivePlaybackTest { } returns holder coEvery { store.fetchInitializationData(holder, video) } returns null coEvery { store.fetchInitializationData(holder, audio) } returns null - coEvery { store.ensureWarmed(holder, 8) } returns Unit every { store.startPump(holder) } returns Unit val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) @@ -67,8 +66,8 @@ class SabrTransitioningLivePlaybackTest { assertTrue(holder.expectsLive()) assertTrue(result.startTimeMs > 0L) assertNull(holder.terminalFailure()) - assertEquals(11_050_200L, result.startTimeMs) - coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + assertEquals(11_068_200L, result.startTimeMs) + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 1) { store.startPump(holder) } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt diff --git a/server-portability/build.gradle.kts b/server-portability/build.gradle.kts new file mode 100644 index 00000000..c46ef7c2 --- /dev/null +++ b/server-portability/build.gradle.kts @@ -0,0 +1,33 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-utils-jvm:3.5.2") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("com.fasterxml.jackson.core:jackson-databind:2.22.2") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt index ecc999e6..046058f4 100644 --- a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import kotlinx.serialization.json.jsonObject -internal object FlowPortabilityReader { +object FlowPortabilityReader { fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = sink.readFlowArray(PortabilityCategory.SUBSCRIPTIONS, parser, token) { item -> val id = item.string("channelId") diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt index f087b3bf..2f2cc4ea 100644 --- a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object FlowPortabilityWriter { +object FlowPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt index 29fba1d8..3c2b6bd6 100644 --- a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import java.util.zip.ZipFile -internal object GrayjayPortabilityReader { +object GrayjayPortabilityReader { fun read(zip: ZipFile, sink: PortabilityRecordSink) { readStrings(zip, "stores/subscriptions") { readSubscription(it, sink) } readStrings(zip, "stores/subscription_groups") { readGroup(it, sink) } diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt index da23f917..621587a3 100644 --- a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt @@ -6,7 +6,7 @@ import java.util.UUID import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -internal object GrayjayPortabilityWriter { +object GrayjayPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { ZipOutputStream(output.buffered()).use { zip -> writeObjectEntry(zip, "exportInfo", mapOf("version" to "1")) diff --git a/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt index afb7c4f2..c16a20bc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt @@ -5,7 +5,7 @@ import com.fasterxml.jackson.core.JsonToken import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.jsonObject -internal object LibreTubePortabilityReader { +object LibreTubePortabilityReader { fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { sink.readArray(PortabilityCategory.HISTORY, parser, token) { element -> val item = element.jsonObject diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt index f3eb1915..568fdd9e 100644 --- a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object LibreTubePortabilityWriter { +object LibreTubePortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt index 3c99df75..17a517fc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt @@ -1,6 +1,6 @@ package dev.typetype.server.portability -internal fun newPipeArchiveCapabilities(): Set = +fun newPipeArchiveCapabilities(): Set = archiveImportCapabilities().mapTo(linkedSetOf()) { capability -> PortabilityCapability( capability.category, @@ -12,7 +12,7 @@ internal fun newPipeArchiveCapabilities(): Set = ) } -internal fun newPipeArchiveExportIssues( +fun newPipeArchiveExportIssues( source: PortabilityRecordSource, categories: Set, target: NewPipeArchiveTarget, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt index ad822179..d0ea698a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt @@ -7,7 +7,7 @@ import java.sql.Connection import java.sql.DriverManager import java.util.zip.ZipFile -internal object NewPipeArchiveDatabase { +object NewPipeArchiveDatabase { private val databaseNames = setOf("newpipe.db", "pipepipe.db") fun userVersion(input: PortabilityInput): Int? { diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt index 96d2b206..ae448eb7 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt @@ -6,7 +6,7 @@ import java.sql.DriverManager import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -internal object NewPipeArchivePortabilityWriter { +object NewPipeArchivePortabilityWriter { fun write( source: PortabilityRecordSource, output: OutputStream, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt index 8c08762a..d5835e3f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.sql.Connection -internal class NewPipeArchiveRecordWriter( +class NewPipeArchiveRecordWriter( private val db: Connection, private val target: NewPipeArchiveTarget, ) { diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt index e83b543f..336dcca1 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.sql.Connection -internal object NewPipeArchiveSchema { +object NewPipeArchiveSchema { fun create(db: Connection, target: NewPipeArchiveTarget) { db.createStatement().use { statement -> statement.execute("PRAGMA foreign_keys = ON") diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt index 61bf8305..49794bba 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt @@ -1,6 +1,6 @@ package dev.typetype.server.portability -internal enum class NewPipeArchiveTarget( +enum class NewPipeArchiveTarget( val databaseVersion: Int, val identityHash: String, val pipePipe: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt index 87d057f8..67d3c6dc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import java.sql.Connection import java.sql.ResultSet -internal object NewPipeDatabasePortabilityReader { +object NewPipeDatabasePortabilityReader { fun read(connection: Connection, sink: PortabilityRecordSink) { subscriptions(connection, sink) groups(connection, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt index 084cf6bc..742793b0 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.net.URI -internal object NewPipeProvider { +object NewPipeProvider { fun serviceId(url: String): Int? { val host = runCatching { URI(url.trim()).host?.lowercase() }.getOrNull() ?: return null return when { diff --git a/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt index e5bb46fb..1ea4f22f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt @@ -36,7 +36,7 @@ class PipePipePortabilityAdapter : PortabilityAdapter { } } -internal fun archiveImportCapabilities(): Set = setOf( +fun archiveImportCapabilities(): Set = setOf( PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityCategory.HISTORY, diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt index 30703bf9..c0c4d449 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt @@ -6,7 +6,7 @@ import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.jsonObject -internal object PipedPortabilityReader { +object PipedPortabilityReader { fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken = JsonToken.START_ARRAY) { require(token == JsonToken.START_ARRAY) { "Invalid Piped subscriptions" } sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt index 304e6cde..d6d65011 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object PipedPortabilityWriter { +object PipedPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt index 10dec38b..ad3236a1 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt @@ -32,6 +32,20 @@ interface PortabilityRecordSource { fun categories(): Set fun counts(): Map fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) + fun readBatch(category: PortabilityCategory, cursor: Long?, limit: Int): PortabilityRecordBatch { + require(limit > 0) + var position = 0L + var nextCursor = cursor + val records = ArrayList(limit) + forEach(category) { record -> + if (position > (cursor ?: -1L) && records.size < limit) { + records += record + nextCursor = position + } + position++ + } + return PortabilityRecordBatch(records, nextCursor) + } fun forEachChild(category: PortabilityCategory, parentKey: String, block: (PortabilityRecord) -> Unit) { forEach(category) { record -> if (record.parentKey() == parentKey.trim().lowercase()) block(record) @@ -39,6 +53,11 @@ interface PortabilityRecordSource { } } +data class PortabilityRecordBatch( + val records: List, + val nextCursor: Long?, +) + interface PortabilityAdapter { val descriptor: PortabilityAdapterDescriptor val autoDetect: Boolean get() = true diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt new file mode 100644 index 00000000..2b37752a --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.portability + +internal class PortabilityBatchRecordSource( + private val category: PortabilityCategory, + private val records: List, +) : PortabilityRecordSource { + init { + require(records.all { it.category == category }) + } + + override fun categories(): Set = setOf(category) + + override fun counts(): Map = mapOf(category to records.size.toLong()) + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + if (category == this.category) records.forEach(block) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt index 0757eb72..065c1fd3 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt @@ -6,6 +6,7 @@ interface PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + onCategoryProgress: (PortabilityCategory, Long) -> Unit = { _, _ -> }, ): Map suspend fun export( diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index 9d4d83a2..c0fbafb8 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -10,7 +10,7 @@ import kotlinx.coroutines.withContext import java.nio.file.Files import java.nio.file.Path -class PortabilityEngine internal constructor( +class PortabilityEngine constructor( private val registry: PortabilityRegistry, private val dataPort: PortabilityDataPort, private val store: PortabilityJobStore, @@ -124,14 +124,25 @@ class PortabilityEngine internal constructor( private suspend fun apply(job: PortabilityJob, request: PortabilityImportRequest) { runJob(job, null) { + val source = requireNotNull(job.spool) + val counts = source.counts() + val total = request.categories.sumOf { counts[it] ?: 0L } val progress = PortabilityProgressReporter( job, PortabilityProgressPhase.APPLYING, - PortabilityProgressUnit.CATEGORIES, - request.categories.size.toLong(), - interval = 1L, + PortabilityProgressUnit.RECORDS, + total, + interval = portabilityProgressInterval(total), + ) + val result = PortabilityImportCoordinator(dataPort).apply( + job.ownerId, + source, + request, + job.snapshot().preview?.detection?.format, + counts, + progress, + job::updateResult, ) - val result = dataPort.import(job.ownerId, requireNotNull(job.spool), request) { _, _ -> progress.add() } progress.finish() job.transition(setOf(PortabilityJobState.APPLYING), PortabilityJobState.COMPLETED, result = result) } diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt new file mode 100644 index 00000000..363dfa2e --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.portability + +internal class PortabilityImportCoordinator( + private val dataPort: PortabilityDataPort, +) { + suspend fun apply( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + format: PortabilityFormat?, + counts: Map, + progress: PortabilityProgressReporter, + onCommitted: (Map) -> Unit, + ): Map { + if (format == PortabilityFormat.YOUTUBE_TAKEOUT) { + return YoutubeTakeoutImportWorkflow(dataPort).apply( + userId, + source, + request, + progress, + onCommitted, + ) + } + val completed = linkedMapOf() + return dataPort.import( + userId, + source, + request, + onCategoryProgress = { category, count -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + progress.add(count) + }, + onCategoryComplete = { category, count -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + completed[category.wireName] = count + onCommitted(completed.toMap()) + progress.checkpoint() + }, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt index 100ecf63..c0ca4127 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -6,7 +6,7 @@ import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.atomic.AtomicReference -internal class PortabilityJob( +class PortabilityJob( val id: String, val ownerId: String, val kind: PortabilityJobKind, @@ -46,6 +46,15 @@ internal class PortabilityJob( } } + fun updateResult(result: Map) { + while (true) { + val current = value.get() + if (current.state in TERMINAL_STATES) return + val next = current.copy(updatedAt = clock(), result = result) + if (value.compareAndSet(current, next)) return + } + } + fun transition( expected: Set, state: PortabilityJobState, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt index 97fffcfa..726ed8ef 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt @@ -53,12 +53,31 @@ enum class PortabilityProgressUnit { BYTES, } +@Serializable +enum class PortabilityImportStage { + @SerialName("subscriptions") + SUBSCRIPTIONS, + @SerialName("recentHistory") + RECENT_HISTORY, + @SerialName("playlists") + PLAYLISTS, + @SerialName("history") + HISTORY, + @SerialName("remaining") + REMAINING, +} + @Serializable data class PortabilityJobProgress( val phase: PortabilityProgressPhase, val unit: PortabilityProgressUnit, val processed: Long, val total: Long? = null, + val category: PortabilityCategory? = null, + val stage: PortabilityImportStage? = null, + val stageProcessed: Long = 0L, + val stageTotal: Long? = null, + val checkpoint: Long = 0L, ) @Serializable diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt index feb3eae2..d837db37 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt @@ -5,7 +5,7 @@ import java.nio.file.Path import java.util.UUID import java.util.concurrent.ConcurrentHashMap -internal class PortabilityJobStore( +class PortabilityJobStore( private val root: Path, private val clock: () -> Long = System::currentTimeMillis, private val retentionMs: Long = DEFAULT_RETENTION_MS, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt index 8e0f00b1..aaf6271d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt @@ -10,7 +10,7 @@ import kotlinx.serialization.json.JsonElement import java.nio.file.Files import java.io.ByteArrayOutputStream -internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() +val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() .streamReadConstraints( StreamReadConstraints.builder() .maxNestingDepth(100) @@ -21,22 +21,22 @@ internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) .build() -internal inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = +inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = Files.newInputStream(path).buffered().use { input -> PortabilityJsonFactory.createParser(input).use(block) } -internal fun JsonParser.requireObject() { +fun JsonParser.requireObject() { require(nextToken() == JsonToken.START_OBJECT) { "Backup root must be a JSON object" } } -internal fun JsonParser.textOrEmpty(): String = +fun JsonParser.textOrEmpty(): String = if (currentToken().isScalarValue) valueAsString.orEmpty() else "" -internal fun JsonParser.longOrZero(): Long = +fun JsonParser.longOrZero(): Long = if (currentToken().isNumeric) longValue else valueAsString?.toLongOrNull() ?: 0L -internal fun JsonParser.readJsonElement(): JsonElement { +fun JsonParser.readJsonElement(): JsonElement { val bytes = ByteArrayOutputStream() PortabilityJsonFactory.createGenerator(bytes).use { generator -> generator.copyCurrentStructure(this) @@ -45,7 +45,7 @@ internal fun JsonParser.readJsonElement(): JsonElement { return CacheJson.parseToJsonElement(bytes.toString(Charsets.UTF_8)) } -internal fun youtubeChannelUrl(value: String): String { +fun youtubeChannelUrl(value: String): String { val trimmed = value.trim() return when { trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed @@ -54,7 +54,7 @@ internal fun youtubeChannelUrl(value: String): String { } } -internal fun youtubeVideoUrl(value: String): String { +fun youtubeVideoUrl(value: String): String { val trimmed = value.trim() return when { trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed @@ -63,6 +63,6 @@ internal fun youtubeVideoUrl(value: String): String { } } -internal fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { +fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { it != '&' && it != '?' && it != '#' } diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt similarity index 56% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt index a6d0dc0d..28836960 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt @@ -8,24 +8,24 @@ import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonPrimitive -internal fun JsonObject.string(name: String): String = get(name)?.let { element -> +fun JsonObject.string(name: String): String = get(name)?.let { element -> runCatching { element.jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") }.orEmpty() -internal fun JsonObject.long(name: String): Long = get(name)?.let { element -> +fun JsonObject.long(name: String): Long = get(name)?.let { element -> runCatching { element.jsonPrimitive.contentOrNull?.toLongOrNull() ?: 0L }.getOrDefault(0L) } ?: 0L -internal fun JsonObject.int(name: String): Int = long(name).toInt() +fun JsonObject.int(name: String): Int = long(name).toInt() -internal fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> +fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> runCatching { element.jsonArray }.getOrNull() } ?: JsonArray(emptyList()) -internal fun JsonElement.stringValue(): String = runCatching { +fun JsonElement.stringValue(): String = runCatching { jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") -internal fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject +fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject -internal fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive +fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt new file mode 100644 index 00000000..0eca3eed --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.ChannelUrlCanonicalizer +import dev.typetype.server.services.SubscriptionAvatarRepairer +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.neq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +object PortabilityMediaAvatarPropagation { + fun propagate(userId: String): Int { + val subscriptions = SubscriptionsTable.selectAll() + .where { + SubscriptionsTable.userId eq userId + } + .limit(MAX_SUBSCRIPTIONS) + .map { + SubscriptionItem( + channelUrl = it[SubscriptionsTable.channelUrl], + name = it[SubscriptionsTable.name], + avatarUrl = it[SubscriptionsTable.avatarUrl], + subscribedAt = it[SubscriptionsTable.subscribedAt], + ) + } + val avatars = SubscriptionAvatarRepairer.repairImported(userId, subscriptions) + .filter { it.avatarUrl.isNotBlank() } + .associate { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) to it.avatarUrl + } + return avatars.entries.sumOf { (channelUrl, avatarUrl) -> + HistoryTable.update({ + (HistoryTable.userId eq userId) and + (HistoryTable.channelUrl eq channelUrl) and + (HistoryTable.channelAvatar eq "") + }) { it[HistoryTable.channelAvatar] = avatarUrl } + + PlaylistVideosTable.update({ + (PlaylistVideosTable.userId eq userId) and + (PlaylistVideosTable.channelUrl eq channelUrl) and + (PlaylistVideosTable.channelAvatar eq "") + }) { it[PlaylistVideosTable.channelAvatar] = avatarUrl } + + WatchLaterTable.update({ + (WatchLaterTable.userId eq userId) and + (WatchLaterTable.channelUrl eq channelUrl) and + (WatchLaterTable.channelAvatar eq "") + }) { it[WatchLaterTable.channelAvatar] = avatarUrl } + + FavoritesTable.update({ + (FavoritesTable.userId eq userId) and + (FavoritesTable.channelUrl eq channelUrl) and + (FavoritesTable.channelAvatar eq "") + }) { it[FavoritesTable.channelAvatar] = avatarUrl } + } + } + + private const val MAX_SUBSCRIPTIONS = 2_000 +} diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt similarity index 65% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt index 265850d2..55abca6c 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.CancellationException import java.io.FilterOutputStream import java.io.OutputStream -internal class PortabilityProgressReporter( +class PortabilityProgressReporter( private val job: PortabilityJob, private val phase: PortabilityProgressPhase, private val unit: PortabilityProgressUnit, @@ -13,18 +13,39 @@ internal class PortabilityProgressReporter( ) { private var processed = 0L private var published = -1L + private var category: PortabilityCategory? = null + private var stage: PortabilityImportStage? = null + private var stageProcessed = 0L + private var stageTotal: Long? = null + private var checkpoint = 0L init { publish(force = true) } + fun setStage(stage: PortabilityImportStage, category: PortabilityCategory, total: Long?) { + if (this.stage == stage && this.category == category && stageTotal == total) return + this.stage = stage + this.category = category + stageProcessed = 0L + stageTotal = total + publish(force = true) + } + fun add(count: Long = 1L) { ensureActive() require(count >= 0L) processed = Math.addExact(processed, count) + stageProcessed = Math.addExact(stageProcessed, count) publish(force = false) } + fun checkpoint() { + ensureActive() + checkpoint = Math.addExact(checkpoint, 1L) + publish(force = true) + } + fun finish() { ensureActive() publish(force = true) @@ -36,12 +57,28 @@ internal class PortabilityProgressReporter( private fun publish(force: Boolean) { if (!force && processed - published < interval) return - job.updateProgress(PortabilityJobProgress(phase, unit, processed, total)) + job.updateProgress( + PortabilityJobProgress( + phase, + unit, + processed, + total, + category, + stage, + stageProcessed, + stageTotal, + checkpoint, + ), + ) published = processed } } -internal class ProgressRecordSink( +fun portabilityProgressInterval(total: Long?): Long = total + ?.let { (it / 100L).coerceIn(1L, 100L) } + ?: 100L + +class ProgressRecordSink( private val delegate: PortabilityRecordSink, private val progress: PortabilityProgressReporter, ) : PortabilityRecordSink { @@ -74,7 +111,7 @@ internal class ProgressRecordSink( (delegate as? PortabilityRecordSource)?.counts()?.get(category) ?: 0L } -internal class ProgressOutputStream( +class ProgressOutputStream( output: OutputStream, private val progress: PortabilityProgressReporter, ) : FilterOutputStream(output) { diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt index 7ffa01a1..668991fa 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt @@ -63,7 +63,7 @@ class PortabilityFormatMismatchException(format: PortabilityFormat) : class PortabilityUploadTooLargeException : PortabilityContractException("portability_upload_too_large", "Backup exceeds the upload limit") -internal fun portabilityErrorCode(error: Exception): String = when (error) { +fun portabilityErrorCode(error: Exception): String = when (error) { is PortabilityContractException -> error.code is PortabilityJobNotFoundException -> "portability_job_not_found" is IllegalStateException -> "portability_invalid_state" @@ -71,7 +71,7 @@ internal fun portabilityErrorCode(error: Exception): String = when (error) { else -> "portability_failed" } -internal fun portabilityErrorMessage(error: Exception): String = when (error) { +fun portabilityErrorMessage(error: Exception): String = when (error) { is PortabilityContractException, is IllegalArgumentException -> error.message ?: "Invalid portability data" else -> "Portability operation failed" diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt index 24acc1d5..a6261358 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt @@ -157,6 +157,28 @@ class PortabilitySpool private constructor( } } + override fun readBatch(category: PortabilityCategory, cursor: Long?, limit: Int): PortabilityRecordBatch { + require(limit > 0) + flush() + connection.prepareStatement( + "SELECT ordinal, payload FROM records WHERE category = ? AND ordinal > ? ORDER BY ordinal LIMIT ?", + ).use { statement -> + statement.setString(1, category.wireName) + statement.setLong(2, cursor ?: 0L) + statement.setInt(3, limit) + statement.executeQuery().use { rows -> + var nextCursor = cursor + val records = buildList { + while (rows.next()) { + nextCursor = rows.getLong(1) + add(CacheJson.decodeFromString(rows.getString(2))) + } + } + return PortabilityRecordBatch(records, nextCursor) + } + } + } + override fun forEachChild( category: PortabilityCategory, parentKey: String, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt similarity index 63% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt index de0ff23d..9d3f4bf4 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt @@ -3,10 +3,10 @@ package dev.typetype.server.portability import java.security.MessageDigest import java.util.HexFormat -internal fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") +fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") .digest(value.toByteArray(Charsets.UTF_8)) .let(HexFormat.of()::formatHex) -internal fun portabilityCategoryByWireName(value: String): PortabilityCategory = +fun portabilityCategoryByWireName(value: String): PortabilityCategory = PortabilityCategory.entries.firstOrNull { it.wireName == value } ?: error("Unknown portability category: $value") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt index 7f81f08e..ccedeb7a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt @@ -5,7 +5,7 @@ import java.nio.file.Path import java.sql.Connection import java.sql.DriverManager -internal object PortabilitySpoolSchema { +object PortabilitySpoolSchema { fun create(directory: Path, factory: (Path, Connection) -> T): T { Files.createDirectories(directory) val path = Files.createTempFile(directory, "portability-", ".sqlite") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt index c71f3a18..2c5c8421 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt @@ -15,7 +15,7 @@ data class PortabilityVideo( val publishedAt: Long = -1L, ) -internal fun PortabilityVideo.normalized(): PortabilityVideo = copy( +fun PortabilityVideo.normalized(): PortabilityVideo = copy( url = url.trim(), title = title.trim(), thumbnailUrl = thumbnailUrl.trim(), diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt index 08c313c3..ce9b6b4f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityCoreExport { +object TypeTypePortabilityCoreExport { fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { when (category) { PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt similarity index 54% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt index 5f867ea3..d0c99e05 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt @@ -1,6 +1,5 @@ package dev.typetype.server.portability -import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable @@ -23,11 +22,11 @@ internal object TypeTypePortabilityCoreImport { category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long = when (category) { - PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy) - PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy) - PortabilityCategory.HISTORY -> history(userId, source, policy) - PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy) + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy, onRecord) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy, onRecord) + PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy, onRecord) else -> error("Unsupported core portability category") } @@ -35,6 +34,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { SubscriptionMutationLock.acquire(userId) if (policy == PortabilityDuplicatePolicy.REPLACE) { @@ -42,15 +42,17 @@ internal object TypeTypePortabilityCoreImport { } var count = 0L source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> - if (record !is PortabilitySubscription) return@forEach - val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) - count += SubscriptionsTable.insertIgnore { - it[SubscriptionsTable.userId] = userId - it[SubscriptionsTable.channelUrl] = channelUrl - it[SubscriptionsTable.name] = record.name - it[SubscriptionsTable.avatarUrl] = record.avatarUrl - it[SubscriptionsTable.subscribedAt] = record.subscribedAt - }.insertedCount + if (record is PortabilitySubscription) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + count += SubscriptionsTable.insertIgnore { + it[SubscriptionsTable.userId] = userId + it[SubscriptionsTable.channelUrl] = channelUrl + it[SubscriptionsTable.name] = record.name + it[SubscriptionsTable.avatarUrl] = record.avatarUrl + it[SubscriptionsTable.subscribedAt] = record.subscribedAt + }.insertedCount + } + onRecord() } return count } @@ -59,6 +61,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { SubscriptionMutationLock.acquire(userId) if (policy == PortabilityDuplicatePolicy.REPLACE) { @@ -67,69 +70,43 @@ internal object TypeTypePortabilityCoreImport { } var count = 0L source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> - if (record !is PortabilitySubscriptionGroup) return@forEach - val normalized = record.name.trim().lowercase(Locale.ROOT) - require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } - count += SubscriptionGroupsTable.insertIgnore { - it[id] = stableId(userId, "group:$normalized") - it[SubscriptionGroupsTable.userId] = userId - it[name] = record.name.trim() - it[normalizedName] = normalized - it[createdAt] = 0L - it[updatedAt] = 0L - }.insertedCount + if (record is PortabilitySubscriptionGroup) { + val normalized = record.name.trim().lowercase(Locale.ROOT) + require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } + count += SubscriptionGroupsTable.insertIgnore { + it[id] = stableId(userId, "group:$normalized") + it[SubscriptionGroupsTable.userId] = userId + it[name] = record.name.trim() + it[normalizedName] = normalized + it[createdAt] = 0L + it[updatedAt] = 0L + }.insertedCount + } + onRecord() } source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> - if (record !is PortabilitySubscriptionGroupMembership) return@forEach - val normalized = record.groupName.trim().lowercase(Locale.ROOT) - val groupId = SubscriptionGroupsTable.selectAll().where { - (SubscriptionGroupsTable.userId eq userId) and - (SubscriptionGroupsTable.normalizedName eq normalized) - }.singleOrNull()?.get(SubscriptionGroupsTable.id) ?: return@forEach - val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) - val subscribed = SubscriptionsTable.selectAll().where { - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) - }.empty().not() - if (!subscribed) return@forEach - count += SubscriptionGroupMembershipsTable.insertIgnore { - it[SubscriptionGroupMembershipsTable.groupId] = groupId - it[SubscriptionGroupMembershipsTable.userId] = userId - it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl - it[addedAt] = 0L - }.insertedCount - } - return count - } - - private fun history( - userId: String, - source: PortabilityRecordSource, - policy: PortabilityDuplicatePolicy, - ): Long { - if (policy == PortabilityDuplicatePolicy.REPLACE) HistoryTable.deleteWhere { HistoryTable.userId eq userId } - var count = 0L - source.forEach(PortabilityCategory.HISTORY) { record -> - if (record !is PortabilityHistory) return@forEach - val exists = HistoryTable.selectAll().where { - (HistoryTable.userId eq userId) and - (HistoryTable.url eq record.video.url) and - (HistoryTable.watchedAt eq record.watchedAt) - }.empty().not() - if (!exists) { - count += HistoryTable.insertIgnore { - it[id] = UUID.randomUUID().toString() - it[HistoryTable.userId] = userId - it[url] = record.video.url - it[title] = record.video.title - it[thumbnail] = record.video.thumbnailUrl - it[channelName] = record.video.channelName - it[channelUrl] = record.video.channelUrl - it[channelAvatar] = record.video.channelAvatarUrl - it[duration] = record.video.durationSeconds - it[progress] = record.positionSeconds - it[watchedAt] = record.watchedAt - }.insertedCount + if (record is PortabilitySubscriptionGroupMembership) { + val normalized = record.groupName.trim().lowercase(Locale.ROOT) + val groupId = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalized) + }.singleOrNull()?.get(SubscriptionGroupsTable.id) + if (groupId != null) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.empty().not() + if (subscribed) { + count += SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = 0L + }.insertedCount + } + } } + onRecord() } return count } @@ -138,6 +115,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { if (policy == PortabilityDuplicatePolicy.REPLACE) { PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } @@ -149,6 +127,7 @@ internal object TypeTypePortabilityCoreImport { } source.forEach(PortabilityCategory.PLAYLISTS) { record -> if (record is PortabilityPlaylistVideo) count += insertPlaylistVideo(userId, record) + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt index 6d133576..0b095fc0 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityCoreWriter { +object TypeTypePortabilityCoreWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, @@ -109,7 +109,7 @@ internal object TypeTypePortabilityCoreWriter { } } -internal fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { +fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { writeStringField("title", video.title) writeStringField("thumbnail", video.thumbnailUrl) writeNumberField("duration", video.durationSeconds) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt index 343a103b..fedd6e0a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt @@ -9,11 +9,14 @@ class TypeTypePortabilityDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map { val result = linkedMapOf() request.categories.sortedBy(PortabilityCategory::wireName).forEach { category -> val imported = DatabaseFactory.query { - TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) + TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) { + onCategoryProgress(category, 1L) + } } result[category.wireName] = imported onCategoryComplete(category, imported) @@ -21,6 +24,7 @@ class TypeTypePortabilityDataPort : PortabilityDataPort { if (PortabilityCategory.SUBSCRIPTIONS in request.categories) { SubscriptionFeedCacheInvalidation.invalidate(userId) } + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(userId) } return result } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt index a575fa7b..2cb96b9d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityFilterExport { +object TypeTypePortabilityFilterExport { private const val USER_SCOPE = "user" fun write(userId: String, sink: PortabilityRecordSink) { diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt index 819dcc67..ffd07b1d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt @@ -22,11 +22,13 @@ internal object TypeTypePortabilityFilterImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { if (policy == PortabilityDuplicatePolicy.REPLACE) clear(userId) var count = 0L source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> if (record is PortabilityContentFilter) count += insert(userId, record) + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt index 5e37bb41..3d51d893 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityFilterWriter { +object TypeTypePortabilityFilterWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt new file mode 100644 index 00000000..17aa9d3d --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt @@ -0,0 +1,64 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.select +import java.util.UUID + +object TypeTypePortabilityHistoryImport { + private const val BATCH_SIZE = 500 + + fun write( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) { + HistoryTable.deleteWhere { HistoryTable.userId eq userId } + } + val knownKeys = existingKeys(userId, policy) + val pending = ArrayList(BATCH_SIZE) + var count = 0L + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record is PortabilityHistory && knownKeys.add(HistoryKey(record.video.url, record.watchedAt))) { + pending += record + } + onRecord() + if (pending.size >= BATCH_SIZE) { + count += insertBatch(userId, pending) + pending.clear() + } + } + return count + insertBatch(userId, pending) + } + + private fun existingKeys(userId: String, policy: PortabilityDuplicatePolicy): MutableSet { + if (policy == PortabilityDuplicatePolicy.REPLACE) return HashSet() + return HistoryTable.select(HistoryTable.url, HistoryTable.watchedAt) + .where { HistoryTable.userId eq userId } + .mapTo(HashSet()) { HistoryKey(it[HistoryTable.url], it[HistoryTable.watchedAt]) } + } + + private fun insertBatch(userId: String, records: List): Long { + if (records.isEmpty()) return 0L + HistoryTable.batchInsert(records, shouldReturnGeneratedValues = false) { record -> + this[HistoryTable.id] = UUID.randomUUID().toString() + this[HistoryTable.userId] = userId + this[HistoryTable.url] = record.video.url + this[HistoryTable.title] = record.video.title + this[HistoryTable.thumbnail] = record.video.thumbnailUrl + this[HistoryTable.channelName] = record.video.channelName + this[HistoryTable.channelUrl] = record.video.channelUrl + this[HistoryTable.channelAvatar] = record.video.channelAvatarUrl + this[HistoryTable.duration] = record.video.durationSeconds + this[HistoryTable.progress] = record.positionSeconds + this[HistoryTable.watchedAt] = record.watchedAt + } + return records.size.toLong() + } + + private data class HistoryKey(val url: String, val watchedAt: Long) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt index 1fa94f38..9f365481 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt @@ -6,19 +6,20 @@ internal object TypeTypePortabilityImport { category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long = when (category) { PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.SUBSCRIPTION_GROUPS, - PortabilityCategory.HISTORY, PortabilityCategory.PLAYLISTS, - -> TypeTypePortabilityCoreImport.write(userId, category, source, policy) + -> TypeTypePortabilityCoreImport.write(userId, category, source, policy, onRecord) + PortabilityCategory.HISTORY -> TypeTypePortabilityHistoryImport.write(userId, source, policy, onRecord) PortabilityCategory.WATCH_LATER, PortabilityCategory.FAVORITES, PortabilityCategory.PROGRESS, PortabilityCategory.SEARCH_HISTORY, PortabilityCategory.SAVED_PLAYLISTS, - -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy) - PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source) - PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy) + -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy, onRecord) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source, onRecord) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy, onRecord) } } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt index fb7d4dd3..805a0682 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt @@ -9,7 +9,7 @@ import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityLibraryExport { +object TypeTypePortabilityLibraryExport { fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { when (category) { PortabilityCategory.WATCH_LATER -> watchLater(userId, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt index d1b3ae65..46de4f9d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt @@ -13,12 +13,13 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import java.nio.charset.StandardCharsets import java.util.UUID -internal object TypeTypePortabilityLibraryImport { +object TypeTypePortabilityLibraryImport { fun write( userId: String, category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { clearIfReplacing(userId, category, policy) var count = 0L @@ -31,6 +32,7 @@ internal object TypeTypePortabilityLibraryImport { is PortabilitySavedPlaylist -> savedPlaylist(userId, record) else -> 0 } + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt index 31d2946f..2139dd39 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityLibraryWriter { +object TypeTypePortabilityLibraryWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt index f42d596b..43ffc01d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt @@ -9,7 +9,7 @@ import kotlinx.serialization.json.jsonObject import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilitySettingsExport { +object TypeTypePortabilitySettingsExport { fun write(userId: String, sink: PortabilityRecordSink) { val settings = SettingsTable.selectAll().where { SettingsTable.userId eq userId } .singleOrNull()?.toSettingsItem() ?: SettingsItem() diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt new file mode 100644 index 00000000..c45f91fb --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.portability + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.services.TypeTypeBackupLibraryRestore + +object TypeTypePortabilitySettingsImport { + fun write(userId: String, source: PortabilityRecordSource, onRecord: () -> Unit): Long { + var count = 0L + source.forEach(PortabilityCategory.SETTINGS) { record -> + if (record is PortabilitySettings) { + val settings = CacheJson.decodeFromString(record.values.toString()) + count += TypeTypeBackupLibraryRestore.settings(userId, settings) + } + onRecord() + } + return count + } +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt index 0ca1d0f6..91cd9bb5 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilitySettingsWriter { +object TypeTypePortabilitySettingsWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt index 66fa7b75..5d4ea0d7 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION import java.io.OutputStream -internal object TypeTypePortabilityWriter { +object TypeTypePortabilityWriter { fun write( source: PortabilityRecordSource, output: OutputStream, diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt index 337594cc..642ed24d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import java.time.Instant -internal object ViewTubePortabilityReader { +object ViewTubePortabilityReader { fun read(parser: JsonParser, sink: PortabilityRecordSink) { parser.requireObject() while (parser.nextToken() != JsonToken.END_OBJECT) { diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index 14577074..794653fc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -10,13 +10,14 @@ import java.io.InputStreamReader import java.util.zip.ZipEntry import java.util.zip.ZipFile -internal object YoutubeTakeoutCsvPortabilityReader { +object YoutubeTakeoutCsvPortabilityReader { private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" - fun readManifests(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - csvEntries(entries).filter(::isPlaylistManifest).forEach { entry -> + fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + val classified = YoutubeTakeoutCsvSchemaDetector.classify(zip, entries) + classified.filter { it.kind == YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST }.forEach { classifiedEntry -> sink.markCategory(PortabilityCategory.PLAYLISTS) - forEachRow(zip, entry) { header, row -> + forEachRow(zip, classifiedEntry.entry) { header, row -> val playlist = YoutubeTakeoutRowParser.parsePlaylist(header, row) ?: return@forEachRow indexPlaylist(playlist, sink) if (!isSystemPlaylist(playlist)) { @@ -24,12 +25,24 @@ internal object YoutubeTakeoutCsvPortabilityReader { } } } - } - - fun readContent(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - val csv = csvEntries(entries).filterNot(::isPlaylistManifest).toList() - csv.filter(::isSubscriptionFile).forEach { readSubscriptions(zip, it, sink) } - csv.filter(::isPlaylistContent).forEach { readPlaylistItems(zip, it, sink) } + classified.filter { it.kind == YoutubeTakeoutCsvKind.SUBSCRIPTIONS } + .forEach { readSubscriptions(zip, it.entry, sink) } + classified.filter { it.kind == YoutubeTakeoutCsvKind.PLAYLIST_CONTENT } + .forEach { readPlaylistItems(zip, it.entry, sink) } + classified.filter { + it.kind == YoutubeTakeoutCsvKind.OTHER && + it.maybePortable && + isExpectedImportCsv(it.entry.name) + } + .forEach { classifiedEntry -> + sink.issue( + PortabilityIssue( + category = null, + code = "unsupported_takeout_csv", + message = "A YouTube Takeout CSV file could not be classified and was skipped: ${classifiedEntry.entry.name}", + ), + ) + } } private fun readSubscriptions(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { @@ -95,41 +108,50 @@ internal object YoutubeTakeoutCsvPortabilityReader { } } - private fun csvEntries(entries: List) = entries.asSequence().filter { - it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() - } - - private fun isPlaylistManifest(entry: ZipEntry): Boolean = fileStem(entry) in setOf("playlists", "oynatma listeleri") - - private fun isSubscriptionFile(entry: ZipEntry): Boolean = fileStem(entry) in SUBSCRIPTION_NAMES - - private fun isPlaylistContent(entry: ZipEntry): Boolean { - val normalized = YoutubeTakeoutSchemaHints.normalize(entry.name) - return "playlist" in normalized || "oynatma list" in normalized - } - private fun playlistKeyFromPath(path: String): String? = path.substringAfterLast('/').substringBeforeLast('.') .takeUnless { YoutubeTakeoutSchemaHints.normalize(it) == "playlist items" } ?.let { YoutubeTakeoutSystemPlaylist.canonicalKey(it) ?: it } - private fun fileStem(entry: ZipEntry) = YoutubeTakeoutSchemaHints.normalize( - entry.name.substringAfterLast('/').substringBeforeLast('.'), - ) - private fun isSystemPlaylist(item: PlaylistItem) = YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null private fun playlistAliases(name: String): Set = setOf( name, + ) + filenameAliases(name) + localizedPlaylistAliases(name) + + private fun filenameAliases(name: String): Set { + val escaped = name.replace('/', '_').replace('\'', '_') + return setOf( + "$escaped-videos", + "$escaped-vídeos", + ) + } + + private fun localizedPlaylistAliases(name: String): Set = setOf( "Videos from $name", "Videos de $name", + "Vídeos de $name", "Vidéos de $name", "Videos da playlist $name", + "Vídeos da playlist $name", + "Video di $name", + "Video da $name", + "Videos von $name", + "Videos van $name", + "Filmy z playlisty $name", + "Видео из $name", + "動画 - $name", + "$name の動画", + "$name 동영상", + "$name 视频", + "$name 影片", "$name videos", ) private fun PortabilityRecordSink.invalid(category: PortabilityCategory, kind: String) = issue(PortabilityIssue(category, "invalid_takeout_row", "An invalid YouTube Takeout $kind row was skipped")) - private val SUBSCRIPTION_NAMES = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") + private fun isExpectedImportCsv(name: String): Boolean = + YoutubeTakeoutSchemaHints.isSubscriptionText(name) || YoutubeTakeoutSchemaHints.isPlaylistText(name) + } diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt new file mode 100644 index 00000000..80d41e08 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt @@ -0,0 +1,133 @@ +package dev.typetype.server.portability + +import dev.typetype.server.services.YoutubeTakeoutCsvReader +import dev.typetype.server.services.YoutubeTakeoutDateParser +import dev.typetype.server.services.YoutubeTakeoutSchemaHints +import dev.typetype.server.services.YoutubeTakeoutTextNormalizer +import java.io.BufferedReader +import java.io.InputStreamReader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +enum class YoutubeTakeoutCsvKind { + PLAYLIST_MANIFEST, + SUBSCRIPTIONS, + PLAYLIST_CONTENT, + OTHER, +} + +data class YoutubeTakeoutCsvClassification( + val entry: ZipEntry, + val kind: YoutubeTakeoutCsvKind, + val maybePortable: Boolean, +) + +object YoutubeTakeoutCsvSchemaDetector { + fun classify(zip: ZipFile, entries: List): List = entries.asSequence() + .filter { it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() } + .map { entry -> classify(zip, entry) } + .toList() + + private fun classify(zip: ZipFile, entry: ZipEntry): YoutubeTakeoutCsvClassification { + val stem = fileStem(entry) + val path = YoutubeTakeoutSchemaHints.normalize(entry.name) + if (YoutubeTakeoutSchemaHints.isPlaylistManifestName(stem)) { + return YoutubeTakeoutCsvClassification(entry, YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST, maybePortable = false) + } + if (YoutubeTakeoutSchemaHints.isSubscriptionText(stem)) { + return YoutubeTakeoutCsvClassification(entry, YoutubeTakeoutCsvKind.SUBSCRIPTIONS, maybePortable = false) + } + if (YoutubeTakeoutSchemaHints.isPlaylistText(path)) { + val shape = inspect(zip, entry) + val kind = if (shape.isManifest()) YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST else YoutubeTakeoutCsvKind.PLAYLIST_CONTENT + return YoutubeTakeoutCsvClassification(entry, kind, maybePortable = false) + } + val shape = inspect(zip, entry) + val kind = when { + shape.isManifest() -> YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST + shape.isSubscription() -> YoutubeTakeoutCsvKind.SUBSCRIPTIONS + shape.isContent() -> YoutubeTakeoutCsvKind.PLAYLIST_CONTENT + else -> YoutubeTakeoutCsvKind.OTHER + } + return YoutubeTakeoutCsvClassification(entry, kind, shape.maybePortable()) + } + + private fun inspect(zip: ZipFile, entry: ZipEntry): CsvShape { + var header = emptyList() + val rows = ArrayList>(SAMPLE_ROWS) + zip.getInputStream(entry).use { input -> + val reader = BufferedReader(InputStreamReader(input, Charsets.UTF_8)) + YoutubeTakeoutCsvReader.forEach(reader, { header = it }) { row -> + if (rows.size < SAMPLE_ROWS) rows += row + } + } + return CsvShape(header, rows) + } + + private fun fileStem(entry: ZipEntry): String = YoutubeTakeoutSchemaHints.normalize( + entry.name.substringAfterLast('/').substringBeforeLast('.'), + ) + + private data class CsvShape( + val header: List, + val rows: List>, + ) { + private val hasChannelId by lazy { + header.any(YoutubeTakeoutSchemaHints::isChannelIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeChannelId) + } + private val hasChannelUrl by lazy { + header.any(YoutubeTakeoutSchemaHints::isChannelUrlHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::containsChannelUrl) + } + private val hasVideoId by lazy { + header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(::isLikelyVideoId) + } + private val hasPlaylistId by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikePlaylistId) + } + private val hasPlaylistKey by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + } + private val hasAddedAt by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistItemAddedAtHeader) || rows.hasValue { + YoutubeTakeoutDateParser.parseEpochMillis(it) != null + } + } + + fun isSubscription(): Boolean = hasChannelId && hasChannelUrl && !hasVideoId && !hasPlaylistId + + fun isManifest(): Boolean = hasPlaylistId && !hasVideoId && + (header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) || rows.any { row -> row.any(::isTextValue) }) + + fun isContent(): Boolean { + if (!hasVideoId || hasChannelId) return false + return hasPlaylistKey || header.size <= 3 && (hasAddedAt || rows.isNotEmpty()) + } + + fun maybePortable(): Boolean = isSubscription() || isManifest() || isContent() || + hasChannelUrl || hasPlaylistId || hasVideoId && !hasChannelId && header.size <= 3 + + private fun isTextValue(value: String): Boolean { + val trimmed = value.trim() + if (trimmed.isBlank() || YoutubeTakeoutSchemaHints.looksLikeChannelId(trimmed) || + YoutubeTakeoutSchemaHints.looksLikePlaylistId(trimmed) || + isLikelyVideoId(trimmed) || + YoutubeTakeoutSchemaHints.containsChannelUrl(trimmed) || + YoutubeTakeoutSchemaHints.containsWatchUrl(trimmed) || + YoutubeTakeoutDateParser.parseEpochMillis(trimmed) != null + ) return false + return YoutubeTakeoutTextNormalizer.normalize(trimmed) !in NON_TEXT_VALUES + } + + private fun isLikelyVideoId(value: String): Boolean = + YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(value) + + private fun List>.hasValue(predicate: (String) -> Boolean): Boolean = any { row -> row.any(predicate) } + } + + private val NON_TEXT_VALUES = setOf( + "true", "false", "yes", "no", "oui", "non", "faux", "vrai", "public", "private", "prive", + "publique", "privee", "unlisted", "non repertorie", + ) + private const val SAMPLE_ROWS = 64 +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt new file mode 100644 index 00000000..54cdc1e2 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -0,0 +1,72 @@ +package dev.typetype.server.portability + +import dev.typetype.server.services.YoutubeTakeoutActivitySignalService +import dev.typetype.server.services.YoutubeTakeoutHistoryParser +import dev.typetype.server.services.YoutubeTakeoutPathHints +import java.io.Reader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +object YoutubeTakeoutHtmlPortabilityReader { + fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> + zip.getInputStream(entry).bufferedReader().use { reader -> + val historyEntry = isHistoryEntry(entry) + val write = { html: String -> + writeWindow(html, sink, includeActivitySignals = !historyEntry, historyEntry) + } + if (entry.size in 1..WHOLE_ENTRY_BYTES) write(reader.readText()) + else readWindows(reader, write) + } + } + } + + private fun writeWindow( + html: String, + sink: PortabilityRecordSink, + includeActivitySignals: Boolean, + historyEntry: Boolean, + ) { + val parsedHistory = YoutubeTakeoutHistoryParser.parseWithDiagnostics(html, requireWatchedMarker = !historyEntry) + parsedHistory.items.forEach { sink.write(it.toPortability()) } + if (parsedHistory.invalidDates > 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "invalid_takeout_date", + "YouTube Takeout HTML rows with an invalid date were skipped", + parsedHistory.invalidDates, + ), + ) + } + if (includeActivitySignals) { + val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) + subscriptions.forEach { sink.write(it.toPortability()) } + favorites.forEach { sink.write(it.toPortability()) } + } + } + + private fun readWindows(reader: Reader, block: (String) -> Unit) { + val buffer = CharArray(READ_CHARS) + val window = StringBuilder(WINDOW_CHARS + READ_CHARS) + while (true) { + val read = reader.read(buffer) + if (read < 0) break + window.append(buffer, 0, read) + if (window.length >= WINDOW_CHARS) { + block(window.toString()) + window.delete(0, window.length - OVERLAP_CHARS) + } + } + if (window.isNotEmpty()) block(window.toString()) + } + + private fun isYoutubeHtml(entry: ZipEntry): Boolean = YoutubeTakeoutPathHints.isYoutubeHtml(entry.name) + + private fun isHistoryEntry(entry: ZipEntry): Boolean = YoutubeTakeoutPathHints.isHistoryEntry(entry.name) + + private const val READ_CHARS = 32 * 1024 + private const val WINDOW_CHARS = 4 * 1024 * 1024 + private const val OVERLAP_CHARS = 256 * 1024 + private const val WHOLE_ENTRY_BYTES = 128L * 1024L * 1024L +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt new file mode 100644 index 00000000..cb54c335 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt @@ -0,0 +1,185 @@ +package dev.typetype.server.portability + +import java.util.PriorityQueue + +internal class YoutubeTakeoutImportWorkflow( + private val dataPort: PortabilityDataPort, +) { + suspend fun apply( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + progress: PortabilityProgressReporter, + onCommitted: (Map) -> Unit, + ): Map { + val counts = source.counts() + val selected = request.categories + val result = linkedMapOf() + val started = mutableSetOf() + val recentHistoryKeys = hashSetOf() + var remainingHistory = 0L + + suspend fun commit( + category: PortabilityCategory, + records: List, + stage: PortabilityImportStage, + stageTotal: Long?, + ) { + progress.setStage(stage, category, stageTotal) + val policy = if (started.add(category)) request.duplicatePolicy else PortabilityDuplicatePolicy.SKIP + val imported = dataPort.import( + userId, + PortabilityBatchRecordSource(category, records), + PortabilityImportRequest(setOf(category), policy), + onCategoryProgress = { _, count -> progress.add(count) }, + )[category.wireName] ?: 0L + result[category.wireName] = Math.addExact(result[category.wireName] ?: 0L, imported) + onCommitted(result.toMap()) + progress.checkpoint() + } + + if (PortabilityCategory.SUBSCRIPTIONS in selected) { + val total = counts[PortabilityCategory.SUBSCRIPTIONS] ?: 0L + val processed = processPaged( + source, + PortabilityCategory.SUBSCRIPTIONS, + progress, + SUBSCRIPTION_BATCH_SIZE, + { true }, + ) { records -> commit(PortabilityCategory.SUBSCRIPTIONS, records, PortabilityImportStage.SUBSCRIPTIONS, total) } + if (processed == 0L) { + commit(PortabilityCategory.SUBSCRIPTIONS, emptyList(), PortabilityImportStage.SUBSCRIPTIONS, total) + } + } + + if (PortabilityCategory.HISTORY in selected) { + val total = counts[PortabilityCategory.HISTORY] ?: 0L + val recent = latestHistory(source, progress) + recentHistoryKeys += recent.map { it.stableKey() } + commit(PortabilityCategory.HISTORY, recent, PortabilityImportStage.RECENT_HISTORY, recent.size.toLong()) + remainingHistory = (total - recent.size).coerceAtLeast(0L) + } + + if (PortabilityCategory.PLAYLISTS in selected) { + val total = counts[PortabilityCategory.PLAYLISTS] ?: 0L + var processed = processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_BATCH_SIZE, + { it is PortabilityPlaylist }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + processed += processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_VIDEO_BATCH_SIZE, + { it is PortabilityPlaylistVideo }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + processed += processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_VIDEO_BATCH_SIZE, + { it !is PortabilityPlaylist && it !is PortabilityPlaylistVideo }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + if (processed == 0L) { + commit(PortabilityCategory.PLAYLISTS, emptyList(), PortabilityImportStage.PLAYLISTS, total) + } + } + + if (PortabilityCategory.HISTORY in selected && remainingHistory > 0L) { + processPaged( + source, + PortabilityCategory.HISTORY, + progress, + HISTORY_BATCH_SIZE, + { it !is PortabilityHistory || it.stableKey() !in recentHistoryKeys }, + ) { records -> + commit(PortabilityCategory.HISTORY, records, PortabilityImportStage.HISTORY, remainingHistory) + } + } + + selected + .filterNot { + it == PortabilityCategory.SUBSCRIPTIONS || + it == PortabilityCategory.HISTORY || + it == PortabilityCategory.PLAYLISTS + } + .sortedBy(PortabilityCategory::wireName) + .forEach { category -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + dataPort.import( + userId, + source, + PortabilityImportRequest(setOf(category), request.duplicatePolicy), + onCategoryProgress = { _, count -> progress.add(count) }, + onCategoryComplete = { _, imported -> + result[category.wireName] = imported + onCommitted(result.toMap()) + progress.checkpoint() + }, + ) + } + return result + } + + private suspend fun processPaged( + source: PortabilityRecordSource, + category: PortabilityCategory, + progress: PortabilityProgressReporter, + batchSize: Int, + include: (PortabilityRecord) -> Boolean, + commit: suspend (List) -> Unit, + ): Long { + var cursor: Long? = null + var matched = 0L + val pending = ArrayList(batchSize) + while (true) { + progress.ensureActive() + val page = source.readBatch(category, cursor, READ_PAGE_SIZE) + if (page.records.isEmpty()) break + check(page.nextCursor != null && page.nextCursor != cursor) { "Portability source cursor did not advance" } + cursor = page.nextCursor + page.records.forEach { record -> + if (include(record)) { + matched++ + pending += record + if (pending.size == batchSize) { + commit(pending.toList()) + pending.clear() + } + } + } + } + if (pending.isNotEmpty()) commit(pending) + return matched + } + + private fun latestHistory( + source: PortabilityRecordSource, + progress: PortabilityProgressReporter, + ): List { + val oldestFirst = compareBy({ it.watchedAt }, { it.stableKey() }) + val recent = PriorityQueue(RECENT_HISTORY_LIMIT, oldestFirst) + source.forEach(PortabilityCategory.HISTORY) { record -> + progress.ensureActive() + if (record !is PortabilityHistory) return@forEach + if (recent.size < RECENT_HISTORY_LIMIT) recent += record + else if (oldestFirst.compare(record, recent.peek()) > 0) { + recent.remove() + recent += record + } + } + return recent.sortedWith(compareByDescending { it.watchedAt }.thenBy { it.stableKey() }) + } + + private companion object { + const val READ_PAGE_SIZE = 500 + const val SUBSCRIPTION_BATCH_SIZE = 50 + const val RECENT_HISTORY_LIMIT = 20 + const val HISTORY_BATCH_SIZE = 500 + const val PLAYLIST_BATCH_SIZE = 5 + const val PLAYLIST_VIDEO_BATCH_SIZE = 50 + } +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt new file mode 100644 index 00000000..c15c867a --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt @@ -0,0 +1,108 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken + +object YoutubeTakeoutJsonEntryReader { + fun read(parser: JsonParser): YoutubeTakeoutJsonEntry { + var title = "" + var titleUrl = "" + var time = "" + var description = "" + var header = "" + val controls = mutableListOf() + val products = mutableListOf() + val subtitles = mutableListOf() + val details = mutableListOf() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val name = parser.currentName() + when (parser.nextToken()) { + JsonToken.VALUE_STRING, JsonToken.VALUE_NUMBER_INT, JsonToken.VALUE_NUMBER_FLOAT -> when (name) { + "title" -> title = parser.textOrEmpty() + "titleUrl" -> titleUrl = parser.textOrEmpty() + "time", "publishedAt" -> time = parser.textOrEmpty() + "description" -> description = parser.textOrEmpty() + "header" -> header = parser.textOrEmpty() + } + JsonToken.START_OBJECT -> if (name == "snippet") { + val nested = read(parser) + if (title.isBlank()) title = nested.title + if (titleUrl.isBlank()) titleUrl = nested.titleUrl + if (time.isBlank()) time = nested.time + if (description.isBlank()) description = nested.description + if (header.isBlank()) header = nested.header + controls += nested.controls + products += nested.products + subtitles += nested.subtitles + details += nested.details + } else parser.skipChildren() + JsonToken.START_ARRAY -> when (name) { + "activityControls" -> readStrings(parser, controls) + "products" -> readStrings(parser, products) + "subtitles" -> readSubtitles(parser, subtitles) + "details" -> readDetails(parser, details) + else -> parser.skipChildren() + } + else -> parser.skipChildren() + } + } + return YoutubeTakeoutJsonEntry(title, titleUrl, time, description, header, controls, products, subtitles, details) + } + + private fun readStrings(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() == JsonToken.VALUE_STRING) target += parser.textOrEmpty() + else parser.skipChildren() + } + } + + private fun readSubtitles(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + var name = "" + var url = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName() + when (parser.nextToken()) { + JsonToken.VALUE_STRING -> when (field) { + "name" -> name = parser.textOrEmpty() + "url" -> url = parser.textOrEmpty() + } + else -> parser.skipChildren() + } + } + target += YoutubeTakeoutJsonSubtitle(name, url) + } + } + + private fun readDetails(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName() + if (parser.nextToken() == JsonToken.VALUE_STRING && field == "name") target += parser.textOrEmpty() + else parser.skipChildren() + } + } + } +} + +data class YoutubeTakeoutJsonEntry( + val title: String, + val titleUrl: String, + val time: String, + val description: String, + val header: String, + val controls: List, + val products: List, + val subtitles: List, + val details: List, +) + +data class YoutubeTakeoutJsonSubtitle(val name: String, val url: String) diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt new file mode 100644 index 00000000..af42b6a3 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt @@ -0,0 +1,172 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.services.YoutubeTakeoutActivityClassifier +import dev.typetype.server.services.YoutubeTakeoutDateParser +import dev.typetype.server.services.YoutubeTakeoutPathHints +import dev.typetype.server.services.YoutubeTakeoutTextNormalizer +import java.io.InputStream + +object YoutubeTakeoutJsonPortabilityReader { + private val videoUrlRegex = Regex( + """(?:https?://|URLs://)(?:www\.|music\.)?youtube\.com/(?:watch\?(?:[^#\s]*&)?v=|shorts/|live/)[A-Za-z0-9_-]{6,}|https?://youtu\.be/[A-Za-z0-9_-]{6,}""", + RegexOption.IGNORE_CASE, + ) + private val channelUrlRegex = Regex( + """https?://(?:www\.)?youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+|c/[A-Za-z0-9._-]+|user/[A-Za-z0-9._-]+)""", + RegexOption.IGNORE_CASE, + ) + private val actionPrefixRegex = Regex( + """^(?:you\s+watched|watched|viewed|you\s+liked|liked|has\s+visto|vous\s+avez\s+regard[eé]|vous\s+avez\s+aime[eé]|te\s+ha\s+gustado|izl[eə]nildi)(?:\s+|$)""", + RegexOption.IGNORE_CASE, + ) + private val actionSuffixRegex = Regex( + """\s+(?:izl[eə]nildi|を視聴しました|を再生しました)\s*$""", + RegexOption.IGNORE_CASE, + ) + + fun isCandidate(path: String): Boolean { + if (!path.endsWith(".json", ignoreCase = true)) return false + val normalized = YoutubeTakeoutTextNormalizer.normalize(path) + val filename = normalized.substringAfterLast('/') + val namedHistory = filename in setOf( + "watch history json", + "watch_history json", + "watch history", + "myactivity json", + "my activity json", + ) || filename.startsWith("myactivity ") || filename.startsWith("my activity ") + return namedHistory || YoutubeTakeoutPathHints.isHistoryEntry(path) || "youtube" in normalized + } + + fun read(input: InputStream, sink: PortabilityRecordSink) { + PortabilityJsonFactory.createParser(input.buffered()).use { parser -> + read(parser, sink) + } + } + + private fun read(parser: JsonParser, sink: PortabilityRecordSink) { + if (parser.nextToken() != JsonToken.START_ARRAY) { + parser.skipChildren() + return + } + var count = 0 + var invalidDates = 0L + var historyRecords = 0L + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "YouTube Takeout JSON contains too many records" } + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + val entry = YoutubeTakeoutJsonEntryReader.read(parser) + if (entry.isAd()) continue + if (!entry.isYoutubeActivity()) continue + val directUrl = findVideoUrl(entry.titleUrl) + val embeddedUrl = findVideoUrl(entry.title) ?: findVideoUrl(entry.description) + val url = directUrl ?: embeddedUrl + val timestamp = entry.time.takeIf(String::isNotBlank)?.let(YoutubeTakeoutDateParser::parseEpochMillis) + val history = url != null && entry.shouldImportHistory(directUrl, embeddedUrl) + val liked = YoutubeTakeoutActivityClassifier.isLikedAction(entry.title) || + entry.controls.any(YoutubeTakeoutActivityClassifier::isLiked) + if (url != null && timestamp != null && history) { + val video = entry.video(url) + sink.write(PortabilityHistory(video, timestamp)) + historyRecords += 1 + if (liked) sink.write(PortabilityFavorite(video, timestamp)) + } else if (url != null && timestamp != null && liked) { + sink.write(PortabilityFavorite(entry.video(url), timestamp)) + } else if (url != null && history) { + invalidDates += 1 + } + if (YoutubeTakeoutActivityClassifier.isSubscribedAction(entry.title) || + entry.controls.any(YoutubeTakeoutActivityClassifier::isSubscribed) + ) { + entry.channel()?.let { sink.write(PortabilitySubscription(it.url, it.name)) } + } + } + if (historyRecords > 0) sink.markCategory(PortabilityCategory.HISTORY) + if (invalidDates > 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "invalid_takeout_date", + "YouTube Takeout JSON rows with an invalid date were skipped", + invalidDates, + ), + ) + } + } + + private fun YoutubeTakeoutJsonEntry.video(url: String): PortabilityVideo { + val title = actionSuffixRegex.replace(actionPrefixRegex.replace(substringBeforeUrl(), "").trim(), "").trim().ifBlank { + "YouTube video ${youtubeId(url)}" + } + val channel = channel() + return PortabilityVideo( + url = url, + title = title, + thumbnailUrl = "https://i.ytimg.com/vi/${youtubeId(url)}/hqdefault.jpg", + channelName = channel?.name.orEmpty(), + channelUrl = channel?.url.orEmpty(), + ) + } + + private fun YoutubeTakeoutJsonEntry.channel(): YoutubeTakeoutJsonSubtitle? { + subtitles.firstOrNull { channelUrlRegex.containsMatchIn(it.url) }?.let { return it } + val url = channelUrlRegex.find(titleUrl)?.value ?: return null + val name = actionSuffixRegex.replace(actionPrefixRegex.replace(title, "").trim(), "").trim() + return YoutubeTakeoutJsonSubtitle(name, url) + } + + private fun YoutubeTakeoutJsonEntry.substringBeforeUrl(): String = title.replace(videoUrlRegex, "").trim() + + private fun YoutubeTakeoutJsonEntry.isYoutubeActivity(): Boolean { + val markers = buildList { + if (header.isNotBlank()) add(header) + addAll(products) + addAll(controls) + } + return markers.isEmpty() || markers.any { YoutubeTakeoutTextNormalizer.normalize(it).contains("youtube") } + } + + private fun YoutubeTakeoutJsonEntry.shouldImportHistory(directUrl: String?, embeddedUrl: String?): Boolean { + if (isNavigationActivity()) return false + if (YoutubeTakeoutActivityClassifier.isWatchedAction(title)) return true + if (controls.any(YoutubeTakeoutPathHints::isHistoryEntry)) return true + if (YoutubeTakeoutActivityClassifier.isLikedAction(title) || + YoutubeTakeoutActivityClassifier.isSubscribedAction(title) || + controls.any(YoutubeTakeoutActivityClassifier::isLiked) || + controls.any(YoutubeTakeoutActivityClassifier::isSubscribed) + ) return false + return directUrl != null && header.isBlank() && products.isEmpty() && controls.isEmpty() && embeddedUrl == null + } + + private fun YoutubeTakeoutJsonEntry.isNavigationActivity(): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(title) + return normalized.startsWith("visited ") || normalized.startsWith("searched ") || + normalized.startsWith("viewed a post") || normalized.startsWith("viewed ads") + } + + private fun YoutubeTakeoutJsonEntry.isAd(): Boolean = details.any { + val normalized = YoutubeTakeoutTextNormalizer.normalize(it) + normalized in setOf("ads", "advertisement", "advertisements", "annonces", "werbung") || + normalized.contains("google ads") + } || YoutubeTakeoutTextNormalizer.normalize(title).let { + it.startsWith("viewed ads on youtube") || it.startsWith("watched ads on youtube") + } + + private fun findVideoUrl(value: String): String? = videoUrlRegex.find(value)?.value?.let { + when { + it.startsWith("URLs://", ignoreCase = true) -> "https://${it.substringAfter("://")}" + it.startsWith("http://", ignoreCase = true) -> "https://${it.substringAfter("://")}" + else -> it + } + } + + private fun youtubeId(url: String): String = url.substringAfter("v=", "").substringBefore('&').ifBlank { + url.substringAfterLast('/').substringBefore('?') + } + +} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt index b335dc2b..1fa7531d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt @@ -6,14 +6,14 @@ import dev.typetype.server.models.PlaylistVideoItem import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.YoutubeTypeTypeMapper -internal fun SubscriptionItem.toPortability() = PortabilitySubscription( +fun SubscriptionItem.toPortability() = PortabilitySubscription( channelUrl = channelUrl, name = name, avatarUrl = avatarUrl, subscribedAt = subscribedAt, ) -internal fun HistoryItem.toPortability() = PortabilityHistory( +fun HistoryItem.toPortability() = PortabilityHistory( video = PortabilityVideo( url = url, title = title, @@ -27,7 +27,7 @@ internal fun HistoryItem.toPortability() = PortabilityHistory( positionSeconds = progress, ) -internal fun FavoriteItem.toPortability() = PortabilityFavorite( +fun FavoriteItem.toPortability() = PortabilityFavorite( video = PortabilityVideo( url = videoUrl, title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(videoUrl) }, @@ -42,7 +42,7 @@ internal fun FavoriteItem.toPortability() = PortabilityFavorite( favoritedAt = favoritedAt, ) -internal fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( +fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( url = url, title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(url) }, thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt index 50e3cbfc..3cd932f5 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt @@ -1,12 +1,13 @@ package dev.typetype.server.portability import java.io.OutputStream +import java.nio.file.Files import java.util.zip.ZipFile class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { override val descriptor = PortabilityAdapterDescriptor( format = PortabilityFormat.YOUTUBE_TAKEOUT, - adapterVersion = 1, + adapterVersion = 3, capabilities = TAKEOUT_CATEGORIES.mapTo(linkedSetOf()) { category -> PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) }, @@ -15,28 +16,46 @@ class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { ) override fun detect(input: PortabilityInput): PortabilityDetection? { - val archive = input.archive ?: return null + val archive = input.archive + if (archive == null) { + if (!YoutubeTakeoutJsonPortabilityReader.isCandidate(input.filename)) return null + return PortabilityDetection( + PortabilityFormat.YOUTUBE_TAKEOUT, + null, + 90, + "YouTube Takeout JSON activity file", + ) + } val names = archive.names.map(String::lowercase) val youtubeFiles = names.count { name -> - "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html")) + "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html") || name.endsWith(".json")) } - if (youtubeFiles == 0) return null + val jsonFiles = names.count { YoutubeTakeoutJsonPortabilityReader.isCandidate(it) } + if (youtubeFiles + jsonFiles == 0) return null val hasTakeoutRoot = names.any { it.startsWith("takeout/") } return PortabilityDetection( PortabilityFormat.YOUTUBE_TAKEOUT, null, if (hasTakeoutRoot) 99 else 90, - "YouTube Takeout CSV or activity files", + "YouTube Takeout CSV, HTML, or JSON activity files", ) } override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { requireNotNull(detect(input)) { "Unsupported YouTube Takeout archive" } + if (input.archive == null) { + Files.newInputStream(input.path).use { YoutubeTakeoutJsonPortabilityReader.read(it, sink) } + return + } ZipFile(input.path.toFile()).use { zip -> val entries = zip.entries().asSequence().filterNot { it.isDirectory }.toList() - YoutubeTakeoutCsvPortabilityReader.readManifests(zip, entries, sink) + YoutubeTakeoutCsvPortabilityReader.read(zip, entries, sink) YoutubeTakeoutHtmlPortabilityReader.read(zip, entries, sink) - YoutubeTakeoutCsvPortabilityReader.readContent(zip, entries, sink) + entries.filter { YoutubeTakeoutJsonPortabilityReader.isCandidate(it.name) }.forEach { entry -> + zip.getInputStream(entry).use { inputStream -> + YoutubeTakeoutJsonPortabilityReader.read(inputStream, sink) + } + } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt index 3df60b25..4b1c5082 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt @@ -12,7 +12,7 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import java.util.Locale import java.util.UUID -internal object SubscriptionGroupBackupRepository { +object SubscriptionGroupBackupRepository { suspend fun export( userId: String, subscriptionUrls: Set, diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt index b8209cb9..18a89cfc 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -4,7 +4,7 @@ import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.models.TypeTypeBackupItem import dev.typetype.server.models.TypeTypeRestoreSummary -internal object TypeTypeBackupRestoreWriter { +object TypeTypeBackupRestoreWriter { suspend fun restore( userId: String, backup: TypeTypeBackupItem, diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt new file mode 100644 index 00000000..e6461848 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt @@ -0,0 +1,168 @@ +package dev.typetype.server.services + +object YoutubeTakeoutActivityClassifier { + private val watchedPhrases = setOf( + "You watched", + "Viewed", + "Vous avez regardé", + "Has visto", + "Has visto el vídeo", + "Ha visto", + "Você assistiu", + "Você assistiu a", + "Você assistiu ao vídeo", + "hai guardato", + "Hai guardato", + "Hai guardato il video", + "İzlediniz", + "du hast angesehen", + "Du hast dir angesehen", + "Du hast dir das angesehen", + "je hebt gekeken", + "je hebt gekeken naar", + "obejrzałeś", + "obejrzałaś", + "obejrzano", + "вы смотрели", + "вы посмотрели", + "ви дивилися", + "ви переглянули", + "du tittade på", + "du har sett", + "du så på", + "katsoit", + "sledovali jste", + "ai vizionat", + "anda menonton", + "bạn đã xem", + "आपने देखा", + "あなたが視聴した動画", + "視聴した動画", + "視聴しました", + "動画を視聴しました", + "시청한 동영상", + "동영상을 시청했습니다", + "已观看", + "已觀看", + "观看了", + "觀看了", + "شاهدت", + "لقد شاهدت", + ) + + private val watchedSuffixes = setOf( + "izlənildi", + "を視聴しました", + "を再生しました", + ) + + private val likedPhrases = setOf( + "You liked", + "Liked", + "Vous avez aimé", + "A aimé", + "te ha gustado", + "te gustó", + "ha gustado", + "Você gostou", + "Você curtiu", + "Você marcou como gostei", + "hai messo mi piace", + "ti è piaciuto", + "Beğendiniz", + "gefällt mir", + "dir gefällt", + "je hebt dit leuk gevonden", + "podobał ci się", + "понравилось", + "вы поставили отметку нравится", + "сподобалося", + "du gillade", + "du kunne godt lide", + "du likte", + "pidit", + "líbilo se vám", + "ți-a plăcut", + "anda menyukai", + "bạn đã thích", + "आपको पसंद आया", + "高く評価しました", + "좋아요 표시함", + "点赞了", + "按讚", + "您喜欢了", + "أعجبك", + ) + + private val subscribedPhrases = setOf( + "You subscribed to", + "Vous vous êtes abonné à", + "te has suscrito a", + "Você se inscreveu em", + "Você se inscreveu no canal", + "ti sei iscritto a", + "ti sei iscritta a", + "Abone oldunuz", + "du hast abonniert", + "du hast den kanal abonniert", + "je hebt je geabonneerd op", + "zasubskrybowałeś", + "zasubskrybowałaś", + "вы подписались на", + "вы подписались на канал", + "ви підписалися на", + "du prenumererade på", + "du abonnerede på", + "du abonnerte på", + "tilasit", + "te-ai abonat la", + "anda berlangganan", + "bạn đã đăng ký", + "आपने सदस्यता ली", + "登録しました", + "チャンネル登録しました", + "구독함", + "구독했습니다", + "已订阅", + "已訂閱", + "订阅了", + "訂閱了", + "اشتركت في", + "لقد اشتركت في", + ) + private val normalizedWatchedPhrases = watchedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedWatchedSuffixes = watchedSuffixes.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedLikedPhrases = likedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedSubscribedPhrases = subscribedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + + fun isWatched(value: String): Boolean = containsAny(value, normalizedWatchedPhrases) + + fun isWatchedAction(value: String): Boolean = startsWithAny(value, normalizedWatchedPhrases) || + normalizedWatchedSuffixes.any { YoutubeTakeoutTextNormalizer.normalize(value).endsWith(" $it") } + + fun isLiked(value: String): Boolean = containsAny(value, normalizedLikedPhrases) + + fun isLikedAction(value: String): Boolean = startsWithAny(value, normalizedLikedPhrases) + + fun isSubscribed(value: String): Boolean = containsAny(value, normalizedSubscribedPhrases) + + fun isSubscribedAction(value: String): Boolean = startsWithAny(value, normalizedSubscribedPhrases) + + val watchedPattern: String = pattern(watchedPhrases) + + val likedPattern: String = pattern(likedPhrases) + + val subscribedPattern: String = pattern(subscribedPhrases) + + private fun containsAny(value: String, phrases: Set): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(value) + return phrases.any { YoutubeTakeoutTextNormalizer.normalize(it) in normalized } + } + + private fun startsWithAny(value: String, phrases: Set): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(value) + return phrases.any { normalized == it || normalized.startsWith("$it ") } + } + + private fun pattern(phrases: Set): String = phrases.joinToString("|") { Regex.escape(it) } +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt new file mode 100644 index 00000000..4a520859 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.FavoriteItem +import dev.typetype.server.models.SubscriptionItem +import java.nio.file.Path +import java.util.zip.ZipFile + +object YoutubeTakeoutActivitySignalService { + private val activityLinkRegex = Regex("""]*>([^<]*)""", RegexOption.IGNORE_CASE) + private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) + private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/|live/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") + private val channelUrlRegex = Regex("""https?://www\.youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+)""") + private val spacesRegex = Regex("""\s+""") + + fun parse(zipPath: Path): Pair, List> { + val subscriptions = mutableListOf() + val favorites = mutableListOf() + ZipFile(zipPath.toFile()).use { zip -> + zip.entries().asSequence().forEach { entry -> + if (entry.isDirectory || !YoutubeTakeoutPathHints.isYoutubeHtml(entry.name) || isHistoryEntry(entry.name)) return@forEach + val html = zip.getInputStream(entry).bufferedReader().use { it.readText() }.replace("\u00a0", " ") + val parsed = parseHtml(html) + subscriptions += parsed.first + favorites += parsed.second + } + } + return subscriptions.distinctBy { it.channelUrl } to favorites.distinctBy { it.videoUrl } + } + + internal fun parseHtml(html: String): Pair, List> = + parseLinks(html) + + private fun parseLinks(html: String): Pair, List> { + val resolvedHtml = html.replace("\u00a0", " ") + val subscriptions = mutableListOf() + val favorites = mutableListOf() + activityLinkRegex.findAll(resolvedHtml).forEach { match -> + val prefixStart = (match.range.first - PREFIX_CHARS).coerceAtLeast(0) + val prefix = resolvedHtml.substring(prefixStart, match.range.first) + val href = decode(match.groupValues[1]) + val text = decode(match.groupValues[2]) + channelUrlRegex.find(href)?.value?.let { channelUrl -> + if (YoutubeTakeoutActivityClassifier.isSubscribed(prefix)) { + subscriptions += SubscriptionItem(channelUrl.replace("http://", "https://"), text, "") + } + } + val videoUrl = watchUrlRegex.find(href)?.value?.replace("http://", "https://") ?: return@forEach + if (!YoutubeTakeoutActivityClassifier.isLiked(prefix) || YoutubeTakeoutUnavailableItem.matches(text)) return@forEach + val tailStart = match.range.last + 1 + val tail = rowTailRegex.find(resolvedHtml, tailStart)?.takeIf { it.range.first == tailStart } ?: return@forEach + val favoritedAt = YoutubeTakeoutDateParser.parseEpochMillis(decode(tail.groupValues[3])) + ?: return@forEach + favorites += FavoriteItem( + videoUrl = videoUrl, + favoritedAt = favoritedAt, + title = text, + channelName = decode(tail.groupValues[2]), + channelUrl = decode(tail.groupValues[1]).replace("http://", "https://"), + ) + } + return subscriptions to favorites + } + + private fun decode(value: String): String { + return value + .replace(" ", " ") + .replace("'", "'") + .replace("&", "&") + .replace(""", "\"") + .replace("<", "<") + .replace(">", ">") + .replace(spacesRegex, " ") + .trim() + } + + private fun isHistoryEntry(name: String): Boolean = YoutubeTakeoutPathHints.isHistoryEntry(name) + + private const val PREFIX_CHARS = 160 +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt new file mode 100644 index 00000000..4b6d0395 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -0,0 +1,206 @@ +package dev.typetype.server.services + +import java.time.Instant +import java.time.LocalDate +import java.time.Month +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeFormatterBuilder +import java.time.format.TextStyle +import java.util.Locale + +object YoutubeTakeoutDateParser { + private val ACTIVITY_PATTERNS = listOf( + "d MMM yyyy, HH:mm:ss z", + "d MMM yyyy, H:mm:ss z", + "d MMMM yyyy, HH:mm:ss z", + "d MMMM yyyy, H:mm:ss z", + "d. MMM yyyy, HH:mm:ss z", + "d. MMM yyyy, H:mm:ss z", + "d. MMMM yyyy, HH:mm:ss z", + "d. MMMM yyyy, H:mm:ss z", + "MMM d, yyyy, HH:mm:ss z", + "MMMM d, yyyy, HH:mm:ss z", + "d MMM yyyy, h:mm:ss a z", + "d MMMM yyyy, h:mm:ss a z", + "MMM d, yyyy, h:mm:ss a z", + "MMMM d, yyyy, h:mm:ss a z", + "d MMM yyyy HH:mm:ss z", + "d MMMM yyyy HH:mm:ss z", + "MMM d, yyyy HH:mm:ss z", + "MMMM d, yyyy HH:mm:ss z", + "d/M/uuuu, HH:mm:ss z", + "M/d/uuuu, HH:mm:ss z", + "d-M-uuuu, HH:mm:ss z", + "M-d-uuuu, HH:mm:ss z", + "d.M.uuuu, HH:mm:ss z", + "M.d.uuuu, HH:mm:ss z", + "yyyy/M/d, HH:mm:ss z", + "yyyy/M/d, H:mm:ss z", + "yyyy/M/d HH:mm:ss z", + "yyyy/M/d H:mm:ss z", + "yyyy-MM-dd, HH:mm:ss z", + "yyyy-MM-dd, H:mm:ss z", + "yyyy-MM-dd HH:mm:ss z", + "yyyy-MM-dd H:mm:ss z", + "yyyy.MM.dd, HH:mm:ss z", + "yyyy.MM.dd HH:mm:ss z", + "yyyy年M月d日 HH:mm:ss z", + "yyyy年M月d日 H:mm:ss z", + "yyyy年M月d日 a h:mm:ss z", + "yyyy년 M월 d일 HH:mm:ss z", + "yyyy년 M월 d일 H:mm:ss z", + "yyyy년 M월 d일 a h:mm:ss z", + "d M月 yyyy, HH:mm:ss z", + "d M月 yyyy HH:mm:ss z", + "d M월 yyyy, HH:mm:ss z", + "d M월 yyyy HH:mm:ss z", + ) + private val canonicalActivityFormatters = ACTIVITY_PATTERNS.map(::formatter) + private val activityFormatters by lazy { + localeCandidates().flatMap { locale -> ACTIVITY_PATTERNS.map { formatter(it, locale) } } + } + private val monthAliases by lazy(::buildMonthAliases) + + fun parseEpochMillis(value: String): Long? { + val trimmed = value.replace("\u00a0", " ").trim() + if (trimmed.isBlank()) return null + parseOffset(trimmed)?.let { return it } + parseInstant(trimmed)?.let { return it } + parseDate(trimmed)?.let { return it } + return parseActivityDate(trimmed) + } + + private fun parseOffset(value: String): Long? = runCatching { + OffsetDateTime.parse(value).toInstant().toEpochMilli() + }.getOrNull() + + private fun parseInstant(value: String): Long? = runCatching { + Instant.parse(value).toEpochMilli() + }.getOrNull() + + private fun parseDate(value: String): Long? = runCatching { + LocalDate.parse(value).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli() + }.getOrNull() + + private fun parseActivityDate(value: String): Long? { + val normalized = normalizeActivityText(value) + val canonical = canonicalizeMonths(normalized) + canonicalActivityFormatters.asSequence() + .mapNotNull { parseWith(canonical, it) } + .firstOrNull() + ?.let { return it } + if (!hasKnownMonth(normalized)) return null + return activityFormatters.asSequence().mapNotNull { parseWith(normalized, it) }.firstOrNull() + } + + private fun hasKnownMonth(value: String): Boolean = MONTH_TOKEN_REGEX.findAll(value) + .any { monthAliases.containsKey(normalizeMonth(it.value)) } + + private fun normalizeActivityText(value: String): String = value + .replace('\u060C', ',') + .replace('\uFF0C', ',') + .replace(DAY_PERIOD_REGEX) { DAY_PERIODS[it.value] ?: it.value } + .replace(ACTIVITY_CONNECTOR_REGEX, " ") + .replace(ACTIVITY_SPACES_REGEX, " ") + .trim() + + private fun parseWith(value: String, formatter: DateTimeFormatter): Long? = runCatching { + ZonedDateTime.parse(value, formatter).toInstant().toEpochMilli() + }.getOrNull() + + private fun canonicalizeMonths(value: String): String = MONTH_TOKEN_REGEX.replace(value) { match -> + val token = normalizeMonth(match.value) + if (token in DAY_PERIOD_TOKENS) match.value else monthAliases[token] ?: match.value + } + + private fun buildMonthAliases(): Map { + val aliases = mutableMapOf>() + localeCandidates().forEach { locale -> + Month.entries.forEach { month -> + TextStyle.entries.filter { it != TextStyle.NARROW }.forEach { style -> + val alias = normalizeMonth(month.getDisplayName(style, locale)) + if (alias.length >= 2) aliases.getOrPut(alias) { mutableSetOf() }.add(month.value) + } + } + } + return aliases.mapNotNull { (alias, months) -> + months.singleOrNull()?.let { alias to Month.of(it).getDisplayName(TextStyle.SHORT, Locale.ENGLISH) } + }.toMap() + } + + private fun normalizeMonth(value: String): String = + YoutubeTakeoutTextNormalizer.normalize(value).replace(" ", "") + + private fun localeCandidates(): List { + val preferred = listOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.forLanguageTag("es"), + Locale.GERMAN, + Locale.ITALIAN, + Locale.forLanguageTag("pt"), + Locale.forLanguageTag("tr"), + Locale.forLanguageTag("nl"), + Locale.forLanguageTag("pl"), + Locale.forLanguageTag("ru"), + Locale.forLanguageTag("uk"), + Locale.forLanguageTag("ja"), + Locale.KOREAN, + Locale.SIMPLIFIED_CHINESE, + Locale.TRADITIONAL_CHINESE, + ) + val available = Locale.getAvailableLocales().asSequence() + .filter { it.language.isNotBlank() } + .map { Locale.forLanguageTag(it.language) } + .distinctBy(Locale::getLanguage) + .toList() + return (preferred + available).distinctBy(Locale::toLanguageTag) + } + + private fun formatter(pattern: String): DateTimeFormatter = formatter(pattern, Locale.ENGLISH) + + private fun formatter(pattern: String, locale: Locale): DateTimeFormatter = + DateTimeFormatterBuilder().parseCaseInsensitive().appendPattern(pattern).toFormatter(locale) + + private val MONTH_TOKEN_REGEX = Regex("[\\p{L}\\p{M}][\\p{L}\\p{M}.]*") + private val ACTIVITY_CONNECTOR_REGEX = Regex("\\s+(?:de|del)\\s+", RegexOption.IGNORE_CASE) + private val ACTIVITY_SPACES_REGEX = Regex("\\s+") + private val DAY_PERIOD_REGEX = Regex( + "凌晨|清晨|早晨|早上|上午|中午|下午|晚上|半夜|午前|午後|오전|오후|" + + "صباحًا|صباحا|صباح|مساءً|مساءا|مساء|قبل الظهر|بعد الظهر|" + + "पूर्वाह्न|अपराह्न|सुबह|दोपहर|शाम|रात", + ) + private val DAY_PERIODS = mapOf( + "凌晨" to "AM ", + "清晨" to "AM ", + "早晨" to "AM ", + "早上" to "AM ", + "上午" to "AM ", + "中午" to "PM ", + "下午" to "PM ", + "晚上" to "PM ", + "半夜" to "AM ", + "午前" to "AM ", + "午後" to "PM ", + "오전" to "AM ", + "오후" to "PM ", + "صباحًا" to "AM ", + "صباحا" to "AM ", + "صباح" to "AM ", + "مساءً" to "PM ", + "مساءا" to "PM ", + "مساء" to "PM ", + "قبل الظهر" to "AM ", + "بعد الظهر" to "PM ", + "पूर्वाह्न" to "AM ", + "अपराह्न" to "PM ", + "सुबह" to "AM ", + "दोपहर" to "PM ", + "शाम" to "PM ", + "रात" to "PM ", + ) + private val DAY_PERIOD_TOKENS = setOf("am", "pm") +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt new file mode 100644 index 00000000..4dcee75e --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.HistoryItem + +object YoutubeTakeoutHistoryParser { + private val videoLinkRegex = Regex( + """]*>([^<]*)""", + RegexOption.IGNORE_CASE, + ) + private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) + private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/|live/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") + private val tagRegex = Regex("<[^>]+>") + private val spacesRegex = Regex("\\s+") + + fun parse(html: String, requireWatchedMarker: Boolean = true): List = + parseWithDiagnostics(html, requireWatchedMarker).items + + fun parseWithDiagnostics(html: String, requireWatchedMarker: Boolean = true): YoutubeTakeoutHistoryParseResult { + val resolvedHtml = html.replace("\u00a0", " ") + var invalidDates = 0L + val items = videoLinkRegex.findAll(resolvedHtml).mapNotNull { match -> + val prefixStart = (match.range.first - WATCHED_PREFIX_CHARS).coerceAtLeast(0) + if (requireWatchedMarker && !YoutubeTakeoutActivityClassifier.isWatched( + resolvedHtml.substring(prefixStart, match.range.first), + ) + ) { + return@mapNotNull null + } + val tailStart = match.range.last + 1 + val tail = rowTailRegex.matchAt(resolvedHtml, tailStart) ?: return@mapNotNull null + val url = extractUrl(match.groupValues[1]) ?: return@mapNotNull null + val title = decode(match.groupValues[2]) + if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null + val channelUrl = tail.groupValues[1].takeIf { it.isNotBlank() }.orEmpty() + val channelName = decode(tail.groupValues[2]).ifBlank { "Unknown channel" } + val dateText = decode(tail.groupValues[3]) + val watchedAt = parseDate(dateText) ?: run { + invalidDates += 1 + return@mapNotNull null + } + HistoryItem( + url = url, + title = title, + thumbnail = "", + channelName = channelName, + channelUrl = channelUrl, + channelAvatar = "", + duration = 0, + progress = 0, + watchedAt = watchedAt, + ) + }.toList().distinctBy { it.url to it.watchedAt } + return YoutubeTakeoutHistoryParseResult(items, invalidDates) + } + + private fun parseDate(value: String): Long? = YoutubeTakeoutDateParser.parseEpochMillis(value) + + private fun extractUrl(value: String): String? { + val decoded = decode(value) + val direct = urlRegex.find(decoded)?.value + if (direct != null) return direct + val fromText = urlRegex.find(decode(tagRegex.replace(decoded, " ")))?.value + return fromText + } + + private fun decode(value: String): String = tagRegex.replace(value, " ") + .replace(" ", " ") + .replace("'", "'") + .replace("&", "&") + .replace(""", "\"") + .replace("<", "<") + .replace(">", ">") + .replace(spacesRegex, " ") + .trim() + + private const val WATCHED_PREFIX_CHARS = 160 +} + +data class YoutubeTakeoutHistoryParseResult( + val items: List, + val invalidDates: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt new file mode 100644 index 00000000..692bf522 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.channels.Channel +import java.util.concurrent.atomic.AtomicInteger + +class YoutubeTakeoutImportProgress(private val total: Long) { + private val updates = Channel(Channel.UNLIMITED) + private val published = AtomicInteger(-1) + + fun offer(processed: Long) { + val percent = if (total <= 0L) 99 else { + ((processed.toDouble() / total.toDouble()) * 99.0).toInt().coerceIn(0, 99) + } + publish(percent) + } + + fun finish() { + publish(99) + } + + fun close() { + updates.close() + } + + suspend fun drain(publish: suspend (Int) -> Unit) { + var applied = -1 + for (value in updates) { + if (value <= applied) continue + applied = value + publish(value) + } + } + + private fun publish(percent: Int) { + while (true) { + val current = published.get() + if (percent <= current) return + if (published.compareAndSet(current, percent)) { + updates.trySend(percent) + return + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt new file mode 100644 index 00000000..9ec0ac7b --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +object YoutubeTakeoutPathHints { + fun isYoutubeHtml(path: String): Boolean = + path.endsWith(".html", ignoreCase = true) && "youtube" in path.lowercase() + + fun isHistoryEntry(path: String): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(path) + return NORMALIZED_HISTORY_MARKERS.any { it in normalized } + } + + private val HISTORY_MARKERS = setOf( + "watch history", + "historique", + "historique des videos regardees", + "historico", + "historico de visualizacao", + "historico de exibicao", + "historial", + "historial de reproduccion", + "historial de visualizacion", + "cronologia", + "cronologia de reproduccion", + "verlauf", + "wiedergabeverlauf", + "kijkgeschiedenis", + "historia ogladania", + "izleme gecmisi", + "lich su xem", + "ประวัติการดู", + "ประวัติการรับชม", + "история просмотров", + "история просмотра", + "історія перегляду", + "視聴履歴", + "再生履歴", + "시청 기록", + "观看记录", + "觀看記錄", + "سجل المشاهدة", + "تاریخچه تماشا", + "देखने का इतिहास", + ) + + private val NORMALIZED_HISTORY_MARKERS = HISTORY_MARKERS.map(YoutubeTakeoutTextNormalizer::normalize) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt index d9cf8d5f..98ab2f14 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt @@ -88,7 +88,7 @@ object YoutubeTakeoutRowParser { value.isNotBlank() && !YoutubeTakeoutSchemaHints.looksLikeChannelId(value) && !YoutubeTakeoutSchemaHints.looksLikePlaylistId(value) && - !YoutubeTakeoutSchemaHints.looksLikeVideoId(value) && + !YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(value) && !YoutubeTakeoutSchemaHints.containsChannelUrl(value) && !YoutubeTakeoutSchemaHints.containsWatchUrl(value) && !isMetadataValue(value) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt similarity index 50% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt index e2a784c7..1d758272 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt @@ -5,38 +5,16 @@ object YoutubeTakeoutSchemaHints { private val videoIdRegex = Regex("""^[A-Za-z0-9_-]{6,}$""") private val playlistIdRegex = Regex("""^(PL|UU|LL|RD|OLAK5uy_)[A-Za-z0-9_-]{6,}$""") private val channelUrlRegex = Regex("""youtube\.com/(channel/|@|c/|user/)""", RegexOption.IGNORE_CASE) - private val titleWords = setOf( - "title", - "titles", - "titre", - "titres", - "titulo", - "titulos", - "baslik", - "baslık", - "başlık", - "basligi", - "baslıgı", - "başlığı", - "adi", - "adı", - "nombre", - "nome", - "nom", - "name", - "names", - "titel", - "titolo", - ) - private val playlistWords = setOf("playlist", "playlists", "oynatma listesi", "oynatma listeleri") - private val channelWords = setOf("channel", "chaine", "canal", "kanal", "kanaal", "канал", "チャンネル", "채널", "频道", "頻道") - + private val titleWords = YoutubeTakeoutSchemaTerms.titleWords + private val playlistWords = YoutubeTakeoutSchemaTerms.playlistWords + private val channelWords = YoutubeTakeoutSchemaTerms.channelWords + private val videoWords = YoutubeTakeoutSchemaTerms.videoWords fun isChannelIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "channel id" || normalized == "id des chaines" || normalized == "id de la chaine" || - (channelWords.any { it in normalized } && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (channelWords.any { normalize(it) in normalized } && hasIdentifier(normalized)) } fun isChannelUrlHeader(value: String): Boolean { @@ -44,47 +22,48 @@ object YoutubeTakeoutSchemaHints { return normalized == "channel url" || normalized == "url des chaines" || normalized == "url de la chaine" || - ("url" in words(normalized) && channelWords.any { it in normalized }) + ("url" in words(normalized) && channelWords.any { normalize(it) in normalized }) } fun isChannelTitleHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "titres des chaines" || normalized == "titre de la chaine" || - titleWords.any { it in words(normalized) } && - (channelWords.any { it in normalized } || normalized in titleWords) + titleWords.any { normalize(it) in normalized } && + (channelWords.any { normalize(it) in normalized } || titleWords.any { normalize(it) == normalized }) } fun isPlaylistIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "playlist id" || normalized == "id de la playlist" || - (isPlaylistText(normalized) && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (isPlaylistText(normalized) && hasIdentifier(normalized)) } fun isPlaylistTitleHeader(value: String): Boolean { val normalized = normalize(value) - return isPlaylistText(normalized) && titleWords.any { it in words(normalized) } + return isPlaylistText(normalized) && titleWords.any { normalize(it) in normalized } } fun isVideoIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "video id" || normalized == "id video" || - ("video" in normalized && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (videoWords.any { normalize(it) in normalized } && hasIdentifier(normalized)) } fun isVideoTitleHeader(value: String): Boolean { val normalized = normalize(value) - return "video" in normalized && titleWords.any { it in words(normalized) } + return videoWords.any { normalize(it) in normalized } && titleWords.any { normalize(it) in normalized } } fun isPlaylistItemAddedAtHeader(value: String): Boolean { val normalized = normalize(value) - val parts = words(normalized) - return ("added" in parts && parts.any { it == "at" || it == "date" || it == "time" || it == "timestamp" }) || - ("creation" in parts && "video" in parts) || - ("created" in parts && "at" in parts) || - ("timestamp" in parts && ("playlist" in normalized || "video" in normalized)) + val hasTemporalWord = TEMPORAL_WORDS.any { normalize(it) in normalized } + return hasTemporalWord && ( + isPlaylistText(normalized) || videoWords.any { normalize(it) in normalized } || + "added at" in normalized || "date added" in normalized || "created at" in normalized || + normalized == "timestamp" + ) } fun isUrlHeader(value: String): Boolean = "url" in words(normalize(value)) @@ -96,13 +75,16 @@ object YoutubeTakeoutSchemaHints { return videoIdRegex.matches(trimmed) && !looksLikeChannelId(trimmed) && !looksLikePlaylistId(trimmed) } + fun looksLikeLikelyVideoId(value: String): Boolean = value.trim().length >= 10 && looksLikeVideoId(value) + fun looksLikePlaylistId(value: String): Boolean = playlistIdRegex.matches(value.trim()) fun containsChannelUrl(value: String): Boolean = channelUrlRegex.containsMatchIn(value) fun containsWatchUrl(value: String): Boolean { val normalized = value.lowercase() - return "youtube.com/watch?v=" in normalized || "youtube.com/shorts/" in normalized || "youtu.be/" in normalized + return "youtube.com/watch?v=" in normalized || "youtube.com/shorts/" in normalized || + "youtube.com/live/" in normalized || "youtu.be/" in normalized } fun normalize(value: String): String = YoutubeTakeoutTextNormalizer.normalize(value) @@ -112,5 +94,28 @@ object YoutubeTakeoutSchemaHints { return playlistWords.any { normalize(it) in normalized } } + fun isPlaylistManifestName(value: String): Boolean = + YoutubeTakeoutSchemaTerms.playlistManifestNames.contains(normalize(value)) + + fun isSubscriptionText(value: String): Boolean = + YoutubeTakeoutSchemaTerms.subscriptionWords.any { normalize(it) in normalize(value) } + + private fun hasIdentifier(normalized: String): Boolean { + val parts = words(normalized) + return "id" in parts || "identifier" in parts || "identificador" in parts || + "identifiant" in parts || "kimligi" in parts || "kimliği" in parts || + "идентификатор" in normalized || "identyfikator" in parts || "identifikator" in parts || + "kennung" in parts || "識別子" in normalized || "標識符" in normalized || + "标识符" in normalized || "معرف" in normalized + } + private fun words(value: String): Set = value.split(' ').filter { it.isNotBlank() }.toSet() + + private val TEMPORAL_WORDS = setOf( + "added", "date", "time", "timestamp", "creation", "created", "fecha", "marca de tiempo", "creacion", + "ajoute", "datum", "zeit", "erstellt", "data", "ora", "creazione", "criacao", "eklenme", "olusturma", + "дата", "время", "создания", "добавления", "日時", "时间", "日期", "创建", "추가", "생성", "التاريخ", "وقت", + "إضافة", "बनाने", "समय", "ημερομηνία", "เวลา", + ) + } diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt new file mode 100644 index 00000000..d20e8dbc --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +object YoutubeTakeoutSchemaTerms { + val titleWords = setOf( + "title", "titles", "titre", "titres", "titulo", "titulos", "baslik", "baslık", "başlık", + "basligi", "baslıgı", "başlığı", "adi", "adı", "nombre", "nome", "nom", "name", "names", + "titel", "titolo", "titulo original", "tytul", "название", "наименование", "название видео", + "動画タイトル", "動画のタイトル", "제목", "视频标题", "影片標題", "naam", "عنوان", "शीर्षक", "τίτλος", + ) + + val playlistWords = setOf( + "playlist", "playlists", "oynatma listesi", "oynatma listeleri", "playlistler", + "lista de reproduccion", "listas de reproduccion", "liste de lecture", "listes de lecture", + "lista di riproduzione", "liste di riproduzione", "lista de reproducao", "listas de reproducao", + "wiedergabeliste", "wiedergabelisten", "afspeellijst", "afspeellijsten", + "список воспроизведения", "списки воспроизведения", "плейлист", "плейлисты", "再生リスト", + "재생목록", "播放列表", "播放清單", "قائمة التشغيل", "قوائم التشغيل", "प्लेलिस्ट", "प्लेलिस्टें", + "λίστα αναπαραγωγής", "λίστες αναπαραγωγής", "danh sach phat", "รายการเล่น", "เพลย์ลิสต์", + "רשימת השמעה", "רשימות השמעה", + ) + + val channelWords = setOf( + "channel", "chaine", "canal", "canale", "kanal", "kanaal", "канал", "канали", "kanał", + "チャンネル", "채널", "频道", "頻道", "قناة", "चैनल", "κανάλι", "kenh", "ช่อง", "ערוץ", + ) + + val videoWords = setOf( + "video", "videos", "vidéo", "vidéos", "vídeo", "vídeos", "動画", "동영상", "비디오", "видео", + "视频", "影片", "film", "filmy", "فيديو", "فيديوهات", "वीडियो", "βίντεο", "วิดีโอ", "ভিডিও", "סרטון", + ) + + val subscriptionWords = setOf( + "subscriptions", "suscripciones", "inscricoes", "inscrições", "iscrizioni", "abbonamenti", "abos", + "abonelikler", "abonnements", "abonnementen", "abonnierte kanäle", "subskrypcje", "подписки", + "підписки", "登録チャンネル", "チャンネル登録", "구독", "订阅", "訂閱", "الاشتراكات", + "การสมัครรับข้อมูล", "การติดตาม", "מינויים", "সাবস্ক্রিপশন", "kenh da dang ky", + ) + + val playlistManifestNames = setOf( + "playlists", "oynatma listeleri", "oynatma listesi", "listas de reproduccion", "listes de lecture", + "liste di riproduzione", "listas de reproducao", "wiedergabelisten", "wiedergabeliste", + "afspeellijsten", "afspeellijst", "списки воспроизведения", "список воспроизведения", "再生リスト", + "재생목록", "播放列表", "播放清單", "قوائم التشغيل", "قائمة التشغيل", "danh sach phat", + "รายการเล่น", "เพลย์ลิสต์", "רשימת השמעה", "רשימות השמעה", + ).mapTo(mutableSetOf(), YoutubeTakeoutTextNormalizer::normalize) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt similarity index 69% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt index 791bc126..13dcaa89 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt @@ -13,11 +13,21 @@ object YoutubeTakeoutSystemPlaylist { "daha sonra izlenecekler", "spater ansehen", "später ansehen", + "later bekijken", + "obejrzyj pozniej", "посмотреть позже", + "смотреть позже", "後で見る", "나중에 볼 동영상", "稍后观看", "稍後觀看", + "بعدا تماشا کنید", + "مشاهدة لاحقا", + "المشاهدة لاحقا", + "переглянути пізніше", + "बाद में देखें", + "ดูภายหลัง", + "παρακολούθηση αργότερα", ) private val likedPhrases = setOf( @@ -28,7 +38,10 @@ object YoutubeTakeoutSystemPlaylist { "videos que me gustan", "videos que gostei", "videos curtidos", + "videos de que gostei", + "videos que eu gostei", "video piaciuti", + "video che mi piacciono", "begendigim videolar", "begendiğim videolar", "beğendigim videolar", @@ -37,11 +50,19 @@ object YoutubeTakeoutSystemPlaylist { "beğenilen videolar", "mit gefallt", "gefällt mir", + "videos die mir gefallen", + "leuke videos", + "polubione filmy", "понравившиеся видео", + "понравились видео", "高く評価した動画", "좋아요 표시한 동영상", "喜欢的视频", "喜歡的影片", + "الفيديوهات التي أعجبتني", + "वीडियो जो मुझे पसंद हैं", + "βίντεο που μου αρέσουν", + "улюблені відео", ) fun canonicalKey(value: String): String? { diff --git a/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt new file mode 100644 index 00000000..4520edf7 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt @@ -0,0 +1,15 @@ +package dev.typetype.server.services + +import java.text.Normalizer +import java.util.Locale + +object YoutubeTakeoutTextNormalizer { + fun normalize(value: String): String = Normalizer + .normalize(value.lowercase(Locale.ROOT), Normalizer.Form.NFD) + .replace(MARKS_REGEX, "") + .replace(NON_ALPHANUMERIC_REGEX, " ") + .trim() + + private val MARKS_REGEX = Regex("\\p{M}+") + private val NON_ALPHANUMERIC_REGEX = Regex("[^\\p{L}\\p{N}]+") +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt index 808d72b7..7740a3c9 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object YoutubeTakeoutUnavailableItem { +object YoutubeTakeoutUnavailableItem { fun matches(title: String): Boolean = YoutubeTakeoutTextNormalizer.normalize(title) in TITLES diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt index c1fe13ec..66cdbace 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt @@ -29,7 +29,7 @@ object YoutubeTakeoutZipScanner { if (subscriptionsHeader.isEmpty()) subscriptionsHeader = header subscriptionsRows += rows } - isPlaylistsHeader(entry.name, header, rows) -> { + isPlaylistsHeader(header, rows) -> { if (playlistsHeader.isEmpty()) playlistsHeader = header playlistsRows += rows } @@ -66,15 +66,22 @@ object YoutubeTakeoutZipScanner { return hasId && (hasUrl || (hasTitle && isSubscriptionPath(path))) } - private fun isPlaylistsHeader(path: String, header: List, rows: List>): Boolean { + private fun isPlaylistsHeader(header: List, rows: List>): Boolean { val hasId = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikePlaylistId) - val hasTitle = header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) - return hasId && hasTitle && isMainPlaylistsFile(path) + val hasTitle = header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) || rows.hasValue(::looksLikeTextValue) + val hasVideo = header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeLikelyVideoId) + return hasId && hasTitle && !hasVideo } private fun isPlaylistItemsEntry(path: String, header: List, rows: List>): Boolean { - if ((!isPlaylistPath(path) && !isPlaylistItemsFile(path)) || isMainPlaylistsFile(path)) return false - return header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeVideoId) + if (isMainPlaylistsFile(path)) return false + val hasVideo = header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeVideoId) + val hasPlaylistKey = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + val hasAddedAt = header.any(YoutubeTakeoutSchemaHints::isPlaylistItemAddedAtHeader) || + rows.hasValue { YoutubeTakeoutDateParser.parseEpochMillis(it) != null } + val compactPlaylistRows = header.size <= 3 && hasAddedAt + return hasVideo && (isPlaylistPath(path) || isPlaylistItemsFile(path) || hasPlaylistKey || compactPlaylistRows) } private fun extractPlaylistSourceKey(path: String, header: List): String? { @@ -84,8 +91,7 @@ object YoutubeTakeoutZipScanner { } private fun isSubscriptionPath(path: String): Boolean { - val normalized = YoutubeTakeoutSchemaHints.normalize(path) - return SUBSCRIPTION_PATH_MARKERS.any { it in normalized } + return YoutubeTakeoutSchemaHints.isSubscriptionText(path) } private fun isPlaylistPath(path: String): Boolean = YoutubeTakeoutSchemaHints.isPlaylistText(path) @@ -95,11 +101,20 @@ object YoutubeTakeoutZipScanner { private fun isMainPlaylistsFile(path: String): Boolean = path.substringAfterLast('/').substringBeforeLast('.').let { fileName -> - val normalized = YoutubeTakeoutSchemaHints.normalize(fileName) - normalized == "playlists" || normalized == "oynatma listeleri" + YoutubeTakeoutSchemaHints.isPlaylistManifestName(fileName) } private fun List>.hasValue(predicate: (String) -> Boolean): Boolean = any { row -> row.any(predicate) } - private val SUBSCRIPTION_PATH_MARKERS = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") + private fun looksLikeTextValue(value: String): Boolean { + val trimmed = value.trim() + return trimmed.isNotBlank() && + !YoutubeTakeoutSchemaHints.looksLikeChannelId(trimmed) && + !YoutubeTakeoutSchemaHints.looksLikePlaylistId(trimmed) && + !YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(trimmed) && + !YoutubeTakeoutSchemaHints.containsChannelUrl(trimmed) && + !YoutubeTakeoutSchemaHints.containsWatchUrl(trimmed) && + YoutubeTakeoutDateParser.parseEpochMillis(trimmed) == null + } + } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt similarity index 87% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt index 845f712d..c5ae9bca 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt +++ b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt @@ -44,6 +44,19 @@ class YoutubeTakeoutActivitySignalServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse omits favorites with an unknown activity date`() { + val zip = createZip( + """ + You liked Video
+ 8 Foo 2026, 19:11:02 CEST
+ """.trimIndent(), + ) + + assertTrue(YoutubeTakeoutActivitySignalService.parse(zip).second.isEmpty()) + Files.deleteIfExists(zip) + } + private fun createZip( html: String = """ Vous vous êtes abonné à Channel Name
diff --git a/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt new file mode 100644 index 00000000..7f8fef4a --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt @@ -0,0 +1,23 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutImportProgress +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class YoutubeTakeoutImportProgressTest { + @Test + fun `publishes increasing percentages once and reaches ninety nine`() = runBlocking { + val progress = YoutubeTakeoutImportProgress(total = 10) + val observed = mutableListOf() + val draining = launch { progress.drain { observed += it } } + + repeat(10) { progress.offer(it + 1L) } + progress.finish() + progress.close() + draining.join() + + assertEquals((9..99 step 10).toList(), observed) + } +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt diff --git a/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt new file mode 100644 index 00000000..13e46cc2 --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt @@ -0,0 +1,14 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutPathHints +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class YoutubeTakeoutPathHintsTest { + @Test + fun `recognizes localized history paths after accent normalization`() { + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/lịch sử xem.html")) + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/ประวัติการรับชม.html")) + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/Wiedergabeverlauf.html")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt similarity index 94% rename from src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt index 4df0f853..83298256 100644 --- a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt @@ -41,6 +41,9 @@ class PortabilityEngineTest { ) val completed = awaitState(engine, "owner-a", started.id, PortabilityJobState.COMPLETED) assertEquals(1L, completed.result?.get("subscriptions")) + assertEquals(PortabilityProgressUnit.RECORDS, completed.progress?.unit) + assertEquals(1L, completed.progress?.processed) + assertEquals(1L, completed.progress?.total) engine.close() } @@ -187,8 +190,13 @@ private class FakeDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map = source.counts().mapKeys { it.key.wireName }.also { result -> - request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + request.categories.forEach { category -> + val count = result[category.wireName] ?: 0L + onCategoryProgress(category, count) + onCategoryComplete(category, count) + } } override suspend fun export( diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt new file mode 100644 index 00000000..2e9370c5 --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityProgressTest { + @TempDir + lateinit var directory: Path + + @Test + fun `progress interval publishes every record for small imports`() { + val job = job() + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.RECORDS, + total = 47L, + interval = portabilityProgressInterval(47L), + ) + + progress.add() + + assertEquals(1L, job.snapshot().progress?.processed) + assertEquals(47L, job.snapshot().progress?.total) + } + + @Test + fun `progress interval stays bounded for large imports`() { + assertEquals(1L, portabilityProgressInterval(0L)) + assertEquals(1L, portabilityProgressInterval(47L)) + assertEquals(8L, portabilityProgressInterval(839L)) + assertEquals(100L, portabilityProgressInterval(42_622L)) + assertEquals(100L, portabilityProgressInterval(null)) + } + + private fun job() = PortabilityJob( + id = "progress-test", + ownerId = "owner", + kind = PortabilityJobKind.IMPORT, + directory = directory, + requestId = null, + clock = System::currentTimeMillis, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt new file mode 100644 index 00000000..72f31a1f --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt @@ -0,0 +1,58 @@ +package dev.typetype.server.portability + +import dev.typetype.server.TEST_USER_ID +import dev.typetype.server.TestDatabase +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PlaylistVideosTable +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class TypeTypePortabilityCoreImportTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `playlist progress counts each source record once`() = runBlocking { + val source = PortabilityBatchRecordSource( + PortabilityCategory.PLAYLISTS, + listOf( + PortabilityPlaylist("favorites", "Favorites"), + PortabilityPlaylistVideo("favorites", 0, video()), + ), + ) + var processed = 0 + + val imported = DatabaseFactory.query { + TypeTypePortabilityCoreImport.write( + TEST_USER_ID, + PortabilityCategory.PLAYLISTS, + source, + PortabilityDuplicatePolicy.SKIP, + ) { processed++ } + } + + assertEquals(2L, imported) + assertEquals(2, processed) + assertEquals(1L, transaction { PlaylistVideosTable.selectAll().count() }) + } + + private fun video() = PortabilityVideo( + url = "https://youtube.com/watch?v=playlist-test", + title = "Playlist video", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://youtube.com/channel/channel", + ) +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt new file mode 100644 index 00000000..74a9da40 --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.portability + +import dev.typetype.server.TEST_USER_ID +import dev.typetype.server.TestDatabase +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.HistoryTable +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.UUID + +class TypeTypePortabilityHistoryImportTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `history import batches records and skips existing and source duplicates`() = runBlocking { + insertExisting(history(0)) + val records = (0..1_000).map(::history) + history(1_000) + var processed = 0L + + val imported = DatabaseFactory.query { + TypeTypePortabilityHistoryImport.write( + TEST_USER_ID, + Source(records), + PortabilityDuplicatePolicy.SKIP, + ) { processed++ } + } + + assertEquals(1_000L, imported) + assertEquals(1_002L, processed) + assertEquals(1_001L, transaction { HistoryTable.selectAll().count() }) + } + + private suspend fun insertExisting(record: PortabilityHistory) { + DatabaseFactory.query { + HistoryTable.insert { + it[HistoryTable.id] = UUID.randomUUID().toString() + it[HistoryTable.userId] = TEST_USER_ID + it[HistoryTable.url] = record.video.url + it[HistoryTable.title] = record.video.title + it[HistoryTable.thumbnail] = record.video.thumbnailUrl + it[HistoryTable.channelName] = record.video.channelName + it[HistoryTable.channelUrl] = record.video.channelUrl + it[HistoryTable.channelAvatar] = record.video.channelAvatarUrl + it[HistoryTable.duration] = record.video.durationSeconds + it[HistoryTable.progress] = record.positionSeconds + it[HistoryTable.watchedAt] = record.watchedAt + } + } + } + + private fun history(index: Int) = PortabilityHistory( + video = PortabilityVideo( + url = "https://video.example/$index", + title = "Video $index", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://channel.example", + ), + watchedAt = index.toLong(), + ) +} + +private class Source(private val records: List) : PortabilityRecordSource { + override fun categories() = setOf(PortabilityCategory.HISTORY) + + override fun counts() = mapOf(PortabilityCategory.HISTORY to records.size.toLong()) + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + if (category == PortabilityCategory.HISTORY) records.forEach(block) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt new file mode 100644 index 00000000..20e2e0e3 --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt @@ -0,0 +1,77 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutCsvSchemaDetectorTest { + @TempDir + lateinit var directory: Path + + @Test + fun `classifies unknown file names from ids urls and row shape`() { + val archive = directory.resolve("takeout-unknown-language.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/section/data-a.csv", + "column-a,column-b,column-c\nUC123456789012,Channel,https://www.youtube.com/channel/UC123456789012\n", + ) + output.entry( + "Takeout/YouTube/section/data-b.csv", + "column-a,column-b\nPL123456789,My list\n", + ) + output.entry( + "Takeout/YouTube/My list.csv", + "column-a,column-b,column-c\nvideo000001,Video,2026-09-16T18:02:08Z\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + val videos = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { videos += it } + assertEquals(1, videos.size) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + + @Test + fun `ignores unrelated Takeout CSV files that contain channel URLs`() { + val archive = directory.resolve("takeout-comments.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/comments/comments.csv", + "Comment,Author channel URL\nHello,https://www.youtube.com/channel/UC123456789012\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(emptyMap(), spool.counts()) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt new file mode 100644 index 00000000..2e54d70e --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutHtmlPortabilityReaderTest { + @TempDir + lateinit var directory: Path + + @Test + fun `reports invalid html dates without creating epoch history`() { + val archive = directory.resolve("takeout-invalid-date.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("Takeout/YouTube/Watch history.html")) + output.write( + ("You watched Video
" + + "16 Foo 2026, 12:00:00 CET
").toByteArray(), + ) + output.closeEntry() + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(0L, spool.counts()[PortabilityCategory.HISTORY] ?: 0L) + val issue = spool.issues().single() + assertEquals("invalid_takeout_date", issue.code) + assertEquals(1L, issue.count) + val history = mutableListOf() + spool.forEach(PortabilityCategory.HISTORY) { history += it } + assertTrue(history.isEmpty()) + } finally { + spool.delete() + } + } +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt new file mode 100644 index 00000000..28cf301e --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt @@ -0,0 +1,141 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class YoutubeTakeoutImportWorkflowTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports subscriptions then recent history and playlists in committed batches`() = runBlocking { + val source = PortabilitySpool.create(directory) + try { + repeat(125) { source.write(PortabilitySubscription("https://youtube.com/channel/UC$it")) } + repeat(1_025) { index -> source.write(history(index)) } + source.write(PortabilityPlaylist("favorites", "Favorites")) + repeat(55) { index -> + source.write( + PortabilityPlaylistVideo( + "favorites", + index, + video(index), + ), + ) + } + + val dataPort = RecordingDataPort() + val job = PortabilityJob( + id = "takeout-workflow", + ownerId = "owner", + kind = PortabilityJobKind.IMPORT, + directory = directory, + requestId = null, + clock = System::currentTimeMillis, + ) + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.RECORDS, + total = 1_206L, + interval = 100L, + ) + var partial = emptyMap() + + val result = YoutubeTakeoutImportWorkflow(dataPort).apply( + "owner", + source, + PortabilityImportRequest( + setOf(PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.HISTORY, PortabilityCategory.PLAYLISTS), + PortabilityDuplicatePolicy.REPLACE, + ), + progress, + ) { + partial = it + job.updateResult(it) + } + + assertEquals( + listOf( + PortabilityCategory.SUBSCRIPTIONS to 50, + PortabilityCategory.SUBSCRIPTIONS to 50, + PortabilityCategory.SUBSCRIPTIONS to 25, + PortabilityCategory.HISTORY to 20, + PortabilityCategory.PLAYLISTS to 1, + PortabilityCategory.PLAYLISTS to 50, + PortabilityCategory.PLAYLISTS to 5, + PortabilityCategory.HISTORY to 500, + PortabilityCategory.HISTORY to 500, + PortabilityCategory.HISTORY to 5, + ), + dataPort.batches.map { it.category to it.records.size }, + ) + assertEquals((1_024 downTo 1_005).map(Int::toLong), dataPort.batches[3].records.map { (it as PortabilityHistory).watchedAt }) + assertEquals((0 until 500).map(Int::toLong), dataPort.batches[7].records.map { (it as PortabilityHistory).watchedAt }) + assertEquals(125L, result["subscriptions"]) + assertEquals(1_025L, result["history"]) + assertEquals(56L, result["playlists"]) + assertEquals(result, partial) + assertEquals(result, job.snapshot().result) + assertEquals(1_206L, job.snapshot().progress?.processed) + assertEquals(1_206L, job.snapshot().progress?.total) + assertEquals(1_005L, job.snapshot().progress?.stageProcessed) + assertEquals(1_005L, job.snapshot().progress?.stageTotal) + assertEquals(10L, job.snapshot().progress?.checkpoint) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches.first().policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[1].policy) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches[3].policy) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches[4].policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[5].policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[7].policy) + } finally { + source.delete() + } + } + + private fun history(index: Int) = PortabilityHistory(video(index), index.toLong()) + + private fun video(index: Int) = PortabilityVideo( + url = "https://youtube.com/watch?v=$index", + title = "Video $index", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://youtube.com/channel/channel", + ) +} + +private class RecordingDataPort : PortabilityDataPort { + val batches = mutableListOf() + + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, + ): Map { + val category = request.categories.single() + val records = buildList { source.forEach(category, ::add) } + batches += Batch(category, records, request.duplicatePolicy) + onCategoryProgress(category, records.size.toLong()) + onCategoryComplete(category, records.size.toLong()) + return mapOf(category.wireName to records.size.toLong()) + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) = Unit + + data class Batch( + val category: PortabilityCategory, + val records: List, + val policy: PortabilityDuplicatePolicy, + ) +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt new file mode 100644 index 00000000..af6624ce --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -0,0 +1,245 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutPortabilityAdapterTest { + @TempDir + lateinit var directory: Path + + @Test + fun `adapter streams takeout categories and keeps playlist order`() { + val archive = directory.resolve("takeout.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube and YouTube Music/subscriptions/subscriptions.csv", + "Channel Id,Channel Url,Channel Title\nUC123456789012,https://youtube.com/channel/UC123456789012,Channel\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/playlists.csv", + "Playlist ID,Playlist Title\nPL123456789,Imported\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Videos de Imported.csv", + "Video ID,Video Title,Video Added Timestamp\nvideo000001,First,2026-01-02T00:00:00Z\nvideo000002,Second,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Watch later.csv", + "Video ID,Video Title\nwatch000001,Later\n", + ) + output.entry( + "Takeout/YouTube and YouTube Music/playlists/Liked videos.csv", + "Video ID,Video Title\nliked000001,Liked\n", + ) + output.entry( + "Takeout/My Activity/YouTube/watch-history.html", + "You watched Seen
1 Jan 2026, 12:00:00 CET
", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + val adapter = YoutubeTakeoutPortabilityAdapter() + + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + val positions = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> + positions += (record as PortabilityPlaylistVideo).position + } + assertEquals(listOf(0, 1), positions) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + @Test + fun `adapter detects spanish playlist paths and activity dates`() { + val archive = directory.resolve("takeout-es.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube y YouTube Music/suscripciones/suscripciones.csv", + "ID de canal,URL del canal,Título del canal\nUC123456789012,https://youtube.com/channel/UC123456789012,Canal\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/catalogo.csv", + "ID de la lista de reproducción,Título de la lista de reproducción\nPL123456789,Importada\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Videos de Importada.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000001,2026-01-02T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Ver más tarde.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000002,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/Mon actividad/YouTube/watch-history.html", + "Has visto Watched
16 sept 2026, 18:02:08 CEST
", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout-es.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + assertEquals(1_789_574_528_000L, (spoolRecord(spool, PortabilityCategory.HISTORY) as PortabilityHistory).watchedAt) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + @Test + fun `adapter associates spanish playlist item files with manifest ids`() { + val archive = directory.resolve("takeout-es-manifest.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Listas de reproducción.csv", + "ID de lista de reproducción,Título de la lista de reproducción (original)\n" + + "PL123456789,Importada\nPL987654321,It's/Luna\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Importada-vídeos.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\n" + + "video000001,2026-01-02T00:00:00Z\n" + + "video000002,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/It_s_Luna-vídeos.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000003,2026-01-03T00:00:00Z\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(5L, spool.counts()[PortabilityCategory.PLAYLISTS]) + val videos = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> + videos += record as PortabilityPlaylistVideo + } + assertEquals(listOf("video000001", "video000002"), videos.map { it.video.url.substringAfter("v=") }) + val escaped = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL987654321") { record -> + escaped += record as PortabilityPlaylistVideo + } + assertEquals(listOf("video000003"), escaped.map { it.video.url.substringAfter("v=") }) + assertTrue(spool.issues().none { it.code == "playlist_manifest_missing" }) + spool.delete() + } + + @Test + fun `adapter streams My Activity JSON including embedded URLs`() { + val json = directory.resolve("watch-history.json") + Files.writeString( + json, + """[ + {"header":"YouTube","title":"Watched First title","titleUrl":"https://www.youtube.com/watch?v=watched01","subtitles":[{"name":"Channel","url":"https://www.youtube.com/channel/UC123456789012"}],"time":"2026-09-16T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Liked Second title","titleUrl":"https://www.youtube.com/watch?v=liked01","time":"2026-09-15T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Watched URLs://www.youtube.com/watch?v=embedded1","time":"2026-09-14T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Titre japonais を視聴しました","titleUrl":"https://music.youtube.com/watch?v=japan01","time":"2026-09-13T18:02:08Z","activityControls":["視聴履歴"]}, + {"header":"YouTube","title":"Visited https://youtu.be/visited1","time":"2026-09-12T18:02:08Z","activityControls":["Web & App Activity","YouTube watch history"]}, + {"header":"YouTube","title":"Watched I liked this title","titleUrl":"https://www.youtube.com/watch?v=falsefav1","time":"2026-09-11T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"You subscribed to Channel","titleUrl":"https://www.youtube.com/channel/UC987654321098","subtitles":[{"name":"Channel","url":"https://www.youtube.com/channel/UC987654321098"}],"time":"2026-09-10T18:02:08Z","activityControls":["YouTube subscriptions"]}, + {"header":"YouTube","title":"You subscribed to Direct channel","titleUrl":"https://www.youtube.com/@directchannel","time":"2026-09-09T18:02:08Z","activityControls":["YouTube subscriptions"]} + ]""".trimIndent(), + ) + val input = PortabilityInputFactory.create(json, json.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = YoutubeTakeoutPortabilityAdapter() + + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(5L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + val history = mutableListOf() + spool.forEach(PortabilityCategory.HISTORY) { history += it as PortabilityHistory } + assertEquals("https://www.youtube.com/watch?v=embedded1", history[2].video.url) + assertEquals("YouTube video embedded1", history[2].video.title) + assertEquals( + "Titre japonais", + history.first { it.video.url.contains("japan01") }.video.title, + ) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + @Test + fun readsLegacySnippetDatesAndSkipsAdRows() { + val json = directory.resolve("legacy-watch-history.json") + Files.writeString( + json, + """[ + {"snippet":{"title":"Watched Legacy title","titleUrl":"https://www.youtube.com/watch?v=legacy01","publishedAt":"2026-09-16T18:02:08Z"}}, + {"title":"Watched advertisement","titleUrl":"https://www.youtube.com/watch?v=adrow01","time":"2026-09-16T18:02:08Z","details":[{"name":"Ads"}]} + ]""".trimIndent(), + ) + val input = PortabilityInputFactory.create(json, "watch-history.json", "application/json") + val spool = PortabilitySpool.create(directory) + try { + assertEquals( + PortabilityFormat.YOUTUBE_TAKEOUT, + requireNotNull(YoutubeTakeoutPortabilityAdapter().detect(input)).format, + ) + assertNotNull( + YoutubeTakeoutPortabilityAdapter().detect( + PortabilityInputFactory.create(json, "再生履歴.json", "application/json"), + ), + ) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + + @Test + fun `adapter reads JSON activity entries from a Takeout archive`() { + val archive = directory.resolve("takeout-json.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/マイ アクティビティ/再生履歴.json", + """[{"header":"YouTube","title":"動画 を視聴しました","titleUrl":"https://www.youtube.com/watch?v=archive01","time":"2026-09-16T18:02:08Z"}]""", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout-json.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + try { + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(YoutubeTakeoutPortabilityAdapter().detect(input)).format) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + } finally { + spool.delete() + } + } + + private fun spoolRecord(spool: PortabilitySpool, category: PortabilityCategory): PortabilityRecord { + var result: PortabilityRecord? = null + spool.forEach(category) { result = it } + return requireNotNull(result) + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts new file mode 100644 index 00000000..7266fc45 --- /dev/null +++ b/server-sabr/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + kotlin("jvm") +} + +dependencies { + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + api("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java b/server-sabr/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java similarity index 100% rename from src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java rename to server-sabr/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt index 3ec926d3..30cfc882 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInf import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe /** The only server entry point for PipePipe's SABR extraction API. */ -internal object SabrAdapter { +object SabrAdapter { fun fetchSabrInfo( videoId: String, profile: YoutubeSabrClientProfile, diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt new file mode 100644 index 00000000..11b4e668 --- /dev/null +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt @@ -0,0 +1,4 @@ +package dev.typetype.server.sabr + +typealias SabrProtocolException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException +typealias SabrRecoverableException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt index 79d46ef0..30f72996 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt @@ -3,8 +3,8 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat /** TypeType's stable representation of a provider media format. */ -internal class YoutubeSabrFormat internal constructor( - internal val delegate: PipeFormat, +class YoutubeSabrFormat constructor( + val delegate: PipeFormat, ) { val isAudio: Boolean get() = delegate.isAudio val isVideo: Boolean get() = delegate.isVideo diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt index 68c9785f..b6fe26a4 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt @@ -5,8 +5,8 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as P import java.util.IdentityHashMap /** Provider-independent SABR metadata used by TypeType's orchestration layer. */ -internal class YoutubeSabrInfo internal constructor( - internal val delegate: PipeInfo, +class YoutubeSabrInfo constructor( + val delegate: PipeInfo, private val formatCache: IdentityHashMap = IdentityHashMap(), ) { val profile: YoutubeSabrClientProfile diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt index 23b74cee..a023b4fa 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt @@ -4,7 +4,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader as Pip import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment as PipeSegment import java.io.InputStream -internal class SabrMediaHeader internal constructor( +class SabrMediaHeader constructor( private val delegate: PipeHeader, ) { val headerId: Int get() = delegate.headerId @@ -27,8 +27,8 @@ internal class SabrMediaHeader internal constructor( fun summarize(): String = delegate.summarize() } -internal class SabrMediaSegment private constructor( - internal val delegate: PipeSegment, +class SabrMediaSegment private constructor( + val delegate: PipeSegment, ) { val header: SabrMediaHeader = SabrMediaHeader(delegate.header) val data: ByteArray get() = delegate.data diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt index 1effadf6..0a916ef3 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrNextRequestPolicy as PipePolicy -internal class SabrNextRequestPolicy internal constructor( +class SabrNextRequestPolicy constructor( private val delegate: PipePolicy, ) { val targetAudioReadaheadMs: Int get() = delegate.targetAudioReadaheadMs diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt index 0c07023a..3dbd9182 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile as PipeProfile -internal enum class YoutubeSabrClientProfile(internal val delegate: PipeProfile) { +enum class YoutubeSabrClientProfile(internal val delegate: PipeProfile) { WEB(PipeProfile.WEB), MWEB(PipeProfile.MWEB), WEB_EMBEDDED(PipeProfile.WEB_EMBEDDED), diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt index 6a8d95ac..528a334b 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange as PipeRange -internal class SabrBufferedRange internal constructor( +class SabrBufferedRange constructor( val itag: Int, val lastModified: Long, val xtags: String?, diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt index b7685c29..6a08db1f 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest as PipeRequest -internal class SabrSegmentRequest private constructor( +class SabrSegmentRequest private constructor( val format: YoutubeSabrFormat, val isInitializationSegment: Boolean, val sequenceNumber: Int, @@ -10,7 +10,7 @@ internal class SabrSegmentRequest private constructor( ) { private val delegateValue: PipeRequest by lazy(LazyThreadSafetyMode.SYNCHRONIZED, delegateFactory) - internal val delegate: PipeRequest + val delegate: PipeRequest get() = delegateValue companion object { diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt index 314820ae..27e3e081 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as Pip import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession as PipeSession import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState -internal class YoutubeSabrSession( +class YoutubeSabrSession( info: YoutubeSabrInfo, audioFormat: YoutubeSabrFormat, videoFormat: YoutubeSabrFormat, @@ -80,10 +80,11 @@ internal class YoutubeSabrSession( fun prepareForForwardJump(request: SabrSegmentRequest, value: Long): Unit = delegate.prepareForForwardJump(request.delegate, value) fun prepareForMissingSegment(request: SabrSegmentRequest): Unit = delegate.prepareForMissingSegment(request.delegate) - internal class DemandResponseResult internal constructor( + class DemandResponseResult internal constructor( private val delegate: PipeSession.DemandResponseResult, ) { val segmentCount: Int get() = delegate.segmentCount val targetTrackSegmentCount: Int get() = delegate.targetTrackSegmentCount + val requestPerformed: Boolean get() = delegate.wasRequestPerformed() } } diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt index 5689bd70..2db9e245 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt @@ -2,8 +2,8 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState -internal class YoutubeSabrStreamState private constructor( - internal val delegate: PipeState, +class YoutubeSabrStreamState private constructor( + val delegate: PipeState, ) { companion object { const val TRACK_MODE_VIDEO_AND_AUDIO: Int = PipeState.TRACK_MODE_VIDEO_AND_AUDIO diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt index 2d06cfb3..3aa915d9 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt @@ -1,5 +1,5 @@ package dev.typetype.server.sabr -internal fun interface SabrPoTokenProvider { +fun interface SabrPoTokenProvider { fun getPoToken(info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState): ByteArray? } diff --git a/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt b/server-sabr/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt rename to server-sabr/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts new file mode 100644 index 00000000..1270b6fb --- /dev/null +++ b/server-services/build.gradle.kts @@ -0,0 +1,107 @@ +import java.time.Instant +plugins { + `java-test-fixtures` + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-domain")) + api(project(":server-playback")) + api(project(":server-auth")) + api(project(":server-downloader")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-downloader")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-client-core-jvm:3.5.2") + implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") + implementation("org.slf4j:slf4j-api:2.0.16") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") + testImplementation("org.testcontainers:testcontainers:2.0.5") + testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testFixturesImplementation(project(":server-cache")) + testFixturesImplementation(project(":server-db")) + testFixturesImplementation(testFixtures(project(":server-db"))) + testFixturesImplementation("org.jetbrains.exposed:exposed-core:1.5.0") + testFixturesImplementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testFixturesImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") +"testFixturesImplementation"("io.lettuce:lettuce-core:7.7.0.RELEASE") + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +val applicationVersion = providers.gradleProperty("appVersion").get() +val buildInfoVersion = applicationVersion.trim().takeUnless { it.isBlank() || it == "unspecified" } ?: "0.0.0-dev" +fun gitRevisionOrUnknown(): String = runCatching { + providers.exec { commandLine("git", "rev-parse", "HEAD") } + .standardOutput + .asText + .get() + .trim() + .ifBlank { "unknown" } +}.getOrElse { "unknown" } +val buildInfoRevision = providers.environmentVariable("GITHUB_SHA") + .map { it.trim().ifBlank { "unknown" } } + .getOrElse(gitRevisionOrUnknown()) +val buildInfoShortRevision = buildInfoRevision.takeIf { it != "unknown" }?.take(12) ?: "unknown" +val buildInfoBuildTime = providers.environmentVariable("BUILD_TIME") + .orElse(providers.provider { Instant.now().toString() }) + .get() +val generatedBuildInfoDir = layout.buildDirectory.dir("generated/sources/buildInfo/main") +val generateBuildInfo = tasks.register("generateBuildInfo") { + inputs.property("version", buildInfoVersion) + inputs.property("revision", buildInfoRevision) + inputs.property("shortRevision", buildInfoShortRevision) + inputs.property("buildTime", buildInfoBuildTime) + outputs.dir(generatedBuildInfoDir) + doLast { + val output = generatedBuildInfoDir.get().file("dev/typetype/server/BuildInfo.kt").asFile + output.parentFile.mkdirs() + output.writeText(""" + package dev.typetype.server + + object BuildInfo { + const val VERSION: String = "${buildInfoVersion.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val REVISION: String = "${buildInfoRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val SHORT_REVISION: String = "${buildInfoShortRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val BUILD_TIME: String = "${buildInfoBuildTime.replace("\\", "\\\\").replace("\"", "\\\"")}" + } + """.trimIndent()) + } +} + +sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } + +tasks.named("compileKotlin") { dependsOn(generateBuildInfo) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/AppMetrics.kt rename to server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt index 0de33b49..75cb8467 100644 --- a/src/main/kotlin/dev/typetype/server/AppMetrics.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -51,6 +51,8 @@ object AppMetrics { } fun metricPath(path: String): String = when { + path.startsWith("/api/sabr/playback/") -> "/api/sabr/playback/{sessionId}" + path.startsWith("/sabr/playback/") -> "/sabr/playback/{sessionId}" path.startsWith("/downloader/jobs/") && path.endsWith("/events") -> "/downloader/jobs/{id}/events" path.startsWith("/downloader/jobs/") && path.endsWith("/artifact") -> "/downloader/jobs/{id}/artifact" path.startsWith("/downloader/jobs/") && path.endsWith("/cancel") -> "/downloader/jobs/{id}/cancel" diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index d2f72dc5..1c983bba 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -27,6 +27,7 @@ import dev.typetype.server.services.PipePipePodcastService import dev.typetype.server.services.PipePipePublicPlaylistService import dev.typetype.server.services.PipePipeSearchService import dev.typetype.server.services.PipePipeStreamService +import dev.typetype.server.services.PipePipeLiveHlsStreamService import dev.typetype.server.services.PipePipeSuggestionService import dev.typetype.server.services.PipePipeTrendingService import dev.typetype.server.services.ProviderMediaHandleService @@ -44,6 +45,8 @@ import dev.typetype.server.services.StreamYouTubeSubtitleResolver import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient import dev.typetype.server.services.YoutubePlayerClientStreamService +import dev.typetype.server.services.YoutubeLiveHlsStreamService +import dev.typetype.server.services.YoutubeDirectLiveHlsStreamService import dev.typetype.server.services.YoutubeScopedChannelService import dev.typetype.server.services.YoutubeScopedCommentService import dev.typetype.server.services.YoutubeScopedPublicPlaylistService @@ -74,6 +77,11 @@ internal class ExtractionServiceRegistry( val httpClient = OkHttpClient.Builder() .apply { youtubeProxySelector?.let(::proxySelector) } .build() + val subtitleHttpClient = httpClient.newBuilder() + .connectTimeout(5, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .callTimeout(15, TimeUnit.SECONDS) + .build() val proxyHttpClient: OkHttpClient = httpClient.newBuilder() .dispatcher(proxyDispatcher()) .connectionPool(ConnectionPool(64, 5, TimeUnit.MINUTES)) @@ -82,7 +90,7 @@ internal class ExtractionServiceRegistry( .followRedirects(true) .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) - val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) + val youtubeSubtitleService = YouTubeSubtitleService(subtitleHttpClient, subtitleServiceUrl) private val bilibiliRelatedService = BilibiliRelatedService() private val directPipePipeStreamService = PipePipeStreamService( cache, @@ -93,15 +101,20 @@ internal class ExtractionServiceRegistry( cache, youtubeSubtitleService, bilibiliRelatedService, - sabrSessionStore::rememberExtractedInfo, + sabrSessionStore::rememberExtractedInfoAsync, + fetchSupplementalSubtitles = false, ) private val publicStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, YoutubePlayerClient.VISIONOS, ) - private val authenticatedStreamService = YoutubePlayerClientStreamService( - directPipePipeStreamService, - YoutubePlayerClient.MWEB, + private val liveHlsStreamService = YoutubePlayerClientStreamService( + PipePipeLiveHlsStreamService(), + YoutubePlayerClient.TV_DOWNGRADED, + ) + private val authenticatedStreamService = YoutubeLiveHlsStreamService( + YoutubePlayerClientStreamService(directPipePipeStreamService, YoutubePlayerClient.MWEB), + liveHlsStreamService, ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( sabrPipePipeStreamService, @@ -114,6 +127,11 @@ internal class ExtractionServiceRegistry( ) private val hlsTokenService = youtubeSessionSecret?.let(::SignedHlsManifestTokenService) private val tokenYoutubeSessionClient = TypetypeTokenYoutubeSessionClient(subtitleServiceUrl, httpClient) + private val youtubeSabrFallbackStreamService = SabrFallbackStreamService( + sabrPublicStreamService, + sabrSessionStore, + tokenYoutubeSessionClient, + ) val youtubeSessionStreamService = hlsTokenService?.let { YoutubeSessionStreamService(authenticatedStreamService, youtubeSessionService, cache, it) } @@ -122,22 +140,31 @@ internal class ExtractionServiceRegistry( } val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( - SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), + YoutubeLiveHlsStreamService( + youtubeSabrFallbackStreamService, + liveHlsStreamService, + ), ), cache, "stream-youtube-sabr:v1", ) + val youtubeLiveHlsStreamService = YoutubeScopedStreamService( + YoutubeDirectLiveHlsStreamService( + liveHlsStreamService, + youtubeSabrFallbackStreamService, + ), + ) val youtubeSubtitleDeliveryService = YouTubeSubtitleDeliveryService( StreamYouTubeSubtitleResolver(youtubeSabrStreamService, youtubeSubtitleService::fetchSubtitleInventory), TokenYouTubeSubtitleContentFetcher( - httpClient, + subtitleHttpClient, subtitleServiceUrl, - OkHttpYouTubeSubtitleContentFetcher(httpClient), + OkHttpYouTubeSubtitleContentFetcher(subtitleHttpClient), ), YouTubeSubtitleCache(cache), ) val youtubeSabrBootstrapStreamService = YoutubeScopedStreamService( - SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient), + SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient, liveHlsStreamService), ) val nicoNicoStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-niconico:v1") val bilibiliStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-bilibili:v1") diff --git a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt b/server-services/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt rename to server-services/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt diff --git a/src/main/kotlin/dev/typetype/server/RequestObservability.kt b/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/RequestObservability.kt rename to server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt index a14c6db0..f00f0158 100644 --- a/src/main/kotlin/dev/typetype/server/RequestObservability.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt @@ -10,15 +10,10 @@ import kotlinx.coroutines.asContextElement import kotlinx.coroutines.withContext import java.util.UUID -const val REQUEST_ID_HEADER = "X-Request-ID" - private val requestIdAttribute = AttributeKey("requestId") private val requestStartNanosAttribute = AttributeKey("requestStartNanos") -private val requestIdContext = ThreadLocal() private val requestIdRegex = Regex("^[A-Za-z0-9._-]{8,128}$") -fun currentRequestId(): String? = requestIdContext.get() - fun ApplicationCall.requestId(): String = attributeOrNull(requestIdAttribute) ?: currentRequestId() ?: "unknown" fun ApplicationCall.requestDurationMs(): Long { @@ -30,13 +25,18 @@ fun Application.installRequestObservability() { intercept(ApplicationCallPipeline.Setup) { val applicationCall = context val requestId = resolveRequestId(applicationCall.request.headers[REQUEST_ID_HEADER]) + val traceId = resolvePlaybackTraceId(applicationCall.request.headers[PLAYBACK_TRACE_ID_HEADER]) applicationCall.attributes.put(requestIdAttribute, requestId) applicationCall.attributes.put(requestStartNanosAttribute, System.nanoTime()) applicationCall.response.headers.append(REQUEST_ID_HEADER, requestId, safeOnly = false) - withContext(requestIdContext.asContextElement(requestId)) { + traceId?.let { applicationCall.response.headers.append(PLAYBACK_TRACE_ID_HEADER, it, safeOnly = false) } + withContext(requestContextElement(requestId, traceId)) { try { proceed() } finally { + traceId?.let { + PlaybackTraceLog.record(it, requestId, "http_complete", "method=${applicationCall.request.httpMethod.value} route=${metricPath(applicationCall.request.path())} status=${applicationCall.response.status()?.value ?: 0} durationMs=${applicationCall.requestDurationMs()}") + } AppMetrics.record(applicationCall) } } @@ -54,5 +54,8 @@ fun requestLogLine(call: ApplicationCall): String = listOf( private fun resolveRequestId(raw: String?): String = raw?.takeIf { requestIdRegex.matches(it) } ?: UUID.randomUUID().toString() +private fun resolvePlaybackTraceId(raw: String?): String? = + raw?.takeIf { requestIdRegex.matches(it) } + private fun ApplicationCall.attributeOrNull(key: AttributeKey): T? = if (attributes.contains(key)) attributes[key] else null diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/ServiceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 5ca5063c..4fd731f6 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -8,6 +8,9 @@ import dev.typetype.server.services.AllowedChannelsService import dev.typetype.server.services.AllowedPlaylistsService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService +import dev.typetype.server.services.BiliBiliSessionCrypto +import dev.typetype.server.services.BiliBiliSessionService +import dev.typetype.server.services.BiliBiliSessionStreamService import dev.typetype.server.services.BlockedService import dev.typetype.server.services.CustomAvatarService import dev.typetype.server.services.DeArrowService @@ -32,16 +35,17 @@ import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsBlendService import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionAvatarWarmupService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.TypeTypeBackupService import dev.typetype.server.services.UserVideoMetadataRepairService import dev.typetype.server.services.VideoMetadataResolver import dev.typetype.server.services.WatchLaterService import dev.typetype.server.services.YoutubeTakeoutFactory import java.net.ProxySelector -internal class ServiceRegistry( +class ServiceRegistry( cache: DragonflyService, subtitleServiceUrl: String, youtubeSessionEncryptionKey: String?, @@ -64,13 +68,23 @@ internal class ServiceRegistry( youtubeProxySelector, ) val youtubeSessionService = extraction.youtubeSessionService + val bilibiliSessionService = BiliBiliSessionService( + System.getenv("BILIBILI_SESSION_ENCRYPTION_KEY")?.let { secret -> + if (secret.length >= 32) BiliBiliSessionCrypto.fromSecret(secret) else null + }, + ) val authenticatedSabrInfoService = extraction.authenticatedSabrInfoService val youtubeSessionStreamService = extraction.youtubeSessionStreamService val youtubeSessionSabrStreamService = extraction.youtubeSessionSabrStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService + val youtubeLiveHlsStreamService = extraction.youtubeLiveHlsStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService val bilibiliStreamService = extraction.bilibiliStreamService + val bilibiliSessionStreamService = BiliBiliSessionStreamService( + bilibiliStreamService, + bilibiliSessionService, + ) val streamService = extraction.streamService val searchService = extraction.searchService val trendingService = extraction.trendingService @@ -92,6 +106,7 @@ internal class ServiceRegistry( val sabrSessionStore = extraction.sabrSessionStore val historyService = HistoryService() val subscriptionsService = SubscriptionsService() + val subscriptionAvatarWarmupService = SubscriptionAvatarWarmupService(channelService) val subscriptionGroupsService = SubscriptionGroupsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) val subscriptionShortsFeedService = SubscriptionShortsFeedService( @@ -102,7 +117,7 @@ internal class ServiceRegistry( ) init { SubscriptionFeedCacheInvalidation.configure( - SubscriptionFeedCacheInvalidator(cache, subscriptionFeedService), + SubscriptionFeedCacheInvalidatorImpl(cache, subscriptionFeedService), ) } val notificationsService = NotificationsService(subscriptionFeedService) diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt b/server-services/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt rename to server-services/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccessControlService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt index c5b81acd..bac06e5b 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt @@ -7,28 +7,28 @@ import dev.typetype.server.models.PublicPlaylistResponse import dev.typetype.server.models.SearchPageResponse import dev.typetype.server.models.StreamResponse -internal fun SearchPageResponse.filterAllowed(profile: AccessControlProfile): SearchPageResponse = copy( +fun SearchPageResponse.filterAllowed(profile: AccessControlProfile): SearchPageResponse = copy( items = items.filterAllowed(profile), channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, playlists = playlists.filter { profile.allowsPlaylist(it.url) || profile.allowsChannel(url = "", name = it.uploaderName) }, ) -internal fun StreamResponse.filterAllowed(profile: AccessControlProfile): StreamResponse = copy( +fun StreamResponse.filterAllowed(profile: AccessControlProfile): StreamResponse = copy( relatedStreams = relatedStreams.filterAllowed(profile), ) -internal fun PublicPlaylistResponse.filterAllowed(profile: AccessControlProfile): PublicPlaylistResponse = copy( +fun PublicPlaylistResponse.filterAllowed(profile: AccessControlProfile): PublicPlaylistResponse = copy( videos = if (profile.allowsPlaylist(playlist.url)) videos else videos.filterAllowed(profile), ) -internal fun ChannelResponse.filterAllowed(profile: AccessControlProfile): ChannelResponse = copy( +fun ChannelResponse.filterAllowed(profile: AccessControlProfile): ChannelResponse = copy( videos = videos.filterAllowed(profile), ) -internal fun ChannelPlaylistsResponse.filterAllowed(profile: AccessControlProfile): ChannelPlaylistsResponse = copy( +fun ChannelPlaylistsResponse.filterAllowed(profile: AccessControlProfile): ChannelPlaylistsResponse = copy( playlists = playlists.filter { profile.allowsPlaylist(it.url) || profile.allowsChannel(url = "", name = it.uploaderName) }, ) -internal fun HomeRecommendationsResponse.filterAllowed(profile: AccessControlProfile): HomeRecommendationsResponse = copy( +fun HomeRecommendationsResponse.filterAllowed(profile: AccessControlProfile): HomeRecommendationsResponse = copy( items = items.filterAllowed(profile), ) diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt similarity index 58% rename from src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt index 0a222b4d..79507b52 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt @@ -2,8 +2,8 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem -internal fun List.filterAllowed(profile: AccessControlProfile): List = +fun List.filterAllowed(profile: AccessControlProfile): List = filter { profile.allowsUploader(url = it.uploaderUrl, name = it.uploaderName) } -internal fun VideoItem.isAllowedBy(profile: AccessControlProfile): Boolean = +fun VideoItem.isAllowedBy(profile: AccessControlProfile): Boolean = profile.allowsUploader(url = uploaderUrl, name = uploaderName) diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt index 7a761a05..407f123a 100644 --- a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt @@ -17,7 +17,7 @@ class AdminSettingsService( ) { suspend fun get(): AdminSettingsItem { - cachedSettings?.let { return it } + AdminSettingsCache.get()?.let { return it } val settings = DatabaseFactory.query { AdminSettingsTable.selectAll().singleOrNull()?.let { AdminSettingsItem( @@ -43,7 +43,7 @@ class AdminSettingsService( ).normalized() } ?: defaultSettings().normalized() } - cachedSettings = settings + AdminSettingsCache.set(settings) return settings } @@ -98,7 +98,7 @@ class AdminSettingsService( } } } - cachedSettings = settings + AdminSettingsCache.set(settings) return settings } @@ -141,11 +141,8 @@ class AdminSettingsService( } companion object { - @Volatile - private var cachedSettings: AdminSettingsItem? = null - fun clearCache() { - cachedSettings = null + AdminSettingsCache.clear() } } } diff --git a/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt index 222bb64a..2c31135d 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt @@ -11,7 +11,6 @@ import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll -import java.net.URI class AllowedChannelsService { suspend fun getChannels(userId: String): List = DatabaseFactory.query { @@ -81,33 +80,3 @@ private fun toAllowedChannelItem(row: ResultRow): AllowedChannelItem = AllowedCh allowedAt = row[AllowedChannelsTable.allowedAt], global = row[AllowedChannelsTable.scope] == ALLOW_SCOPE_GLOBAL, ) - -internal fun normalizeChannelKey(value: String): String = value.trim() - .substringBefore('#') - .substringBefore('?') - .removeSuffix("/") - .replace("http://", "https://") - .replace( - Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), - "https://youtube.com", - ) - .withoutYoutubeTab() - -private fun String.withoutYoutubeTab(): String { - val uri = runCatching { URI(this) }.getOrNull() ?: return this - if (!uri.host.equals("youtube.com", ignoreCase = true)) return this - val segments = uri.path.split('/').filter(String::isNotBlank) - if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this - val path = "/${segments.dropLast(1).joinToString("/")}" - return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() -} - -private val YOUTUBE_CHANNEL_TABS = setOf( - "featured", - "videos", - "shorts", - "streams", - "playlists", - "community", - "about", -) diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt index aba44687..82079358 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt @@ -71,8 +71,3 @@ private fun toAllowedPlaylistItem(row: ResultRow): AllowedPlaylistItem = Allowed allowedAt = row[AllowedPlaylistsTable.allowedAt], global = row[AllowedPlaylistsTable.scope] == ALLOW_SCOPE_GLOBAL, ) - -internal fun normalizePlaylistKey(value: String): String = value.trim() - .substringBefore('#') - .removeSuffix("/") - .replace("http://", "https://") diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt index b9d9b005..e24cc981 100644 --- a/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.AudioStreamItem -internal fun hlsFallbackStream(url: String): AudioStreamItem = AudioStreamItem( +fun hlsFallbackStream(url: String): AudioStreamItem = AudioStreamItem( url = url, mimeType = "application/vnd.apple.mpegurl", format = "HLS", diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt index d0f1f103..37235e79 100644 --- a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt @@ -62,10 +62,10 @@ class AudioOnlyStreamResolver( if (sabr != null) return ExtractionResult.Success( AudioOnlyStreamSelection(response, sabr, AudioOnlyStreamKind.SabrHls) ) - if (selectedItag != null) return ExtractionResult.Failure("No audio-only stream is available") + if (selectedItag != null) return noAudioStream() val hls = (if (allowHls) hlsCandidate(response, preferOriginal, preferredLocale) else null) if (hls != null) return ExtractionResult.Success(AudioOnlyStreamSelection(response, hls, AudioOnlyStreamKind.Hls)) - return ExtractionResult.Failure("No audio-only stream is available") + return noAudioStream() } private fun hlsCandidate(response: StreamResponse, preferOriginal: Boolean, preferredLocale: String?) = @@ -78,6 +78,9 @@ class AudioOnlyStreamResolver( private fun dev.typetype.server.models.AudioStreamItem.matchesSelected(itag: Int, trackId: String?): Boolean = this.itag == itag && this.audioTrackId == trackId + private fun noAudioStream(): ExtractionResult.Failure = + ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") + private fun ExtractionResult?.resolveWith( sessionResult: ExtractionResult?, ): ExtractionResult { diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt new file mode 100644 index 00000000..9d64292e --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt @@ -0,0 +1,105 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import okhttp3.OkHttpClient +import okhttp3.Request + +class BiliBiliQrLoginService( + private val client: OkHttpClient = OkHttpClient(), + private val nowMillis: () -> Long = System::currentTimeMillis, +) { + private val json = Json { ignoreUnknownKeys = true } + + suspend fun generate(): BiliBiliQrGenerateResult = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(GENERATE_URL) + .header("User-Agent", WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + val response = client.newCall(request).execute() + response.use { + val body = it.body.string() ?: return@withContext BiliBiliQrGenerateResult.Error("Empty response") + val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() + ?: return@withContext BiliBiliQrGenerateResult.Error("Invalid JSON") + val data = root["data"]?.jsonObject + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing data") + val url = data["url"]?.jsonPrimitive?.content + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing QR URL") + val key = data["qrcode_key"]?.jsonPrimitive?.content + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing QR key") + BiliBiliQrGenerateResult.Success( + qrUrl = url, + qrcodeKey = key, + expiresAt = nowMillis() + QR_TTL_MS, + ) + } + } + + suspend fun poll(qrcodeKey: String): BiliBiliQrPollResult = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url("$POLL_URL?qrcode_key=$qrcodeKey") + .header("User-Agent", WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + val response = client.newCall(request).execute() + response.use { httpResponse -> + val body = httpResponse.body?.string() ?: return@withContext BiliBiliQrPollResult.Error("Empty response") + val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() + ?: return@withContext BiliBiliQrPollResult.Error("Invalid JSON") + val data = root["data"]?.jsonObject + ?: return@withContext BiliBiliQrPollResult.Error("Missing data") + val code = data["code"]?.jsonPrimitive?.content?.toIntOrNull() + ?: return@withContext BiliBiliQrPollResult.Error("Missing code") + when (code) { + 0 -> { + val cookies = httpResponse.headers("Set-Cookie") + .filter { it.contains("SESSDATA=") || it.contains("bili_jct=") || it.contains("buvid3=") } + .mapNotNull { cookie -> + val parts = cookie.split(";").firstOrNull()?.trim() + parts?.takeIf { it.isNotEmpty() } + } + val cookieHeader = cookies.joinToString("; ") + if (cookieHeader.contains("SESSDATA=")) { + BiliBiliQrPollResult.Confirmed(cookieHeader) + } else { + BiliBiliQrPollResult.Error("Missing SESSDATA in response") + } + } + else -> classifyBiliBiliQrCode(code) + } + } + } + + companion object { + private const val GENERATE_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/generate" + private const val POLL_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/poll" + private const val QR_TTL_MS = 180_000L + internal const val WEB_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + } +} + +internal fun classifyBiliBiliQrCode(code: Int): BiliBiliQrPollResult = when (code) { + 86038 -> BiliBiliQrPollResult.Expired + 86090 -> BiliBiliQrPollResult.Scanned + // The current web endpoint uses 86101 while the QR is still waiting to be scanned. + 86001, 86101 -> BiliBiliQrPollResult.Waiting + else -> BiliBiliQrPollResult.Error("Unexpected code: $code") +} + +sealed class BiliBiliQrGenerateResult { + data class Success(val qrUrl: String, val qrcodeKey: String, val expiresAt: Long) : BiliBiliQrGenerateResult() + data class Error(val message: String) : BiliBiliQrGenerateResult() +} + +sealed class BiliBiliQrPollResult { + data class Confirmed(val cookieHeader: String) : BiliBiliQrPollResult() + object Scanned : BiliBiliQrPollResult() + object Waiting : BiliBiliQrPollResult() + object Expired : BiliBiliQrPollResult() + data class Error(val message: String) : BiliBiliQrPollResult() +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt new file mode 100644 index 00000000..816735cf --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 +import javax.crypto.Cipher +import javax.crypto.spec.GCMParameterSpec +import javax.crypto.spec.SecretKeySpec + +class BiliBiliSessionCrypto private constructor(private val key: SecretKeySpec) { + private val random = SecureRandom() + + fun encrypt(value: String): String { + val nonce = ByteArray(NONCE_BYTES) + random.nextBytes(nonce) + val cipher = Cipher.getInstance(TRANSFORMATION) + cipher.init(Cipher.ENCRYPT_MODE, key, GCMParameterSpec(TAG_BITS, nonce)) + cipher.updateAAD(AAD) + val encrypted = cipher.doFinal(value.toByteArray(Charsets.UTF_8)) + return PREFIX + encoder.encodeToString(nonce + encrypted) + } + + fun decrypt(value: String): String { + require(value.startsWith(PREFIX)) { "Invalid encrypted payload format" } + val bytes = decoder.decode(value.removePrefix(PREFIX)) + require(bytes.size > NONCE_BYTES) { "Invalid encrypted payload" } + val nonce = bytes.copyOfRange(0, NONCE_BYTES) + val encrypted = bytes.copyOfRange(NONCE_BYTES, bytes.size) + val cipher = Cipher.getInstance(TRANSFORMATION) + cipher.init(Cipher.DECRYPT_MODE, key, GCMParameterSpec(TAG_BITS, nonce)) + cipher.updateAAD(AAD) + return cipher.doFinal(encrypted).toString(Charsets.UTF_8) + } + + companion object { + private const val PREFIX = "gcm256." + private const val NONCE_BYTES = 12 + private const val TAG_BITS = 128 + private const val MIN_SECRET_LENGTH = 32 + private const val TRANSFORMATION = "AES/GCM/NoPadding" + private val AAD = "typetype.bilibili-session".toByteArray(Charsets.UTF_8) + private val encoder = Base64.getUrlEncoder().withoutPadding() + private val decoder = Base64.getUrlDecoder() + + fun fromSecret(secret: String): BiliBiliSessionCrypto { + require(secret.length >= MIN_SECRET_LENGTH) { "BiliBili session encryption key is too short" } + val digest = MessageDigest.getInstance("SHA-256") + .digest(secret.toByteArray(Charsets.UTF_8)) + return BiliBiliSessionCrypto(SecretKeySpec(digest, "AES")) + } + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt new file mode 100644 index 00000000..b2e8944f --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt @@ -0,0 +1,81 @@ +package dev.typetype.server.services + +import dev.typetype.server.downloader.BilibiliCookieContext +import org.schabi.newpipe.extractor.ServiceList +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.util.concurrent.atomic.AtomicBoolean + +object BiliBiliSessionScope { + private const val PUBLIC_PERMITS = 64 + private val permits = Semaphore(PUBLIC_PERMITS, true) + private val userRequestTimestamps = ConcurrentHashMap>() + private const val MAX_REQUESTS_PER_WINDOW = 30 + private const val RATE_WINDOW_MS = 60_000L + + suspend fun withCredentials(userId: String, cookieHeader: String, block: suspend () -> T): T { + checkRateLimit(userId) + return withPermits(PUBLIC_PERMITS) { + val bilibili = ServiceList.BiliBili + try { + bilibili.setTokens(cookieHeader) + bilibili.setCookieFunctions(BILIBILI_COOKIE_FUNCTIONS) + BilibiliCookieContext.set(cookieHeader) + block() + } finally { + bilibili.setTokens("") + bilibili.setCookieFunctions(emptySet()) + BilibiliCookieContext.set(null) + } + } + } + + suspend fun withoutCredentials(block: suspend () -> T): T = + withPermits(1) { + val bilibili = ServiceList.BiliBili + bilibili.setTokens("") + bilibili.setCookieFunctions(emptySet()) + BilibiliCookieContext.set(null) + try { + block() + } finally { + BilibiliCookieContext.set(null) + } + } + + private fun checkRateLimit(userId: String) { + val now = System.currentTimeMillis() + val queue = userRequestTimestamps.computeIfAbsent(userId) { ArrayDeque() } + synchronized(queue) { + while (queue.isNotEmpty() && queue.first() < now - RATE_WINDOW_MS) queue.removeFirst() + if (queue.size >= MAX_REQUESTS_PER_WINDOW) { + throw BiliBiliRateLimitException(MAX_REQUESTS_PER_WINDOW) + } + queue.addLast(now) + } + } + + private suspend fun withPermits(count: Int, block: suspend () -> T): T { + val acquired = AtomicBoolean(false) + return try { + withContext(Dispatchers.IO) { + if (!permits.tryAcquire(count, PERMIT_ACQUIRE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + error("Timed out waiting for BiliBili extraction permits") + } + acquired.set(true) + } + block() + } finally { + if (acquired.get()) permits.release(count) + } + } + + private const val PERMIT_ACQUIRE_TIMEOUT_MS = 15_000L + private val BILIBILI_COOKIE_FUNCTIONS = setOf("high_res", "ai_subtitle") +} + +class BiliBiliRateLimitException(val maxRequests: Int) : + RuntimeException("BiliBili rate limit exceeded: max $maxRequests requests per minute") diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt new file mode 100644 index 00000000..93194b0c --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt @@ -0,0 +1,147 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BiliBiliQrLoginResponse +import dev.typetype.server.models.BiliBiliQrPollResponse +import dev.typetype.server.models.BiliBiliSessionStatusResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import okhttp3.OkHttpClient +import okhttp3.Request + +class BiliBiliSessionService( + private val crypto: BiliBiliSessionCrypto?, + private val store: BiliBiliSessionStore = BiliBiliSessionStore(), + private val qrLoginService: BiliBiliQrLoginService = BiliBiliQrLoginService(), + private val nowMillis: () -> Long = System::currentTimeMillis, +) { + private val json = Json { ignoreUnknownKeys = true } + private val client = OkHttpClient() + + val isConfigured: Boolean = crypto != null + + suspend fun startQrLogin(): BiliBiliQrLoginResult { + if (!isConfigured) return BiliBiliQrLoginResult.Unavailable + return when (val result = qrLoginService.generate()) { + is BiliBiliQrGenerateResult.Success -> + BiliBiliQrLoginResult.Success( + BiliBiliQrLoginResponse( + qrUrl = result.qrUrl, + qrcodeKey = result.qrcodeKey, + expiresAt = result.expiresAt, + ), + ) + is BiliBiliQrGenerateResult.Error -> BiliBiliQrLoginResult.Error(result.message) + } + } + + suspend fun pollQrLogin(userId: String, qrcodeKey: String): BiliBiliQrPollResponse { + if (!isConfigured) return BiliBiliQrPollResponse("unavailable", "BiliBili session is unavailable") + val crypto = crypto ?: return BiliBiliQrPollResponse("unavailable", "BiliBili session is unavailable") + return when (val result = qrLoginService.poll(qrcodeKey)) { + is BiliBiliQrPollResult.Confirmed -> { + val cookies = BilibiliCookieConfig.fromRaw(result.cookieHeader) + if (!cookies.isConfigured) { + return BiliBiliQrPollResponse("error", "Invalid BiliBili cookies received") + } + store.completeForUser( + userId = userId, + encryptedCookies = crypto.encrypt(result.cookieHeader), + expiresAt = parseSessDataExpiry(result.cookieHeader) ?: estimateExpiry(), + ) + BiliBiliQrPollResponse("confirmed") + } + is BiliBiliQrPollResult.Scanned -> BiliBiliQrPollResponse("scanned") + is BiliBiliQrPollResult.Waiting -> BiliBiliQrPollResponse("waiting") + is BiliBiliQrPollResult.Expired -> BiliBiliQrPollResponse("expired") + is BiliBiliQrPollResult.Error -> BiliBiliQrPollResponse("error", result.message) + } + } + + suspend fun status(userId: String): BiliBiliSessionStatusResponse { + if (!isConfigured) return BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + val stored = store.status(userId) + if (stored.status == BiliBiliSessionStatus.Connected.value && stored.expiresAt > 0 && stored.expiresAt < nowMillis()) { + store.markNeedsReconnect(userId) + return stored.copy(status = BiliBiliSessionStatus.NeedsReconnect.value) + } + return stored + } + + suspend fun healthCheck(userId: String): BiliBiliHealthResult { + if (!isConfigured) return BiliBiliHealthResult.Unconfigured + val cookies = connectedCookies(userId) ?: return BiliBiliHealthResult.Disconnected + return withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(NAV_API_URL) + .header("Cookie", cookies) + .header("User-Agent", BiliBiliQrLoginService.WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + runCatching { + client.newCall(request).execute().use { response -> + if (response.code == 412) return@runCatching BiliBiliHealthResult.RateLimited + val body = response.body?.string() ?: return@runCatching BiliBiliHealthResult.Error("Empty response") + val root = json.parseToJsonElement(body).jsonObject + val code = root["code"]?.jsonPrimitive?.content?.toIntOrNull() ?: -1 + when (code) { + 0 -> BiliBiliHealthResult.Healthy + -101 -> BiliBiliHealthResult.Expired + -352 -> BiliBiliHealthResult.RateLimited + else -> BiliBiliHealthResult.Error("Unexpected code: $code") + } + } + }.getOrElse { BiliBiliHealthResult.Error(it.message ?: "Network error") } + } + } + + suspend fun delete(userId: String): Boolean = store.delete(userId) + + suspend fun connectedCookies(userId: String): String? { + val crypto = crypto ?: return null + val encrypted = store.connectedEncrypted(userId) ?: return null + val decrypted = runCatching { crypto.decrypt(encrypted) }.getOrNull() + if (decrypted == null) store.markNeedsReconnect(userId) + return decrypted + } + + suspend fun markUsed(userId: String) = store.markUsed(userId) + + suspend fun markNeedsReconnect(userId: String) = store.markNeedsReconnect(userId) + + private fun parseSessDataExpiry(cookieHeader: String): Long? { + val sessData = cookieHeader.split(";") + .map { it.trim() } + .firstOrNull { it.startsWith("SESSDATA=", ignoreCase = true) } + ?.substringAfter("=") + ?: return null + val parts = sessData.split(",") + val timestamp = parts.getOrNull(1)?.trim()?.toLongOrNull() ?: return null + if (timestamp < 1_000_000_000L) return null + return timestamp * 1000 + } + + private fun estimateExpiry(): Long = nowMillis() + DEFAULT_COOKIE_TTL_MS + + companion object { + private const val DEFAULT_COOKIE_TTL_MS = 30L * 24 * 60 * 60 * 1000 + private const val NAV_API_URL = "https://api.bilibili.com/x/web-interface/nav" + } +} + +sealed class BiliBiliHealthResult { + object Healthy : BiliBiliHealthResult() + object Expired : BiliBiliHealthResult() + object RateLimited : BiliBiliHealthResult() + object Disconnected : BiliBiliHealthResult() + object Unconfigured : BiliBiliHealthResult() + data class Error(val message: String) : BiliBiliHealthResult() +} + +sealed class BiliBiliQrLoginResult { + data class Success(val response: BiliBiliQrLoginResponse) : BiliBiliQrLoginResult() + data class Error(val message: String) : BiliBiliQrLoginResult() + object Unavailable : BiliBiliQrLoginResult() +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt new file mode 100644 index 00000000..04d5596a --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.services + +enum class BiliBiliSessionStatus(val value: String) { + Connected("connected"), + NeedsReconnect("needs_reconnect"), + Disconnected("disconnected"); + + companion object { + fun from(value: String): BiliBiliSessionStatus = + entries.firstOrNull { it.value == value } ?: Disconnected + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt new file mode 100644 index 00000000..0e0bb325 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt @@ -0,0 +1,88 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.BiliBiliSessionsTable +import dev.typetype.server.models.BiliBiliSessionStatusResponse +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +class BiliBiliSessionStore( + private val nowMillis: () -> Long = System::currentTimeMillis, +) { + suspend fun completeForUser( + userId: String, + encryptedCookies: String, + expiresAt: Long, + ): Unit = DatabaseFactory.query { + upsertSession(userId, encryptedCookies, expiresAt, nowMillis()) + } + + suspend fun status(userId: String): BiliBiliSessionStatusResponse = DatabaseFactory.query { + BiliBiliSessionsTable.selectAll() + .where { BiliBiliSessionsTable.userId eq userId } + .singleOrNull() + ?.let { + BiliBiliSessionStatusResponse( + status = BiliBiliSessionStatus.from(it[BiliBiliSessionsTable.status]).value, + updatedAt = it[BiliBiliSessionsTable.updatedAt], + lastUsedAt = it[BiliBiliSessionsTable.lastUsedAt], + expiresAt = it[BiliBiliSessionsTable.expiresAt], + ) + } + ?: BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + } + + suspend fun delete(userId: String): Boolean = DatabaseFactory.query { + BiliBiliSessionsTable.deleteWhere { BiliBiliSessionsTable.userId eq userId } > 0 + } + + suspend fun connectedEncrypted(userId: String): String? = DatabaseFactory.query { + BiliBiliSessionsTable.selectAll() + .where { BiliBiliSessionsTable.userId eq userId } + .singleOrNull() + ?.takeIf { BiliBiliSessionStatus.from(it[BiliBiliSessionsTable.status]) == BiliBiliSessionStatus.Connected } + ?.get(BiliBiliSessionsTable.encryptedCookies) + } + + suspend fun markUsed(userId: String): Unit = DatabaseFactory.query { + BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { it[lastUsedAt] = nowMillis() } + } + + suspend fun markNeedsReconnect(userId: String): Unit = DatabaseFactory.query { + val now = nowMillis() + BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { + it[status] = BiliBiliSessionStatus.NeedsReconnect.value + it[updatedAt] = now + it[lastUsedAt] = now + } + } + + private fun upsertSession( + userId: String, + encryptedCookies: String, + expiresAt: Long, + now: Long, + ) { + val updated = BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { + it[BiliBiliSessionsTable.encryptedCookies] = encryptedCookies + it[BiliBiliSessionsTable.status] = BiliBiliSessionStatus.Connected.value + it[BiliBiliSessionsTable.updatedAt] = now + it[BiliBiliSessionsTable.lastUsedAt] = 0 + it[BiliBiliSessionsTable.expiresAt] = expiresAt + } + if (updated == 0) { + BiliBiliSessionsTable.insert { + it[BiliBiliSessionsTable.userId] = userId + it[BiliBiliSessionsTable.encryptedCookies] = encryptedCookies + it[BiliBiliSessionsTable.status] = BiliBiliSessionStatus.Connected.value + it[BiliBiliSessionsTable.createdAt] = now + it[BiliBiliSessionsTable.updatedAt] = now + it[BiliBiliSessionsTable.lastUsedAt] = 0 + it[BiliBiliSessionsTable.expiresAt] = expiresAt + } + } + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt new file mode 100644 index 00000000..4415b3bc --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class BiliBiliSessionStreamService( + private val streamService: StreamService, + private val sessionService: BiliBiliSessionService, +) { + suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { + if (!isBiliBiliUrl(url)) return null + val cookies = sessionService.connectedCookies(userId) ?: return null + return try { + BiliBiliSessionScope.withCredentials(userId, cookies) { + val result = streamService.getStreamInfo(url) + if (result is ExtractionResult.Success) sessionService.markUsed(userId) + if (result is ExtractionResult.Failure && requiresReconnect(result)) sessionService.markNeedsReconnect(userId) + result + } + } catch (e: BiliBiliRateLimitException) { + ExtractionResult.Failure( + BILIBILI_RATE_LIMIT_MESSAGE, + BILIBILI_RATE_LIMIT_CODE, + ) + } + } + + private fun isBiliBiliUrl(url: String): Boolean = + url.contains("bilibili.com") || url.contains("b23.tv") + + private fun requiresReconnect(result: ExtractionResult.Failure): Boolean = + result.message.contains("412") || + result.message.contains("-352") || + result.message.contains("risk", ignoreCase = true) + + companion object { + const val BILIBILI_RATE_LIMIT_MESSAGE = "BiliBili is temporarily limiting requests. Try again later." + const val BILIBILI_RATE_LIMIT_CODE = "bilibili_rate_limited" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 0f04f09a..3b502cf2 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -8,7 +8,7 @@ import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 -internal fun readBilibiliRangeWithRetry( +fun readBilibiliRangeWithRetry( execute: (Request) -> Response, request: Request, checkActive: () -> Unit = {}, diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt index 38a55888..a7e67476 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt @@ -16,7 +16,7 @@ private const val RELATED_BASE_URL = "https://api.bilibili.com/x/web-interface/a private const val SPACE_BASE_URL = "https://space.bilibili.com/" private val BVID_REGEX = Regex("""/(BV[0-9A-Za-z]+)""") -internal class BilibiliRelatedService( +class BilibiliRelatedService( private val relatedLookupOverride: (suspend (String) -> Map)? = null, ) { private val uploaderUrlCache = BoundedExpiringCache>( diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt new file mode 100644 index 00000000..8770d11e --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt @@ -0,0 +1,42 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse +import dev.typetype.server.models.HomeRecommendationsResponse +import dev.typetype.server.models.PublicPlaylistResponse +import dev.typetype.server.models.SearchPageResponse +import dev.typetype.server.models.StreamResponse + +fun HomeRecommendationsResponse.filterBlocked(profile: BlockedContentProfile): HomeRecommendationsResponse = copy( + items = items.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( + items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, + channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, + playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, +) + +fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse = copy( + relatedStreams = relatedStreams.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun PublicPlaylistResponse.filterBlocked(profile: BlockedContentProfile): PublicPlaylistResponse = copy( + videos = videos.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun ChannelResponse.filterBlocked(profile: BlockedContentProfile): ChannelResponse = copy( + videos = videos.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun ChannelPlaylistsResponse.filterBlocked(profile: BlockedContentProfile): ChannelPlaylistsResponse = copy( + playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, +) diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt index 9595097f..6447fff0 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt @@ -38,7 +38,7 @@ data class BlockedContentProfile( private val YOUTUBE_VIDEO_ID = Regex("(?:[?&]v=|/(?:shorts|embed|live)/|youtu\\.be/)([A-Za-z0-9_-]{6,})", RegexOption.IGNORE_CASE) -internal fun normalizeBlockedVideoKey(value: String): String { +fun normalizeBlockedVideoKey(value: String): String { val trimmed = value.trim() val youtubeId = YOUTUBE_VIDEO_ID.find(trimmed)?.groupValues?.get(1) return youtubeId?.let { "youtube:video:$it" } ?: normalizeChannelKey(trimmed) diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/BlockedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt index 3d91f801..31472655 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt @@ -161,10 +161,7 @@ class BlockedService { } } -internal fun normalizeBlockedKeyword(value: String): String = - Normalizer.normalize(value, Normalizer.Form.NFKC).trim().lowercase() - -internal fun containsBlockedKeyword(title: String, keyword: String): Boolean { +fun containsBlockedKeyword(title: String, keyword: String): Boolean { val normalizedKeyword = normalizeBlockedKeyword(keyword) return normalizedKeyword.isNotEmpty() && normalizeBlockedKeyword(title).contains(normalizedKeyword) } diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt index d79f937a..8f5c3f50 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt @@ -7,7 +7,7 @@ import dev.typetype.server.models.BugReportContextItem import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow -internal object BugReportRowMapper { +object BugReportRowMapper { fun toAdminListItem(row: ResultRow, userEmail: String): AdminBugReportItem = AdminBugReportItem( id = row[BugReportsTable.id], category = row[BugReportsTable.category], diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BugReportService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt index cc5a8cdc..62edbf12 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object BugReportStatusFlow { +object BugReportStatusFlow { private val nextStatus: Map = mapOf( "new" to "triaged", "triaged" to "in_progress", diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt index e19625ff..4fe95c7a 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.BugReportContextItem -internal object BugReportValidation { +object BugReportValidation { private val categories = setOf("player", "audio_language", "subtitles", "ui", "functionality") private val statuses = setOf("new", "triaged", "in_progress", "fixed", "closed") @@ -36,10 +36,12 @@ internal object BugReportValidation { context.screenHeight, ).filterNotNull() if (dimensions.any { it !in 1..100_000 }) return "Invalid display dimensions" - if (context.devicePixelRatio != null && context.devicePixelRatio !in 0.1..100.0) { + val devicePixelRatio = context.devicePixelRatio + if (devicePixelRatio != null && devicePixelRatio !in 0.1..100.0) { return "Invalid device pixel ratio" } - if (context.timezone != null && context.timezone.length > 128) return "Invalid timezone" + val timezone = context.timezone + if (timezone != null && timezone.length > 128) return "Invalid timezone" return null } } diff --git a/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt index 10968e4e..3e4a2d60 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt @@ -10,7 +10,7 @@ import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class ChannelNotificationPreferenceService( +class ChannelNotificationPreferenceService( private val subscriptionsService: SubscriptionsService, ) { suspend fun list(userId: String): List { @@ -26,7 +26,7 @@ internal class ChannelNotificationPreferenceService( val row = stored[channelUrl] ChannelNotificationPreference( channelUrl, - row?.get(ChannelNotificationPreferencesTable.enabled) ?: true, + row?.get(ChannelNotificationPreferencesTable.enabled) ?: false, row?.get(ChannelNotificationPreferencesTable.updatedAt) ?: 0L, ) } @@ -76,7 +76,7 @@ internal class ChannelNotificationPreferenceService( } } -internal sealed interface PreferenceUpdateResult { +sealed interface PreferenceUpdateResult { data class Updated(val preference: ChannelNotificationPreference) : PreferenceUpdateResult data object NotSubscribed : PreferenceUpdateResult } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt index 3ef4acf3..d358342a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt @@ -9,7 +9,7 @@ import org.schabi.newpipe.extractor.channel.ChannelTabInfo import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( +fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( name = name ?: "", description = description ?: "", avatarUrl = avatarUrl ?: "", @@ -20,7 +20,7 @@ internal fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( +fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( name = "", description = "", avatarUrl = "", @@ -31,7 +31,7 @@ internal fun InfoItemsPage.toChannelResponse(): ChannelResponse nextpage = nextPage?.toCursor(), ) -internal fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): ChannelResponse = ChannelResponse( +fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): ChannelResponse = ChannelResponse( name = metadata?.name ?: name ?: "", description = metadata?.description ?: "", avatarUrl = metadata?.avatarUrl ?: "", @@ -42,7 +42,7 @@ internal fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( +fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( name = "", description = "", avatarUrl = "", @@ -53,12 +53,12 @@ internal fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = C nextpage = nextPage?.toCursor(), ) -internal fun ChannelTabInfo.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( +fun ChannelTabInfo.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( playlists = relatedItems.filterIsInstance().map { it.toPlaylistResultItem() }, nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( +fun InfoItemsPage.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( playlists = items.filterIsInstance().map { it.toPlaylistResultItem() }, nextpage = nextPage?.toCursor(), ) diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt index 37622ff0..c251943a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ChannelResultItem import org.schabi.newpipe.extractor.channel.ChannelInfoItem -internal fun ChannelInfoItem.toChannelResultItem(): ChannelResultItem = ChannelResultItem( +fun ChannelInfoItem.toChannelResultItem(): ChannelResultItem = ChannelResultItem( id = url, name = name, url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt index 446fb83a..3641ce0a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs import org.schabi.newpipe.extractor.search.filter.Filter import org.schabi.newpipe.extractor.search.filter.FilterItem -internal fun StreamingService.channelTabExtractor( +fun StreamingService.channelTabExtractor( url: String, channelId: String, tab: String, @@ -17,3 +17,8 @@ internal fun StreamingService.channelTabExtractor( val linkHandler = channelTabLHFactory.fromQuery(channelId, contentFilter, sort.toYouTubeChannelTabSortFilter()) return getChannelTabExtractor(linkHandler) } + +fun StreamingService.toChannelTab(url: String, sort: String?): String? { + if (channelTabLHFactory == null) return null + return url.toChannelTab(sort) +} diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt index 3abffd24..7b51f61b 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.linkhandler.ChannelTabs -internal fun String.toChannelTab(sort: String?): String? { +fun String.toChannelTab(sort: String?): String? { if (contains("/shorts", ignoreCase = true)) return ChannelTabs.SHORTS if (contains("/streams", ignoreCase = true)) return ChannelTabs.LIVESTREAMS if (contains("/livestreams", ignoreCase = true)) return ChannelTabs.LIVESTREAMS @@ -10,7 +10,7 @@ internal fun String.toChannelTab(sort: String?): String? { return if (sort != null) ChannelTabs.VIDEOS else null } -internal fun String.toBaseChannelUrl(tab: String): String = substringBefore("/$tab") +fun String.toBaseChannelUrl(tab: String): String = substringBefore("/$tab") .substringBefore("/streams") .substringBefore('?') .substringBefore('#') diff --git a/src/main/kotlin/dev/typetype/server/services/CommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt index ec7a5407..64f679dd 100644 --- a/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoStreamItem -internal fun isDashManifestVideoStream(stream: VideoStreamItem): Boolean { +fun isDashManifestVideoStream(stream: VideoStreamItem): Boolean { val codec = stream.codec ?: return false val height = stream.height.takeIf { it > 0 } ?: dashResolutionHeight(stream.resolution) return stream.url.isNotBlank() && codec.startsWith("avc1") && (height <= 0 || height <= 1080) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt new file mode 100644 index 00000000..c270d75e --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt @@ -0,0 +1,58 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import java.util.concurrent.TimeUnit + +interface DeArrowRemote { + suspend fun branding(videoId: String): String? + suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? + suspend fun fallbackThumbnail(videoId: String): ByteArray? +} + +class DeArrowClient( + private val client: OkHttpClient = defaultClient(), +) : DeArrowRemote { + private val limiter = Semaphore(MAX_CONCURRENT_REQUESTS) + + override suspend fun branding(videoId: String): String? = + get("https://sponsor.ajay.app/api/branding?videoID=$videoId&fetchAll=true")?.decodeToString() + + override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? = + get("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=$videoId&time=$timestamp") + + override suspend fun fallbackThumbnail(videoId: String): ByteArray? = + get("https://i.ytimg.com/vi/$videoId/hqdefault.jpg") + + private suspend fun get(url: String): ByteArray? = withContext(Dispatchers.IO) { + limiter.withPermit { + runCatching { + client.newCall( + Request.Builder() + .url(url) + .header("User-Agent", USER_AGENT) + .get() + .build(), + ).execute().use { response -> + if (!response.isSuccessful) return@use null + response.body.bytes().takeIf { it.isNotEmpty() } + } + }.getOrNull() + } + } + + private companion object { + private const val MAX_CONCURRENT_REQUESTS = 6 + private const val USER_AGENT = "TypeType-DeArrow/1.0" + + fun defaultClient(): OkHttpClient = OkHttpClient.Builder() + .connectTimeout(4, TimeUnit.SECONDS) + .readTimeout(8, TimeUnit.SECONDS) + .callTimeout(10, TimeUnit.SECONDS) + .build() + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/DeArrowService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt index f354b6d2..a1d0805d 100644 --- a/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt @@ -14,32 +14,53 @@ import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import java.util.Base64 +class DeArrowUnavailableException : RuntimeException("DeArrow is temporarily unavailable") + +data class DeArrowThumbnail( + val bytes: ByteArray, + val fallback: Boolean, +) + class DeArrowService( private val cache: CacheService, private val client: DeArrowRemote = DeArrowClient(), ) { suspend fun get(videoId: String): DeArrowItem? { if (!isValidVideoId(videoId)) return null - cache.get("dearrow:branding:v2:$videoId")?.let { + cache.get("dearrow:branding:v3:$videoId")?.let { return runCatching { CacheJson.decodeFromString(DeArrowItem.serializer(), it) }.getOrNull() } - val item = client.branding(videoId)?.let { parse(videoId, it) } ?: DeArrowItem(videoId) + val raw = client.branding(videoId) ?: throw DeArrowUnavailableException() + val item = parse(videoId, raw) cache.set( - "dearrow:branding:v2:$videoId", + "dearrow:branding:v3:$videoId", CacheJson.encodeToString(DeArrowItem.serializer(), item), BRANDING_TTL_SECONDS, ) return item } - suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? { + suspend fun thumbnail(videoId: String, timestamp: Double): DeArrowThumbnail? { if (!isValidVideoId(videoId) || !timestamp.isFinite() || timestamp < 0.0) return null val normalizedTime = "%.3f".format(java.util.Locale.ROOT, timestamp) val key = "dearrow:thumbnail:$videoId:$normalizedTime" - cache.get(key)?.let { return runCatching { Base64.getDecoder().decode(it) }.getOrNull() } - val bytes = client.thumbnail(videoId, timestamp)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } ?: return null - cache.set(key, Base64.getEncoder().encodeToString(bytes), THUMBNAIL_TTL_SECONDS) - return bytes + cache.get(key)?.let { return decodeThumbnail(it) } + val generated = client.thumbnail(videoId, timestamp)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } + val fallback = generated ?: client.fallbackThumbnail(videoId)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } + val bytes = fallback ?: return null + val isFallback = generated == null + val ttl = if (isFallback) FALLBACK_THUMBNAIL_TTL_SECONDS else THUMBNAIL_TTL_SECONDS + val prefix = if (isFallback) FALLBACK_CACHE_PREFIX else GENERATED_CACHE_PREFIX + cache.set(key, prefix + Base64.getEncoder().encodeToString(bytes), ttl) + return DeArrowThumbnail(bytes, isFallback) + } + + private fun decodeThumbnail(value: String): DeArrowThumbnail? { + val fallback = value.startsWith(FALLBACK_CACHE_PREFIX) + val payload = value.removePrefix(FALLBACK_CACHE_PREFIX).removePrefix(GENERATED_CACHE_PREFIX) + return runCatching { + DeArrowThumbnail(Base64.getDecoder().decode(payload), fallback) + }.getOrNull() } private fun parse(videoId: String, raw: String): DeArrowItem { @@ -115,7 +136,10 @@ class DeArrowService( companion object { private const val BRANDING_TTL_SECONDS = 86_400L private const val THUMBNAIL_TTL_SECONDS = 604_800L + private const val FALLBACK_THUMBNAIL_TTL_SECONDS = 900L private const val MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024 + private const val FALLBACK_CACHE_PREFIX = "fallback:" + private const val GENERATED_CACHE_PREFIX = "generated:" private val VIDEO_ID_REGEX = Regex("^[A-Za-z0-9_-]{11}$") private val TITLE_MARKER_REGEX = Regex(">(?=\\S)") } diff --git a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt similarity index 75% rename from src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt index b7c21e3e..3f4a0016 100644 --- a/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt @@ -1,13 +1,17 @@ package dev.typetype.server.services +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException -internal suspend fun withExtractionRetry( +suspend fun withExtractionRetry( attempts: Int = 3, initialDelayMs: Long = 250, block: suspend () -> T, @@ -27,11 +31,15 @@ internal suspend fun withExtractionRetry( } private fun Throwable.isRetriableExtractionError(): Boolean = when (this) { + is CancellationException, + is ContentNotAvailableException, + is AgeRestrictedContentException, is GeographicRestrictionException, is PaidContentException, - is NeedLoginException, - is AgeRestrictedContentException, is PrivateContentException, + is NeedLoginException, + is AntiBotException, + is ReCaptchaException, is IllegalArgumentException -> false else -> true } diff --git a/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt index d98b6e2f..8b1315ba 100644 --- a/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt @@ -2,5 +2,5 @@ package dev.typetype.server.services import dev.typetype.server.models.FavoriteItem -internal fun FavoriteItem.withYoutubeFallbackTitle(): FavoriteItem = +fun FavoriteItem.withYoutubeFallbackTitle(): FavoriteItem = if (title.isNotBlank()) this else copy(title = YoutubeTypeTypeMapper.titleForUrl(videoUrl)) diff --git a/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/FavoritesService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt index dd06bcc9..140ab4d0 100644 --- a/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt @@ -54,7 +54,8 @@ class GitHubIssueService( appendLine("## API errors") report.context.apiErrors.take(10).forEach { error -> appendLine("- endpoint=${redactDomains(error.endpoint)} status=${error.status} requestId=${redactDomains(error.requestId ?: "n/a")} code=${error.code ?: "n/a"} timestamp=${error.timestamp}") - if (!error.message.isNullOrBlank()) appendLine(" message=${redactDomains(error.message)}") + val errorMessage = error.message + if (!errorMessage.isNullOrBlank()) appendLine(" message=${redactDomains(errorMessage)}") } } } diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt index 766b240c..031de29e 100644 --- a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt @@ -9,7 +9,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal object HistoryProgressMapper { +object HistoryProgressMapper { fun toHistoryItems(userId: String, rows: List): List { val items = rows.map { it.toHistoryItem() } val savedProgress = savedProgressSeconds(userId, items.map { it.url }) diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt similarity index 70% rename from src/main/kotlin/dev/typetype/server/services/HistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt index 4d065d51..f2109400 100644 --- a/src/main/kotlin/dev/typetype/server/services/HistoryService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt @@ -3,14 +3,17 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.HistoryItem import org.jetbrains.exposed.v1.core.LowerCase import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.greaterEq +import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.less import org.jetbrains.exposed.v1.core.like +import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.andWhere import org.jetbrains.exposed.v1.jdbc.batchInsert @@ -26,20 +29,49 @@ class HistoryService { .orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC) .toList() HistoryProgressMapper.toHistoryItemsForExport(userId, rows) + }.withSubscriptionAvatars(userId) + + suspend fun search(userId: String, q: String?, from: Long?, to: Long?, limit: Int, offset: Int): Pair, Long> { + val (items, total) = DatabaseFactory.query { + val query = HistoryTable.selectAll().where { HistoryTable.userId eq userId } + if (!q.isNullOrBlank()) { + val pattern = "%${q.lowercase()}%" + query.andWhere { (LowerCase(HistoryTable.title) like pattern) or (LowerCase(HistoryTable.channelName) like pattern) } + } + if (from != null) query.andWhere { HistoryTable.watchedAt greaterEq from } + if (to != null) query.andWhere { HistoryTable.watchedAt less to } + val total = query.count() + val rows = query.orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC).limit(limit).offset(offset.toLong()).toList() + HistoryProgressMapper.toHistoryItems(userId, rows) to total + } + return items.withSubscriptionAvatars(userId) to total } - suspend fun search(userId: String, q: String?, from: Long?, to: Long?, limit: Int, offset: Int): Pair, Long> = DatabaseFactory.query { - val query = HistoryTable.selectAll().where { HistoryTable.userId eq userId } - if (!q.isNullOrBlank()) { - val pattern = "%${q.lowercase()}%" - query.andWhere { (LowerCase(HistoryTable.title) like pattern) or (LowerCase(HistoryTable.channelName) like pattern) } + private suspend fun List.withSubscriptionAvatars(userId: String): List { + val missingUrls = filter { it.channelAvatar.isBlank() && it.channelUrl.isNotBlank() } + .map { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + .distinct() + if (missingUrls.isEmpty()) return this + val avatars = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { + (SubscriptionsTable.userId eq userId) and + (SubscriptionsTable.channelUrl inList missingUrls) and + (SubscriptionsTable.avatarUrl neq "") + } + .map { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) to + it[SubscriptionsTable.avatarUrl] + } + .toMap() + } + if (avatars.isEmpty()) return this + return map { item -> + if (item.channelAvatar.isNotBlank()) item + else avatars[ChannelUrlCanonicalizer.canonicalize(item.channelUrl)]?.let { + item.copy(channelAvatar = it) + } ?: item } - if (from != null) query.andWhere { HistoryTable.watchedAt greaterEq from } - if (to != null) query.andWhere { HistoryTable.watchedAt less to } - val total = query.count() - val rows = query.orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC).limit(limit).offset(offset.toLong()).toList() - val items = HistoryProgressMapper.toHistoryItems(userId, rows) - items to total } suspend fun add(userId: String, item: HistoryItem): HistoryItem = insert(userId, item, System.currentTimeMillis()) diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt index 428be477..8d2fea2b 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -internal class HlsManifestCache(private val cache: CacheService) { +class HlsManifestCache(private val cache: CacheService) { suspend fun get(manifestUrl: String): String? = runCatching { cache.get(key(manifestUrl)) }.getOrNull() diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt index 4a3daa14..7d7878c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt @@ -4,7 +4,7 @@ import java.net.URLEncoder import java.net.URI import java.nio.charset.StandardCharsets -internal fun isManifestUrl(url: String): Boolean { +fun isManifestUrl(url: String): Boolean { val uri = runCatching { URI(url) }.getOrNull() ?: return false if (uri.scheme != "http" && uri.scheme != "https") return false if (uri.path.contains("/file/seg.ts")) return false @@ -12,7 +12,7 @@ internal fun isManifestUrl(url: String): Boolean { return host == "manifest.googlevideo.com" || uri.path.endsWith(".m3u8", ignoreCase = true) } -internal fun rewriteYouTubeHlsManifest( +fun rewriteYouTubeHlsManifest( manifest: String, mapManifestUrl: (String) -> String = ::toHlsProxyUrl, ): String { @@ -27,7 +27,7 @@ internal fun rewriteYouTubeHlsManifest( } } -internal fun toHlsProxyUrl(url: String): String { +fun toHlsProxyUrl(url: String): String { val encoded = URLEncoder.encode(url, StandardCharsets.UTF_8) return if (isManifestUrl(url)) "hls-manifest?url=$encoded" else "../proxy?url=$encoded" } diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index 5a07b108..a4b62da9 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -49,7 +49,9 @@ class HlsManifestService( } private suspend fun cachedOrFetch(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult { - val cache = manifestCache.takeUnless { isProviderManifestUrl(manifestUrl) } + val cache = manifestCache.takeUnless { + isProviderManifestUrl(manifestUrl) || isYoutubeLiveManifest(manifestUrl) + } val cacheKey = if (signManifestLinks) "signed:$manifestUrl" else manifestUrl cache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } val pending = CompletableDeferred>() @@ -82,13 +84,20 @@ class HlsManifestService( result: ExtractionResult, allowAttestedYoutubeHls: Boolean = false, ): ExtractionResult { - if (result is ExtractionResult.BadRequest) return result - if (result !is ExtractionResult.Success) return ExtractionResult.Failure("No HLS stream available for this video") - if (allowAttestedYoutubeHls && result.data.isLive) { - attestedYoutubeHls(result.data.id)?.let { return ExtractionResult.Success(it) } + val success = when (result) { + is ExtractionResult.BadRequest -> return result + is ExtractionResult.Failure -> return ExtractionResult.Failure(result.message, result.code, result.kind) + is ExtractionResult.Success -> result + } + if (allowAttestedYoutubeHls && success.data.isLive) { + attestedYoutubeHls(success.data.id)?.let { return ExtractionResult.Success(it) } + } + val hls = success.data.hlsUrl + return if (hls.isNotBlank()) { + ExtractionResult.Success(hls) + } else { + ExtractionResult.Failure("No HLS stream available for this video", "no_playable_streams") } - val hls = result.data.hlsUrl - return if (hls.isNotBlank()) ExtractionResult.Success(hls) else ExtractionResult.Failure("No HLS stream available for this video") } private suspend fun fetchAndRewrite(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult = @@ -151,4 +160,11 @@ class HlsManifestService( private fun isProviderManifestUrl(url: String): Boolean = isNicoNicoManifest(url) || isBilibiliManifest(url) + + private fun isYoutubeLiveManifest(url: String): Boolean = runCatching { + val uri = URI(url) + val host = uri.host?.lowercase().orEmpty() + val isYoutubeHost = host == "googlevideo.com" || host.endsWith(".googlevideo.com") + isYoutubeHost && uri.path.orEmpty().split('/').any { it == "yt_live_broadcast" } + }.getOrDefault(false) } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt index dc488378..4ecf90ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt @@ -5,6 +5,7 @@ data class HomeRecommendationCursor( val discoveryIndex: Int = 0, val subscriptionRun: Int = 0, val preferDiscovery: Boolean = true, + val rotationSeed: Long = 0L, val recentChannels: List = emptyList(), val recentSemanticKeys: List = emptyList(), val creatorMomentum: Map = emptyMap(), diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt index 748aba72..4a0fe7e4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt @@ -27,6 +27,7 @@ object HomeRecommendationCursorCodec { discoveryIndex = full.d, subscriptionRun = full.r, preferDiscovery = full.p == 1, + rotationSeed = full.f, recentChannels = full.c, recentSemanticKeys = full.k, creatorMomentum = full.m, @@ -75,6 +76,7 @@ object HomeRecommendationCursorCodec { d = cursor.discoveryIndex, r = cursor.subscriptionRun, p = if (cursor.preferDiscovery) 1 else 0, + f = cursor.rotationSeed, c = cursor.recentChannels, k = cursor.recentSemanticKeys, m = cursor.creatorMomentum, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt index cb017b2b..f759dcd3 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt @@ -6,6 +6,7 @@ object HomeRecommendationCursorFactory { discoveryIndex: Int, subscriptionRun: Int, preferDiscovery: Boolean, + rotationSeed: Long, personaState: HomeRecommendationPersonaState, snapshot: HomeRecommendationCursorMemory, ): String = HomeRecommendationCursorCodec.encode( @@ -14,6 +15,7 @@ object HomeRecommendationCursorFactory { discoveryIndex = discoveryIndex, subscriptionRun = subscriptionRun, preferDiscovery = preferDiscovery, + rotationSeed = rotationSeed, recentChannels = snapshot.recentChannels, recentSemanticKeys = snapshot.recentSemanticKeys, creatorMomentum = snapshot.creatorMomentum, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt index fd7a4d05..2275d6f4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt @@ -8,6 +8,7 @@ data class HomeRecommendationCursorPayload( val d: Int, val r: Int, val p: Int, + val f: Long = 0L, val c: List = emptyList(), val k: List = emptyList(), val m: Map = emptyMap(), diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt index 6dbbc459..910ccc2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem class HomeRecommendationMixState(cursor: HomeRecommendationCursor, context: HomeRecommendationSessionContext) { + val rotationSeed = cursor.rotationSeed val channelCount = mutableMapOf() val memory = HomeRecommendationMomentumMemory(cursor) var personaState = HomeRecommendationPersonaDrift.seed(context, cursor.personaState) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt index b96a9ee8..b85bc1fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt @@ -17,6 +17,7 @@ object HomeRecommendationNextCursor { discoveryIndex = state.discoveryIndex, subscriptionRun = state.subscriptionRun, preferDiscovery = state.preferDiscovery, + rotationSeed = state.rotationSeed, personaState = state.personaState, snapshot = snapshot, ) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt similarity index 67% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt index 1e1a1483..62bf3af4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt @@ -8,20 +8,30 @@ object HomeRecommendationPageBuilder { mode: HomeRecommendationPoolMode, poolResolver: HomeRecommendationPoolResolver, ): HomeRecommendationsResponse { - val pool = poolResolver.resolve( + val cursor = if (mode == HomeRecommendationPoolMode.FULL) { + args.cursor.withRotationSeed() + } else { + args.cursor + } + val resolvedPool = poolResolver.resolve( userId = args.userId, serviceId = args.serviceId, mode = mode, context = args.context, ) + val pool = if (mode == HomeRecommendationPoolMode.FULL && cursor.rotationSeed != 0L) { + HomeRecommendationPoolRotation.apply(resolvedPool, cursor.rotationSeed) + } else { + resolvedPool + } val page = HomeRecommendationMixer.mix( pool = pool, - cursor = args.cursor, + cursor = cursor, limit = args.limit, context = args.context.sessionContext, sourceWeights = HomeRecommendationExploreBonus.apply( sourceWeights = pool.sourceWeights, - pageIndex = HomeRecommendationCursorPageIndex.from(args.cursor, args.limit), + pageIndex = HomeRecommendationCursorPageIndex.from(cursor, args.limit), ), mode = mode, userId = args.userId, @@ -31,19 +41,19 @@ object HomeRecommendationPageBuilder { val refresh = HomeRecommendationShortsRefresher.refresh( pool = pool, page = page, - cursor = args.cursor, + cursor = cursor, ) if (refresh.pool == pool && refresh.cursorOverride == null) { page } else { HomeRecommendationMixer.mix( pool = refresh.pool, - cursor = refresh.cursorOverride ?: args.cursor, + cursor = refresh.cursorOverride ?: cursor, limit = args.limit, context = args.context.sessionContext, sourceWeights = HomeRecommendationExploreBonus.apply( sourceWeights = refresh.pool.sourceWeights, - pageIndex = HomeRecommendationCursorPageIndex.from(refresh.cursorOverride ?: args.cursor, args.limit), + pageIndex = HomeRecommendationCursorPageIndex.from(refresh.cursorOverride ?: cursor, args.limit), ), mode = mode, userId = args.userId, @@ -64,4 +74,14 @@ object HomeRecommendationPageBuilder { }, ) } + + private fun HomeRecommendationCursor.withRotationSeed(): HomeRecommendationCursor = + if (rotationSeed != 0L || !isInitialPage()) { + this + } else { + copy(rotationSeed = HomeRecommendationPoolRotation.newSeed()) + } + + private fun HomeRecommendationCursor.isInitialPage(): Boolean = + subscriptionIndex == 0 && discoveryIndex == 0 && subscriptionRun == 0 && recentUrls.isEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt new file mode 100644 index 00000000..bb11c2b1 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt @@ -0,0 +1,38 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.HomeRecommendationPool +import dev.typetype.server.models.VideoItem +import java.util.Random +import java.util.concurrent.ThreadLocalRandom + +object HomeRecommendationPoolRotation { + private const val ROTATION_WINDOW_SIZE = 40 + private const val SUBSCRIPTION_SALT = 0x5EED1234L + private const val DISCOVERY_SALT = 0xD15C0A7L + + fun newSeed(): Long { + var seed = ThreadLocalRandom.current().nextLong() + while (seed == 0L) seed = ThreadLocalRandom.current().nextLong() + return seed + } + + fun apply(pool: HomeRecommendationPool, seed: Long): HomeRecommendationPool = pool.copy( + subscriptions = rotate(pool.subscriptions, seed xor SUBSCRIPTION_SALT), + discovery = rotate(pool.discovery, seed xor DISCOVERY_SALT), + ) + + private fun rotate(source: List, seed: Long): List { + if (source.size <= 1) return source + val windowSize = minOf(source.size, ROTATION_WINDOW_SIZE) + if (windowSize <= 1) return source + val window = source.take(windowSize).toMutableList() + val random = Random(seed) + for (index in window.lastIndex downTo 1) { + val swapIndex = random.nextInt(index + 1) + val item = window[index] + window[index] = window[swapIndex] + window[swapIndex] = item + } + return window + source.drop(windowSize) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt index a78aea45..3a244b6c 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class HomeWarmupTracker( +class HomeWarmupTracker( private val throttleMs: Long, private val activeTtlMs: Long, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/InstanceService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/InstanceService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/InstanceService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt b/server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt similarity index 61% rename from src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt index ff5a46d3..077fccc9 100644 --- a/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.stream.StreamType -internal data class StreamLiveMetadata( +data class StreamLiveMetadata( val streamType: String, val hlsUrl: String, val dashMpdUrl: String, @@ -12,9 +12,9 @@ internal data class StreamLiveMetadata( val hasLiveManifest: Boolean, ) -internal fun StreamType?.toApiStreamType(fallback: String = ""): String = this?.name?.lowercase() ?: fallback +fun StreamType?.toApiStreamType(fallback: String = ""): String = this?.name?.lowercase() ?: fallback -internal fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMpdUrl: String?): StreamLiveMetadata { +fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMpdUrl: String?): StreamLiveMetadata { val apiStreamType = streamType.toApiStreamType() val apiHlsUrl = hlsUrl?.takeIf { it.startsWith("http") } ?: "" val apiDashMpdUrl = dashMpdUrl?.takeIf { it.startsWith("http") } ?: "" @@ -29,8 +29,8 @@ internal fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMp ) } -internal fun String.isLiveStreamType(): Boolean = this == "live_stream" || this == "audio_live_stream" +fun String.isLiveStreamType(): Boolean = this == "live_stream" || this == "audio_live_stream" -internal fun String.isPostLiveStreamType(): Boolean = this == "post_live_stream" || this == "post_live_audio_stream" +fun String.isPostLiveStreamType(): Boolean = this == "post_live_stream" || this == "post_live_audio_stream" -internal fun String.isLiveContentType(): Boolean = isLiveStreamType() || isPostLiveStreamType() +fun String.isLiveContentType(): Boolean = isLiveStreamType() || isPostLiveStreamType() diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt index 578edcd2..58dd025b 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt @@ -5,14 +5,14 @@ import dev.typetype.server.models.VideoStreamItem import org.schabi.newpipe.extractor.stream.AudioStream import org.schabi.newpipe.extractor.stream.VideoStream -internal fun VideoStreamItem.manifestRepresentationId(index: Int): String = "v-${itag.takeIf { it > 0 } ?: index}" +fun VideoStreamItem.manifestRepresentationId(index: Int): String = "v-${itag.takeIf { it > 0 } ?: index}" -internal fun AudioStreamItem.manifestRepresentationId(index: Int): String = +fun AudioStreamItem.manifestRepresentationId(index: Int): String = stableAudioRepresentationId(itag = itag, trackId = audioTrackId, index = index) -internal fun VideoStream.manifestRepresentationId(index: Int): String = "v-${getItag().takeIf { it > 0 } ?: index}" +fun VideoStream.manifestRepresentationId(index: Int): String = "v-${getItag().takeIf { it > 0 } ?: index}" -internal fun AudioStream.manifestRepresentationId(index: Int): String = +fun AudioStream.manifestRepresentationId(index: Int): String = stableAudioRepresentationId(itag = getItag(), trackId = getAudioTrackId(), index = index) private fun stableAudioRepresentationId(itag: Int, trackId: String?, index: Int): String { diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/ManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ManifestService.kt index 81c2ed36..c4ac4583 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestService.kt @@ -17,7 +17,7 @@ class ManifestService( val videos = compatibleVideoStreams(info.videoOnlyStreams) val audios = compatibleAudioStreams(info.audioStreams, info.preferredDefaultAudioTrackId) if (videos.isEmpty() && audios.isEmpty()) - return ExtractionResult.Failure("No compatible streams found for DASH manifest") + return ExtractionResult.Failure("No compatible streams found for DASH manifest", "no_playable_streams") return ExtractionResult.Success(buildMpd(videos, audios, info.duration)) } @@ -129,7 +129,7 @@ class ManifestService( URLEncoder.encode(url, StandardCharsets.UTF_8) private fun ExtractionResult.recast(): ExtractionResult = when (this) { is ExtractionResult.Success -> ExtractionResult.Success(data.toString()) - is ExtractionResult.BadRequest -> ExtractionResult.BadRequest(message) - is ExtractionResult.Failure -> ExtractionResult.Failure(message) + is ExtractionResult.BadRequest -> ExtractionResult.BadRequest(message, code) + is ExtractionResult.Failure -> ExtractionResult.Failure(message, code, kind) } } diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt index 97b855b0..bdef81ff 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream import org.schabi.newpipe.extractor.stream.DeliveryMethod import org.schabi.newpipe.extractor.stream.VideoStream -internal object NativeManifestBuilder { +object NativeManifestBuilder { fun build(videos: List, audios: List, duration: Long, preferredAudioTrackId: String?): String { val sb = StringBuilder() diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt index 0d216102..bd8c57c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt @@ -40,7 +40,7 @@ class NativeManifestService { val audios = compatibleAudioStreams(info.audioStreams) val preferredAudioTrackId = resolvePreferredAudioTrackId(audios) if (videos.isEmpty() && audios.isEmpty()) - return ExtractionResult.Failure("No compatible streams found") + return ExtractionResult.Failure("No compatible streams found", "no_playable_streams") return runCatching { ExtractionResult.Success( NativeManifestBuilder.build(videos, audios, info.duration, preferredAudioTrackId) diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt index 3b4e62d0..92203121 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt @@ -3,10 +3,10 @@ package dev.typetype.server.services import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal val REPRESENTATION_REGEX = Regex("""""") +val REPRESENTATION_REGEX = Regex("""""") private val MANIFEST_GOOGLEVIDEO_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/[^"<\s]+""") -internal fun rewriteManifestUrls(manifest: String): String = +fun rewriteManifestUrls(manifest: String): String = manifest.replace(MANIFEST_GOOGLEVIDEO_REGEX) { match -> val raw = match.value.replace("&", "&") val withPlaceholders = raw @@ -20,14 +20,14 @@ internal fun rewriteManifestUrls(manifest: String): String = .replace("TMPL_TIME", "\$Time\$") } -internal fun encodeUrl(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) +fun encodeUrl(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) -internal fun videoMimeType(codec: String): String = when { +fun videoMimeType(codec: String): String = when { codec.startsWith("vp9") || codec.startsWith("vp09") || codec.startsWith("av01") -> "video/webm" else -> "video/mp4" } -internal fun audioMimeType(codec: String): String = when { +fun audioMimeType(codec: String): String = when { codec.startsWith("opus") || codec.startsWith("vorbis") -> "audio/webm" else -> "audio/mp4" } diff --git a/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 8bb0c0f9..5f8b1a60 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -13,7 +13,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit -internal fun parseNicoCookie(fragment: String): String? { +fun parseNicoCookie(fragment: String): String? { val decoded = URLDecoder.decode(fragment, StandardCharsets.UTF_8) val cookieParam = decoded.split("&") .firstOrNull { it.startsWith("cookie=") } @@ -23,7 +23,7 @@ internal fun parseNicoCookie(fragment: String): String? { return cookieParam.substring(eqIdx + 1) } -internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: String? = null, proxyPath: String = "nicovideo"): String { +fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: String? = null, proxyPath: String = "nicovideo"): String { val base = URI(baseUrl) val uriAttr = Regex("""URI="([^"]+)"""") val bidSuffix = if (domandBid != null) "&domand_bid=${URLEncoder.encode(domandBid, StandardCharsets.UTF_8)}" else "" diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NotificationsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 9b906a23..0eea6e4c 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -9,18 +9,20 @@ import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request import java.io.ByteArrayInputStream +import java.net.URI import java.net.URLEncoder import java.nio.charset.StandardCharsets +import java.util.concurrent.TimeUnit -internal val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") +val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") private val CPN_TRACKING_PARAM_REGEX = Regex("[&?]cpn=[^&]*") private val PPPID_TRACKING_PARAM_REGEX = Regex("[&?]pppid=[^&]*") -internal fun stripTrackingParams(url: String): String = +fun stripTrackingParams(url: String): String = url.replace(CPN_TRACKING_PARAM_REGEX, "") .replace(PPPID_TRACKING_PARAM_REGEX, "") -internal fun rewriteHlsManifest(manifest: String): String = +fun rewriteHlsManifest(manifest: String): String = manifest.replace(GOOGLEVIDEO_URL_REGEX) { match -> "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } @@ -30,6 +32,11 @@ class OkHttpProxyService( private val mediaHandleService: ProviderMediaHandleService? = null, ) : ProxyService, ProviderMediaAwareProxyService { private val executor = ProxyHttpExecutor(client) + private val boundedExecutor = ProxyHttpExecutor( + client.newBuilder() + .callTimeout(30, TimeUnit.SECONDS) + .build(), + ) override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = false) @@ -53,6 +60,7 @@ class OkHttpProxyService( val fragment = if (hashIdx >= 0) url.substring(hashIdx + 1) else "" val resolvedDomandBid = domandBid ?: if (fragment.isNotBlank()) parseNicoCookie(fragment) else null validateProxyUrl(fetchUrl)?.let { return@withContext ExtractionResult.BadRequest(it) } + val requestExecutor = if (rangeHeader == null) boundedExecutor else executor runCatching { val cleanUrl = stripTrackingParams(fetchUrl) val bilibili = isBilibili(cleanUrl) @@ -67,9 +75,9 @@ class OkHttpProxyService( if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() if (bilibili && rangeHeader != null) { - return@withContext readBilibiliRangeWithRetry(executor::execute, request, requestContext::ensureActive) + return@withContext readBilibiliRangeWithRetry(requestExecutor::execute, request, requestContext::ensureActive) } - executor.execute(request) + requestExecutor.execute(request) }.fold( onSuccess = { response -> val body = response.body @@ -149,7 +157,7 @@ class OkHttpProxyService( } private fun isHls(contentType: String, url: String): Boolean = - contentType.contains("mpegurl", ignoreCase = true) || url.contains(".m3u8", ignoreCase = true) + contentType.contains("mpegurl", ignoreCase = true) || runCatching { URI(url).path?.endsWith(".m3u8", ignoreCase = true) == true }.getOrDefault(false) companion object { private const val BILIBILI_REFERER = "https://www.bilibili.com" @@ -162,7 +170,7 @@ class OkHttpProxyService( } } -internal fun acceptsYoutubeThumbnailFallback(url: String, status: Int, contentType: String?): Boolean { +fun acceptsYoutubeThumbnailFallback(url: String, status: Int, contentType: String?): Boolean { val host = runCatching { java.net.URI(url).host.orEmpty() }.getOrDefault("") return status == 404 && host.endsWith("ytimg.com") && contentType?.startsWith("image/") == true } diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt index 4a321af8..0c485ee2 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt @@ -15,9 +15,6 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response -import okhttp3.WebSocket -import okhttp3.WebSocketListener -import okio.ByteString class OkHttpYoutubeRemoteBrowserClient( private val serviceUrl: String, @@ -55,12 +52,17 @@ class OkHttpYoutubeRemoteBrowserClient( ): Unit = coroutineScope { val done = CompletableDeferred() val outbound = Channel(config.outboundQueueSize, BufferOverflow.DROP_OLDEST) - val socket = client.newWebSocket(tokenWebSocketRequest(tokenSessionId, internalToken), listener(outbound, done, config)) + val listener = YoutubeRemoteBrowserBridgeListener(tokenSessionId, outbound, done, config) + val socket = client.newWebSocket(tokenWebSocketRequest(tokenSessionId, internalToken), listener) + var inputs = 0 val outboundJob = launch { for (frame in outbound) serverSession.send(frame) } val inboundJob = launch { for (frame in serverSession.incoming) { if (frame is Frame.Text) { - YoutubeRemoteBrowserMessageGuard.frontendText(frame.readText(), config.maxInputBytes)?.let(socket::send) + YoutubeRemoteBrowserMessageGuard.frontendText(frame.readText(), config.maxInputBytes)?.let { + inputs += 1 + socket.send(it) + } } if (frame is Frame.Close) done.complete(Unit) } @@ -72,6 +74,7 @@ class OkHttpYoutubeRemoteBrowserClient( outbound.close() outboundJob.cancel() inboundJob.cancel() + YoutubeRemoteBrowserAudit.bridgeEnded(tokenSessionId, listener.texts.get(), listener.frames.get(), inputs) } private fun decodeStartResponse(response: Response): YoutubeRemoteBrowserTokenStartResponse? { @@ -85,27 +88,6 @@ class OkHttpYoutubeRemoteBrowserClient( .header(INTERNAL_HEADER, internalToken) .build() - private fun listener( - outbound: Channel, - done: CompletableDeferred, - config: YoutubeRemoteBrowserConfig, - ): WebSocketListener = object : WebSocketListener() { - override fun onMessage(webSocket: WebSocket, text: String) { - YoutubeRemoteBrowserMessageGuard.tokenText(text)?.let { outbound.trySend(Frame.Text(it)) } - } - - override fun onMessage(webSocket: WebSocket, bytes: ByteString) { - if (bytes.size <= config.maxFrameBytes) outbound.trySend(Frame.Binary(true, bytes.toByteArray())) - } - - override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - done.complete(Unit) - } - - override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - done.complete(Unit) - } - } private fun webSocketBaseUrl(): String = serviceUrl.trimEnd('/').replaceFirst("https://", "wss://").replaceFirst("http://", "ws://") diff --git a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt index dca441e0..5c2db6a3 100644 --- a/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt @@ -8,7 +8,7 @@ import kotlinx.serialization.json.Json import org.schabi.newpipe.extractor.Page @Serializable -internal data class PageSurrogate( +data class PageSurrogate( val url: String?, val id: String?, val ids: List?, @@ -16,7 +16,7 @@ internal data class PageSurrogate( val body: String?, ) -internal fun Page.toCursor(): String { +fun Page.toCursor(): String { val surrogate = PageSurrogate( url = url, id = id, @@ -27,7 +27,7 @@ internal fun Page.toCursor(): String { return Base64.getEncoder().encodeToString(Json.encodeToString(surrogate).toByteArray()) } -internal fun String.toPage(): Page { +fun String.toPage(): Page { val json = String(Base64.getDecoder().decode(this)) val surrogate = Json.decodeFromString(json) return Page( diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt index 162d57ef..053939e6 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt @@ -61,7 +61,7 @@ class PipePipeChannelService : ChannelService { private fun extractFirstPage(url: String, sort: String?): ChannelResponse { val service = NewPipe.getServiceByUrl(url) - val tab = url.toChannelTab(sort) + val tab = service.toChannelTab(url, sort) if (tab != null) { val channelUrl = url.toBaseChannelUrl(tab) val metadata = runCatching { ChannelInfo.getInfo(channelUrl) }.getOrNull() @@ -74,7 +74,7 @@ class PipePipeChannelService : ChannelService { private fun extractMorePage(url: String, page: Page, sort: String?): ChannelResponse { val service = NewPipe.getServiceByUrl(url) - val tab = url.toChannelTab(sort) + val tab = service.toChannelTab(url, sort) if (tab != null) { val extractor = service.channelTabExtractor(url, channelId(url.toBaseChannelUrl(tab), service), tab, sort) return extractor.getPage(page).toChannelTabResponse() diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt index 422b215d..648ebc1a 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.comments.CommentsInfo import org.schabi.newpipe.extractor.comments.CommentsInfoItem -internal fun String.normalizeHttpSchema(): String = replace("httpss://", "https://") +fun String.normalizeHttpSchema(): String = replace("httpss://", "https://") class PipePipeCommentService : CommentService { diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt new file mode 100644 index 00000000..285da922 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt @@ -0,0 +1,108 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.stream.StreamExtractor + +class PipePipeLiveHlsStreamService( + private val extractorForUrl: (String) -> StreamExtractor = ::createStreamExtractor, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") + + return withContext(Dispatchers.IO) { + try { + val extractor = extractorForUrl(url) + val response = withTimeout(LIVE_EXTRACTION_TIMEOUT_MS) { + runPipePipeCall { + extractor.fetchPage() + val streamType = extractor.streamType + val type = streamType.toApiStreamType() + if (!type.isLiveStreamType()) return@runPipePipeCall null + + val hlsUrl = extractor.hlsUrl + if (hlsUrl.isBlank() || extractor.requiresMembership()) { + return@runPipePipeCall null + } + extractor.toLiveHlsResponse(type, hlsUrl) + } + } + response?.let { ExtractionResult.Success(it) } + ?: ExtractionResult.Failure( + "No active YouTube live stream is available", + "live_stream_unavailable", + ) + } catch (_: TimeoutCancellationException) { + ExtractionResult.Failure("YouTube live extraction timed out", "live_stream_unavailable") + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + StreamExtractionErrorMapper.map(error, sourceUrl = url) + } + } + } + + private fun StreamExtractor.toLiveHlsResponse(streamType: String, hlsUrl: String) = StreamResponse( + id = id, + title = name.orEmpty(), + uploaderName = optionalExtractorValue("") { uploaderName }, + uploaderUrl = optionalExtractorValue("") { uploaderUrl }, + uploaderAvatarUrl = optionalExtractorValue("") { uploaderAvatarUrl }, + thumbnailUrl = optionalExtractorValue("") { thumbnailUrl }, + description = optionalExtractorValue("") { description.content.orEmpty() }, + duration = optionalExtractorValue(0L) { length.coerceAtLeast(0L) }, + viewCount = optionalExtractorValue(-1L) { viewCount }, + likeCount = -1L, + dislikeCount = -1L, + uploadDate = "", + uploaded = -1L, + uploaderSubscriberCount = -1L, + uploaderVerified = false, + category = "", + license = "", + visibility = "public", + tags = emptyList(), + streamType = streamType, + isLive = true, + isPostLive = false, + isLiveContent = true, + hasLiveManifest = true, + isShortFormContent = optionalExtractorValue(false) { isShortFormContent }, + requiresMembership = false, + startPosition = optionalExtractorValue(0L) { timeStamp.coerceAtLeast(0L) }, + streamSegments = emptyList(), + hlsUrl = hlsUrl, + dashMpdUrl = "", + videoStreams = emptyList(), + audioStreams = emptyList(), + originalAudioTrackId = null, + preferredDefaultAudioTrackId = null, + videoOnlyStreams = emptyList(), + subtitles = emptyList(), + previewFrames = emptyList(), + sponsorBlockSegments = emptyList(), + relatedStreams = emptyList(), + ) + + private companion object { + const val LIVE_EXTRACTION_TIMEOUT_MS = 8_000L + + fun createStreamExtractor(url: String): StreamExtractor { + val service = NewPipe.getServiceByUrl(url) + return service.getStreamExtractor(service.streamLHFactory.fromUrl(url)) + } + + inline fun optionalExtractorValue(default: T, value: () -> T): T = + try { + value() + } catch (_: Exception) { + default + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt index 0d2579f7..7621ca58 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.playlist.PlaylistInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal class PipePipePodcastEpisodesService { +class PipePipePodcastEpisodesService { suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult = withContext(Dispatchers.IO) { diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt similarity index 53% rename from src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index 85209166..1aadb0f9 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -1,10 +1,12 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.cache.CacheJson import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.SponsorBlockSegmentItem import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -31,37 +33,74 @@ private val ALL_SPONSOR_BLOCK_SETTINGS = SponsorBlockApiSettings().also { it.includeFillerCategory = true } -internal class PipePipeStreamService( +class PipePipeStreamService( private val cache: CacheService, private val subtitleService: YouTubeSubtitleService, private val bilibiliRelatedService: BilibiliRelatedService, - private val sabrInfoSink: (suspend (String, YoutubeSabrInfo) -> Unit)? = null, + private val sabrInfoSink: ((String, YoutubeSabrInfo) -> Unit)? = null, + private val fetchSupplementalSubtitles: Boolean = true, ) : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult = withContext(Dispatchers.IO) { runCatching { withExtractionRetry { - val service = NewPipe.getServiceByUrl(url) - val linkHandler = service.streamLHFactory.fromUrl(url) - val extractor: StreamExtractor = service.getStreamExtractor(linkHandler) - withTimeout(30_000L) { runPipePipeCall { extractor.fetchPage() } } + val (service, linkHandler, extractor) = playbackTracePhase("extractor_setup") { + val service = NewPipe.getServiceByUrl(url) + val linkHandler = service.streamLHFactory.fromUrl(url) + val extractor: StreamExtractor = service.getStreamExtractor(linkHandler) + Triple(service, linkHandler, extractor) + } + playbackTracePhase("extractor_fetch_page") { + withTimeout(30_000L) { runPipePipeCall { extractor.fetchPage() } } + } coroutineScope { val streamInfoDeferred = async { - withTimeout(30_000L) { runPipePipeCall { StreamInfo.getInfo(extractor) } } + playbackTracePhase("stream_info") { + withTimeout(30_000L) { runPipePipeCall { StreamInfo.getInfo(extractor) } } + } + } + val segmentsDeferred = async { + playbackTracePhase("sponsorblock") { resolveSegments(extractor) } } - val segmentsDeferred = async { resolveSegments(extractor) } val streamInfo = streamInfoDeferred.await() rememberSabrInfo(streamInfo) streamInfo.setSponsorBlockSegments(segmentsDeferred.await()) val response = StreamAudioContractResolver.apply(streamInfo.toStreamResponse()) - val withSubtitles = if (response.subtitles.isEmpty() && service.serviceId == 0) { - response.copy(subtitles = subtitleService.fetchSubtitles(streamInfo.id)) - } else { - response + val withSubtitles = when { + !fetchSupplementalSubtitles -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=disabled") + } + response.subtitles.isNotEmpty() -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=already_present") + } + service.serviceId != 0 -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=not_applicable") + } + else -> { + var subtitleStatus = "loaded" + val subtitles = playbackTracePhase("supplemental_subtitle_fetch") { + try { + subtitleService.fetchSubtitles(streamInfo.id) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + subtitleStatus = "failed:${error.javaClass.simpleName}" + emptyList() + } + } + PlaybackTraceLog.record( + "supplemental_subtitles", + "status=$subtitleStatus count=${subtitles.size}", + ) + response.copy(subtitles = subtitles) + } } - if (service.serviceId == BILIBILI_SERVICE_ID) bilibiliRelatedService.patchRelatedStreams(withSubtitles, linkHandler.url) - else withSubtitles + if (service.serviceId == BILIBILI_SERVICE_ID) { + playbackTracePhase("bilibili_related_streams") { + bilibiliRelatedService.patchRelatedStreams(withSubtitles, linkHandler.url) + } + } else withSubtitles } } }.fold( @@ -70,7 +109,7 @@ internal class PipePipeStreamService( ) } - private suspend fun rememberSabrInfo(streamInfo: StreamInfo): Unit { + private fun rememberSabrInfo(streamInfo: StreamInfo): Unit { val sink = sabrInfoSink ?: return val info = sequence { streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } @@ -119,5 +158,24 @@ internal class PipePipeStreamService( } } -internal suspend fun runPipePipeCall(block: () -> T): T = +private suspend fun playbackTracePhase(phase: String, block: suspend () -> T): T { + val startedAt = System.nanoTime() + var outcome = "ok" + try { + return block() + } catch (error: CancellationException) { + outcome = "cancelled" + throw error + } catch (error: Exception) { + outcome = "error:${error.javaClass.simpleName}" + throw error + } finally { + PlaybackTraceLog.record( + "pipepipe_phase", + "phase=$phase outcome=$outcome durationMs=${(System.nanoTime() - startedAt) / 1_000_000}", + ) + } +} + +suspend fun runPipePipeCall(block: () -> T): T = runInterruptible(context = Dispatchers.IO, block = block) diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt index 0652c8c2..e29a0aa8 100644 --- a/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.PlaylistResultItem import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem -internal fun PlaylistInfoItem.toPlaylistResultItem(): PlaylistResultItem = PlaylistResultItem( +fun PlaylistInfoItem.toPlaylistResultItem(): PlaylistResultItem = PlaylistResultItem( id = url ?: "", title = name ?: "", url = url ?: "", diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/PlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt index df2e9ad5..fe4c7e15 100644 --- a/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt @@ -29,8 +29,23 @@ class PlaylistService { .where { PlaylistVideosTable.userId eq userId } .groupBy(PlaylistVideosTable.playlistId) .associate { row -> row[PlaylistVideosTable.playlistId] to row[videoCount].toInt() } + val firstVideos = PlaylistVideosTable + .selectAll() + .withDistinctOn(PlaylistVideosTable.playlistId) + .where { PlaylistVideosTable.userId eq userId } + .orderBy( + PlaylistVideosTable.playlistId to SortOrder.ASC, + PlaylistVideosTable.position to SortOrder.ASC, + ) + .associate { row -> + row[PlaylistVideosTable.playlistId] to row.toPlaylistVideoItem(emptyMap()) + } playlists.map { row -> - row.toPlaylistSummary(videoCounts[row[PlaylistsTable.id]] ?: 0) + val playlistId = row[PlaylistsTable.id] + row.toPlaylistSummary( + videoCount = videoCounts[playlistId] ?: 0, + firstVideo = firstVideos[playlistId], + ) } } diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt index 1d07f770..94a0baf8 100644 --- a/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt @@ -12,7 +12,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal fun playlistProgressByUrl(userId: String, videoUrls: List): Map { +fun playlistProgressByUrl(userId: String, videoUrls: List): Map { val urls = videoUrls.distinct().filter { it.isNotBlank() } if (urls.isEmpty()) return emptyMap() return ProgressTable.selectAll() @@ -26,15 +26,19 @@ internal fun playlistProgressByUrl(userId: String, videoUrls: List): Map } } -internal fun ResultRow.toPlaylistSummary(videoCount: Int): PlaylistItem = PlaylistItem( +fun ResultRow.toPlaylistSummary( + videoCount: Int, + firstVideo: PlaylistVideoItem?, +): PlaylistItem = PlaylistItem( id = this[PlaylistsTable.id], name = this[PlaylistsTable.name], description = this[PlaylistsTable.description], + videos = listOfNotNull(firstVideo), videoCount = videoCount, createdAt = this[PlaylistsTable.createdAt], ) -internal fun ResultRow.toPlaylistVideoItem(progressByUrl: Map): PlaylistVideoItem { +fun ResultRow.toPlaylistVideoItem(progressByUrl: Map): PlaylistVideoItem { val url = this[PlaylistVideosTable.url] val duration = this[PlaylistVideosTable.duration] val progress = progressByUrl[url] diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt index 0ca1ba28..e6b918a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.PodcastItem import org.schabi.newpipe.extractor.playlist.PlaylistInfo import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem -internal fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( +fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -14,7 +14,7 @@ internal fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( +fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -24,7 +24,7 @@ internal fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun emptyPodcastItem(url: String): PodcastItem = PodcastItem( +fun emptyPodcastItem(url: String): PodcastItem = PodcastItem( id = url, title = "", url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PodcastService.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt new file mode 100644 index 00000000..8aba8d37 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt @@ -0,0 +1,124 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PresenceKeysTable +import dev.typetype.server.models.PresenceKeyCreatedResponse +import dev.typetype.server.models.PresenceKeyItem +import java.util.UUID +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +data class PresenceTokenPrincipal( + val keyId: String, + val userId: String, +) + +class PresenceKeyLimitException(message: String) : IllegalStateException(message) + +class PresenceKeyService( + private val nowProvider: () -> Long = System::currentTimeMillis, +) { + suspend fun create(userId: String, requestedName: String?): PresenceKeyCreatedResponse { + val name = requestedName?.trim().orEmpty() + require(name.isNotEmpty() && name.length <= MAX_NAME_LENGTH) { + "Presence key name must contain 1 to $MAX_NAME_LENGTH characters" + } + return DatabaseFactory.query { + val activeCount = PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .count() + if (activeCount >= MAX_KEYS_PER_USER) { + throw PresenceKeyLimitException("Presence key limit reached") + } + val now = nowProvider() + val id = UUID.randomUUID().toString() + val token = PresenceTokenCodec.issue() + val tokenPrefix = PresenceTokenCodec.prefix(token) + PresenceKeysTable.insert { + it[PresenceKeysTable.id] = id + it[PresenceKeysTable.userId] = userId + it[PresenceKeysTable.name] = name + it[PresenceKeysTable.tokenPrefix] = tokenPrefix + it[PresenceKeysTable.tokenHash] = PresenceTokenCodec.hash(token) + it[createdAt] = now + it[lastUsedAt] = null + } + PresenceKeyCreatedResponse( + key = item(id, name, tokenPrefix, now, null), + token = token, + ) + } + } + + suspend fun list(userId: String): List = DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .sortedBy { it[PresenceKeysTable.createdAt] } + .map { + item( + id = it[PresenceKeysTable.id], + name = it[PresenceKeysTable.name], + tokenPrefix = it[PresenceKeysTable.tokenPrefix], + createdAt = it[PresenceKeysTable.createdAt], + lastUsedAt = it[PresenceKeysTable.lastUsedAt], + ) + } + } + + suspend fun revoke(userId: String, keyId: String): Boolean = DatabaseFactory.query { + PresenceKeysTable.deleteWhere { + PresenceKeysTable.id eq keyId + PresenceKeysTable.userId eq userId + } > 0 + } + + suspend fun resolve(token: String): PresenceTokenPrincipal? { + if (!token.startsWith(PresenceTokenCodec.PREFIX)) return null + val hash = PresenceTokenCodec.hash(token) + val now = nowProvider() + return DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.tokenHash eq hash } + .singleOrNull() + ?.let { + PresenceKeysTable.update({ PresenceKeysTable.id eq it[PresenceKeysTable.id] }) { + it[lastUsedAt] = now + } + PresenceTokenPrincipal( + keyId = it[PresenceKeysTable.id], + userId = it[PresenceKeysTable.userId], + ) + } + } + } + + suspend fun hasActiveKey(userId: String): Boolean = DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .empty() + .not() + } + + private fun item( + id: String, + name: String, + tokenPrefix: String, + createdAt: Long, + lastUsedAt: Long?, + ) = PresenceKeyItem( + id = id, + name = name, + tokenPrefix = tokenPrefix, + scope = dev.typetype.server.models.PresenceScopes.READ, + createdAt = createdAt, + lastUsedAt = lastUsedAt, + ) + + private companion object { + const val MAX_KEYS_PER_USER = 10L + const val MAX_NAME_LENGTH = 50 + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PresenceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceService.kt new file mode 100644 index 00000000..30cc7feb --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceService.kt @@ -0,0 +1,48 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ActiveSessionNowPlayingItem +import dev.typetype.server.models.SessionPlaybackProgressRequest +import dev.typetype.server.models.SessionPlaybackStartRequest +import java.util.concurrent.ConcurrentHashMap + +class PresenceService( + private val nowProvider: () -> Long = System::currentTimeMillis, + private val hasActiveKey: suspend (String) -> Boolean = { false }, +) { + private val nowPlaying = ConcurrentHashMap() + + suspend fun reportPlaybackStart(userId: String, request: SessionPlaybackStartRequest): Unit = + record(userId, ActiveSessionNowPlayingMapper.fromStart(request, nowProvider())) + + suspend fun reportPlaybackProgress(userId: String, request: SessionPlaybackProgressRequest): Unit { + val current = nowPlaying[userId] + record(userId, ActiveSessionNowPlayingMapper.fromProgress(current, request, nowProvider())) + } + + fun reportPlaybackStop(userId: String): Unit { + nowPlaying.remove(userId) + } + + suspend fun current(userId: String): ActiveSessionNowPlayingItem? { + val now = nowProvider() + pruneExpired(now) + return nowPlaying[userId]?.takeIf { now - it.updatedAt <= ACTIVITY_TTL_MS } + } + + private suspend fun record(userId: String, value: ActiveSessionNowPlayingItem?) { + if (value == null) return + if (!hasActiveKey(userId)) { + nowPlaying.remove(userId) + return + } + nowPlaying[userId] = value + } + + private fun pruneExpired(now: Long): Unit { + nowPlaying.entries.removeIf { now - it.value.updatedAt > ACTIVITY_TTL_MS } + } + + companion object { + const val ACTIVITY_TTL_MS = 120_000L + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt new file mode 100644 index 00000000..2d49e1da --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 + +object PresenceTokenCodec { + const val PREFIX = "ttp1_" + private val random = SecureRandom() + + fun issue(): String { + val bytes = ByteArray(32) + random.nextBytes(bytes) + return PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) + } + + fun prefix(token: String): String = token.take(PREFIX.length + 6) + + fun hash(token: String): String = Base64.getUrlEncoder().withoutPadding().encodeToString( + MessageDigest.getInstance("SHA-256").digest(token.toByteArray(Charsets.UTF_8)), + ) + + fun sameHash(left: String, right: String): Boolean = + MessageDigest.isEqual(left.toByteArray(Charsets.UTF_8), right.toByteArray(Charsets.UTF_8)) +} diff --git a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProgressService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProgressService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProgressService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt index e7f2a361..cdb29c5d 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import java.net.URI -internal suspend fun rewriteProviderHlsManifest( +suspend fun rewriteProviderHlsManifest( manifest: String, baseUrl: String, mapUrl: suspend (String) -> String, diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt index 04d46dff..07f54644 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt @@ -19,13 +19,13 @@ import java.security.MessageDigest import java.util.Base64 import java.util.concurrent.ConcurrentHashMap -internal enum class ProviderMediaType { +enum class ProviderMediaType { BILIBILI, NICONICO, } @Serializable -internal data class ProviderMediaTarget( +data class ProviderMediaTarget( val url: String, val domandBid: String? = null, ) @@ -37,7 +37,7 @@ class ProviderMediaHandleService( private val writeLimiter = Semaphore(MAX_CONCURRENT_CACHE_WRITES) private val inFlightWrites = ConcurrentHashMap>() - internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { + suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { val hls = async { response.hlsUrl.handleIfRemote(provider) } val dash = async { response.dashMpdUrl.handleIfRemote(provider) } val video = async { response.videoStreams.withHandledVideoUrls(provider) } @@ -52,7 +52,7 @@ class ProviderMediaHandleService( ) } - internal suspend fun createPath(rawUrl: String, domandBid: String? = null): String { + suspend fun createPath(rawUrl: String, domandBid: String? = null): String { val (cleanUrl, fragment) = rawUrl.splitFragment() val resolvedBid = domandBid?.takeIf { it.isNotBlank() } ?: fragment.takeIf { it.isNotBlank() }?.let(::parseNicoCookie) @@ -81,14 +81,14 @@ class ProviderMediaHandleService( } } - internal suspend fun resolve(handle: String): ProviderMediaTarget? { + suspend fun resolve(handle: String): ProviderMediaTarget? { if (!HANDLE_PATTERN.matches(handle)) return null return cache.get(cacheKey(handle))?.let { encoded -> runCatching { CacheJson.decodeFromString(encoded) }.getOrNull() } } - internal fun relativeManifestPath(path: String): String = + fun relativeManifestPath(path: String): String = "../media/${path.substringAfterLast('/')}" private suspend fun String.handleIfRemote(provider: ProviderMediaType): String { diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt index 4f27c759..350eea33 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt @@ -12,7 +12,7 @@ import java.net.Proxy import java.net.UnknownHostException import java.util.concurrent.ConcurrentHashMap -internal class ProxyHttpExecutor( +class ProxyHttpExecutor( client: OkHttpClient, private val maxRedirects: Int = 5, ) { @@ -59,7 +59,7 @@ internal class ProxyHttpExecutor( } } -internal class ValidatingProxyDns(private val delegate: Dns) : Dns { +class ValidatingProxyDns(private val delegate: Dns) : Dns { private val trustedTransportHosts = ConcurrentHashMap.newKeySet() override fun lookup(hostname: String): List { diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/ProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt index 1286bc5e..5f677b1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt @@ -7,7 +7,7 @@ interface ProxyService { suspend fun pipe(url: String, rangeHeader: String?, domandBid: String? = null): ExtractionResult } -internal interface ProviderMediaAwareProxyService { +interface ProviderMediaAwareProxyService { suspend fun pipeProviderMedia( url: String, rangeHeader: String?, diff --git a/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt index 7513ab18..5e139ca8 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object PublicCachePolicy { +object PublicCachePolicy { fun trendingTtl(serviceId: Int): Long = when (serviceId) { BILIBILI_SERVICE_ID, NICONICO_SERVICE_ID -> 600L YOUTUBE_SERVICE_ID -> 1_800L diff --git a/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt index 8771fc29..24f4d574 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt @@ -6,7 +6,7 @@ import dev.typetype.server.models.ExtractionResult import kotlinx.serialization.KSerializer import org.slf4j.LoggerFactory -internal object PublicExtractionCache { +object PublicExtractionCache { private val logger = LoggerFactory.getLogger(PublicExtractionCache::class.java) suspend fun getOrLoad( diff --git a/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt index b9b5fb9b..9563e378 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt @@ -50,7 +50,7 @@ class PublicHlsManifestTokenService( private fun decode(value: String): String? = runCatching { String(decoder.decode(value), Charsets.UTF_8) }.getOrNull() companion object { - const val TTL_SECONDS = 900L + const val TTL_SECONDS = 86_400L private const val VERSION = "ph1" private const val PART_COUNT = 4 private const val MAX_TOKEN_LENGTH = 4096 diff --git a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt index 4b7d2c17..4ff44bb4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.PublicPlaylistItem import org.schabi.newpipe.extractor.playlist.PlaylistInfo -internal fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPlaylistItem( +fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPlaylistItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -13,7 +13,7 @@ internal fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPla playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun emptyPublicPlaylistItem(url: String): PublicPlaylistItem = PublicPlaylistItem( +fun emptyPublicPlaylistItem(url: String): PublicPlaylistItem = PublicPlaylistItem( id = url, title = "", url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt index 3233cad5..a98c34a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt @@ -1,5 +1,5 @@ package dev.typetype.server.services -internal object PublishedAtMapper { +object PublishedAtMapper { fun fromUploaded(uploaded: Long): Long? = uploaded.takeIf { it > 0L } } diff --git a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt index 5ad17817..f7a67f17 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt @@ -14,7 +14,7 @@ import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class PushDeviceRegistry( +class PushDeviceRegistry( private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), private val maxDevicesPerAccount: Int = DEFAULT_MAX_DEVICES, ) { @@ -135,9 +135,9 @@ internal class PushDeviceRegistry( } } -internal data class PushDevice(val id: String, val userId: String, val endpoint: String) +data class PushDevice(val id: String, val userId: String, val endpoint: String) -internal sealed interface DeviceRegistrationResult { +sealed interface DeviceRegistrationResult { data class Success(val response: PushDeviceRegistrationResponse) : DeviceRegistrationResult data class Invalid(val reason: String) : DeviceRegistrationResult data object UnsupportedPlatform : DeviceRegistrationResult diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt index 5d5315e3..921444fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt @@ -13,7 +13,7 @@ import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class PushNotificationDeliveryStore( +class PushNotificationDeliveryStore( private val clock: () -> Long, ) { fun markSeen(userId: String, candidate: PushCandidate, now: Long): Boolean = @@ -97,7 +97,7 @@ internal class PushNotificationDeliveryStore( } } -internal object PushNotificationDeliveryStatus { +object PushNotificationDeliveryStatus { const val PENDING = "pending" const val DELIVERED = "delivered" const val INVALID = "invalid" diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt index e82a5285..62bd2778 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -internal class PushNotificationScheduler( +class PushNotificationScheduler( private val service: PushNotificationService, private val intervalMs: Long = configuredIntervalMs(), private val initialDelayMs: Long = 10_000L, diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt index dd0ad996..fe0b4696 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt @@ -13,7 +13,7 @@ import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll -internal class PushNotificationService( +class PushNotificationService( private val subscriptionsService: SubscriptionsService, private val subscriptionFeedService: SubscriptionFeedService, private val preferenceService: ChannelNotificationPreferenceService, @@ -76,7 +76,7 @@ internal class PushNotificationService( } } - internal suspend fun pollUser(userId: String) { + suspend fun pollUser(userId: String) { val subscriptions = subscriptionsService.getAll(userId) if (subscriptions.isEmpty()) return val devices = deviceRegistry.activeDevices(userId, clock()) @@ -153,7 +153,7 @@ internal class PushNotificationService( } } -internal data class PushCandidate( +data class PushCandidate( val serviceId: Int, val channelId: String, val videoId: String, @@ -161,4 +161,4 @@ internal data class PushCandidate( val video: VideoItem, ) -internal data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) +data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt index fc1d548e..753f8f54 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.UnifiedPushNotificationPayload import dev.typetype.server.models.VideoItem import java.security.MessageDigest -internal object PushNotificationSupport { +object PushNotificationSupport { fun candidates( video: VideoItem, sources: Map>, diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt index 63bcdbac..059abc73 100644 --- a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.net.URI -internal fun recommendationServiceId(url: String): Int { +fun recommendationServiceId(url: String): Int { val host = runCatching { URI(url).host.orEmpty().lowercase() }.getOrDefault("") return when { host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> BILIBILI_SERVICE_ID diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt index bb546633..ea85822d 100644 --- a/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt @@ -11,16 +11,16 @@ private val JAPANESE_DATE_REGEX = Regex("""\d{4}年\d{2}月\d{2}日 \d{2}[：:]\ private fun String.extractJapaneseDate(): String = JAPANESE_DATE_REGEX.find(this)?.value ?: this -internal fun String?.toAbsoluteUrl(): String = if (this?.startsWith("http") == true) this else "" +fun String?.toAbsoluteUrl(): String = if (this?.startsWith("http") == true) this else "" -internal fun SponsorBlockSegment.toSegmentItem(): SponsorBlockSegmentItem = SponsorBlockSegmentItem( +fun SponsorBlockSegment.toSegmentItem(): SponsorBlockSegmentItem = SponsorBlockSegmentItem( startTime = startTime, endTime = endTime, category = category.apiName, action = action.apiName, ) -internal fun List.toSponsorBlockSegments(): Array = +fun List.toSponsorBlockSegments(): Array = mapNotNull { item -> runCatching { SponsorBlockSegment( @@ -34,7 +34,7 @@ internal fun List.toSponsorBlockSegments(): Array Response, private val request: Request, initialResponse: Response, diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt index 169281fa..4d6defd6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -12,7 +12,7 @@ import java.time.format.DateTimeFormatter import javax.xml.stream.XMLOutputFactory import javax.xml.stream.XMLStreamWriter -internal object RssDocumentRenderer { +object RssDocumentRenderer { fun render( feed: RssFeedItem, videos: List, diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt index 76816a96..0bdccb9f 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt @@ -10,7 +10,7 @@ import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal class RssFeedAdminRepository { +class RssFeedAdminRepository { suspend fun list(page: Int, limit: Int): Pair, Long> = DatabaseFactory.query { val total = RssFeedsTable.selectAll().count() val rows = RssFeedsTable.selectAll() diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt index 868f6a30..08eb7ab7 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt @@ -8,7 +8,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.util.UUID -class RssFeedManagementService internal constructor( +class RssFeedManagementService public constructor( private val settings: AdminSettingsService, private val subscriptions: SubscriptionsService, private val repository: RssFeedRepository = RssFeedRepository(), diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt index eae437df..818c3e33 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.VideoItem import java.security.MessageDigest import java.util.HexFormat -class RssFeedReaderService internal constructor( +class RssFeedReaderService public constructor( private val settings: AdminSettingsService, private val subscriptionFeed: SubscriptionFeedService, private val blocked: BlockedService, @@ -36,7 +36,8 @@ class RssFeedReaderService internal constructor( val lastModified = RssDocumentRenderer.lastModified(stored.item, videos, now) val bytes = RssDocumentRenderer.render(stored.item, videos, baseUrl, lastModified) val etag = "\"${HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes))}\"" - if (stored.item.lastUsedAt == null || now - stored.item.lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { + val lastUsedAt = stored.item.lastUsedAt + if (lastUsedAt == null || now - lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { repository.touch(feedId, now) } return RssFeedReadResult.Ready(bytes, etag, lastModified, config.rssMinimumPollMinutes * 60) diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt index d8fab751..e5fc87dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt @@ -17,9 +17,9 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager import org.jetbrains.exposed.v1.jdbc.update -internal data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) +data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) -internal class RssFeedRepository { +class RssFeedRepository { suspend fun list(userId: String): List = DatabaseFactory.query { val rows = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt index 55421556..219256a6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt @@ -4,10 +4,10 @@ import dev.typetype.server.db.tables.RssFeedsTable import dev.typetype.server.models.RssFeedItem import org.jetbrains.exposed.v1.core.ResultRow -internal fun ResultRow.toStoredFeed(): StoredRssFeed = +fun ResultRow.toStoredFeed(): StoredRssFeed = toStoredFeed(loadRssFeedSelections(listOf(this[RssFeedsTable.id]))) -internal fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { +fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { val id = this[RssFeedsTable.id] return StoredRssFeed( item = RssFeedItem( diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt index 563dda78..a947c75d 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt @@ -4,7 +4,7 @@ import java.security.MessageDigest import java.security.SecureRandom import java.util.Base64 -internal class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { +class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { fun create(): String { val bytes = ByteArray(32) random.nextBytes(bytes) diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt index 17343952..af575ab6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt @@ -5,12 +5,12 @@ import dev.typetype.server.db.tables.RssFeedServicesTable import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal data class RssFeedSelections( +data class RssFeedSelections( val channels: Map>, val services: Map>, ) -internal fun loadRssFeedSelections(feedIds: List): RssFeedSelections { +fun loadRssFeedSelections(feedIds: List): RssFeedSelections { if (feedIds.isEmpty()) return RssFeedSelections(emptyMap(), emptyMap()) val channels = RssFeedChannelsTable.selectAll() .where { RssFeedChannelsTable.feedId inList feedIds } diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt index 87e81317..c27af1c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger -internal class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { +class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { private val windows = ConcurrentHashMap() private val acquisitions = AtomicInteger() diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt index 047eaaab..7b59c322 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem import java.net.URI -internal object RssVideoMetadata { +object RssVideoMetadata { fun serviceId(video: VideoItem): Int { val host = runCatching { URI(video.url).host.orEmpty().lowercase() }.getOrDefault("") return when { diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt index a7919c07..4db8b756 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.RssFeedItem import dev.typetype.server.models.VideoItem -internal object RssVideoTypeFilter { +object RssVideoTypeFilter { fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { video.isLive -> feed.includeLive video.isUpcomingAt(now) -> feed.includeUpcoming diff --git a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt index 3760acf4..4c0e828c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt @@ -4,7 +4,7 @@ import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.models.PublicPlaylistItem import org.jetbrains.exposed.v1.core.statements.UpdateBuilder -internal fun UpdateBuilder<*>.writePlaylist(playlist: PublicPlaylistItem, savedAt: Long): Unit { +fun UpdateBuilder<*>.writePlaylist(playlist: PublicPlaylistItem, savedAt: Long): Unit { this[SavedPlaylistsTable.publicPlaylistId] = playlist.id this[SavedPlaylistsTable.url] = playlist.url this[SavedPlaylistsTable.title] = playlist.title diff --git a/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt index bfe703ac..59ed00e8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt @@ -3,20 +3,20 @@ package dev.typetype.server.services import dev.typetype.server.models.SearchPageResponse import org.schabi.newpipe.extractor.search.filter.FilterItem -internal enum class SearchContentKind { +enum class SearchContentKind { All, Videos, Playlists, Channels, } -internal fun String?.toSearchContentKind(fallbackFilters: List): SearchContentKind = this +fun String?.toSearchContentKind(fallbackFilters: List): SearchContentKind = this ?.substringAfterLast('|') ?.toSearchContentKind() ?.takeUnless { it == SearchContentKind.All } ?: fallbackFilters.toSearchContentKind() -internal fun List.toSearchContentKind(): SearchContentKind = firstOrNull() +fun List.toSearchContentKind(): SearchContentKind = firstOrNull() ?.name ?.lowercase() ?.toSearchContentKind() @@ -29,7 +29,7 @@ private fun String.toSearchContentKind(): SearchContentKind = when (lowercase()) else -> SearchContentKind.All } -internal fun SearchPageResponse.filteredBy(kind: SearchContentKind): SearchPageResponse = when (kind) { +fun SearchPageResponse.filteredBy(kind: SearchContentKind): SearchPageResponse = when (kind) { SearchContentKind.All -> this SearchContentKind.Videos -> copy(playlists = emptyList(), channels = emptyList()) SearchContentKind.Playlists -> copy(items = emptyList(), channels = emptyList()) diff --git a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt similarity index 82% rename from src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt index 80223e8e..1cdd9db1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt @@ -6,22 +6,22 @@ import org.schabi.newpipe.extractor.search.filter.Filter import org.schabi.newpipe.extractor.search.filter.FilterGroup import org.schabi.newpipe.extractor.search.filter.FilterItem -internal sealed interface SearchFilterResolution { +sealed interface SearchFilterResolution { data class Valid(val items: List) : SearchFilterResolution data class Invalid(val message: String) : SearchFilterResolution } -internal fun Filter?.toSearchContentFilterOptions(): List = entries() +fun Filter?.toSearchContentFilterOptions(): List = entries() .mapIndexed { index, entry -> entry.toOption(isDefault = index == 0, includeGroup = true) } -internal fun Filter?.toSearchFilterOptions(): List = this?.filterGroups +fun Filter?.toSearchFilterOptions(): List = this?.filterGroups ?.flatMap { group -> group.filterItems.mapIndexed { index, item -> FilterEntry(group, item).toOption(group.onlyOneCheckable && index == 0, includeGroup = true) } }.orEmpty() -internal fun Filter?.toSearchFilterGroups(): List = this?.filterGroups +fun Filter?.toSearchFilterGroups(): List = this?.filterGroups ?.map { group -> SearchFilterGroup( key = "${group.groupName.orEmpty()}|${group.identifier}", @@ -33,11 +33,11 @@ internal fun Filter?.toSearchFilterGroups(): List = this?.fil ) }.orEmpty() -internal fun Filter?.findSearchFilter(value: String?): List = value?.let { raw -> +fun Filter?.findSearchFilter(value: String?): List = value?.let { raw -> entries().firstOrNull { it.value == raw }?.let { listOf(it.item) } } ?: emptyList() -internal fun Filter?.resolveSearchFilters(values: List): SearchFilterResolution { +fun Filter?.resolveSearchFilters(values: List): SearchFilterResolution { if (values.isEmpty()) return SearchFilterResolution.Valid(emptyList()) val requested = values.toSet() val selected = entries().filter { it.value in requested } @@ -52,12 +52,12 @@ internal fun Filter?.resolveSearchFilters(values: List): SearchFilterRes return SearchFilterResolution.Valid(selected.map(FilterEntry::item)) } -internal fun Filter?.defaultSearchFilters(): List = this?.filterGroups +fun Filter?.defaultSearchFilters(): List = this?.filterGroups ?.filter(FilterGroup::onlyOneCheckable) ?.mapNotNull { it.filterItems.firstOrNull() } .orEmpty() -internal fun Filter?.defaultSearchFilter(): List = this?.filterGroups +fun Filter?.defaultSearchFilter(): List = this?.filterGroups ?.firstOrNull() ?.filterItems ?.firstOrNull() diff --git a/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt index efd2feeb..847967aa 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem import org.schabi.newpipe.extractor.search.SearchInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( +fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( items = relatedItems.filterIsInstance().map { it.toVideoItem() }, nextpage = nextPage?.toCursor(), searchSuggestion = searchSuggestion?.takeIf { it.isNotBlank() }, @@ -17,7 +17,7 @@ internal fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageR channels = relatedItems.filterIsInstance().map { it.toChannelResultItem() }, ) -internal fun InfoItemsPage.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( +fun InfoItemsPage.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( items = items.filterIsInstance().map { it.toVideoItem() }, nextpage = nextPage?.toCursor(), searchSuggestion = null, diff --git a/src/main/kotlin/dev/typetype/server/services/SearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ServiceIds.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SettingsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index d609931b..8fc51f21 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,7 +35,7 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } - internal suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { + suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { SubscriptionFeedVisibility( hideLiveStreams = it[SettingsTable.hideSubscriptionLiveStreams], @@ -54,7 +54,7 @@ class SettingsService { } } -internal data class SubscriptionFeedVisibility( +data class SubscriptionFeedVisibility( val hideLiveStreams: Boolean = false, val hideMembersOnlyContent: Boolean = false, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt index ba3f8fc7..3c39f9d4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt @@ -67,7 +67,7 @@ class SignedHlsManifestTokenService( runCatching { String(decoder.decode(value), Charsets.UTF_8) }.getOrNull() companion object { - const val TTL_SECONDS = 900L + const val TTL_SECONDS = 86_400L private const val VERSION = "v1" private const val PART_COUNT = 6 private const val MAX_TOKEN_LENGTH = 4096 diff --git a/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt index 69cd3304..111e8dca 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -11,7 +11,8 @@ private const val BILIBILI_SIGNED_STREAM_MAX_TTL_SECONDS = 3_600L private const val SIGNED_STREAM_DEADLINE_SAFETY_SECONDS = 300L private const val MIN_CACHEABLE_STREAM_TTL_SECONDS = 60L -internal fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { +fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { + if (isLive || isLiveContent) return 0L val deadline = signedMediaUrls().mapNotNull { it.bilibiliDeadline() }.minOrNull() ?: return stableMetadataTtlSeconds() val ttl = deadline - nowEpochSeconds - SIGNED_STREAM_DEADLINE_SAFETY_SECONDS diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt similarity index 57% rename from src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 2be09e73..7bd91b79 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -3,16 +3,29 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ExtractionFailureKind import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException +import org.schabi.newpipe.extractor.exceptions.LiveNotStartException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException -internal object StreamExtractionErrorMapper { +object StreamExtractionErrorMapper { + const val UPSTREAM_FAILURE_CODE = "upstream_failure" const val MEMBERS_ONLY_FALLBACK = "This video is only available for members" const val PAID_CONTENT_FALLBACK = "This video is a paid video" + const val GEOGRAPHIC_RESTRICTION_CODE = "geographic_restriction" + const val PROVIDER_ACCESS_BLOCKED_CODE = "provider_access_blocked" + const val PRIVATE_CONTENT_CODE = "private_content" + const val GEOGRAPHIC_RESTRICTION_FALLBACK = + "This video is not available in the server's region. A VPN or another outbound network may help." + const val PROVIDER_ACCESS_BLOCKED_FALLBACK = + "The video provider is blocking requests from this TypeType server. Try a VPN or another outbound network for the server, then retry." + const val PRIVATE_CONTENT_FALLBACK = "This video is private or no longer available" fun map(error: Throwable, sourceUrl: String? = null, fallback: String = "Extraction failed"): ExtractionResult = mapByType(error, fallback) @@ -27,18 +40,42 @@ internal object StreamExtractionErrorMapper { sanitize(error.message) ?: PAID_CONTENT_FALLBACK, "paid_content", ) + is LiveNotStartException -> ExtractionResult.Failure( + sanitize(error.message) ?: "This live event has not started yet", + "live_not_started", + ExtractionFailureKind.LiveEventNotStarted, + ) is VideoNotReleaseException -> ExtractionResult.Failure( sanitize(error.message) ?: "This premiere has not started yet", "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, ) is AgeRestrictedContentException -> ExtractionResult.BadRequest( sanitize(error.message) ?: "This video is age-restricted", "age_restricted", ) - is GeographicRestrictionException, - is PrivateContentException -> ExtractionResult.BadRequest(sanitize(error.message) ?: "Content not available") + is GeographicRestrictionException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: GEOGRAPHIC_RESTRICTION_FALLBACK, + GEOGRAPHIC_RESTRICTION_CODE, + ) + is PrivateContentException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: PRIVATE_CONTENT_FALLBACK, + PRIVATE_CONTENT_CODE, + ) + is AntiBotException, + is ReCaptchaException -> ExtractionResult.Failure( + PROVIDER_ACCESS_BLOCKED_FALLBACK, + PROVIDER_ACCESS_BLOCKED_CODE, + ExtractionFailureKind.ProviderAccessBlocked, + ) + is ContentNotAvailableException -> ExtractionResult.Failure( + sanitize(error.message) ?: "This content is not available", + "content_unavailable", + ExtractionFailureKind.ContentUnavailable, + ) else -> ExtractionResult.Failure( sanitize(error.message) ?: fallback, + if (error.isYoutubeSessionRejected()) "youtube_session_rejected" else UPSTREAM_FAILURE_CODE, kind = if (error.isYoutubeSessionRejected()) { ExtractionFailureKind.YoutubeSessionRejected } else { diff --git a/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt index 7ed667c6..5ebe5add 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.extractor.stream.SubtitlesStream -internal fun StreamInfo.toStreamResponse(): StreamResponse { +fun StreamInfo.toStreamResponse(): StreamResponse { val uploaded = uploadDate?.offsetDateTime()?.toInstant()?.toEpochMilli() ?: -1L val liveMetadata = streamLiveMetadata(streamType, hlsUrl, dashMpdUrl) return StreamResponse( @@ -57,7 +57,7 @@ internal fun StreamInfo.toStreamResponse(): StreamResponse { ) } -internal fun SubtitlesStream.toSubtitleItem(): SubtitleItem = SubtitleItem( +fun SubtitlesStream.toSubtitleItem(): SubtitleItem = SubtitleItem( url = getContent() ?: "", mimeType = getFormat()?.getMimeType() ?: "", languageTag = getLanguageTag() ?: "", diff --git a/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt index bc38188d..07262bca 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean): VideoStreamItem { +fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean): VideoStreamItem { val method = deliveryMethodName() return VideoStreamItem( url = playableUrl(method), @@ -32,7 +32,7 @@ internal fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean ) } -internal fun AudioStream.toAudioStreamItem(videoId: String): AudioStreamItem { +fun AudioStream.toAudioStreamItem(videoId: String): AudioStreamItem { val method = deliveryMethodName() return AudioStreamItem( url = playableUrl(method), @@ -85,10 +85,10 @@ private fun audioSessionUrl(videoId: String, itag: Int, trackId: String?): Strin return "/sabr/session/$videoId?audioItag=$itag$track" } -internal fun VideoStreamItem.isSupportedPlaybackStream(): Boolean = +fun VideoStreamItem.isSupportedPlaybackStream(): Boolean = deliveryMethod != "sabr" || isSupportedSabrVideo() -internal fun AudioStreamItem.isSupportedPlaybackStream(): Boolean = +fun AudioStreamItem.isSupportedPlaybackStream(): Boolean = deliveryMethod != "sabr" || isSupportedSabrAudio() private fun VideoStreamItem.isSupportedSabrVideo(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt index a89f3b7d..ad7b561f 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.StreamSegmentItem import org.schabi.newpipe.extractor.stream.Frameset import org.schabi.newpipe.extractor.stream.StreamSegment -internal fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( +fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( urls = getUrls() ?: emptyList(), frameWidth = getFrameWidth(), frameHeight = getFrameHeight(), @@ -15,7 +15,7 @@ internal fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( framesPerPageY = getFramesPerPageY(), ) -internal fun StreamSegment.toStreamSegmentItem(): StreamSegmentItem = StreamSegmentItem( +fun StreamSegment.toStreamSegmentItem(): StreamSegmentItem = StreamSegmentItem( title = getTitle() ?: "", startTimeSeconds = getStartTimeSeconds(), channelName = getChannelName(), diff --git a/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt index 85643b1f..5050fc30 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt @@ -8,11 +8,11 @@ import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -internal fun interface YouTubeSubtitleTrackResolver { +fun interface YouTubeSubtitleTrackResolver { suspend fun resolve(selection: YouTubeSubtitleSelection): YouTubeSubtitleResolution } -internal class StreamYouTubeSubtitleResolver( +class StreamYouTubeSubtitleResolver( private val streamService: StreamService, private val fetchInventory: suspend (String) -> YouTubeSubtitleInventoryResult, ) : YouTubeSubtitleTrackResolver { @@ -57,7 +57,7 @@ private suspend fun StreamService.streamForSubtitle(videoId: String): StreamResp private fun StreamResponse.isLiveSubtitleContent(): Boolean = isLive || isLiveContent -internal fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelection): Boolean { +fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelection): Boolean { val parsedUrl = url.toHttpUrlOrNull() val sourceLanguage = parsedUrl?.queryParameter("lang") ?: languageTag val requestedSource = selection.sourceLanguage ?: selection.language @@ -70,7 +70,7 @@ internal fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelec return selection.trackName == null || parsedUrl?.queryParameter("name") == selection.trackName } -internal fun SubtitleItem.contentForYouTubeSubtitle(selection: YouTubeSubtitleSelection): String? { +fun SubtitleItem.contentForYouTubeSubtitle(selection: YouTubeSubtitleSelection): String? { val parsedUrl = url.toHttpUrlOrNull()?.takeIf { isYouTubeTimedTextUrl(it.toString()) } ?: return null return parsedUrl.newBuilder() .setQueryParameter("fmt", selection.format.value) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt new file mode 100644 index 00000000..08fe1a78 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt @@ -0,0 +1,93 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubscriptionItem +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.update +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +class SubscriptionAvatarWarmupService(private val channelService: ChannelService) { + private val logger = LoggerFactory.getLogger(SubscriptionAvatarWarmupService::class.java) + private val inFlight = ConcurrentHashMap.newKeySet() + private val recentlyFailed = ConcurrentHashMap() + + fun schedule(scope: CoroutineScope, userId: String, items: List) { + val now = System.currentTimeMillis() + val candidates = items.asSequence() + .filter { it.avatarUrl.isBlank() } + .map(SubscriptionItem::channelUrl) + .distinct() + .filter { url -> + val failedAt = recentlyFailed[url] + failedAt == null || now - failedAt >= FAILURE_COOLDOWN_MS + } + .filter(inFlight::add) + .take(MAX_AVATARS_PER_REQUEST) + .toList() + if (candidates.isEmpty()) return + scope.launch { + try { + warm(userId, candidates) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + logger.warn("Background subscription avatar warmup failed", error) + } finally { + inFlight.removeAll(candidates.toSet()) + } + } + } + + internal suspend fun warm(userId: String, channelUrls: List) { + channelUrls.chunked(MAX_CONCURRENT_LOOKUPS).forEach { chunk -> + coroutineScope { + chunk.map { channelUrl -> + async { resolve(userId, channelUrl) } + }.awaitAll() + } + delay(BATCH_DELAY_MS) + } + } + + private suspend fun resolve(userId: String, channelUrl: String) { + when (val result = channelService.getChannel(url = channelUrl, nextpage = null)) { + is ExtractionResult.Success -> { + val avatarUrl = result.data.avatarUrl.trim() + if (avatarUrl.isBlank()) { + recentlyFailed[channelUrl] = System.currentTimeMillis() + return + } + DatabaseFactory.query { + SubscriptionsTable.update({ + (SubscriptionsTable.userId eq userId) and + (SubscriptionsTable.channelUrl eq channelUrl) and + (SubscriptionsTable.avatarUrl eq "") + }) { + it[SubscriptionsTable.avatarUrl] = avatarUrl + } + } + recentlyFailed.remove(channelUrl) + } + is ExtractionResult.BadRequest, + is ExtractionResult.Failure -> recentlyFailed[channelUrl] = System.currentTimeMillis() + } + } + + private companion object { + const val MAX_AVATARS_PER_REQUEST = 8 + const val MAX_CONCURRENT_LOOKUPS = 2 + const val BATCH_DELAY_MS = 750L + const val FAILURE_COOLDOWN_MS = 15 * 60 * 1000L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt index e29e0beb..33863129 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt @@ -7,7 +7,7 @@ data class SubscriptionFeedAvailability( val available: Boolean, ) -internal data class SubscriptionFeedWithSources( +data class SubscriptionFeedWithSources( val videos: List, val available: Boolean, val sourceChannelUrls: Map>, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index 95beed0d..dbe8154d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -11,7 +11,7 @@ import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withTimeoutOrNull import java.net.URI -internal class SubscriptionFeedBuilder(private val channelService: ChannelService) { +class SubscriptionFeedBuilder(private val channelService: ChannelService) { private val semaphore = Semaphore(MAX_CONCURRENT_FETCHES) suspend fun build(subscriptions: List): SubscriptionFeedBuildResult = coroutineScope { @@ -122,7 +122,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } } -internal data class SubscriptionFeedBuildResult( +data class SubscriptionFeedBuildResult( val videos: List, val sourceChannelUrls: Map>, val successfulSources: Int, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt similarity index 64% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt index d570645a..4979d8b9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt @@ -2,16 +2,16 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -class SubscriptionFeedCacheInvalidator( +class SubscriptionFeedCacheInvalidatorImpl( private val cache: CacheService, private val feedService: SubscriptionFeedService, -) { - suspend fun invalidate(userId: String) { +) : SubscriptionFeedCacheInvalidator { + override suspend fun invalidate(userId: String) { feedService.invalidate(userId) runCatching { cache.delete(SubscriptionFeedCacheKeys.shorts(userId)) } } - suspend fun awaitRefresh(userId: String) { + override suspend fun awaitRefresh(userId: String) { feedService.awaitRefresh(userId) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt index a1dc2b9d..786c5cef 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem -internal class SubscriptionFeedOrderer { +class SubscriptionFeedOrderer { fun order( videos: List, previous: SubscriptionFeedSnapshot?, @@ -41,15 +41,14 @@ internal class SubscriptionFeedOrderer { private fun VideoItem.feedTimestamp(): Long = when { uploaded >= 0L -> uploaded - publishedAt != null && publishedAt >= 0L -> publishedAt - else -> Long.MIN_VALUE + else -> publishedAt?.takeIf { it >= 0L } ?: Long.MIN_VALUE } } -internal data class SubscriptionFeedOrdering( +data class SubscriptionFeedOrdering( val videos: List, val livePromotedAt: Map, ) -internal fun VideoItem.subscriptionFeedKey(): String = +fun VideoItem.subscriptionFeedKey(): String = url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt index b8bc1737..3d2e8ac5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt @@ -5,7 +5,7 @@ import dev.typetype.server.cache.CacheService import kotlinx.serialization.Serializable import java.security.MessageDigest -internal class SubscriptionFeedSelectionStore( +class SubscriptionFeedSelectionStore( private val cache: CacheService, private val subscriptions: SubscriptionsService, ) { @@ -98,7 +98,7 @@ private data class SubscriptionFeedSelectionIdentity( val channelUrls: List, ) -internal data class SubscriptionFeedSelectionSnapshot( +data class SubscriptionFeedSelectionSnapshot( val token: String?, val channelUrls: Set?, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 019bf267..81b661ab 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -35,7 +35,7 @@ class SubscriptionFeedService( private val refreshJobs = ConcurrentHashMap() private val generation = AtomicLong(clock()) - internal suspend fun getPage( + suspend fun getPage( userId: String, page: Int, limit: Int, @@ -124,7 +124,7 @@ class SubscriptionFeedService( ) } - internal suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = + suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = getAllWithAvailability(userId).let { SubscriptionFeedWithSources(it.videos, it.available, store.current(userId)?.sourceChannelUrls.orEmpty()) } suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { @@ -132,7 +132,7 @@ class SubscriptionFeedService( return snapshot.page(page * limit, limit, isRefreshing(userId)) } - internal suspend fun getCachedAll(userId: String): List? = + suspend fun getCachedAll(userId: String): List? = store.current(userId)?.videos suspend fun invalidate(userId: String) { @@ -141,7 +141,7 @@ class SubscriptionFeedService( scheduleRefresh(userId, currentRequestId()) } - internal suspend fun awaitRefresh(userId: String) { + suspend fun awaitRefresh(userId: String) { while (true) { val jobs = listOfNotNull(refreshJobs[userId]) if (jobs.isEmpty()) return @@ -149,7 +149,7 @@ class SubscriptionFeedService( } } - internal fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true + fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true fun close() { refreshJobs.values.forEach(Job::cancel) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 436d7948..3ce28437 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -7,7 +7,7 @@ import kotlinx.serialization.Serializable import java.util.Base64 @Serializable -internal data class SubscriptionFeedSnapshot( +data class SubscriptionFeedSnapshot( val generation: Long, val generatedAt: Long, val stale: Boolean, @@ -27,7 +27,7 @@ private data class SubscriptionFeedCursor( val selectionToken: String? = null, ) -internal object SubscriptionFeedCursorCodec { +object SubscriptionFeedCursorCodec { fun encode( generation: Long, offset: Int, @@ -70,7 +70,7 @@ internal object SubscriptionFeedCursorCodec { private val SELECTION_TOKEN = Regex("[0-9a-f]{64}") } -internal data class SubscriptionFeedCursorState( +data class SubscriptionFeedCursorState( val generation: Long, val offset: Int, val limit: Int, @@ -80,10 +80,10 @@ internal data class SubscriptionFeedCursorState( val selectionToken: String?, ) -internal fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = +fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = videos.all { sourceChannelUrls.containsKey(it.subscriptionFeedKey()) } -internal fun SubscriptionFeedSnapshot.page( +fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, @@ -138,7 +138,7 @@ private fun SubscriptionFeedSnapshot.projectedVideos( } } -internal sealed interface SubscriptionFeedPageResult { +sealed interface SubscriptionFeedPageResult { data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult data object InvalidCursor : SubscriptionFeedPageResult diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt index 365e4308..a86a2a23 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt @@ -5,7 +5,7 @@ import dev.typetype.server.cache.CacheService import dev.typetype.server.models.VideoItem import kotlinx.serialization.builtins.ListSerializer -internal class SubscriptionFeedSnapshotStore( +class SubscriptionFeedSnapshotStore( private val cache: CacheService, private val clock: () -> Long, ) { diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt new file mode 100644 index 00000000..20aa8494 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt @@ -0,0 +1,35 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.models.SubscriptionGroupItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.count +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object SubscriptionGroupQueries { + fun all(userId: String): List { + val count = SubscriptionGroupMembershipsTable.channelUrl.count() + val counts = SubscriptionGroupMembershipsTable.select(SubscriptionGroupMembershipsTable.groupId, count) + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .groupBy(SubscriptionGroupMembershipsTable.groupId) + .associate { it[SubscriptionGroupMembershipsTable.groupId] to it[count].toInt() } + return SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) + .map { row -> SubscriptionGroupItem( + id = row[SubscriptionGroupsTable.id], + name = row[SubscriptionGroupsTable.name], + channelCount = counts[row[SubscriptionGroupsTable.id]] ?: 0, + createdAt = row[SubscriptionGroupsTable.createdAt], + updatedAt = row[SubscriptionGroupsTable.updatedAt], + ) } + } + + fun exists(userId: String, groupId: String): Boolean = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.any() +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 5c92769b..a93e926c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -5,7 +5,6 @@ import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.SubscriptionGroupItem -import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -21,18 +20,11 @@ import java.util.UUID class SubscriptionGroupsService { suspend fun getAll(userId: String): List = DatabaseFactory.query { - val counts = SubscriptionGroupMembershipsTable.selectAll() - .where { SubscriptionGroupMembershipsTable.userId eq userId } - .groupingBy { it[SubscriptionGroupMembershipsTable.groupId] } - .eachCount() - SubscriptionGroupsTable.selectAll() - .where { SubscriptionGroupsTable.userId eq userId } - .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) - .map { it.toItem(counts[it[SubscriptionGroupsTable.id]] ?: 0) } + SubscriptionGroupQueries.all(userId) } suspend fun exists(userId: String, groupId: String): Boolean = DatabaseFactory.query { - groupExists(userId, groupId) + SubscriptionGroupQueries.exists(userId, groupId) } suspend fun create(userId: String, rawName: String): SubscriptionGroupWriteResult { @@ -176,10 +168,7 @@ class SubscriptionGroupsService { .map { it[SubscriptionGroupMembershipsTable.channelUrl] } } - private fun groupExists(userId: String, groupId: String): Boolean = - SubscriptionGroupsTable.selectAll().where { - (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) - }.any() + private fun groupExists(userId: String, groupId: String): Boolean = SubscriptionGroupQueries.exists(userId, groupId) private fun nameExists(userId: String, normalizedName: String): Boolean = SubscriptionGroupsTable.selectAll().where { @@ -193,14 +182,6 @@ class SubscriptionGroupsService { (SubscriptionGroupMembershipsTable.groupId eq groupId) }.count().toInt() - private fun ResultRow.toItem(channelCount: Int): SubscriptionGroupItem = SubscriptionGroupItem( - id = this[SubscriptionGroupsTable.id], - name = this[SubscriptionGroupsTable.name], - channelCount = channelCount, - createdAt = this[SubscriptionGroupsTable.createdAt], - updatedAt = this[SubscriptionGroupsTable.updatedAt], - ) - private fun normalizeDisplayName(value: String): String? = value.trim().takeIf { it.length in 1..MAX_GROUP_NAME_LENGTH } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt new file mode 100644 index 00000000..61a24eab --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.services + +data class SubscriptionMembershipFilter( + val page: Int = 0, + val limit: Int = 20, + val search: String = "", + val groupId: String? = null, + val ungrouped: Boolean = false, + val excluded: Boolean = false, +) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt new file mode 100644 index 00000000..b8252c11 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupMembershipItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.SubscriptionMembershipPage +import org.jetbrains.exposed.v1.core.Op +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.exists +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.like +import org.jetbrains.exposed.v1.core.lowerCase +import org.jetbrains.exposed.v1.core.notExists +import org.jetbrains.exposed.v1.core.or +import org.jetbrains.exposed.v1.core.stringParam +import org.jetbrains.exposed.v1.jdbc.Query +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll + +class SubscriptionMembershipPageService { + suspend fun getPage(userId: String, filter: SubscriptionMembershipFilter): SubscriptionMembershipPage = + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(userId) + val owned = SubscriptionsTable.userId eq userId + val ungrouped = notExists(matchingMemberships(userId)) + val matching = owned and filterCondition(userId, filter) + val total = SubscriptionsTable.selectAll().where { matching }.count() + val items = SubscriptionsTable.selectAll().where { matching } + .orderBy(SubscriptionsTable.name.lowerCase() to SortOrder.ASC, SubscriptionsTable.channelUrl to SortOrder.ASC) + .limit(filter.limit).offset(filter.page.toLong() * filter.limit) + .map { it.toItem() } + SubscriptionMembershipPage( + items = withMemberships(userId, items), + total = total, + totalSubscriptions = SubscriptionsTable.selectAll().where { owned }.count(), + ungroupedCount = SubscriptionsTable.selectAll().where { owned and ungrouped }.count(), + page = filter.page, + limit = filter.limit, + ) + } + + suspend fun lookup(userId: String, channelUrls: List): List = + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(userId) + val urls = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + val items = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl inList urls) + }.map { it.toItem() } + withMemberships(userId, items) + } + + private fun filterCondition(userId: String, filter: SubscriptionMembershipFilter): Op { + val memberships = matchingMemberships(userId, filter.groupId) + val membership = when { + filter.ungrouped || filter.excluded -> notExists(memberships) + filter.groupId != null -> exists(memberships) + else -> Op.TRUE + } + val search = filter.search.trim() + if (search.isEmpty()) return membership + val escaped = search.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + val pattern = stringParam("%$escaped%").lowerCase() + return membership and ((SubscriptionsTable.name.lowerCase() like pattern) or + (SubscriptionsTable.channelUrl.lowerCase() like pattern)) + } + + private fun matchingMemberships(userId: String, groupId: String? = null): Query = + SubscriptionGroupMembershipsTable.select(SubscriptionGroupMembershipsTable.channelUrl).where { + val matching = (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq SubscriptionsTable.channelUrl) + groupId?.let { matching and (SubscriptionGroupMembershipsTable.groupId eq it) } ?: matching + } + + private fun withMemberships(userId: String, items: List): List { + if (items.isEmpty()) return emptyList() + val urls = items.map { it.channelUrl } + val groups = SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList urls) + }.groupBy({ it[SubscriptionGroupMembershipsTable.channelUrl] }, { it[SubscriptionGroupMembershipsTable.groupId] }) + return SubscriptionAvatarRepairer.resolve(userId = userId, items = items).map { item -> + SubscriptionGroupMembershipItem( + channelUrl = item.channelUrl, + name = item.name, + avatarUrl = item.avatarUrl, + subscribedAt = item.subscribedAt, + groupIds = groups[item.channelUrl].orEmpty().sorted(), + ) + } + } + + private fun ResultRow.toItem(): SubscriptionItem = SubscriptionItem( + channelUrl = this[SubscriptionsTable.channelUrl], + name = this[SubscriptionsTable.name], + avatarUrl = this[SubscriptionsTable.avatarUrl], + subscribedAt = this[SubscriptionsTable.subscribedAt], + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index b0bb5dd9..840e7d68 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -61,6 +61,7 @@ class SubscriptionsService { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) val now = System.currentTimeMillis() DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) SubscriptionsTable.insert { it[SubscriptionsTable.userId] = userId it[channelUrl] = canonicalUrl diff --git a/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/TrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/TrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt index 4b59038e..4e1ce959 100644 --- a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt @@ -4,7 +4,7 @@ import java.net.InetAddress import java.net.URI import java.net.UnknownHostException -internal class UnifiedPushEndpointValidator( +class UnifiedPushEndpointValidator( private val resolver: (String) -> Array = InetAddress::getAllByName, ) { fun validate(raw: String): EndpointValidationResult { @@ -61,7 +61,7 @@ internal class UnifiedPushEndpointValidator( } } -internal sealed interface EndpointValidationResult { +sealed interface EndpointValidationResult { data class Valid(val uri: URI, val addresses: List) : EndpointValidationResult data class Invalid(val reason: String) : EndpointValidationResult } diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt index ac0a1d74..048fa262 100644 --- a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt @@ -9,7 +9,7 @@ import okhttp3.RequestBody.Companion.toRequestBody import java.net.InetAddress import java.net.UnknownHostException -internal class UnifiedPushSender( +class UnifiedPushSender( private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), client: OkHttpClient = OkHttpClient(), ) : PushEndpointSender { @@ -57,11 +57,11 @@ internal class UnifiedPushSender( } } -internal interface PushEndpointSender { +interface PushEndpointSender { suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult } -internal sealed interface PushSendResult { +sealed interface PushSendResult { data object Delivered : PushSendResult data object InvalidEndpoint : PushSendResult data class Retry(val statusCode: Int?) : PushSendResult diff --git a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/UrlValidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt index d7b488cc..95e7c1e3 100644 --- a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt @@ -5,18 +5,18 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.net.InetAddress import java.net.URI -internal enum class ProxyProvider { +enum class ProxyProvider { YOUTUBE, BILIBILI, NICONICO, } -internal data class ProxyTarget( +data class ProxyTarget( val url: HttpUrl, val provider: ProxyProvider, ) -internal class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) +class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) private val BLOCKED_IPV4_RANGES = listOf( ipv4(0, 0, 0, 0) to 8, @@ -44,12 +44,12 @@ private val BLOCKED_IPV6_RANGES = listOf( byteArrayOf(0x20, 0x02) to 16, ) -internal fun validateProxyUrl(raw: String): String? = +fun validateProxyUrl(raw: String): String? = runCatching { requireProxyTarget(raw) } .exceptionOrNull() ?.message -internal fun requireProxyTarget(raw: String): ProxyTarget { +fun requireProxyTarget(raw: String): ProxyTarget { val uri = runCatching { URI(raw) }.getOrElse { throw ProxyTargetRejectedException("Malformed URL") } val scheme = uri.scheme?.lowercase() ?: throw ProxyTargetRejectedException("Missing URL scheme") if (scheme != "https") throw ProxyTargetRejectedException("Unsupported URL scheme: $scheme") @@ -65,7 +65,7 @@ internal fun requireProxyTarget(raw: String): ProxyTarget { return ProxyTarget(url, provider) } -internal fun providerForProxyHost(rawHost: String): ProxyProvider? { +fun providerForProxyHost(rawHost: String): ProxyProvider? { val host = rawHost.lowercase().trimEnd('.') return when { host.matchesHost("googlevideo.com") || @@ -75,13 +75,14 @@ internal fun providerForProxyHost(rawHost: String): ProxyProvider? { host.matchesHost("bilivideo.com") || host.matchesHost("bilivideo.cn") || host.matchesHost("hdslb.com") || + host.matchesHost("live.bilibili.com") || host == "upos-hz-mirrorakam.akamaized.net" -> ProxyProvider.BILIBILI host.matchesHost("nicovideo.jp") || host.matchesHost("nimg.jp") -> ProxyProvider.NICONICO else -> null } } -internal fun isPublicProxyAddress(address: InetAddress): Boolean { +fun isPublicProxyAddress(address: InetAddress): Boolean { if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || address.isSiteLocalAddress || address.isMulticastAddress ) return false diff --git a/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/UserAdminService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt index 2f1757ec..f67681d6 100644 --- a/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt @@ -7,6 +7,7 @@ import dev.typetype.server.db.tables.WatchLaterTable import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.launch import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -24,22 +25,35 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private val lastScheduledAt = ConcurrentHashMap() fun schedulePlaylists(scope: CoroutineScope, userId: String): Unit = schedule(scope, "playlists:$userId") { - repairPlaylists(userId) + drain { excluded -> repairPlaylists(userId, excluded) } } fun scheduleWatchLater(scope: CoroutineScope, userId: String): Unit = schedule(scope, "watch-later:$userId") { - repairWatchLater(userId) + drain { excluded -> repairWatchLater(userId, excluded) } } fun scheduleFavorites(scope: CoroutineScope, userId: String): Unit = schedule(scope, "favorites:$userId") { - repairFavorites(userId) + drain { excluded -> repairFavorites(userId, excluded) } } - suspend fun repairPlaylists(userId: String): Int = repair(userId, ::playlistCandidateUrls) + private suspend fun repairPlaylists(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::playlistCandidateUrls) - suspend fun repairWatchLater(userId: String): Int = repair(userId, ::watchLaterCandidateUrls) + private suspend fun repairWatchLater(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::watchLaterCandidateUrls) - suspend fun repairFavorites(userId: String): Int = repair(userId, ::favoriteCandidateUrls) + private suspend fun repairFavorites(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::favoriteCandidateUrls) + + private suspend fun drain(repair: suspend (Set) -> RepairOutcome) { + val excluded = mutableSetOf() + repeat(MAX_BATCHES_PER_RUN) { + val outcome = repair(excluded) + if (outcome.attempted == 0) return + excluded += outcome.attemptedUrls + delay(BATCH_DELAY_MS) + } + } private fun schedule(scope: CoroutineScope, key: String, repair: suspend () -> Unit) { val now = System.currentTimeMillis() @@ -60,22 +74,33 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver } } - private suspend fun repair(userId: String, candidates: suspend (String) -> List): Int { - val urls = candidates(userId).take(MAX_REPAIR_PER_REQUEST) - if (urls.isEmpty()) return 0 - val metadata = resolver.resolve(urls) - if (metadata.isEmpty()) return 0 - return DatabaseFactory.query { - metadata.values.sumOf { item -> + private suspend fun repair( + userId: String, + excluded: Set, + candidates: suspend (String) -> List, + ): RepairOutcome { + val urls = candidates(userId).filterNot { it in excluded }.take(MAX_REPAIR_PER_REQUEST) + if (urls.isEmpty()) return RepairOutcome(attemptedUrls = emptySet(), updated = 0) + val first = resolver.resolveDetailed(urls) + var updated = applyMetadata(userId, first) + if (first.retryableUrls.isNotEmpty()) { + delay(RETRY_DELAY_MS) + val retried = resolver.resolveDetailed(first.retryableUrls) + updated += applyMetadata(userId, retried) + } + return RepairOutcome(attemptedUrls = urls.toSet(), updated = updated) + } + + private suspend fun applyMetadata(userId: String, resolution: VideoMetadataResolution): Int = + DatabaseFactory.query { + resolution.metadata.values.sumOf { item -> updatePlaylistVideos(userId, item) + updateWatchLater(userId, item) + updateFavorites(userId, item) } } - } private suspend fun playlistCandidateUrls(userId: String): List = DatabaseFactory.query { PlaylistVideosTable.selectAll() .where { (PlaylistVideosTable.userId eq userId) and playlistNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[PlaylistVideosTable.url] } .distinct() } @@ -83,7 +108,6 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private suspend fun watchLaterCandidateUrls(userId: String): List = DatabaseFactory.query { WatchLaterTable.selectAll() .where { (WatchLaterTable.userId eq userId) and watchLaterNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[WatchLaterTable.url] } .distinct() } @@ -91,7 +115,6 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private suspend fun favoriteCandidateUrls(userId: String): List = DatabaseFactory.query { FavoritesTable.selectAll() .where { (FavoritesTable.userId eq userId) and favoriteNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[FavoritesTable.videoUrl] } .distinct() } @@ -144,7 +167,17 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private companion object { const val FALLBACK_TITLE_PATTERN = "YouTube video %" const val YOUTUBE_THUMB_PATTERN = "https://i.ytimg.com/vi/%" - const val MAX_REPAIR_PER_REQUEST = 25 + const val MAX_REPAIR_PER_REQUEST = 8 + const val MAX_BATCHES_PER_RUN = 3 + const val BATCH_DELAY_MS = 1_500L + const val RETRY_DELAY_MS = 3_000L const val REPAIR_COOLDOWN_MS = 5 * 60 * 1000L } } + +private data class RepairOutcome( + val attemptedUrls: Set, + val updated: Int, +) { + val attempted: Int get() = attemptedUrls.size +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt new file mode 100644 index 00000000..975e6c9b --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +fun VideoItem.isUpcomingAt(now: Long): Boolean = + !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now + +fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) + +fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = + isLive || isPostLive || isLiveContent || isUpcomingAt(now) diff --git a/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt diff --git a/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt similarity index 69% rename from src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt index 7a604287..9d81dd3b 100644 --- a/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt @@ -10,6 +10,11 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +internal data class VideoMetadataResolution( + val metadata: Map, + val retryableUrls: Set, +) + class VideoMetadataResolver(private val streamService: StreamService) { suspend fun enrichPlaylistVideos(videos: List): List { val metadata = resolve(videos.filter(::shouldEnrich).map { it.url }) @@ -27,23 +32,37 @@ class VideoMetadataResolver(private val streamService: StreamService) { return distinct.map { item -> metadata[item.videoUrl]?.toFavorite(item) ?: item.withYoutubeFallbackTitle() } } - suspend fun resolve(urls: Collection): Map = coroutineScope { + suspend fun resolve(urls: Collection): Map = + resolveDetailed(urls).metadata + + internal suspend fun resolveDetailed(urls: Collection): VideoMetadataResolution = coroutineScope { val semaphore = Semaphore(MAX_CONCURRENT_RESOLUTIONS) - urls.map { it.trim() }.filter { it.isNotBlank() }.distinct().map { url -> + val results = urls.map { it.trim() }.filter { it.isNotBlank() }.distinct().map { url -> async { semaphore.withPermit { resolve(url) } } - }.awaitAll().filterNotNull().toMap() + }.awaitAll().filterNotNull() + val metadata = mutableMapOf() + val retryableUrls = mutableSetOf() + results.forEach { result -> + when (result) { + is ResolutionResult.Resolved -> metadata[result.url] = result.metadata + is ResolutionResult.Failed -> if (result.retryable) { + retryableUrls += result.url + } + } + } + VideoMetadataResolution(metadata, retryableUrls) } - private suspend fun resolve(url: String): Pair? = try { + private suspend fun resolve(url: String): ResolutionResult? = try { when (val result = streamService.getStreamInfo(url)) { - is ExtractionResult.Success -> url to result.data.toMetadata(url) - is ExtractionResult.BadRequest -> null - is ExtractionResult.Failure -> null + is ExtractionResult.Success -> ResolutionResult.Resolved(url, result.data.toMetadata(url)) + is ExtractionResult.BadRequest -> ResolutionResult.Failed(url, retryable = false) + is ExtractionResult.Failure -> ResolutionResult.Failed(url, retryable = true) } } catch (e: CancellationException) { throw e } catch (_: Exception) { - null + ResolutionResult.Failed(url, retryable = true) } private fun shouldEnrich(video: PlaylistVideoItem): Boolean = @@ -88,7 +107,19 @@ class VideoMetadataResolver(private val streamService: StreamService) { private companion object { const val FALLBACK_TITLE_PREFIX = "YouTube video " - const val MAX_CONCURRENT_RESOLUTIONS = 8 + const val MAX_CONCURRENT_RESOLUTIONS = 2 const val YOUTUBE_THUMB_PREFIX = "https://i.ytimg.com/vi/" } } + +private sealed interface ResolutionResult { + data class Resolved( + val url: String, + val metadata: VideoMetadataItem, + ) : ResolutionResult + + data class Failed( + val url: String, + val retryable: Boolean, + ) : ResolutionResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt index 525c7e55..e741eb0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.search.filter.FilterItem private val VALID_YOUTUBE_CHANNEL_TAB_SORTS = setOf("latest", "popular", "oldest") -internal fun String?.toYouTubeChannelTabSortFilter(): List? { +fun String?.toYouTubeChannelTabSortFilter(): List? { val normalized = this?.lowercase()?.takeIf { it.isNotBlank() } ?: return null if (normalized !in VALID_YOUTUBE_CHANNEL_TAB_SORTS) throw IllegalArgumentException("Invalid 'sort' parameter") return listOf(FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, normalized)) diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt index 03297d24..b0a54d21 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt @@ -7,7 +7,7 @@ import java.security.MessageDigest import java.time.Duration import java.util.HexFormat -internal class YouTubeSubtitleCache(private val sharedCache: CacheService?) { +class YouTubeSubtitleCache(private val sharedCache: CacheService?) { private val vod = BoundedExpiringCache( maxEntries = 128, maxWeight = MAX_MEMORY_BYTES, diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt index acb9e662..65b14152 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt @@ -12,11 +12,11 @@ import okhttp3.Response import java.io.IOException import kotlin.coroutines.resume -internal fun interface YouTubeSubtitleContentFetcher { +fun interface YouTubeSubtitleContentFetcher { suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult } -internal class OkHttpYouTubeSubtitleContentFetcher( +class OkHttpYouTubeSubtitleContentFetcher( private val client: OkHttpClient, ) : YouTubeSubtitleContentFetcher { override suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult { @@ -33,7 +33,7 @@ internal class OkHttpYouTubeSubtitleContentFetcher( } } -internal class TokenYouTubeSubtitleContentFetcher( +class TokenYouTubeSubtitleContentFetcher( private val client: OkHttpClient, baseUrl: String, private val directFetcher: YouTubeSubtitleContentFetcher, @@ -92,9 +92,9 @@ private fun readSubtitleResponse(response: Response, format: YouTubeSubtitleForm } private const val YOUTUBE_ORIGIN = "https://m.youtube.com" -internal const val MAX_SUBTITLE_BYTES = 5 * 1024 * 1024 +const val MAX_SUBTITLE_BYTES = 5 * 1024 * 1024 -internal fun isValidSubtitlePayload(content: ByteArray, format: YouTubeSubtitleFormat): Boolean { +fun isValidSubtitlePayload(content: ByteArray, format: YouTubeSubtitleFormat): Boolean { if (content.isEmpty() || content.size > MAX_SUBTITLE_BYTES) return false val text = content.toString(Charsets.UTF_8).trimStart() return when (format) { diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt index 1b60b8c3..3de2b424 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.schabi.newpipe.extractor.MediaFormat -internal enum class YouTubeSubtitleFormat( +enum class YouTubeSubtitleFormat( val value: String, val mediaFormat: MediaFormat, val contentType: String, @@ -16,7 +16,7 @@ internal enum class YouTubeSubtitleFormat( } } -internal enum class YouTubeSubtitleVariant(val value: String) { +enum class YouTubeSubtitleVariant(val value: String) { Manual("manual"), Auto("auto"); @@ -25,7 +25,7 @@ internal enum class YouTubeSubtitleVariant(val value: String) { } } -internal data class YouTubeSubtitleSelection( +data class YouTubeSubtitleSelection( val videoId: String, val language: String, val variant: YouTubeSubtitleVariant, @@ -45,20 +45,20 @@ internal data class YouTubeSubtitleSelection( ).joinToString(":") } -internal data class ResolvedYouTubeSubtitle( +data class ResolvedYouTubeSubtitle( val content: String, val isUrl: Boolean, val isLive: Boolean, ) -internal sealed interface YouTubeSubtitleResolution { +sealed interface YouTubeSubtitleResolution { data class Ready(val track: ResolvedYouTubeSubtitle) : YouTubeSubtitleResolution data object NotFound : YouTubeSubtitleResolution data object Throttled : YouTubeSubtitleResolution data object Unavailable : YouTubeSubtitleResolution } -internal sealed interface YouTubeSubtitleFetchResult { +sealed interface YouTubeSubtitleFetchResult { data class Ready(val content: ByteArray) : YouTubeSubtitleFetchResult data object Expired : YouTubeSubtitleFetchResult data object Throttled : YouTubeSubtitleFetchResult @@ -66,7 +66,7 @@ internal sealed interface YouTubeSubtitleFetchResult { data object Unavailable : YouTubeSubtitleFetchResult } -internal sealed interface YouTubeSubtitleContentResult { +sealed interface YouTubeSubtitleContentResult { data class Ready( val content: ByteArray, val format: YouTubeSubtitleFormat, @@ -81,7 +81,7 @@ internal sealed interface YouTubeSubtitleContentResult { data object Unavailable : YouTubeSubtitleContentResult } -internal fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleSelection? { +fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleSelection? { val url = rawUrl.toHttpUrlOrNull()?.takeIf { it.isYouTubeTimedText() } ?: return null val videoId = url.queryParameter("v")?.trim().orEmpty() val sourceLanguage = url.queryParameter("lang")?.trim().orEmpty() @@ -100,13 +100,13 @@ internal fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleS ) } -internal fun isYouTubeTimedTextUrl(rawUrl: String): Boolean = +fun isYouTubeTimedTextUrl(rawUrl: String): Boolean = rawUrl.toHttpUrlOrNull()?.isYouTubeTimedText() == true -internal fun isValidYouTubeVideoId(value: String): Boolean = +fun isValidYouTubeVideoId(value: String): Boolean = value.length == 11 && value.all { it.isLetterOrDigit() || it == '_' || it == '-' } -internal fun isValidSubtitleTag(value: String): Boolean = +fun isValidSubtitleTag(value: String): Boolean = value.isNotEmpty() && value.length <= 64 && value.all { it.isLetterOrDigit() || it in "-_." } private fun okhttp3.HttpUrl.isYouTubeTimedText(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt index 1e5ff2a2..713f7dc7 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt @@ -1,10 +1,11 @@ package dev.typetype.server.services import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CancellationException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Semaphore -internal class YouTubeSubtitleDeliveryService( +class YouTubeSubtitleDeliveryService( private val resolver: YouTubeSubtitleTrackResolver, private val fetcher: YouTubeSubtitleContentFetcher, private val cache: YouTubeSubtitleCache, @@ -28,8 +29,12 @@ internal class YouTubeSubtitleDeliveryService( pending.complete(result) result } catch (error: Throwable) { - pending.completeExceptionally(error) - throw error + if (error is CancellationException) { + pending.cancel(error) + throw error + } + pending.complete(YouTubeSubtitleContentResult.Unavailable) + YouTubeSubtitleContentResult.Unavailable } finally { inFlight.remove(selection.cacheKey, pending) upstreamPermits.release() diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt index 0d1f5367..0a0f504c 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt @@ -13,7 +13,7 @@ import okhttp3.Response import java.io.IOException import kotlin.coroutines.resume -internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, private val baseUrl: String) { +class YouTubeSubtitleService(private val httpClient: OkHttpClient, private val baseUrl: String) { suspend fun fetchSubtitles(videoId: String): List = when (val result = fetchSubtitleInventory(videoId)) { @@ -55,7 +55,7 @@ internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, priv } -internal sealed interface YouTubeSubtitleInventoryResult { +sealed interface YouTubeSubtitleInventoryResult { data class Ready(val tracks: List) : YouTubeSubtitleInventoryResult data object Unavailable : YouTubeSubtitleInventoryResult } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt new file mode 100644 index 00000000..134623a0 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt @@ -0,0 +1,215 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.selects.select +import kotlinx.coroutines.withTimeoutOrNull +import org.slf4j.LoggerFactory + +class YoutubeDirectLiveHlsStreamService( + private val liveHlsService: StreamService, + private val fallbackService: StreamService? = null, +) : StreamService { + private val logger = LoggerFactory.getLogger(YoutubeDirectLiveHlsStreamService::class.java) + + override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { + if (!isYoutubeUrl(url)) return@coroutineScope ExtractionResult.BadRequest("Invalid YouTube URL") + + val videoId = youtubeVideoId(url) ?: "unknown" + val totalStartedAt = System.nanoTime() + val directStartedAt = System.nanoTime() + val directAttempt = async { timedExtraction(liveHlsService, url, directStartedAt) } + val directBeforeHedge = withTimeoutOrNull(HEDGE_DELAY_MILLIS) { directAttempt.await() } + + if (directBeforeHedge != null) { + return@coroutineScope resolveDirect( + url = url, + videoId = videoId, + direct = directBeforeHedge, + totalStartedAt = totalStartedAt, + ) + } + + val fallback = fallbackService + if (fallback == null) { + return@coroutineScope resolveDirect( + url = url, + videoId = videoId, + direct = directAttempt.await(), + totalStartedAt = totalStartedAt, + ) + } + + val fallbackStartedAt = System.nanoTime() + val fallbackAttempt = async { timedExtraction(fallback, url, fallbackStartedAt) } + try { + val first = select { + directAttempt.onAwait { CompletedAttempt(AttemptSource.DIRECT, it) } + fallbackAttempt.onAwait { CompletedAttempt(AttemptSource.FALLBACK, it) } + } + if (first.source == AttemptSource.FALLBACK && first.extraction.result.isLiveHls()) { + val totalElapsedMs = elapsedMsSince(totalStartedAt) + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} directCode=pending result=live_hls source=fallback", + videoId, + elapsedMsSince(directStartedAt), + first.extraction.elapsedMs, + totalElapsedMs, + ) + return@coroutineScope first.extraction.result + } + + val direct = if (first.source == AttemptSource.DIRECT) { + first.extraction + } else { + directAttempt.await() + } + resolveDirect( + url = url, + videoId = videoId, + direct = direct, + totalStartedAt = totalStartedAt, + fallbackAttempt = fallbackAttempt, + completedFallback = first.extraction.takeIf { first.source == AttemptSource.FALLBACK }, + fallbackStartedAt = fallbackStartedAt, + ) + } finally { + directAttempt.cancel() + fallbackAttempt.cancel() + } + } + + private suspend fun resolveDirect( + url: String, + videoId: String, + direct: TimedExtraction, + totalStartedAt: Long, + fallbackAttempt: Deferred? = null, + completedFallback: TimedExtraction? = null, + fallbackStartedAt: Long? = null, + ): ExtractionResult = when (val directResult = direct.result) { + is ExtractionResult.Success -> { + if (directResult.data.isLive && directResult.data.hlsUrl.isNotBlank()) { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=direct_hls", + videoId, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), + ) + directResult + } else { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=not_live", + videoId, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), + ) + ExtractionResult.Failure( + "No active YouTube live stream is available", + "live_stream_unavailable", + ) + } + } + is ExtractionResult.BadRequest -> { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=bad_request code={}", + videoId, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), + directResult.code, + ) + directResult + } + is ExtractionResult.Failure -> resolveFailure( + url = url, + videoId = videoId, + direct = direct, + failure = directResult, + totalStartedAt = totalStartedAt, + fallbackAttempt = fallbackAttempt, + completedFallback = completedFallback, + ) + } + + private suspend fun resolveFailure( + url: String, + videoId: String, + direct: TimedExtraction, + failure: ExtractionResult.Failure, + totalStartedAt: Long, + fallbackAttempt: Deferred?, + completedFallback: TimedExtraction?, + ): ExtractionResult { + val fallback = fallbackService + if (fallback == null) { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 totalMs={} result=direct_failure directCode={}", + videoId, + direct.elapsedMs, + elapsedMsSince(totalStartedAt), + failure.code, + ) + return failure + } + + val extraction = completedFallback ?: fallbackAttempt?.await() ?: timedExtraction(fallback, url) + val liveFallback = (extraction.result as? ExtractionResult.Success) + ?.takeIf { it.data.isLive && it.data.hlsUrl.isNotBlank() } + val fallbackStatus = when (val result = extraction.result) { + is ExtractionResult.Success -> if (liveFallback != null) "live_hls" else "not_live" + is ExtractionResult.BadRequest -> "bad_request:${result.code}" + is ExtractionResult.Failure -> "failure:${result.code}" + } + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} directCode={} result={}", + videoId, + direct.elapsedMs, + extraction.elapsedMs, + elapsedMsSince(totalStartedAt), + failure.code, + fallbackStatus, + ) + return liveFallback ?: failure + } + + private suspend fun timedExtraction( + service: StreamService, + url: String, + startedAt: Long = System.nanoTime(), + ): TimedExtraction { + val result = service.getStreamInfo(url) + return TimedExtraction(result, elapsedMsSince(startedAt)) + } + + private fun ExtractionResult.isLiveHls(): Boolean = + (this as? ExtractionResult.Success)?.data?.let { it.isLive && it.hlsUrl.isNotBlank() } == true + + private fun fallbackElapsedMs(completed: TimedExtraction?, startedAt: Long?): Long = + completed?.elapsedMs ?: startedAt?.let(::elapsedMsSince) ?: 0 + + private fun elapsedMsSince(startedAt: Long): Long = + (System.nanoTime() - startedAt) / NANOS_PER_MILLI + + private data class TimedExtraction( + val result: ExtractionResult, + val elapsedMs: Long, + ) + + private data class CompletedAttempt( + val source: AttemptSource, + val extraction: TimedExtraction, + ) + + private enum class AttemptSource { DIRECT, FALLBACK } + + private companion object { + const val HEDGE_DELAY_MILLIS = 100L + const val NANOS_PER_MILLI = 1_000_000L + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt new file mode 100644 index 00000000..bc43771f --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt @@ -0,0 +1,21 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class YoutubeLiveHlsStreamService( + private val metadataService: StreamService, + private val liveHlsService: StreamService, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return metadataService.getStreamInfo(url) + + val liveResult = liveHlsService.getStreamInfo(url) + val liveResponse = (liveResult as? ExtractionResult.Success)?.data + if (liveResponse?.isLive == true && !liveResponse.requiresMembership && liveResponse.hlsUrl.isNotBlank()) { + return liveResult + } + + return metadataService.getStreamInfo(url) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt index 90fda099..1955bc14 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class YoutubePlayerClient(val value: String) { +enum class YoutubePlayerClient(val value: String) { MWEB("mweb"), VISIONOS("visionos"), TV_DOWNGRADED("tv_downgraded"), diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt index 957af39c..a8d4e5a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -internal class YoutubePlayerClientFallbackStreamService( +class YoutubePlayerClientFallbackStreamService( private val delegate: StreamService, private val clients: List, ) : StreamService { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt index 770951f2..b66046ca 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.schabi.newpipe.extractor.NewPipe -internal object YoutubePlayerClientScope { +object YoutubePlayerClientScope { private const val CONCURRENT_SABR_PERMITS = 64 private val permits = Semaphore(CONCURRENT_SABR_PERMITS, true) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt index ac916861..0b814010 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -internal class YoutubePlayerClientStreamService( +class YoutubePlayerClientStreamService( private val delegate: StreamService, private val client: YoutubePlayerClient, ) : StreamService { diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt new file mode 100644 index 00000000..5ec135fc --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.YoutubeRemoteBrowserCompleteRequest +import org.slf4j.LoggerFactory + +object YoutubeRemoteBrowserAudit { + private val log = LoggerFactory.getLogger("YoutubeRemoteLogin") + private val typePattern = Regex("\"type\"\\s*:\\s*\"([a-z_]{1,32})\"") + + fun startResult(userId: String, result: String) { + log.info("remote-login start user={} result={}", short(userId), result) + } + + fun bridgeOpened(tokenSessionId: String) { + log.info("remote-login bridge session={} token socket open", short(tokenSessionId)) + } + + fun bridgeClosed(tokenSessionId: String, code: Int, reason: String) { + log.info("remote-login bridge session={} token socket closed code={} reason={}", short(tokenSessionId), code, reason) + } + + fun bridgeFailed(tokenSessionId: String, error: Throwable) { + log.warn("remote-login bridge session={} token socket failed: {}", short(tokenSessionId), error.message ?: error.javaClass.simpleName) + } + + fun bridgeEnded(tokenSessionId: String, texts: Int, frames: Int, inputs: Int) { + log.info("remote-login bridge session={} ended texts={} frames={} inputs={}", short(tokenSessionId), texts, frames, inputs) + } + + fun droppedTokenMessage(tokenSessionId: String, text: String) { + val type = typePattern.find(text)?.groupValues?.get(1) ?: "unknown" + log.warn("remote-login bridge session={} dropped token message type={} bytes={}", short(tokenSessionId), type, text.length) + } + + fun completion(request: YoutubeRemoteBrowserCompleteRequest, result: YoutubeRemoteBrowserCompleteResult) { + val lines = request.cookies.lineSequence().count { it.isNotBlank() && !it.startsWith("#") } + val sapisid = request.cookies.contains("SAPISID") + val loginInfo = request.cookies.contains("LOGIN_INFO") + log.info( + "remote-login completion session={} result={} cookies={} sapisid={} loginInfo={} poToken={} authUser={}", + short(request.sessionId), result::class.simpleName, lines, sapisid, loginInfo, request.poToken.length, request.authUser, + ) + } + + private fun short(value: String): String = value.take(8) +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt new file mode 100644 index 00000000..8580388f --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt @@ -0,0 +1,50 @@ +package dev.typetype.server.services + +import io.ktor.websocket.Frame +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.channels.Channel +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import java.util.concurrent.atomic.AtomicInteger + +class YoutubeRemoteBrowserBridgeListener( + private val tokenSessionId: String, + private val outbound: Channel, + private val done: CompletableDeferred, + private val config: YoutubeRemoteBrowserConfig, +) : WebSocketListener() { + val texts = AtomicInteger() + val frames = AtomicInteger() + + override fun onOpen(webSocket: WebSocket, response: Response) { + YoutubeRemoteBrowserAudit.bridgeOpened(tokenSessionId) + } + + override fun onMessage(webSocket: WebSocket, text: String) { + val accepted = YoutubeRemoteBrowserMessageGuard.tokenText(text) + if (accepted == null) { + YoutubeRemoteBrowserAudit.droppedTokenMessage(tokenSessionId, text) + return + } + texts.incrementAndGet() + outbound.trySend(Frame.Text(accepted)) + } + + override fun onMessage(webSocket: WebSocket, bytes: ByteString) { + if (bytes.size > config.maxFrameBytes) return + frames.incrementAndGet() + outbound.trySend(Frame.Binary(true, bytes.toByteArray())) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + YoutubeRemoteBrowserAudit.bridgeClosed(tokenSessionId, code, reason) + done.complete(Unit) + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + YoutubeRemoteBrowserAudit.bridgeFailed(tokenSessionId, t) + done.complete(Unit) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt new file mode 100644 index 00000000..eba375c5 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.services + +sealed interface YoutubeRemoteBrowserCompleteResult { + data object Completed : YoutubeRemoteBrowserCompleteResult + data object Unauthorized : YoutubeRemoteBrowserCompleteResult + data object NotFound : YoutubeRemoteBrowserCompleteResult + data object InvalidPayload : YoutubeRemoteBrowserCompleteResult + data object InvalidCredentials : YoutubeRemoteBrowserCompleteResult + data object Unavailable : YoutubeRemoteBrowserCompleteResult +} + +fun YoutubeSessionCompleteResult.toRemoteBrowserResult(): YoutubeRemoteBrowserCompleteResult = when (this) { + YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed + YoutubeSessionCompleteResult.InvalidCode, + YoutubeSessionCompleteResult.ExpiredCode, + YoutubeSessionCompleteResult.InvalidCredentials -> YoutubeRemoteBrowserCompleteResult.InvalidCredentials + YoutubeSessionCompleteResult.Unavailable -> YoutubeRemoteBrowserCompleteResult.Unavailable +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt index a5d6d429..f65c6d1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt @@ -22,11 +22,13 @@ class YoutubeRemoteBrowserService( return YoutubeRemoteBrowserStartResult.Misconfigured } val internalToken = config.internalToken ?: return YoutubeRemoteBrowserStartResult.Misconfigured - return when (val reserved = sessions.reserve(userId, config)) { + val result = when (val reserved = sessions.reserve(userId, config)) { YoutubeRemoteBrowserReserveResult.AlreadyActive -> YoutubeRemoteBrowserStartResult.AlreadyActive YoutubeRemoteBrowserReserveResult.CapacityReached -> YoutubeRemoteBrowserStartResult.CapacityReached is YoutubeRemoteBrowserReserveResult.Reserved -> startTokenSession(reserved, userId, internalToken, returnTo) } + YoutubeRemoteBrowserAudit.startResult(userId, result::class.simpleName ?: "unknown") + return result } suspend fun cancel(userId: String, sessionId: String): Boolean { @@ -44,21 +46,12 @@ class YoutubeRemoteBrowserService( if (request.status != "completed") return YoutubeRemoteBrowserCompleteResult.InvalidPayload if (!youtubeSessionService.isConfigured) return YoutubeRemoteBrowserCompleteResult.Unavailable val session = sessions.complete(request.sessionId, request.tokenSessionId) - ?: return YoutubeRemoteBrowserCompleteResult.NotFound - return when ( - youtubeSessionService.completeRemote( - session.userId, - request.cookies, - request.poToken, - request.authUser, - ) - ) { - YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed - YoutubeSessionCompleteResult.InvalidCode, - YoutubeSessionCompleteResult.ExpiredCode, - YoutubeSessionCompleteResult.InvalidCredentials -> YoutubeRemoteBrowserCompleteResult.InvalidCredentials - YoutubeSessionCompleteResult.Unavailable -> YoutubeRemoteBrowserCompleteResult.Unavailable - } + ?: return YoutubeRemoteBrowserCompleteResult.NotFound.also { YoutubeRemoteBrowserAudit.completion(request, it) } + val result = youtubeSessionService + .completeRemote(session.userId, request.cookies, request.poToken, request.authUser) + .toRemoteBrowserResult() + YoutubeRemoteBrowserAudit.completion(request, result) + return result } suspend fun bridge(sessionId: String, wsToken: String?, serverSession: DefaultWebSocketServerSession): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt similarity index 71% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index 962039b7..b93c2667 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -5,10 +5,10 @@ import dev.typetype.server.models.StreamResponse import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeout -internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" -internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" +const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" +const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" -internal class YoutubeSessionSabrStreamService( +class YoutubeSessionSabrStreamService( private val metadataService: YoutubeSessionStreamService, private val infoService: AuthenticatedSabrInfoService, private val timeoutMs: Long = AuthenticatedSabrPolicy.STREAM_TIMEOUT_MS, @@ -18,6 +18,7 @@ internal class YoutubeSessionSabrStreamService( withTimeout(timeoutMs) { val metadata = metadataService.getStreamInfo(userId, url) ?: return@withTimeout null if (metadata !is ExtractionResult.Success) return@withTimeout metadata + if (metadata.data.isLive) return@withTimeout null val videoId = youtubeVideoId(url) ?: return@withTimeout ExtractionResult.BadRequest("Invalid YouTube URL") when (val info = infoService.fetch(userId, videoId)) { is AuthenticatedSabrInfoResult.Ready -> @@ -25,20 +26,22 @@ internal class YoutubeSessionSabrStreamService( AuthenticatedSabrInfoResult.Failed -> ExtractionResult.Failure("Authenticated SABR playback unavailable") AuthenticatedSabrInfoResult.TimedOut -> - ExtractionResult.Failure( - "Authenticated SABR preparation timed out", - AuthenticatedSabrPolicy.TIMEOUT_CODE, - ) + reconnectResult(userId) AuthenticatedSabrInfoResult.NotConnected -> null } } } catch (error: TimeoutCancellationException) { - ExtractionResult.Failure("Authenticated SABR preparation timed out", AuthenticatedSabrPolicy.TIMEOUT_CODE) + reconnectResult(userId) } } + + private suspend fun reconnectResult(userId: String): ExtractionResult { + metadataService.markYoutubeSessionNeedsReconnect(userId) + return ExtractionResult.BadRequest(YOUTUBE_SESSION_RECONNECT_ERROR, YOUTUBE_SESSION_RECONNECT_CODE) + } } -internal fun ExtractionResult.requiresYoutubeSession(): Boolean = +fun ExtractionResult.requiresYoutubeSession(): Boolean = when (this) { is ExtractionResult.Success -> data.requiresMembership is ExtractionResult.BadRequest -> code == "age_restricted" || code == "members_only" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index d9ca60b6..214d68bb 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -45,6 +45,10 @@ class YoutubeSessionStreamService( return result } + suspend fun markYoutubeSessionNeedsReconnect(userId: String) { + youtubeSessionService.markNeedsReconnect(userId) + } + private fun StreamResponse.withSignedHlsUrl( credentials: YoutubeSessionCredentials, url: String, @@ -79,4 +83,4 @@ class YoutubeSessionStreamService( } } -internal const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" +const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt index e8a665b6..d94d39dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt @@ -5,6 +5,8 @@ import dev.typetype.server.models.YoutubeTakeoutImportReportItem import dev.typetype.server.models.YoutubeTakeoutPreviewItem import dev.typetype.server.models.YoutubeTakeoutCommitRequest import dev.typetype.server.models.YoutubeTakeoutCommitPlan +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import java.nio.file.Path @@ -60,7 +62,7 @@ class YoutubeTakeoutImportJobService( if (flags.importStarted) return statusStore.getStatus(userId, jobId) ?: error("Missing job") val plan = YoutubeTakeoutCommitPlanner.fromRequest(request) flagsStore.setImportStarted(jobId) - statusStore.updateStatus(jobId, "running", "importing", 75) + statusStore.updateStatus(jobId, "running", "importing", 0) engine.startCommit(jobId, plan) { runCommit(userId, jobId, it) } return statusStore.getStatus(userId, jobId) ?: error("Missing job") } @@ -69,7 +71,7 @@ class YoutubeTakeoutImportJobService( try { val flags = flagsStore.getFlags(userId, jobId) if (!flags.parseCompleted) preview(userId, jobId) - statusStore.updateStatus(jobId, "running", "importing", 75) + statusStore.updateStatus(jobId, "running", "importing", 0) val parsed = cache.getParsed(jobId) ?: parser.parse(Path.of(archiveStore.getArchivePath(userId, jobId))).also { cache.setParsed(jobId, it) val preview = previewService.build(userId, it) @@ -77,7 +79,19 @@ class YoutubeTakeoutImportJobService( cache.setPreview(jobId, preview) flagsStore.setParseCompleted(jobId) } - val report = importerService.commit(userId, parsed, plan) + val report = coroutineScope { + val progress = YoutubeTakeoutImportProgress(importTotal(parsed, plan)) + val publisher = launch { + progress.drain { value -> statusStore.updateStatus(jobId, "running", "importing", value) } + } + try { + importerService.commit(userId, parsed, plan) { processed, _ -> progress.offer(processed) } + .also { progress.finish() } + } finally { + progress.close() + publisher.join() + } + } reportStore.persistReport(jobId, Json.encodeToString(report)) flagsStore.setImportCompleted(jobId) statusStore.updateStatus(jobId, "completed", "completed", 100) @@ -95,6 +109,15 @@ class YoutubeTakeoutImportJobService( return "$name: $detail" } + private fun importTotal(parsed: dev.typetype.server.models.YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): Long = buildList { + if (plan.importSubscriptions) add(parsed.subscriptions.size.toLong()) + if (plan.importPlaylists) add(parsed.playlists.size.toLong()) + if (plan.importPlaylistItems) add(parsed.playlistItems.values.sumOf { it.size }.toLong()) + if (plan.importFavorites) add(parsed.favorites.size.toLong()) + if (plan.importWatchLater) add(parsed.watchLater.size.toLong()) + if (plan.importHistory) add(parsed.history.size.toLong()) + }.sum() + suspend fun report(userId: String, jobId: String): YoutubeTakeoutImportReportItem? { val reportJson = reportStore.getReport(userId, jobId) ?: return null return Json.decodeFromString(YoutubeTakeoutImportReportItem.serializer(), reportJson) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt index e1beb779..179627af 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt @@ -9,6 +9,7 @@ import dev.typetype.server.models.YoutubeTakeoutImportStats import dev.typetype.server.models.YoutubeTakeoutParsedData import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import java.util.concurrent.atomic.AtomicLong class YoutubeTakeoutImporterService( private val subscriptionsService: SubscriptionsService, @@ -16,8 +17,16 @@ class YoutubeTakeoutImporterService( private val signalImportService: YoutubeTakeoutSignalImportService, private val playlistKeyService: YoutubeTakeoutPlaylistKeyService = YoutubeTakeoutPlaylistKeyService(), ) { - suspend fun commit(userId: String, parsed: YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): YoutubeTakeoutImportReportItem = coroutineScope { + suspend fun commit( + userId: String, + parsed: YoutubeTakeoutParsedData, + plan: YoutubeTakeoutCommitPlan, + onProgress: (processed: Long, total: Long) -> Unit = { _, _ -> }, + ): YoutubeTakeoutImportReportItem = coroutineScope { val (issues, issueSummary) = YoutubeTakeoutIssueService.build(parsed.warnings, parsed.errors, stage = "commit") + val total = importTotal(parsed, plan) + val processed = AtomicLong() + val progress = { onProgress(processed.incrementAndGet(), total) } val existingSubsDeferred = async { subscriptionsService.getAll(userId).map { it.channelUrl }.toSet() } val existingPlaylistsDeferred = async { playlistService.getAll(userId) } val sourceMappingsDeferred = async { playlistKeyService.getMappings(userId).toMutableMap() } @@ -35,6 +44,7 @@ class YoutubeTakeoutImporterService( existingSubs += canonicalUrl subImported += 1 } + progress() } } var plImported = 0 @@ -47,6 +57,7 @@ class YoutubeTakeoutImporterService( parsed.playlists.forEach { item -> if (YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null) { plSkipped += 1 + progress() return@forEach } val nameKey = item.name.lowercase() @@ -68,6 +79,7 @@ class YoutubeTakeoutImporterService( playlistKeyService.putMapping(userId, idKey, playlist.id) sourceMappings[idKey] = playlist.id } + progress() } } if (plan.importPlaylistItems) { @@ -75,11 +87,17 @@ class YoutubeTakeoutImporterService( val normalizedKey = playlistKey.lowercase() if (YoutubeTakeoutSystemPlaylist.canonicalKey(normalizedKey) != null) { itemSkipped += videos.size + repeat(videos.size) { progress() } return@forEach } val mappedId = sourceMappings[normalizedKey].orEmpty() val mappedPlaylist = if (mappedId.isBlank()) null else playlistService.getById(userId, mappedId) - val playlist = mappedPlaylist ?: createdBySource[normalizedKey] ?: return@forEach + val playlist = mappedPlaylist ?: createdBySource[normalizedKey] + if (playlist == null) { + itemSkipped += videos.size + repeat(videos.size) { progress() } + return@forEach + } val existingUrls = playlistService.getById(userId, playlist.id)?.videos?.map { it.url }.orEmpty().toMutableSet() videos.forEach { video -> if (video.url in existingUrls) itemSkipped += 1 else { @@ -87,12 +105,19 @@ class YoutubeTakeoutImporterService( itemImported += 1 existingUrls += video.url } + progress() } } } - val favoriteDeferred = if (plan.importFavorites) async { signalImportService.importFavorites(userId, favorites) } else null - val watchLaterDeferred = if (plan.importWatchLater) async { signalImportService.importWatchLater(userId, parsed.watchLater) } else null - val historyDeferred = if (plan.importHistory) async { signalImportService.importHistory(userId, parsed.history) } else null + val favoriteDeferred = if (plan.importFavorites) async { + signalImportService.importFavorites(userId, favorites) { progress() } + } else null + val watchLaterDeferred = if (plan.importWatchLater) async { + signalImportService.importWatchLater(userId, parsed.watchLater) { progress() } + } else null + val historyDeferred = if (plan.importHistory) async { + signalImportService.importHistory(userId, parsed.history) { progress() } + } else null val emptyStats = YoutubeTakeoutImportStats(0, 0, 0) val favoriteStats = favoriteDeferred?.await() ?: emptyStats val watchLaterStats = watchLaterDeferred?.await() ?: emptyStats @@ -114,4 +139,13 @@ class YoutubeTakeoutImporterService( if (plan.importSubscriptions) SubscriptionFeedCacheInvalidation.invalidate(userId) report } + + private fun importTotal(parsed: YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): Long = buildList { + if (plan.importSubscriptions) add(parsed.subscriptions.size.toLong()) + if (plan.importPlaylists) add(parsed.playlists.size.toLong()) + if (plan.importPlaylistItems) add(parsed.playlistItems.values.sumOf { it.size }.toLong()) + if (plan.importFavorites) add(parsed.favorites.size.toLong()) + if (plan.importWatchLater) add(parsed.watchLater.size.toLong()) + if (plan.importHistory) add(parsed.history.size.toLong()) + }.sum() } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt index 1196917a..6de323d4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt @@ -92,17 +92,20 @@ class YoutubeTakeoutParserService { private fun parseHistory(zipPath: Path, warnings: MutableList): List { ZipFile(zipPath.toFile()).use { zip -> val entries = zip.entries().asSequence().filter { item -> - val normalized = item.name.lowercase() - !item.isDirectory && normalized.endsWith(".html") && normalized.contains("youtube") + !item.isDirectory && YoutubeTakeoutPathHints.isYoutubeHtml(item.name) }.toList() - val entry = entries.firstOrNull { it.name.lowercase().contains("watch-history") } - ?: entries.firstOrNull { it.name.lowercase().contains("monactiv") } + val entry = entries.firstOrNull { YoutubeTakeoutPathHints.isHistoryEntry(it.name) } + ?: entries.firstOrNull { YoutubeTakeoutTextNormalizer.normalize(it.name).contains("monactiv") } ?: entries.firstOrNull() if (entry == null) return emptyList() val html = zip.getInputStream(entry).bufferedReader().use { it.readText() } - val parsed = YoutubeTakeoutHistoryParser.parse(html) - if (parsed.isEmpty()) warnings += "No watch history rows detected" - return parsed.map(YoutubeTypeTypeMapper::historyItem) + val parsed = YoutubeTakeoutHistoryParser.parseWithDiagnostics( + html, + requireWatchedMarker = !YoutubeTakeoutPathHints.isHistoryEntry(entry.name), + ) + if (parsed.items.isEmpty()) warnings += "No watch history rows detected" + if (parsed.invalidDates > 0) warnings += "Skipped ${parsed.invalidDates} watch history rows with invalid dates" + return parsed.items.map(YoutubeTypeTypeMapper::historyItem) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt index 30412a7d..e9b09df4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt @@ -11,7 +11,11 @@ class YoutubeTakeoutSignalImportService( private val watchLaterService: WatchLaterService, private val historyService: HistoryService, ) { - suspend fun importFavorites(userId: String, items: List): YoutubeTakeoutImportStats { + suspend fun importFavorites( + userId: String, + items: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var imported = 0 var skipped = 0 val existing = favoritesService.getAll(userId).map { it.videoUrl }.toMutableSet() @@ -21,11 +25,16 @@ class YoutubeTakeoutSignalImportService( imported += 1 existing += item.videoUrl } + onProgress() } return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) } - suspend fun importWatchLater(userId: String, videos: List): YoutubeTakeoutImportStats { + suspend fun importWatchLater( + userId: String, + videos: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var imported = 0 var skipped = 0 val existing = watchLaterService.getAll(userId).map { it.url }.toMutableSet() @@ -35,11 +44,16 @@ class YoutubeTakeoutSignalImportService( imported += 1 existing += video.url } + onProgress() } return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) } - suspend fun importHistory(userId: String, items: List): YoutubeTakeoutImportStats { + suspend fun importHistory( + userId: String, + items: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var skipped = 0 val existing = historyService.dedupKeys(userId).toMutableSet() val toInsert = mutableListOf() @@ -49,6 +63,7 @@ class YoutubeTakeoutSignalImportService( toInsert += item existing += key } + onProgress() } val imported = historyService.addImportedBatch(userId, toInsert) return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt similarity index 73% rename from src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt index 59cdf9e2..449954c3 100644 --- a/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt @@ -16,6 +16,14 @@ class ChannelTabResolverTest { assertEquals("https://www.youtube.com/@test", url.toBaseChannelUrl(ChannelTabs.LIVESTREAMS)) } + @Test + fun `niconico livestreams url maps to livestreams tab`() { + val url = "https://www.nicovideo.jp/user/123/livestreams" + + assertEquals(ChannelTabs.LIVESTREAMS, url.toChannelTab(null)) + assertEquals("https://www.nicovideo.jp/user/123", url.toBaseChannelUrl(ChannelTabs.LIVESTREAMS)) + } + @Test fun `channel sort maps base url to videos tab`() { val url = "https://www.youtube.com/@test" diff --git a/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt new file mode 100644 index 00000000..ece0601a --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt @@ -0,0 +1,99 @@ +package dev.typetype.server + +import dev.typetype.server.services.DeArrowUnavailableException +import dev.typetype.server.services.DeArrowRemote +import dev.typetype.server.services.DeArrowService +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class DeArrowServiceTest { + @Test + fun `selects accepted title and thumbnail and caches branding`() = runBlocking { + val remote = FakeDeArrowRemote() + val service = DeArrowService(FakeCacheService(), remote) + val first = service.get("stZ3ZoR_8eg") + val second = service.get("stZ3ZoR_8eg") + assertEquals("Clear title", first?.title) + assertEquals("/dearrow/thumbnail?videoId=stZ3ZoR_8eg&time=12.5", first?.thumbnailUrl) + assertEquals(listOf("Rejected title", "Clear title", "Original title"), first?.titles?.map { it.title }) + assertEquals(listOf(-2, 2, 0), first?.titles?.map { it.votes }) + assertEquals(listOf(8.0, 12.5, null), first?.thumbnails?.map { it.timestamp }) + assertEquals(0.4, first?.randomTime) + assertEquals(100.0, first?.videoDuration) + assertEquals(first, second) + assertEquals(1, remote.brandingCalls) + } + + @Test + fun `rejects invalid video id without remote call`() = runBlocking { + val remote = FakeDeArrowRemote() + val service = DeArrowService(FakeCacheService(), remote) + assertNull(service.get("invalid")) + assertEquals(0, remote.brandingCalls) + } + + @Test + fun `does not cache upstream failures`() = runBlocking { + val cache = FakeCacheService() + val remote = FakeDeArrowRemote().apply { brandingResult = null } + val service = DeArrowService(cache, remote) + + assertThrows(DeArrowUnavailableException::class.java) { + runBlocking { service.get("stZ3ZoR_8eg") } + } + assertTrue(cache.keys().isEmpty()) + + remote.brandingResult = FakeDeArrowRemote.BRANDING + assertEquals("Clear title", service.get("stZ3ZoR_8eg")?.title) + assertEquals(2, remote.brandingCalls) + } + + @Test + fun `falls back to the original thumbnail when generation fails`() = runBlocking { + val cache = FakeCacheService() + val remote = FakeDeArrowRemote().apply { thumbnailResult = null } + val service = DeArrowService(cache, remote) + + val first = service.thumbnail("stZ3ZoR_8eg", 12.5) + val second = service.thumbnail("stZ3ZoR_8eg", 12.5) + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, first?.bytes) + assertEquals(true, first?.fallback) + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, second?.bytes) + assertEquals(true, second?.fallback) + assertEquals(1, remote.thumbnailCalls) + assertEquals(1, remote.fallbackThumbnailCalls) + } +} + +private class FakeDeArrowRemote : DeArrowRemote { + var brandingCalls = 0 + var thumbnailCalls = 0 + var fallbackThumbnailCalls = 0 + var brandingResult: String? = BRANDING + var thumbnailResult: ByteArray? = byteArrayOf(1, 2, 3) + + override suspend fun branding(videoId: String): String? { + brandingCalls += 1 + return brandingResult + } + + override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? { + thumbnailCalls += 1 + return thumbnailResult + } + + override suspend fun fallbackThumbnail(videoId: String): ByteArray { + fallbackThumbnailCalls += 1 + return FALLBACK_THUMBNAIL + } + + companion object { + const val BRANDING = """{"titles":[{"title":"Rejected title","votes":-2,"locked":false,"original":false,"UUID":"rejected"},{"title":"Clear title","votes":2,"locked":false,"original":false,"UUID":"accepted"},{"title":"Original title","votes":0,"locked":false,"original":true,"UUID":"original"}],"thumbnails":[{"timestamp":8.0,"votes":-1,"locked":false,"original":false,"UUID":"rejected-thumb"},{"timestamp":12.5,"votes":1,"locked":false,"original":false,"UUID":"accepted-thumb"},{"votes":0,"locked":false,"original":true,"UUID":"original-thumb"}],"videoDuration":100,"randomTime":0.4}""" + val FALLBACK_THUMBNAIL = byteArrayOf(9, 8, 7) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt new file mode 100644 index 00000000..b01072df --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt @@ -0,0 +1,114 @@ +package dev.typetype.server + +import dev.typetype.server.services.withExtractionRetry +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException + +class ExtractionResilienceTest { + @Test + fun `returns result without retries when block succeeds`() = runBlocking { + var calls = 0 + val result = withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls += 1 + "ok" + } + assertEquals("ok", result) + assertEquals(1, calls) + } + + @Test + fun `retries retriable error then succeeds`() = runBlocking { + var calls = 0 + val result = withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls += 1 + if (calls < 3) throw RuntimeException("temporary") + "ok" + } + assertEquals("ok", result) + assertEquals(3, calls) + } + + @Test + fun `does not retry illegal argument exception`() = runBlocking { + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls += 1 + throw IllegalArgumentException("bad input") + } + }.exceptionOrNull() + assertEquals(IllegalArgumentException::class.java, error?.javaClass) + assertEquals(1, calls) + } + + @Test + fun `does not retry content availability failures`() = runBlocking { + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw ContentNotAvailableException("unavailable") + } + }.exceptionOrNull() + + assertEquals(ContentNotAvailableException::class.java, error?.javaClass) + assertEquals(1, calls) + } + + @Test + fun `does not retry provider bot challenges`() = runBlocking { + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw AntiBotException("challenge") + } + }.exceptionOrNull() + + assertEquals(AntiBotException::class.java, error?.javaClass) + assertEquals(1, calls) + } + + @Test + fun `propagates cancellation without retry`() = runBlocking { + val cancelled = CancellationException("cancelled") + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw cancelled + } + }.exceptionOrNull() + + assertSame(cancelled, error) + assertEquals(1, calls) + } + + @Test + fun `does not retry access restriction failures`() = runBlocking { + val failures = listOf( + org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException("age restricted"), + org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException("geographic restriction"), + org.schabi.newpipe.extractor.exceptions.PaidContentException("paid content"), + org.schabi.newpipe.extractor.exceptions.PrivateContentException("private content"), + ) + + failures.forEach { failure -> + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw failure + } + }.exceptionOrNull() + + assertSame(failure, error) + assertEquals(1, calls) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt similarity index 96% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt index 3a88482a..e94dd418 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt @@ -24,6 +24,7 @@ class HomeRecommendationCursorCodecTest { discoveryIndex = 12, subscriptionRun = 2, preferDiscovery = false, + rotationSeed = 123456789L, recentChannels = listOf("c1", "c2"), recentSemanticKeys = listOf("linux|kernel"), creatorMomentum = mapOf("https://yt.com/c/a" to 2), @@ -38,6 +39,7 @@ class HomeRecommendationCursorCodecTest { assertEquals(12, decoded?.discoveryIndex) assertEquals(2, decoded?.subscriptionRun) assertEquals(false, decoded?.preferDiscovery) + assertEquals(123456789L, decoded?.rotationSeed) assertEquals(listOf("c1", "c2"), decoded?.recentChannels) assertEquals(listOf("linux|kernel"), decoded?.recentSemanticKeys) assertEquals(2, decoded?.creatorMomentum?.get("https://yt.com/c/a")) diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt new file mode 100644 index 00000000..52746c5c --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server + +import dev.typetype.server.models.HomeRecommendationPool +import dev.typetype.server.services.HomeRecommendationApiArgs +import dev.typetype.server.services.HomeRecommendationContext +import dev.typetype.server.services.HomeRecommendationCursor +import dev.typetype.server.services.HomeRecommendationCursorCodec +import dev.typetype.server.services.HomeRecommendationMixer +import dev.typetype.server.services.HomeRecommendationPageBuilder +import dev.typetype.server.services.HomeRecommendationPoolMode +import dev.typetype.server.services.HomeRecommendationPoolResolver +import dev.typetype.server.services.HomeRecommendationPoolRotation +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test + +class HomeRecommendationPoolRotationTest { + @Test + fun `same seed keeps the pool order stable`() { + val pool = pool() + + val first = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + val second = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + + assertEquals(first, second) + } + + @Test + fun `different seeds rotate the candidate window`() { + val pool = pool() + + val first = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + val second = HomeRecommendationPoolRotation.apply(pool, seed = 12L) + + assertNotEquals(first.subscriptions, second.subscriptions) + assertNotEquals(first.discovery, second.discovery) + assertEquals(pool.subscriptions.map { it.url }.toSet(), first.subscriptions.map { it.url }.toSet()) + assertEquals(pool.discovery.map { it.url }.toSet(), first.discovery.map { it.url }.toSet()) + assertEquals(pool.subscriptions.drop(40), first.subscriptions.drop(40)) + assertEquals(pool.discovery.drop(40), first.discovery.drop(40)) + } + + @Test + fun `the rotation seed stays with the cursor across pages`() { + val pool = HomeRecommendationPoolRotation.apply(pool(), seed = 11L) + val first = HomeRecommendationMixer.mix( + pool = pool, + cursor = HomeRecommendationCursor(rotationSeed = 11L), + limit = 20, + context = HomeRecommendationItemFixtures.context, + ) + val nextCursor = HomeRecommendationCursorCodec.decode(first.nextCursor) + val second = HomeRecommendationMixer.mix( + pool = pool, + cursor = nextCursor ?: error("expected a next cursor"), + limit = 20, + context = HomeRecommendationItemFixtures.context, + ) + + assertEquals(11L, nextCursor.rotationSeed) + assertEquals(emptySet(), first.items.map { it.url }.toSet() intersect second.items.map { it.url }.toSet()) + } + + @Test + fun `initial home pages receive a fresh seed while a seeded page is stable`() = runTest { + val resolver = mockk() + coEvery { resolver.resolve(any(), any(), any(), any()) } returns pool() + val args = HomeRecommendationApiArgs( + userId = "user", + serviceId = 0, + limit = 20, + cursor = HomeRecommendationCursor(), + context = HomeRecommendationContext(serviceId = 0, sessionContext = HomeRecommendationItemFixtures.context), + ) + + val first = HomeRecommendationPageBuilder.build(args, HomeRecommendationPoolMode.FULL, resolver) + val second = HomeRecommendationPageBuilder.build(args, HomeRecommendationPoolMode.FULL, resolver) + val firstCursor = HomeRecommendationCursorCodec.decode(first.nextCursor) + + assertNotEquals(first.items, second.items) + assertNotEquals(0L, firstCursor?.rotationSeed) + + val seededArgs = args.copy(cursor = HomeRecommendationCursor(rotationSeed = 11L)) + val seededFirst = HomeRecommendationPageBuilder.build(seededArgs, HomeRecommendationPoolMode.FULL, resolver) + val seededSecond = HomeRecommendationPageBuilder.build(seededArgs, HomeRecommendationPoolMode.FULL, resolver) + assertEquals(seededFirst.items, seededSecond.items) + } + + private fun pool(): HomeRecommendationPool = HomeRecommendationPool( + subscriptions = (1..50).map { index -> HomeRecommendationItemFixtures.video("s$index", "s$index") }, + discovery = (1..50).map { index -> HomeRecommendationItemFixtures.video("d$index", "d$index") }, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt diff --git a/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt b/server-services/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt index 298da50b..db07b3f2 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt @@ -97,6 +97,7 @@ class ManifestServiceTest { val result = service.dashManifest("https://youtube.com/watch?v=test") assertTrue(result is ExtractionResult.Failure) + assertEquals("no_playable_streams", (result as ExtractionResult.Failure).code) } @Test diff --git a/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt b/server-services/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt new file mode 100644 index 00000000..4c1a934c --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt @@ -0,0 +1,75 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.OkHttpProxyService +import kotlinx.coroutines.test.runTest +import okhttp3.Dns +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class OkHttpProxyServiceTest { + @Test + fun `live HLS segment URL containing m3u8 path preserves binary bytes`() = runTest { + val mediaBytes = byteArrayOf( + 0x47, 0x40, 0x00, 0x30, 0xff.toByte(), 0x00, 0xff.toByte(), 0x10, + 0x80.toByte(), 0x7f, 0x00, 0x01, + ) + val service = OkHttpProxyService(clientFor(mediaBytes)) + val result = service.pipe( + "https://rr8---sn-n4g-jqbe6.googlevideo.com/videoplayback/playlist/index.m3u8/sq/42/file/seg.ts", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + val response = (result as ExtractionResult.Success).data + try { + assertEquals(200, response.status) + assertArrayEquals(mediaBytes, response.stream.readBytes()) + } finally { + response.close() + } + } + + @Test + fun `m3u8 manifest path is still rewritten when upstream MIME is generic`() = runTest { + val manifest = "#EXTM3U\n#EXTINF:5.0,\nhttps://rr8---sn-n4g-jqbe6.googlevideo.com/segment.ts\n" + val service = OkHttpProxyService(clientFor(manifest.toByteArray())) + val result = service.pipe( + "https://manifest.googlevideo.com/api/manifest/hls_variant/playlist/index.m3u8", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + val response = (result as ExtractionResult.Success).data + try { + assertTrue(response.stream.readBytes().decodeToString().contains("/proxy?url=")) + } finally { + response.close() + } + } + + private fun clientFor(body: ByteArray): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(Interceptor { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .header("Content-Type", "application/octet-stream") + .body(body.toResponseBody("application/octet-stream".toMediaType())) + .build() + }) + .build() +} diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt similarity index 83% rename from src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt index 3d85a071..a4d49834 100644 --- a/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt @@ -40,13 +40,36 @@ class PlaylistServiceTest { } @Test - fun `getAll returns playlist metadata without videos populated`() = runBlocking { + fun `getAll returns playlist count and first video thumbnail`() = runBlocking { val playlist = service.create(TEST_USER_ID, PlaylistItem(name = "Test")) - service.addVideo(TEST_USER_ID, playlist.id, PlaylistVideoItem(url = "https://yt.com", title = "T", thumbnail = "", duration = 100L, channelName = "C", channelUrl = "https://c", channelAvatar = "avatar", viewCount = 123L)) + service.addVideo( + TEST_USER_ID, + playlist.id, + PlaylistVideoItem( + url = "https://yt.com/1", + title = "First", + thumbnail = "first-thumb", + duration = 100L, + channelName = "C", + channelUrl = "https://c", + channelAvatar = "avatar", + viewCount = 123L, + ), + ) + service.addVideo( + TEST_USER_ID, + playlist.id, + PlaylistVideoItem( + url = "https://yt.com/2", + title = "Second", + thumbnail = "second-thumb", + duration = 200L, + ), + ) val all = service.getAll(TEST_USER_ID) assertEquals(1, all.size) - assertEquals(1, all[0].videoCount) - assertTrue(all[0].videos.isEmpty()) + assertEquals(2, all[0].videoCount) + assertEquals("first-thumb", all[0].videos.single().thumbnail) } @Test diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt similarity index 81% rename from src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt index 3eec1114..60210b1a 100644 --- a/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt @@ -17,7 +17,7 @@ class SignedHlsManifestTokenServiceTest { assertEquals("user-1", payload.userId) assertEquals("https://youtube.com/watch?v=test", payload.videoUrl) assertEquals("fingerprint", payload.fingerprint) - assertEquals(901_000L, payload.expiresAt) + assertEquals(1_000L + SignedHlsManifestTokenService.TTL_SECONDS * 1_000L, payload.expiresAt) } @Test @@ -28,11 +28,13 @@ class SignedHlsManifestTokenServiceTest { } @Test - fun `token expires`() { + fun `token remains valid for a long live session and then expires`() { var now = 1_000L val service = SignedHlsManifestTokenService("secret", nowMillis = { now }) val token = service.createToken("user-1", "https://youtube.com/watch?v=test", "fingerprint") - now = 901_000L + now = 1_801_000L + assertTrue(service.verify(token) is SignedHlsManifestTokenResult.Valid) + now = 1_000L + SignedHlsManifestTokenService.TTL_SECONDS * 1_000L assertEquals(SignedHlsManifestTokenResult.Expired, service.verify(token)) } } diff --git a/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt similarity index 84% rename from src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt index 72ebc0aa..e72445c5 100644 --- a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt @@ -18,6 +18,16 @@ class StreamCacheTtlResolverTest { assertEquals(300L, response("https://example.com/video.mp4", dislikeCount = -1L).streamCacheTtlSeconds(nowEpochSeconds = 1_000L)) } + @Test + fun `live content urls are not cached across clients`() { + val response = response( + "https://example.com/video.mp4", + isLive = true, + hlsUrl = "/streams/hls-manifest?token=live", + ) + assertEquals(0L, response.streamCacheTtlSeconds(nowEpochSeconds = 1_000L)) + } + @Test fun `bilibili stream ttl follows signed deadline`() { val url = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=10000&upsig=x" @@ -42,7 +52,12 @@ class StreamCacheTtlResolverTest { assertEquals(1_700L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) } - private fun response(url: String, dislikeCount: Long = 0L): StreamResponse = StreamResponse( + private fun response( + url: String, + dislikeCount: Long = 0L, + isLive: Boolean = false, + hlsUrl: String = "", + ): StreamResponse = StreamResponse( id = "id", title = "title", uploaderName = "uploader", @@ -67,12 +82,14 @@ class StreamCacheTtlResolverTest { requiresMembership = false, startPosition = 0L, streamSegments = emptyList(), - hlsUrl = "", + hlsUrl = hlsUrl, dashMpdUrl = "", videoStreams = emptyList(), audioStreams = emptyList(), originalAudioTrackId = null, preferredDefaultAudioTrackId = null, + isLive = isLive, + isLiveContent = isLive, videoOnlyStreams = listOf(video(url)), subtitles = emptyList(), previewFrames = emptyList(), diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt similarity index 60% rename from src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index 25db135c..1b3d44d5 100644 --- a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -8,9 +8,14 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException +import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException +import org.schabi.newpipe.extractor.exceptions.LiveNotStartException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException @@ -46,14 +51,49 @@ class StreamExtractionErrorMapperTest { @Test fun `maps upcoming premieres to a stable availability code`() { val result = StreamExtractionErrorMapper.map(VideoNotReleaseException("Premieres in 200 days")) - assertEquals(ExtractionResult.Failure("Premieres in 200 days", "scheduled_premiere"), result) + assertEquals( + ExtractionResult.Failure( + "Premieres in 200 days", + "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, + ), + result, + ) + } + + @Test + fun `maps upcoming live events to a stable availability code`() { + val message = "This live event will begin in 20 minutes." + val result = StreamExtractionErrorMapper.map(LiveNotStartException(message)) + + assertEquals( + ExtractionResult.Failure(message, "live_not_started", ExtractionFailureKind.LiveEventNotStarted), + result, + ) + } + + @Test + fun `maps generic content unavailability to a stable kind`() { + val message = "This content is not available" + val result = StreamExtractionErrorMapper.map(ContentNotAvailableException(message)) + + assertEquals( + ExtractionResult.Failure(message, "content_unavailable", ExtractionFailureKind.ContentUnavailable), + result, + ) } @Test fun `does not rewrite youtube timeout message to members-only fallback`() { val timeout = IllegalStateException("Error occurs when fetching the page. Try increase the loading timeout in Settings.") val mapped = StreamExtractionErrorMapper.map(timeout, sourceUrl = "https://www.youtube.com/watch?v=test") - assertEquals(ExtractionResult.Failure("Error occurs when fetching the page. Try increase the loading timeout in Settings."), mapped) + assertEquals( + ExtractionResult.Failure( + "Error occurs when fetching the page. Try increase the loading timeout in Settings.", + StreamExtractionErrorMapper.UPSTREAM_FAILURE_CODE, + ), + mapped, + ) } @Test @@ -68,7 +108,27 @@ class StreamExtractionErrorMapperTest { @Test fun `maps content restrictions to bad request with extractor message`() { val result = StreamExtractionErrorMapper.map(PrivateContentException("private video")) - assertEquals(ExtractionResult.BadRequest("private video"), result) + assertEquals(ExtractionResult.BadRequest("private video", "private_content"), result) + } + + @Test + fun `maps geographic restrictions to a stable access code`() { + val result = StreamExtractionErrorMapper.map(GeographicRestrictionException("Only available in Japan")) + assertEquals(ExtractionResult.BadRequest("Only available in Japan", "geographic_restriction"), result) + } + + @Test + fun `maps provider bot blocks to a vpn-aware failure`() { + val antiBot = StreamExtractionErrorMapper.map(AntiBotException("YouTube requested CAPTCHA verification")) + val captcha = StreamExtractionErrorMapper.map(ReCaptchaException("reCaptcha requested", "https://youtube.com")) + + val expected = ExtractionResult.Failure( + StreamExtractionErrorMapper.PROVIDER_ACCESS_BLOCKED_FALLBACK, + "provider_access_blocked", + ExtractionFailureKind.ProviderAccessBlocked, + ) + assertEquals(expected, antiBot) + assertEquals(expected, captcha) } @Test @@ -84,8 +144,10 @@ class StreamExtractionErrorMapperTest { fun `maps unknown exceptions to failure`() { val result = StreamExtractionErrorMapper.map(IllegalStateException("boom")) assertTrue(result is ExtractionResult.Failure) - assertEquals("boom", (result as ExtractionResult.Failure).message) - assertEquals(ExtractionFailureKind.Unknown, result.kind) + val failure = result as ExtractionResult.Failure + assertEquals("boom", failure.message) + assertEquals(ExtractionFailureKind.Unknown, failure.kind) + assertEquals(StreamExtractionErrorMapper.UPSTREAM_FAILURE_CODE, failure.code) } @Test @@ -100,6 +162,8 @@ class StreamExtractionErrorMapperTest { val result = StreamExtractionErrorMapper.map(error) assertTrue(result is ExtractionResult.Failure) - assertEquals(ExtractionFailureKind.YoutubeSessionRejected, (result as ExtractionResult.Failure).kind) + val failure = result as ExtractionResult.Failure + assertEquals(ExtractionFailureKind.YoutubeSessionRejected, failure.kind) + assertEquals("youtube_session_rejected", failure.code) } } diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt new file mode 100644 index 00000000..b8228a67 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt @@ -0,0 +1,97 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SubscriptionAvatarWarmupService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionAvatarWarmupServiceTest { + private val channelUrl = "https://www.youtube.com/channel/UCWarmup" + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `warmup persists a resolved channel avatar`() = runBlocking { + insertSubscription() + val service = SubscriptionAvatarWarmupService(channelService("https://avatar.test/warmup.jpg")) + + service.warm(TEST_USER_ID, listOf(channelUrl)) + + assertEquals("https://avatar.test/warmup.jpg", storedAvatar()) + } + + @Test + fun `warmup leaves a blank avatar when channel resolution has no avatar`() = runBlocking { + insertSubscription() + val service = SubscriptionAvatarWarmupService(channelService("")) + + service.warm(TEST_USER_ID, listOf(channelUrl)) + + assertEquals("", storedAvatar()) + } + + private suspend fun insertSubscription(): Unit { + SubscriptionsService().add( + TEST_USER_ID, + SubscriptionItem( + channelUrl = channelUrl, + name = "OHIOBOSS SATOYU", + avatarUrl = "", + ), + ) + } + + private suspend fun storedAvatar(): String = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { + (SubscriptionsTable.userId eq TEST_USER_ID) and + (SubscriptionsTable.channelUrl eq channelUrl) + } + .single()[SubscriptionsTable.avatarUrl] + } + + private fun channelService(avatarUrl: String): ChannelService = object : ChannelService { + override suspend fun getChannel( + url: String, + nextpage: String?, + sort: String?, + ): ExtractionResult = ExtractionResult.Success( + ChannelResponse( + name = "OHIOBOSS SATOYU", + description = "", + avatarUrl = avatarUrl, + bannerUrl = "", + subscriberCount = 0, + isVerified = false, + videos = emptyList(), + nextpage = null, + ), + ) + + override suspend fun getPlaylists( + url: String, + nextpage: String?, + ): ExtractionResult = + ExtractionResult.BadRequest("Not used") + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt new file mode 100644 index 00000000..ee8f52e8 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt @@ -0,0 +1,155 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import dev.typetype.server.services.SubscriptionMutationLock +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class SubscriptionMembershipConcurrencyTest { + private val subscriptions = SubscriptionsService() + private val pages = SubscriptionMembershipPageService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `page reads and additions wait for the same account mutation`() = runTest { + val lockHeld = CompletableDeferred() + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + SubscriptionMutationLock.acquire(TEST_USER_ID) + val pid = TransactionManager.current().exec("SELECT pg_backend_pid()") { result -> + result.next() + result.getInt(1) + } + lockHeld.complete(requireNotNull(pid)) + check(releaseLock.await(10, TimeUnit.SECONDS)) + } + } + try { + val holderPid = lockHeld.await() + val addition = async(Dispatchers.IO) { subscriptions.add(TEST_USER_ID, subscription(0)) } + val read = async(Dispatchers.IO) { pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()) } + val otherAccount = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { subscriptions.add("other-account", subscription(0)) } + } + assertEquals(subscription(0).channelUrl, otherAccount?.channelUrl) + val bothWaited = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { + while (!addition.isCompleted && !read.isCompleted && waitingTransactions(holderPid) < 2) { + delay(10) + } + waitingTransactions(holderPid) >= 2 + } ?: false + } + assertTrue(bothWaited, "page reads and additions must wait for the same account-scoped lock") + releaseLock.countDown() + holder.await() + addition.await() + val page = read.await() + assertEquals(page.total, page.totalSubscriptions) + assertEquals(page.total, page.ungroupedCount) + assertEquals(page.total.toInt(), page.items.size) + assertEquals(1L, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).total) + } finally { + releaseLock.countDown() + } + } + + @Test + fun `same-account page reads share the lock while additions still wait`() = runTest { + val lockHeld = CompletableDeferred() + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(TEST_USER_ID) + val pid = TransactionManager.current().exec("SELECT pg_backend_pid()") { result -> + result.next() + result.getInt(1) + } + lockHeld.complete(requireNotNull(pid)) + check(releaseLock.await(10, TimeUnit.SECONDS)) + } + } + try { + val holderPid = lockHeld.await() + val pageRead = async(Dispatchers.IO) { pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()) } + val page = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { pageRead.await() } + } + assertEquals(0L, page?.total, "a concurrent page read must not wait for another reader") + + val addition = async(Dispatchers.IO) { subscriptions.add(TEST_USER_ID, subscription(0)) } + val additionWaited = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { + while (!addition.isCompleted && waitingTransactions(holderPid) < 1) delay(10) + waitingTransactions(holderPid) >= 1 + } ?: false + } + assertTrue(additionWaited, "an addition must wait for active page readers") + releaseLock.countDown() + holder.await() + addition.await() + } finally { + releaseLock.countDown() + } + } + + @Test + fun `page rows and counts remain consistent during concurrent additions`() = runTest { + val start = CompletableDeferred() + val reader = async(Dispatchers.IO) { + start.await() + repeat(100) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 20)) + assertEquals(page.total, page.totalSubscriptions) + assertEquals(page.total, page.ungroupedCount) + assertEquals(minOf(page.total, 20L).toInt(), page.items.size) + } + } + val writer = async(Dispatchers.IO) { + start.await() + repeat(100) { subscriptions.add(TEST_USER_ID, subscription(it)) } + } + start.complete(Unit) + reader.await() + writer.await() + assertEquals(100L, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).total) + } + + private suspend fun waitingTransactions(holderPid: Int): Int = DatabaseFactory.query { + TransactionManager.current().exec( + "SELECT count(*) FROM pg_stat_activity WHERE $holderPid = ANY(pg_blocking_pids(pid))", + ) { result -> + result.next() + result.getInt(1) + } ?: 0 + } + + private fun subscription(index: Int): SubscriptionItem = + SubscriptionItem("https://example.com/channel/$index", "Channel $index", "avatar") +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt new file mode 100644 index 00000000..eab35b5c --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt @@ -0,0 +1,160 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionMembershipPageServiceTest { + private val pages = SubscriptionMembershipPageService() + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `large libraries return only the requested stable page with account totals`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.batchInsert(0..1000, shouldReturnGeneratedValues = false) { index -> + this[SubscriptionsTable.userId] = TEST_USER_ID + this[SubscriptionsTable.channelUrl] = url(index) + this[SubscriptionsTable.name] = "Channel ${index.toString().padStart(4, '0')}" + this[SubscriptionsTable.avatarUrl] = "avatar" + this[SubscriptionsTable.subscribedAt] = 1L + } + } + subscriptions.add("foreign", SubscriptionItem(url(0), "Foreign", "avatar")) + val first = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 7)) + val next = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(page = 1, limit = 7)) + assertEquals(1001L, first.total) + assertEquals(1001L, first.totalSubscriptions) + assertEquals(1001L, first.ungroupedCount) + assertEquals((0..6).map(::url), first.items.map { it.channelUrl }) + assertEquals((7..13).map(::url), next.items.map { it.channelUrl }) + assertTrue(pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(page = 999, limit = 7)).items.isEmpty()) + assertEquals(first, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 7))) + } + + @Test + fun `membership filters intersect literal search while returning all groups on each row`() = runTest { + for (index in 0..3) subscriptions.add(TEST_USER_ID, SubscriptionItem(url(index), "SAME $index", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(4), "100%_\\done", "avatar")) + val a = (groups.create(TEST_USER_ID, "A") as SubscriptionGroupWriteResult.Success).group + val b = (groups.create(TEST_USER_ID, "B") as SubscriptionGroupWriteResult.Success).group + groups.addSubscriptions(TEST_USER_ID, a.id, listOf(url(0), url(1))) + groups.addSubscriptions(TEST_USER_ID, b.id, listOf(url(1), url(2))) + val foreign = (groups.create("foreign", "Private") as SubscriptionGroupWriteResult.Success).group + subscriptions.add("foreign", SubscriptionItem(url(3), "Foreign channel", "avatar")) + groups.addSubscription("foreign", foreign.id, url(3)) + val inGroup = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(groupId = a.id, search = "same")) + assertEquals(listOf(url(0), url(1)), inGroup.items.map { it.channelUrl }) + assertEquals(listOf(a.id, b.id).sorted(), inGroup.items[1].groupIds) + assertEquals(2L, inGroup.total) + assertEquals(5L, inGroup.totalSubscriptions) + assertEquals(2L, inGroup.ungroupedCount) + assertEquals(listOf(url(2), url(3)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(groupId = a.id, excluded = true, search = " same ")).items.map { it.channelUrl }) + assertEquals(listOf(url(3)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(ungrouped = true, search = "SAME")).items.map { it.channelUrl }) + assertEquals(listOf(url(4)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = "%_\\")).items.map { it.channelUrl }) + assertEquals(listOf(url(2)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = url(2))).items.map { it.channelUrl }) + } + + @Test + fun `lookup refreshes off-page selections and omits deleted or foreign subscriptions`() = runTest { + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(1), "One", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(2), "Two", "avatar")) + subscriptions.add("foreign", SubscriptionItem(url(3), "Foreign", "avatar")) + val group = (groups.create(TEST_USER_ID, "A") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, url(2)) + subscriptions.delete(TEST_USER_ID, url(1)) + val selected = pages.lookup(TEST_USER_ID, listOf(url(1), url(2), url(2), url(3))) + assertEquals(1, selected.size) + assertEquals(url(2), selected.single().channelUrl) + assertEquals(listOf(group.id), selected.single().groupIds) + assertEquals(1, groups.getAll(TEST_USER_ID).single().channelCount) + } + + @Test + fun `page and lookup resolve avatars without persisting repairs`() = runTest { + val item = subscriptions.add(TEST_USER_ID, SubscriptionItem(url(1), "One", "")) + val avatarUrl = "https://example.com/avatar.jpg" + DatabaseFactory.query { + HistoryTable.insert { + it[HistoryTable.id] = "membership-avatar" + it[HistoryTable.userId] = TEST_USER_ID + it[HistoryTable.url] = "https://www.youtube.com/watch?v=avatar-source" + it[HistoryTable.title] = "Video" + it[HistoryTable.thumbnail] = "thumbnail" + it[HistoryTable.channelName] = "One" + it[HistoryTable.channelUrl] = item.channelUrl + it[HistoryTable.channelAvatar] = avatarUrl + it[HistoryTable.duration] = 1L + it[HistoryTable.progress] = 0L + it[HistoryTable.watchedAt] = 1L + } + } + + assertEquals(avatarUrl, pages.lookup(TEST_USER_ID, listOf(item.channelUrl)).single().avatarUrl) + assertEquals(avatarUrl, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).items.single().avatarUrl) + val storedAvatar = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq TEST_USER_ID } + .single()[SubscriptionsTable.avatarUrl] + } + assertEquals("", storedAvatar) + } + + @Test + fun `Unicode names match exact search without treating wildcards as patterns`() = runTest { + val names = listOf("École", "İstanbul", "ΟΣ", "İzmir_%\\'News") + for ((index, name) in names.withIndex()) { + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(index), name, "avatar")) + } + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(99), "İzmirAXNews", "avatar")) + for ((index, name) in names.withIndex()) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(search = name)) + assertEquals(1L, page.total, name) + assertEquals(listOf(url(index)), page.items.map { it.channelUrl }, name) + } + assertEquals("École", pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = "ÉCOLE")).items.single().name) + } + + @Test + fun `Unicode channel URLs use the same search normalization`() = runTest { + val item = subscriptions.add(TEST_USER_ID, + SubscriptionItem("https://example.com/İstanbul", "Ordinary channel", "avatar")) + for (search in listOf("İstanbul", "istanbul")) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(search = search)) + assertEquals(1L, page.total, search) + assertEquals(item.channelUrl, page.items.single().channelUrl) + } + } + + private fun url(index: Int): String = "https://www.youtube.com/channel/test${index.toString().padStart(4, '0')}" +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt new file mode 100644 index 00000000..55cf501a --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt @@ -0,0 +1,78 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import kotlin.time.Duration.Companion.seconds +import kotlin.time.measureTimedValue + +class SubscriptionMembershipScaleTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `pages stay responsive with 25000 channels and 112500 memberships`() = runTest { + seedLibrary() + val pages = SubscriptionMembershipPageService() + val filters = listOf( + SubscriptionMembershipFilter(limit = 7) to 25_000L, + SubscriptionMembershipFilter(limit = 7, ungrouped = true) to 2_500L, + SubscriptionMembershipFilter(limit = 7, groupId = "scale-1", excluded = true) to 2_500L, + SubscriptionMembershipFilter(limit = 7, groupId = "scale-1") to 22_500L, + SubscriptionMembershipFilter(limit = 7, search = "Channel 1") to 10_000L, + ) + for ((filter, total) in filters) { + val (page, elapsed) = measureTimedValue { pages.getPage(TEST_USER_ID, filter) } + assertTrue(elapsed < 3.seconds, "$filter page read took $elapsed") + assertEquals(total, page.total) + assertEquals(25_000L, page.totalSubscriptions) + assertEquals(2_500L, page.ungroupedCount) + assertEquals(7, page.items.size) + assertEquals(7, page.items.map { it.channelUrl }.distinct().size) + assertTrue(page.items.all { channel -> + when { + filter.ungrouped || filter.excluded -> channel.groupIds.isEmpty() + filter.groupId != null -> channel.groupIds.size == 5 + else -> true + } + }) + } + } + + private suspend fun seedLibrary(): Unit = DatabaseFactory.query { + // This membership set exceeds PostgreSQL's default NOT IN hash budget. + val transaction = TransactionManager.current() + transaction.exec(""" + INSERT INTO subscriptions (user_id, channel_url, name, avatar_url, subscribed_at) + SELECT '$TEST_USER_ID', 'https://www.youtube.com/channel/UC' || lpad(n::text, 22, '0'), + 'Channel ' || lpad(n::text, 5, '0'), 'avatar', 1 + FROM generate_series(1, 25000) n + """.trimIndent()) + transaction.exec(""" + INSERT INTO subscription_groups (id, user_id, name, normalized_name, created_at, updated_at) + SELECT 'scale-' || n, '$TEST_USER_ID', 'Group ' || n, 'group ' || n, 1, 1 + FROM generate_series(1, 5) n + """.trimIndent()) + transaction.exec(""" + INSERT INTO subscription_group_memberships (group_id, user_id, channel_url, added_at) + SELECT g.id, s.user_id, s.channel_url, 1 + FROM subscriptions s JOIN subscription_groups g ON g.user_id = s.user_id + WHERE s.user_id = '$TEST_USER_ID' AND right(s.channel_url, 1) <> '0' + """.trimIndent()) + transaction.exec("ANALYZE subscriptions") + transaction.exec("ANALYZE subscription_group_memberships") + } +} diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt similarity index 95% rename from src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt index fc3d0252..19df7677 100644 --- a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt @@ -28,6 +28,10 @@ class UrlValidatorTest { assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://i.ytimg.com/image.jpg").provider) assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://yt3.googleusercontent.com/avatar").provider) assertEquals(ProxyProvider.BILIBILI, requireProxyTarget("https://i2.hdslb.com/image.jpg").provider) + assertEquals( + ProxyProvider.BILIBILI, + requireProxyTarget("https://api.live.bilibili.com/xlive/play-gateway/master/url?cid=1").provider, + ) assertEquals( ProxyProvider.BILIBILI, requireProxyTarget("https://upos-hz-mirrorakam.akamaized.net/video.m4s").provider, diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt similarity index 66% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index 5b960a53..5c3fcb04 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.services.YoutubeTakeoutParserService +import dev.typetype.server.services.YoutubeTakeoutDateParser import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -70,6 +71,77 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse detects spanish takeout dates and playlist names`() { + val zip = Files.createTempFile("yt-takeout-spanish-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/suscripciones/suscripciones.csv")) + out.write("ID de canal,URL del canal,Título del canal\nUC123456789012,https://www.youtube.com/channel/UC123456789012,Canal\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/listas de reproducción.csv")) + out.write("ID de la lista de reproducción,Título de la lista de reproducción\nPL123456789,Importada\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/Videos de Importada.csv")) + out.write("ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000001,2026-01-02T00:00:00Z\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/Ver más tarde.csv")) + out.write("ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000002,2026-01-01T00:00:00Z\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/Mon actividad/YouTube/watch-history.html")) + out.write("Has visto Watched
16 sept 2026, 18:02:08 CEST
".toByteArray()) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertEquals(1, parsed.subscriptions.size) + assertEquals(1, parsed.playlists.size) + assertEquals(1, parsed.playlistItems["Importada"]?.size) + assertEquals(1, parsed.watchLater.size) + assertEquals(1, parsed.history.size) + assertEquals(1_789_574_528_000L, parsed.history.single().watchedAt) + assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 septiembre 2026, 18:02:08 CEST")) + assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 de septiembre de 2026, 18:02:08 CEST")) + assertEquals(1_789_250_639_000L, YoutubeTakeoutDateParser.parseEpochMillis("13 sept 2026, 0:03:59 CEST")) + Files.deleteIfExists(zip) + } + + @Test + fun `parse skips history rows with an unknown activity date`() { + val zip = Files.createTempFile("yt-takeout-invalid-date-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/My Activity/YouTube/watch-history.html")) + out.write("You watched Unknown date
16 Foo 2026, 12:00:00 CET
".toByteArray()) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertTrue(parsed.history.isEmpty()) + assertTrue(parsed.history.none { it.watchedAt == 0L }) + Files.deleteIfExists(zip) + } + + @Test + fun `parse accepts a localized history file without an english activity marker`() { + val zip = Files.createTempFile("yt-takeout-localized-history-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube/история просмотров/история просмотров.html")) + out.write( + ("Localized title
" + + "Channel
" + + "16. September 2026, 18:02:08 CEST
").toByteArray(), + ) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertEquals(listOf("localized1"), parsed.history.map { it.url.substringAfter("v=") }) + assertTrue(parsed.history.none { it.watchedAt == 0L }) + Files.deleteIfExists(zip) + } + @Test fun `parse preserves takeout playlist order and added dates`() { val zip = Files.createTempFile("yt-takeout-playlist-order-", ".zip") @@ -178,6 +250,24 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse preserves live video history urls`() { + val zip = Files.createTempFile("yt-takeout-live-history-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.writeEntry( + "Takeout/My Activity/YouTube/watch-history.html", + "You watched Live stream
" + + "1 Jan 2026, 12:00:00 CET
", + ) + } + + val history = YoutubeTakeoutParserService().parse(zip).history + + assertEquals("https://www.youtube.com/live/live000001", history.single().url) + assertEquals("https://i.ytimg.com/vi/live000001/hqdefault.jpg", history.single().thumbnail) + Files.deleteIfExists(zip) + } + private fun createZip(): Path { val zip = Files.createTempFile("yt-takeout-parser-", ".zip") ZipOutputStream(Files.newOutputStream(zip)).use { out -> diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt new file mode 100644 index 00000000..2f2a2cee --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class BiliBiliQrLoginServiceTest { + @Test + fun code_86101_is_waiting() { + assertTrue(classifyBiliBiliQrCode(86101) is BiliBiliQrPollResult.Waiting) + } + + @Test + fun code_86090_is_scanned() { + assertTrue(classifyBiliBiliQrCode(86090) is BiliBiliQrPollResult.Scanned) + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt new file mode 100644 index 00000000..12cdb5a6 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class BiliBiliSessionCryptoTest { + private val crypto = BiliBiliSessionCrypto.fromSecret("test-secret-key-at-least-32-chars!!") + + @Test + fun `encrypt and decrypt round-trips`() { + val cookie = "SESSDATA=abc123; bili_jct=csrf456; buvid3=device789" + val encrypted = crypto.encrypt(cookie) + assertNotEquals(cookie, encrypted) + assertEquals(cookie, crypto.decrypt(encrypted)) + } + + @Test + fun `encryption produces different ciphertexts`() { + val cookie = "SESSDATA=same" + assertNotEquals(crypto.encrypt(cookie), crypto.encrypt(cookie)) + } + + @Test + fun `decrypt rejects tampered payload`() { + val encrypted = crypto.encrypt("SESSDATA=test") + val tampered = encrypted.dropLast(4) + "AAAA" + assertThrows { crypto.decrypt(tampered) } + } + + @Test + fun `rejects short secret`() { + assertThrows { BiliBiliSessionCrypto.fromSecret("short") } + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt new file mode 100644 index 00000000..34d37602 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt @@ -0,0 +1,28 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.ServiceList + +class BiliBiliSessionScopeTest { + @Test + fun `withCredentials sets and clears tokens`() = runBlocking { + val bilibili = ServiceList.BiliBili + BiliBiliSessionScope.withCredentials("test-user", "SESSDATA=test; bili_jct=csrf") { + assertEquals("SESSDATA=test; bili_jct=csrf", bilibili.tokens) + } + assertEquals("", bilibili.tokens) + } + + @Test + fun `withoutCredentials clears tokens`() = runBlocking { + val bilibili = ServiceList.BiliBili + bilibili.setTokens("leftover") + BiliBiliSessionScope.withoutCredentials { + assertEquals("", bilibili.tokens) + } + assertEquals("", bilibili.tokens) + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt new file mode 100644 index 00000000..d25627d2 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.PlaylistResultItem +import dev.typetype.server.models.PublicPlaylistItem +import dev.typetype.server.models.PublicPlaylistResponse +import dev.typetype.server.testVideoItem +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class BlockedContentFiltersTest { + private val blocked = BlockedContentProfile( + videos = listOf(BlockedItem("https://youtube.com/watch?v=blocked", blockedAt = 1)), + channels = listOf(BlockedItem("https://youtube.com/@blocked", "Blocked Channel", null, 1)), + keywords = listOf(BlockedKeywordItem("spoiler", 1)), + ) + + @Test + fun `public playlist hides blocked videos channels and keywords`() { + val response = PublicPlaylistResponse( + playlist = PublicPlaylistItem("playlist", "Playlist", "url", "", "Owner", 4, "normal"), + videos = listOf( + video("https://youtube.com/watch?v=blocked", "Visible"), + video("https://youtube.com/watch?v=channel", "Visible", "Blocked Channel"), + video("https://youtube.com/watch?v=keyword", "Major spoiler"), + video("https://youtube.com/watch?v=allowed", "Allowed"), + ), + nextpage = null, + ) + + assertEquals( + listOf("https://youtube.com/watch?v=allowed"), + response.filterBlocked(blocked).videos.map { it.url }, + ) + } + + @Test + fun `channel playlist list hides playlists from blocked uploaders`() { + val response = ChannelPlaylistsResponse( + playlists = listOf( + playlist("blocked", "Blocked Channel"), + playlist("allowed", "Allowed Channel"), + ), + nextpage = null, + ) + + assertEquals(listOf("allowed"), response.filterBlocked(blocked).playlists.map { it.id }) + } + + private fun video(url: String, title: String, uploaderName: String = "Allowed Channel") = + testVideoItem().copy( + id = url, + url = url, + title = title, + uploaderName = uploaderName, + uploaderUrl = "https://youtube.com/@${uploaderName.lowercase().replace(' ', '-')}", + ) + + private fun playlist(id: String, uploaderName: String) = PlaylistResultItem( + id = id, + title = id, + url = "https://youtube.com/playlist?list=$id", + thumbnailUrl = "", + uploaderName = uploaderName, + streamCount = 1, + playlistType = "normal", + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..1796abfe --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.runs +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.stream.StreamExtractor +import org.schabi.newpipe.extractor.stream.StreamType + +class PipePipeLiveHlsStreamServiceTest { + @Test + fun returnsLiveHlsWithoutEnumeratingFormats() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.LIVE_STREAM + every { extractor.hlsUrl } returns HLS_URL + every { extractor.requiresMembership() } returns false + every { extractor.id } returns VIDEO_ID + every { extractor.name } returns "Live title" + val service = PipePipeLiveHlsStreamService { extractor } + + val response = (service.getStreamInfo(YOUTUBE_URL) as ExtractionResult.Success).data + + assertEquals(HLS_URL, response.hlsUrl) + assertTrue(response.isLive) + assertTrue(response.hasLiveManifest) + assertTrue(response.videoStreams.isEmpty()) + assertTrue(response.audioStreams.isEmpty()) + verify(exactly = 1) { extractor.fetchPage() } + verify(exactly = 0) { extractor.audioStreams } + verify(exactly = 0) { extractor.videoStreams } + } + + @Test + fun rejectsVodBeforeReadingManifest() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.VIDEO_STREAM + val service = PipePipeLiveHlsStreamService { extractor } + + val result = service.getStreamInfo(YOUTUBE_URL) + + assertTrue(result is ExtractionResult.Failure) + assertEquals("live_stream_unavailable", (result as ExtractionResult.Failure).code) + verify(exactly = 0) { extractor.hlsUrl } + } + + @Test + fun doesNotExposeMembersOnlyManifest() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.LIVE_STREAM + every { extractor.hlsUrl } returns HLS_URL + every { extractor.requiresMembership() } returns true + val service = PipePipeLiveHlsStreamService { extractor } + + val result = service.getStreamInfo(YOUTUBE_URL) + + assertTrue(result is ExtractionResult.Failure) + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt similarity index 94% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt index b09e96ca..c04a60fa 100644 --- a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt @@ -122,6 +122,16 @@ class YouTubeSubtitleDeliveryServiceTest { assertEquals(0, fetches.get()) } + @Test + fun `upstream subtitle failure is isolated as unavailable`() = runTest { + val service = service( + resolver = { readyTrack() }, + fetcher = { _, _ -> error("ReCaptcha required") }, + ) + + assertEquals(YouTubeSubtitleContentResult.Unavailable, service.fetch(SELECTION)) + } + private fun service( resolver: suspend (YouTubeSubtitleSelection) -> YouTubeSubtitleResolution, fetcher: suspend (String, YouTubeSubtitleFormat) -> YouTubeSubtitleFetchResult, diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..28b34e83 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt @@ -0,0 +1,101 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class YoutubeDirectLiveHlsStreamServiceTest { + @Test + fun returnsLiveWithHlsWithoutCallingFallback() = runTest { + val liveHlsService = mockk() + val fallbackService = mockk() + val response = liveResponse() + val expected = ExtractionResult.Success(response) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + coVerify(exactly = 0) { fallbackService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun rejectsNonLiveResult() = runTest { + val liveHlsService = mockk() + val response = mockk() + every { response.isLive } returns false + every { response.hlsUrl } returns HLS_URL + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService).getStreamInfo(YOUTUBE_URL) + + assertEquals("live_stream_unavailable", (result as ExtractionResult.Failure).code) + } + + @Test + fun usesFallbackWhenDirectLiveExtractionFails() = runTest { + val liveHlsService = mockk() + val fallbackService = mockk() + val directFailure = ExtractionResult.Failure("provider blocked", "provider_access_blocked") + val expected = ExtractionResult.Success(liveResponse()) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns directFailure + coEvery { fallbackService.getStreamInfo(YOUTUBE_URL) } returns expected + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + } + + @Test + fun startsFallbackAfterHedgeDelayAndCancelsSlowDirectAttempt() = runTest { + val liveHlsService = mockk() + val fallbackService = mockk() + val directCancelled = CompletableDeferred() + val fallbackStartedAt = CompletableDeferred() + val scheduler = testScheduler + val expected = ExtractionResult.Success(liveResponse()) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } coAnswers { + try { + delay(10_000) + ExtractionResult.Failure("late direct failure", "provider_access_blocked") + } finally { + directCancelled.complete(Unit) + } + } + coEvery { fallbackService.getStreamInfo(YOUTUBE_URL) } coAnswers { + fallbackStartedAt.complete(scheduler.currentTime) + delay(150) + expected + } + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + assertEquals(100L, fallbackStartedAt.await()) + assertTrue(directCancelled.isCompleted) + } + + private fun liveResponse(): StreamResponse = mockk().also { + every { it.isLive } returns true + every { it.hlsUrl } returns HLS_URL + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..32f5125b --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeLiveHlsStreamServiceTest { + @Test + fun retriesActiveLiveWithLiveHlsClientWhenMwebHasNoManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "") + val liveHls = streamResponse(isLive = true, manifestUrl = HLS_URL) + val expected = ExtractionResult.Success(liveHls) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(metadata) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun prefersLiveHlsClientWhenMwebAlreadyHasManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "https://mweb.example/live.m3u8") + val liveHls = streamResponse(isLive = true, manifestUrl = HLS_URL) + val expected = ExtractionResult.Success(liveHls) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(metadata) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun fallsBackToMwebManifestWhenLiveHlsClientHasNoManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "https://mweb.example/live.m3u8") + val liveHls = streamResponse(isLive = true, manifestUrl = "") + val expected = ExtractionResult.Success(metadata) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns expected + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(liveHls) + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun fallsBackToMwebAfterFastProbeFindsVod() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val vod = streamResponse(isLive = false, manifestUrl = "") + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure(YOUTUBE_URL) + val expected = ExtractionResult.Success(vod) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + private fun streamResponse(isLive: Boolean, manifestUrl: String): StreamResponse { + val response = mockk() + every { response.isLive } returns isLive + every { response.hlsUrl } returns manifestUrl + every { response.requiresMembership } returns false + return response + } + + private companion object { + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=GlzleRbo5E0" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt new file mode 100644 index 00000000..87a7dfa4 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt @@ -0,0 +1,33 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class YoutubeSessionSabrStreamServiceTest { + @Test + fun `returns null for live metadata so the public HLS path is used`() = runTest { + val metadataService = mockk() + val infoService = mockk() + val live = mockk() + every { live.isLive } returns true + val expected = ExtractionResult.Success(live) + coEvery { metadataService.getStreamInfo(USER_ID, YOUTUBE_URL) } returns expected + val service = YoutubeSessionSabrStreamService(metadataService, infoService) + + assertEquals(null, service.getStreamInfo(USER_ID, YOUTUBE_URL)) + coVerify(exactly = 0) { infoService.fetch(USER_ID, VIDEO_ID) } + } + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/FakeChannelService.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/FakeChannelService.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeChannelService.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/FakeChannelService.kt diff --git a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt index 0d65cdde..6237f6dc 100644 --- a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.AdminBugReportDetailResponse import dev.typetype.server.models.BugApiErrorItem import dev.typetype.server.models.BugReportContextItem -internal object GitHubIssueServiceTestReports { +object GitHubIssueServiceTestReports { fun sampleReport(): AdminBugReportDetailResponse = AdminBugReportDetailResponse( id = "report-id", category = "player", diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt index 73902f88..601a2096 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.HomeRecommendationSessionIntent import dev.typetype.server.services.HomeRecommendationSourceTag import dev.typetype.server.services.HomeRecommendationTaggedVideo -internal object HomeRecommendationItemFixtures { +object HomeRecommendationItemFixtures { val context: HomeRecommendationSessionContext = HomeRecommendationSessionContext( intent = HomeRecommendationSessionIntent.AUTO, deviceClass = HomeRecommendationDeviceClass.UNKNOWN, diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt similarity index 96% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt index a44151fb..44501ded 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.models.VideoItem -internal object SubscriptionFeedTestFixtures { +object SubscriptionFeedTestFixtures { fun video( uploaded: Long, channel: String = "Ch", diff --git a/src/test/kotlin/dev/typetype/server/TestCommentFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/TestCommentFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestCommentFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/TestCommentFixtures.kt diff --git a/server-test-support/build.gradle.kts b/server-test-support/build.gradle.kts new file mode 100644 index 00000000..a7327d7c --- /dev/null +++ b/server-test-support/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + kotlin("jvm") +} + +dependencies { + api(project(":server-core")) + api(project(":server-db")) + api(project(":server-cache")) + api("org.testcontainers:testcontainers:2.0.5") + api("org.testcontainers:testcontainers-postgresql:2.0.5") + api("io.mockk:mockk:1.14.11") + api("org.junit.jupiter:junit-jupiter:6.1.3") + api("io.ktor:ktor-utils-jvm:3.5.2") + api("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + api("io.ktor:ktor-server-test-host-jvm:3.5.2") + api("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + api("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + api(project(":server-services")) + api(project(":server-playback")) + api(project(":server-token-gateway")) + api(project(":server-sabr")) +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt index 2ce9c2a1..0efabf12 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt @@ -1,5 +1,7 @@ package dev.typetype.server.services +import dev.typetype.server.sabr.SabrProtocolException +import dev.typetype.server.sabr.SabrRecoverableException import io.mockk.every import io.mockk.coEvery import io.mockk.coVerify diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt similarity index 84% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt index f4981740..3bb64d82 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt @@ -6,30 +6,30 @@ private const val DEFAULT_SABR_PROBE_VIDEO_ITAG = 137 private const val DEFAULT_SABR_PROBE_AUDIO_ITAG = 140 private const val DEFAULT_SABR_PROBE_TIMEOUT_MS = 60_000L -internal fun sabrProbeTokenServiceUrl(): String = +fun sabrProbeTokenServiceUrl(): String = envValue("SUBTITLE_SERVICE_URL") ?: "http://localhost:8081" -internal fun sabrProbeVideoId(): String = +fun sabrProbeVideoId(): String = envValue("SABR_PROBE_VIDEO") ?: envValues("SABR_PROBE_VIDEOS").firstOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ID -internal fun sabrProbeVideoIds(): List = +fun sabrProbeVideoIds(): List = envValues("SABR_PROBE_VIDEOS").ifEmpty { listOf(sabrProbeVideoId()) } -internal fun sabrProbePlayerTimeMs(): Long = +fun sabrProbePlayerTimeMs(): Long = envValue("SABR_PROBE_PLAYER_TIME_MS")?.toLongOrNull()?.takeIf { it >= 0L } ?: DEFAULT_SABR_PROBE_PLAYER_TIME_MS -internal fun sabrProbeTimeoutMs(): Long = +fun sabrProbeTimeoutMs(): Long = envValue("SABR_PROBE_FETCH_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } ?: envValue("SABR_PROBE_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } ?: DEFAULT_SABR_PROBE_TIMEOUT_MS -internal fun sabrProbeAudioItag(): Int = +fun sabrProbeAudioItag(): Int = envValue("SABR_PROBE_AUDIO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_AUDIO_ITAG -internal fun sabrProbeVideoItags(): List { +fun sabrProbeVideoItags(): List { val explicit = envIntValues("SABR_PROBE_VIDEO_ITAGS") val primary = envValue("SABR_PROBE_VIDEO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ITAG val base = explicit.ifEmpty { listOf(primary) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt similarity index 91% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt index 7c28b7f1..1150edaf 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { +fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { println( "$label format itag=${format.itag} audio=${format.isAudio} video=${format.isVideo} " + "size=${format.width}x${format.height} bitrate=${format.bitrate} mime=${format.mimeType} " + @@ -14,7 +14,7 @@ internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Uni ) } -internal fun printSabrProbeFetch( +fun printSabrProbeFetch( label: String, holder: SabrSessionHolder, request: SabrSegmentRequest, @@ -34,7 +34,7 @@ internal fun printSabrProbeFetch( result.error?.let { println("$label error ${it.javaClass.simpleName}: ${it.message}") } } -internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { +fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { val header = segment.header return "headerId=${header.headerId} videoId=${header.videoId} itag=${header.itag} " + "seq=${header.sequenceNumber} init=${header.isInitSegment} startMs=${header.startMs} " + @@ -42,7 +42,7 @@ internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { "contentLength=${header.contentLength} bytes=${segment.length} summary=${header.summarize()}" } -internal fun sabrProbeRequestSummary( +fun sabrProbeRequestSummary( holder: SabrSessionHolder, request: SabrSegmentRequest, ): String { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt similarity index 87% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt index 86f2cf76..8a9f6272 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt @@ -5,7 +5,14 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal suspend fun fetchSabrProbeSegment( +data class SabrProbeFetchResult( + val segment: SabrMediaSegment?, + val elapsedMs: Long, + val timedOut: Boolean, + val error: Throwable?, +) + +suspend fun fetchSabrProbeSegment( store: SabrSessionStore, holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts new file mode 100644 index 00000000..195077a4 --- /dev/null +++ b/server-token-gateway/build.gradle.kts @@ -0,0 +1,34 @@ +plugins { + kotlin("jvm") +} + +dependencies { + api(project(":server-sabr")) + api(project(":server-core")) + implementation(project(":server-db")) + implementation(project(":server-downloader")) + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.slf4j:slf4j-api:2.0.16") + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-client-core-jvm:3.5.2") + implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt index 2c2d55db..763c9ad8 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper -internal object AuthenticatedYoutubeVisitorData { +object AuthenticatedYoutubeVisitorData { fun fetch( localization: Localization = Localization("en", "US"), contentCountry: ContentCountry = ContentCountry("US"), diff --git a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt new file mode 100644 index 00000000..454496ab --- /dev/null +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt @@ -0,0 +1,47 @@ +package dev.typetype.server.services + +/** Validates cookies captured for an authenticated BiliBili user session. */ +data class BilibiliCookieConfig internal constructor( + val cookieHeader: String?, +) { + val isConfigured: Boolean + get() = cookieHeader != null + + companion object { + private const val MAX_COOKIE_LENGTH = 32 * 1024 + private val COOKIE_NAME = Regex("[!#\\$%&'*+.^_`|~0-9A-Za-z-]+") + private val ACCOUNT_COOKIE_NAMES = setOf("SESSDATA", "bili_jct", "buvid3", "DedeUserID") + + fun fromRaw(raw: String?): BilibiliCookieConfig { + val supplied = raw?.trim()?.takeIf { it.isNotEmpty() } + ?: return BilibiliCookieConfig(cookieHeader = null) + return BilibiliCookieConfig(cookieHeader = normalize(supplied)) + } + + private fun normalize(raw: String): String? { + if (raw.length > MAX_COOKIE_LENGTH || raw.any { it == '\r' || it == '\n' }) return null + val withoutHeader = if (raw.startsWith("Cookie:", ignoreCase = true)) { + raw.substringAfter(':').trim() + } else { + raw + } + if (withoutHeader.isEmpty()) return null + val names = HashSet() + val pairs = withoutHeader.split(';') + .map(String::trim) + .filter(String::isNotEmpty) + .map { pair -> + val separator = pair.indexOf('=') + if (separator <= 0) return null + val name = pair.substring(0, separator).trim() + val value = pair.substring(separator + 1).trim() + if (!COOKIE_NAME.matches(name) || value.isEmpty() || + value.any { it == '\r' || it == '\n' } || !names.add(name) + ) return null + name to value + } + if (pairs.isEmpty() || pairs.none { it.first in ACCOUNT_COOKIE_NAMES }) return null + return pairs.joinToString("; ") { (name, value) -> "$name=$value" } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt index 7a35ef16..b259d320 100644 --- a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.time.Duration import java.util.LinkedHashMap -internal class BoundedExpiringCache( +class BoundedExpiringCache( private val maxEntries: Int, private val maxWeight: Long = Long.MAX_VALUE, ttl: Duration, diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index b6cc04b7..c1770226 100644 --- a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -9,6 +9,7 @@ object NewPipeInitializer { @Volatile private var initialized = false @Volatile private var decoderServiceUrl: String? = null + @Synchronized fun init( tokenServiceUrl: String? = null, youtubeProxySelector: ProxySelector? = null, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt index d0d661e4..1835ba99 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrInfo -internal class SabrPreparedInfo( +class SabrPreparedInfo( val info: YoutubeSabrInfo, val initialToken: SabrTokenBundle?, val isLive: Boolean = false, @@ -10,5 +10,5 @@ internal class SabrPreparedInfo( val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) -internal fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = +fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = info.formats.any { it.isAudio } && info.formats.any { it.isVideo } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt similarity index 65% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt index 93b526e1..3e7b353f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrPreparedSource { +enum class SabrPreparedSource { PUBLIC, AUTHENTICATED_YOUTUBE, } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt index f4fa83f9..e72ba958 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import dev.typetype.server.sabr.YoutubeSabrInfo import java.util.Base64 -internal class SabrTokenBundle( +class SabrTokenBundle( val videoId: String, val visitorBoundPoToken: String, val visitorBoundPoTokenBytes: ByteArray, @@ -63,10 +63,10 @@ internal class SabrTokenBundle( } } -internal fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = +fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = YoutubeSessionPoToken(sessionBinding ?: visitorData, sessionBoundPoToken ?: visitorBoundPoToken) -internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = +fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = takeIf { it.videoId == info.videoId && (it.visitorData == info.visitorData || it.sessionBinding == info.visitorData) @@ -74,4 +74,4 @@ internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): By ?.streamingPoTokenBytes ?.takeIf { it.isNotEmpty() } -internal const val SABR_TOKEN_BINDING_FAILURE = "SABR token does not match session visitorData" +const val SABR_TOKEN_BINDING_FAILURE = "SABR token does not match session visitorData" diff --git a/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt index 713f16a2..2668542a 100644 --- a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrInfo -internal data class TokenYoutubeSession( +data class TokenYoutubeSession( val info: YoutubeSabrInfo, val token: SabrTokenBundle?, val title: String, @@ -19,7 +19,7 @@ internal data class TokenYoutubeSession( val hlsUrl: String = "", ) -internal fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { +fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { val boundToken = token?.takeIf { it.visitorData == info.visitorData } ?: return null return SabrPreparedInfo(info, boundToken, isLive, isLiveContent) .takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index 6daab550..b4b1b1d3 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -5,10 +5,15 @@ import okhttp3.MediaType.Companion.toMediaType import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject +import dev.typetype.server.PLAYBACK_TRACE_ID_HEADER +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.currentPlaybackTraceId +import dev.typetype.server.currentRequestId import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal class TypetypeTokenSabrTokenClient( +class TypetypeTokenSabrTokenClient( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) { @@ -34,9 +39,15 @@ internal class TypetypeTokenSabrTokenClient( val request = Request.Builder() .url("${tokenServiceUrl.trimEnd('/')}/potoken/session") .post(body) + .apply { + currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { header(PLAYBACK_TRACE_ID_HEADER, it) } + } .build() + val startedAt = System.nanoTime() return try { client.newCall(request).execute().use { response -> + PlaybackTraceLog.record("token_http", "endpoint=potoken_session status=${response.code} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (!response.isSuccessful) return null SabrTokenBundle.fromSessionResponse(videoId, sessionBinding, JSONObject(response.body.string())) } @@ -56,8 +67,14 @@ internal class TypetypeTokenSabrTokenClient( ): SabrTokenBundle? { val url = buildUrl(binding, forceRefresh, refreshVideo) val target = if (logIdentifier) " for $binding" else " for session binding" + val startedAt = System.nanoTime() + val request = Request.Builder().url(url).get().apply { + currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { header(PLAYBACK_TRACE_ID_HEADER, it) } + }.build() return try { - client.newCall(Request.Builder().url(url).get().build()).execute().use { resp -> + client.newCall(request).execute().use { resp -> + PlaybackTraceLog.record("token_http", "endpoint=potoken status=${resp.code} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (!resp.isSuccessful) { System.err.println("[TypetypeTokenSabrTokenClient] /potoken HTTP ${resp.code}$target") return null diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt index 4f7b5cd3..5756317a 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException import org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptDecoder -internal class TypetypeTokenYoutubeJavaScriptDecoder( +class TypetypeTokenYoutubeJavaScriptDecoder( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) : YoutubeJavaScriptDecoder { diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt index 43074ff1..fa465918 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt @@ -11,6 +11,11 @@ import okhttp3.Response import org.json.JSONObject import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.PLAYBACK_TRACE_ID_HEADER +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.currentPlaybackTraceId +import dev.typetype.server.currentRequestId import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo import java.io.IOException @@ -18,7 +23,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import kotlin.coroutines.resume -internal class TypetypeTokenYoutubeSessionClient( +class TypetypeTokenYoutubeSessionClient( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) { @@ -67,17 +72,27 @@ internal class TypetypeTokenYoutubeSessionClient( val url = "${tokenServiceUrl.trimEnd('/')}/youtube/sabr/session?videoId=$encodedVideoId&client=MWEB" + if (isolated) "&isolated=true" else "" return suspendCancellableCoroutine { continuation -> - val call = client.newCall(Request.Builder().url(url).get().build()) + val requestBuilder = Request.Builder().url(url).get() + currentRequestId()?.let { requestBuilder.header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { requestBuilder.header(PLAYBACK_TRACE_ID_HEADER, it) } + val traceId = currentPlaybackTraceId() + val requestId = currentRequestId() + val startedAt = System.nanoTime() + PlaybackTraceLog.record("token_session_start", "operation=youtube_sabr_session isolated=$isolated") + val call = client.newCall(requestBuilder.build()) continuation.invokeOnCancellation { call.cancel() } call.enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { + PlaybackTraceLog.record(traceId, requestId, "token_session_complete", "status=network_error durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (continuation.isActive) continuation.resume(null) } override fun onResponse(call: Call, response: Response) { + val status = response.code val session = response.use { if (!it.isSuccessful) null else runCatching { JSONObject(it.body.string()) }.getOrNull() } + PlaybackTraceLog.record(traceId, requestId, "token_session_complete", "status=$status result=${if (session == null) "empty" else "ok"} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (continuation.isActive) continuation.resume(session) } }) diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt index 15b63405..3ddc8d7f 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -9,7 +9,7 @@ import java.security.MessageDigest import java.util.concurrent.CompletableFuture import java.util.concurrent.ConcurrentHashMap -internal class TypetypeTokenYoutubeSessionPoTokenProvider( +class TypetypeTokenYoutubeSessionPoTokenProvider( private val boundTokenFetcher: (String) -> String?, private val visitorDataFetcher: (Localization, ContentCountry) -> String, private val nowMillis: () -> Long = System::currentTimeMillis, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index 74351ac3..019e2b94 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider -internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { +object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { private val scopedToken = ThreadLocal() @Volatile private var authenticatedProvider: YoutubeSessionPoTokenProvider? = null diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt index 3c9e475c..31ca2538 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt @@ -12,7 +12,7 @@ import kotlinx.serialization.json.jsonPrimitive object YoutubeRemoteBrowserMessageGuard { private val json = Json { ignoreUnknownKeys = true } private val clientTypes = setOf("resize", "pointer", "wheel", "key", "text", "cancel") - private val tokenTypes = setOf("status", "error") + private val tokenTypes = setOf("status", "error", "log") private val phases = setOf("opening", "awaiting_login", "capturing_session", "connected") private val pointerEvents = setOf("down", "up", "move") private val keyEvents = setOf("down", "up") @@ -45,6 +45,7 @@ object YoutubeRemoteBrowserMessageGuard { when (obj.string("type")?.takeIf { it in tokenTypes }) { "status" -> obj.string("phase")?.let { it in phases } == true "error" -> (obj.string("message")?.length ?: Int.MAX_VALUE) <= 200 + "log" -> obj.int("at").inRange(0..Int.MAX_VALUE) && (obj.string("message")?.length ?: Int.MAX_VALUE) <= 1000 else -> false } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt diff --git a/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionClientTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt similarity index 73% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt index 2469785f..ad58db12 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt +++ b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt @@ -30,4 +30,14 @@ class YoutubeRemoteBrowserMessageGuardTest { assertNull(YoutubeRemoteBrowserMessageGuard.tokenText(complete)) assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"status","phase":"unknown"}""")) } + + @Test + fun `token diagnostics logs are forwarded when bounded`() { + val log = """{"type":"log","at":1200,"message":"login check url=https://accounts.google.com/signin cookies=google.com=[SID]"}""" + + assertEquals(log, YoutubeRemoteBrowserMessageGuard.tokenText(log)) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","message":"missing at"}""")) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","at":-1,"message":"negative"}""")) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","at":5,"message":"${"x".repeat(1001)}"}""")) + } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt diff --git a/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt new file mode 100644 index 00000000..5ebba5ea --- /dev/null +++ b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class BilibiliCookieConfigTest { + @Test + fun `normalizes a browser cookie header`() { + val config = BilibiliCookieConfig.fromRaw( + "Cookie: SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", + ) + + assertTrue(config.isConfigured) + assertEquals( + "SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", + config.cookieHeader, + ) + } + + @Test + fun `rejects malformed and unrelated cookies`() { + assertFalse(BilibiliCookieConfig.fromRaw("theme=dark").isConfigured) + assertNull(BilibiliCookieConfig.fromRaw("SESSDATA=session; SESSDATA=duplicate").cookieHeader) + assertNull(BilibiliCookieConfig.fromRaw("SESSDATA=session\r\nInjected: yes").cookieHeader) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 0a997ae6..86c5ea82 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,5 +1,28 @@ rootProject.name = "typetype-server" +dependencyResolutionManagement { + repositories { + mavenLocal() + mavenCentral() + maven { url = uri("https://jitpack.io") } + } +} + +include(":server-admin") +include(":server-auth") +include(":server-cache") +include(":server-core") +include(":server-db") +include(":server-downloader") +include(":server-domain") +include(":server-http") +include(":server-playback") +include(":server-portability") +include(":server-sabr") +include(":server-test-support") +include(":server-services") +include(":server-token-gateway") + val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull ?.let { file(it) } diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 3e382395..8bc5297c 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -12,6 +12,8 @@ import dev.typetype.server.services.GitHubIssueService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.InstanceService @@ -56,6 +58,8 @@ fun Application.module() { val gitHubIssueService = GitHubIssueService() val adminSettingsService = AdminSettingsService() val activeSessionService = ActiveSessionService(adminSettingsService) + val presenceKeyService = PresenceKeyService() + val presenceService = PresenceService(hasActiveKey = presenceKeyService::hasActiveKey) val restoreService = PipePipeBackupImporterService() val downloaderServiceUrl = System.getenv("DOWNLOADER_SERVICE_URL") ?: "http://typetype-downloader:18093" val youtubeSessionEncryptionKey = SecretConfigReader.read("YOUTUBE_SESSION_ENCRYPTION_KEY") @@ -118,6 +122,8 @@ fun Application.module() { authSessionConfig = authSessionConfig, adminSettingsService = adminSettingsService, activeSessionService = activeSessionService, + presenceKeyService = presenceKeyService, + presenceService = presenceService, downloaderGatewayService = downloaderGatewayService, gitHubIssueService = gitHubIssueService, instanceService = instanceService, diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index ef71d1cf..c9c6a102 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -21,6 +21,8 @@ import io.ktor.server.request.path import io.ktor.server.response.respond import io.ktor.server.websocket.WebSockets import io.ktor.util.AttributeKey +import dev.typetype.server.configureStatusPages +import dev.typetype.server.routes.TooManyRequestsBodyAttribute import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory import kotlin.time.Duration.Companion.minutes @@ -34,15 +36,7 @@ private const val PROXY_STORYBOARD_RATE_LIMIT = 1200 private const val USER_DATA_RATE_LIMIT = 120 private const val MAX_WEBSOCKET_FRAME_BYTES = 64L * 1024L * 1024L private val RATE_LIMIT_WINDOW = 1.minutes -private val preserveTooManyRequestsBodyAttribute = AttributeKey("preserveTooManyRequestsBody") -val EXTRACTION_ZONE = RateLimitName("extraction") -val DEARROW_ZONE = RateLimitName("dearrow") -val STREAMS_ZONE = RateLimitName("streams") -val CHANNEL_ZONE = RateLimitName("channel") -val PROXY_ZONE = RateLimitName("proxy") -val PROXY_STORYBOARD_ZONE = RateLimitName("proxy-storyboard") -val USER_DATA_ZONE = RateLimitName("user-data") fun Application.configurePlugins(authService: AuthService) { installRequestObservability() @@ -100,33 +94,3 @@ fun Application.configurePlugins(authService: AuthService) { } configureStatusPages() } - -internal fun ApplicationCall.preserveTooManyRequestsBody() { - attributes.put(preserveTooManyRequestsBodyAttribute, Unit) -} - -internal fun Application.configureStatusPages() { - val log = LoggerFactory.getLogger("RequestLogger") - install(StatusPages) { - status(HttpStatusCode.TooManyRequests) { call, status -> - if (call.attributes.contains(preserveTooManyRequestsBodyAttribute)) return@status - if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") - call.respond(status, ErrorResponse("Too many requests", "rate_limited")) - } - exception { call, cause -> - log.warn("Bad request: ${cause.message}") - call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request", "bad_request")) - } - exception { call, cause -> - if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception - if (cause is kotlinx.coroutines.CancellationException) throw cause - // Ktor's multipart producer can fail outside the route's receive block. - if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) { - call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException()) - return@exception - } - log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) - call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) - } - } -} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt deleted file mode 100644 index 9479bf65..00000000 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt +++ /dev/null @@ -1,17 +0,0 @@ -package dev.typetype.server.portability - -import dev.typetype.server.cache.CacheJson -import dev.typetype.server.models.SettingsItem -import dev.typetype.server.services.TypeTypeBackupLibraryRestore - -internal object TypeTypePortabilitySettingsImport { - fun write(userId: String, source: PortabilityRecordSource): Long { - var count = 0L - source.forEach(PortabilityCategory.SETTINGS) { record -> - if (record !is PortabilitySettings) return@forEach - val settings = CacheJson.decodeFromString(record.values.toString()) - count += TypeTypeBackupLibraryRestore.settings(userId, settings) - } - return count - } -} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt deleted file mode 100644 index e48116cf..00000000 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt +++ /dev/null @@ -1,46 +0,0 @@ -package dev.typetype.server.portability - -import dev.typetype.server.services.YoutubeTakeoutActivitySignalService -import dev.typetype.server.services.YoutubeTakeoutHistoryParser -import java.io.Reader -import java.util.zip.ZipEntry -import java.util.zip.ZipFile - -internal object YoutubeTakeoutHtmlPortabilityReader { - fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> - zip.getInputStream(entry).bufferedReader().use { reader -> - readWindows(reader) { html -> writeWindow(html, sink) } - } - } - } - - private fun writeWindow(html: String, sink: PortabilityRecordSink) { - YoutubeTakeoutHistoryParser.parse(html).forEach { sink.write(it.toPortability()) } - val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) - subscriptions.forEach { sink.write(it.toPortability()) } - favorites.forEach { sink.write(it.toPortability()) } - } - - private fun readWindows(reader: Reader, block: (String) -> Unit) { - val buffer = CharArray(READ_CHARS) - val window = StringBuilder(WINDOW_CHARS + READ_CHARS) - while (true) { - val read = reader.read(buffer) - if (read < 0) break - window.append(buffer, 0, read) - if (window.length >= WINDOW_CHARS) { - block(window.toString()) - window.delete(0, window.length - OVERLAP_CHARS) - } - } - if (window.isNotEmpty()) block(window.toString()) - } - - private fun isYoutubeHtml(entry: ZipEntry): Boolean = - entry.name.endsWith(".html", ignoreCase = true) && "youtube" in entry.name.lowercase() - - private const val READ_CHARS = 32 * 1024 - private const val WINDOW_CHARS = 512 * 1024 - private const val OVERLAP_CHARS = 128 * 1024 -} diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt deleted file mode 100644 index fd73c3dc..00000000 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt +++ /dev/null @@ -1,4 +0,0 @@ -package dev.typetype.server.sabr - -internal typealias SabrProtocolException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -internal typealias SabrRecoverableException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException diff --git a/src/main/kotlin/dev/typetype/server/services/AccessMode.kt b/src/main/kotlin/dev/typetype/server/services/AccessMode.kt deleted file mode 100644 index a0fbd39b..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AccessMode.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dev.typetype.server.services - -internal const val ACCESS_MODE_UNRESTRICTED = "unrestricted" -internal const val ACCESS_MODE_ALLOW_LIST = "allow_list" -internal const val ALLOW_SCOPE_USER = "user" -internal const val ALLOW_SCOPE_GLOBAL = "global" - -internal fun String.toAccessMode(): String = when (this) { - ACCESS_MODE_ALLOW_LIST -> ACCESS_MODE_ALLOW_LIST - else -> ACCESS_MODE_UNRESTRICTED -} diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt deleted file mode 100644 index 0048911d..00000000 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt +++ /dev/null @@ -1,23 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.HomeRecommendationsResponse -import dev.typetype.server.models.SearchPageResponse -import dev.typetype.server.models.StreamResponse - -internal fun HomeRecommendationsResponse.filterBlocked(profile: BlockedContentProfile): HomeRecommendationsResponse = copy( - items = items.filter { - profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) - }, -) - -internal fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( - items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, - channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, - playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, -) - -internal fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse = copy( - relatedStreams = relatedStreams.filter { - profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) - }, -) diff --git a/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt b/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt deleted file mode 100644 index edd5e0a7..00000000 --- a/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt +++ /dev/null @@ -1,28 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import okhttp3.OkHttpClient -import okhttp3.Request - -interface DeArrowRemote { - suspend fun branding(videoId: String): String? - suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? -} - -class DeArrowClient(private val client: OkHttpClient = OkHttpClient()) : DeArrowRemote { - override suspend fun branding(videoId: String): String? = - get("https://sponsor.ajay.app/api/branding?videoID=$videoId&fetchAll=true")?.decodeToString() - - override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? = - get("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=$videoId&time=$timestamp") - - private suspend fun get(url: String): ByteArray? = withContext(Dispatchers.IO) { - runCatching { - client.newCall(Request.Builder().url(url).get().build()).execute().use { response -> - if (!response.isSuccessful) return@use null - response.body.bytes().takeIf { it.isNotEmpty() } - } - }.getOrNull() - } -} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt deleted file mode 100644 index 771aed8c..00000000 --- a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ /dev/null @@ -1,24 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.models.StreamResponse - -internal class SabrBootstrapStreamService( - private val sessionStore: SabrSessionStore, - private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, -) : StreamService { - override suspend fun getStreamInfo(url: String): ExtractionResult { - val videoId = youtubeVideoId(url) - ?: return ExtractionResult.BadRequest("Invalid YouTube URL") - val metadata = tokenSessionClient.fetchPlaybackSession(videoId) - ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") - val prepared = metadata.preparedSabrInfo() - ?: sessionStore.fetchInfo(videoId, cachedFirst = true) - ?: return ExtractionResult.Failure("SABR playback formats unavailable") - sessionStore.rememberPreparedInfo(videoId, prepared) - return ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) - } -} - -internal fun youtubeVideoId(url: String): String? = - Regex("(?:[?&]v=|/shorts/|youtu\\.be/)([A-Za-z0-9_-]{6,})").find(url)?.groupValues?.get(1) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt deleted file mode 100644 index 2a0e3874..00000000 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ /dev/null @@ -1,82 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.delay - -internal class SabrDemandWatchdog( - private val clock: () -> Long = System::currentTimeMillis, - private val intervalMs: Long = SabrPumpPolicy.IDLE_POLL_MS, -) { - suspend fun monitor(isAlive: () -> Boolean, holder: SabrSessionHolder): Boolean { - val deadline = SabrDemandDeadline(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS) - while (isAlive()) { - val state = holder.playbackState() - if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return false - val inFlightDemand = holder.inFlightSegmentDemand() - if (inFlightDemand != null) { - if (holder.isLiveDemandOutsideRecoverableWindow(inFlightDemand.request) && - SabrDemandAttemptFinisher.expireStalledInFlightDemand(holder, inFlightDemand, recoverable = true) - ) { - return true - } - if (inFlightDemand.futureLiveRequest) holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) - val nowMs = clock() - val backoffRemainingMs = holder.session.demandBackoffRemainingMs - val lastProgressAtMs = inFlightDemand.observeProgress(holder.session.mediaProgressVersion, nowMs) - val completedIdle = holder.session.getCachedSegment(inFlightDemand.request) != null && - nowMs - lastProgressAtMs >= SabrPumpPolicy.COMPLETED_DEMAND_IDLE_MS - if (completedIdle && SabrDemandAttemptFinisher.interruptCompletedInFlightDemand(holder, inFlightDemand)) { - return true - } - if (deadline.isExpired( - inFlightDemand.identity, - inFlightDemand.registeredAtMs, - nowMs, - backoffRemainingMs, - ) && - SabrDemandAttemptFinisher.expireStalledInFlightDemand(holder, inFlightDemand) - ) { - return true - } - delay(nextCheckDelayMs(backoffRemainingMs, inFlightDemand.futureLiveRequest)) - continue - } - val request = holder.nextSegmentDemand() - if (request == null) { - delay(intervalMs) - continue - } - val outsideLiveWindow = holder.isLiveDemandOutsideRecoverableWindow(request) - val futureLiveRequest = holder.isFutureLiveRequest(request) - if (outsideLiveWindow) { - val identity = holder.segmentDemandIdentity(request) - if (identity != null && - SabrDemandAttemptFinisher.expireStalledDemand(holder, request, identity, recoverable = true) - ) { - return true - } - } - if (futureLiveRequest) { - holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) - } - val identity = holder.segmentDemandIdentity(request) - val registeredAtMs = identity?.let { holder.segmentDemandRegisteredAtMs(request, it) } - val nowMs = clock() - val backoffRemainingMs = holder.session.demandBackoffRemainingMs - if (identity != null && registeredAtMs != null && deadline.isExpired( - identity, - registeredAtMs, - nowMs, - backoffRemainingMs, - ) && - SabrDemandAttemptFinisher.expireStalledDemand(holder, request, identity, futureLiveRequest) - ) { - return true - } - delay(nextCheckDelayMs(backoffRemainingMs, futureLiveRequest)) - } - return false - } - - private fun nextCheckDelayMs(backoffRemainingMs: Long, futureLiveRequest: Boolean): Long = - maxOf(intervalMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) -} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt b/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt deleted file mode 100644 index e0f6ae20..00000000 --- a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt +++ /dev/null @@ -1,71 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.sabr.SabrSegmentRequest -import java.util.concurrent.ConcurrentHashMap - -internal class SabrInFlightDemand( - val request: SabrSegmentRequest, - val identity: String, - val registeredAtMs: Long, - val futureLiveRequest: Boolean, -) { - private var lastProgressVersion = Long.MIN_VALUE - private var lastProgressAtMs = registeredAtMs - - fun observeProgress(version: Long, observedAtMs: Long): Long { - if (version != lastProgressVersion) { - lastProgressVersion = version - lastProgressAtMs = observedAtMs - } - return lastProgressAtMs - } -} - -internal object SabrInFlightDemandTracker { - private val demands = ConcurrentHashMap() - - fun begin( - holder: SabrSessionHolder, - request: SabrSegmentRequest, - identity: String, - futureLiveRequest: Boolean, - ): Boolean { - val registeredAtMs = holder.segmentDemandRegisteredAtMs(request, identity) ?: return false - val demand = SabrInFlightDemand( - request, - identity, - registeredAtMs, - futureLiveRequest, - ) - return demands.putIfAbsent(holder.sessionToken, demand) == null - } - - fun current(holder: SabrSessionHolder): SabrInFlightDemand? = demands[holder.sessionToken] - - fun finish(holder: SabrSessionHolder, identity: String): Boolean { - val demand = demands[holder.sessionToken] ?: return false - if (demand.identity != identity) return false - return demands.remove(holder.sessionToken, demand) - } - - fun clear(holder: SabrSessionHolder): Unit { - demands.remove(holder.sessionToken) - } - - fun clearAll(): Unit = demands.clear() -} - -internal fun SabrSessionHolder.beginInFlightSegmentDemand( - request: SabrSegmentRequest, - identity: String, - futureLiveRequest: Boolean, -): Boolean = SabrInFlightDemandTracker.begin(this, request, identity, futureLiveRequest) - -internal fun SabrSessionHolder.inFlightSegmentDemand(): SabrInFlightDemand? = - SabrInFlightDemandTracker.current(this) - -internal fun SabrSessionHolder.finishInFlightSegmentDemand(identity: String): Boolean = - SabrInFlightDemandTracker.finish(this, identity) - -internal fun SabrSessionHolder.clearInFlightSegmentDemand(): Unit = - SabrInFlightDemandTracker.clear(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt deleted file mode 100644 index 10064135..00000000 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt +++ /dev/null @@ -1,36 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.withTimeoutOrNull - -internal data class SabrPlaybackInitializationPreloadResult( - val video: ByteArray?, - val audio: ByteArray?, -) { - fun isComplete(audioOnly: Boolean): Boolean = - audio != null && (audioOnly || video != null) - - fun missingTracks(audioOnly: Boolean, videoItag: Int, audioItag: Int): String = - buildList { - if (!audioOnly && video == null) add("video:$videoItag") - if (audio == null) add("audio:$audioItag") - }.joinToString() -} - -internal object SabrPlaybackInitializationPreloader { - suspend fun preload( - sessionStore: SabrSessionStore, - holder: SabrSessionHolder, - audioOnly: Boolean, - timeoutMs: Long, - ): SabrPlaybackInitializationPreloadResult = withTimeoutOrNull(timeoutMs) { - coroutineScope { - val video = holder.videoFormat - .takeUnless { audioOnly } - ?.let { format -> async { sessionStore.fetchInitializationData(holder, format) } } - val audio = async { sessionStore.fetchInitializationData(holder, holder.audioFormat) } - SabrPlaybackInitializationPreloadResult(video?.await(), audio.await()) - } - } ?: SabrPlaybackInitializationPreloadResult(null, null) -} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt deleted file mode 100644 index 34c7a309..00000000 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt +++ /dev/null @@ -1,92 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.db.tables.FavoritesTable -import dev.typetype.server.db.tables.HistoryTable -import dev.typetype.server.db.tables.PlaylistVideosTable -import dev.typetype.server.db.tables.SubscriptionsTable -import dev.typetype.server.db.tables.WatchLaterTable -import dev.typetype.server.models.SubscriptionItem -import org.jetbrains.exposed.v1.core.Column -import org.jetbrains.exposed.v1.core.ResultRow -import org.jetbrains.exposed.v1.core.SortOrder -import org.jetbrains.exposed.v1.core.and -import org.jetbrains.exposed.v1.core.eq -import org.jetbrains.exposed.v1.core.inList -import org.jetbrains.exposed.v1.core.neq -import org.jetbrains.exposed.v1.jdbc.selectAll -import org.jetbrains.exposed.v1.jdbc.update - -object SubscriptionAvatarRepairer { - fun repair(userId: String, items: List): List { - val candidateUrls = items.filter { it.avatarUrl.isBlank() } - .map { it.channelUrl } - .distinct() - if (candidateUrls.isEmpty()) return items - val avatars = knownAvatars(userId = userId, channelUrls = candidateUrls) - .entries - .take(MAX_AVATAR_REPAIR_PER_REQUEST) - .associate { it.toPair() } - if (avatars.isEmpty()) return items - avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } - return items.map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } - } - - private fun knownAvatars(userId: String, channelUrls: List): Map { - val avatars = linkedMapOf() - historyAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - playlistAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - watchLaterAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - favoriteAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - return avatars - } - - private fun historyAvatars(userId: String, channelUrls: List): Map = HistoryTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, HistoryTable.userId, HistoryTable.channelUrl, HistoryTable.channelAvatar) } - .orderBy(HistoryTable.watchedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(HistoryTable.channelUrl, HistoryTable.channelAvatar) - - private fun playlistAvatars(userId: String, channelUrls: List): Map = PlaylistVideosTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, PlaylistVideosTable.userId, PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) } - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) - - private fun watchLaterAvatars(userId: String, channelUrls: List): Map = WatchLaterTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, WatchLaterTable.userId, WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) } - .orderBy(WatchLaterTable.addedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) - - private fun favoriteAvatars(userId: String, channelUrls: List): Map = FavoritesTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, FavoritesTable.userId, FavoritesTable.channelUrl, FavoritesTable.channelAvatar) } - .orderBy(FavoritesTable.favoritedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(FavoritesTable.channelUrl, FavoritesTable.channelAvatar) - - private fun updateAvatar(userId: String, channelUrl: String, avatarUrl: String): Int = SubscriptionsTable.update({ - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) and (SubscriptionsTable.avatarUrl eq "") - }) { - it[SubscriptionsTable.avatarUrl] = avatarUrl - } - - private fun avatarSourceFilter( - userId: String, - channelUrls: List, - userColumn: Column, - urlColumn: Column, - avatarColumn: Column, - ) = (userColumn eq userId) and (urlColumn inList channelUrls) and (avatarColumn neq "") - - private fun Iterable.associateAvatarRows( - urlColumn: Column, - avatarColumn: Column, - ): Map = mapNotNull { row -> - val avatar = row[avatarColumn].trim() - if (avatar.isProxyableAvatar()) ChannelUrlCanonicalizer.canonicalize(row[urlColumn]) to avatar else null - }.distinctBy { it.first }.toMap() - - private fun String.isProxyableAvatar(): Boolean = startsWith("https://") || startsWith("http://") - - private const val MAX_AVATAR_REPAIR_PER_REQUEST = 25 - private const val MAX_AVATAR_SOURCE_ROWS = 100 -} diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt deleted file mode 100644 index e8f1837f..00000000 --- a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.VideoItem - -internal fun VideoItem.isUpcomingAt(now: Long): Boolean = - !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now - -internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) - -internal fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = - isLive || isPostLive || isLiveContent || isUpcomingAt(now) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt deleted file mode 100644 index 06325041..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt +++ /dev/null @@ -1,10 +0,0 @@ -package dev.typetype.server.services - -sealed interface YoutubeRemoteBrowserCompleteResult { - data object Completed : YoutubeRemoteBrowserCompleteResult - data object Unauthorized : YoutubeRemoteBrowserCompleteResult - data object NotFound : YoutubeRemoteBrowserCompleteResult - data object InvalidPayload : YoutubeRemoteBrowserCompleteResult - data object InvalidCredentials : YoutubeRemoteBrowserCompleteResult - data object Unavailable : YoutubeRemoteBrowserCompleteResult -} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt deleted file mode 100644 index 6a01dcfe..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt +++ /dev/null @@ -1,69 +0,0 @@ -package dev.typetype.server.services - -object YoutubeTakeoutActivityClassifier { - private val watchedPhrases = setOf( - "You watched", - "Vous avez regardé", - "Has visto", - "Você assistiu", - "hai guardato", - "İzlediniz", - "du hast angesehen", - "вы смотрели", - "視聴しました", - "시청한 동영상", - "已观看", - "已觀看", - ) - - private val likedPhrases = setOf( - "You liked", - "Liked", - "Vous avez aimé", - "A aimé", - "te ha gustado", - "Você gostou", - "hai messo mi piace", - "Beğendiniz", - "gefällt mir", - "понравилось", - "高く評価しました", - "좋아요 표시함", - "点赞了", - "按讚", - ) - - private val subscribedPhrases = setOf( - "You subscribed to", - "Vous vous êtes abonné à", - "te has suscrito a", - "Você se inscreveu em", - "ti sei iscritto a", - "Abone oldunuz", - "du hast abonniert", - "вы подписались на", - "登録しました", - "구독함", - "已订阅", - "已訂閱", - ) - - fun isWatched(value: String): Boolean = containsAny(value, watchedPhrases) - - fun isLiked(value: String): Boolean = containsAny(value, likedPhrases) - - fun isSubscribed(value: String): Boolean = containsAny(value, subscribedPhrases) - - val watchedPattern: String = pattern(watchedPhrases) - - val likedPattern: String = pattern(likedPhrases) - - val subscribedPattern: String = pattern(subscribedPhrases) - - private fun containsAny(value: String, phrases: Set): Boolean { - val normalized = YoutubeTakeoutTextNormalizer.normalize(value) - return phrases.any { YoutubeTakeoutTextNormalizer.normalize(it) in normalized } - } - - private fun pattern(phrases: Set): String = phrases.joinToString("|") { Regex.escape(it) } -} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt deleted file mode 100644 index 4846bce3..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ /dev/null @@ -1,70 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.FavoriteItem -import dev.typetype.server.models.SubscriptionItem -import java.nio.file.Path -import java.util.zip.ZipFile - -object YoutubeTakeoutActivitySignalService { - private val subscribedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.subscribedPattern})\s*([\s\S]*?)
""", RegexOption.IGNORE_CASE) - private val likedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.likedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) - private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") - private val channelUrlRegex = Regex("""https?://www\.youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+)""") - private val spacesRegex = Regex("""\s+""") - - fun parse(zipPath: Path): Pair, List> { - val subscriptions = mutableListOf() - val favorites = mutableListOf() - ZipFile(zipPath.toFile()).use { zip -> - zip.entries().asSequence().forEach { entry -> - val normalized = entry.name.lowercase() - if (entry.isDirectory || !normalized.endsWith(".html") || !normalized.contains("youtube")) return@forEach - val html = zip.getInputStream(entry).bufferedReader().use { it.readText() }.replace("\u00a0", " ") - val parsed = parseHtml(html) - subscriptions += parsed.first - favorites += parsed.second - } - } - return subscriptions.distinctBy { it.channelUrl } to favorites.distinctBy { it.videoUrl } - } - - internal fun parseHtml(html: String): Pair, List> = - parseSubscriptions(html) to parseFavorites(html) - - private fun parseSubscriptions(html: String): List { - return subscribedRegex.findAll(html).mapNotNull { match -> - val url = channelUrlRegex.find(decode(match.groupValues[1]))?.value ?: return@mapNotNull null - val name = decode(match.groupValues[2]) - SubscriptionItem(channelUrl = url.replace("http://", "https://"), name = name, avatarUrl = "") - }.toList() - } - - private fun parseFavorites(html: String): List { - return likedRegex.findAll(html).mapNotNull { match -> - val title = decode(match.groupValues[2]) - if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null - val source = decode(match.groupValues[1]) + " " + title - val videoUrl = watchUrlRegex.find(source)?.value?.replace("http://", "https://") - ?: return@mapNotNull null - FavoriteItem( - videoUrl = videoUrl, - favoritedAt = YoutubeTakeoutDateParser.parseEpochMillis(decode(match.groupValues[5])) ?: 0L, - title = title, - channelName = decode(match.groupValues[4]), - channelUrl = decode(match.groupValues[3]).replace("http://", "https://"), - ) - }.toList() - } - - private fun decode(value: String): String { - return value - .replace(" ", " ") - .replace("'", "'") - .replace("&", "&") - .replace(""", "\"") - .replace("<", "<") - .replace(">", ">") - .replace(spacesRegex, " ") - .trim() - } -} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt deleted file mode 100644 index 2364cdc1..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ /dev/null @@ -1,50 +0,0 @@ -package dev.typetype.server.services - -import java.time.Instant -import java.time.LocalDate -import java.time.OffsetDateTime -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.util.Locale - -object YoutubeTakeoutDateParser { - private val activityFormatters = listOf( - Locale.ENGLISH, - Locale.FRENCH, - ).flatMap { locale -> - listOf("d MMM yyyy, HH:mm:ss z", "d MMMM yyyy, HH:mm:ss z").map { pattern -> - DateTimeFormatter.ofPattern(pattern, locale) - } - } - - fun parseEpochMillis(value: String): Long? { - val trimmed = value.replace("\u00a0", " ").trim() - if (trimmed.isBlank()) return null - parseOffset(trimmed)?.let { return it } - parseInstant(trimmed)?.let { return it } - parseDate(trimmed)?.let { return it } - return parseActivityDate(trimmed) - } - - private fun parseOffset(value: String): Long? = runCatching { - OffsetDateTime.parse(value).toInstant().toEpochMilli() - }.getOrNull() - - private fun parseInstant(value: String): Long? = runCatching { - Instant.parse(value).toEpochMilli() - }.getOrNull() - - private fun parseDate(value: String): Long? = runCatching { - LocalDate.parse(value).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli() - }.getOrNull() - - private fun parseActivityDate(value: String): Long? { - activityFormatters.forEach { formatter -> - runCatching { ZonedDateTime.parse(value, formatter).toInstant().toEpochMilli() } - .getOrNull() - ?.let { return it } - } - return null - } -} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt deleted file mode 100644 index 1cee2b5a..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ /dev/null @@ -1,55 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.HistoryItem - -object YoutubeTakeoutHistoryParser { - private val rowRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.watchedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) - private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") - private val tagRegex = Regex("<[^>]+>") - private val spacesRegex = Regex("\\s+") - - fun parse(html: String): List { - val resolvedHtml = html.replace("\u00a0", " ") - return rowRegex.findAll(resolvedHtml).mapNotNull { match -> - val url = extractUrl(match.groupValues[1]) ?: return@mapNotNull null - val title = decode(match.groupValues[2]) - if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null - val channelUrl = match.groupValues[3].takeIf { it.isNotBlank() }.orEmpty() - val channelName = decode(match.groupValues[4]).ifBlank { "Unknown channel" } - val watchedAt = parseDate(decode(match.groupValues[5])) - HistoryItem( - url = url, - title = title, - thumbnail = "", - channelName = channelName, - channelUrl = channelUrl, - channelAvatar = "", - duration = 0, - progress = 0, - watchedAt = watchedAt, - ) - }.toList().distinctBy { it.url to it.watchedAt } - } - - private fun parseDate(value: String): Long { - return YoutubeTakeoutDateParser.parseEpochMillis(value) ?: 0L - } - - private fun extractUrl(value: String): String? { - val decoded = decode(value) - val direct = urlRegex.find(decoded)?.value - if (direct != null) return direct - val fromText = urlRegex.find(decode(tagRegex.replace(decoded, " ")))?.value - return fromText - } - - private fun decode(value: String): String = tagRegex.replace(value, " ") - .replace(" ", " ") - .replace("'", "'") - .replace("&", "&") - .replace(""", "\"") - .replace("<", "<") - .replace(">", ">") - .replace(spacesRegex, " ") - .trim() -} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt deleted file mode 100644 index 5bfe477b..00000000 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dev.typetype.server.services - -import java.text.Normalizer - -object YoutubeTakeoutTextNormalizer { - fun normalize(value: String): String = Normalizer - .normalize(value.lowercase(), Normalizer.Form.NFD) - .replace(Regex("\\p{M}+"), "") - .replace(Regex("[^\\p{L}\\p{N}]+"), " ") - .trim() -} diff --git a/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt b/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt new file mode 100644 index 00000000..543825e1 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt @@ -0,0 +1,37 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.ChannelNotificationPreferenceService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class ChannelNotificationPreferenceServiceTest { + private val subscriptions = SubscriptionsService() + private val preferences = ChannelNotificationPreferenceService(subscriptions) + + companion object { + private const val CHANNEL_URL = "https://www.youtube.com/channel/UCNotification" + + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `channel notifications are disabled until explicitly enabled`() = runTest { + subscriptions.add(TEST_USER_ID, SubscriptionItem(CHANNEL_URL, "Channel", "")) + + assertEquals(false, preferences.list(TEST_USER_ID).single().enabled) + + preferences.set(TEST_USER_ID, CHANNEL_URL, true) + + assertEquals(true, preferences.list(TEST_USER_ID).single().enabled) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt b/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt new file mode 100644 index 00000000..fbdc0031 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt @@ -0,0 +1,30 @@ +package dev.typetype.server + +import dev.typetype.server.services.toChannelTab +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs + +class ChannelTabCapabilityTest { + @Test + fun `services without channel tab support fall back to channel info`() { + val tab = ServiceList.BiliBili.toChannelTab( + "https://space.bilibili.com/946974", + "latest", + ) + + assertNull(tab) + } + + @Test + fun `services with channel tab support keep sorted videos`() { + val tab = ServiceList.YouTube.toChannelTab( + "https://www.youtube.com/@test", + "latest", + ) + + assertEquals(ChannelTabs.VIDEOS, tab) + } +} diff --git a/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt b/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt deleted file mode 100644 index bd1bcff4..00000000 --- a/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -package dev.typetype.server - -import dev.typetype.server.services.DeArrowRemote -import dev.typetype.server.services.DeArrowService -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Test - -class DeArrowServiceTest { - @Test - fun `selects accepted title and thumbnail and caches branding`() = runBlocking { - val remote = FakeDeArrowRemote() - val service = DeArrowService(FakeCacheService(), remote) - val first = service.get("stZ3ZoR_8eg") - val second = service.get("stZ3ZoR_8eg") - assertEquals("Clear title", first?.title) - assertEquals("/dearrow/thumbnail?videoId=stZ3ZoR_8eg&time=12.5", first?.thumbnailUrl) - assertEquals(listOf("Rejected title", "Clear title", "Original title"), first?.titles?.map { it.title }) - assertEquals(listOf(-2, 2, 0), first?.titles?.map { it.votes }) - assertEquals(listOf(8.0, 12.5, null), first?.thumbnails?.map { it.timestamp }) - assertEquals(0.4, first?.randomTime) - assertEquals(100.0, first?.videoDuration) - assertEquals(first, second) - assertEquals(1, remote.brandingCalls) - } - - @Test - fun `rejects invalid video id without remote call`() = runBlocking { - val remote = FakeDeArrowRemote() - val service = DeArrowService(FakeCacheService(), remote) - assertNull(service.get("invalid")) - assertEquals(0, remote.brandingCalls) - } -} - -private class FakeDeArrowRemote : DeArrowRemote { - var brandingCalls = 0 - - override suspend fun branding(videoId: String): String { - brandingCalls += 1 - return BRANDING - } - - override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray = byteArrayOf(1, 2, 3) - - companion object { - private const val BRANDING = """{"titles":[{"title":"Rejected title","votes":-2,"locked":false,"original":false,"UUID":"rejected"},{"title":"Clear title","votes":2,"locked":false,"original":false,"UUID":"accepted"},{"title":"Original title","votes":0,"locked":false,"original":true,"UUID":"original"}],"thumbnails":[{"timestamp":8.0,"votes":-1,"locked":false,"original":false,"UUID":"rejected-thumb"},{"timestamp":12.5,"votes":1,"locked":false,"original":false,"UUID":"accepted-thumb"},{"votes":0,"locked":false,"original":true,"UUID":"original-thumb"}],"videoDuration":100,"randomTime":0.4}""" - } -} diff --git a/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt b/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt deleted file mode 100644 index 6572e686..00000000 --- a/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt +++ /dev/null @@ -1,44 +0,0 @@ -package dev.typetype.server - -import dev.typetype.server.services.withExtractionRetry -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test - -class ExtractionResilienceTest { - @Test - fun `returns result without retries when block succeeds`() = runBlocking { - var calls = 0 - val result = withExtractionRetry(attempts = 3, initialDelayMs = 1) { - calls += 1 - "ok" - } - assertEquals("ok", result) - assertEquals(1, calls) - } - - @Test - fun `retries retriable error then succeeds`() = runBlocking { - var calls = 0 - val result = withExtractionRetry(attempts = 3, initialDelayMs = 1) { - calls += 1 - if (calls < 3) throw RuntimeException("temporary") - "ok" - } - assertEquals("ok", result) - assertEquals(3, calls) - } - - @Test - fun `does not retry illegal argument exception`() = runBlocking { - var calls = 0 - val error = runCatching { - withExtractionRetry(attempts = 3, initialDelayMs = 1) { - calls += 1 - throw IllegalArgumentException("bad input") - } - }.exceptionOrNull() - assertEquals(IllegalArgumentException::class.java, error?.javaClass) - assertEquals(1, calls) - } -} diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index 2a54a629..9ac771dd 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -14,6 +14,7 @@ import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.net.InetAddress @@ -41,6 +42,32 @@ class HlsManifestServiceCacheTest { assertEquals(1, calls) } + @Test + fun `youtube live manifests are fetched again instead of using the cached playlist`() = runTest { + var calls = 0 + val client = proxyTestClient(Interceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:$calls\nsegment.ts" + .toResponseBody("application/vnd.apple.mpegurl".toMediaType()), + ) + .build() + }) + val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) + val url = "https://manifest.googlevideo.com/api/manifest/hls_variant/source/yt_live_broadcast/id/live-video/index.m3u8" + + val first = service.hlsManifest(url) + val second = service.hlsManifest(url) + + assertEquals(2, calls) + assertNotEquals(first, second) + } + @Test fun `provider manifests are fetched again instead of serving signed cache entries`() = runTest { var calls = 0 diff --git a/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt index bfb280ee..c15385b9 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt @@ -29,7 +29,8 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class ManifestHlsPublicTokenRoutesTest { - private val tokenService = PublicHlsManifestTokenService("test-secret") + private var now = 1_000L + private val tokenService = PublicHlsManifestTokenService("test-secret", nowMillis = { now }) private val cache = InMemoryPublicHlsCache() @Test @@ -49,6 +50,15 @@ class ManifestHlsPublicTokenRoutesTest { assertEquals(2, Regex("hls-manifest\\?token=").findAll(body).count()) assertTrue(body.contains("../proxy?url=")) assertFalse(body.contains("hls-manifest?url=")) + + val childToken = Regex("hls-manifest\\?token=([A-Za-z0-9._-]+)") + .find(body)?.groupValues?.get(1) + assertTrue(childToken != null) + now += 30 * 60 * 1_000L + + val childResponse = client.get("/streams/hls-manifest?token=$childToken") + + assertEquals(HttpStatusCode.OK, childResponse.status) } @Test diff --git a/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt new file mode 100644 index 00000000..bdb14538 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt @@ -0,0 +1,31 @@ +package dev.typetype.server + +import dev.typetype.server.portability.PortabilityCategory +import dev.typetype.server.portability.PortabilityInputFactory +import dev.typetype.server.portability.PortabilityLimits +import dev.typetype.server.portability.PortabilitySpool +import dev.typetype.server.portability.YoutubeTakeoutPortabilityAdapter +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import java.nio.file.Files +import java.nio.file.Path + +@EnabledIfEnvironmentVariable(named = "PORTABILITY_REPRO_JSON", matches = ".+") +class PortabilityJsonReproductionTest { + @Test + fun `public My Activity JSON reaches preview without account writes`() { + val path = Path.of(System.getenv("PORTABILITY_REPRO_JSON")) + require(Files.size(path) <= PortabilityLimits.MAX_UPLOAD_BYTES) + val input = PortabilityInputFactory.create(path, "watch-history.json", "application/json") + val spool = PortabilitySpool.create(path.parent ?: Path.of("/tmp")) + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + val counts = spool.counts() + assertTrue((counts[PortabilityCategory.HISTORY] ?: 0L) > 0L, counts.toString()) + println("JSON preview counts: ${counts.filterKeys { it in setOf(PortabilityCategory.HISTORY, PortabilityCategory.FAVORITES, PortabilityCategory.SUBSCRIPTIONS) }}") + } finally { + spool.delete() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt new file mode 100644 index 00000000..27fe8b0b --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt @@ -0,0 +1,91 @@ +package dev.typetype.server + +import dev.typetype.server.routes.settingsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SettingsService +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SettingsAutoplayOnOpenRoutesTest { + private val service = SettingsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() { TestDatabase.setup() } + } + + @BeforeEach + fun clean() { TestDatabase.truncateAll() } + + @Test + fun `autoplay on open defaults independently from autoplay next`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + assertEquals(HttpStatusCode.OK, response.status) + val body = response.bodyAsText() + assertTrue(body.contains("\"autoplay\":true")) + assertTrue(body.contains("\"autoplayOnOpen\":true")) + assertTrue(body.contains("\"autoplayCountdownSeconds\":10")) + } + + @Test + fun `autoplay next can be disabled while autoplay on open stays enabled`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"autoplay":false,"autoplayOnOpen":true}""") + } + val body = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText() + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"autoplay\":false")) + assertTrue(body.contains("\"autoplayOnOpen\":true")) + } + + @Test + fun `autoplay countdown is persisted with safe bounds`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"autoplayCountdownSeconds":120}""") + } + val body = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText() + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"autoplayCountdownSeconds\":60")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index 143ff781..c3d19480 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -27,6 +27,7 @@ class StreamRoutesDeliveryModeTest { private val sabrService: StreamService = mockk() private val nicoNicoService: StreamService = mockk() private val bilibiliService: StreamService = mockk() + private val liveHlsService: StreamService = mockk() @Test fun `removed classic endpoints are not registered`() = testApplication { @@ -42,10 +43,10 @@ class StreamRoutesDeliveryModeTest { coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) var sabrFilterCalled = false application { - installRoutes { _, data -> + installRoutes(sabrFilter = { _, data -> sabrFilterCalled = true data - } + }) } val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") @@ -70,12 +71,12 @@ class StreamRoutesDeliveryModeTest { val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertEquals(HttpStatusCode.BadGateway, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } @Test - fun `sabr endpoint removes direct live manifest`() = testApplication { + fun `sabr endpoint exposes only HLS for live streams`() = testApplication { val live = sabrResponse().copy( hlsUrl = "/streams/hls-manifest?url=live", isLive = true, @@ -88,8 +89,39 @@ class StreamRoutesDeliveryModeTest { val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("\"hlsUrl\":\"\"")) - assertFalse(response.bodyAsText().contains("hls-manifest")) + assertTrue(response.bodyAsText().contains("\"hlsUrl\":\"/streams/hls-manifest?url=live\"")) + assertTrue(response.bodyAsText().contains("\"videoStreams\":[]")) + assertTrue(response.bodyAsText().contains("\"videoOnlyStreams\":[]")) + assertTrue(response.bodyAsText().contains("\"audioStreams\":[]")) + assertFalse(response.bodyAsText().contains("\"deliveryMethod\":\"sabr\"")) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + } + + @Test + fun knownLiveEndpointReturnsOnlyHls() = testApplication { + val live = mixedResponse().copy( + hlsUrl = "/streams/hls-manifest?url=live", + dashMpdUrl = "/manifest/live.mpd", + isLive = true, + isLiveContent = true, + hasLiveManifest = true, + ) + coEvery { liveHlsService.getStreamInfo(any()) } returns ExtractionResult.Success(live) + application { installRoutes(liveHls = liveHlsService) } + + val response = client.get("/streams/youtube/live?url=$VIDEO_URL") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?url=live\"")) + assertTrue(body.contains("\"dashMpdUrl\":\"\"")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertTrue(body.contains("\"hasLiveManifest\":true")) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(VIDEO_URL) } + coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } } @Test @@ -148,6 +180,7 @@ class StreamRoutesDeliveryModeTest { private fun Application.installRoutes( providerMediaHandleService: ProviderMediaHandleService? = null, sabrFilter: suspend (String, StreamResponse) -> StreamResponse = { _, data -> data }, + liveHls: StreamService = liveHlsService, ) { install(ContentNegotiation) { json() } routing { @@ -155,6 +188,7 @@ class StreamRoutesDeliveryModeTest { streamService = sabrService, nicoNicoStreamService = nicoNicoService, bilibiliStreamService = bilibiliService, + youtubeLiveHlsStreamService = liveHls, providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrFilter, ) diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt index d2fa50ba..b39af026 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt @@ -63,7 +63,7 @@ class StreamRoutesSignedHlsTest { } @Test - fun `anonymous sabr removes live hls url`() = testApplication { + fun `anonymous sabr live response keeps only signed HLS`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( publicHlsStream().copy( videoOnlyStreams = listOf( @@ -83,9 +83,13 @@ class StreamRoutesSignedHlsTest { val body = response.bodyAsText() assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"hlsUrl\":\"\"")) - assertFalse(body.contains("hls-manifest")) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?token=")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) assertFalse(body.contains(MANIFEST_URL)) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) } @Test diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt deleted file mode 100644 index d9d6cac6..00000000 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ /dev/null @@ -1,144 +0,0 @@ -package dev.typetype.server - -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.routes.streamRoutes -import dev.typetype.server.services.StreamService -import io.ktor.client.request.get -import io.ktor.client.statement.bodyAsText -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.routing.routing -import io.ktor.server.testing.testApplication -import io.mockk.coEvery -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test - -class StreamRoutesTest { - - private val streamService: StreamService = mockk() - - @Test - fun `GET streams without url returns 400`() = testApplication { - application { - install(ContentNegotiation) { json() } - routing { streamRoutes(streamService) } - } - val response = client.get("/streams/youtube/sabr") - assertEquals(HttpStatusCode.BadRequest, response.status) - } - - @Test - fun `GET streams with valid url returns 200 on Success`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Success(sabrResponse()) - application { - install(ContentNegotiation) { json() } - routing { streamRoutes(streamService) } - } - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") - assertEquals(HttpStatusCode.OK, response.status) - assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) - } - - @Test - fun `GET streams returns 422 on Failure`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Failure("Premieres in 200 days", "scheduled_premiere") - application { - install(ContentNegotiation) { json() } - routing { streamRoutes(streamService) } - } - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"scheduled_premiere\"")) - } - - @Test - fun `GET streams returns 400 on BadRequest`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.BadRequest("This video is a paid video", "paid_content") - application { - install(ContentNegotiation) { json() } - routing { streamRoutes(streamService) } - } - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=paid") - assertEquals(HttpStatusCode.BadRequest, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) - } - - @Test - fun `GET restricted YouTube streams asks guests to connect YouTube`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.BadRequest("Sign in to confirm your age", "age_restricted") - application { - install(ContentNegotiation) { json() } - routing { - streamRoutes( - streamService = streamService, - youtubeSessionSabrStreamInfo = { _, _ -> null }, - ) - } - } - - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=restricted") - - assertEquals(HttpStatusCode.BadRequest, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) - } - - @Test - fun `GET members-only metadata asks guests to connect YouTube`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Success(sabrResponse().copy(requiresMembership = true)) - application { - install(ContentNegotiation) { json() } - routing { - streamRoutes( - streamService = streamService, - youtubeSessionSabrStreamInfo = { _, _ -> null }, - ) - } - } - - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=members") - - assertEquals(HttpStatusCode.BadRequest, response.status) - assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) - } - - @Test - fun `GET sabr streams returns 422 when final response has no playable source`() = testApplication { - coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Success(testStreamResponse()) - application { - install(ContentNegotiation) { json() } - routing { - streamRoutes(streamService) { _, data -> - data.copy( - videoStreams = emptyList(), - videoOnlyStreams = emptyList(), - audioStreams = emptyList(), - hlsUrl = "", - dashMpdUrl = "", - ) - } - } - } - - val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") - val body = response.bodyAsText() - - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) - assertTrue(body.contains("\"code\":\"no_playable_streams\"")) - } - - private fun sabrResponse() = testStreamResponse( - videoOnlyStreams = listOf(testVideoStream().copy(deliveryMethod = "sabr")), - audioStreams = listOf(testAudioStream(deliveryMethod = "sabr")), - ) -} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt index dc0b9ece..844b4e7c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt @@ -6,7 +6,7 @@ import dev.typetype.server.routes.subscriptionFeedRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.forms.MultiPartFormDataContent @@ -51,7 +51,7 @@ class SubscriptionFeedCacheInvalidationPipePipeTest { fun clean() = runBlocking { TestDatabase.truncateAll() cache.clear() - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt index 7e06f65d..9e83b903 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt @@ -4,7 +4,7 @@ import dev.typetype.server.services.PipePipeBackupPersisterService import dev.typetype.server.services.PipePipeBackupSnapshotItem import dev.typetype.server.services.PipePipeBackupSubscriptionItem import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedCacheKeys import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService @@ -29,7 +29,7 @@ class SubscriptionFeedCacheInvalidationPipePipeUnitTest { TestDatabase.truncateAll() cache.clear() val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt index 7cabdca9..83c8bfc6 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.FavoritesService import dev.typetype.server.services.HistoryService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedCacheKeys import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService @@ -36,7 +36,7 @@ class SubscriptionFeedCacheInvalidationTakeoutUnitTest { TestDatabase.truncateAll() cache.clear() val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index de4341ef..f2cb6e0d 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -6,11 +6,13 @@ import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.portability.PortabilityMediaAvatarPropagation import dev.typetype.server.services.SubscriptionAvatarRepairer import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.HistoryService import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -104,6 +106,65 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(WATCH_AVATAR_URL, repaired.last().avatarUrl) } + @Test + fun `propagates subscription avatars to imported history and collections`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = WATCH_AVATAR_URL + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "", watchedAt = 1L) + addWatchLater(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "") + addFavorite(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "") + + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(TEST_USER_ID) } + + assertEquals(WATCH_AVATAR_URL, storedHistoryAvatar()) + assertEquals(WATCH_AVATAR_URL, storedWatchLaterAvatar()) + assertEquals(WATCH_AVATAR_URL, storedFavoriteAvatar()) + } + + @Test + fun `import propagation repairs subscription avatars from media first`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = "" + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL, watchedAt = 1L) + + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(TEST_USER_ID) } + + assertEquals(WATCH_AVATAR_URL, storedAvatar(WATCH_CHANNEL_URL)) + assertEquals(WATCH_AVATAR_URL, storedHistoryAvatar()) + } + + @Test + fun `history reads subscription avatars without waiting for a repair write`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = WATCH_AVATAR_URL + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "", watchedAt = 1L) + + val item = HistoryService().search(TEST_USER_ID, null, null, null, 1, 0).first.single() + + assertEquals(WATCH_AVATAR_URL, item.channelAvatar) + } + private suspend fun addSubscription(channelUrl: String): Unit { service.add(TEST_USER_ID, SubscriptionItem(channelUrl = channelUrl, name = "Channel", avatarUrl = "")) } @@ -114,6 +175,24 @@ class SubscriptionsAvatarRepairServiceTest { }.single()[SubscriptionsTable.avatarUrl] } + private suspend fun storedHistoryAvatar(): String = DatabaseFactory.query { + HistoryTable.selectAll().where { + (HistoryTable.userId eq TEST_USER_ID) and (HistoryTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[HistoryTable.channelAvatar] + } + + private suspend fun storedWatchLaterAvatar(): String = DatabaseFactory.query { + WatchLaterTable.selectAll().where { + (WatchLaterTable.userId eq TEST_USER_ID) and (WatchLaterTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[WatchLaterTable.channelAvatar] + } + + private suspend fun storedFavoriteAvatar(): String = DatabaseFactory.query { + FavoritesTable.selectAll().where { + (FavoritesTable.userId eq TEST_USER_ID) and (FavoritesTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[FavoritesTable.channelAvatar] + } + private suspend fun addWatchLater(channelUrl: String, avatarUrl: String): Unit = DatabaseFactory.query { WatchLaterTable.insert { it[userId] = TEST_USER_ID; it[url] = "https://video.test/watch"; it[title] = "Video"; it[thumbnail] = "" diff --git a/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt b/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt index 2af7e2b3..0ef78652 100644 --- a/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt @@ -138,7 +138,61 @@ class UserVideoMetadataRepairRoutesTest { repair.schedulePlaylists(scope, TEST_USER_ID) delay(100) - assertEquals(1, attempts.get()) + assertEquals(2, attempts.get()) + } finally { + scope.cancel() + } + } + + @Test + fun `repair continues across more than one metadata batch`() = runBlocking { + val attempts = AtomicInteger() + val repair = UserVideoMetadataRepairService(VideoMetadataResolver(alwaysSuccessfulStreamService(attempts))) + val playlist = playlists.create(TEST_USER_ID, PlaylistItem(name = "Imported", description = "")) + repeat(12) { index -> + val url = "https://www.youtube.com/watch?v=video$index" + playlists.addVideo(TEST_USER_ID, playlist.id, fallbackVideo(url)) + } + val scope = CoroutineScope(SupervisorJob()) + + try { + repair.schedulePlaylists(scope, TEST_USER_ID) + withTimeout(10_000) { + while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.count { it.title.startsWith("Resolved") } != 12) { + delay(20) + } + } + assertEquals(12, attempts.get()) + } finally { + scope.cancel() + } + } + + @Test + fun `transient metadata failure is retried once`() = runBlocking { + val attempts = AtomicInteger() + val streamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + return if (attempts.incrementAndGet() == 1) { + ExtractionResult.Failure("Provider blocked", "provider_access_blocked") + } else { + ExtractionResult.Success(stream(url)) + } + } + } + val repair = UserVideoMetadataRepairService(VideoMetadataResolver(streamService)) + val playlist = playlists.create(TEST_USER_ID, PlaylistItem(name = "Imported", description = "")) + playlists.addVideo(TEST_USER_ID, playlist.id, fallbackVideo(VIDEO_URL)) + val scope = CoroutineScope(SupervisorJob()) + + try { + repair.schedulePlaylists(scope, TEST_USER_ID) + withTimeout(10_000) { + while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.single()?.title != "Resolved abc123") { + delay(20) + } + } + assertEquals(2, attempts.get()) } finally { scope.cancel() } @@ -146,11 +200,18 @@ class UserVideoMetadataRepairRoutesTest { private fun fallbackVideo(url: String): PlaylistVideoItem = PlaylistVideoItem( url = url, - title = "YouTube video abc123", - thumbnail = "https://i.ytimg.com/vi/abc123/hqdefault.jpg", + title = "YouTube video ${url.substringAfterLast('=')}", + thumbnail = "https://i.ytimg.com/vi/${url.substringAfterLast('=')}/hqdefault.jpg", duration = 0L, ) + private fun alwaysSuccessfulStreamService(attempts: AtomicInteger): StreamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + attempts.incrementAndGet() + return ExtractionResult.Success(stream(url)) + } + } + private fun fakeStreamService(started: CompletableDeferred, release: CompletableDeferred): StreamService = object : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult { started.complete(Unit) diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt new file mode 100644 index 00000000..8f75bb3e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt @@ -0,0 +1,29 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutDateParser +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test + +class YoutubeTakeoutDateParserLocaleTest { + @Test + fun `parses CLDR periods used by Chinese Japanese and Korean Takeout exports`() { + assertEquals( + 1_788_576_327_000L, + YoutubeTakeoutDateParser.parseEpochMillis("2026年9月4日 晚上9:45:27 CST"), + ) + assertEquals( + 1_788_525_927_000L, + YoutubeTakeoutDateParser.parseEpochMillis("2026年9月4日 午後9:45:27 JST"), + ) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026년 9월 4일 오후 9:45:27 KST")) + } + + @Test + fun `parses year first and localized day period variants`() { + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026/09/16 18:02:08 CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026-09-16, 18:02:08 CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("4 سبتمبر 2026, 9:45:27 مساءً CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("4 सितंबर 2026, 9:45:27 शाम CEST")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt new file mode 100644 index 00000000..a5909279 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt @@ -0,0 +1,46 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutParserService +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutZipScannerSchemaTest { + @Test + fun `parse recognizes csv schemas without translated paths`() { + val archive = Files.createTempFile("yt-takeout-structure-", ".zip") + try { + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/section/data-a.csv", + "column-a,column-b,column-c\nUC1234567890,Channel,https://www.youtube.com/channel/UC1234567890\n", + ) + output.entry( + "Takeout/YouTube/section/data-b.csv", + "column-a,column-b\nPL123456,My list\n", + ) + output.entry( + "Takeout/YouTube/section/My list.csv", + "column-a,column-b,column-c\nvideo000001,Video,2026-09-16T18:02:08Z\n", + ) + } + + val parsed = YoutubeTakeoutParserService().parse(archive) + + assertEquals(1, parsed.subscriptions.size) + assertEquals(1, parsed.playlists.size) + assertEquals(1, parsed.playlistItems["My list"]?.size) + } finally { + Files.deleteIfExists(archive) + } + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt new file mode 100644 index 00000000..0dbfccb6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutJsonSchemaDetectionTest { + @TempDir + lateinit var directory: Path + + @Test + fun `detects an activity array with an unknown localized filename`() { + val archive = directory.resolve("takeout-json.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("Takeout/YouTube/donnees.json")) + output.write( + """[{"title":"動画を視聴しました","titleUrl":"https://www.youtube.com/watch?v=unknown01","time":"2026-09-16T18:02:08Z"}]""".toByteArray(), + ) + output.closeEntry() + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, YoutubeTakeoutPortabilityAdapter().detect(input)?.format) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + } finally { + spool.delete() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt deleted file mode 100644 index 84fb2371..00000000 --- a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt +++ /dev/null @@ -1,71 +0,0 @@ -package dev.typetype.server.portability - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.io.TempDir -import java.nio.file.Files -import java.nio.file.Path -import java.util.zip.ZipEntry -import java.util.zip.ZipOutputStream - -class YoutubeTakeoutPortabilityAdapterTest { - @TempDir - lateinit var directory: Path - - @Test - fun `adapter streams takeout categories and keeps playlist order`() { - val archive = directory.resolve("takeout.zip") - ZipOutputStream(Files.newOutputStream(archive)).use { output -> - output.entry( - "Takeout/YouTube and YouTube Music/subscriptions/subscriptions.csv", - "Channel Id,Channel Url,Channel Title\nUC123456789012,https://youtube.com/channel/UC123456789012,Channel\n", - ) - output.entry( - "Takeout/YouTube and YouTube Music/playlists/playlists.csv", - "Playlist ID,Playlist Title\nPL123456789,Imported\n", - ) - output.entry( - "Takeout/YouTube and YouTube Music/playlists/Videos de Imported.csv", - "Video ID,Video Title,Video Added Timestamp\nvideo000001,First,2026-01-02T00:00:00Z\nvideo000002,Second,2026-01-01T00:00:00Z\n", - ) - output.entry( - "Takeout/YouTube and YouTube Music/playlists/Watch later.csv", - "Video ID,Video Title\nwatch000001,Later\n", - ) - output.entry( - "Takeout/YouTube and YouTube Music/playlists/Liked videos.csv", - "Video ID,Video Title\nliked000001,Liked\n", - ) - output.entry( - "Takeout/My Activity/YouTube/watch-history.html", - "You watched Seen
1 Jan 2026, 12:00:00 CET
", - ) - } - val input = PortabilityInputFactory.create(archive, "takeout.zip", "application/zip") - val spool = PortabilitySpool.create(directory) - val adapter = YoutubeTakeoutPortabilityAdapter() - - assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) - adapter.decode(input, spool) - - assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) - assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) - assertEquals(3L, spool.counts()[PortabilityCategory.PLAYLISTS]) - assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) - assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) - val positions = mutableListOf() - spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> - positions += (record as PortabilityPlaylistVideo).position - } - assertEquals(listOf(0, 1), positions) - assertTrue(spool.issues().isEmpty()) - spool.delete() - } - - private fun ZipOutputStream.entry(name: String, value: String) { - putNextEntry(ZipEntry(name)) - write(value.toByteArray()) - closeEntry() - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt index 32501d89..c0e3840d 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt @@ -1,7 +1,9 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.delay @@ -70,11 +72,30 @@ class AuthenticatedSabrTimeoutTest { mockk(relaxed = true), timeoutMs = 20L, ) + coEvery { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } returns Unit val result = withTimeout(1_000L) { service.getStreamInfo(USER_ID, URL) } - assertTrue(result is ExtractionResult.Failure) - assertEquals(AuthenticatedSabrPolicy.TIMEOUT_CODE, (result as ExtractionResult.Failure).code) + assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) + coVerify { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } + } + + @Test + fun `authenticated SABR probe timeout marks session needs reconnect`() = runTest { + val metadata = mockk() + val info = mockk() + val stream = mockk(relaxed = true) + coEvery { metadata.getStreamInfo(USER_ID, URL) } returns ExtractionResult.Success(stream) + coEvery { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } returns Unit + coEvery { info.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.TimedOut + val service = YoutubeSessionSabrStreamService(metadata, info) + + val result = service.getStreamInfo(USER_ID, URL) + + assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) + coVerify { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } } private fun service( diff --git a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt deleted file mode 100644 index c5c0c8f0..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt +++ /dev/null @@ -1,176 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.testStreamResponse -import dev.typetype.server.models.ExtractionResult -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo - -class SabrFallbackStreamServiceTest { - @Test - fun `enriches empty youtube extraction with token sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) - assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) - assertEquals("fr-FR.4", enriched.originalAudioTrackId) - assertTrue(enriched.audioStreams.single().sabrSessionUrl?.contains("audioTrackId=fr-FR.4") == true) - } - - @Test - fun `replaces classic extraction with prepared sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) - assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } - coVerify(exactly = 0) { tokenSessionClient.fetchPlaybackSession(any()) } - } - - @Test - fun `enriches a live hls extraction with prepared sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse( - videoOnlyStreams = emptyList(), - audioStreams = emptyList(), - hlsUrl = LIVE_HLS_URL, - ).copy(isLive = true, isLiveContent = true, hasLiveManifest = true, streamType = "live_stream") - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(LIVE_HLS_URL, enriched.hlsUrl) - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - } - - @Test - fun `recovers youtube extraction failure with token playback session`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("MWEB player response is not valid") - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val response = (result as ExtractionResult.Success).data - assertEquals(VIDEO_ID, response.id) - assertEquals("Fallback title", response.title) - assertEquals("Fallback channel", response.uploaderName) - assertEquals("https://example.com/avatar.jpg", response.uploaderAvatarUrl) - assertEquals(3554L, response.duration) - assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), response.audioStreams.map { it.itag }) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } - } - - @Test - fun `preserves token hls when recovering a live stream`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("blocked") - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession( - isLive = true, - hlsUrl = LIVE_HLS_URL, - ) - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val response = (result as ExtractionResult.Success).data - assertEquals(LIVE_HLS_URL, response.hlsUrl) - assertTrue(response.isLive) - assertTrue(response.hasLiveManifest) - } - - private fun preparedInfo(): SabrPreparedInfo { - val video = mockk(relaxed = true) - every { video.isVideo } returns true - every { video.isAudio } returns false - every { video.itag } returns 137 - every { video.mimeType } returns "video/mp4; codecs=\"avc1.640028\"" - every { video.qualityLabel } returns "1080p" - every { video.width } returns 1920 - every { video.height } returns 1080 - every { video.bitrate } returns 4_000_000 - val audio = mockk(relaxed = true) - every { audio.isVideo } returns false - every { audio.isAudio } returns true - every { audio.itag } returns 140 - every { audio.mimeType } returns "audio/mp4; codecs=\"mp4a.40.2\"" - every { audio.audioQuality } returns "AUDIO_QUALITY_MEDIUM" - every { audio.audioTrackId } returns "fr-FR.4" - every { audio.audioTrackDisplayName } returns "French (original)" - every { audio.isOriginalAudio } returns true - every { audio.isAudioDefault } returns true - every { audio.bitrate } returns 129_000 - val info = mockk() - every { info.formats } returns listOf(video, audio) - return SabrPreparedInfo(info, null) - } - - private fun tokenSession(isLive: Boolean = false, hlsUrl: String = ""): TokenYoutubeSession { - val prepared = preparedInfo() - return TokenYoutubeSession( - info = prepared.info, - token = null, - title = "Fallback title", - author = "Fallback channel", - channelId = "channel-id", - channelAvatarUrl = "https://example.com/avatar.jpg", - description = "Fallback description", - durationMs = 3_554_000L, - viewCount = 42L, - thumbnailUrl = "https://example.com/thumb.jpg", - tags = listOf("tag"), - isLive = isLive, - isLiveContent = isLive, - hlsUrl = hlsUrl, - ) - } - - private companion object { - const val VIDEO_ID = "Vj6ReOur1Kk" - const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" - const val LIVE_HLS_URL = "https://example.com/live.m3u8" - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt index 2ce89bfa..0270fcf6 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt @@ -1,27 +1,175 @@ package dev.typetype.server.services +import dev.typetype.server.testStreamResponse +import dev.typetype.server.models.ExtractionResult +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every import io.mockk.mockk +import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo -class SabrInitializationPolicyTest { +class SabrFallbackStreamServiceTest { @Test - fun `audio only warmup excludes video initialization`() { - val audio = mockk() - val video = mockk() + fun `enriches empty youtube extraction with token sabr formats`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - assertEquals(listOf(audio), SabrInitializationPolicy.warmFormats(true, audio, video)) - assertEquals(listOf(video, audio), SabrInitializationPolicy.warmFormats(false, audio, video)) + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) + assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) + assertEquals("fr-FR.4", enriched.originalAudioTrackId) + assertTrue(enriched.audioStreams.single().sabrSessionUrl?.contains("audioTrackId=fr-FR.4") == true) + } + + @Test + fun `replaces classic extraction with prepared sabr formats`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) + assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) + coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { tokenSessionClient.fetchPlaybackSession(any()) } + } + + @Test + fun `keeps live hls extraction on hls without sabr fallback`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse( + videoOnlyStreams = emptyList(), + audioStreams = emptyList(), + hlsUrl = LIVE_HLS_URL, + ).copy(isLive = true, isLiveContent = true, hasLiveManifest = true, streamType = "live_stream") + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(LIVE_HLS_URL, enriched.hlsUrl) + assertTrue(enriched.videoOnlyStreams.isEmpty()) + assertTrue(enriched.audioStreams.isEmpty()) + } + + @Test + fun `recovers youtube extraction failure with token playback session`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("MWEB player response is not valid") + coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val response = (result as ExtractionResult.Success).data + assertEquals(VIDEO_ID, response.id) + assertEquals("Fallback title", response.title) + assertEquals("Fallback channel", response.uploaderName) + assertEquals("https://example.com/avatar.jpg", response.uploaderAvatarUrl) + assertEquals(3554L, response.duration) + assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), response.audioStreams.map { it.itag }) + coVerify(exactly = 1) { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } } @Test - fun `audio only fetch never waits for video initialization`() { - assertFalse(SabrInitializationPolicy.requiresVideoFirst(true, true, 60_000L)) - assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, true, 0L)) - assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, false, 60_000L)) - assertTrue(SabrInitializationPolicy.requiresVideoFirst(false, true, 60_000L)) + fun `preserves token hls when recovering a live stream`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("blocked") + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession( + isLive = true, + hlsUrl = LIVE_HLS_URL, + ) + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val response = (result as ExtractionResult.Success).data + assertEquals(LIVE_HLS_URL, response.hlsUrl) + assertTrue(response.isLive) + assertTrue(response.hasLiveManifest) + } + + private fun preparedInfo(): SabrPreparedInfo { + val video = mockk(relaxed = true) + every { video.isVideo } returns true + every { video.isAudio } returns false + every { video.itag } returns 137 + every { video.mimeType } returns "video/mp4; codecs=\"avc1.640028\"" + every { video.qualityLabel } returns "1080p" + every { video.width } returns 1920 + every { video.height } returns 1080 + every { video.bitrate } returns 4_000_000 + val audio = mockk(relaxed = true) + every { audio.isVideo } returns false + every { audio.isAudio } returns true + every { audio.itag } returns 140 + every { audio.mimeType } returns "audio/mp4; codecs=\"mp4a.40.2\"" + every { audio.audioQuality } returns "AUDIO_QUALITY_MEDIUM" + every { audio.audioTrackId } returns "fr-FR.4" + every { audio.audioTrackDisplayName } returns "French (original)" + every { audio.isOriginalAudio } returns true + every { audio.isAudioDefault } returns true + every { audio.bitrate } returns 129_000 + val info = mockk() + every { info.formats } returns listOf(video, audio) + return SabrPreparedInfo(info, null) + } + + private fun tokenSession(isLive: Boolean = false, hlsUrl: String = ""): TokenYoutubeSession { + val prepared = preparedInfo() + return TokenYoutubeSession( + info = prepared.info, + token = null, + title = "Fallback title", + author = "Fallback channel", + channelId = "channel-id", + channelAvatarUrl = "https://example.com/avatar.jpg", + description = "Fallback description", + durationMs = 3_554_000L, + viewCount = 42L, + thumbnailUrl = "https://example.com/thumb.jpg", + tags = listOf("tag"), + isLive = isLive, + isLiveContent = isLive, + hlsUrl = hlsUrl, + ) + } + + private companion object { + const val VIDEO_ID = "Vj6ReOur1Kk" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val LIVE_HLS_URL = "https://example.com/live.m3u8" } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt deleted file mode 100644 index 94fe8c6a..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt +++ /dev/null @@ -1,300 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant - -class SabrLivePlaybackTest { - @AfterEach - fun clearDemands(): Unit = SabrSegmentDemandTracker.clearAll() - - @Test - fun `active live starts from media that the session can retrieve`() { - val fixture = fixture() - listOf(fixture.audio, fixture.video).forEach { format -> - fixture.holder.observeMediaSegment(mockk { - every { header } returns mockk { - every { isInitSegment } returns false - every { itag } returns format.itag - every { sequenceNumber } returns 199 - every { startMs } returns 1_003_000L - every { durationMs } returns 2_000L - } - }) - } - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertTrue(live.active) - assertFalse(live.postLiveDvr) - assertEquals(1_005_000L, live.seekableEndMs) - assertEquals(1_003_000L, fixture.holder.resolvePlaybackStartMs(0L)) - assertEquals(1_005_000L, fixture.holder.resolvePlaybackStartMs(1_100_000L)) - } - - @Test - fun `reported live head wins over sequence based duration estimates`() { - val fixture = fixture() - every { fixture.state.getSegmentEndMs(fixture.video, 200) } returns 9_011_868_000L - every { fixture.state.getBufferedEndMs(fixture.video) } returns 9_011_868_000L - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertEquals(1_005_000L, live.headTimeMs) - assertEquals(1_005_000L, live.seekableEndMs) - } - - @Test - fun `active live maps time from an observed sabr segment for every codec`() { - val fixture = fixture() - every { fixture.state.getMaxSegment(fixture.video) } returns 180 - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns 2_000L - } - val segment = mockk { - every { this@mockk.header } returns header - } - fixture.holder.observeMediaSegment(segment) - - assertEquals(200L, requireNotNull(fixture.holder.livePlaybackSnapshot()).headSequence) - assertEquals(segment, fixture.holder.observedMediaSegment(fixture.video)) - assertEquals(965_000L, segment.header.startMs) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_001L)) - assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) - assertEquals(179, fixture.holder.playbackStartSequence(fixture.video, 964_999L)) - assertEquals(200, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) - every { fixture.state.getMaxSegment(fixture.video) } returns 200 - val nearHead = mockk { - every { this@mockk.header } returns mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 199 - every { startMs } returns 1_003_000L - every { durationMs } returns 2_000L - } - } - fixture.holder.observeMediaSegment(nearHead) - assertEquals(199, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) - } - - @Test - fun `active live derives missing segment duration from the sabr head`() { - val fixture = fixture() - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns -1L - } - val segment = mockk { - every { this@mockk.header } returns header - } - fixture.holder.observeMediaSegment(segment) - - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) - } - - @Test - fun `active live maps time before the head sequence is reported`() { - val fixture = fixture() - every { fixture.session.liveHeadSequenceNumber } returns 0L - every { fixture.state.liveHeadSequenceNumber } returns 0L - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns 2_000L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - } - - @Test - fun `live duration tolerates millisecond drift between media and head timestamps`() { - val fixture = fixture() - every { fixture.state.liveHeadTimeMs } returns 1_004_999L - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns -1L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - - assertEquals(2_000L, fixture.holder.playbackSegmentDurationMs(fixture.video, 180)) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - } - - @Test - fun `only the next live media segments wait for production`() { - val fixture = fixture() - - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 202))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 101))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 102))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 203))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 103))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 100))) - } - - @Test - fun `advertised live head waits until its media is complete`() { - val fixture = fixture() - every { fixture.state.liveHeadTimeMs } returns 1_025_000L - fixture.holder.observeMediaSegment(mockk { - every { header } returns mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 200 - every { startMs } returns 1_002_000L - every { durationMs } returns 2_000L - } - }) - - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) - } - - @Test - fun `in flight live media waits beyond the reported head`() { - val fixture = fixture() - val request = SabrSegmentRequest.media(fixture.audio, 103) - every { fixture.session.getReadableSegment(request) } returns mockk() - - assertTrue(fixture.holder.isFutureLiveRequest(request)) - } - - @Test - fun `live head can advance beyond the last complete media segment`() { - val fixture = fixture() - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 198 - every { startMs } returns 998_000L - every { durationMs } returns 2_000L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - every { fixture.state.liveHeadTimeMs } returns 1_025_000L - fixture.holder.setLastServedSequence(fixture.video.itag, 200) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - assertFalse(fixture.holder.isHistoricalLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - } - - @Test - fun `future live demand remains retryable after repeated responses`() { - val fixture = fixture() - val request = SabrSegmentRequest.media(fixture.video, 201) - fixture.holder.requestSegmentDemand(request, registeredAtMs = 0L) - val identity = requireNotNull(fixture.holder.segmentDemandIdentity(request)) - val result = mockk { - every { segmentCount } returns 2 - every { targetTrackSegmentCount } returns 1 - } - val runtime = SabrPumpRuntime { 20_000L } - val wasFutureLiveRequest = fixture.holder.isFutureLiveRequest(request) - every { fixture.state.getMaxSegment(fixture.video) } returns 201 - - repeat(4) { - assertFalse( - SabrDemandAttemptFinisher.finish( - fixture.holder, - request, - identity, - result, - runtime, - wasFutureLiveRequest, - ), - ) - } - - assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) - assertNull(fixture.holder.terminalFailure()) - assertEquals("299:201", fixture.holder.pendingSegmentDemandSummary()) - } - - @Test - fun `post live dvr is finite instead of an active live edge`() { - val fixture = fixture(postLiveDvr = true) - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertFalse(live.active) - assertTrue(live.postLiveDvr) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - } - - private fun fixture(postLiveDvr: Boolean = false): Fixture { - val audio = format(140, true) - val video = format(299, false) - val state = mockk(relaxed = true) - val session = mockk(relaxed = true) - every { session.streamState } returns state - every { session.isLive } returns !postLiveDvr - every { session.isAtLiveEdge } returns !postLiveDvr - every { session.liveHeadSequenceNumber } returns 200L - every { session.getCachedSegment(any()) } returns null - every { session.getReadableSegment(any()) } returns null - every { state.isLive } returns !postLiveDvr - every { state.isPostLiveDvr } returns postLiveDvr - every { state.liveHeadSequenceNumber } returns 200L - every { state.liveHeadTimeMs } returns 1_005_000L - every { state.getMaxSegment(audio) } returns 100 - every { state.getMaxSegment(video) } returns 200 - every { state.getSegmentEndMs(audio, 100) } returns 1_000_000L - every { state.getSegmentEndMs(video, 200) } returns 1_002_000L - every { state.getBufferedEndMs(audio) } returns 1_000_000L - every { state.getBufferedEndMs(video) } returns 1_002_000L - every { state.getMinBufferedEndMs() } returns 1_000_000L - val holder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - return Fixture(holder, session, state, audio, video) - } - - private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 - return format - } - - private data class Fixture( - val holder: SabrSessionHolder, - val session: YoutubeSabrSession, - val state: YoutubeSabrStreamState, - val audio: YoutubeSabrFormat, - val video: YoutubeSabrFormat, - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt index c4877083..94fe8c6a 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt @@ -1,170 +1,300 @@ package dev.typetype.server.services -import kotlinx.coroutines.runBlocking +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -import java.security.MessageDigest +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrLivePlaybackTest { + @AfterEach + fun clearDemands(): Unit = SabrSegmentDemandTracker.clearAll() -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrLiveProtocolProbeTest { @Test - fun `retrieves every requested live sabr codec`(): Unit = runBlocking { - val videoId = sabrProbeVideoId() - val tokenServiceUrl = sabrProbeTokenServiceUrl() - NewPipeInitializer.init(tokenServiceUrl) - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - try { - val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) - val prepared = SabrInfoFetcher( - tokenClient, - TypetypeTokenYoutubeSessionClient(tokenServiceUrl), - ).fetchInfo(videoId) ?: error("Missing SABR player metadata") - println( - "profile=${prepared.info.profile} clientVersion=${prepared.info.clientVersion} " + - "visitorMatches=${prepared.info.visitorData == prepared.initialToken?.visitorData}" - ) - val audio = prepared.info.formats.first { it.itag == sabrProbeAudioItag() && it.isAudio } - val requestedVideoItags = sabrProbeVideoItags() - val videos = requestedVideoItags.map { itag -> - prepared.info.formats.first { it.itag == itag && it.isVideo } - } - assertEquals(requestedVideoItags, videos.map { it.itag }) - videos.forEach { video -> - probe(store, videoId, prepared, audio, video) - } - } finally { - store.release() + fun `active live starts from media that the session can retrieve`() { + val fixture = fixture() + listOf(fixture.audio, fixture.video).forEach { format -> + fixture.holder.observeMediaSegment(mockk { + every { header } returns mockk { + every { isInitSegment } returns false + every { itag } returns format.itag + every { sequenceNumber } returns 199 + every { startMs } returns 1_003_000L + every { durationMs } returns 2_000L + } + }) } + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertTrue(live.active) + assertFalse(live.postLiveDvr) + assertEquals(1_005_000L, live.seekableEndMs) + assertEquals(1_003_000L, fixture.holder.resolvePlaybackStartMs(0L)) + assertEquals(1_005_000L, fixture.holder.resolvePlaybackStartMs(1_100_000L)) } - private suspend fun probe( - store: SabrSessionStore, - videoId: String, - prepared: SabrPreparedInfo, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - ) { - val label = "${video.codecFamily()}-${video.itag}" - val holder = store.getOrCreate( - videoId = videoId, - userId = "live-protocol-probe-$label", - info = prepared.info, - audioFormat = audio, - videoFormat = video, - initialToken = prepared.initialToken, - startPump = false, - ) - holder.markExpectedLive() - store.ensureWarmed(holder, maxPumps = 8) - val live = requireNotNull(holder.livePlaybackSnapshot()) - val segments = listOfNotNull( - holder.observedMediaSegment(audio), - holder.observedMediaSegment(video), - ) - println( - "$label requests=${holder.session.requestNumber} segments=${segments.size} " + - "headSeq=${live.headSequence} headMs=${live.headTimeMs} startMs=${holder.resolvePlaybackStartMs(0L)}" - ) - println("$label trace=${holder.session.diagnosticTrace}") - assertEquals(setOf(audio.itag, video.itag), segments.map { it.header.itag }.toSet()) - segments.forEach { segment -> - val header = segment.header - val format = if (header.itag == audio.itag) audio else video - val parts = requireNotNull(SabrLiveMediaNormalizer.split(format.mimeType.orEmpty(), segment.data)) { - "$label could not split live ${format.mimeType}" + @Test + fun `reported live head wins over sequence based duration estimates`() { + val fixture = fixture() + every { fixture.state.getSegmentEndMs(fixture.video, 200) } returns 9_011_868_000L + every { fixture.state.getBufferedEndMs(fixture.video) } returns 9_011_868_000L + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertEquals(1_005_000L, live.headTimeMs) + assertEquals(1_005_000L, live.seekableEndMs) + } + + @Test + fun `active live maps time from an observed sabr segment for every codec`() { + val fixture = fixture() + every { fixture.state.getMaxSegment(fixture.video) } returns 180 + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns 2_000L + } + val segment = mockk { + every { this@mockk.header } returns header + } + fixture.holder.observeMediaSegment(segment) + + assertEquals(200L, requireNotNull(fixture.holder.livePlaybackSnapshot()).headSequence) + assertEquals(segment, fixture.holder.observedMediaSegment(fixture.video)) + assertEquals(965_000L, segment.header.startMs) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_001L)) + assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) + assertEquals(179, fixture.holder.playbackStartSequence(fixture.video, 964_999L)) + assertEquals(200, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) + every { fixture.state.getMaxSegment(fixture.video) } returns 200 + val nearHead = mockk { + every { this@mockk.header } returns mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 199 + every { startMs } returns 1_003_000L + every { durationMs } returns 2_000L } - assertTrue(header.sequenceNumber > 0, "$label returned bootstrap media as playable media") - assertTrue(parts.initialization.isNotEmpty(), "$label returned an empty initialization") - assertTrue(parts.media.isNotEmpty(), "$label returned empty media") - assertTrue(holder.liveInitialization(format)?.isNotEmpty() == true, "$label did not retain initialization") - println( - " itag=${header.itag} seq=${header.sequenceNumber} startMs=${header.startMs} " + - "durationMs=${header.durationMs} bytes=${segment.length} sha256=${fingerprint(segment.data)} " + - "boxes=${mp4BoxNames(segment.data)} tfdt=${mp4DecodeTimes(segment.data, header.timeRangeTimescale)}" - ) } + fixture.holder.observeMediaSegment(nearHead) + assertEquals(199, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) } - private fun YoutubeSabrFormat.codecFamily(): String { - val mime = mimeType.orEmpty().lowercase() - return when { - "avc1" in mime -> "H.264" - "vp09" in mime || "vp9" in mime -> "VP9" - "av01" in mime -> "AV1" - else -> error("Unsupported probe codec for itag $itag: $mime") + @Test + fun `active live derives missing segment duration from the sabr head`() { + val fixture = fixture() + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns -1L } + val segment = mockk { + every { this@mockk.header } returns header + } + fixture.holder.observeMediaSegment(segment) + + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) + assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) } - private fun mp4BoxNames(data: ByteArray): String { - val names = mutableListOf() - var offset = 0 - while (offset + 8 <= data.size && names.size < 12) { - val size = ((data[offset].toLong() and 0xff) shl 24) or - ((data[offset + 1].toLong() and 0xff) shl 16) or - ((data[offset + 2].toLong() and 0xff) shl 8) or - (data[offset + 3].toLong() and 0xff) - val type = String(data, offset + 4, 4, Charsets.US_ASCII) - if (size < 8L || size > data.size - offset) break - names += "$type:$size" - offset += size.toInt() + @Test + fun `active live maps time before the head sequence is reported`() { + val fixture = fixture() + every { fixture.session.liveHeadSequenceNumber } returns 0L + every { fixture.state.liveHeadSequenceNumber } returns 0L + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns 2_000L } - return names.joinToString(",") + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) } - private fun fingerprint(data: ByteArray): String = MessageDigest.getInstance("SHA-256") - .digest(data) - .take(6) - .joinToString("") { "%02x".format(it) } - - private fun mp4DecodeTimes(data: ByteArray, timescale: Int): String { - if (timescale <= 0) return "unavailable" - val decodeTimes = mutableListOf() - collectDecodeTimes(data, 0, data.size, decodeTimes) - if (decodeTimes.isEmpty()) return "none" - val firstMs = decodeTimes.first() * 1_000L / timescale - val lastMs = decodeTimes.last() * 1_000L / timescale - return "count=${decodeTimes.size},firstMs=$firstMs,lastMs=$lastMs" + @Test + fun `live duration tolerates millisecond drift between media and head timestamps`() { + val fixture = fixture() + every { fixture.state.liveHeadTimeMs } returns 1_004_999L + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns -1L + } + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + + assertEquals(2_000L, fixture.holder.playbackSegmentDurationMs(fixture.video, 180)) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) } - private fun collectDecodeTimes(data: ByteArray, start: Int, end: Int, output: MutableList) { - var offset = start - while (offset + 8 <= end) { - val size = data.readUnsignedInt(offset) - if (size < 8L || size > end - offset) return - val type = String(data, offset + 4, 4, Charsets.US_ASCII) - val payloadStart = offset + 8 - val boxEnd = offset + size.toInt() - when (type) { - "moof", "traf" -> collectDecodeTimes(data, payloadStart, boxEnd, output) - "tfdt" -> data.readTfdt(payloadStart, boxEnd)?.let(output::add) + @Test + fun `only the next live media segments wait for production`() { + val fixture = fixture() + + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 202))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 101))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 102))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 203))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 103))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 100))) + } + + @Test + fun `advertised live head waits until its media is complete`() { + val fixture = fixture() + every { fixture.state.liveHeadTimeMs } returns 1_025_000L + fixture.holder.observeMediaSegment(mockk { + every { header } returns mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 200 + every { startMs } returns 1_002_000L + every { durationMs } returns 2_000L } - offset = boxEnd + }) + + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) + } + + @Test + fun `in flight live media waits beyond the reported head`() { + val fixture = fixture() + val request = SabrSegmentRequest.media(fixture.audio, 103) + every { fixture.session.getReadableSegment(request) } returns mockk() + + assertTrue(fixture.holder.isFutureLiveRequest(request)) + } + + @Test + fun `live head can advance beyond the last complete media segment`() { + val fixture = fixture() + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 198 + every { startMs } returns 998_000L + every { durationMs } returns 2_000L } + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + every { fixture.state.liveHeadTimeMs } returns 1_025_000L + fixture.holder.setLastServedSequence(fixture.video.itag, 200) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) + assertFalse(fixture.holder.isHistoricalLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) } - private fun ByteArray.readTfdt(offset: Int, end: Int): Long? { - if (offset + 8 > end) return null - return if (this[offset].toInt() == 1) { - if (offset + 12 > end) null else readUnsignedLong(offset + 4) - } else { - readUnsignedInt(offset + 4) + @Test + fun `future live demand remains retryable after repeated responses`() { + val fixture = fixture() + val request = SabrSegmentRequest.media(fixture.video, 201) + fixture.holder.requestSegmentDemand(request, registeredAtMs = 0L) + val identity = requireNotNull(fixture.holder.segmentDemandIdentity(request)) + val result = mockk { + every { segmentCount } returns 2 + every { targetTrackSegmentCount } returns 1 + } + val runtime = SabrPumpRuntime { 20_000L } + val wasFutureLiveRequest = fixture.holder.isFutureLiveRequest(request) + every { fixture.state.getMaxSegment(fixture.video) } returns 201 + + repeat(4) { + assertFalse( + SabrDemandAttemptFinisher.finish( + fixture.holder, + request, + identity, + result, + runtime, + wasFutureLiveRequest, + ), + ) } + + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) + assertNull(fixture.holder.terminalFailure()) + assertEquals("299:201", fixture.holder.pendingSegmentDemandSummary()) + } + + @Test + fun `post live dvr is finite instead of an active live edge`() { + val fixture = fixture(postLiveDvr = true) + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertFalse(live.active) + assertTrue(live.postLiveDvr) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) } - private fun ByteArray.readUnsignedInt(offset: Int): Long = - ((this[offset].toLong() and 0xff) shl 24) or - ((this[offset + 1].toLong() and 0xff) shl 16) or - ((this[offset + 2].toLong() and 0xff) shl 8) or - (this[offset + 3].toLong() and 0xff) + private fun fixture(postLiveDvr: Boolean = false): Fixture { + val audio = format(140, true) + val video = format(299, false) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns !postLiveDvr + every { session.isAtLiveEdge } returns !postLiveDvr + every { session.liveHeadSequenceNumber } returns 200L + every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null + every { state.isLive } returns !postLiveDvr + every { state.isPostLiveDvr } returns postLiveDvr + every { state.liveHeadSequenceNumber } returns 200L + every { state.liveHeadTimeMs } returns 1_005_000L + every { state.getMaxSegment(audio) } returns 100 + every { state.getMaxSegment(video) } returns 200 + every { state.getSegmentEndMs(audio, 100) } returns 1_000_000L + every { state.getSegmentEndMs(video, 200) } returns 1_002_000L + every { state.getBufferedEndMs(audio) } returns 1_000_000L + every { state.getBufferedEndMs(video) } returns 1_002_000L + every { state.getMinBufferedEndMs() } returns 1_000_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + return Fixture(holder, session, state, audio, video) + } - private fun ByteArray.readUnsignedLong(offset: Int): Long { - var value = 0L - repeat(8) { index -> value = (value shl 8) or (this[offset + index].toLong() and 0xff) } - return value + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + return format } + + private data class Fixture( + val holder: SabrSessionHolder, + val session: YoutubeSabrSession, + val state: YoutubeSabrStreamState, + val audio: YoutubeSabrFormat, + val video: YoutubeSabrFormat, + ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt index 70915d31..c4877083 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt @@ -1,84 +1,170 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertFalse +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment +import org.junit.jupiter.api.condition.EnabledIfSystemProperty import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import java.time.Instant +import java.security.MessageDigest -class SabrLivePumpStepTest { +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrLiveProtocolProbeTest { @Test - fun `live pump fills below read ahead cushion`() = runTest { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) - var pumps = 0 - - val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } - - assertTrue(pumps == 1) - assertFalse(immediate) + fun `retrieves every requested live sabr codec`(): Unit = runBlocking { + val videoId = sabrProbeVideoId() + val tokenServiceUrl = sabrProbeTokenServiceUrl() + NewPipeInitializer.init(tokenServiceUrl) + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + try { + val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) + val prepared = SabrInfoFetcher( + tokenClient, + TypetypeTokenYoutubeSessionClient(tokenServiceUrl), + ).fetchInfo(videoId) ?: error("Missing SABR player metadata") + println( + "profile=${prepared.info.profile} clientVersion=${prepared.info.clientVersion} " + + "visitorMatches=${prepared.info.visitorData == prepared.initialToken?.visitorData}" + ) + val audio = prepared.info.formats.first { it.itag == sabrProbeAudioItag() && it.isAudio } + val requestedVideoItags = sabrProbeVideoItags() + val videos = requestedVideoItags.map { itag -> + prepared.info.formats.first { it.itag == itag && it.isVideo } + } + assertEquals(requestedVideoItags, videos.map { it.itag }) + videos.forEach { video -> + probe(store, videoId, prepared, audio, video) + } + } finally { + store.release() + } } - @Test - fun `live pump idles above read ahead cushion`() = runTest { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 130_000L) - var pumps = 0 + private suspend fun probe( + store: SabrSessionStore, + videoId: String, + prepared: SabrPreparedInfo, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ) { + val label = "${video.codecFamily()}-${video.itag}" + val holder = store.getOrCreate( + videoId = videoId, + userId = "live-protocol-probe-$label", + info = prepared.info, + audioFormat = audio, + videoFormat = video, + initialToken = prepared.initialToken, + startPump = false, + ) + holder.markExpectedLive() + store.ensureWarmed(holder, maxPumps = 8) + val live = requireNotNull(holder.livePlaybackSnapshot()) + val segments = listOfNotNull( + holder.observedMediaSegment(audio), + holder.observedMediaSegment(video), + ) + println( + "$label requests=${holder.session.requestNumber} segments=${segments.size} " + + "headSeq=${live.headSequence} headMs=${live.headTimeMs} startMs=${holder.resolvePlaybackStartMs(0L)}" + ) + println("$label trace=${holder.session.diagnosticTrace}") + assertEquals(setOf(audio.itag, video.itag), segments.map { it.header.itag }.toSet()) + segments.forEach { segment -> + val header = segment.header + val format = if (header.itag == audio.itag) audio else video + val parts = requireNotNull(SabrLiveMediaNormalizer.split(format.mimeType.orEmpty(), segment.data)) { + "$label could not split live ${format.mimeType}" + } + assertTrue(header.sequenceNumber > 0, "$label returned bootstrap media as playable media") + assertTrue(parts.initialization.isNotEmpty(), "$label returned an empty initialization") + assertTrue(parts.media.isNotEmpty(), "$label returned empty media") + assertTrue(holder.liveInitialization(format)?.isNotEmpty() == true, "$label did not retain initialization") + println( + " itag=${header.itag} seq=${header.sequenceNumber} startMs=${header.startMs} " + + "durationMs=${header.durationMs} bytes=${segment.length} sha256=${fingerprint(segment.data)} " + + "boxes=${mp4BoxNames(segment.data)} tfdt=${mp4DecodeTimes(segment.data, header.timeRangeTimescale)}" + ) + } + } - val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + private fun YoutubeSabrFormat.codecFamily(): String { + val mime = mimeType.orEmpty().lowercase() + return when { + "avc1" in mime -> "H.264" + "vp09" in mime || "vp9" in mime -> "VP9" + "av01" in mime -> "AV1" + else -> error("Unsupported probe codec for itag $itag: $mime") + } + } - assertTrue(pumps == 0) - assertFalse(immediate) + private fun mp4BoxNames(data: ByteArray): String { + val names = mutableListOf() + var offset = 0 + while (offset + 8 <= data.size && names.size < 12) { + val size = ((data[offset].toLong() and 0xff) shl 24) or + ((data[offset + 1].toLong() and 0xff) shl 16) or + ((data[offset + 2].toLong() and 0xff) shl 8) or + (data[offset + 3].toLong() and 0xff) + val type = String(data, offset + 4, 4, Charsets.US_ASCII) + if (size < 8L || size > data.size - offset) break + names += "$type:$size" + offset += size.toInt() + } + return names.joinToString(",") } - private fun fixture(playerTimeMs: Long, observedEndMs: Long): Fixture { - val audio = format(140, true) - val video = format(299, false) - val session = mockk(relaxed = true) - every { session.streamState } returns mockk(relaxed = true) - every { session.getCachedSegment(any()) } returns null - val holder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session-token", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - holder.setPlayerTimeMs(playerTimeMs) - holder.observeMediaSegment(segment(audio.itag, observedEndMs - SEGMENT_DURATION_MS)) - holder.observeMediaSegment(segment(video.itag, observedEndMs - SEGMENT_DURATION_MS)) - return Fixture(holder) + private fun fingerprint(data: ByteArray): String = MessageDigest.getInstance("SHA-256") + .digest(data) + .take(6) + .joinToString("") { "%02x".format(it) } + + private fun mp4DecodeTimes(data: ByteArray, timescale: Int): String { + if (timescale <= 0) return "unavailable" + val decodeTimes = mutableListOf() + collectDecodeTimes(data, 0, data.size, decodeTimes) + if (decodeTimes.isEmpty()) return "none" + val firstMs = decodeTimes.first() * 1_000L / timescale + val lastMs = decodeTimes.last() * 1_000L / timescale + return "count=${decodeTimes.size},firstMs=$firstMs,lastMs=$lastMs" } - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { lastModified } returns 1L - every { xtags } returns null + private fun collectDecodeTimes(data: ByteArray, start: Int, end: Int, output: MutableList) { + var offset = start + while (offset + 8 <= end) { + val size = data.readUnsignedInt(offset) + if (size < 8L || size > end - offset) return + val type = String(data, offset + 4, 4, Charsets.US_ASCII) + val payloadStart = offset + 8 + val boxEnd = offset + size.toInt() + when (type) { + "moof", "traf" -> collectDecodeTimes(data, payloadStart, boxEnd, output) + "tfdt" -> data.readTfdt(payloadStart, boxEnd)?.let(output::add) + } + offset = boxEnd + } } - private fun segment(itag: Int, startMs: Long): SabrMediaSegment { - val header = mockk { - every { this@mockk.itag } returns itag - every { sequenceNumber } returns 100 - every { this@mockk.startMs } returns startMs - every { durationMs } returns SEGMENT_DURATION_MS - every { isInitSegment } returns false + private fun ByteArray.readTfdt(offset: Int, end: Int): Long? { + if (offset + 8 > end) return null + return if (this[offset].toInt() == 1) { + if (offset + 12 > end) null else readUnsignedLong(offset + 4) + } else { + readUnsignedInt(offset + 4) } - return mockk { every { this@mockk.header } returns header } } - private data class Fixture(val holder: SabrSessionHolder) + private fun ByteArray.readUnsignedInt(offset: Int): Long = + ((this[offset].toLong() and 0xff) shl 24) or + ((this[offset + 1].toLong() and 0xff) shl 16) or + ((this[offset + 2].toLong() and 0xff) shl 8) or + (this[offset + 3].toLong() and 0xff) - private companion object { - const val SEGMENT_DURATION_MS = 2_000L + private fun ByteArray.readUnsignedLong(offset: Int): Long { + var value = 0L + repeat(8) { index -> value = (value shl 8) or (this[offset + index].toLong() and 0xff) } + return value } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt deleted file mode 100644 index 55d799d9..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt +++ /dev/null @@ -1,10 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.sabr.SabrMediaSegment - -internal data class SabrProbeFetchResult( - val segment: SabrMediaSegment?, - val elapsedMs: Long, - val timedOut: Boolean, - val error: Throwable?, -) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt index 9b764213..d82d9350 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt @@ -1,93 +1,3 @@ package dev.typetype.server.services -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization -import dev.typetype.server.sabr.YoutubeSabrClientProfile -import dev.typetype.server.sabr.SabrAdapter -import dev.typetype.server.sabr.YoutubeSabrSession - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrProbeTest { - - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() - - private val probeVideos: List = - sabrProbeVideoIds() - - @Test - fun probeSabr(): Unit { - NewPipeInitializer.init() - val loc = Localization("en", "US") - val country = ContentCountry("US") - val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) - val profile = YoutubeSabrClientProfile.WEB - val playerTimeMs = sabrProbePlayerTimeMs() - val audioItag = sabrProbeAudioItag() - val videoItags = sabrProbeVideoItags() - println( - "config videos=${probeVideos.joinToString(",")} playerTimeMs=$playerTimeMs " + - "audioItag=$audioItag videoItags=${videoItags.joinToString(",")}" - ) - - for (videoId in probeVideos) { - println("\n========== SABR probe: $videoId ==========") - try { - val token = tokenClient.fetch(videoId) ?: error("No SABR token") - val info = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { - SabrAdapter.fetchSabrInfo(videoId, profile, loc, country) - } - println("serverAbrStreamingUrl present: ${!info.serverAbrStreamingUrl.isNullOrEmpty()}") - println("videoPlaybackUstreamerConfig present: ${!info.videoPlaybackUstreamerConfig.isNullOrEmpty()}") - println("--- formats (itag | A/V | height×width | bitrate | mime | audioTrack | approxDurMs) ---") - info.formats.forEach { f -> - val kind = when { - f.isAudio -> "A" - f.isVideo -> "V" - else -> "?" - } - println( - " ${f.itag} | $kind | ${f.height}×${f.width} | br=${f.bitrate} | " + - "${f.mimeType} | track=${f.audioTrackId} | dur≈${f.approxDurationMs}ms" - ) - } - val hasAvc = info.formats.any { it.itag == 137 } - println("AVC itag 137 present: $hasAvc") - for (videoItag in videoItags) { - val configuredVideo = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } - println("configured video itag $videoItag present: ${configuredVideo != null}") - configuredVideo?.let { printSabrProbeFormat("configured video[$videoItag]", it) } - } - - val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } - ?: info.findBestAudioFormat() - val video = info.formats.firstOrNull { it.itag == videoItags.first() && it.isVideo } - ?: info.findLowestVideoFormat() - println("selected pair: audio=${audio?.itag} video=${video?.itag}") - if (audio == null || video == null) { - println("no usable audio/video pair — skipping session") - continue - } - - val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, token) - val session = YoutubeSabrSession(info, audio, video, provider) - session.streamState.setActiveTrackTypes(true, true) - session.streamState.setPlayerTimeMs(playerTimeMs) - println("--- pumpOnce #1 ---") - val segments = session.pumpOnce(loc) - println("segments returned: ${segments.size}") - segments.take(8).forEach { s -> - println(" ${sabrProbeSegmentHeader(s)}") - } - println("session complete: ${session.isComplete}") - } catch (e: Exception) { - println("FAILED on $videoId: ${e.javaClass.simpleName}: ${e.message}") - e.printStackTrace() - } - } - } -} +import dev.typetype.server.sabr.SabrMediaSegment diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt index 0552eddf..9b764213 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt @@ -1,49 +1,93 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.sabr.YoutubeSabrSession -class SabrProtectedContextRecoveryTest { - @Test - fun `refreshes the active rejected context`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns token("rejected") - every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrProbeTest { - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } - } + private val probeVideos: List = + sabrProbeVideoIds() @Test - fun `keeps a context already refreshed by another recovery`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns token("fresh") + fun probeSabr(): Unit { + NewPipeInitializer.init() + val loc = Localization("en", "US") + val country = ContentCountry("US") + val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) + val profile = YoutubeSabrClientProfile.WEB + val playerTimeMs = sabrProbePlayerTimeMs() + val audioItag = sabrProbeAudioItag() + val videoItags = sabrProbeVideoItags() + println( + "config videos=${probeVideos.joinToString(",")} playerTimeMs=$playerTimeMs " + + "audioItag=$audioItag videoItags=${videoItags.joinToString(",")}" + ) - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + for (videoId in probeVideos) { + println("\n========== SABR probe: $videoId ==========") + try { + val token = tokenClient.fetch(videoId) ?: error("No SABR token") + val info = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { + SabrAdapter.fetchSabrInfo(videoId, profile, loc, country) + } + println("serverAbrStreamingUrl present: ${!info.serverAbrStreamingUrl.isNullOrEmpty()}") + println("videoPlaybackUstreamerConfig present: ${!info.videoPlaybackUstreamerConfig.isNullOrEmpty()}") + println("--- formats (itag | A/V | height×width | bitrate | mime | audioTrack | approxDurMs) ---") + info.formats.forEach { f -> + val kind = when { + f.isAudio -> "A" + f.isVideo -> "V" + else -> "?" + } + println( + " ${f.itag} | $kind | ${f.height}×${f.width} | br=${f.bitrate} | " + + "${f.mimeType} | track=${f.audioTrackId} | dur≈${f.approxDurationMs}ms" + ) + } + val hasAvc = info.formats.any { it.itag == 137 } + println("AVC itag 137 present: $hasAvc") + for (videoItag in videoItags) { + val configuredVideo = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } + println("configured video itag $videoItag present: ${configuredVideo != null}") + configuredVideo?.let { printSabrProbeFormat("configured video[$videoItag]", it) } + } - verify(exactly = 0) { tokenClient.fetch("video", forceRefresh = true) } - } + val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } + ?: info.findBestAudioFormat() + val video = info.formats.firstOrNull { it.itag == videoItags.first() && it.isVideo } + ?: info.findLowestVideoFormat() + println("selected pair: audio=${audio?.itag} video=${video?.itag}") + if (audio == null || video == null) { + println("no usable audio/video pair — skipping session") + continue + } - @Test - fun `refreshes when the current context cannot be read`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns null - every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") - - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") - - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, token) + val session = YoutubeSabrSession(info, audio, video, provider) + session.streamState.setActiveTrackTypes(true, true) + session.streamState.setPlayerTimeMs(playerTimeMs) + println("--- pumpOnce #1 ---") + val segments = session.pumpOnce(loc) + println("segments returned: ${segments.size}") + segments.take(8).forEach { s -> + println(" ${sabrProbeSegmentHeader(s)}") + } + println("session complete: ${session.isComplete}") + } catch (e: Exception) { + println("FAILED on $videoId: ${e.javaClass.simpleName}: ${e.message}") + e.printStackTrace() + } + } } - - private fun token(visitorData: String): SabrTokenBundle = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = "player-$visitorData", - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video-$visitorData", - videoBoundPoTokenBytes = byteArrayOf(2), - ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt deleted file mode 100644 index 162d2c52..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt +++ /dev/null @@ -1,130 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrNextRequestPolicy -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState - -class SabrPumpRuntimeTest { - @Test - fun `startup and seek cushions follow PipePipe policy`() { - var now = 1_000L - val holder = holder(policy(targetAudioMs = 4_000, targetVideoMs = 7_000), playerTimeMs = 1_000L) - val runtime = SabrPumpRuntime { now } - - assertEquals(25_000L, runtime.targetReadaheadCushionMs(holder)) - now += 25_001L - assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) - - runtime.activateSeekMode() - assertEquals(5_000L, runtime.targetReadaheadCushionMs(holder)) - now += 8_000L - assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) - } - - @Test - fun `readahead cushions scale with playback rate while remaining bounded`() { - var now = 1_000L - val holder = holder( - policy(targetAudioMs = 4_000, targetVideoMs = 7_000), - playerTimeMs = 1_000L, - playbackRate = 4.0f, - ) - val runtime = SabrPumpRuntime { now } - - assertEquals(60_000L, runtime.targetReadaheadCushionMs(holder)) - now += 25_001L - assertEquals(28_000L, runtime.targetReadaheadCushionMs(holder)) - runtime.activateSeekMode() - assertEquals(20_000L, runtime.targetReadaheadCushionMs(holder)) - } - - @Test - fun `server heartbeat bypasses time throttling`() { - var now = 1_000L - val holder = holder( - policy(targetAudioMs = 3_000, targetVideoMs = 3_000, maximumRequestGapMs = 5_000), - playerTimeMs = 1_000L, - edgeMs = 13_000L, - ) - val runtime = SabrPumpRuntime { now } - now += 25_001L - - assertTrue(runtime.isThrottled(holder)) - runtime.recordRequest() - now += 4_999L - assertTrue(runtime.isThrottled(holder)) - now += 1L - assertFalse(runtime.isThrottled(holder)) - } - - @Test - fun `startup request caps reported server ahead`() { - var now = 1_000L - val holder = holder(policy(), playerTimeMs = 1_000L) - val runtime = SabrPumpRuntime { now } - - assertEquals(34_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) - now += 25_000L - assertEquals(1_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) - assertEquals(34_000L, runtime.demandPlayerTimeMs(holder, edgeMs = 50_000L)) - } - - @Test - fun `slow response without media stays recoverable for watchdog`() { - var now = 1_000L - val runtime = SabrPumpRuntime { now } - runtime.beginDemand("140:44") - now += SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS - - assertEquals(SabrDemandRecoveryAction.WAIT, runtime.demandRecoveryAction("140:44", 0, false)) - } - - @Test - fun `repeated responses without demanded segment stay retryable`() { - val runtime = SabrPumpRuntime { 1_000L } - runtime.beginDemand("140:44") - - assertEquals(SabrDemandRecoveryAction.READVERTISE_TRACK, runtime.demandRecoveryAction("140:44", 1, false)) - repeat(5) { - assertEquals(SabrDemandRecoveryAction.WAIT, runtime.demandRecoveryAction("140:44", 1, false)) - } - } - - private fun holder( - policy: SabrNextRequestPolicy, - playerTimeMs: Long, - edgeMs: Long = 0L, - playbackRate: Float = 1.0f, - ): SabrSessionHolder { - val holder = mockk() - val session = mockk() - val state = mockk() - every { holder.session } returns session - every { holder.playerTimeMs() } returns playerTimeMs - every { holder.playbackRate() } returns playbackRate - every { holder.readerTailMs() } returns 1L - every { session.streamState } returns state - every { session.cachedBytes } returns 0L - every { state.getMinBufferedEndMs() } returns edgeMs - every { state.nextRequestPolicy } returns policy - return holder - } - - private fun policy( - targetAudioMs: Int = -1, - targetVideoMs: Int = -1, - maximumRequestGapMs: Int = -1, - ): SabrNextRequestPolicy { - val policy = mockk() - every { policy.targetAudioReadaheadMs } returns targetAudioMs - every { policy.targetVideoReadaheadMs } returns targetVideoMs - every { policy.maxTimeSinceLastRequestMs } returns maximumRequestGapMs - return policy - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt index 3b058d58..ed54c1d1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt @@ -1,159 +1,140 @@ package dev.typetype.server.services -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeoutOrNull +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.NewPipe -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.stream.StreamInfo - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrRandomAccessProbeTest { - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() +import dev.typetype.server.sabr.SabrNextRequestPolicy +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +class SabrPumpRuntimeTest { @Test - fun `fetches post seek stateful responses`(): Unit = runBlocking { - NewPipeInitializer.init() - val videoId = sabrProbeVideoId() - val videoItags = sabrProbeVideoItags() - val audioItag = sabrProbeAudioItag() - val playerTimeMs = sabrProbePlayerTimeMs() - val timeoutMs = sabrProbeTimeoutMs() - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - try { - println( - "config videoId=$videoId playerTimeMs=$playerTimeMs audioItag=$audioItag " + - "videoItags=${videoItags.joinToString(",")} timeoutMs=$timeoutMs " + - "contract=stateful-pump" - ) - if (System.getenv("SABR_PROBE_EXTRACT_FIRST") != "false") { - store.rememberExtractedInfo(videoId, extractSabrInfo(videoId)) - } - val prepared = store.fetchInfo(videoId, playerTimeMs, cachedFirst = true) ?: error("SABR probe failed") - val info = prepared.info - val audio = requireAudioFormat(info.formats, audioItag) - printSabrProbeFormat("audio", audio) - for (videoItag in videoItags) { - val video = requireVideoFormat(info.formats, videoItag) - printSabrProbeFormat("video", video) - val startedAt = System.nanoTime() - val preparation = SabrPlaybackSessionService(store).prepare( - videoId = videoId, - userId = "sabr-random-access-video-$videoItag", - prepared = prepared, - audio = audio, - video = video, - startTimeMs = playerTimeMs, - ) - val holder = preparation.holder - assertInitData(store, holder, audio) - assertInitData(store, holder, video) - holder.setActiveTracks(videoActive = true, audioActive = true) - holder.setPlayerTimeMs(playerTimeMs) - val requests = listOf(audio, video).map { format -> - SabrSegmentRequest.media(format, holder.playbackStartSequence(format, playerTimeMs)) - } - requests.forEach { request -> - println("pump target[$videoItag] ${sabrProbeRequestSummary(holder, request)}") - } - val segments = withTimeoutOrNull(timeoutMs) { - requests.map { request -> - store.requestSegmentDemand(holder, request, holder.activeGeneration()) - awaitCachedSegment(store, holder, request) - } - } - val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L - println("pump[$videoItag] elapsedMs=$elapsedMs segments=${segments?.size ?: -1}") - println("pump[$videoItag] trace=${holder.session.diagnosticTrace}") - segments.orEmpty().forEach { - println( - "pump[$videoItag] itag=${it.itag} seq=${it.sequence} startMs=${it.startMs} " + - "durationMs=${it.durationMs} bytes=${it.length}" - ) - } - assertTrue( - segments.orEmpty().any { it.covers(video, playerTimeMs) }, - "video[$videoItag] pump bytes", - ) - assertTrue( - segments.orEmpty().any { it.covers(audio, playerTimeMs) }, - "audio[$videoItag] pump bytes", - ) - } - } finally { - store.release() - } + fun `startup and seek cushions follow PipePipe policy`() { + var now = 1_000L + val holder = holder(policy(targetAudioMs = 4_000, targetVideoMs = 7_000), playerTimeMs = 1_000L) + val runtime = SabrPumpRuntime { now } + + assertEquals(25_000L, runtime.targetReadaheadCushionMs(holder)) + now += 25_001L + assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) + + runtime.activateSeekMode() + assertEquals(5_000L, runtime.targetReadaheadCushionMs(holder)) + now += 8_000L + assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) } - private suspend fun awaitCachedSegment( - store: SabrSessionStore, - holder: SabrSessionHolder, - request: SabrSegmentRequest, - ): CachedSabrSegment { - var segment = store.cachedSegment(holder, request) - while (segment == null) { - delay(50L) - segment = store.cachedSegment(holder, request) - } - return segment + @Test + fun `readahead cushions scale with playback rate while remaining bounded`() { + var now = 1_000L + val holder = holder( + policy(targetAudioMs = 4_000, targetVideoMs = 7_000), + playerTimeMs = 1_000L, + playbackRate = 4.0f, + ) + val runtime = SabrPumpRuntime { now } + + assertEquals(60_000L, runtime.targetReadaheadCushionMs(holder)) + now += 25_001L + assertEquals(28_000L, runtime.targetReadaheadCushionMs(holder)) + runtime.activateSeekMode() + assertEquals(20_000L, runtime.targetReadaheadCushionMs(holder)) + } + + @Test + fun `server heartbeat bypasses time throttling`() { + var now = 1_000L + val holder = holder( + policy(targetAudioMs = 3_000, targetVideoMs = 3_000, maximumRequestGapMs = 5_000), + playerTimeMs = 1_000L, + edgeMs = 13_000L, + ) + val runtime = SabrPumpRuntime { now } + now += 25_001L + + assertTrue(runtime.isThrottled(holder)) + runtime.recordRequest() + now += 4_999L + assertTrue(runtime.isThrottled(holder)) + now += 1L + assertFalse(runtime.isThrottled(holder)) } - private suspend fun assertInitData( - store: SabrSessionStore, - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - ): Unit { - val data = store.fetchInitializationData(holder, format) - println("init itag=${format.itag} bytes=${data?.size ?: -1}") - if (data == null || data.isEmpty()) println("init trace=${holder.session.diagnosticTrace}") - assertTrue(data?.isNotEmpty() == true, "init bytes for itag ${format.itag}") + @Test + fun `startup request caps reported server ahead`() { + var now = 1_000L + val holder = holder(policy(), playerTimeMs = 1_000L) + val runtime = SabrPumpRuntime { now } + + assertEquals(34_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) + now += 25_000L + assertEquals(1_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) + assertEquals(34_000L, runtime.demandPlayerTimeMs(holder, edgeMs = 50_000L)) + } + + @Test + fun `deferred request stays recoverable for watchdog`() { + var now = 1_000L + val runtime = SabrPumpRuntime { now } + runtime.beginDemand("140:44") + now += SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS + + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = false, resolved = false), + ) + } + + @Test + fun `response without demanded segment is readvertised once`() { + val runtime = SabrPumpRuntime { 1_000L } + runtime.beginDemand("140:44") + + assertEquals( + SabrDemandRecoveryAction.READVERTISE_TRACK, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) + repeat(5) { + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) + } } - private fun requireAudioFormat( - formats: List, - audioItag: Int, - ): YoutubeSabrFormat = - formats.filter { it.itag == audioItag && it.isAudio } - .maxWithOrNull(compareBy { it.isOriginalAudio } - .thenBy { it.xtags.isNullOrBlank() } - .thenBy { !it.isDrc } - .thenBy { it.bitrate }) - ?: error("No SABR audio format for itag $audioItag") - - private fun extractSabrInfo(videoId: String): YoutubeSabrInfo { - val service = NewPipe.getServiceByUrl("https://www.youtube.com/watch?v=$videoId") - val linkHandler = service.streamLHFactory.fromUrl("https://www.youtube.com/watch?v=$videoId") - val extractor = service.getStreamExtractor(linkHandler) - extractor.fetchPage() - val streamInfo = StreamInfo.getInfo(extractor) - return sequence { - streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } - streamInfo.videoOnlyStreams.forEach { yield(it.deliveryMethodInfo) } - streamInfo.audioStreams.forEach { yield(it.deliveryMethodInfo) } - }.filterIsInstance() - .first { it.videoId == videoId } + private fun holder( + policy: SabrNextRequestPolicy, + playerTimeMs: Long, + edgeMs: Long = 0L, + playbackRate: Float = 1.0f, + ): SabrSessionHolder { + val holder = mockk() + val session = mockk() + val state = mockk() + every { holder.session } returns session + every { holder.expectsLive() } returns false + every { holder.playerTimeMs() } returns playerTimeMs + every { holder.playbackRate() } returns playbackRate + every { holder.readerTailMs() } returns 1L + every { session.streamState } returns state + every { session.cachedBytes } returns 0L + every { state.getMinBufferedEndMs() } returns edgeMs + every { state.nextRequestPolicy } returns policy + return holder } - private fun requireVideoFormat( - formats: List, - videoItag: Int, - ): YoutubeSabrFormat = - formats.firstOrNull { it.itag == videoItag && it.isVideo } - ?: error("No SABR video format for itag $videoItag") - - private fun CachedSabrSegment.covers(format: YoutubeSabrFormat, playerTimeMs: Long): Boolean { - if (length <= 0 || init || itag != format.itag) return false - val startMs = this.startMs - val durationMs = this.durationMs - return startMs >= 0 && durationMs > 0 && - playerTimeMs >= startMs && playerTimeMs < startMs + durationMs + private fun policy( + targetAudioMs: Int = -1, + targetVideoMs: Int = -1, + maximumRequestGapMs: Int = -1, + ): SabrNextRequestPolicy { + val policy = mockk() + every { policy.targetAudioReadaheadMs } returns targetAudioMs + every { policy.targetVideoReadaheadMs } returns targetVideoMs + every { policy.maxTimeSinceLastRequestMs } returns maximumRequestGapMs + return policy } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt index a477f8dc..3b058d58 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt @@ -1,67 +1,159 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrRecoverableException +import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import org.schabi.newpipe.extractor.NewPipe +import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant +import org.schabi.newpipe.extractor.stream.StreamInfo -@OptIn(ExperimentalCoroutinesApi::class) -class SabrRecoverablePumpFailureTest { - @Test - fun `recoverable media failure requests a fresh session`() = runTest { - val holder = pumpFailure("Unexpected EOF") +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrRandomAccessProbeTest { + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertTrue(holder.terminalFailure().orEmpty().startsWith(SABR_RECOVERABLE_FAILURE_PREFIX)) + @Test + fun `fetches post seek stateful responses`(): Unit = runBlocking { + NewPipeInitializer.init() + val videoId = sabrProbeVideoId() + val videoItags = sabrProbeVideoItags() + val audioItag = sabrProbeAudioItag() + val playerTimeMs = sabrProbePlayerTimeMs() + val timeoutMs = sabrProbeTimeoutMs() + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + try { + println( + "config videoId=$videoId playerTimeMs=$playerTimeMs audioItag=$audioItag " + + "videoItags=${videoItags.joinToString(",")} timeoutMs=$timeoutMs " + + "contract=stateful-pump" + ) + if (System.getenv("SABR_PROBE_EXTRACT_FIRST") != "false") { + store.rememberExtractedInfo(videoId, extractSabrInfo(videoId)) + } + val prepared = store.fetchInfo(videoId, playerTimeMs, cachedFirst = true) ?: error("SABR probe failed") + val info = prepared.info + val audio = requireAudioFormat(info.formats, audioItag) + printSabrProbeFormat("audio", audio) + for (videoItag in videoItags) { + val video = requireVideoFormat(info.formats, videoItag) + printSabrProbeFormat("video", video) + val startedAt = System.nanoTime() + val preparation = SabrPlaybackSessionService(store).prepare( + videoId = videoId, + userId = "sabr-random-access-video-$videoItag", + prepared = prepared, + audio = audio, + video = video, + startTimeMs = playerTimeMs, + ) + val holder = preparation.holder + assertInitData(store, holder, audio) + assertInitData(store, holder, video) + holder.setActiveTracks(videoActive = true, audioActive = true) + holder.setPlayerTimeMs(playerTimeMs) + val requests = listOf(audio, video).map { format -> + SabrSegmentRequest.media(format, holder.playbackStartSequence(format, playerTimeMs)) + } + requests.forEach { request -> + println("pump target[$videoItag] ${sabrProbeRequestSummary(holder, request)}") + } + val segments = withTimeoutOrNull(timeoutMs) { + requests.map { request -> + store.requestSegmentDemand(holder, request, holder.activeGeneration()) + awaitCachedSegment(store, holder, request) + } + } + val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L + println("pump[$videoItag] elapsedMs=$elapsedMs segments=${segments?.size ?: -1}") + println("pump[$videoItag] trace=${holder.session.diagnosticTrace}") + segments.orEmpty().forEach { + println( + "pump[$videoItag] itag=${it.itag} seq=${it.sequence} startMs=${it.startMs} " + + "durationMs=${it.durationMs} bytes=${it.length}" + ) + } + assertTrue( + segments.orEmpty().any { it.covers(video, playerTimeMs) }, + "video[$videoItag] pump bytes", + ) + assertTrue( + segments.orEmpty().any { it.covers(audio, playerTimeMs) }, + "audio[$videoItag] pump bytes", + ) + } + } finally { + store.release() + } } - @Test - fun `local spool failure remains terminal without fresh session recovery`() = runTest { - val holder = pumpFailure("Could not write SABR spool file") + private suspend fun awaitCachedSegment( + store: SabrSessionStore, + holder: SabrSessionHolder, + request: SabrSegmentRequest, + ): CachedSabrSegment { + var segment = store.cachedSegment(holder, request) + while (segment == null) { + delay(50L) + segment = store.cachedSegment(holder, request) + } + return segment + } - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertEquals("Could not write SABR spool file", holder.terminalFailure()) + private suspend fun assertInitData( + store: SabrSessionStore, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + ): Unit { + val data = store.fetchInitializationData(holder, format) + println("init itag=${format.itag} bytes=${data?.size ?: -1}") + if (data == null || data.isEmpty()) println("init trace=${holder.session.diagnosticTrace}") + assertTrue(data?.isNotEmpty() == true, "init bytes for itag ${format.itag}") } - private suspend fun pumpFailure(message: String): SabrSessionHolder { - val session = mockk(relaxed = true) - val streamState = mockk(relaxed = true) - every { session.streamState } returns streamState - every { session.requestNumber } returns 1 - every { session.isComplete } returns false - every { session.cachedBytes } returns 0L - every { streamState.nextRequestPolicy } returns null - every { session.pumpOnceStreaming(any()) } throws SabrRecoverableException(message) - val holder = holder(session) + private fun requireAudioFormat( + formats: List, + audioItag: Int, + ): YoutubeSabrFormat = + formats.filter { it.itag == audioItag && it.isAudio } + .maxWithOrNull(compareBy { it.isOriginalAudio } + .thenBy { it.xtags.isNullOrBlank() } + .thenBy { !it.isDrc } + .thenBy { it.bitrate }) + ?: error("No SABR audio format for itag $audioItag") - SabrSessionPump().pumpLoop({ true }, holder, intervalMs = 0L) - return holder + private fun extractSabrInfo(videoId: String): YoutubeSabrInfo { + val service = NewPipe.getServiceByUrl("https://www.youtube.com/watch?v=$videoId") + val linkHandler = service.streamLHFactory.fromUrl("https://www.youtube.com/watch?v=$videoId") + val extractor = service.getStreamExtractor(linkHandler) + extractor.fetchPage() + val streamInfo = StreamInfo.getInfo(extractor) + return sequence { + streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } + streamInfo.videoOnlyStreams.forEach { yield(it.deliveryMethodInfo) } + streamInfo.audioStreams.forEach { yield(it.deliveryMethodInfo) } + }.filterIsInstance() + .first { it.videoId == videoId } } - private fun holder(session: YoutubeSabrSession): SabrSessionHolder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = format(140, true), - videoFormat = format(137, false), - sessionToken = "session-token", - key = SabrSessionKey("video", "user", 140, null, 137, 0L), - lastRequestAt = Instant.EPOCH, - ) + private fun requireVideoFormat( + formats: List, + videoItag: Int, + ): YoutubeSabrFormat = + formats.firstOrNull { it.itag == videoItag && it.isVideo } + ?: error("No SABR video format for itag $videoItag") - private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 - return format + private fun CachedSabrSegment.covers(format: YoutubeSabrFormat, playerTimeMs: Long): Boolean { + if (length <= 0 || init || itag != format.itag) return false + val startMs = this.startMs + val durationMs = this.durationMs + return startMs >= 0 && durationMs > 0 && + playerTimeMs >= startMs && playerTimeMs < startMs + durationMs } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt deleted file mode 100644 index 85254a98..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ /dev/null @@ -1,78 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import dev.typetype.server.sabr.SabrSegmentRequest - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrSessionStoreTest { - - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() - - @Test - fun storeRoundTrip(): Unit = runBlocking { - NewPipeInitializer.init() - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - val videoId = sabrProbeVideoId() - val playerTimeMs = sabrProbePlayerTimeMs() - val timeoutMs = sabrProbeTimeoutMs() - val audioItag = sabrProbeAudioItag() - val videoItag = sabrProbeVideoItags().first() - val userId = "sabr-probe-user" - - println( - "\n========== SABR store round-trip: $videoId " + - "playerTimeMs=$playerTimeMs timeoutMs=$timeoutMs ==========" - ) - val prepared = store.fetchInfo(videoId) ?: error("SABR probe failed") - val info = prepared.info - val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } - ?: info.findBestAudioFormat() - val video = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } - ?: info.findLowestVideoFormat() - println("probe picked: audio=${audio?.itag} video=${video?.itag}") - check(audio != null && video != null) - printSabrProbeFormat("audio", audio) - printSabrProbeFormat("video", video) - - val holder = store.getOrCreate(videoId, userId, info, audio, video, prepared.initialToken) - store.ensureWarmed(holder) - holder.setActiveTracks(videoActive = true, audioActive = true) - holder.setPlayerTimeMs(playerTimeMs) - println("session complete (cold): ${holder.session.isComplete}") - - val requests = listOf( - SabrSegmentRequest.initialization(video), - SabrSegmentRequest.initialization(audio), - ) + mediaRequestsForProbe(holder, video, audio, playerTimeMs) - for ((i, req) in requests.withIndex()) { - val result = fetchSabrProbeSegment(store, holder, req, timeoutMs) - printSabrProbeFetch("req[$i]", holder, req, result) - } - println("session complete after fetches: ${holder.session.isComplete}") - - val looked = store.lookup(videoId, userId, audio.itag, video.itag) - println("lookup same key: ${looked === holder}") - - store.release() - } - - private fun mediaRequestsForProbe( - holder: SabrSessionHolder, - video: dev.typetype.server.sabr.YoutubeSabrFormat, - audio: dev.typetype.server.sabr.YoutubeSabrFormat, - playerTimeMs: Long, - ): List { - val videoSequence = System.getenv("SABR_PROBE_VIDEO_SEQUENCE")?.toIntOrNull() - val audioSequence = System.getenv("SABR_PROBE_AUDIO_SEQUENCE")?.toIntOrNull() - if (videoSequence == null && audioSequence == null) return holder.mediaRequestsAt(playerTimeMs) - return listOfNotNull( - videoSequence?.let { SabrSegmentRequest.media(video, it) }, - audioSequence?.let { SabrSegmentRequest.media(audio, it) }, - ) - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt index 55509e38..85254a98 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt @@ -1,159 +1,78 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment +import org.junit.jupiter.api.condition.EnabledIfSystemProperty import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant -class SabrSessionTimeRequestsTest { - @Test - fun `mediaRequestsAt returns active audio and video requests for player time`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk() - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 321_601L) } returns 33 - val holder = holder(session, audio, video) - - val requests = holder.mediaRequestsAt(321_601L) +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrSessionStoreTest { - assertEquals(listOf(137, 140), requests.map { it.format.itag }) - assertEquals(listOf(64, 33), requests.map { it.sequenceNumber }) - } + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() @Test - fun `mediaRequestsAt uses mapped video and audio sequences`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 247, isAudio = false) - val session = mockk() - val state = mockk() - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 340_000L) } returns 64 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 340_000L) } returns 35 - val holder = holder(session, audio, video) - - val requests = holder.mediaRequestsAt(340_000L) - - assertEquals(listOf(247, 140), requests.map { it.format.itag }) - assertEquals(listOf(64, 35), requests.map { it.sequenceNumber }) + fun storeRoundTrip(): Unit = runBlocking { + NewPipeInitializer.init() + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + val videoId = sabrProbeVideoId() + val playerTimeMs = sabrProbePlayerTimeMs() + val timeoutMs = sabrProbeTimeoutMs() + val audioItag = sabrProbeAudioItag() + val videoItag = sabrProbeVideoItags().first() + val userId = "sabr-probe-user" + + println( + "\n========== SABR store round-trip: $videoId " + + "playerTimeMs=$playerTimeMs timeoutMs=$timeoutMs ==========" + ) + val prepared = store.fetchInfo(videoId) ?: error("SABR probe failed") + val info = prepared.info + val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } + ?: info.findBestAudioFormat() + val video = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } + ?: info.findLowestVideoFormat() + println("probe picked: audio=${audio?.itag} video=${video?.itag}") + check(audio != null && video != null) + printSabrProbeFormat("audio", audio) + printSabrProbeFormat("video", video) + + val holder = store.getOrCreate(videoId, userId, info, audio, video, prepared.initialToken) + store.ensureWarmed(holder) + holder.setActiveTracks(videoActive = true, audioActive = true) + holder.setPlayerTimeMs(playerTimeMs) + println("session complete (cold): ${holder.session.isComplete}") + + val requests = listOf( + SabrSegmentRequest.initialization(video), + SabrSegmentRequest.initialization(audio), + ) + mediaRequestsForProbe(holder, video, audio, playerTimeMs) + for ((i, req) in requests.withIndex()) { + val result = fetchSabrProbeSegment(store, holder, req, timeoutMs) + printSabrProbeFetch("req[$i]", holder, req, result) + } + println("session complete after fetches: ${holder.session.isComplete}") + + val looked = store.lookup(videoId, userId, audio.itag, video.itag) + println("lookup same key: ${looked === holder}") + + store.release() } - @Test - fun `mediaRequestsAt excludes inactive tracks`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk() - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 - val holder = holder(session, audio, video) - holder.setActiveTracks(videoActive = true, audioActive = false) - - val requests = holder.mediaRequestsAt(321_601L) - - assertEquals(listOf(137), requests.map { it.format.itag }) - assertEquals(listOf(64), requests.map { it.sequenceNumber }) + private fun mediaRequestsForProbe( + holder: SabrSessionHolder, + video: dev.typetype.server.sabr.YoutubeSabrFormat, + audio: dev.typetype.server.sabr.YoutubeSabrFormat, + playerTimeMs: Long, + ): List { + val videoSequence = System.getenv("SABR_PROBE_VIDEO_SEQUENCE")?.toIntOrNull() + val audioSequence = System.getenv("SABR_PROBE_AUDIO_SEQUENCE")?.toIntOrNull() + if (videoSequence == null && audioSequence == null) return holder.mediaRequestsAt(playerTimeMs) + return listOfNotNull( + videoSequence?.let { SabrSegmentRequest.media(video, it) }, + audioSequence?.let { SabrSegmentRequest.media(audio, it) }, + ) } - - @Test - fun `reposition keeps a non adjacent live boundary request pending`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 248, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentStartMs(audio, 100) } returns 995_000L - every { session.getCachedSegment(any()) } returns null - val holder = holder(session, audio, video) - val observed = mediaSegment(audio.itag, sequence = 102, startMs = 995_010L) - holder.markExpectedLive() - holder.observeMediaSegment(observed) - every { - session.getCachedSegment(match { - it.format.itag == audio.itag && it.sequenceNumber == 102 - }) - } returns observed - val request = SabrSegmentRequest.media(audio, 100) - - val missing = holder.repositionTargets(listOf(request), playerTimeMs = 995_000L, generation = 0L) - - assertEquals(listOf(request), missing) - } - - @Test - fun `reposition advances to the next warmed live segment from inside a boundary`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 248, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { session.getCachedSegment(any()) } returns null - val holder = holder(session, audio, video) - val observed = mediaSegment(audio.itag, sequence = 101, startMs = 1_000_000L, durationMs = 5_000L) - holder.markExpectedLive() - holder.observeMediaSegment(observed) - every { - session.getCachedSegment(match { - it.format.itag == audio.itag && it.sequenceNumber == 101 - }) - } returns observed - val request = SabrSegmentRequest.media(audio, 100) - - val missing = holder.repositionTargets(listOf(request), playerTimeMs = 996_200L, generation = 0L) - - assertEquals(emptyList(), missing) - assertEquals(1_000_000L, holder.readerPosition(audio)) - } - - private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.isVideo } returns !isAudio - return format - } - - private fun mediaSegment( - itag: Int, - sequence: Int, - startMs: Long, - durationMs: Long = 0L, - ): SabrMediaSegment { - val header = mockk(relaxed = true) - every { header.itag } returns itag - every { header.sequenceNumber } returns sequence - every { header.startMs } returns startMs - every { header.durationMs } returns durationMs - every { header.isInitSegment } returns false - return mockk { every { this@mockk.header } returns header } - } - - private fun holder( - session: YoutubeSabrSession, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - ): SabrSessionHolder = SabrSessionHolder( - session, - mockk(), - audio, - video, - "session", - SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - Instant.now(), - ) } diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt deleted file mode 100644 index 3a6a12e4..00000000 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ /dev/null @@ -1,96 +0,0 @@ -package dev.typetype.server.services - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider - -class TypetypeYoutubeSessionPoTokenProviderTest { - @AfterEach - fun clearAuthenticatedProvider(): Unit = - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) - - @Test - fun `exposes the session token only inside its scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - assertEquals("visitor", currentToken()?.visitorData) - assertEquals("player-token", currentToken()?.poToken) - } - - assertNull(currentToken()) - } - - @Test - fun `restores the outer token after a nested scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("outer", "outer-token")) { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("inner", "inner-token")) { - assertEquals("inner", currentToken()?.visitorData) - } - assertEquals("outer", currentToken()?.visitorData) - } - - assertNull(currentToken()) - } - - @Test - fun `clears the token when the scoped call fails`() { - runCatching { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - error("failed") - } - } - - assertNull(currentToken()) - } - - @Test - fun `uses the authenticated provider outside a SABR scope`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) - - assertEquals("auth", currentToken()?.visitorData) - assertEquals("auth-token", currentToken()?.poToken) - } - - @Test - fun `prefers the SABR token over the authenticated provider`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) - - TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { - assertEquals("sabr", currentToken()?.visitorData) - assertEquals("sabr-token", currentToken()?.poToken) - } - } - - private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( - "MWEB", - "2.20260801.00.00", - "test-user-agent", - Localization("en", "US"), - ContentCountry("US"), - false, - ) - - private fun token(visitorData: String, playerToken: String) = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = playerToken, - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video-token", - videoBoundPoTokenBytes = byteArrayOf(2), - ) - - private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { - override fun getSessionPoToken( - clientName: String, - clientVersion: String, - userAgent: String?, - localization: Localization, - contentCountry: ContentCountry, - loggedIn: Boolean, - ) = YoutubeSessionPoToken(visitorData, poToken) - } -}