From 7270e88780888f0873a5d263388f54faca4495a2 Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 18:52:38 +0200 Subject: [PATCH 01/11] add external Stremio catalogs Let users bring curated catalogs from external addons into librarySync so they can be filtered like built-in catalogs, including hiding watched items. Cache and refresh them on a recurring worker job so addon manifests stay fast and stable. --- .env.example | 5 +- .../librarysync/api/routes_stremio_addon.py | 340 +++++++++++++- .../api/routes_stremio_addon_public.py | 109 ++++- backend/src/librarysync/config.py | 8 + .../src/librarysync/core/external_catalog.py | 267 +++++++++++ ...7c8f9a1e0_add_stremio_external_catalogs.py | 151 ++++++ backend/src/librarysync/db/models.py | 91 ++++ .../jobs/external_catalog_refresh.py | 73 +++ .../librarysync/static/page-stremio-addon.js | 439 ++++++++++++++++++ .../librarysync/templates/stremio-addon.html | 41 +- backend/src/librarysync/worker.py | 4 + backend/tests/test_stremio_addon_catalogs.py | 46 +- 12 files changed, 1561 insertions(+), 13 deletions(-) create mode 100644 backend/src/librarysync/core/external_catalog.py create mode 100644 backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py create mode 100644 backend/src/librarysync/jobs/external_catalog_refresh.py diff --git a/.env.example b/.env.example index a35d535..7407c41 100644 --- a/.env.example +++ b/.env.example @@ -28,13 +28,16 @@ SIMKL_CLIENT_SECRET=your_simkl_client_secret ANILIST_CLIENT_ID=your_anilist_client_id ANILIST_CLIENT_SECRET=your_anilist_client_secret -# Worker modes (comma-separated subset of `outbox`, `metadata`, `metadata_backfill`, `metadata_cache`, `quick_import`, `import_all`, `watchlist`, `merge_history`, `merge_all_history`) +# Worker modes (comma-separated subset of `outbox`, `metadata`, `metadata_backfill`, `metadata_cache`, `quick_import`, `import_all`, `external_catalog_refresh`, `watchlist`, `merge_history`, `merge_all_history`) LIBRARYSYNC_WORKER_MODES=all LIBRARYSYNC_WORKER_OUTBOX_CONCURRENCY=1 LIBRARYSYNC_WORKER_METADATA_CONCURRENCY=1 LIBRARYSYNC_WORKER_QUICK_IMPORT_CONCURRENCY=1 LIBRARYSYNC_WORKER_IMPORT_ALL_CONCURRENCY=1 LIBRARYSYNC_WORKER_METADATA_CACHE_CONCURRENCY=1 +LIBRARYSYNC_WORKER_EXTERNAL_CATALOG_REFRESH_CONCURRENCY=1 +LIBRARYSYNC_EXTERNAL_CATALOG_REFRESH_HOURS=3 +LIBRARYSYNC_EXTERNAL_CATALOG_MAX_ITEMS=500 # Rate limits (per user, per provider, per minute) LIBRARYSYNC_TRAKT_RATE_LIMIT_PER_MINUTE=60 diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index a352981..80cb90a 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Literal +import httpx from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel from sqlalchemy import func, select @@ -15,6 +16,18 @@ from librarysync.api.deps import get_current_user, get_db from librarysync.config import settings from librarysync.core.catalog_ordering import CatalogOrderBy +from librarysync.core.external_catalog import ( + discover_external_catalogs, + external_catalog_out, + mark_external_catalog_refresh_failed, + normalize_external_filters, + normalize_external_manifest_url, + normalize_external_order_by, + normalize_external_order_dir, + normalize_external_page_size, + normalize_external_show_in_home, + refresh_external_catalog, +) from librarysync.core.stremio_addon import ( ensure_addon_config, normalize_default_catalogs, @@ -30,6 +43,7 @@ StremioAddonConfig, StremioCustomCatalog, StremioCustomCatalogItem, + StremioExternalCatalog, User, ) @@ -42,6 +56,7 @@ *CatalogOrderBy.__args__, } CUSTOM_ORDER_DIR = {"asc", "desc"} +EXTERNAL_CATALOG_TYPES = {"movie", "series"} class StremioCatalogFilters(BaseModel): @@ -101,6 +116,35 @@ class StremioCustomCatalogReorder(BaseModel): media_item_ids: list[str] +class StremioExternalCatalogDiscoverPayload(BaseModel): + manifest_url: str + + +class StremioExternalCatalogCreate(BaseModel): + name: str + manifest_url: str + addon_name: str | None = None + source_catalog_id: str + source_catalog_type: Literal["movie", "series"] + media_type: Literal["movie", "tv", "anime"] | None = None + enabled: bool | None = None + filters: StremioCatalogFilters | None = None + order_by: str | None = None + order_dir: Literal["asc", "desc"] | None = None + page_size: int | None = None + show_in_home: bool | None = None + + +class StremioExternalCatalogUpdate(BaseModel): + name: str | None = None + enabled: bool | None = None + filters: StremioCatalogFilters | None = None + order_by: str | None = None + order_dir: Literal["asc", "desc"] | None = None + page_size: int | None = None + show_in_home: bool | None = None + + def _resolve_base_url(request: Request) -> str: base = settings.base_url or str(request.base_url) return base.rstrip("/") @@ -130,19 +174,33 @@ async def _build_unique_slug( user_id: str, value: str, *, - exclude_catalog_id: str | None = None, + exclude_custom_catalog_id: str | None = None, + exclude_external_catalog_id: str | None = None, ) -> str: base_slug = _truncate_slug(_slugify(value)) - result = await db.execute( + custom_result = await db.execute( select(StremioCustomCatalog.id, StremioCustomCatalog.slug).where( StremioCustomCatalog.user_id == user_id ) ) + external_result = await db.execute( + select(StremioExternalCatalog.id, StremioExternalCatalog.slug).where( + StremioExternalCatalog.user_id == user_id + ) + ) existing = { slug - for catalog_id, slug in result.all() - if slug and (not exclude_catalog_id or catalog_id != exclude_catalog_id) + for catalog_id, slug in custom_result.all() + if slug and (not exclude_custom_catalog_id or catalog_id != exclude_custom_catalog_id) } + existing.update( + { + slug + for catalog_id, slug in external_result.all() + if slug + and (not exclude_external_catalog_id or catalog_id != exclude_external_catalog_id) + } + ) if base_slug not in existing: return base_slug for index in range(2, 100): @@ -280,6 +338,23 @@ async def _load_custom_catalog( return catalog +async def _load_external_catalog( + db: AsyncSession, + user_id: str, + catalog_id: str, +) -> StremioExternalCatalog: + result = await db.execute( + select(StremioExternalCatalog).where( + StremioExternalCatalog.user_id == user_id, + StremioExternalCatalog.id == catalog_id, + ) + ) + catalog = result.scalars().first() + if not catalog: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") + return catalog + + async def _resolve_custom_media_item( db: AsyncSession, payload: StremioCustomCatalogItemCreate, @@ -378,6 +453,40 @@ async def _resolve_custom_media_item( return media_item +def _normalize_external_media_type( + source_catalog_type: str, + requested_media_type: str | None, +) -> str: + if source_catalog_type == "movie": + return "movie" + normalized = (requested_media_type or "tv").strip().lower() + return normalized if normalized in {"tv", "anime"} else "tv" + + +async def _refresh_external_catalog_or_502( + db: AsyncSession, + catalog: StremioExternalCatalog, +) -> int: + try: + count = await refresh_external_catalog(db, catalog) + await db.commit() + await db.refresh(catalog) + return count + except httpx.HTTPError as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"External catalog refresh failed: {exc}", + ) from exc + except Exception as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise + + @router.get( "/config", summary="Get Stremio addon config", @@ -392,14 +501,25 @@ async def get_stremio_addon_config( catalogs = await _ensure_default_catalogs(db, config) custom_result = await db.execute( - select(StremioCustomCatalog).where(StremioCustomCatalog.user_id == current_user.id) + select(StremioCustomCatalog) + .where(StremioCustomCatalog.user_id == current_user.id) + .order_by(StremioCustomCatalog.created_at.asc()) ) custom_catalogs = [_custom_catalog_out(catalog) for catalog in custom_result.scalars().all()] + external_result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == current_user.id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + external_catalogs = [ + external_catalog_out(catalog) for catalog in external_result.scalars().all() + ] return { "addon_id": config.id, "is_enabled": bool(config.is_enabled), "catalogs": catalogs, + "external_catalogs": external_catalogs, "custom_catalogs": custom_catalogs, **_build_manifest_links(_resolve_base_url(request), config.id), } @@ -437,6 +557,203 @@ async def update_stremio_addon_config( } +@router.post( + "/external-catalogs/discover", + summary="Discover external catalogs", + description="Fetch an external Stremio manifest and list supported catalogs.", +) +async def discover_stremio_external_catalogs( + payload: StremioExternalCatalogDiscoverPayload, + _: User = Depends(get_current_user), +) -> dict: + manifest_url = normalize_external_manifest_url(payload.manifest_url) + if not manifest_url: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Manifest URL is required", + ) + try: + return await discover_external_catalogs(manifest_url) + except httpx.HTTPError as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to fetch addon manifest: {exc}", + ) from exc + + +@router.get( + "/external-catalogs", + summary="List external catalogs", + description="List external Stremio catalogs configured for the current user.", +) +async def list_external_catalogs( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == current_user.id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + return { + "external_catalogs": [external_catalog_out(catalog) for catalog in result.scalars().all()] + } + + +@router.post( + "/external-catalogs", + status_code=status.HTTP_201_CREATED, + summary="Create external catalog", + description="Create a new external Stremio catalog backed by another addon.", +) +async def create_external_catalog( + payload: StremioExternalCatalogCreate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + name = payload.name.strip() + if not name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") + manifest_url = normalize_external_manifest_url(payload.manifest_url) + if not manifest_url: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Manifest URL is required", + ) + source_catalog_id = payload.source_catalog_id.strip() + if not source_catalog_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Source catalog ID is required", + ) + if payload.source_catalog_type not in EXTERNAL_CATALOG_TYPES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid source catalog type", + ) + + catalog = StremioExternalCatalog( + user_id=current_user.id, + name=name, + slug=await _build_unique_slug(db, current_user.id, name), + addon_name=(payload.addon_name or "").strip() or None, + manifest_url=manifest_url, + source_catalog_id=source_catalog_id, + source_catalog_type=payload.source_catalog_type, + media_type=_normalize_external_media_type( + payload.source_catalog_type, + payload.media_type, + ), + enabled=True if payload.enabled is None else bool(payload.enabled), + filters=normalize_external_filters( + payload.filters.model_dump(exclude_none=True) if payload.filters else None + ), + order_by=normalize_external_order_by(payload.order_by), + order_dir=normalize_external_order_dir(payload.order_dir), + page_size=normalize_external_page_size(payload.page_size), + show_in_home=normalize_external_show_in_home(payload.show_in_home), + ) + db.add(catalog) + try: + await db.flush() + await refresh_external_catalog(db, catalog) + await db.commit() + await db.refresh(catalog) + except httpx.HTTPError as exc: + await db.rollback() + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"External catalog refresh failed: {exc}", + ) from exc + except Exception: + await db.rollback() + raise + return external_catalog_out(catalog) + + +@router.patch( + "/external-catalogs/{catalog_id}", + summary="Update external catalog", + description="Update an external Stremio catalog.", +) +async def update_external_catalog( + catalog_id: str, + payload: StremioExternalCatalogUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + fields = payload.model_fields_set + + if "name" in fields: + name = (payload.name or "").strip() + if not name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") + if name != catalog.name: + catalog.slug = await _build_unique_slug( + db, + current_user.id, + name, + exclude_external_catalog_id=catalog.id, + ) + catalog.name = name + if "enabled" in fields: + catalog.enabled = bool(payload.enabled) + if "filters" in fields: + catalog.filters = normalize_external_filters( + payload.filters.model_dump(exclude_none=True) if payload.filters else None + ) + if "order_by" in fields: + catalog.order_by = normalize_external_order_by(payload.order_by) + if "order_dir" in fields: + catalog.order_dir = normalize_external_order_dir(payload.order_dir) + if "page_size" in fields: + catalog.page_size = normalize_external_page_size(payload.page_size) + if "show_in_home" in fields: + catalog.show_in_home = normalize_external_show_in_home(payload.show_in_home) + + catalog.updated_at = datetime.now(timezone.utc) + db.add(catalog) + await db.commit() + await db.refresh(catalog) + return external_catalog_out(catalog) + + +@router.delete( + "/external-catalogs/{catalog_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Delete external catalog", + description="Delete an external Stremio catalog and its cached items.", +) +async def delete_external_catalog( + catalog_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + await db.delete(catalog) + await db.commit() + + +@router.post( + "/external-catalogs/{catalog_id}/refresh", + summary="Refresh external catalog", + description="Fetch the latest items for an external Stremio catalog.", +) +async def refresh_stremio_external_catalog( + catalog_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + item_count = await _refresh_external_catalog_or_502(db, catalog) + return { + "status": "ok", + "item_count": item_count, + "catalog": external_catalog_out(catalog), + } + + @router.post( "/custom-catalogs", status_code=status.HTTP_201_CREATED, @@ -486,7 +803,7 @@ async def update_custom_catalog( raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") if name != catalog.name: catalog.slug = await _build_unique_slug( - db, current_user.id, name, exclude_catalog_id=catalog.id + db, current_user.id, name, exclude_custom_catalog_id=catalog.id ) catalog.name = name @@ -495,6 +812,17 @@ async def update_custom_catalog( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid media type" ) + if payload.media_type != catalog.media_type: + item_count_result = await db.execute( + select(func.count(StremioCustomCatalogItem.id)).where( + StremioCustomCatalogItem.catalog_id == catalog.id + ) + ) + if int(item_count_result.scalar_one() or 0) > 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot change media type while the catalog still has items", + ) catalog.media_type = payload.media_type if "order_by" in fields: diff --git a/backend/src/librarysync/api/routes_stremio_addon_public.py b/backend/src/librarysync/api/routes_stremio_addon_public.py index aad113c..f5e642b 100644 --- a/backend/src/librarysync/api/routes_stremio_addon_public.py +++ b/backend/src/librarysync/api/routes_stremio_addon_public.py @@ -10,6 +10,10 @@ from librarysync.api.deps import get_db from librarysync.core.catalog_ordering import apply_catalog_ordering, build_show_progress_subquery +from librarysync.core.external_catalog import ( + external_catalog_manifest_entry, + normalize_external_filters, +) from librarysync.core.stremio_addon import ( DEFAULT_PAGE_SIZE, DEFAULT_SHOW_IN_HOME, @@ -21,6 +25,9 @@ MediaItem, StremioCustomCatalog, StremioCustomCatalogItem, + StremioExternalCatalog, + StremioExternalCatalogItem, + WatchedItem, WatchlistItem, ) @@ -197,6 +204,7 @@ def _coerce_show_in_home(catalog: dict | None) -> bool: def _build_manifest( catalogs: list[dict], + external_catalogs: list[StremioExternalCatalog], custom_catalogs: list[StremioCustomCatalog], ) -> dict[str, Any]: manifest_catalogs: list[dict[str, Any]] = [] @@ -226,6 +234,12 @@ def _build_manifest( ) seen_types.add(stremio_type) + for external in external_catalogs: + if not external.enabled: + continue + manifest_catalogs.append(external_catalog_manifest_entry(external)) + seen_types.add(external.source_catalog_type) + for custom in custom_catalogs: stremio_type = _resolve_stremio_type(custom.media_type) if not stremio_type: @@ -357,6 +371,43 @@ async def _build_custom_catalog_query( return query +async def _build_external_catalog_query( + user_id: str, + catalog: StremioExternalCatalog, + search: str | None, +): + query = ( + select(MediaItem) + .join(StremioExternalCatalogItem, StremioExternalCatalogItem.media_item_id == MediaItem.id) + .where( + StremioExternalCatalogItem.catalog_id == catalog.id, + StremioExternalCatalogItem.media_item_id.is_not(None), + ) + ) + filters = normalize_external_filters(catalog.filters) + if not filters.get("show_watched", False): + now_date = datetime.now(timezone.utc).date() + progress_subq = build_show_progress_subquery(user_id, now_date) + query = query.outerjoin(progress_subq, progress_subq.c.media_item_id == MediaItem.id) + directly_watched_exists = ( + select(WatchedItem.id) + .where( + WatchedItem.user_id == user_id, + WatchedItem.media_item_id == MediaItem.id, + ) + .exists() + ) + completed_show_clause = and_( + MediaItem.media_type.in_(["tv", "anime"]), + progress_subq.c.total_released > 0, + progress_subq.c.watched_count >= progress_subq.c.total_released, + ) + query = query.where(~or_(directly_watched_exists, completed_show_clause)) + if search: + query = _apply_search_filter(query, search) + return query + + def _apply_ordering( query, *, @@ -392,10 +443,18 @@ async def stremio_addon_manifest( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") catalogs = _normalize_catalogs(config.default_catalogs) custom_result = await db.execute( - select(StremioCustomCatalog).where(StremioCustomCatalog.user_id == config.user_id) + select(StremioCustomCatalog) + .where(StremioCustomCatalog.user_id == config.user_id) + .order_by(StremioCustomCatalog.created_at.asc()) ) custom_catalogs = custom_result.scalars().all() - return _build_manifest(catalogs, custom_catalogs) + external_result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == config.user_id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + external_catalogs = external_result.scalars().all() + return _build_manifest(catalogs, external_catalogs, custom_catalogs) @router.get("/{addon_id}/catalog/{catalog_type}/{catalog_id}.json", include_in_schema=False) @@ -440,8 +499,29 @@ async def _serve_catalog( catalogs = _normalize_catalogs(config.default_catalogs) catalog = _catalogs_by_id(catalogs).get(catalog_id) custom_catalog = None + external_catalog = None if not catalog: + external_result = await db.execute( + select(StremioExternalCatalog).where( + StremioExternalCatalog.user_id == config.user_id, + StremioExternalCatalog.slug == catalog_id, + ) + ) + external_catalog = external_result.scalars().first() + if external_catalog: + if not external_catalog.enabled: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Catalog not found", + ) + if external_catalog.source_catalog_type != catalog_type: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Catalog not found", + ) + + if not catalog and not external_catalog: custom_result = await db.execute( select(StremioCustomCatalog).where( StremioCustomCatalog.user_id == config.user_id, @@ -463,7 +543,30 @@ async def _serve_catalog( skip, limit, search = _resolve_pagination(request, extra_overrides) - if custom_catalog: + if external_catalog: + query = await _build_external_catalog_query(config.user_id, external_catalog, search) + order_by = external_catalog.order_by or "source" + order_dir = external_catalog.order_dir or "asc" + if order_by == "source": + order_clause = StremioExternalCatalogItem.position.asc() + if order_dir == "desc": + order_clause = StremioExternalCatalogItem.position.desc() + query = query.order_by(order_clause, MediaItem.id) + elif order_by == "random": + query = query.order_by(func.random()) + else: + release_date_expr = func.coalesce(MediaItem.release_date, MediaItem.first_air_date) + query = _apply_ordering( + query, + order_by=order_by, + order_dir=order_dir, + user_id=config.user_id, + date_added_col=StremioExternalCatalogItem.fetched_at, + release_date_col=release_date_expr, + base_media_id_col=MediaItem.id, + tie_breaker_col=MediaItem.id, + ) + elif custom_catalog: query = await _build_custom_catalog_query(custom_catalog, search) order_by = custom_catalog.order_by or "manual" order_dir = custom_catalog.order_dir or "asc" diff --git a/backend/src/librarysync/config.py b/backend/src/librarysync/config.py index f77ed1b..3fe52a9 100644 --- a/backend/src/librarysync/config.py +++ b/backend/src/librarysync/config.py @@ -42,6 +42,8 @@ class Settings: enable_dashboard_stats: bool trakt_max_batch_size: int simkl_max_batch_size: int + external_catalog_refresh_hours: int + external_catalog_max_items: int def load_settings() -> Settings: @@ -115,6 +117,12 @@ def load_settings() -> Settings: simkl_max_batch_size=int( _get_env("LIBRARYSYNC_SIMKL_MAX_BATCH_SIZE", "750") or "750" ), + external_catalog_refresh_hours=int( + _get_env("LIBRARYSYNC_EXTERNAL_CATALOG_REFRESH_HOURS", "3") or "3" + ), + external_catalog_max_items=int( + _get_env("LIBRARYSYNC_EXTERNAL_CATALOG_MAX_ITEMS", "500") or "500" + ), ) diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py new file mode 100644 index 0000000..57052f4 --- /dev/null +++ b/backend/src/librarysync/core/external_catalog.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import delete, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from librarysync.config import settings +from librarysync.core.catalog_ordering import CatalogOrderBy +from librarysync.core.http_client import get_http_client +from librarysync.db.models import MediaItem, StremioExternalCatalog, StremioExternalCatalogItem + +EXTERNAL_CATALOG_ORDER_BY = {"source", "random", *CatalogOrderBy.__args__} +EXTERNAL_CATALOG_ORDER_DIR = {"asc", "desc"} +EXTERNAL_CATALOG_DEFAULT_PAGE_SIZE = 30 +EXTERNAL_CATALOG_DEFAULT_SHOW_IN_HOME = True +EXTERNAL_CATALOG_DEFAULT_FILTERS = {"show_watched": False, "statuses": []} +EXTERNAL_CATALOG_FETCH_PAGE_SIZE = 100 + + +def normalize_external_manifest_url(value: str) -> str: + manifest_url = (value or "").strip() + if not manifest_url: + return "" + if manifest_url.endswith("/manifest.json"): + return manifest_url + return f"{manifest_url.rstrip('/')}/manifest.json" + + +def normalize_external_filters(filters: dict[str, Any] | None) -> dict[str, Any]: + base = filters if isinstance(filters, dict) else {} + statuses = base.get("statuses") if isinstance(base.get("statuses"), list) else [] + show_watched = base.get("show_watched") if isinstance(base.get("show_watched"), bool) else False + return {"statuses": [str(value) for value in statuses if value], "show_watched": show_watched} + + +def normalize_external_order_by(value: str | None) -> str: + normalized = value.strip().lower() if value else "source" + return normalized if normalized in EXTERNAL_CATALOG_ORDER_BY else "source" + + +def normalize_external_order_dir(value: str | None) -> str: + normalized = value.strip().lower() if value else "asc" + return normalized if normalized in EXTERNAL_CATALOG_ORDER_DIR else "asc" + + +def normalize_external_page_size(value: int | None) -> int: + if isinstance(value, int) and value > 0: + return min(value, 100) + return EXTERNAL_CATALOG_DEFAULT_PAGE_SIZE + + +def normalize_external_show_in_home(value: bool | None) -> bool: + return value if isinstance(value, bool) else EXTERNAL_CATALOG_DEFAULT_SHOW_IN_HOME + + +def external_catalog_out(catalog: StremioExternalCatalog) -> dict[str, Any]: + return { + "id": catalog.id, + "name": catalog.name, + "slug": catalog.slug, + "addon_name": catalog.addon_name, + "manifest_url": catalog.manifest_url, + "source_catalog_id": catalog.source_catalog_id, + "source_catalog_type": catalog.source_catalog_type, + "media_type": catalog.media_type, + "enabled": catalog.enabled, + "filters": normalize_external_filters(catalog.filters), + "order_by": catalog.order_by, + "order_dir": catalog.order_dir, + "page_size": catalog.page_size, + "show_in_home": catalog.show_in_home, + "last_refreshed_at": catalog.last_refreshed_at, + "last_refresh_error": catalog.last_refresh_error, + "created_at": catalog.created_at, + "updated_at": catalog.updated_at, + } + + +def external_catalog_manifest_entry(catalog: StremioExternalCatalog) -> dict[str, Any]: + return { + "type": catalog.source_catalog_type, + "id": catalog.slug, + "name": catalog.name, + "extraSupported": ["search", "skip", "limit"], + "pageSize": normalize_external_page_size(catalog.page_size), + "showInHome": normalize_external_show_in_home(catalog.show_in_home), + } + + +def stremio_type_to_media_types(stremio_type: str) -> tuple[str, ...]: + if stremio_type == "movie": + return ("movie",) + if stremio_type == "series": + return ("tv", "anime") + return () + + +async def discover_external_catalogs(manifest_url: str) -> dict[str, Any]: + normalized_url = normalize_external_manifest_url(manifest_url) + if not normalized_url: + raise ValueError("Manifest URL is required") + + async with get_http_client() as client: + response = await client.get(normalized_url) + response.raise_for_status() + payload = response.json() + + addon_name = str(payload.get("name") or "External addon").strip() or "External addon" + discovered: list[dict[str, str]] = [] + for catalog in payload.get("catalogs") or []: + if not isinstance(catalog, dict): + continue + catalog_id = str(catalog.get("id") or "").strip() + catalog_type = str(catalog.get("type") or "").strip().lower() + if not catalog_id or catalog_type not in {"movie", "series"}: + continue + discovered.append( + { + "id": catalog_id, + "name": str(catalog.get("name") or catalog_id), + "type": catalog_type, + } + ) + + return { + "manifest_url": normalized_url, + "addon_name": addon_name, + "catalogs": sorted(discovered, key=lambda entry: (entry["type"], entry["name"].lower())), + } + + +async def refresh_external_catalog( + db: AsyncSession, + catalog: StremioExternalCatalog, + *, + max_items: int | None = None, +) -> int: + normalized_manifest_url = normalize_external_manifest_url(catalog.manifest_url) + max_catalog_items = max_items or settings.external_catalog_max_items + fetched_at = datetime.now(timezone.utc) + + async with get_http_client() as client: + manifest_response = await client.get(normalized_manifest_url) + manifest_response.raise_for_status() + manifest_payload = manifest_response.json() + catalog.addon_name = ( + str(manifest_payload.get("name") or catalog.addon_name or "").strip() or None + ) + + base_url = normalized_manifest_url[: -len("/manifest.json")] + metas: list[dict[str, Any]] = [] + skip = 0 + while len(metas) < max_catalog_items: + limit = min(EXTERNAL_CATALOG_FETCH_PAGE_SIZE, max_catalog_items - len(metas)) + catalog_url = ( + f"{base_url}/catalog/{catalog.source_catalog_type}/{catalog.source_catalog_id}" + f"/skip={skip}&limit={limit}.json" + ) + response = await client.get(catalog_url) + response.raise_for_status() + payload = response.json() + batch = payload.get("metas") if isinstance(payload, dict) else None + if not isinstance(batch, list) or not batch: + break + metas.extend(item for item in batch if isinstance(item, dict)) + if len(batch) < limit: + break + skip += limit + + await db.execute( + delete(StremioExternalCatalogItem).where( + StremioExternalCatalogItem.catalog_id == catalog.id + ) + ) + + created = 0 + for position, meta in enumerate(metas): + stremio_id = str(meta.get("id") or "").strip() + stremio_type = str(meta.get("type") or catalog.source_catalog_type or "").strip().lower() + if not stremio_id or stremio_type not in {"movie", "series"}: + continue + imdb_id = _extract_imdb_id(meta, stremio_id) + media_item_id = await _resolve_external_media_item_id(db, stremio_id, imdb_id, stremio_type) + db.add( + StremioExternalCatalogItem( + id=str(uuid.uuid4()), + catalog_id=catalog.id, + media_item_id=media_item_id, + stremio_id=stremio_id, + stremio_type=stremio_type, + title=_coerce_text(meta.get("name")), + year=_coerce_year(meta.get("year")), + poster_url=_coerce_text(meta.get("poster")), + imdb_id=imdb_id, + position=position, + fetched_at=fetched_at, + ) + ) + created += 1 + + catalog.manifest_url = normalized_manifest_url + catalog.last_refreshed_at = fetched_at + catalog.last_refresh_error = None + catalog.updated_at = fetched_at + db.add(catalog) + await db.flush() + return created + + +async def mark_external_catalog_refresh_failed( + db: AsyncSession, + catalog: StremioExternalCatalog, + error: Exception, +) -> None: + catalog.last_refresh_error = str(error) + catalog.updated_at = datetime.now(timezone.utc) + db.add(catalog) + await db.flush() + + +def _coerce_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _coerce_year(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip().isdigit(): + return int(value.strip()) + return None + + +def _extract_imdb_id(meta: dict[str, Any], stremio_id: str) -> str | None: + imdb_id = _coerce_text(meta.get("imdb_id")) + if imdb_id: + return imdb_id + if stremio_id.startswith("tt"): + return stremio_id + return None + + +async def _resolve_external_media_item_id( + db: AsyncSession, + stremio_id: str, + imdb_id: str | None, + stremio_type: str, +) -> str | None: + media_types = stremio_type_to_media_types(stremio_type) + if not media_types: + return None + + conditions = [MediaItem.raw["stremio_id"].as_string() == stremio_id] + if imdb_id: + conditions.append(MediaItem.imdb_id == imdb_id) + result = await db.execute( + select(MediaItem.id) + .where(MediaItem.media_type.in_(media_types), or_(*conditions)) + .limit(1) + ) + media_item_id = result.scalars().first() + return str(media_item_id) if media_item_id else None diff --git a/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py b/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py new file mode 100644 index 0000000..3a09fa9 --- /dev/null +++ b/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py @@ -0,0 +1,151 @@ +"""add stremio external catalogs + +Revision ID: d2b7c8f9a1e0 +Revises: a1d4e6f8b9c0 +Create Date: 2026-04-17 14:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "d2b7c8f9a1e0" +down_revision = "a1d4e6f8b9c0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "stremio_external_catalogs", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=64), nullable=False), + sa.Column("addon_name", sa.String(length=255), nullable=True), + sa.Column("manifest_url", sa.String(length=500), nullable=False), + sa.Column("source_catalog_id", sa.String(length=255), nullable=False), + sa.Column("source_catalog_type", sa.String(length=32), nullable=False), + sa.Column("media_type", sa.String(length=32), nullable=False, server_default="movie"), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("filters", sa.JSON(), nullable=True), + sa.Column("order_by", sa.String(length=32), nullable=False, server_default="source"), + sa.Column("order_dir", sa.String(length=8), nullable=False, server_default="asc"), + sa.Column("page_size", sa.Integer(), nullable=False, server_default="30"), + sa.Column("show_in_home", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("last_refreshed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_refresh_error", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "slug", name="uq_stremio_external_catalogs_user_slug"), + ) + op.create_index( + op.f("ix_stremio_external_catalogs_user_id"), + "stremio_external_catalogs", + ["user_id"], + ) + + op.create_table( + "stremio_external_catalog_items", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("catalog_id", sa.String(length=36), nullable=False), + sa.Column("media_item_id", sa.String(length=36), nullable=True), + sa.Column("stremio_id", sa.String(length=255), nullable=False), + sa.Column("stremio_type", sa.String(length=32), nullable=False), + sa.Column("title", sa.String(length=255), nullable=True), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column("poster_url", sa.String(length=500), nullable=True), + sa.Column("imdb_id", sa.String(length=32), nullable=True), + sa.Column("position", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "fetched_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.ForeignKeyConstraint( + ["catalog_id"], ["stremio_external_catalogs.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["media_item_id"], ["media_items.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "catalog_id", + "position", + name="uq_stremio_external_catalog_items_catalog_position", + ), + sa.UniqueConstraint( + "catalog_id", + "stremio_id", + name="uq_stremio_external_catalog_items_catalog_stremio", + ), + ) + op.create_index( + "ix_stremio_external_catalog_items_catalog_position", + "stremio_external_catalog_items", + ["catalog_id", "position"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_catalog_id"), + "stremio_external_catalog_items", + ["catalog_id"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_media_item_id"), + "stremio_external_catalog_items", + ["media_item_id"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_imdb_id"), + "stremio_external_catalog_items", + ["imdb_id"], + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_stremio_external_catalog_items_imdb_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + op.f("ix_stremio_external_catalog_items_media_item_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + op.f("ix_stremio_external_catalog_items_catalog_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + "ix_stremio_external_catalog_items_catalog_position", + table_name="stremio_external_catalog_items", + ) + op.drop_table("stremio_external_catalog_items") + op.drop_index( + op.f("ix_stremio_external_catalogs_user_id"), + table_name="stremio_external_catalogs", + ) + op.drop_table("stremio_external_catalogs") diff --git a/backend/src/librarysync/db/models.py b/backend/src/librarysync/db/models.py index 54eddef..3801e71 100644 --- a/backend/src/librarysync/db/models.py +++ b/backend/src/librarysync/db/models.py @@ -465,6 +465,97 @@ class StremioCustomCatalogItem(Base): ) +class StremioExternalCatalog(Base): + __tablename__ = "stremio_external_catalogs" + __table_args__ = ( + UniqueConstraint( + "user_id", + "slug", + name="uq_stremio_external_catalogs_user_slug", + ), + Index("ix_stremio_external_catalogs_user_id", "user_id"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + user_id: Mapped[str] = mapped_column( + String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + name: Mapped[str] = mapped_column(String(255)) + slug: Mapped[str] = mapped_column(String(64)) + addon_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + manifest_url: Mapped[str] = mapped_column(String(500)) + source_catalog_id: Mapped[str] = mapped_column(String(255)) + source_catalog_type: Mapped[str] = mapped_column(String(32)) + media_type: Mapped[str] = mapped_column(String(32), default="movie") + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + filters: Mapped[dict | None] = mapped_column(JSON, nullable=True) + order_by: Mapped[str] = mapped_column(String(32), default="source") + order_dir: Mapped[str] = mapped_column(String(8), default="asc") + page_size: Mapped[int] = mapped_column(Integer, default=30) + show_in_home: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + last_refreshed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + last_refresh_error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + + +class StremioExternalCatalogItem(Base): + __tablename__ = "stremio_external_catalog_items" + __table_args__ = ( + UniqueConstraint( + "catalog_id", + "position", + name="uq_stremio_external_catalog_items_catalog_position", + ), + UniqueConstraint( + "catalog_id", + "stremio_id", + name="uq_stremio_external_catalog_items_catalog_stremio", + ), + Index( + "ix_stremio_external_catalog_items_catalog_position", + "catalog_id", + "position", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + catalog_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("stremio_external_catalogs.id", ondelete="CASCADE"), + index=True, + ) + media_item_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("media_items.id", ondelete="SET NULL"), index=True, nullable=True + ) + stremio_id: Mapped[str] = mapped_column(String(255)) + stremio_type: Mapped[str] = mapped_column(String(32)) + title: Mapped[str | None] = mapped_column(String(255), nullable=True) + year: Mapped[int | None] = mapped_column(Integer, nullable=True) + poster_url: Mapped[str | None] = mapped_column(String(500), nullable=True) + imdb_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) + position: Mapped[int] = mapped_column(Integer, default=0) + fetched_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + + class WatchlistItem(Base): __tablename__ = "watchlist_items" __table_args__ = ( diff --git a/backend/src/librarysync/jobs/external_catalog_refresh.py b/backend/src/librarysync/jobs/external_catalog_refresh.py new file mode 100644 index 0000000..d128636 --- /dev/null +++ b/backend/src/librarysync/jobs/external_catalog_refresh.py @@ -0,0 +1,73 @@ +import logging +from datetime import timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from librarysync.config import settings +from librarysync.core.external_catalog import ( + mark_external_catalog_refresh_failed, + refresh_external_catalog, +) +from librarysync.core.scheduler import ( + claim_scheduled_job, + complete_scheduled_job, + extend_scheduled_job, + release_scheduled_job, +) +from librarysync.db.models import ScheduledJob, StremioExternalCatalog +from librarysync.db.session import SessionLocal, init_session_factory + +logger = logging.getLogger(__name__) + +EXTERNAL_CATALOG_REFRESH_JOB = "external_catalog_refresh" +EXTERNAL_CATALOG_REFRESH_INTERVAL = timedelta(hours=settings.external_catalog_refresh_hours) +EXTERNAL_CATALOG_REFRESH_LEASE = timedelta(minutes=30) +EXTERNAL_CATALOG_REFRESH_RETRY_DELAY = timedelta(minutes=5) + + +async def process_external_catalog_refresh_once() -> int: + init_session_factory() + async with SessionLocal() as db: + job = await claim_scheduled_job( + db, + EXTERNAL_CATALOG_REFRESH_JOB, + EXTERNAL_CATALOG_REFRESH_INTERVAL, + EXTERNAL_CATALOG_REFRESH_LEASE, + ) + if not job: + return 0 + try: + refreshed = await run_external_catalog_refresh(db, job) + except Exception: + logger.exception("External catalog refresh failed") + await release_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_RETRY_DELAY) + return 0 + await complete_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_INTERVAL) + return refreshed + + +async def run_external_catalog_refresh(db: AsyncSession, job: ScheduledJob) -> int: + logger.info("Starting external catalog refresh") + result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.enabled.is_(True)) + .order_by(StremioExternalCatalog.updated_at.asc()) + ) + catalogs = result.scalars().all() + refreshed = 0 + + for catalog in catalogs: + try: + await refresh_external_catalog(db, catalog) + await db.commit() + refreshed += 1 + except Exception as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + logger.warning("External catalog refresh failed for %s: %s", catalog.slug, exc) + await extend_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_LEASE) + + logger.info("Finished external catalog refresh") + return refreshed diff --git a/backend/src/librarysync/static/page-stremio-addon.js b/backend/src/librarysync/static/page-stremio-addon.js index 3dd773b..5798ba6 100644 --- a/backend/src/librarysync/static/page-stremio-addon.js +++ b/backend/src/librarysync/static/page-stremio-addon.js @@ -1,7 +1,11 @@ const addonState = { config: null, catalogs: [], + externalCatalogs: [], customCatalogs: [], + discoveredExternalCatalogs: [], + discoveredExternalManifestUrl: "", + discoveredExternalAddonName: "", selectedCatalogId: null, items: [], candidates: [], @@ -61,6 +65,16 @@ const CUSTOM_ORDER_OPTIONS = [ ...ORDER_OPTIONS, ]; +const EXTERNAL_ORDER_OPTIONS = [ + { value: "source", label: "Source order" }, + ...ORDER_OPTIONS, +]; + +const EXTERNAL_TYPE_LABELS = { + movie: "Movie", + series: "Series", +}; + function setAddonMessage(message, isError = false) { setMessage("stremio-addon-message", message, isError); } @@ -338,6 +352,238 @@ function renderBuiltInCatalogs() { container.appendChild(fragment); } +function getDiscoveredExternalCatalog(key) { + return ( + addonState.discoveredExternalCatalogs.find( + (catalog) => `${catalog.type}::${catalog.id}` === key + ) || null + ); +} + +function renderExternalCatalogDiscovery() { + const form = document.getElementById("external-catalog-form"); + if (!form) { + return; + } + const select = form.querySelector('[name="catalog_key"]'); + if (!select) { + return; + } + const currentValue = select.value; + select.innerHTML = ""; + if (!addonState.discoveredExternalCatalogs.length) { + const option = document.createElement("option"); + option.value = ""; + option.textContent = "Discover catalogs first"; + select.appendChild(option); + select.disabled = true; + return; + } + const placeholder = document.createElement("option"); + placeholder.value = ""; + placeholder.textContent = "Choose a catalog"; + select.appendChild(placeholder); + addonState.discoveredExternalCatalogs.forEach((catalog) => { + const option = document.createElement("option"); + option.value = `${catalog.type}::${catalog.id}`; + option.textContent = `${catalog.name} · ${EXTERNAL_TYPE_LABELS[catalog.type] || catalog.type}`; + if (option.value === currentValue) { + option.selected = true; + } + select.appendChild(option); + }); + select.disabled = false; +} + +function renderExternalCatalogs() { + const container = document.getElementById("external-catalog-list"); + if (!container) { + return; + } + container.innerHTML = ""; + if (!addonState.externalCatalogs.length) { + const empty = document.createElement("p"); + empty.className = "empty-state"; + empty.textContent = "No external catalogs yet. Discover an addon above."; + container.appendChild(empty); + return; + } + + const fragment = document.createDocumentFragment(); + addonState.externalCatalogs.forEach((catalog) => { + const card = document.createElement("div"); + card.className = "rounded-2xl border border-line/60 bg-surface/80 p-5"; + card.dataset.externalCatalogId = catalog.id; + + const header = document.createElement("div"); + header.className = "flex flex-wrap items-start justify-between gap-3"; + const info = document.createElement("div"); + const title = document.createElement("h3"); + title.className = "font-display text-base font-semibold text-ink"; + title.textContent = catalog.name; + const subtitle = document.createElement("p"); + subtitle.className = "text-xs text-muted"; + const addonName = catalog.addon_name || "External addon"; + subtitle.textContent = `${addonName} · ${catalog.source_catalog_id} · ${EXTERNAL_TYPE_LABELS[catalog.source_catalog_type] || catalog.source_catalog_type}`; + info.appendChild(title); + info.appendChild(subtitle); + + const actions = document.createElement("div"); + actions.className = "flex flex-wrap items-center gap-2"; + const refreshButton = document.createElement("button"); + refreshButton.type = "button"; + refreshButton.className = "btn btn-secondary btn-sm"; + refreshButton.dataset.externalCatalogRefresh = catalog.id; + refreshButton.textContent = "Refresh"; + const deleteButton = document.createElement("button"); + deleteButton.type = "button"; + deleteButton.className = "btn btn-ghost btn-sm"; + deleteButton.dataset.externalCatalogDelete = catalog.id; + deleteButton.textContent = "Delete"; + actions.appendChild(refreshButton); + actions.appendChild(deleteButton); + + header.appendChild(info); + header.appendChild(actions); + + const body = document.createElement("div"); + body.className = "mt-4 grid gap-4 lg:grid-cols-[minmax(0,2fr)_minmax(0,1fr)]"; + + const visibilityBlock = document.createElement("div"); + const visibilityLabel = document.createElement("p"); + visibilityLabel.className = "text-xs font-semibold uppercase tracking-[0.2em] text-muted"; + visibilityLabel.textContent = "Visibility"; + const visibilityGroup = document.createElement("div"); + visibilityGroup.className = "mt-2 space-y-2"; + + const enabledLabel = document.createElement("label"); + enabledLabel.className = "inline-control"; + const enabledInput = document.createElement("input"); + enabledInput.type = "checkbox"; + enabledInput.checked = catalog.enabled !== false; + enabledInput.dataset.externalCatalogEnabled = "true"; + const enabledText = document.createElement("span"); + enabledText.textContent = "Enabled"; + enabledLabel.appendChild(enabledInput); + enabledLabel.appendChild(enabledText); + + const watchedLabel = document.createElement("label"); + watchedLabel.className = "inline-control"; + const watchedInput = document.createElement("input"); + watchedInput.type = "checkbox"; + watchedInput.checked = normalizeShowWatched(catalog); + watchedInput.dataset.externalCatalogShowWatched = "true"; + const watchedText = document.createElement("span"); + watchedText.textContent = "Show watched"; + watchedLabel.appendChild(watchedInput); + watchedLabel.appendChild(watchedText); + + const homeLabel = document.createElement("label"); + homeLabel.className = "inline-control"; + const homeInput = document.createElement("input"); + homeInput.type = "checkbox"; + homeInput.checked = !!catalog.show_in_home; + homeInput.dataset.externalCatalogShowInHome = "true"; + const homeText = document.createElement("span"); + homeText.textContent = "Show in home"; + homeLabel.appendChild(homeInput); + homeLabel.appendChild(homeText); + + visibilityGroup.appendChild(enabledLabel); + visibilityGroup.appendChild(watchedLabel); + visibilityGroup.appendChild(homeLabel); + visibilityBlock.appendChild(visibilityLabel); + visibilityBlock.appendChild(visibilityGroup); + + const configBlock = document.createElement("div"); + configBlock.className = "space-y-3"; + + const nameField = document.createElement("label"); + nameField.className = "field"; + const nameLabel = document.createElement("span"); + nameLabel.textContent = "Name"; + const nameInput = document.createElement("input"); + nameInput.className = "input"; + nameInput.type = "text"; + nameInput.value = catalog.name || ""; + nameInput.dataset.externalCatalogName = "true"; + nameField.appendChild(nameLabel); + nameField.appendChild(nameInput); + + const orderField = document.createElement("label"); + orderField.className = "field"; + const orderLabel = document.createElement("span"); + orderLabel.textContent = "Order"; + const orderSelect = buildSelect(EXTERNAL_ORDER_OPTIONS, catalog.order_by || "source"); + orderSelect.dataset.externalCatalogOrderBy = "true"; + orderField.appendChild(orderLabel); + orderField.appendChild(orderSelect); + + const dirField = document.createElement("label"); + dirField.className = "field"; + const dirLabel = document.createElement("span"); + dirLabel.textContent = "Direction"; + const dirSelect = buildSelect( + [ + { value: "asc", label: "Ascending" }, + { value: "desc", label: "Descending" }, + ], + catalog.order_dir || "asc" + ); + dirSelect.dataset.externalCatalogOrderDir = "true"; + dirField.appendChild(dirLabel); + dirField.appendChild(dirSelect); + + const sizeField = document.createElement("label"); + sizeField.className = "field"; + const sizeLabel = document.createElement("span"); + sizeLabel.textContent = "Page size"; + const sizeInput = document.createElement("input"); + sizeInput.className = "input"; + sizeInput.type = "number"; + sizeInput.min = "1"; + sizeInput.max = "100"; + sizeInput.value = catalog.page_size || 30; + sizeInput.dataset.externalCatalogPageSize = "true"; + sizeField.appendChild(sizeLabel); + sizeField.appendChild(sizeInput); + + configBlock.appendChild(nameField); + configBlock.appendChild(orderField); + configBlock.appendChild(dirField); + configBlock.appendChild(sizeField); + + body.appendChild(visibilityBlock); + body.appendChild(configBlock); + + const footer = document.createElement("div"); + footer.className = "mt-4 flex flex-wrap items-center justify-between gap-3"; + const status = document.createElement("span"); + status.className = "text-xs text-muted"; + if (catalog.last_refresh_error) { + status.textContent = `Last refresh failed: ${catalog.last_refresh_error}`; + } else if (catalog.last_refreshed_at) { + status.textContent = `Refreshed ${formatMetadataDate(catalog.last_refreshed_at)}`; + } else { + status.textContent = "Never refreshed"; + } + const saveButton = document.createElement("button"); + saveButton.type = "button"; + saveButton.className = "btn btn-secondary btn-sm"; + saveButton.dataset.externalCatalogSave = catalog.id; + saveButton.textContent = "Save changes"; + footer.appendChild(status); + footer.appendChild(saveButton); + + card.appendChild(header); + card.appendChild(body); + card.appendChild(footer); + fragment.appendChild(card); + }); + + container.appendChild(fragment); +} + function renderCustomCatalogs() { const container = document.getElementById("custom-catalog-list"); if (!container) { @@ -845,6 +1091,165 @@ async function handleCustomItemMove(mediaItemId, direction) { } } +async function handleExternalCatalogDiscover() { + const form = document.getElementById("external-catalog-form"); + if (!form) { + return; + } + const manifestInput = form.querySelector('[name="manifest_url"]'); + const nameInput = form.querySelector('[name="name"]'); + const manifestUrl = manifestInput ? manifestInput.value.trim() : ""; + if (!manifestUrl) { + setMessage("external-catalog-message", "Manifest URL is required.", true); + return; + } + try { + setMessage("external-catalog-message", "Discovering catalogs..."); + const response = await requestJSON("/api/stremio-addon/external-catalogs/discover", { + method: "POST", + body: JSON.stringify({ manifest_url: manifestUrl }), + }); + addonState.discoveredExternalCatalogs = Array.isArray(response.catalogs) + ? response.catalogs + : []; + addonState.discoveredExternalManifestUrl = response.manifest_url || manifestUrl; + addonState.discoveredExternalAddonName = response.addon_name || ""; + renderExternalCatalogDiscovery(); + if (nameInput && !nameInput.value.trim() && addonState.discoveredExternalCatalogs.length) { + nameInput.value = addonState.discoveredExternalCatalogs[0].name || ""; + } + setMessage( + "external-catalog-message", + addonState.discoveredExternalCatalogs.length + ? `Found ${addonState.discoveredExternalCatalogs.length} catalog(s).` + : "No supported catalogs found." + ); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogCreate(data, form) { + const catalogKey = data.get("catalog_key") || ""; + const selectedCatalog = getDiscoveredExternalCatalog(String(catalogKey)); + const name = (data.get("name") || "").trim(); + if (!addonState.discoveredExternalManifestUrl) { + setMessage("external-catalog-message", "Discover catalogs first.", true); + return; + } + if (!selectedCatalog) { + setMessage("external-catalog-message", "Choose a discovered catalog.", true); + return; + } + if (!name) { + setMessage("external-catalog-message", "Name is required.", true); + return; + } + try { + const response = await requestJSON("/api/stremio-addon/external-catalogs", { + method: "POST", + body: JSON.stringify({ + name, + manifest_url: addonState.discoveredExternalManifestUrl, + addon_name: addonState.discoveredExternalAddonName || null, + source_catalog_id: selectedCatalog.id, + source_catalog_type: selectedCatalog.type, + }), + }); + addonState.externalCatalogs = [...addonState.externalCatalogs, response]; + renderExternalCatalogs(); + if (form) { + form.reset(); + } + addonState.discoveredExternalCatalogs = []; + addonState.discoveredExternalManifestUrl = ""; + addonState.discoveredExternalAddonName = ""; + renderExternalCatalogDiscovery(); + setMessage("external-catalog-message", "External catalog added."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogUpdate(catalogId, card) { + const nameInput = card.querySelector("[data-external-catalog-name]"); + const enabledInput = card.querySelector("[data-external-catalog-enabled]"); + const showWatchedInput = card.querySelector("[data-external-catalog-show-watched]"); + const showInHomeInput = card.querySelector("[data-external-catalog-show-in-home]"); + const orderBySelect = card.querySelector("[data-external-catalog-order-by]"); + const orderDirSelect = card.querySelector("[data-external-catalog-order-dir]"); + const pageSizeInput = card.querySelector("[data-external-catalog-page-size]"); + const payload = { + name: nameInput ? nameInput.value.trim() : "", + enabled: enabledInput ? enabledInput.checked : true, + filters: { + show_watched: showWatchedInput ? showWatchedInput.checked : false, + statuses: [], + }, + order_by: orderBySelect ? orderBySelect.value : "source", + order_dir: orderDirSelect ? orderDirSelect.value : "asc", + page_size: pageSizeInput ? Number(pageSizeInput.value || 30) : 30, + show_in_home: showInHomeInput ? showInHomeInput.checked : true, + }; + if (!payload.name) { + setMessage("external-catalog-message", "Name is required.", true); + return; + } + try { + const response = await requestJSON(`/api/stremio-addon/external-catalogs/${catalogId}`, { + method: "PATCH", + body: JSON.stringify(payload), + }); + addonState.externalCatalogs = addonState.externalCatalogs.map((entry) => + entry.id === catalogId ? response : entry + ); + renderExternalCatalogs(); + setMessage("external-catalog-message", "External catalog updated."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogDelete(catalogId) { + const confirmDelete = window.confirm( + "Delete this external catalog? Cached items will be removed from librarySync." + ); + if (!confirmDelete) { + return; + } + try { + await requestJSON(`/api/stremio-addon/external-catalogs/${catalogId}`, { + method: "DELETE", + }); + addonState.externalCatalogs = addonState.externalCatalogs.filter( + (entry) => entry.id !== catalogId + ); + renderExternalCatalogs(); + setMessage("external-catalog-message", "External catalog deleted."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogRefresh(catalogId) { + try { + const response = await requestJSON( + `/api/stremio-addon/external-catalogs/${catalogId}/refresh`, + { method: "POST" } + ); + addonState.externalCatalogs = addonState.externalCatalogs.map((entry) => + entry.id === catalogId ? response.catalog : entry + ); + renderExternalCatalogs(); + setMessage( + "external-catalog-message", + `External catalog refreshed (${response.item_count || 0} items).` + ); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + async function handleCustomCatalogCreate(data, form) { setMessage("custom-catalog-message", ""); const payload = { @@ -1009,11 +1414,16 @@ async function loadAddonConfig() { const data = await requestJSON("/api/stremio-addon/config"); addonState.config = data; addonState.catalogs = Array.isArray(data.catalogs) ? data.catalogs : []; + addonState.externalCatalogs = Array.isArray(data.external_catalogs) + ? data.external_catalogs + : []; addonState.customCatalogs = Array.isArray(data.custom_catalogs) ? data.custom_catalogs : []; updateInstallLinks(data); renderInstallSection(); renderControlSection(); renderBuiltInCatalogs(); + renderExternalCatalogDiscovery(); + renderExternalCatalogs(); renderCustomCatalogs(); } catch (error) { setAddonMessage(error.message, true); @@ -1075,6 +1485,34 @@ function bindAddonActions() { }); } + const externalDiscover = document.getElementById("external-catalog-discover"); + if (externalDiscover) { + externalDiscover.addEventListener("click", handleExternalCatalogDiscover); + } + + const externalList = document.getElementById("external-catalog-list"); + if (externalList) { + externalList.addEventListener("click", (event) => { + const update = event.target.closest("[data-external-catalog-save]"); + if (update) { + const card = update.closest("[data-external-catalog-id]"); + if (card) { + handleExternalCatalogUpdate(update.dataset.externalCatalogSave, card); + } + return; + } + const refresh = event.target.closest("[data-external-catalog-refresh]"); + if (refresh) { + handleExternalCatalogRefresh(refresh.dataset.externalCatalogRefresh); + return; + } + const remove = event.target.closest("[data-external-catalog-delete]"); + if (remove) { + handleExternalCatalogDelete(remove.dataset.externalCatalogDelete); + } + }); + } + const customList = document.getElementById("custom-catalog-list"); if (customList) { customList.addEventListener("click", (event) => { @@ -1149,6 +1587,7 @@ window.librarysyncPageInit = async ({ user }) => { if (!user) { return; } + bindForm("external-catalog-form", handleExternalCatalogCreate); bindForm("custom-catalog-form", handleCustomCatalogCreate); bindForm("custom-catalog-items-lookup-form", handleCustomLookupSubmit); bindAddonActions(); diff --git a/backend/src/librarysync/templates/stremio-addon.html b/backend/src/librarysync/templates/stremio-addon.html index b2ef12f..0545ef1 100644 --- a/backend/src/librarysync/templates/stremio-addon.html +++ b/backend/src/librarysync/templates/stremio-addon.html @@ -7,7 +7,7 @@

Publish your watchlists.

- Turn librarySync lists into Stremio catalogs with per-catalog filters and custom picks. + Turn librarySync lists and external addon feeds into Stremio catalogs with per-catalog filters.

@@ -72,6 +72,45 @@

Built-in catalogs

Loading...
+
+
+
+

External catalogs

+

Pull catalogs from other Stremio addons and filter out watched items.

+
+
+
+ + + +
+ + + +
+
+
Loading...
+
+
diff --git a/backend/src/librarysync/worker.py b/backend/src/librarysync/worker.py index 09c1360..8a3e2b7 100644 --- a/backend/src/librarysync/worker.py +++ b/backend/src/librarysync/worker.py @@ -5,6 +5,7 @@ from typing import Awaitable, Callable from librarysync.config import settings +from librarysync.jobs.external_catalog_refresh import process_external_catalog_refresh_once from librarysync.jobs.imports import process_import_all_once, process_quick_import_once from librarysync.jobs.merge_history import ( process_merge_all_history_once, @@ -34,6 +35,9 @@ class ModeConfig: "metadata_cache": ModeConfig("metadata_cache", process_metadata_cache_refresh_once, 60.0, 5.0), "quick_import": ModeConfig("quick_import", process_quick_import_once, 2.0, 0.5), "import_all": ModeConfig("import_all", process_import_all_once, 2.0, 0.5), + "external_catalog_refresh": ModeConfig( + "external_catalog_refresh", process_external_catalog_refresh_once, 60.0, 10.0 + ), "watchlist": ModeConfig("watchlist", process_watchlist_refresh_once, 60.0, 10.0), "merge_history": ModeConfig("merge_history", process_merge_history_once, 60.0, 10.0), "merge_all_history": ModeConfig( diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index a8cf25a..fe6b407 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -30,21 +30,63 @@ def test_merge_catalog_updates_does_not_mutate_existing(self) -> None: def test_build_manifest_includes_custom_and_enabled_catalogs(self) -> None: catalogs = build_default_catalogs() + external_catalog = SimpleNamespace( + name="Best Movies", + slug="best-movies", + enabled=True, + source_catalog_type="movie", + page_size=30, + show_in_home=True, + ) custom_catalog = SimpleNamespace( name="Curated Picks", slug="curated_picks", media_type="movie" ) - manifest = routes_stremio_addon_public._build_manifest(catalogs, [custom_catalog]) + manifest = routes_stremio_addon_public._build_manifest( + catalogs, + [external_catalog], + [custom_catalog], + ) - catalog_ids = {catalog["id"] for catalog in manifest["catalogs"]} + catalog_ids = [catalog["id"] for catalog in manifest["catalogs"]] self.assertIn("watchlist_movies", catalog_ids) self.assertIn("watchlist_shows", catalog_ids) self.assertIn("in_progress_shows", catalog_ids) + self.assertIn("best-movies", catalog_ids) self.assertIn("curated_picks", catalog_ids) self.assertNotIn("watchlist_anime", catalog_ids) + self.assertLess(catalog_ids.index("best-movies"), catalog_ids.index("curated_picks")) for catalog in manifest["catalogs"]: self.assertIn("extraSupported", catalog) + def test_external_catalog_query_hides_watched_items_by_default(self) -> None: + catalog = SimpleNamespace(id="external-1", filters={"show_watched": False}) + query = asyncio.run( + routes_stremio_addon_public._build_external_catalog_query( + "user-id", + catalog, + None, + ) + ) + + compiled = str(query.compile(compile_kwargs={"literal_binds": True})).lower() + self.assertIn("stremio_external_catalog_items", compiled) + self.assertIn("exists (select watched_items.id", compiled) + + def test_external_catalog_query_can_include_watched_items(self) -> None: + catalog = SimpleNamespace(id="external-1", filters={"show_watched": True}) + query = asyncio.run( + routes_stremio_addon_public._build_external_catalog_query( + "user-id", + catalog, + None, + ) + ) + + compiled = str(query.compile(compile_kwargs={"literal_binds": True})).lower() + self.assertIn("stremio_external_catalog_items", compiled) + self.assertNotIn("not (exists", compiled) + def test_build_meta_prefers_stremio_id(self) -> None: media_item = MediaItem( title="Example Movie", From 0dc45eac803fd1cb8ed00fc0252170481ea1294d Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:08:17 +0200 Subject: [PATCH 02/11] better list handling --- .../librarysync/api/routes_stremio_addon.py | 61 +- .../api/routes_stremio_addon_public.py | 74 +- .../src/librarysync/core/external_catalog.py | 641 +++++++++++++++++- .../src/librarysync/core/watchlist_links.py | 190 ++++++ ...c2d3_add_external_catalog_source_fields.py | 32 + backend/src/librarysync/db/models.py | 2 + .../src/librarysync/jobs/letterboxd_import.py | 8 + .../librarysync/static/page-stremio-addon.js | 44 +- .../librarysync/templates/stremio-addon.html | 10 +- backend/tests/test_stremio_addon_catalogs.py | 86 ++- pyproject.toml | 2 +- 11 files changed, 1099 insertions(+), 51 deletions(-) create mode 100644 backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index 80cb90a..f6fd557 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -15,9 +15,10 @@ from librarysync.api.deps import get_current_user, get_db from librarysync.config import settings +from librarysync.connectors.services.letterboxd import LetterboxdError from librarysync.core.catalog_ordering import CatalogOrderBy from librarysync.core.external_catalog import ( - discover_external_catalogs, + discover_external_catalog_source, external_catalog_out, mark_external_catalog_refresh_failed, normalize_external_filters, @@ -26,6 +27,8 @@ normalize_external_order_dir, normalize_external_page_size, normalize_external_show_in_home, + normalize_external_source_kind, + normalize_external_source_provider, refresh_external_catalog, ) from librarysync.core.stremio_addon import ( @@ -117,12 +120,16 @@ class StremioCustomCatalogReorder(BaseModel): class StremioExternalCatalogDiscoverPayload(BaseModel): - manifest_url: str + source_url: str | None = None + manifest_url: str | None = None class StremioExternalCatalogCreate(BaseModel): name: str - manifest_url: str + manifest_url: str | None = None + source_url: str | None = None + source_kind: Literal["manifest", "list"] | None = None + source_provider: Literal["tmdb", "tvdb", "letterboxd", "mdblist"] | None = None addon_name: str | None = None source_catalog_id: str source_catalog_type: Literal["movie", "series"] @@ -472,7 +479,7 @@ async def _refresh_external_catalog_or_502( await db.commit() await db.refresh(catalog) return count - except httpx.HTTPError as exc: + except (httpx.HTTPError, LetterboxdError) as exc: await db.rollback() await mark_external_catalog_refresh_failed(db, catalog, exc) await db.commit() @@ -564,20 +571,26 @@ async def update_stremio_addon_config( ) async def discover_stremio_external_catalogs( payload: StremioExternalCatalogDiscoverPayload, - _: User = Depends(get_current_user), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), ) -> dict: - manifest_url = normalize_external_manifest_url(payload.manifest_url) - if not manifest_url: + source_url = (payload.source_url or payload.manifest_url or "").strip() + if not source_url: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="Manifest URL is required", + detail="Source URL is required", ) try: - return await discover_external_catalogs(manifest_url) - except httpx.HTTPError as exc: + return await discover_external_catalog_source(db, current_user.id, source_url) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except (httpx.HTTPError, LetterboxdError) as exc: raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"Failed to fetch addon manifest: {exc}", + detail=f"Failed to fetch external source: {exc}", ) from exc @@ -614,12 +627,21 @@ async def create_external_catalog( name = payload.name.strip() if not name: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") - manifest_url = normalize_external_manifest_url(payload.manifest_url) + source_kind = normalize_external_source_kind(payload.source_kind) + source_provider = normalize_external_source_provider(payload.source_provider) + source_url = (payload.source_url or payload.manifest_url or "").strip() + manifest_url = ( + normalize_external_manifest_url(source_url) + if source_kind == "manifest" + else source_url + ) if not manifest_url: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Manifest URL is required", + detail = ( + "Manifest URL is required" + if source_kind == "manifest" + else "Source URL is required" ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) source_catalog_id = payload.source_catalog_id.strip() if not source_catalog_id: raise HTTPException( @@ -631,11 +653,18 @@ async def create_external_catalog( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid source catalog type", ) + if source_kind == "list" and not source_provider: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="List provider is required", + ) catalog = StremioExternalCatalog( user_id=current_user.id, name=name, slug=await _build_unique_slug(db, current_user.id, name), + source_kind=source_kind, + source_provider=source_provider, addon_name=(payload.addon_name or "").strip() or None, manifest_url=manifest_url, source_catalog_id=source_catalog_id, @@ -659,7 +688,7 @@ async def create_external_catalog( await refresh_external_catalog(db, catalog) await db.commit() await db.refresh(catalog) - except httpx.HTTPError as exc: + except (httpx.HTTPError, LetterboxdError) as exc: await db.rollback() raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, diff --git a/backend/src/librarysync/api/routes_stremio_addon_public.py b/backend/src/librarysync/api/routes_stremio_addon_public.py index f5e642b..e830695 100644 --- a/backend/src/librarysync/api/routes_stremio_addon_public.py +++ b/backend/src/librarysync/api/routes_stremio_addon_public.py @@ -79,6 +79,25 @@ def _build_meta(media_item: MediaItem, catalog_type: str) -> dict[str, Any] | No return meta +def _build_external_item_meta( + item: StremioExternalCatalogItem, + media_item: MediaItem | None, + catalog_type: str, +) -> dict[str, Any] | None: + if media_item is not None: + return _build_meta(media_item, catalog_type) + stremio_id = str(item.stremio_id or "").strip() + stremio_type = str(item.stremio_type or "").strip().lower() + if not stremio_id or not stremio_type or stremio_type != catalog_type: + return None + meta = {"id": stremio_id, "type": stremio_type, "name": item.title or stremio_id} + if item.poster_url: + meta["poster"] = item.poster_url + if item.year: + meta["year"] = item.year + return meta + + def _extract_extra_param(request: Request, name: str) -> str | None: return request.query_params.get(name) or request.query_params.get(f"extra[{name}]") @@ -377,11 +396,11 @@ async def _build_external_catalog_query( search: str | None, ): query = ( - select(MediaItem) - .join(StremioExternalCatalogItem, StremioExternalCatalogItem.media_item_id == MediaItem.id) + select(StremioExternalCatalogItem, MediaItem) + .select_from(StremioExternalCatalogItem) + .outerjoin(MediaItem, StremioExternalCatalogItem.media_item_id == MediaItem.id) .where( StremioExternalCatalogItem.catalog_id == catalog.id, - StremioExternalCatalogItem.media_item_id.is_not(None), ) ) filters = normalize_external_filters(catalog.filters) @@ -402,9 +421,32 @@ async def _build_external_catalog_query( progress_subq.c.total_released > 0, progress_subq.c.watched_count >= progress_subq.c.total_released, ) - query = query.where(~or_(directly_watched_exists, completed_show_clause)) + query = query.where( + or_( + StremioExternalCatalogItem.media_item_id.is_(None), + ~or_(directly_watched_exists, completed_show_clause), + ) + ) if search: - query = _apply_search_filter(query, search) + normalized_search = search.strip() + if normalized_search: + like_value = f"%{normalized_search}%" + search_clauses = [ + StremioExternalCatalogItem.title.ilike(like_value), + StremioExternalCatalogItem.imdb_id.ilike(like_value), + MediaItem.title.ilike(like_value), + MediaItem.imdb_id.ilike(like_value), + MediaItem.tmdb_id.ilike(like_value), + MediaItem.tvdb_id.ilike(like_value), + MediaItem.tvmaze_id.ilike(like_value), + MediaItem.kitsu_id.ilike(like_value), + MediaItem.myanimelist_id.ilike(like_value), + MediaItem.anilist_id.ilike(like_value), + ] + if normalized_search.isdigit() and len(normalized_search) == 4: + search_clauses.append(StremioExternalCatalogItem.year == int(normalized_search)) + search_clauses.append(MediaItem.year == int(normalized_search)) + query = query.where(or_(*search_clauses)) return query @@ -607,13 +649,21 @@ async def _serve_catalog( ) result = await db.execute(query.offset(skip).limit(limit + 1)) - media_items = result.scalars().all() - has_more = len(media_items) > limit + rows = result.all() + has_more = len(rows) > limit - metas = [ - meta - for media in media_items[:limit] - if (meta := _build_meta(media, catalog_type)) is not None - ] + if external_catalog: + metas = [ + meta + for item, media in rows[:limit] + if (meta := _build_external_item_meta(item, media, catalog_type)) is not None + ] + else: + media_items = [row[0] if isinstance(row, tuple) else row for row in rows] + metas = [ + meta + for media in media_items[:limit] + if (meta := _build_meta(media, catalog_type)) is not None + ] return {"metas": metas, "hasMore": has_more} if has_more else {"metas": metas} diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index 57052f4..72da883 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -1,16 +1,37 @@ from __future__ import annotations +import asyncio import uuid +from dataclasses import dataclass from datetime import datetime, timezone from typing import Any +import httpx from sqlalchemy import delete, or_, select from sqlalchemy.ext.asyncio import AsyncSession from librarysync.config import settings +from librarysync.connectors.metadata.tmdb import TmdbMetadataProvider +from librarysync.connectors.metadata.tvdb import TvdbMetadataProvider +from librarysync.connectors.services.letterboxd import ( + DEFAULT_LETTERBOXD_API_BASE_URL, + LetterboxdClient, + has_required_letterboxd_fields, +) from librarysync.core.catalog_ordering import CatalogOrderBy from librarysync.core.http_client import get_http_client +from librarysync.core.integrations import load_integration_with_secrets +from librarysync.core.metadata_providers import MetadataProviderService +from librarysync.core.watchlist_links import ( + parse_imdb_chart_urls, + parse_letterboxd_list_urls, + parse_mdblist_urls, + parse_tmdb_chart_urls, + parse_tmdb_list_urls, + parse_tvdb_list_urls, +) from librarysync.db.models import MediaItem, StremioExternalCatalog, StremioExternalCatalogItem +from librarysync.jobs.letterboxd_import import build_letterboxd_list_candidate EXTERNAL_CATALOG_ORDER_BY = {"source", "random", *CatalogOrderBy.__args__} EXTERNAL_CATALOG_ORDER_DIR = {"asc", "desc"} @@ -18,6 +39,31 @@ EXTERNAL_CATALOG_DEFAULT_SHOW_IN_HOME = True EXTERNAL_CATALOG_DEFAULT_FILTERS = {"show_watched": False, "statuses": []} EXTERNAL_CATALOG_FETCH_PAGE_SIZE = 100 +EXTERNAL_SOURCE_KINDS = {"manifest", "list"} +EXTERNAL_LIST_PROVIDERS = {"tmdb", "tvdb", "letterboxd", "mdblist"} +TMDB_LIST_TYPE = "movie" +TMDB_CHART_ENDPOINTS = { + "tmdb-chart:movie:top-rated": ("/movie/top_rated", "movie"), + "tmdb-chart:movie:popular": ("/movie/popular", "movie"), + "tmdb-chart:movie:now-playing": ("/movie/now_playing", "movie"), + "tmdb-chart:movie:upcoming": ("/movie/upcoming", "movie"), + "tmdb-chart:tv:top-rated": ("/tv/top_rated", "series"), + "tmdb-chart:tv:popular": ("/tv/popular", "series"), + "tmdb-chart:tv:on-the-air": ("/tv/on_the_air", "series"), + "tmdb-chart:tv:airing-today": ("/tv/airing_today", "series"), +} + + +@dataclass(frozen=True) +class ExternalCatalogListItem: + stremio_id: str + stremio_type: str + title: str | None + year: int | None + poster_url: str | None + imdb_id: str | None = None + tmdb_id: str | None = None + tvdb_id: str | None = None def normalize_external_manifest_url(value: str) -> str: @@ -29,6 +75,16 @@ def normalize_external_manifest_url(value: str) -> str: return f"{manifest_url.rstrip('/')}/manifest.json" +def normalize_external_source_kind(value: str | None) -> str: + normalized = (value or "manifest").strip().lower() + return normalized if normalized in EXTERNAL_SOURCE_KINDS else "manifest" + + +def normalize_external_source_provider(value: str | None) -> str | None: + normalized = (value or "").strip().lower() + return normalized if normalized in EXTERNAL_LIST_PROVIDERS else None + + def normalize_external_filters(filters: dict[str, Any] | None) -> dict[str, Any]: base = filters if isinstance(filters, dict) else {} statuses = base.get("statuses") if isinstance(base.get("statuses"), list) else [] @@ -57,10 +113,16 @@ def normalize_external_show_in_home(value: bool | None) -> bool: def external_catalog_out(catalog: StremioExternalCatalog) -> dict[str, Any]: + source_kind = normalize_external_source_kind(getattr(catalog, "source_kind", None)) return { "id": catalog.id, "name": catalog.name, "slug": catalog.slug, + "source_kind": source_kind, + "source_provider": normalize_external_source_provider( + getattr(catalog, "source_provider", None) + ), + "source_url": catalog.manifest_url, "addon_name": catalog.addon_name, "manifest_url": catalog.manifest_url, "source_catalog_id": catalog.source_catalog_id, @@ -99,6 +161,49 @@ def stremio_type_to_media_types(stremio_type: str) -> tuple[str, ...]: async def discover_external_catalogs(manifest_url: str) -> dict[str, Any]: + raise NotImplementedError("Use discover_external_catalog_source") + + +async def discover_external_catalog_source( + db: AsyncSession, + user_id: str, + source_url: str, +) -> dict[str, Any]: + cleaned_url = (source_url or "").strip() + if not cleaned_url: + raise ValueError("Source URL is required") + + tmdb_refs = parse_tmdb_list_urls([cleaned_url]) + if tmdb_refs: + return await _discover_tmdb_list_catalogs(db, user_id, tmdb_refs[0]) + + tmdb_chart_refs = parse_tmdb_chart_urls([cleaned_url]) + if tmdb_chart_refs: + return await _discover_tmdb_chart_catalogs(tmdb_chart_refs[0]) + + tvdb_refs = parse_tvdb_list_urls([cleaned_url]) + if tvdb_refs: + return await _discover_tvdb_list_catalogs(db, user_id, tvdb_refs[0]) + + letterboxd_refs = parse_letterboxd_list_urls([cleaned_url]) + if letterboxd_refs: + return await _discover_letterboxd_list_catalogs(db, user_id, letterboxd_refs[0]) + + mdblist_refs = parse_mdblist_urls([cleaned_url]) + if mdblist_refs: + return await _discover_mdblist_catalogs(mdblist_refs[0]) + + imdb_chart_refs = parse_imdb_chart_urls([cleaned_url]) + if imdb_chart_refs: + raise ValueError( + "IMDb chart pages are not supported. Use TMDB charts/lists, TVDB lists, " + "Letterboxd lists, or a Stremio manifest." + ) + + return await _discover_manifest_catalogs(cleaned_url) + + +async def _discover_manifest_catalogs(manifest_url: str) -> dict[str, Any]: normalized_url = normalize_external_manifest_url(manifest_url) if not normalized_url: raise ValueError("Manifest URL is required") @@ -122,10 +227,15 @@ async def discover_external_catalogs(manifest_url: str) -> dict[str, Any]: "id": catalog_id, "name": str(catalog.get("name") or catalog_id), "type": catalog_type, + "source_kind": "manifest", + "source_provider": None, } ) return { + "source_kind": "manifest", + "source_provider": None, + "source_url": normalized_url, "manifest_url": normalized_url, "addon_name": addon_name, "catalogs": sorted(discovered, key=lambda entry: (entry["type"], entry["name"].lower())), @@ -138,6 +248,9 @@ async def refresh_external_catalog( *, max_items: int | None = None, ) -> int: + if normalize_external_source_kind(getattr(catalog, "source_kind", None)) == "list": + return await _refresh_external_list_catalog(db, catalog, max_items=max_items) + normalized_manifest_url = normalize_external_manifest_url(catalog.manifest_url) max_catalog_items = max_items or settings.external_catalog_max_items fetched_at = datetime.now(timezone.utc) @@ -221,6 +334,82 @@ async def mark_external_catalog_refresh_failed( await db.flush() +async def _refresh_external_list_catalog( + db: AsyncSession, + catalog: StremioExternalCatalog, + *, + max_items: int | None = None, +) -> int: + provider = normalize_external_source_provider(catalog.source_provider) + if not provider: + raise ValueError("List provider is required") + + max_catalog_items = max_items or settings.external_catalog_max_items + if provider == "tmdb": + items = await _load_tmdb_list_items( + db, + catalog.user_id, + catalog.manifest_url, + catalog.source_catalog_id, + max_catalog_items, + ) + elif provider == "tvdb": + items = await _load_tvdb_list_items( + db, catalog.user_id, catalog.manifest_url, max_catalog_items + ) + elif provider == "letterboxd": + items = await _load_letterboxd_list_items( + db, catalog.user_id, catalog.manifest_url, max_catalog_items + ) + elif provider == "mdblist": + items = await _load_mdblist_items(catalog.manifest_url, max_catalog_items) + else: + raise ValueError("Unsupported list provider") + + items = [item for item in items if item.stremio_type == catalog.source_catalog_type] + + fetched_at = datetime.now(timezone.utc) + await db.execute( + delete(StremioExternalCatalogItem).where( + StremioExternalCatalogItem.catalog_id == catalog.id + ) + ) + + created = 0 + for position, item in enumerate(items): + media_item_id = await _resolve_external_media_item_id( + db, + item.stremio_id, + item.imdb_id, + item.stremio_type, + tmdb_id=item.tmdb_id, + tvdb_id=item.tvdb_id, + ) + db.add( + StremioExternalCatalogItem( + id=str(uuid.uuid4()), + catalog_id=catalog.id, + media_item_id=media_item_id, + stremio_id=item.stremio_id, + stremio_type=item.stremio_type, + title=item.title, + year=item.year, + poster_url=item.poster_url, + imdb_id=item.imdb_id, + position=position, + fetched_at=fetched_at, + ) + ) + created += 1 + + catalog.last_refreshed_at = fetched_at + catalog.last_refresh_error = None + catalog.updated_at = fetched_at + db.add(catalog) + await db.flush() + return created + + def _coerce_text(value: object) -> str | None: if value is None: return None @@ -231,8 +420,12 @@ def _coerce_text(value: object) -> str | None: def _coerce_year(value: object) -> int | None: if isinstance(value, int): return value - if isinstance(value, str) and value.strip().isdigit(): - return int(value.strip()) + if isinstance(value, str): + cleaned = value.strip() + if cleaned.isdigit(): + return int(cleaned) + if len(cleaned) >= 4 and cleaned[:4].isdigit(): + return int(cleaned[:4]) return None @@ -250,6 +443,9 @@ async def _resolve_external_media_item_id( stremio_id: str, imdb_id: str | None, stremio_type: str, + *, + tmdb_id: str | None = None, + tvdb_id: str | None = None, ) -> str | None: media_types = stremio_type_to_media_types(stremio_type) if not media_types: @@ -258,6 +454,10 @@ async def _resolve_external_media_item_id( conditions = [MediaItem.raw["stremio_id"].as_string() == stremio_id] if imdb_id: conditions.append(MediaItem.imdb_id == imdb_id) + if tmdb_id: + conditions.append(MediaItem.tmdb_id == tmdb_id) + if tvdb_id: + conditions.append(MediaItem.tvdb_id == tvdb_id) result = await db.execute( select(MediaItem.id) .where(MediaItem.media_type.in_(media_types), or_(*conditions)) @@ -265,3 +465,440 @@ async def _resolve_external_media_item_id( ) media_item_id = result.scalars().first() return str(media_item_id) if media_item_id else None + + +async def _discover_tmdb_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + provider = await _load_tmdb_provider(db, user_id) + payload = await provider._get(f"/list/{ref.list_id}", {"page": 1}) + name = str(payload.get("name") or ref.name).strip() or ref.name + return { + "source_kind": "list", + "source_provider": "tmdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TMDB List", + "catalogs": [{"id": ref.external_id, "name": name, "type": TMDB_LIST_TYPE}], + } + + +async def _discover_tmdb_chart_catalogs(ref) -> dict[str, Any]: + return { + "source_kind": "list", + "source_provider": "tmdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TMDB Charts", + "catalogs": [ + { + "id": ref.external_id, + "name": ref.name, + "type": "movie" if ref.media_type == "movie" else "series", + } + ], + } + + +async def _discover_tvdb_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + provider = await _load_tvdb_provider(db, user_id) + payload = await _tvdb_get_list_payload(provider, ref.list_id) + data = payload.get("data") if isinstance(payload, dict) else None + name = str((data or {}).get("name") or ref.name).strip() or ref.name + catalog_types = _detect_tvdb_catalog_types(data) + return { + "source_kind": "list", + "source_provider": "tvdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TVDB List", + "catalogs": [ + { + "id": f"{ref.external_id}:{catalog_type}", + "name": name, + "type": catalog_type, + } + for catalog_type in catalog_types + ], + } + + +async def _discover_letterboxd_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + client, access_token = await _build_letterboxd_client(db, user_id) + payload = await client.fetch_list_by_slug(access_token, ref.username, ref.slug) + name = str(payload.get("name") or ref.name).strip() or ref.name + return { + "source_kind": "list", + "source_provider": "letterboxd", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "Letterboxd List", + "catalogs": [{"id": ref.external_id, "name": name, "type": "movie"}], + } + + +async def _discover_mdblist_catalogs(ref) -> dict[str, Any]: + items = await _load_mdblist_items(ref.url, max_items=50) + catalog_types = sorted({item.stremio_type for item in items}) + if not catalog_types: + catalog_types = ["movie"] + return { + "source_kind": "list", + "source_provider": "mdblist", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "MDBList", + "catalogs": [ + { + "id": f"{ref.external_id}:{catalog_type}", + "name": ref.name.replace("-", " ").title(), + "type": catalog_type, + } + for catalog_type in catalog_types + ], + } + + +async def _load_tmdb_provider(db: AsyncSession, user_id: str) -> TmdbMetadataProvider: + provider = await MetadataProviderService(db, user_id).load_provider("tmdb") + if not isinstance(provider, TmdbMetadataProvider): + raise ValueError("TMDB provider is not enabled or missing API key") + return provider + + +async def _load_tvdb_provider(db: AsyncSession, user_id: str) -> TvdbMetadataProvider: + provider = await MetadataProviderService(db, user_id).load_provider("tvdb") + if not isinstance(provider, TvdbMetadataProvider): + raise ValueError("TVDB provider is not enabled or missing API key") + return provider + + +async def _load_tmdb_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + source_catalog_id: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + provider = await _load_tmdb_provider(db, user_id) + chart_ref = TMDB_CHART_ENDPOINTS.get(source_catalog_id) + if chart_ref: + endpoint, stremio_type = chart_ref + return await _load_tmdb_chart_items(provider, endpoint, stremio_type, max_items) + + refs = parse_tmdb_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported TMDB list URL") + ref = refs[0] + + results: list[dict[str, Any]] = [] + page = 1 + while len(results) < max_items: + payload = await provider._get(f"/list/{ref.list_id}", {"page": page}) + batch = payload.get("items") if isinstance(payload, dict) else None + if not isinstance(batch, list) or not batch: + break + results.extend(item for item in batch if isinstance(item, dict)) + total_pages = int(payload.get("total_pages") or page) + if page >= total_pages: + break + page += 1 + + detail_items = await _load_tmdb_external_ids(provider, results[:max_items]) + return [item for item in detail_items if item.stremio_id] + + +async def _load_tmdb_chart_items( + provider: TmdbMetadataProvider, + endpoint: str, + stremio_type: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + results: list[dict[str, Any]] = [] + page = 1 + while len(results) < max_items: + payload = await provider._get(endpoint, {"page": page}) + batch = payload.get("results") if isinstance(payload, dict) else None + if not isinstance(batch, list) or not batch: + break + results.extend(item for item in batch if isinstance(item, dict)) + total_pages = int(payload.get("total_pages") or page) + if page >= total_pages: + break + page += 1 + return await _load_tmdb_external_ids( + provider, + results[:max_items], + stremio_type=stremio_type, + ) + + +async def _load_tmdb_external_ids( + provider: TmdbMetadataProvider, + items: list[dict[str, Any]], + *, + stremio_type: str = "movie", +) -> list[ExternalCatalogListItem]: + semaphore = asyncio.Semaphore(8) + + async def _build(item: dict[str, Any]) -> ExternalCatalogListItem | None: + tmdb_id = _coerce_text(item.get("id")) + if not tmdb_id: + return None + details_path = "/movie" if stremio_type == "movie" else "/tv" + async with semaphore: + payload = await provider._get(f"{details_path}/{tmdb_id}/external_ids", {}) + imdb_id = _coerce_text(payload.get("imdb_id")) + stremio_id = imdb_id + if not stremio_id: + return None + return ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type=stremio_type, + title=_coerce_text( + item.get("title") + or item.get("original_title") + or item.get("name") + or item.get("original_name") + ), + year=_coerce_year( + item.get("release_date") + or item.get("first_air_date") + or item.get("year") + ), + poster_url=_tmdb_poster_url(item.get("poster_path")), + imdb_id=imdb_id, + tmdb_id=tmdb_id, + ) + + results = await asyncio.gather(*[_build(item) for item in items]) + return [item for item in results if item is not None] + + +async def _load_tvdb_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + refs = parse_tvdb_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported TVDB list URL") + ref = refs[0] + provider = await _load_tvdb_provider(db, user_id) + payload = await _tvdb_get_list_payload(provider, ref.list_id) + data = payload.get("data") if isinstance(payload, dict) else None + entities = (data or {}).get("entities") if isinstance(data, dict) else None + if not isinstance(entities, list): + return [] + + items: list[ExternalCatalogListItem] = [] + for entry in entities[:max_items]: + if not isinstance(entry, dict): + continue + media_type = _normalize_tvdb_entity_type(entry) + if not media_type: + continue + imdb_id = _extract_remote_id(entry, "imdb") + tvdb_id = _coerce_text(entry.get("id")) + stremio_id = imdb_id + if not stremio_id: + continue + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie" if media_type == "movie" else "series", + title=_coerce_text(entry.get("name") or entry.get("title") or entry.get("slug")), + year=_coerce_year( + entry.get("year") + or entry.get("firstAired") + or entry.get("first_aired") + or entry.get("releaseDate") + ), + poster_url=_coerce_text( + entry.get("image") or entry.get("image_url") or entry.get("imageUrl") + ), + imdb_id=imdb_id, + tmdb_id=_extract_remote_id(entry, "tmdb"), + tvdb_id=tvdb_id, + ) + ) + return items + + +async def _load_letterboxd_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + refs = parse_letterboxd_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported Letterboxd list URL") + ref = refs[0] + client, access_token = await _build_letterboxd_client(db, user_id) + entries = await client.get_list_entries( + access_token, + ref.username, + ref.slug, + per_page=50, + max_pages=10, + ) + items: list[ExternalCatalogListItem] = [] + list_context = {"name": ref.name, "url": ref.url, "type": "list"} + for entry in entries: + candidate = build_letterboxd_list_candidate(entry, list_context=list_context) + if not candidate: + continue + imdb_id = candidate.ids.get("imdb_id") + stremio_id = imdb_id + if not stremio_id: + continue + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie", + title=candidate.title, + year=candidate.year, + poster_url=candidate.poster_url, + imdb_id=imdb_id, + tmdb_id=candidate.ids.get("tmdb_id"), + ) + ) + if len(items) >= max_items: + break + return items + + +async def _load_mdblist_items(source_url: str, max_items: int) -> list[ExternalCatalogListItem]: + refs = parse_mdblist_urls([source_url]) + if not refs: + raise ValueError("Unsupported MDBList URL") + ref = refs[0] + json_url = f"https://mdblist.com/lists/{ref.username}/{ref.slug}/json/" + async with get_http_client() as client: + response = await client.get(json_url) + response.raise_for_status() + payload = response.json() + + if not isinstance(payload, list): + raise ValueError("MDBList list payload is invalid") + + items: list[ExternalCatalogListItem] = [] + for entry in payload[:max_items]: + if not isinstance(entry, dict): + continue + imdb_id = _coerce_text(entry.get("imdb_id")) + stremio_id = imdb_id + if not stremio_id: + continue + media_type = _coerce_text(entry.get("mediatype")) or "movie" + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie" if media_type == "movie" else "series", + title=_coerce_text(entry.get("title")), + year=_coerce_year(entry.get("release_year") or entry.get("year")), + poster_url=None, + imdb_id=imdb_id, + tmdb_id=_coerce_text( + entry.get("tmdb_id") or entry.get("tmdbid") or entry.get("id") + ), + tvdb_id=_coerce_text(entry.get("tvdb_id") or entry.get("tvdbid")), + ) + ) + return items + + +async def _build_letterboxd_client( + db: AsyncSession, user_id: str +) -> tuple[LetterboxdClient, str]: + integration, secret_data = await load_integration_with_secrets(db, user_id, "letterboxd") + if not integration or integration.status == "disconnected": + raise ValueError("Letterboxd integration is not connected") + if not secret_data or not has_required_letterboxd_fields(secret_data): + raise ValueError("Letterboxd credentials are incomplete") + api_base_url = DEFAULT_LETTERBOXD_API_BASE_URL + if integration.config and integration.config.get("api_base_url"): + api_base_url = str(integration.config["api_base_url"]) + client = LetterboxdClient( + api_base_url=api_base_url, + client_id=str(secret_data.get("client_id")), + client_secret=str(secret_data.get("client_secret")), + refresh_token=str(secret_data.get("refresh_token")), + ) + access_token = await client.refresh_access_token() + return client, access_token + + +async def _tvdb_get_list_payload(provider: TvdbMetadataProvider, list_id: str) -> dict[str, Any]: + try: + return await provider._get(f"/lists/{list_id}/extended", {}) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + return await provider._get(f"/lists/slug/{list_id}", {}) + + +def _detect_tvdb_catalog_types(data: dict[str, Any] | None) -> list[str]: + entities = (data or {}).get("entities") if isinstance(data, dict) else None + if not isinstance(entities, list): + return ["series"] + has_movie = False + has_series = False + for entry in entities: + media_type = _normalize_tvdb_entity_type(entry) if isinstance(entry, dict) else None + if media_type == "movie": + has_movie = True + elif media_type == "tv": + has_series = True + if has_movie and has_series: + return ["movie", "series"] + if has_movie: + return ["movie"] + return ["series"] + + +def _normalize_tvdb_entity_type(entry: dict[str, Any]) -> str | None: + raw = _coerce_text(entry.get("type") or entry.get("recordType")) + if not raw: + raw = _coerce_text(entry.get("companyType") or entry.get("kind")) + if not raw: + return None + lowered = raw.lower() + if lowered in {"movie", "movies"}: + return "movie" + if lowered in {"series", "tv", "show"}: + return "tv" + return None + + +def _extract_remote_id(entry: dict[str, Any], provider: str) -> str | None: + remote_ids = entry.get("remoteIds") or entry.get("remote_ids") or [] + if not isinstance(remote_ids, list): + return None + for remote in remote_ids: + if not isinstance(remote, dict): + continue + source_name = str(remote.get("sourceName") or "").lower() + source = str(remote.get("source") or "").lower() + entry_type = str(remote.get("type") or "").lower() + if provider == "imdb" and ( + source_name == "imdb" or source == "imdb" or entry_type == "imdb" + ): + return _coerce_text(remote.get("id") or remote.get("value")) + if provider == "tmdb" and ( + source_name == "tmdb" + or "themoviedb" in source_name + or source == "tmdb" + or entry_type == "tmdb" + ): + return _coerce_text(remote.get("id") or remote.get("value")) + return None + + +def _tmdb_poster_url(path: object) -> str | None: + value = _coerce_text(path) + if not value: + return None + if value.startswith("http://") or value.startswith("https://"): + return value + return f"https://image.tmdb.org/t/p/w185{value}" diff --git a/backend/src/librarysync/core/watchlist_links.py b/backend/src/librarysync/core/watchlist_links.py index 93b488f..f454f90 100644 --- a/backend/src/librarysync/core/watchlist_links.py +++ b/backend/src/librarysync/core/watchlist_links.py @@ -1,9 +1,12 @@ from __future__ import annotations +import re from dataclasses import dataclass from typing import Iterable from urllib.parse import urlparse +TMDB_LIST_ID_RE = re.compile(r"^(\d+)") + @dataclass(frozen=True) class TraktListRef: @@ -23,6 +26,46 @@ class LetterboxdListRef: name: str +@dataclass(frozen=True) +class TmdbListRef: + list_id: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class TmdbChartRef: + media_type: str + chart_slug: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class TvdbListRef: + list_id: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class ImdbChartRef: + chart_slug: str + url: str + + +@dataclass(frozen=True) +class MdblistRef: + username: str + slug: str + url: str + external_id: str + name: str + + def parse_trakt_list_urls(urls: Iterable[str]) -> list[TraktListRef]: refs: list[TraktListRef] = [] for url in urls: @@ -68,6 +111,9 @@ def parse_letterboxd_list_urls(urls: Iterable[str]) -> list[LetterboxdListRef]: if not cleaned: continue parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"letterboxd.com", "www.letterboxd.com"}: + continue path = parsed.path.strip("/") segments = [segment for segment in path.split("/") if segment] if len(segments) >= 3 and segments[1] == "list": @@ -85,3 +131,147 @@ def parse_letterboxd_list_urls(urls: Iterable[str]) -> list[LetterboxdListRef]: ) return refs + +def parse_tmdb_list_urls(urls: Iterable[str]) -> list[TmdbListRef]: + refs: list[TmdbListRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"themoviedb.org", "www.themoviedb.org"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + match = ( + TMDB_LIST_ID_RE.match(segments[1]) + if len(segments) >= 2 and segments[0] == "list" + else None + ) + if match: + list_id = match.group(1) + refs.append( + TmdbListRef( + list_id=list_id, + url=cleaned, + external_id=f"tmdb:{list_id}", + name=list_id, + ) + ) + return refs + + +def parse_tmdb_chart_urls(urls: Iterable[str]) -> list[TmdbChartRef]: + refs: list[TmdbChartRef] = [] + chart_names = { + ("movie", "top-rated"): "Top Rated Movies", + ("movie", "popular"): "Popular Movies", + ("movie", "now-playing"): "Now Playing Movies", + ("movie", "upcoming"): "Upcoming Movies", + ("tv", "top-rated"): "Top Rated Series", + ("tv", "popular"): "Popular Series", + ("tv", "on-the-air"): "On The Air Series", + ("tv", "airing-today"): "Airing Today Series", + } + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"themoviedb.org", "www.themoviedb.org"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) < 2: + continue + media_type = segments[0] + chart_slug = segments[1] + name = chart_names.get((media_type, chart_slug)) + if not name: + continue + refs.append( + TmdbChartRef( + media_type=media_type, + chart_slug=chart_slug, + url=cleaned, + external_id=f"tmdb-chart:{media_type}:{chart_slug}", + name=name, + ) + ) + return refs + + +def parse_tvdb_list_urls(urls: Iterable[str]) -> list[TvdbListRef]: + refs: list[TvdbListRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"thetvdb.com", "www.thetvdb.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) >= 2 and segments[0] == "lists": + list_id = segments[1] + refs.append( + TvdbListRef( + list_id=list_id, + url=cleaned, + external_id=f"tvdb:{list_id}", + name=list_id, + ) + ) + return refs + + +def parse_imdb_chart_urls(urls: Iterable[str]) -> list[ImdbChartRef]: + refs: list[ImdbChartRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"imdb.com", "www.imdb.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if not segments: + continue + if len(segments) >= 3 and len(segments[0]) == 2 and segments[1] == "chart": + refs.append(ImdbChartRef(chart_slug=segments[2], url=cleaned)) + continue + if len(segments) >= 2 and segments[0] == "chart": + refs.append(ImdbChartRef(chart_slug=segments[1], url=cleaned)) + return refs + + +def parse_mdblist_urls(urls: Iterable[str]) -> list[MdblistRef]: + refs: list[MdblistRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"mdblist.com", "www.mdblist.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) >= 3 and segments[0] == "lists": + username = segments[1] + slug = segments[2] + refs.append( + MdblistRef( + username=username, + slug=slug, + url=cleaned, + external_id=f"mdblist:{username}:{slug}", + name=slug, + ) + ) + return refs diff --git a/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py b/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py new file mode 100644 index 0000000..af0a231 --- /dev/null +++ b/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py @@ -0,0 +1,32 @@ +"""add external catalog source fields + +Revision ID: e4f6a8b1c2d3 +Revises: d2b7c8f9a1e0 +Create Date: 2026-04-17 16:30:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "e4f6a8b1c2d3" +down_revision = "d2b7c8f9a1e0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "stremio_external_catalogs", + sa.Column("source_kind", sa.String(length=32), nullable=False, server_default="manifest"), + ) + op.add_column( + "stremio_external_catalogs", + sa.Column("source_provider", sa.String(length=32), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("stremio_external_catalogs", "source_provider") + op.drop_column("stremio_external_catalogs", "source_kind") diff --git a/backend/src/librarysync/db/models.py b/backend/src/librarysync/db/models.py index 3801e71..264a5ba 100644 --- a/backend/src/librarysync/db/models.py +++ b/backend/src/librarysync/db/models.py @@ -482,6 +482,8 @@ class StremioExternalCatalog(Base): ) name: Mapped[str] = mapped_column(String(255)) slug: Mapped[str] = mapped_column(String(64)) + source_kind: Mapped[str] = mapped_column(String(32), default="manifest") + source_provider: Mapped[str | None] = mapped_column(String(32), nullable=True) addon_name: Mapped[str | None] = mapped_column(String(255), nullable=True) manifest_url: Mapped[str] = mapped_column(String(500)) source_catalog_id: Mapped[str] = mapped_column(String(255)) diff --git a/backend/src/librarysync/jobs/letterboxd_import.py b/backend/src/librarysync/jobs/letterboxd_import.py index c7d38ef..ee51631 100644 --- a/backend/src/librarysync/jobs/letterboxd_import.py +++ b/backend/src/librarysync/jobs/letterboxd_import.py @@ -494,6 +494,14 @@ def _build_watchlist_candidate( ) +def build_letterboxd_list_candidate( + entry: dict[str, Any], + *, + list_context: dict[str, Any] | None = None, +) -> WatchlistCandidate | None: + return _build_watchlist_candidate(entry, list_context=list_context) + + async def _get_or_create_media_item( db: AsyncSession, film: FilmSummary ) -> MediaItem | None: diff --git a/backend/src/librarysync/static/page-stremio-addon.js b/backend/src/librarysync/static/page-stremio-addon.js index 5798ba6..9c51b55 100644 --- a/backend/src/librarysync/static/page-stremio-addon.js +++ b/backend/src/librarysync/static/page-stremio-addon.js @@ -4,7 +4,9 @@ const addonState = { externalCatalogs: [], customCatalogs: [], discoveredExternalCatalogs: [], - discoveredExternalManifestUrl: "", + discoveredExternalSourceUrl: "", + discoveredExternalSourceKind: "manifest", + discoveredExternalSourceProvider: "", discoveredExternalAddonName: "", selectedCatalogId: null, items: [], @@ -374,7 +376,7 @@ function renderExternalCatalogDiscovery() { if (!addonState.discoveredExternalCatalogs.length) { const option = document.createElement("option"); option.value = ""; - option.textContent = "Discover catalogs first"; + option.textContent = "Discover a source first"; select.appendChild(option); select.disabled = true; return; @@ -423,8 +425,12 @@ function renderExternalCatalogs() { title.textContent = catalog.name; const subtitle = document.createElement("p"); subtitle.className = "text-xs text-muted"; - const addonName = catalog.addon_name || "External addon"; - subtitle.textContent = `${addonName} · ${catalog.source_catalog_id} · ${EXTERNAL_TYPE_LABELS[catalog.source_catalog_type] || catalog.source_catalog_type}`; + const addonName = catalog.addon_name || "External source"; + const sourceLabel = + catalog.source_kind === "list" + ? `${(catalog.source_provider || "list").toUpperCase()} list` + : catalog.source_catalog_id; + subtitle.textContent = `${addonName} · ${sourceLabel} · ${EXTERNAL_TYPE_LABELS[catalog.source_catalog_type] || catalog.source_catalog_type}`; info.appendChild(title); info.appendChild(subtitle); @@ -1098,21 +1104,23 @@ async function handleExternalCatalogDiscover() { } const manifestInput = form.querySelector('[name="manifest_url"]'); const nameInput = form.querySelector('[name="name"]'); - const manifestUrl = manifestInput ? manifestInput.value.trim() : ""; - if (!manifestUrl) { - setMessage("external-catalog-message", "Manifest URL is required.", true); + const sourceUrl = manifestInput ? manifestInput.value.trim() : ""; + if (!sourceUrl) { + setMessage("external-catalog-message", "Manifest or list URL is required.", true); return; } try { - setMessage("external-catalog-message", "Discovering catalogs..."); + setMessage("external-catalog-message", "Discovering source..."); const response = await requestJSON("/api/stremio-addon/external-catalogs/discover", { method: "POST", - body: JSON.stringify({ manifest_url: manifestUrl }), + body: JSON.stringify({ source_url: sourceUrl }), }); addonState.discoveredExternalCatalogs = Array.isArray(response.catalogs) ? response.catalogs : []; - addonState.discoveredExternalManifestUrl = response.manifest_url || manifestUrl; + addonState.discoveredExternalSourceUrl = response.source_url || response.manifest_url || sourceUrl; + addonState.discoveredExternalSourceKind = response.source_kind || "manifest"; + addonState.discoveredExternalSourceProvider = response.source_provider || ""; addonState.discoveredExternalAddonName = response.addon_name || ""; renderExternalCatalogDiscovery(); if (nameInput && !nameInput.value.trim() && addonState.discoveredExternalCatalogs.length) { @@ -1133,8 +1141,8 @@ async function handleExternalCatalogCreate(data, form) { const catalogKey = data.get("catalog_key") || ""; const selectedCatalog = getDiscoveredExternalCatalog(String(catalogKey)); const name = (data.get("name") || "").trim(); - if (!addonState.discoveredExternalManifestUrl) { - setMessage("external-catalog-message", "Discover catalogs first.", true); + if (!addonState.discoveredExternalSourceUrl) { + setMessage("external-catalog-message", "Discover a source first.", true); return; } if (!selectedCatalog) { @@ -1150,7 +1158,13 @@ async function handleExternalCatalogCreate(data, form) { method: "POST", body: JSON.stringify({ name, - manifest_url: addonState.discoveredExternalManifestUrl, + manifest_url: + addonState.discoveredExternalSourceKind === "manifest" + ? addonState.discoveredExternalSourceUrl + : null, + source_url: addonState.discoveredExternalSourceUrl, + source_kind: addonState.discoveredExternalSourceKind, + source_provider: addonState.discoveredExternalSourceProvider || null, addon_name: addonState.discoveredExternalAddonName || null, source_catalog_id: selectedCatalog.id, source_catalog_type: selectedCatalog.type, @@ -1162,7 +1176,9 @@ async function handleExternalCatalogCreate(data, form) { form.reset(); } addonState.discoveredExternalCatalogs = []; - addonState.discoveredExternalManifestUrl = ""; + addonState.discoveredExternalSourceUrl = ""; + addonState.discoveredExternalSourceKind = "manifest"; + addonState.discoveredExternalSourceProvider = ""; addonState.discoveredExternalAddonName = ""; renderExternalCatalogDiscovery(); setMessage("external-catalog-message", "External catalog added."); diff --git a/backend/src/librarysync/templates/stremio-addon.html b/backend/src/librarysync/templates/stremio-addon.html index 0545ef1..c1d0464 100644 --- a/backend/src/librarysync/templates/stremio-addon.html +++ b/backend/src/librarysync/templates/stremio-addon.html @@ -76,24 +76,24 @@

Built-in catalogs

External catalogs

-

Pull catalogs from other Stremio addons and filter out watched items.

+

Pull catalogs from Stremio manifests or supported list URLs and filter out watched items.

diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index fe6b407..0bf88a3 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -8,8 +8,16 @@ routes_stremio_addon, routes_stremio_addon_public, ) +from librarysync.core.external_catalog import _discover_tmdb_chart_catalogs from librarysync.core.stremio_addon import build_default_catalogs -from librarysync.db.models import MediaItem +from librarysync.core.watchlist_links import ( + parse_imdb_chart_urls, + parse_mdblist_urls, + parse_tmdb_chart_urls, + parse_tmdb_list_urls, + parse_tvdb_list_urls, +) +from librarysync.db.models import MediaItem, StremioExternalCatalogItem class TestStremioAddonCatalogs(unittest.TestCase): @@ -138,6 +146,82 @@ def test_reorder_map_requires_all_items(self) -> None: with self.assertRaises(HTTPException): routes_stremio_addon._build_reorder_map(["a", "b"], ["a"]) + def test_build_external_item_meta_uses_cached_fields_when_media_missing(self) -> None: + item = StremioExternalCatalogItem( + stremio_id="tt1234567", + stremio_type="movie", + title="Cached Movie", + year=1999, + poster_url="https://image.example/poster.jpg", + ) + meta = routes_stremio_addon_public._build_external_item_meta(item, None, "movie") + + self.assertEqual( + meta, + { + "id": "tt1234567", + "type": "movie", + "name": "Cached Movie", + "year": 1999, + "poster": "https://image.example/poster.jpg", + }, + ) + + def test_parse_tmdb_list_url(self) -> None: + refs = parse_tmdb_list_urls(["https://www.themoviedb.org/list/12345-best-movies"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].list_id, "12345") + self.assertEqual(refs[0].external_id, "tmdb:12345") + + def test_parse_tvdb_list_url(self) -> None: + refs = parse_tvdb_list_urls(["https://thetvdb.com/lists/top-shows"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].list_id, "top-shows") + self.assertEqual(refs[0].external_id, "tvdb:top-shows") + + def test_parse_tmdb_chart_url(self) -> None: + refs = parse_tmdb_chart_urls(["https://www.themoviedb.org/movie/top-rated"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].media_type, "movie") + self.assertEqual(refs[0].chart_slug, "top-rated") + self.assertEqual(refs[0].external_id, "tmdb-chart:movie:top-rated") + + def test_discover_tmdb_chart_catalog(self) -> None: + ref = parse_tmdb_chart_urls(["https://www.themoviedb.org/movie/top-rated"])[0] + + payload = asyncio.run(_discover_tmdb_chart_catalogs(ref)) + + self.assertEqual(payload["source_provider"], "tmdb") + self.assertEqual(payload["catalogs"][0]["id"], "tmdb-chart:movie:top-rated") + self.assertEqual(payload["catalogs"][0]["type"], "movie") + + def test_parse_imdb_chart_url_with_locale(self) -> None: + refs = parse_imdb_chart_urls(["https://www.imdb.com/de/chart/top/"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].chart_slug, "top") + + def test_discover_payload_accepts_legacy_manifest_url_field(self) -> None: + payload = routes_stremio_addon.StremioExternalCatalogDiscoverPayload( + manifest_url="https://www.themoviedb.org/movie/top-rated" + ) + + self.assertEqual(payload.manifest_url, "https://www.themoviedb.org/movie/top-rated") + self.assertIsNone(payload.source_url) + + def test_parse_mdblist_url(self) -> None: + refs = parse_mdblist_urls([ + "https://mdblist.com/lists/cb2131/emby-imdb-top-rated-movies" + ]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].username, "cb2131") + self.assertEqual(refs[0].slug, "emby-imdb-top-rated-movies") + self.assertEqual(refs[0].external_id, "mdblist:cb2131:emby-imdb-top-rated-movies") + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 0e7cb5f..5e29f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ members = ["backend"] [tool.ruff] -line-length = 100 +line-length = 120 extend-exclude = ["**/db/migrations/**"] [tool.ruff.lint] From 008477db1563fa552ce5b88668fbb02dbb9a406f Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:22:32 +0200 Subject: [PATCH 03/11] fix stremio addons --- .../api/routes_stremio_addon_public.py | 2 +- .../src/librarysync/core/external_catalog.py | 28 +++++ backend/tests/test_stremio_addon_catalogs.py | 108 +++++++++++++++++- 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/backend/src/librarysync/api/routes_stremio_addon_public.py b/backend/src/librarysync/api/routes_stremio_addon_public.py index e830695..b382c50 100644 --- a/backend/src/librarysync/api/routes_stremio_addon_public.py +++ b/backend/src/librarysync/api/routes_stremio_addon_public.py @@ -576,7 +576,7 @@ async def _serve_catalog( if _resolve_stremio_type(custom_catalog.media_type) != catalog_type: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") - else: + elif catalog: if not catalog.get("enabled", True): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index 72da883..e30c96d 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -289,6 +289,7 @@ async def refresh_external_catalog( ) ) + metas = _dedupe_external_metas(metas) created = 0 for position, meta in enumerate(metas): stremio_id = str(meta.get("id") or "").strip() @@ -367,6 +368,7 @@ async def _refresh_external_list_catalog( raise ValueError("Unsupported list provider") items = [item for item in items if item.stremio_type == catalog.source_catalog_type] + items = _dedupe_external_list_items(items) fetched_at = datetime.now(timezone.utc) await db.execute( @@ -410,6 +412,32 @@ async def _refresh_external_list_catalog( return created +def _dedupe_external_metas(metas: list[dict[str, Any]]) -> list[dict[str, Any]]: + deduped: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for meta in metas: + stremio_id = str(meta.get("id") or "").strip() + if not stremio_id or stremio_id in seen_ids: + continue + seen_ids.add(stremio_id) + deduped.append(meta) + return deduped + + +def _dedupe_external_list_items( + items: list[ExternalCatalogListItem], +) -> list[ExternalCatalogListItem]: + deduped: list[ExternalCatalogListItem] = [] + seen_ids: set[str] = set() + for item in items: + stremio_id = item.stremio_id.strip() + if not stremio_id or stremio_id in seen_ids: + continue + seen_ids.add(stremio_id) + deduped.append(item) + return deduped + + def _coerce_text(value: object) -> str | None: if value is None: return None diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index 0bf88a3..7910ddb 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -2,13 +2,19 @@ import copy import unittest from types import SimpleNamespace +from unittest.mock import AsyncMock, patch from fastapi import HTTPException from librarysync.api import ( routes_stremio_addon, routes_stremio_addon_public, ) -from librarysync.core.external_catalog import _discover_tmdb_chart_catalogs +from librarysync.core.external_catalog import ( + ExternalCatalogListItem, + _dedupe_external_list_items, + _dedupe_external_metas, + _discover_tmdb_chart_catalogs, +) from librarysync.core.stremio_addon import build_default_catalogs from librarysync.core.watchlist_links import ( parse_imdb_chart_urls, @@ -222,6 +228,106 @@ def test_parse_mdblist_url(self) -> None: self.assertEqual(refs[0].slug, "emby-imdb-top-rated-movies") self.assertEqual(refs[0].external_id, "mdblist:cb2131:emby-imdb-top-rated-movies") + def test_dedupe_external_metas_keeps_first_stremio_id(self) -> None: + metas = [ + {"id": "tt1", "name": "First"}, + {"id": "tt1", "name": "Duplicate"}, + {"id": "tt2", "name": "Second"}, + ] + + deduped = _dedupe_external_metas(metas) + + self.assertEqual(deduped, [{"id": "tt1", "name": "First"}, {"id": "tt2", "name": "Second"}]) + + def test_dedupe_external_list_items_keeps_first_stremio_id(self) -> None: + items = [ + ExternalCatalogListItem("tt1", "movie", "First", 2000, None), + ExternalCatalogListItem("tt1", "movie", "Duplicate", 2001, None), + ExternalCatalogListItem("tt2", "movie", "Second", 2002, None), + ] + + deduped = _dedupe_external_list_items(items) + + self.assertEqual([item.title for item in deduped], ["First", "Second"]) + + def test_serve_catalog_supports_external_catalog_without_builtin_catalog(self) -> None: + class _FakeScalarResult: + def __init__(self, value): + self._value = value + + def first(self): + return self._value + + class _FakeExecuteResult: + def __init__(self, *, scalar=None, rows=None): + self._scalar = scalar + self._rows = rows or [] + + def scalars(self): + return _FakeScalarResult(self._scalar) + + def all(self): + return self._rows + + class _FakeQuery: + def order_by(self, *args): + return self + + def offset(self, value): + return self + + def limit(self, value): + return self + + class _FakeDb: + def __init__(self): + self._results = [ + _FakeExecuteResult( + scalar=SimpleNamespace( + enabled=True, + source_catalog_type="movie", + order_by="source", + order_dir="asc", + ) + ), + _FakeExecuteResult(rows=[]), + ] + + async def execute(self, query): + return self._results.pop(0) + + request = SimpleNamespace(query_params={}) + config = SimpleNamespace( + is_enabled=True, + user_id="user-1", + default_catalogs=build_default_catalogs(), + ) + + with ( + patch.object( + routes_stremio_addon_public, + "get_addon_config_by_id", + AsyncMock(return_value=config), + ), + patch.object( + routes_stremio_addon_public, + "_build_external_catalog_query", + AsyncMock(return_value=_FakeQuery()), + ), + ): + payload = asyncio.run( + routes_stremio_addon_public._serve_catalog( + "addon-1", + "movie", + "top-rated-movies", + request, + _FakeDb(), + None, + ) + ) + + self.assertEqual(payload, {"metas": []}) + if __name__ == "__main__": unittest.main() From cf5341c756136fc3f5d8f4897406cb813575db7f Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:28:29 +0200 Subject: [PATCH 04/11] fix external catalog error handling --- .../librarysync/api/routes_stremio_addon.py | 20 +++-- .../src/librarysync/core/external_catalog.py | 89 +++++++++++++++++-- backend/tests/test_stremio_addon_catalogs.py | 84 +++++++++++++++++ 3 files changed, 177 insertions(+), 16 deletions(-) diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index f6fd557..a0cc6a5 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -18,6 +18,7 @@ from librarysync.connectors.services.letterboxd import LetterboxdError from librarysync.core.catalog_ordering import CatalogOrderBy from librarysync.core.external_catalog import ( + ExternalCatalogProviderError, discover_external_catalog_source, external_catalog_out, mark_external_catalog_refresh_failed, @@ -479,13 +480,13 @@ async def _refresh_external_catalog_or_502( await db.commit() await db.refresh(catalog) return count - except (httpx.HTTPError, LetterboxdError) as exc: + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: await db.rollback() await mark_external_catalog_refresh_failed(db, catalog, exc) await db.commit() raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"External catalog refresh failed: {exc}", + detail="External catalog refresh failed", ) from exc except Exception as exc: await db.rollback() @@ -587,10 +588,10 @@ async def discover_stremio_external_catalogs( status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc), ) from exc - except (httpx.HTTPError, LetterboxdError) as exc: + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"Failed to fetch external source: {exc}", + detail="Failed to fetch external source", ) from exc @@ -688,11 +689,11 @@ async def create_external_catalog( await refresh_external_catalog(db, catalog) await db.commit() await db.refresh(catalog) - except (httpx.HTTPError, LetterboxdError) as exc: + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: await db.rollback() raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"External catalog refresh failed: {exc}", + detail="External catalog refresh failed", ) from exc except Exception: await db.rollback() @@ -727,7 +728,12 @@ async def update_external_catalog( ) catalog.name = name if "enabled" in fields: - catalog.enabled = bool(payload.enabled) + if payload.enabled is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Enabled must be true or false", + ) + catalog.enabled = payload.enabled if "filters" in fields: catalog.filters = normalize_external_filters( payload.filters.model_dump(exclude_none=True) if payload.filters else None diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index e30c96d..387faa5 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -1,10 +1,13 @@ from __future__ import annotations import asyncio +import ipaddress +import socket import uuid from dataclasses import dataclass from datetime import datetime, timezone from typing import Any +from urllib.parse import urlparse, urlunparse import httpx from sqlalchemy import delete, or_, select @@ -66,13 +69,73 @@ class ExternalCatalogListItem: tvdb_id: str | None = None +class ExternalCatalogProviderError(Exception): + pass + + +def _is_disallowed_host_address(value: str) -> bool: + ip = ipaddress.ip_address(value) + return ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ) + + +def _validate_external_manifest_host(hostname: str | None) -> None: + if not hostname: + raise ValueError("Manifest URL host is required") + if hostname.lower() == "localhost": + raise ValueError("Manifest URL host is not allowed") + + try: + if _is_disallowed_host_address(hostname): + raise ValueError("Manifest URL host is not allowed") + return + except ValueError as exc: + if str(exc) == "Manifest URL host is not allowed": + raise + pass + + try: + resolved = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP) + except socket.gaierror: + return + + for entry in resolved: + address = entry[4][0] + if _is_disallowed_host_address(address): + raise ValueError("Manifest URL host is not allowed") + + +def _decode_external_json(response: httpx.Response, context: str) -> dict[str, Any]: + try: + payload = response.json() + except ValueError as exc: + raise ExternalCatalogProviderError(f"{context} returned invalid JSON") from exc + if not isinstance(payload, dict): + raise ExternalCatalogProviderError(f"{context} returned an invalid payload") + return payload + + def normalize_external_manifest_url(value: str) -> str: manifest_url = (value or "").strip() if not manifest_url: return "" - if manifest_url.endswith("/manifest.json"): - return manifest_url - return f"{manifest_url.rstrip('/')}/manifest.json" + parsed = urlparse(manifest_url) + if parsed.scheme.lower() not in {"http", "https"}: + raise ValueError("Manifest URL must use http or https") + if not parsed.netloc: + raise ValueError("Manifest URL host is required") + _validate_external_manifest_host(parsed.hostname) + + path = parsed.path or "" + if not path.endswith("/manifest.json"): + path = f"{path.rstrip('/')}/manifest.json" if path else "/manifest.json" + return urlunparse(parsed._replace(path=path)) def normalize_external_source_kind(value: str | None) -> str: @@ -87,8 +150,16 @@ def normalize_external_source_provider(value: str | None) -> str | None: def normalize_external_filters(filters: dict[str, Any] | None) -> dict[str, Any]: base = filters if isinstance(filters, dict) else {} - statuses = base.get("statuses") if isinstance(base.get("statuses"), list) else [] - show_watched = base.get("show_watched") if isinstance(base.get("show_watched"), bool) else False + statuses = ( + base.get("statuses") + if isinstance(base.get("statuses"), list) + else EXTERNAL_CATALOG_DEFAULT_FILTERS["statuses"] + ) + show_watched = ( + base.get("show_watched") + if isinstance(base.get("show_watched"), bool) + else EXTERNAL_CATALOG_DEFAULT_FILTERS["show_watched"] + ) return {"statuses": [str(value) for value in statuses if value], "show_watched": show_watched} @@ -211,7 +282,7 @@ async def _discover_manifest_catalogs(manifest_url: str) -> dict[str, Any]: async with get_http_client() as client: response = await client.get(normalized_url) response.raise_for_status() - payload = response.json() + payload = _decode_external_json(response, "External manifest") addon_name = str(payload.get("name") or "External addon").strip() or "External addon" discovered: list[dict[str, str]] = [] @@ -258,7 +329,7 @@ async def refresh_external_catalog( async with get_http_client() as client: manifest_response = await client.get(normalized_manifest_url) manifest_response.raise_for_status() - manifest_payload = manifest_response.json() + manifest_payload = _decode_external_json(manifest_response, "External manifest") catalog.addon_name = ( str(manifest_payload.get("name") or catalog.addon_name or "").strip() or None ) @@ -274,8 +345,8 @@ async def refresh_external_catalog( ) response = await client.get(catalog_url) response.raise_for_status() - payload = response.json() - batch = payload.get("metas") if isinstance(payload, dict) else None + payload = _decode_external_json(response, "External catalog") + batch = payload.get("metas") if not isinstance(batch, list) or not batch: break metas.extend(item for item in batch if isinstance(item, dict)) diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index 7910ddb..f48fd2c 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -11,9 +11,11 @@ ) from librarysync.core.external_catalog import ( ExternalCatalogListItem, + ExternalCatalogProviderError, _dedupe_external_list_items, _dedupe_external_metas, _discover_tmdb_chart_catalogs, + normalize_external_manifest_url, ) from librarysync.core.stremio_addon import build_default_catalogs from librarysync.core.watchlist_links import ( @@ -250,6 +252,88 @@ def test_dedupe_external_list_items_keeps_first_stremio_id(self) -> None: self.assertEqual([item.title for item in deduped], ["First", "Second"]) + def test_normalize_external_manifest_url_preserves_query(self) -> None: + self.assertEqual( + normalize_external_manifest_url("https://addon.example/path?foo=bar"), + "https://addon.example/path/manifest.json?foo=bar", + ) + + def test_normalize_external_manifest_url_rejects_disallowed_host(self) -> None: + with self.assertRaises(ValueError): + normalize_external_manifest_url("http://127.0.0.1/manifest.json") + + def test_refresh_external_catalog_or_502_maps_provider_errors(self) -> None: + class _FakeDb: + def __init__(self): + self.rollback_calls = 0 + self.commit_calls = 0 + + async def rollback(self): + self.rollback_calls += 1 + + async def commit(self): + self.commit_calls += 1 + + async def refresh(self, catalog): + raise AssertionError("refresh should not be called") + + db = _FakeDb() + catalog = SimpleNamespace() + + with ( + patch.object( + routes_stremio_addon, + "refresh_external_catalog", + AsyncMock(side_effect=ExternalCatalogProviderError("bad payload")), + ), + patch.object( + routes_stremio_addon, + "mark_external_catalog_refresh_failed", + AsyncMock(), + ) as mark_failed, + ): + with self.assertRaises(HTTPException) as ctx: + asyncio.run(routes_stremio_addon._refresh_external_catalog_or_502(db, catalog)) + + self.assertEqual(ctx.exception.status_code, 502) + self.assertEqual(ctx.exception.detail, "External catalog refresh failed") + self.assertEqual(db.rollback_calls, 1) + self.assertEqual(db.commit_calls, 1) + mark_failed.assert_awaited_once() + + def test_update_external_catalog_rejects_null_enabled(self) -> None: + class _FakeDb: + async def commit(self): + raise AssertionError("commit should not be called") + + async def refresh(self, catalog): + raise AssertionError("refresh should not be called") + + def add(self, catalog): + raise AssertionError("add should not be called") + + payload = routes_stremio_addon.StremioExternalCatalogUpdate(enabled=None) + catalog = SimpleNamespace(id="catalog-1", name="Catalog") + user = SimpleNamespace(id="user-1") + + with patch.object( + routes_stremio_addon, + "_load_external_catalog", + AsyncMock(return_value=catalog), + ): + with self.assertRaises(HTTPException) as ctx: + asyncio.run( + routes_stremio_addon.update_external_catalog( + "catalog-1", + payload, + user, + _FakeDb(), + ) + ) + + self.assertEqual(ctx.exception.status_code, 400) + self.assertEqual(ctx.exception.detail, "Enabled must be true or false") + def test_serve_catalog_supports_external_catalog_without_builtin_catalog(self) -> None: class _FakeScalarResult: def __init__(self, value): From 755d87fb8083949697cf07c1335a770db6cd7c80 Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:39:50 +0200 Subject: [PATCH 05/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- backend/src/librarysync/api/routes_stremio_addon_public.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/src/librarysync/api/routes_stremio_addon_public.py b/backend/src/librarysync/api/routes_stremio_addon_public.py index b382c50..52bd872 100644 --- a/backend/src/librarysync/api/routes_stremio_addon_public.py +++ b/backend/src/librarysync/api/routes_stremio_addon_public.py @@ -649,17 +649,18 @@ async def _serve_catalog( ) result = await db.execute(query.offset(skip).limit(limit + 1)) - rows = result.all() - has_more = len(rows) > limit if external_catalog: + rows = result.all() + has_more = len(rows) > limit metas = [ meta for item, media in rows[:limit] if (meta := _build_external_item_meta(item, media, catalog_type)) is not None ] else: - media_items = [row[0] if isinstance(row, tuple) else row for row in rows] + media_items = result.scalars().all() + has_more = len(media_items) > limit metas = [ meta for media in media_items[:limit] From 89266f9999436847f4a0b73746535f82c832519f Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:40:13 +0200 Subject: [PATCH 06/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- backend/src/librarysync/api/routes_stremio_addon.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index a0cc6a5..d8164d5 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -695,6 +695,12 @@ async def create_external_catalog( status_code=status.HTTP_502_BAD_GATEWAY, detail="External catalog refresh failed", ) from exc + except ValueError as exc: + await db.rollback() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc except Exception: await db.rollback() raise From d107f3f0af210717ad0b03bf804e96cce001bd05 Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:40:24 +0200 Subject: [PATCH 07/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- backend/src/librarysync/api/routes_stremio_addon.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index d8164d5..cc5c382 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -488,6 +488,14 @@ async def _refresh_external_catalog_or_502( status_code=status.HTTP_502_BAD_GATEWAY, detail="External catalog refresh failed", ) from exc + except ValueError as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc) or "Invalid external catalog configuration", + ) from exc except Exception as exc: await db.rollback() await mark_external_catalog_refresh_failed(db, catalog, exc) From ffe5210323fc2c8428036f6d4d70999a55931afb Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:40:55 +0200 Subject: [PATCH 08/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- backend/src/librarysync/core/external_catalog.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index 387faa5..3aa9a18 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -85,7 +85,7 @@ def _is_disallowed_host_address(value: str) -> bool: ) -def _validate_external_manifest_host(hostname: str | None) -> None: +async def _validate_external_manifest_host(hostname: str | None) -> None: if not hostname: raise ValueError("Manifest URL host is required") if hostname.lower() == "localhost": @@ -101,7 +101,11 @@ def _validate_external_manifest_host(hostname: str | None) -> None: pass try: - resolved = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP) + resolved = await asyncio.get_running_loop().getaddrinfo( + hostname, + None, + proto=socket.IPPROTO_TCP, + ) except socket.gaierror: return From 15a393da48483aca73b09cfbdb044e3a491222fd Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:42:07 +0200 Subject: [PATCH 09/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/librarysync/core/external_catalog.py | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index 3aa9a18..534624c 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -322,6 +322,78 @@ async def refresh_external_catalog( catalog: StremioExternalCatalog, *, max_items: int | None = None, +@dataclass(frozen=True) +class _ExternalMediaItemLookup: + by_stremio_id: dict[tuple[str, str], str] + by_imdb_id: dict[tuple[str, str], str] + + +async def _prefetch_external_media_item_ids( + db: AsyncSession, + metas: list[dict[str, Any]], + default_stremio_type: str | None, +) -> _ExternalMediaItemLookup: + stremio_ids: set[str] = set() + imdb_ids: set[str] = set() + + for meta in metas: + stremio_id = str(meta.get("id") or "").strip() + stremio_type = str(meta.get("type") or default_stremio_type or "").strip().lower() + if not stremio_id or stremio_type not in {"movie", "series"}: + continue + stremio_ids.add(stremio_id) + imdb_id = _extract_imdb_id(meta, stremio_id) + if imdb_id: + imdb_ids.add(imdb_id) + + if not stremio_ids and not imdb_ids: + return _ExternalMediaItemLookup(by_stremio_id={}, by_imdb_id={}) + + filters = [] + if stremio_ids: + filters.append(MediaItem.stremio_id.in_(stremio_ids)) + if imdb_ids: + filters.append(MediaItem.imdb_id.in_(imdb_ids)) + + result = await db.execute(select(MediaItem).where(or_(*filters))) + media_items = result.scalars().all() + + by_stremio_id: dict[tuple[str, str], str] = {} + by_imdb_id: dict[tuple[str, str], str] = {} + for media_item in media_items: + media_type = str(getattr(media_item, "media_type", "") or "").strip().lower() + if media_type not in {"movie", "series"}: + continue + + media_item_id = str(media_item.id) + item_stremio_id = str(getattr(media_item, "stremio_id", "") or "").strip() + if item_stremio_id: + by_stremio_id.setdefault((media_type, item_stremio_id), media_item_id) + + item_imdb_id = str(getattr(media_item, "imdb_id", "") or "").strip() + if item_imdb_id: + by_imdb_id.setdefault((media_type, item_imdb_id), media_item_id) + + return _ExternalMediaItemLookup( + by_stremio_id=by_stremio_id, + by_imdb_id=by_imdb_id, + ) + + +def _lookup_prefetched_external_media_item_id( + lookup: _ExternalMediaItemLookup, + stremio_id: str, + imdb_id: str | None, + stremio_type: str, +) -> str | None: + media_item_id = lookup.by_stremio_id.get((stremio_type, stremio_id)) + if media_item_id is not None: + return media_item_id + if imdb_id: + return lookup.by_imdb_id.get((stremio_type, imdb_id)) + return None + + ) -> int: if normalize_external_source_kind(getattr(catalog, "source_kind", None)) == "list": return await _refresh_external_list_catalog(db, catalog, max_items=max_items) @@ -365,6 +437,11 @@ async def refresh_external_catalog( ) metas = _dedupe_external_metas(metas) + media_item_lookup = await _prefetch_external_media_item_ids( + db, + metas, + str(catalog.source_catalog_type or "").strip().lower() or None, + ) created = 0 for position, meta in enumerate(metas): stremio_id = str(meta.get("id") or "").strip() @@ -372,7 +449,12 @@ async def refresh_external_catalog( if not stremio_id or stremio_type not in {"movie", "series"}: continue imdb_id = _extract_imdb_id(meta, stremio_id) - media_item_id = await _resolve_external_media_item_id(db, stremio_id, imdb_id, stremio_type) + media_item_id = _lookup_prefetched_external_media_item_id( + media_item_lookup, + stremio_id, + imdb_id, + stremio_type, + ) db.add( StremioExternalCatalogItem( id=str(uuid.uuid4()), From a23867a8075216bcd7eaf16bebc74fd92dbf9e7f Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:42:28 +0200 Subject: [PATCH 10/11] set line length --- .github/copilot-instructions.md | 4 ++-- .vscode/settings.json | 2 +- AGENTS.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c837604..bbc27d3 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -11,12 +11,12 @@ librarySync is a self-hosted, Docker-compose-deployable, multi-user hub for watc - **Frontend**: Static HTML + vanilla JavaScript with Tailwind CSS - **Architecture**: Async worker pattern with outbox-based delivery - **Package Manager**: uv (preferred) with pyproject.toml -- **Linter**: Ruff with 100 character line length +- **Linter**: Ruff with 120 character line length ## Code Style & Conventions ### Python -- Line length: 100 characters (configured in pyproject.toml) +- Line length: 120 characters (configured in pyproject.toml) - Use async/await patterns throughout the codebase - Follow existing patterns for database models and SQLAlchemy queries - Use type hints consistently diff --git a/.vscode/settings.json b/.vscode/settings.json index c6f0985..9ae34e9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "python.terminal.activateEnvironment": true, "ruff.enable": true, "ruff.lint.run": "onSave", - "editor.rulers": [100], + "editor.rulers": [120], "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, diff --git a/AGENTS.md b/AGENTS.md index ea21c3d..445392b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,7 +347,7 @@ Primary audit sources: `watch_events`, `outbox`, `sync_attempts`, `watch_syncs` - **HTTP Requests**: Always use `get_http_client()` from `core/http_client.py` to ensure consistent User-Agent headers (`librarySync Version/`) ### Tooling -- **Linter**: Ruff with 100-character line length (see `pyproject.toml`) +- **Linter**: Ruff with 120-character line length (see `pyproject.toml`) - **Styles**: Edit `frontend/input.css` (not `backend/src/librarysync/static/styles.css` directly) - **Build**: Rebuild `backend/src/librarysync/static/styles.css` after Tailwind changes From e88b5f27650bc6b74b343e77366bf4da2a8812a4 Mon Sep 17 00:00:00 2001 From: Thomas Willems Date: Fri, 17 Apr 2026 21:51:56 +0200 Subject: [PATCH 11/11] fixes --- .../librarysync/api/routes_stremio_addon.py | 17 ++- .../src/librarysync/core/external_catalog.py | 102 ++++++++++++------ backend/tests/test_stremio_addon_catalogs.py | 58 +++++++++- 3 files changed, 132 insertions(+), 45 deletions(-) diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index cc5c382..1febd44 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -639,17 +639,16 @@ async def create_external_catalog( source_kind = normalize_external_source_kind(payload.source_kind) source_provider = normalize_external_source_provider(payload.source_provider) source_url = (payload.source_url or payload.manifest_url or "").strip() - manifest_url = ( - normalize_external_manifest_url(source_url) - if source_kind == "manifest" - else source_url - ) - if not manifest_url: - detail = ( - "Manifest URL is required" + try: + manifest_url = ( + await normalize_external_manifest_url(source_url) if source_kind == "manifest" - else "Source URL is required" + else source_url ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + if not manifest_url: + detail = "Manifest URL is required" if source_kind == "manifest" else "Source URL is required" raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) source_catalog_id = payload.source_catalog_id.strip() if not source_catalog_id: diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py index 534624c..fe43989 100644 --- a/backend/src/librarysync/core/external_catalog.py +++ b/backend/src/librarysync/core/external_catalog.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from typing import Any -from urllib.parse import urlparse, urlunparse +from urllib.parse import quote, urlparse, urlunparse import httpx from sqlalchemy import delete, or_, select @@ -125,7 +125,7 @@ def _decode_external_json(response: httpx.Response, context: str) -> dict[str, A return payload -def normalize_external_manifest_url(value: str) -> str: +def _normalize_external_manifest_url_path(value: str) -> str: manifest_url = (value or "").strip() if not manifest_url: return "" @@ -134,7 +134,6 @@ def normalize_external_manifest_url(value: str) -> str: raise ValueError("Manifest URL must use http or https") if not parsed.netloc: raise ValueError("Manifest URL host is required") - _validate_external_manifest_host(parsed.hostname) path = parsed.path or "" if not path.endswith("/manifest.json"): @@ -142,6 +141,14 @@ def normalize_external_manifest_url(value: str) -> str: return urlunparse(parsed._replace(path=path)) +async def normalize_external_manifest_url(value: str) -> str: + normalized_url = _normalize_external_manifest_url_path(value) + if not normalized_url: + return "" + await _validate_external_manifest_host(urlparse(normalized_url).hostname) + return normalized_url + + def normalize_external_source_kind(value: str | None) -> str: normalized = (value or "manifest").strip().lower() return normalized if normalized in EXTERNAL_SOURCE_KINDS else "manifest" @@ -279,7 +286,7 @@ async def discover_external_catalog_source( async def _discover_manifest_catalogs(manifest_url: str) -> dict[str, Any]: - normalized_url = normalize_external_manifest_url(manifest_url) + normalized_url = await normalize_external_manifest_url(manifest_url) if not normalized_url: raise ValueError("Manifest URL is required") @@ -317,44 +324,45 @@ async def _discover_manifest_catalogs(manifest_url: str) -> dict[str, Any]: } -async def refresh_external_catalog( - db: AsyncSession, - catalog: StremioExternalCatalog, - *, - max_items: int | None = None, @dataclass(frozen=True) class _ExternalMediaItemLookup: by_stremio_id: dict[tuple[str, str], str] by_imdb_id: dict[tuple[str, str], str] +def _media_item_type_to_stremio_type(media_type: str) -> str | None: + if media_type == "movie": + return "movie" + if media_type in {"tv", "anime", "series"}: + return "series" + return None + + async def _prefetch_external_media_item_ids( db: AsyncSession, metas: list[dict[str, Any]], default_stremio_type: str | None, ) -> _ExternalMediaItemLookup: - stremio_ids: set[str] = set() - imdb_ids: set[str] = set() + filters = [] for meta in metas: stremio_id = str(meta.get("id") or "").strip() stremio_type = str(meta.get("type") or default_stremio_type or "").strip().lower() - if not stremio_id or stremio_type not in {"movie", "series"}: + media_types = stremio_type_to_media_types(stremio_type) + if not stremio_id or not media_types: continue - stremio_ids.add(stremio_id) + + filters.append( + MediaItem.media_type.in_(media_types) & (MediaItem.raw["stremio_id"].as_string() == stremio_id) + ) + imdb_id = _extract_imdb_id(meta, stremio_id) if imdb_id: - imdb_ids.add(imdb_id) + filters.append(MediaItem.media_type.in_(media_types) & (MediaItem.imdb_id == imdb_id)) - if not stremio_ids and not imdb_ids: + if not filters: return _ExternalMediaItemLookup(by_stremio_id={}, by_imdb_id={}) - filters = [] - if stremio_ids: - filters.append(MediaItem.stremio_id.in_(stremio_ids)) - if imdb_ids: - filters.append(MediaItem.imdb_id.in_(imdb_ids)) - result = await db.execute(select(MediaItem).where(or_(*filters))) media_items = result.scalars().all() @@ -362,22 +370,21 @@ async def _prefetch_external_media_item_ids( by_imdb_id: dict[tuple[str, str], str] = {} for media_item in media_items: media_type = str(getattr(media_item, "media_type", "") or "").strip().lower() - if media_type not in {"movie", "series"}: + stremio_type = _media_item_type_to_stremio_type(media_type) + if stremio_type is None: continue media_item_id = str(media_item.id) - item_stremio_id = str(getattr(media_item, "stremio_id", "") or "").strip() + raw = media_item.raw if isinstance(media_item.raw, dict) else {} + item_stremio_id = str(raw.get("stremio_id") or "").strip() if item_stremio_id: - by_stremio_id.setdefault((media_type, item_stremio_id), media_item_id) + by_stremio_id.setdefault((stremio_type, item_stremio_id), media_item_id) item_imdb_id = str(getattr(media_item, "imdb_id", "") or "").strip() if item_imdb_id: - by_imdb_id.setdefault((media_type, item_imdb_id), media_item_id) + by_imdb_id.setdefault((stremio_type, item_imdb_id), media_item_id) - return _ExternalMediaItemLookup( - by_stremio_id=by_stremio_id, - by_imdb_id=by_imdb_id, - ) + return _ExternalMediaItemLookup(by_stremio_id=by_stremio_id, by_imdb_id=by_imdb_id) def _lookup_prefetched_external_media_item_id( @@ -394,11 +401,36 @@ def _lookup_prefetched_external_media_item_id( return None +def _build_external_manifest_catalog_url( + manifest_url: str, + catalog_type: str, + catalog_id: str, + *, + skip: int, + limit: int, +) -> str: + parsed = urlparse(manifest_url) + if not parsed.path.endswith("/manifest.json"): + raise ValueError("Manifest URL must end with /manifest.json") + + base_path = parsed.path[: -len("/manifest.json")] + catalog_path = ( + f"{base_path}/catalog/{quote(catalog_type, safe='')}/{quote(catalog_id, safe='')}" + f"/skip={skip}&limit={limit}.json" + ) + return urlunparse(parsed._replace(path=catalog_path)) + + +async def refresh_external_catalog( + db: AsyncSession, + catalog: StremioExternalCatalog, + *, + max_items: int | None = None, ) -> int: if normalize_external_source_kind(getattr(catalog, "source_kind", None)) == "list": return await _refresh_external_list_catalog(db, catalog, max_items=max_items) - normalized_manifest_url = normalize_external_manifest_url(catalog.manifest_url) + normalized_manifest_url = await normalize_external_manifest_url(catalog.manifest_url) max_catalog_items = max_items or settings.external_catalog_max_items fetched_at = datetime.now(timezone.utc) @@ -410,14 +442,16 @@ def _lookup_prefetched_external_media_item_id( str(manifest_payload.get("name") or catalog.addon_name or "").strip() or None ) - base_url = normalized_manifest_url[: -len("/manifest.json")] metas: list[dict[str, Any]] = [] skip = 0 while len(metas) < max_catalog_items: limit = min(EXTERNAL_CATALOG_FETCH_PAGE_SIZE, max_catalog_items - len(metas)) - catalog_url = ( - f"{base_url}/catalog/{catalog.source_catalog_type}/{catalog.source_catalog_id}" - f"/skip={skip}&limit={limit}.json" + catalog_url = _build_external_manifest_catalog_url( + normalized_manifest_url, + catalog.source_catalog_type, + catalog.source_catalog_id, + skip=skip, + limit=limit, ) response = await client.get(catalog_url) response.raise_for_status() diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index f48fd2c..4a5f487 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -12,9 +12,11 @@ from librarysync.core.external_catalog import ( ExternalCatalogListItem, ExternalCatalogProviderError, + _build_external_manifest_catalog_url, _dedupe_external_list_items, _dedupe_external_metas, _discover_tmdb_chart_catalogs, + _prefetch_external_media_item_ids, normalize_external_manifest_url, ) from librarysync.core.stremio_addon import build_default_catalogs @@ -254,13 +256,65 @@ def test_dedupe_external_list_items_keeps_first_stremio_id(self) -> None: def test_normalize_external_manifest_url_preserves_query(self) -> None: self.assertEqual( - normalize_external_manifest_url("https://addon.example/path?foo=bar"), + asyncio.run(normalize_external_manifest_url("https://addon.example/path?foo=bar")), "https://addon.example/path/manifest.json?foo=bar", ) def test_normalize_external_manifest_url_rejects_disallowed_host(self) -> None: with self.assertRaises(ValueError): - normalize_external_manifest_url("http://127.0.0.1/manifest.json") + asyncio.run(normalize_external_manifest_url("http://127.0.0.1/manifest.json")) + + def test_build_external_manifest_catalog_url_preserves_query(self) -> None: + self.assertEqual( + _build_external_manifest_catalog_url( + "https://addon.example/path/manifest.json?foo=bar", + "movie", + "top rated", + skip=100, + limit=50, + ), + "https://addon.example/path/catalog/movie/top%20rated/skip=100&limit=50.json?foo=bar", + ) + + def test_prefetch_external_media_item_ids_uses_raw_stremio_id_and_series_type(self) -> None: + class _FakeScalarResult: + def __init__(self, values): + self._values = values + + def all(self): + return self._values + + class _FakeExecuteResult: + def __init__(self, values): + self._values = values + + def scalars(self): + return _FakeScalarResult(self._values) + + class _FakeDb: + async def execute(self, query): + return _FakeExecuteResult( + [ + MediaItem( + id="media-1", + media_type="tv", + title="Example Show", + imdb_id="tt1234567", + raw={"stremio_id": "stremio:series:1"}, + ) + ] + ) + + lookup = asyncio.run( + _prefetch_external_media_item_ids( + _FakeDb(), + [{"id": "stremio:series:1", "type": "series", "imdb_id": "tt1234567"}], + "series", + ) + ) + + self.assertEqual(lookup.by_stremio_id[("series", "stremio:series:1")], "media-1") + self.assertEqual(lookup.by_imdb_id[("series", "tt1234567")], "media-1") def test_refresh_external_catalog_or_502_maps_provider_errors(self) -> None: class _FakeDb: