diff --git a/.env.example b/.env.example index a35d535..7407c41 100644 --- a/.env.example +++ b/.env.example @@ -28,13 +28,16 @@ SIMKL_CLIENT_SECRET=your_simkl_client_secret ANILIST_CLIENT_ID=your_anilist_client_id ANILIST_CLIENT_SECRET=your_anilist_client_secret -# Worker modes (comma-separated subset of `outbox`, `metadata`, `metadata_backfill`, `metadata_cache`, `quick_import`, `import_all`, `watchlist`, `merge_history`, `merge_all_history`) +# Worker modes (comma-separated subset of `outbox`, `metadata`, `metadata_backfill`, `metadata_cache`, `quick_import`, `import_all`, `external_catalog_refresh`, `watchlist`, `merge_history`, `merge_all_history`) LIBRARYSYNC_WORKER_MODES=all LIBRARYSYNC_WORKER_OUTBOX_CONCURRENCY=1 LIBRARYSYNC_WORKER_METADATA_CONCURRENCY=1 LIBRARYSYNC_WORKER_QUICK_IMPORT_CONCURRENCY=1 LIBRARYSYNC_WORKER_IMPORT_ALL_CONCURRENCY=1 LIBRARYSYNC_WORKER_METADATA_CACHE_CONCURRENCY=1 +LIBRARYSYNC_WORKER_EXTERNAL_CATALOG_REFRESH_CONCURRENCY=1 +LIBRARYSYNC_EXTERNAL_CATALOG_REFRESH_HOURS=3 +LIBRARYSYNC_EXTERNAL_CATALOG_MAX_ITEMS=500 # Rate limits (per user, per provider, per minute) LIBRARYSYNC_TRAKT_RATE_LIMIT_PER_MINUTE=60 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c837604..bbc27d3 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -11,12 +11,12 @@ librarySync is a self-hosted, Docker-compose-deployable, multi-user hub for watc - **Frontend**: Static HTML + vanilla JavaScript with Tailwind CSS - **Architecture**: Async worker pattern with outbox-based delivery - **Package Manager**: uv (preferred) with pyproject.toml -- **Linter**: Ruff with 100 character line length +- **Linter**: Ruff with 120 character line length ## Code Style & Conventions ### Python -- Line length: 100 characters (configured in pyproject.toml) +- Line length: 120 characters (configured in pyproject.toml) - Use async/await patterns throughout the codebase - Follow existing patterns for database models and SQLAlchemy queries - Use type hints consistently diff --git a/.vscode/settings.json b/.vscode/settings.json index c6f0985..9ae34e9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "python.terminal.activateEnvironment": true, "ruff.enable": true, "ruff.lint.run": "onSave", - "editor.rulers": [100], + "editor.rulers": [120], "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, diff --git a/AGENTS.md b/AGENTS.md index ea21c3d..445392b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -347,7 +347,7 @@ Primary audit sources: `watch_events`, `outbox`, `sync_attempts`, `watch_syncs` - **HTTP Requests**: Always use `get_http_client()` from `core/http_client.py` to ensure consistent User-Agent headers (`librarySync Version/`) ### Tooling -- **Linter**: Ruff with 100-character line length (see `pyproject.toml`) +- **Linter**: Ruff with 120-character line length (see `pyproject.toml`) - **Styles**: Edit `frontend/input.css` (not `backend/src/librarysync/static/styles.css` directly) - **Build**: Rebuild `backend/src/librarysync/static/styles.css` after Tailwind changes diff --git a/backend/src/librarysync/api/routes_stremio_addon.py b/backend/src/librarysync/api/routes_stremio_addon.py index a352981..1febd44 100644 --- a/backend/src/librarysync/api/routes_stremio_addon.py +++ b/backend/src/librarysync/api/routes_stremio_addon.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Literal +import httpx from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel from sqlalchemy import func, select @@ -14,7 +15,23 @@ from librarysync.api.deps import get_current_user, get_db from librarysync.config import settings +from librarysync.connectors.services.letterboxd import LetterboxdError from librarysync.core.catalog_ordering import CatalogOrderBy +from librarysync.core.external_catalog import ( + ExternalCatalogProviderError, + discover_external_catalog_source, + external_catalog_out, + mark_external_catalog_refresh_failed, + normalize_external_filters, + normalize_external_manifest_url, + normalize_external_order_by, + normalize_external_order_dir, + normalize_external_page_size, + normalize_external_show_in_home, + normalize_external_source_kind, + normalize_external_source_provider, + refresh_external_catalog, +) from librarysync.core.stremio_addon import ( ensure_addon_config, normalize_default_catalogs, @@ -30,6 +47,7 @@ StremioAddonConfig, StremioCustomCatalog, StremioCustomCatalogItem, + StremioExternalCatalog, User, ) @@ -42,6 +60,7 @@ *CatalogOrderBy.__args__, } CUSTOM_ORDER_DIR = {"asc", "desc"} +EXTERNAL_CATALOG_TYPES = {"movie", "series"} class StremioCatalogFilters(BaseModel): @@ -101,6 +120,39 @@ class StremioCustomCatalogReorder(BaseModel): media_item_ids: list[str] +class StremioExternalCatalogDiscoverPayload(BaseModel): + source_url: str | None = None + manifest_url: str | None = None + + +class StremioExternalCatalogCreate(BaseModel): + name: str + manifest_url: str | None = None + source_url: str | None = None + source_kind: Literal["manifest", "list"] | None = None + source_provider: Literal["tmdb", "tvdb", "letterboxd", "mdblist"] | None = None + addon_name: str | None = None + source_catalog_id: str + source_catalog_type: Literal["movie", "series"] + media_type: Literal["movie", "tv", "anime"] | None = None + enabled: bool | None = None + filters: StremioCatalogFilters | None = None + order_by: str | None = None + order_dir: Literal["asc", "desc"] | None = None + page_size: int | None = None + show_in_home: bool | None = None + + +class StremioExternalCatalogUpdate(BaseModel): + name: str | None = None + enabled: bool | None = None + filters: StremioCatalogFilters | None = None + order_by: str | None = None + order_dir: Literal["asc", "desc"] | None = None + page_size: int | None = None + show_in_home: bool | None = None + + def _resolve_base_url(request: Request) -> str: base = settings.base_url or str(request.base_url) return base.rstrip("/") @@ -130,19 +182,33 @@ async def _build_unique_slug( user_id: str, value: str, *, - exclude_catalog_id: str | None = None, + exclude_custom_catalog_id: str | None = None, + exclude_external_catalog_id: str | None = None, ) -> str: base_slug = _truncate_slug(_slugify(value)) - result = await db.execute( + custom_result = await db.execute( select(StremioCustomCatalog.id, StremioCustomCatalog.slug).where( StremioCustomCatalog.user_id == user_id ) ) + external_result = await db.execute( + select(StremioExternalCatalog.id, StremioExternalCatalog.slug).where( + StremioExternalCatalog.user_id == user_id + ) + ) existing = { slug - for catalog_id, slug in result.all() - if slug and (not exclude_catalog_id or catalog_id != exclude_catalog_id) + for catalog_id, slug in custom_result.all() + if slug and (not exclude_custom_catalog_id or catalog_id != exclude_custom_catalog_id) } + existing.update( + { + slug + for catalog_id, slug in external_result.all() + if slug + and (not exclude_external_catalog_id or catalog_id != exclude_external_catalog_id) + } + ) if base_slug not in existing: return base_slug for index in range(2, 100): @@ -280,6 +346,23 @@ async def _load_custom_catalog( return catalog +async def _load_external_catalog( + db: AsyncSession, + user_id: str, + catalog_id: str, +) -> StremioExternalCatalog: + result = await db.execute( + select(StremioExternalCatalog).where( + StremioExternalCatalog.user_id == user_id, + StremioExternalCatalog.id == catalog_id, + ) + ) + catalog = result.scalars().first() + if not catalog: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") + return catalog + + async def _resolve_custom_media_item( db: AsyncSession, payload: StremioCustomCatalogItemCreate, @@ -378,6 +461,48 @@ async def _resolve_custom_media_item( return media_item +def _normalize_external_media_type( + source_catalog_type: str, + requested_media_type: str | None, +) -> str: + if source_catalog_type == "movie": + return "movie" + normalized = (requested_media_type or "tv").strip().lower() + return normalized if normalized in {"tv", "anime"} else "tv" + + +async def _refresh_external_catalog_or_502( + db: AsyncSession, + catalog: StremioExternalCatalog, +) -> int: + try: + count = await refresh_external_catalog(db, catalog) + await db.commit() + await db.refresh(catalog) + return count + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="External catalog refresh failed", + ) from exc + except ValueError as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc) or "Invalid external catalog configuration", + ) from exc + except Exception as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + raise + + @router.get( "/config", summary="Get Stremio addon config", @@ -392,14 +517,25 @@ async def get_stremio_addon_config( catalogs = await _ensure_default_catalogs(db, config) custom_result = await db.execute( - select(StremioCustomCatalog).where(StremioCustomCatalog.user_id == current_user.id) + select(StremioCustomCatalog) + .where(StremioCustomCatalog.user_id == current_user.id) + .order_by(StremioCustomCatalog.created_at.asc()) ) custom_catalogs = [_custom_catalog_out(catalog) for catalog in custom_result.scalars().all()] + external_result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == current_user.id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + external_catalogs = [ + external_catalog_out(catalog) for catalog in external_result.scalars().all() + ] return { "addon_id": config.id, "is_enabled": bool(config.is_enabled), "catalogs": catalogs, + "external_catalogs": external_catalogs, "custom_catalogs": custom_catalogs, **_build_manifest_links(_resolve_base_url(request), config.id), } @@ -437,6 +573,235 @@ async def update_stremio_addon_config( } +@router.post( + "/external-catalogs/discover", + summary="Discover external catalogs", + description="Fetch an external Stremio manifest and list supported catalogs.", +) +async def discover_stremio_external_catalogs( + payload: StremioExternalCatalogDiscoverPayload, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + source_url = (payload.source_url or payload.manifest_url or "").strip() + if not source_url: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Source URL is required", + ) + try: + return await discover_external_catalog_source(db, current_user.id, source_url) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Failed to fetch external source", + ) from exc + + +@router.get( + "/external-catalogs", + summary="List external catalogs", + description="List external Stremio catalogs configured for the current user.", +) +async def list_external_catalogs( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == current_user.id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + return { + "external_catalogs": [external_catalog_out(catalog) for catalog in result.scalars().all()] + } + + +@router.post( + "/external-catalogs", + status_code=status.HTTP_201_CREATED, + summary="Create external catalog", + description="Create a new external Stremio catalog backed by another addon.", +) +async def create_external_catalog( + payload: StremioExternalCatalogCreate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + name = payload.name.strip() + if not name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") + source_kind = normalize_external_source_kind(payload.source_kind) + source_provider = normalize_external_source_provider(payload.source_provider) + source_url = (payload.source_url or payload.manifest_url or "").strip() + try: + manifest_url = ( + await normalize_external_manifest_url(source_url) + if source_kind == "manifest" + else source_url + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + if not manifest_url: + detail = "Manifest URL is required" if source_kind == "manifest" else "Source URL is required" + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=detail) + source_catalog_id = payload.source_catalog_id.strip() + if not source_catalog_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Source catalog ID is required", + ) + if payload.source_catalog_type not in EXTERNAL_CATALOG_TYPES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid source catalog type", + ) + if source_kind == "list" and not source_provider: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="List provider is required", + ) + + catalog = StremioExternalCatalog( + user_id=current_user.id, + name=name, + slug=await _build_unique_slug(db, current_user.id, name), + source_kind=source_kind, + source_provider=source_provider, + addon_name=(payload.addon_name or "").strip() or None, + manifest_url=manifest_url, + source_catalog_id=source_catalog_id, + source_catalog_type=payload.source_catalog_type, + media_type=_normalize_external_media_type( + payload.source_catalog_type, + payload.media_type, + ), + enabled=True if payload.enabled is None else bool(payload.enabled), + filters=normalize_external_filters( + payload.filters.model_dump(exclude_none=True) if payload.filters else None + ), + order_by=normalize_external_order_by(payload.order_by), + order_dir=normalize_external_order_dir(payload.order_dir), + page_size=normalize_external_page_size(payload.page_size), + show_in_home=normalize_external_show_in_home(payload.show_in_home), + ) + db.add(catalog) + try: + await db.flush() + await refresh_external_catalog(db, catalog) + await db.commit() + await db.refresh(catalog) + except (httpx.HTTPError, LetterboxdError, ExternalCatalogProviderError) as exc: + await db.rollback() + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="External catalog refresh failed", + ) from exc + except ValueError as exc: + await db.rollback() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except Exception: + await db.rollback() + raise + return external_catalog_out(catalog) + + +@router.patch( + "/external-catalogs/{catalog_id}", + summary="Update external catalog", + description="Update an external Stremio catalog.", +) +async def update_external_catalog( + catalog_id: str, + payload: StremioExternalCatalogUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + fields = payload.model_fields_set + + if "name" in fields: + name = (payload.name or "").strip() + if not name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") + if name != catalog.name: + catalog.slug = await _build_unique_slug( + db, + current_user.id, + name, + exclude_external_catalog_id=catalog.id, + ) + catalog.name = name + if "enabled" in fields: + if payload.enabled is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Enabled must be true or false", + ) + catalog.enabled = payload.enabled + if "filters" in fields: + catalog.filters = normalize_external_filters( + payload.filters.model_dump(exclude_none=True) if payload.filters else None + ) + if "order_by" in fields: + catalog.order_by = normalize_external_order_by(payload.order_by) + if "order_dir" in fields: + catalog.order_dir = normalize_external_order_dir(payload.order_dir) + if "page_size" in fields: + catalog.page_size = normalize_external_page_size(payload.page_size) + if "show_in_home" in fields: + catalog.show_in_home = normalize_external_show_in_home(payload.show_in_home) + + catalog.updated_at = datetime.now(timezone.utc) + db.add(catalog) + await db.commit() + await db.refresh(catalog) + return external_catalog_out(catalog) + + +@router.delete( + "/external-catalogs/{catalog_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Delete external catalog", + description="Delete an external Stremio catalog and its cached items.", +) +async def delete_external_catalog( + catalog_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + await db.delete(catalog) + await db.commit() + + +@router.post( + "/external-catalogs/{catalog_id}/refresh", + summary="Refresh external catalog", + description="Fetch the latest items for an external Stremio catalog.", +) +async def refresh_stremio_external_catalog( + catalog_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + catalog = await _load_external_catalog(db, current_user.id, catalog_id) + item_count = await _refresh_external_catalog_or_502(db, catalog) + return { + "status": "ok", + "item_count": item_count, + "catalog": external_catalog_out(catalog), + } + + @router.post( "/custom-catalogs", status_code=status.HTTP_201_CREATED, @@ -486,7 +851,7 @@ async def update_custom_catalog( raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Name is required") if name != catalog.name: catalog.slug = await _build_unique_slug( - db, current_user.id, name, exclude_catalog_id=catalog.id + db, current_user.id, name, exclude_custom_catalog_id=catalog.id ) catalog.name = name @@ -495,6 +860,17 @@ async def update_custom_catalog( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid media type" ) + if payload.media_type != catalog.media_type: + item_count_result = await db.execute( + select(func.count(StremioCustomCatalogItem.id)).where( + StremioCustomCatalogItem.catalog_id == catalog.id + ) + ) + if int(item_count_result.scalar_one() or 0) > 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot change media type while the catalog still has items", + ) catalog.media_type = payload.media_type if "order_by" in fields: diff --git a/backend/src/librarysync/api/routes_stremio_addon_public.py b/backend/src/librarysync/api/routes_stremio_addon_public.py index aad113c..52bd872 100644 --- a/backend/src/librarysync/api/routes_stremio_addon_public.py +++ b/backend/src/librarysync/api/routes_stremio_addon_public.py @@ -10,6 +10,10 @@ from librarysync.api.deps import get_db from librarysync.core.catalog_ordering import apply_catalog_ordering, build_show_progress_subquery +from librarysync.core.external_catalog import ( + external_catalog_manifest_entry, + normalize_external_filters, +) from librarysync.core.stremio_addon import ( DEFAULT_PAGE_SIZE, DEFAULT_SHOW_IN_HOME, @@ -21,6 +25,9 @@ MediaItem, StremioCustomCatalog, StremioCustomCatalogItem, + StremioExternalCatalog, + StremioExternalCatalogItem, + WatchedItem, WatchlistItem, ) @@ -72,6 +79,25 @@ def _build_meta(media_item: MediaItem, catalog_type: str) -> dict[str, Any] | No return meta +def _build_external_item_meta( + item: StremioExternalCatalogItem, + media_item: MediaItem | None, + catalog_type: str, +) -> dict[str, Any] | None: + if media_item is not None: + return _build_meta(media_item, catalog_type) + stremio_id = str(item.stremio_id or "").strip() + stremio_type = str(item.stremio_type or "").strip().lower() + if not stremio_id or not stremio_type or stremio_type != catalog_type: + return None + meta = {"id": stremio_id, "type": stremio_type, "name": item.title or stremio_id} + if item.poster_url: + meta["poster"] = item.poster_url + if item.year: + meta["year"] = item.year + return meta + + def _extract_extra_param(request: Request, name: str) -> str | None: return request.query_params.get(name) or request.query_params.get(f"extra[{name}]") @@ -197,6 +223,7 @@ def _coerce_show_in_home(catalog: dict | None) -> bool: def _build_manifest( catalogs: list[dict], + external_catalogs: list[StremioExternalCatalog], custom_catalogs: list[StremioCustomCatalog], ) -> dict[str, Any]: manifest_catalogs: list[dict[str, Any]] = [] @@ -226,6 +253,12 @@ def _build_manifest( ) seen_types.add(stremio_type) + for external in external_catalogs: + if not external.enabled: + continue + manifest_catalogs.append(external_catalog_manifest_entry(external)) + seen_types.add(external.source_catalog_type) + for custom in custom_catalogs: stremio_type = _resolve_stremio_type(custom.media_type) if not stremio_type: @@ -357,6 +390,66 @@ async def _build_custom_catalog_query( return query +async def _build_external_catalog_query( + user_id: str, + catalog: StremioExternalCatalog, + search: str | None, +): + query = ( + select(StremioExternalCatalogItem, MediaItem) + .select_from(StremioExternalCatalogItem) + .outerjoin(MediaItem, StremioExternalCatalogItem.media_item_id == MediaItem.id) + .where( + StremioExternalCatalogItem.catalog_id == catalog.id, + ) + ) + filters = normalize_external_filters(catalog.filters) + if not filters.get("show_watched", False): + now_date = datetime.now(timezone.utc).date() + progress_subq = build_show_progress_subquery(user_id, now_date) + query = query.outerjoin(progress_subq, progress_subq.c.media_item_id == MediaItem.id) + directly_watched_exists = ( + select(WatchedItem.id) + .where( + WatchedItem.user_id == user_id, + WatchedItem.media_item_id == MediaItem.id, + ) + .exists() + ) + completed_show_clause = and_( + MediaItem.media_type.in_(["tv", "anime"]), + progress_subq.c.total_released > 0, + progress_subq.c.watched_count >= progress_subq.c.total_released, + ) + query = query.where( + or_( + StremioExternalCatalogItem.media_item_id.is_(None), + ~or_(directly_watched_exists, completed_show_clause), + ) + ) + if search: + normalized_search = search.strip() + if normalized_search: + like_value = f"%{normalized_search}%" + search_clauses = [ + StremioExternalCatalogItem.title.ilike(like_value), + StremioExternalCatalogItem.imdb_id.ilike(like_value), + MediaItem.title.ilike(like_value), + MediaItem.imdb_id.ilike(like_value), + MediaItem.tmdb_id.ilike(like_value), + MediaItem.tvdb_id.ilike(like_value), + MediaItem.tvmaze_id.ilike(like_value), + MediaItem.kitsu_id.ilike(like_value), + MediaItem.myanimelist_id.ilike(like_value), + MediaItem.anilist_id.ilike(like_value), + ] + if normalized_search.isdigit() and len(normalized_search) == 4: + search_clauses.append(StremioExternalCatalogItem.year == int(normalized_search)) + search_clauses.append(MediaItem.year == int(normalized_search)) + query = query.where(or_(*search_clauses)) + return query + + def _apply_ordering( query, *, @@ -392,10 +485,18 @@ async def stremio_addon_manifest( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") catalogs = _normalize_catalogs(config.default_catalogs) custom_result = await db.execute( - select(StremioCustomCatalog).where(StremioCustomCatalog.user_id == config.user_id) + select(StremioCustomCatalog) + .where(StremioCustomCatalog.user_id == config.user_id) + .order_by(StremioCustomCatalog.created_at.asc()) ) custom_catalogs = custom_result.scalars().all() - return _build_manifest(catalogs, custom_catalogs) + external_result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.user_id == config.user_id) + .order_by(StremioExternalCatalog.created_at.asc()) + ) + external_catalogs = external_result.scalars().all() + return _build_manifest(catalogs, external_catalogs, custom_catalogs) @router.get("/{addon_id}/catalog/{catalog_type}/{catalog_id}.json", include_in_schema=False) @@ -440,8 +541,29 @@ async def _serve_catalog( catalogs = _normalize_catalogs(config.default_catalogs) catalog = _catalogs_by_id(catalogs).get(catalog_id) custom_catalog = None + external_catalog = None if not catalog: + external_result = await db.execute( + select(StremioExternalCatalog).where( + StremioExternalCatalog.user_id == config.user_id, + StremioExternalCatalog.slug == catalog_id, + ) + ) + external_catalog = external_result.scalars().first() + if external_catalog: + if not external_catalog.enabled: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Catalog not found", + ) + if external_catalog.source_catalog_type != catalog_type: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Catalog not found", + ) + + if not catalog and not external_catalog: custom_result = await db.execute( select(StremioCustomCatalog).where( StremioCustomCatalog.user_id == config.user_id, @@ -454,7 +576,7 @@ async def _serve_catalog( if _resolve_stremio_type(custom_catalog.media_type) != catalog_type: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") - else: + elif catalog: if not catalog.get("enabled", True): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Catalog not found") @@ -463,7 +585,30 @@ async def _serve_catalog( skip, limit, search = _resolve_pagination(request, extra_overrides) - if custom_catalog: + if external_catalog: + query = await _build_external_catalog_query(config.user_id, external_catalog, search) + order_by = external_catalog.order_by or "source" + order_dir = external_catalog.order_dir or "asc" + if order_by == "source": + order_clause = StremioExternalCatalogItem.position.asc() + if order_dir == "desc": + order_clause = StremioExternalCatalogItem.position.desc() + query = query.order_by(order_clause, MediaItem.id) + elif order_by == "random": + query = query.order_by(func.random()) + else: + release_date_expr = func.coalesce(MediaItem.release_date, MediaItem.first_air_date) + query = _apply_ordering( + query, + order_by=order_by, + order_dir=order_dir, + user_id=config.user_id, + date_added_col=StremioExternalCatalogItem.fetched_at, + release_date_col=release_date_expr, + base_media_id_col=MediaItem.id, + tie_breaker_col=MediaItem.id, + ) + elif custom_catalog: query = await _build_custom_catalog_query(custom_catalog, search) order_by = custom_catalog.order_by or "manual" order_dir = custom_catalog.order_dir or "asc" @@ -504,13 +649,22 @@ async def _serve_catalog( ) result = await db.execute(query.offset(skip).limit(limit + 1)) - media_items = result.scalars().all() - has_more = len(media_items) > limit - metas = [ - meta - for media in media_items[:limit] - if (meta := _build_meta(media, catalog_type)) is not None - ] + if external_catalog: + rows = result.all() + has_more = len(rows) > limit + metas = [ + meta + for item, media in rows[:limit] + if (meta := _build_external_item_meta(item, media, catalog_type)) is not None + ] + else: + media_items = result.scalars().all() + has_more = len(media_items) > limit + metas = [ + meta + for media in media_items[:limit] + if (meta := _build_meta(media, catalog_type)) is not None + ] return {"metas": metas, "hasMore": has_more} if has_more else {"metas": metas} diff --git a/backend/src/librarysync/config.py b/backend/src/librarysync/config.py index f77ed1b..3fe52a9 100644 --- a/backend/src/librarysync/config.py +++ b/backend/src/librarysync/config.py @@ -42,6 +42,8 @@ class Settings: enable_dashboard_stats: bool trakt_max_batch_size: int simkl_max_batch_size: int + external_catalog_refresh_hours: int + external_catalog_max_items: int def load_settings() -> Settings: @@ -115,6 +117,12 @@ def load_settings() -> Settings: simkl_max_batch_size=int( _get_env("LIBRARYSYNC_SIMKL_MAX_BATCH_SIZE", "750") or "750" ), + external_catalog_refresh_hours=int( + _get_env("LIBRARYSYNC_EXTERNAL_CATALOG_REFRESH_HOURS", "3") or "3" + ), + external_catalog_max_items=int( + _get_env("LIBRARYSYNC_EXTERNAL_CATALOG_MAX_ITEMS", "500") or "500" + ), ) diff --git a/backend/src/librarysync/core/external_catalog.py b/backend/src/librarysync/core/external_catalog.py new file mode 100644 index 0000000..fe43989 --- /dev/null +++ b/backend/src/librarysync/core/external_catalog.py @@ -0,0 +1,1123 @@ +from __future__ import annotations + +import asyncio +import ipaddress +import socket +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from urllib.parse import quote, urlparse, urlunparse + +import httpx +from sqlalchemy import delete, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from librarysync.config import settings +from librarysync.connectors.metadata.tmdb import TmdbMetadataProvider +from librarysync.connectors.metadata.tvdb import TvdbMetadataProvider +from librarysync.connectors.services.letterboxd import ( + DEFAULT_LETTERBOXD_API_BASE_URL, + LetterboxdClient, + has_required_letterboxd_fields, +) +from librarysync.core.catalog_ordering import CatalogOrderBy +from librarysync.core.http_client import get_http_client +from librarysync.core.integrations import load_integration_with_secrets +from librarysync.core.metadata_providers import MetadataProviderService +from librarysync.core.watchlist_links import ( + parse_imdb_chart_urls, + parse_letterboxd_list_urls, + parse_mdblist_urls, + parse_tmdb_chart_urls, + parse_tmdb_list_urls, + parse_tvdb_list_urls, +) +from librarysync.db.models import MediaItem, StremioExternalCatalog, StremioExternalCatalogItem +from librarysync.jobs.letterboxd_import import build_letterboxd_list_candidate + +EXTERNAL_CATALOG_ORDER_BY = {"source", "random", *CatalogOrderBy.__args__} +EXTERNAL_CATALOG_ORDER_DIR = {"asc", "desc"} +EXTERNAL_CATALOG_DEFAULT_PAGE_SIZE = 30 +EXTERNAL_CATALOG_DEFAULT_SHOW_IN_HOME = True +EXTERNAL_CATALOG_DEFAULT_FILTERS = {"show_watched": False, "statuses": []} +EXTERNAL_CATALOG_FETCH_PAGE_SIZE = 100 +EXTERNAL_SOURCE_KINDS = {"manifest", "list"} +EXTERNAL_LIST_PROVIDERS = {"tmdb", "tvdb", "letterboxd", "mdblist"} +TMDB_LIST_TYPE = "movie" +TMDB_CHART_ENDPOINTS = { + "tmdb-chart:movie:top-rated": ("/movie/top_rated", "movie"), + "tmdb-chart:movie:popular": ("/movie/popular", "movie"), + "tmdb-chart:movie:now-playing": ("/movie/now_playing", "movie"), + "tmdb-chart:movie:upcoming": ("/movie/upcoming", "movie"), + "tmdb-chart:tv:top-rated": ("/tv/top_rated", "series"), + "tmdb-chart:tv:popular": ("/tv/popular", "series"), + "tmdb-chart:tv:on-the-air": ("/tv/on_the_air", "series"), + "tmdb-chart:tv:airing-today": ("/tv/airing_today", "series"), +} + + +@dataclass(frozen=True) +class ExternalCatalogListItem: + stremio_id: str + stremio_type: str + title: str | None + year: int | None + poster_url: str | None + imdb_id: str | None = None + tmdb_id: str | None = None + tvdb_id: str | None = None + + +class ExternalCatalogProviderError(Exception): + pass + + +def _is_disallowed_host_address(value: str) -> bool: + ip = ipaddress.ip_address(value) + return ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ) + + +async def _validate_external_manifest_host(hostname: str | None) -> None: + if not hostname: + raise ValueError("Manifest URL host is required") + if hostname.lower() == "localhost": + raise ValueError("Manifest URL host is not allowed") + + try: + if _is_disallowed_host_address(hostname): + raise ValueError("Manifest URL host is not allowed") + return + except ValueError as exc: + if str(exc) == "Manifest URL host is not allowed": + raise + pass + + try: + resolved = await asyncio.get_running_loop().getaddrinfo( + hostname, + None, + proto=socket.IPPROTO_TCP, + ) + except socket.gaierror: + return + + for entry in resolved: + address = entry[4][0] + if _is_disallowed_host_address(address): + raise ValueError("Manifest URL host is not allowed") + + +def _decode_external_json(response: httpx.Response, context: str) -> dict[str, Any]: + try: + payload = response.json() + except ValueError as exc: + raise ExternalCatalogProviderError(f"{context} returned invalid JSON") from exc + if not isinstance(payload, dict): + raise ExternalCatalogProviderError(f"{context} returned an invalid payload") + return payload + + +def _normalize_external_manifest_url_path(value: str) -> str: + manifest_url = (value or "").strip() + if not manifest_url: + return "" + parsed = urlparse(manifest_url) + if parsed.scheme.lower() not in {"http", "https"}: + raise ValueError("Manifest URL must use http or https") + if not parsed.netloc: + raise ValueError("Manifest URL host is required") + + path = parsed.path or "" + if not path.endswith("/manifest.json"): + path = f"{path.rstrip('/')}/manifest.json" if path else "/manifest.json" + return urlunparse(parsed._replace(path=path)) + + +async def normalize_external_manifest_url(value: str) -> str: + normalized_url = _normalize_external_manifest_url_path(value) + if not normalized_url: + return "" + await _validate_external_manifest_host(urlparse(normalized_url).hostname) + return normalized_url + + +def normalize_external_source_kind(value: str | None) -> str: + normalized = (value or "manifest").strip().lower() + return normalized if normalized in EXTERNAL_SOURCE_KINDS else "manifest" + + +def normalize_external_source_provider(value: str | None) -> str | None: + normalized = (value or "").strip().lower() + return normalized if normalized in EXTERNAL_LIST_PROVIDERS else None + + +def normalize_external_filters(filters: dict[str, Any] | None) -> dict[str, Any]: + base = filters if isinstance(filters, dict) else {} + statuses = ( + base.get("statuses") + if isinstance(base.get("statuses"), list) + else EXTERNAL_CATALOG_DEFAULT_FILTERS["statuses"] + ) + show_watched = ( + base.get("show_watched") + if isinstance(base.get("show_watched"), bool) + else EXTERNAL_CATALOG_DEFAULT_FILTERS["show_watched"] + ) + return {"statuses": [str(value) for value in statuses if value], "show_watched": show_watched} + + +def normalize_external_order_by(value: str | None) -> str: + normalized = value.strip().lower() if value else "source" + return normalized if normalized in EXTERNAL_CATALOG_ORDER_BY else "source" + + +def normalize_external_order_dir(value: str | None) -> str: + normalized = value.strip().lower() if value else "asc" + return normalized if normalized in EXTERNAL_CATALOG_ORDER_DIR else "asc" + + +def normalize_external_page_size(value: int | None) -> int: + if isinstance(value, int) and value > 0: + return min(value, 100) + return EXTERNAL_CATALOG_DEFAULT_PAGE_SIZE + + +def normalize_external_show_in_home(value: bool | None) -> bool: + return value if isinstance(value, bool) else EXTERNAL_CATALOG_DEFAULT_SHOW_IN_HOME + + +def external_catalog_out(catalog: StremioExternalCatalog) -> dict[str, Any]: + source_kind = normalize_external_source_kind(getattr(catalog, "source_kind", None)) + return { + "id": catalog.id, + "name": catalog.name, + "slug": catalog.slug, + "source_kind": source_kind, + "source_provider": normalize_external_source_provider( + getattr(catalog, "source_provider", None) + ), + "source_url": catalog.manifest_url, + "addon_name": catalog.addon_name, + "manifest_url": catalog.manifest_url, + "source_catalog_id": catalog.source_catalog_id, + "source_catalog_type": catalog.source_catalog_type, + "media_type": catalog.media_type, + "enabled": catalog.enabled, + "filters": normalize_external_filters(catalog.filters), + "order_by": catalog.order_by, + "order_dir": catalog.order_dir, + "page_size": catalog.page_size, + "show_in_home": catalog.show_in_home, + "last_refreshed_at": catalog.last_refreshed_at, + "last_refresh_error": catalog.last_refresh_error, + "created_at": catalog.created_at, + "updated_at": catalog.updated_at, + } + + +def external_catalog_manifest_entry(catalog: StremioExternalCatalog) -> dict[str, Any]: + return { + "type": catalog.source_catalog_type, + "id": catalog.slug, + "name": catalog.name, + "extraSupported": ["search", "skip", "limit"], + "pageSize": normalize_external_page_size(catalog.page_size), + "showInHome": normalize_external_show_in_home(catalog.show_in_home), + } + + +def stremio_type_to_media_types(stremio_type: str) -> tuple[str, ...]: + if stremio_type == "movie": + return ("movie",) + if stremio_type == "series": + return ("tv", "anime") + return () + + +async def discover_external_catalogs(manifest_url: str) -> dict[str, Any]: + raise NotImplementedError("Use discover_external_catalog_source") + + +async def discover_external_catalog_source( + db: AsyncSession, + user_id: str, + source_url: str, +) -> dict[str, Any]: + cleaned_url = (source_url or "").strip() + if not cleaned_url: + raise ValueError("Source URL is required") + + tmdb_refs = parse_tmdb_list_urls([cleaned_url]) + if tmdb_refs: + return await _discover_tmdb_list_catalogs(db, user_id, tmdb_refs[0]) + + tmdb_chart_refs = parse_tmdb_chart_urls([cleaned_url]) + if tmdb_chart_refs: + return await _discover_tmdb_chart_catalogs(tmdb_chart_refs[0]) + + tvdb_refs = parse_tvdb_list_urls([cleaned_url]) + if tvdb_refs: + return await _discover_tvdb_list_catalogs(db, user_id, tvdb_refs[0]) + + letterboxd_refs = parse_letterboxd_list_urls([cleaned_url]) + if letterboxd_refs: + return await _discover_letterboxd_list_catalogs(db, user_id, letterboxd_refs[0]) + + mdblist_refs = parse_mdblist_urls([cleaned_url]) + if mdblist_refs: + return await _discover_mdblist_catalogs(mdblist_refs[0]) + + imdb_chart_refs = parse_imdb_chart_urls([cleaned_url]) + if imdb_chart_refs: + raise ValueError( + "IMDb chart pages are not supported. Use TMDB charts/lists, TVDB lists, " + "Letterboxd lists, or a Stremio manifest." + ) + + return await _discover_manifest_catalogs(cleaned_url) + + +async def _discover_manifest_catalogs(manifest_url: str) -> dict[str, Any]: + normalized_url = await normalize_external_manifest_url(manifest_url) + if not normalized_url: + raise ValueError("Manifest URL is required") + + async with get_http_client() as client: + response = await client.get(normalized_url) + response.raise_for_status() + payload = _decode_external_json(response, "External manifest") + + addon_name = str(payload.get("name") or "External addon").strip() or "External addon" + discovered: list[dict[str, str]] = [] + for catalog in payload.get("catalogs") or []: + if not isinstance(catalog, dict): + continue + catalog_id = str(catalog.get("id") or "").strip() + catalog_type = str(catalog.get("type") or "").strip().lower() + if not catalog_id or catalog_type not in {"movie", "series"}: + continue + discovered.append( + { + "id": catalog_id, + "name": str(catalog.get("name") or catalog_id), + "type": catalog_type, + "source_kind": "manifest", + "source_provider": None, + } + ) + + return { + "source_kind": "manifest", + "source_provider": None, + "source_url": normalized_url, + "manifest_url": normalized_url, + "addon_name": addon_name, + "catalogs": sorted(discovered, key=lambda entry: (entry["type"], entry["name"].lower())), + } + + +@dataclass(frozen=True) +class _ExternalMediaItemLookup: + by_stremio_id: dict[tuple[str, str], str] + by_imdb_id: dict[tuple[str, str], str] + + +def _media_item_type_to_stremio_type(media_type: str) -> str | None: + if media_type == "movie": + return "movie" + if media_type in {"tv", "anime", "series"}: + return "series" + return None + + +async def _prefetch_external_media_item_ids( + db: AsyncSession, + metas: list[dict[str, Any]], + default_stremio_type: str | None, +) -> _ExternalMediaItemLookup: + filters = [] + + for meta in metas: + stremio_id = str(meta.get("id") or "").strip() + stremio_type = str(meta.get("type") or default_stremio_type or "").strip().lower() + media_types = stremio_type_to_media_types(stremio_type) + if not stremio_id or not media_types: + continue + + filters.append( + MediaItem.media_type.in_(media_types) & (MediaItem.raw["stremio_id"].as_string() == stremio_id) + ) + + imdb_id = _extract_imdb_id(meta, stremio_id) + if imdb_id: + filters.append(MediaItem.media_type.in_(media_types) & (MediaItem.imdb_id == imdb_id)) + + if not filters: + return _ExternalMediaItemLookup(by_stremio_id={}, by_imdb_id={}) + + result = await db.execute(select(MediaItem).where(or_(*filters))) + media_items = result.scalars().all() + + by_stremio_id: dict[tuple[str, str], str] = {} + by_imdb_id: dict[tuple[str, str], str] = {} + for media_item in media_items: + media_type = str(getattr(media_item, "media_type", "") or "").strip().lower() + stremio_type = _media_item_type_to_stremio_type(media_type) + if stremio_type is None: + continue + + media_item_id = str(media_item.id) + raw = media_item.raw if isinstance(media_item.raw, dict) else {} + item_stremio_id = str(raw.get("stremio_id") or "").strip() + if item_stremio_id: + by_stremio_id.setdefault((stremio_type, item_stremio_id), media_item_id) + + item_imdb_id = str(getattr(media_item, "imdb_id", "") or "").strip() + if item_imdb_id: + by_imdb_id.setdefault((stremio_type, item_imdb_id), media_item_id) + + return _ExternalMediaItemLookup(by_stremio_id=by_stremio_id, by_imdb_id=by_imdb_id) + + +def _lookup_prefetched_external_media_item_id( + lookup: _ExternalMediaItemLookup, + stremio_id: str, + imdb_id: str | None, + stremio_type: str, +) -> str | None: + media_item_id = lookup.by_stremio_id.get((stremio_type, stremio_id)) + if media_item_id is not None: + return media_item_id + if imdb_id: + return lookup.by_imdb_id.get((stremio_type, imdb_id)) + return None + + +def _build_external_manifest_catalog_url( + manifest_url: str, + catalog_type: str, + catalog_id: str, + *, + skip: int, + limit: int, +) -> str: + parsed = urlparse(manifest_url) + if not parsed.path.endswith("/manifest.json"): + raise ValueError("Manifest URL must end with /manifest.json") + + base_path = parsed.path[: -len("/manifest.json")] + catalog_path = ( + f"{base_path}/catalog/{quote(catalog_type, safe='')}/{quote(catalog_id, safe='')}" + f"/skip={skip}&limit={limit}.json" + ) + return urlunparse(parsed._replace(path=catalog_path)) + + +async def refresh_external_catalog( + db: AsyncSession, + catalog: StremioExternalCatalog, + *, + max_items: int | None = None, +) -> int: + if normalize_external_source_kind(getattr(catalog, "source_kind", None)) == "list": + return await _refresh_external_list_catalog(db, catalog, max_items=max_items) + + normalized_manifest_url = await normalize_external_manifest_url(catalog.manifest_url) + max_catalog_items = max_items or settings.external_catalog_max_items + fetched_at = datetime.now(timezone.utc) + + async with get_http_client() as client: + manifest_response = await client.get(normalized_manifest_url) + manifest_response.raise_for_status() + manifest_payload = _decode_external_json(manifest_response, "External manifest") + catalog.addon_name = ( + str(manifest_payload.get("name") or catalog.addon_name or "").strip() or None + ) + + metas: list[dict[str, Any]] = [] + skip = 0 + while len(metas) < max_catalog_items: + limit = min(EXTERNAL_CATALOG_FETCH_PAGE_SIZE, max_catalog_items - len(metas)) + catalog_url = _build_external_manifest_catalog_url( + normalized_manifest_url, + catalog.source_catalog_type, + catalog.source_catalog_id, + skip=skip, + limit=limit, + ) + response = await client.get(catalog_url) + response.raise_for_status() + payload = _decode_external_json(response, "External catalog") + batch = payload.get("metas") + if not isinstance(batch, list) or not batch: + break + metas.extend(item for item in batch if isinstance(item, dict)) + if len(batch) < limit: + break + skip += limit + + await db.execute( + delete(StremioExternalCatalogItem).where( + StremioExternalCatalogItem.catalog_id == catalog.id + ) + ) + + metas = _dedupe_external_metas(metas) + media_item_lookup = await _prefetch_external_media_item_ids( + db, + metas, + str(catalog.source_catalog_type or "").strip().lower() or None, + ) + created = 0 + for position, meta in enumerate(metas): + stremio_id = str(meta.get("id") or "").strip() + stremio_type = str(meta.get("type") or catalog.source_catalog_type or "").strip().lower() + if not stremio_id or stremio_type not in {"movie", "series"}: + continue + imdb_id = _extract_imdb_id(meta, stremio_id) + media_item_id = _lookup_prefetched_external_media_item_id( + media_item_lookup, + stremio_id, + imdb_id, + stremio_type, + ) + db.add( + StremioExternalCatalogItem( + id=str(uuid.uuid4()), + catalog_id=catalog.id, + media_item_id=media_item_id, + stremio_id=stremio_id, + stremio_type=stremio_type, + title=_coerce_text(meta.get("name")), + year=_coerce_year(meta.get("year")), + poster_url=_coerce_text(meta.get("poster")), + imdb_id=imdb_id, + position=position, + fetched_at=fetched_at, + ) + ) + created += 1 + + catalog.manifest_url = normalized_manifest_url + catalog.last_refreshed_at = fetched_at + catalog.last_refresh_error = None + catalog.updated_at = fetched_at + db.add(catalog) + await db.flush() + return created + + +async def mark_external_catalog_refresh_failed( + db: AsyncSession, + catalog: StremioExternalCatalog, + error: Exception, +) -> None: + catalog.last_refresh_error = str(error) + catalog.updated_at = datetime.now(timezone.utc) + db.add(catalog) + await db.flush() + + +async def _refresh_external_list_catalog( + db: AsyncSession, + catalog: StremioExternalCatalog, + *, + max_items: int | None = None, +) -> int: + provider = normalize_external_source_provider(catalog.source_provider) + if not provider: + raise ValueError("List provider is required") + + max_catalog_items = max_items or settings.external_catalog_max_items + if provider == "tmdb": + items = await _load_tmdb_list_items( + db, + catalog.user_id, + catalog.manifest_url, + catalog.source_catalog_id, + max_catalog_items, + ) + elif provider == "tvdb": + items = await _load_tvdb_list_items( + db, catalog.user_id, catalog.manifest_url, max_catalog_items + ) + elif provider == "letterboxd": + items = await _load_letterboxd_list_items( + db, catalog.user_id, catalog.manifest_url, max_catalog_items + ) + elif provider == "mdblist": + items = await _load_mdblist_items(catalog.manifest_url, max_catalog_items) + else: + raise ValueError("Unsupported list provider") + + items = [item for item in items if item.stremio_type == catalog.source_catalog_type] + items = _dedupe_external_list_items(items) + + fetched_at = datetime.now(timezone.utc) + await db.execute( + delete(StremioExternalCatalogItem).where( + StremioExternalCatalogItem.catalog_id == catalog.id + ) + ) + + created = 0 + for position, item in enumerate(items): + media_item_id = await _resolve_external_media_item_id( + db, + item.stremio_id, + item.imdb_id, + item.stremio_type, + tmdb_id=item.tmdb_id, + tvdb_id=item.tvdb_id, + ) + db.add( + StremioExternalCatalogItem( + id=str(uuid.uuid4()), + catalog_id=catalog.id, + media_item_id=media_item_id, + stremio_id=item.stremio_id, + stremio_type=item.stremio_type, + title=item.title, + year=item.year, + poster_url=item.poster_url, + imdb_id=item.imdb_id, + position=position, + fetched_at=fetched_at, + ) + ) + created += 1 + + catalog.last_refreshed_at = fetched_at + catalog.last_refresh_error = None + catalog.updated_at = fetched_at + db.add(catalog) + await db.flush() + return created + + +def _dedupe_external_metas(metas: list[dict[str, Any]]) -> list[dict[str, Any]]: + deduped: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for meta in metas: + stremio_id = str(meta.get("id") or "").strip() + if not stremio_id or stremio_id in seen_ids: + continue + seen_ids.add(stremio_id) + deduped.append(meta) + return deduped + + +def _dedupe_external_list_items( + items: list[ExternalCatalogListItem], +) -> list[ExternalCatalogListItem]: + deduped: list[ExternalCatalogListItem] = [] + seen_ids: set[str] = set() + for item in items: + stremio_id = item.stremio_id.strip() + if not stremio_id or stremio_id in seen_ids: + continue + seen_ids.add(stremio_id) + deduped.append(item) + return deduped + + +def _coerce_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _coerce_year(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str): + cleaned = value.strip() + if cleaned.isdigit(): + return int(cleaned) + if len(cleaned) >= 4 and cleaned[:4].isdigit(): + return int(cleaned[:4]) + return None + + +def _extract_imdb_id(meta: dict[str, Any], stremio_id: str) -> str | None: + imdb_id = _coerce_text(meta.get("imdb_id")) + if imdb_id: + return imdb_id + if stremio_id.startswith("tt"): + return stremio_id + return None + + +async def _resolve_external_media_item_id( + db: AsyncSession, + stremio_id: str, + imdb_id: str | None, + stremio_type: str, + *, + tmdb_id: str | None = None, + tvdb_id: str | None = None, +) -> str | None: + media_types = stremio_type_to_media_types(stremio_type) + if not media_types: + return None + + conditions = [MediaItem.raw["stremio_id"].as_string() == stremio_id] + if imdb_id: + conditions.append(MediaItem.imdb_id == imdb_id) + if tmdb_id: + conditions.append(MediaItem.tmdb_id == tmdb_id) + if tvdb_id: + conditions.append(MediaItem.tvdb_id == tvdb_id) + result = await db.execute( + select(MediaItem.id) + .where(MediaItem.media_type.in_(media_types), or_(*conditions)) + .limit(1) + ) + media_item_id = result.scalars().first() + return str(media_item_id) if media_item_id else None + + +async def _discover_tmdb_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + provider = await _load_tmdb_provider(db, user_id) + payload = await provider._get(f"/list/{ref.list_id}", {"page": 1}) + name = str(payload.get("name") or ref.name).strip() or ref.name + return { + "source_kind": "list", + "source_provider": "tmdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TMDB List", + "catalogs": [{"id": ref.external_id, "name": name, "type": TMDB_LIST_TYPE}], + } + + +async def _discover_tmdb_chart_catalogs(ref) -> dict[str, Any]: + return { + "source_kind": "list", + "source_provider": "tmdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TMDB Charts", + "catalogs": [ + { + "id": ref.external_id, + "name": ref.name, + "type": "movie" if ref.media_type == "movie" else "series", + } + ], + } + + +async def _discover_tvdb_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + provider = await _load_tvdb_provider(db, user_id) + payload = await _tvdb_get_list_payload(provider, ref.list_id) + data = payload.get("data") if isinstance(payload, dict) else None + name = str((data or {}).get("name") or ref.name).strip() or ref.name + catalog_types = _detect_tvdb_catalog_types(data) + return { + "source_kind": "list", + "source_provider": "tvdb", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "TVDB List", + "catalogs": [ + { + "id": f"{ref.external_id}:{catalog_type}", + "name": name, + "type": catalog_type, + } + for catalog_type in catalog_types + ], + } + + +async def _discover_letterboxd_list_catalogs(db: AsyncSession, user_id: str, ref) -> dict[str, Any]: + client, access_token = await _build_letterboxd_client(db, user_id) + payload = await client.fetch_list_by_slug(access_token, ref.username, ref.slug) + name = str(payload.get("name") or ref.name).strip() or ref.name + return { + "source_kind": "list", + "source_provider": "letterboxd", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "Letterboxd List", + "catalogs": [{"id": ref.external_id, "name": name, "type": "movie"}], + } + + +async def _discover_mdblist_catalogs(ref) -> dict[str, Any]: + items = await _load_mdblist_items(ref.url, max_items=50) + catalog_types = sorted({item.stremio_type for item in items}) + if not catalog_types: + catalog_types = ["movie"] + return { + "source_kind": "list", + "source_provider": "mdblist", + "source_url": ref.url, + "manifest_url": ref.url, + "addon_name": "MDBList", + "catalogs": [ + { + "id": f"{ref.external_id}:{catalog_type}", + "name": ref.name.replace("-", " ").title(), + "type": catalog_type, + } + for catalog_type in catalog_types + ], + } + + +async def _load_tmdb_provider(db: AsyncSession, user_id: str) -> TmdbMetadataProvider: + provider = await MetadataProviderService(db, user_id).load_provider("tmdb") + if not isinstance(provider, TmdbMetadataProvider): + raise ValueError("TMDB provider is not enabled or missing API key") + return provider + + +async def _load_tvdb_provider(db: AsyncSession, user_id: str) -> TvdbMetadataProvider: + provider = await MetadataProviderService(db, user_id).load_provider("tvdb") + if not isinstance(provider, TvdbMetadataProvider): + raise ValueError("TVDB provider is not enabled or missing API key") + return provider + + +async def _load_tmdb_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + source_catalog_id: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + provider = await _load_tmdb_provider(db, user_id) + chart_ref = TMDB_CHART_ENDPOINTS.get(source_catalog_id) + if chart_ref: + endpoint, stremio_type = chart_ref + return await _load_tmdb_chart_items(provider, endpoint, stremio_type, max_items) + + refs = parse_tmdb_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported TMDB list URL") + ref = refs[0] + + results: list[dict[str, Any]] = [] + page = 1 + while len(results) < max_items: + payload = await provider._get(f"/list/{ref.list_id}", {"page": page}) + batch = payload.get("items") if isinstance(payload, dict) else None + if not isinstance(batch, list) or not batch: + break + results.extend(item for item in batch if isinstance(item, dict)) + total_pages = int(payload.get("total_pages") or page) + if page >= total_pages: + break + page += 1 + + detail_items = await _load_tmdb_external_ids(provider, results[:max_items]) + return [item for item in detail_items if item.stremio_id] + + +async def _load_tmdb_chart_items( + provider: TmdbMetadataProvider, + endpoint: str, + stremio_type: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + results: list[dict[str, Any]] = [] + page = 1 + while len(results) < max_items: + payload = await provider._get(endpoint, {"page": page}) + batch = payload.get("results") if isinstance(payload, dict) else None + if not isinstance(batch, list) or not batch: + break + results.extend(item for item in batch if isinstance(item, dict)) + total_pages = int(payload.get("total_pages") or page) + if page >= total_pages: + break + page += 1 + return await _load_tmdb_external_ids( + provider, + results[:max_items], + stremio_type=stremio_type, + ) + + +async def _load_tmdb_external_ids( + provider: TmdbMetadataProvider, + items: list[dict[str, Any]], + *, + stremio_type: str = "movie", +) -> list[ExternalCatalogListItem]: + semaphore = asyncio.Semaphore(8) + + async def _build(item: dict[str, Any]) -> ExternalCatalogListItem | None: + tmdb_id = _coerce_text(item.get("id")) + if not tmdb_id: + return None + details_path = "/movie" if stremio_type == "movie" else "/tv" + async with semaphore: + payload = await provider._get(f"{details_path}/{tmdb_id}/external_ids", {}) + imdb_id = _coerce_text(payload.get("imdb_id")) + stremio_id = imdb_id + if not stremio_id: + return None + return ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type=stremio_type, + title=_coerce_text( + item.get("title") + or item.get("original_title") + or item.get("name") + or item.get("original_name") + ), + year=_coerce_year( + item.get("release_date") + or item.get("first_air_date") + or item.get("year") + ), + poster_url=_tmdb_poster_url(item.get("poster_path")), + imdb_id=imdb_id, + tmdb_id=tmdb_id, + ) + + results = await asyncio.gather(*[_build(item) for item in items]) + return [item for item in results if item is not None] + + +async def _load_tvdb_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + refs = parse_tvdb_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported TVDB list URL") + ref = refs[0] + provider = await _load_tvdb_provider(db, user_id) + payload = await _tvdb_get_list_payload(provider, ref.list_id) + data = payload.get("data") if isinstance(payload, dict) else None + entities = (data or {}).get("entities") if isinstance(data, dict) else None + if not isinstance(entities, list): + return [] + + items: list[ExternalCatalogListItem] = [] + for entry in entities[:max_items]: + if not isinstance(entry, dict): + continue + media_type = _normalize_tvdb_entity_type(entry) + if not media_type: + continue + imdb_id = _extract_remote_id(entry, "imdb") + tvdb_id = _coerce_text(entry.get("id")) + stremio_id = imdb_id + if not stremio_id: + continue + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie" if media_type == "movie" else "series", + title=_coerce_text(entry.get("name") or entry.get("title") or entry.get("slug")), + year=_coerce_year( + entry.get("year") + or entry.get("firstAired") + or entry.get("first_aired") + or entry.get("releaseDate") + ), + poster_url=_coerce_text( + entry.get("image") or entry.get("image_url") or entry.get("imageUrl") + ), + imdb_id=imdb_id, + tmdb_id=_extract_remote_id(entry, "tmdb"), + tvdb_id=tvdb_id, + ) + ) + return items + + +async def _load_letterboxd_list_items( + db: AsyncSession, + user_id: str, + source_url: str, + max_items: int, +) -> list[ExternalCatalogListItem]: + refs = parse_letterboxd_list_urls([source_url]) + if not refs: + raise ValueError("Unsupported Letterboxd list URL") + ref = refs[0] + client, access_token = await _build_letterboxd_client(db, user_id) + entries = await client.get_list_entries( + access_token, + ref.username, + ref.slug, + per_page=50, + max_pages=10, + ) + items: list[ExternalCatalogListItem] = [] + list_context = {"name": ref.name, "url": ref.url, "type": "list"} + for entry in entries: + candidate = build_letterboxd_list_candidate(entry, list_context=list_context) + if not candidate: + continue + imdb_id = candidate.ids.get("imdb_id") + stremio_id = imdb_id + if not stremio_id: + continue + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie", + title=candidate.title, + year=candidate.year, + poster_url=candidate.poster_url, + imdb_id=imdb_id, + tmdb_id=candidate.ids.get("tmdb_id"), + ) + ) + if len(items) >= max_items: + break + return items + + +async def _load_mdblist_items(source_url: str, max_items: int) -> list[ExternalCatalogListItem]: + refs = parse_mdblist_urls([source_url]) + if not refs: + raise ValueError("Unsupported MDBList URL") + ref = refs[0] + json_url = f"https://mdblist.com/lists/{ref.username}/{ref.slug}/json/" + async with get_http_client() as client: + response = await client.get(json_url) + response.raise_for_status() + payload = response.json() + + if not isinstance(payload, list): + raise ValueError("MDBList list payload is invalid") + + items: list[ExternalCatalogListItem] = [] + for entry in payload[:max_items]: + if not isinstance(entry, dict): + continue + imdb_id = _coerce_text(entry.get("imdb_id")) + stremio_id = imdb_id + if not stremio_id: + continue + media_type = _coerce_text(entry.get("mediatype")) or "movie" + items.append( + ExternalCatalogListItem( + stremio_id=stremio_id, + stremio_type="movie" if media_type == "movie" else "series", + title=_coerce_text(entry.get("title")), + year=_coerce_year(entry.get("release_year") or entry.get("year")), + poster_url=None, + imdb_id=imdb_id, + tmdb_id=_coerce_text( + entry.get("tmdb_id") or entry.get("tmdbid") or entry.get("id") + ), + tvdb_id=_coerce_text(entry.get("tvdb_id") or entry.get("tvdbid")), + ) + ) + return items + + +async def _build_letterboxd_client( + db: AsyncSession, user_id: str +) -> tuple[LetterboxdClient, str]: + integration, secret_data = await load_integration_with_secrets(db, user_id, "letterboxd") + if not integration or integration.status == "disconnected": + raise ValueError("Letterboxd integration is not connected") + if not secret_data or not has_required_letterboxd_fields(secret_data): + raise ValueError("Letterboxd credentials are incomplete") + api_base_url = DEFAULT_LETTERBOXD_API_BASE_URL + if integration.config and integration.config.get("api_base_url"): + api_base_url = str(integration.config["api_base_url"]) + client = LetterboxdClient( + api_base_url=api_base_url, + client_id=str(secret_data.get("client_id")), + client_secret=str(secret_data.get("client_secret")), + refresh_token=str(secret_data.get("refresh_token")), + ) + access_token = await client.refresh_access_token() + return client, access_token + + +async def _tvdb_get_list_payload(provider: TvdbMetadataProvider, list_id: str) -> dict[str, Any]: + try: + return await provider._get(f"/lists/{list_id}/extended", {}) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + return await provider._get(f"/lists/slug/{list_id}", {}) + + +def _detect_tvdb_catalog_types(data: dict[str, Any] | None) -> list[str]: + entities = (data or {}).get("entities") if isinstance(data, dict) else None + if not isinstance(entities, list): + return ["series"] + has_movie = False + has_series = False + for entry in entities: + media_type = _normalize_tvdb_entity_type(entry) if isinstance(entry, dict) else None + if media_type == "movie": + has_movie = True + elif media_type == "tv": + has_series = True + if has_movie and has_series: + return ["movie", "series"] + if has_movie: + return ["movie"] + return ["series"] + + +def _normalize_tvdb_entity_type(entry: dict[str, Any]) -> str | None: + raw = _coerce_text(entry.get("type") or entry.get("recordType")) + if not raw: + raw = _coerce_text(entry.get("companyType") or entry.get("kind")) + if not raw: + return None + lowered = raw.lower() + if lowered in {"movie", "movies"}: + return "movie" + if lowered in {"series", "tv", "show"}: + return "tv" + return None + + +def _extract_remote_id(entry: dict[str, Any], provider: str) -> str | None: + remote_ids = entry.get("remoteIds") or entry.get("remote_ids") or [] + if not isinstance(remote_ids, list): + return None + for remote in remote_ids: + if not isinstance(remote, dict): + continue + source_name = str(remote.get("sourceName") or "").lower() + source = str(remote.get("source") or "").lower() + entry_type = str(remote.get("type") or "").lower() + if provider == "imdb" and ( + source_name == "imdb" or source == "imdb" or entry_type == "imdb" + ): + return _coerce_text(remote.get("id") or remote.get("value")) + if provider == "tmdb" and ( + source_name == "tmdb" + or "themoviedb" in source_name + or source == "tmdb" + or entry_type == "tmdb" + ): + return _coerce_text(remote.get("id") or remote.get("value")) + return None + + +def _tmdb_poster_url(path: object) -> str | None: + value = _coerce_text(path) + if not value: + return None + if value.startswith("http://") or value.startswith("https://"): + return value + return f"https://image.tmdb.org/t/p/w185{value}" diff --git a/backend/src/librarysync/core/watchlist_links.py b/backend/src/librarysync/core/watchlist_links.py index 93b488f..f454f90 100644 --- a/backend/src/librarysync/core/watchlist_links.py +++ b/backend/src/librarysync/core/watchlist_links.py @@ -1,9 +1,12 @@ from __future__ import annotations +import re from dataclasses import dataclass from typing import Iterable from urllib.parse import urlparse +TMDB_LIST_ID_RE = re.compile(r"^(\d+)") + @dataclass(frozen=True) class TraktListRef: @@ -23,6 +26,46 @@ class LetterboxdListRef: name: str +@dataclass(frozen=True) +class TmdbListRef: + list_id: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class TmdbChartRef: + media_type: str + chart_slug: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class TvdbListRef: + list_id: str + url: str + external_id: str + name: str + + +@dataclass(frozen=True) +class ImdbChartRef: + chart_slug: str + url: str + + +@dataclass(frozen=True) +class MdblistRef: + username: str + slug: str + url: str + external_id: str + name: str + + def parse_trakt_list_urls(urls: Iterable[str]) -> list[TraktListRef]: refs: list[TraktListRef] = [] for url in urls: @@ -68,6 +111,9 @@ def parse_letterboxd_list_urls(urls: Iterable[str]) -> list[LetterboxdListRef]: if not cleaned: continue parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"letterboxd.com", "www.letterboxd.com"}: + continue path = parsed.path.strip("/") segments = [segment for segment in path.split("/") if segment] if len(segments) >= 3 and segments[1] == "list": @@ -85,3 +131,147 @@ def parse_letterboxd_list_urls(urls: Iterable[str]) -> list[LetterboxdListRef]: ) return refs + +def parse_tmdb_list_urls(urls: Iterable[str]) -> list[TmdbListRef]: + refs: list[TmdbListRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"themoviedb.org", "www.themoviedb.org"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + match = ( + TMDB_LIST_ID_RE.match(segments[1]) + if len(segments) >= 2 and segments[0] == "list" + else None + ) + if match: + list_id = match.group(1) + refs.append( + TmdbListRef( + list_id=list_id, + url=cleaned, + external_id=f"tmdb:{list_id}", + name=list_id, + ) + ) + return refs + + +def parse_tmdb_chart_urls(urls: Iterable[str]) -> list[TmdbChartRef]: + refs: list[TmdbChartRef] = [] + chart_names = { + ("movie", "top-rated"): "Top Rated Movies", + ("movie", "popular"): "Popular Movies", + ("movie", "now-playing"): "Now Playing Movies", + ("movie", "upcoming"): "Upcoming Movies", + ("tv", "top-rated"): "Top Rated Series", + ("tv", "popular"): "Popular Series", + ("tv", "on-the-air"): "On The Air Series", + ("tv", "airing-today"): "Airing Today Series", + } + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"themoviedb.org", "www.themoviedb.org"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) < 2: + continue + media_type = segments[0] + chart_slug = segments[1] + name = chart_names.get((media_type, chart_slug)) + if not name: + continue + refs.append( + TmdbChartRef( + media_type=media_type, + chart_slug=chart_slug, + url=cleaned, + external_id=f"tmdb-chart:{media_type}:{chart_slug}", + name=name, + ) + ) + return refs + + +def parse_tvdb_list_urls(urls: Iterable[str]) -> list[TvdbListRef]: + refs: list[TvdbListRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"thetvdb.com", "www.thetvdb.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) >= 2 and segments[0] == "lists": + list_id = segments[1] + refs.append( + TvdbListRef( + list_id=list_id, + url=cleaned, + external_id=f"tvdb:{list_id}", + name=list_id, + ) + ) + return refs + + +def parse_imdb_chart_urls(urls: Iterable[str]) -> list[ImdbChartRef]: + refs: list[ImdbChartRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"imdb.com", "www.imdb.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if not segments: + continue + if len(segments) >= 3 and len(segments[0]) == 2 and segments[1] == "chart": + refs.append(ImdbChartRef(chart_slug=segments[2], url=cleaned)) + continue + if len(segments) >= 2 and segments[0] == "chart": + refs.append(ImdbChartRef(chart_slug=segments[1], url=cleaned)) + return refs + + +def parse_mdblist_urls(urls: Iterable[str]) -> list[MdblistRef]: + refs: list[MdblistRef] = [] + for url in urls: + cleaned = url.strip() + if not cleaned: + continue + parsed = urlparse(cleaned) + host = (parsed.netloc or "").lower() + if host and host not in {"mdblist.com", "www.mdblist.com"}: + continue + path = parsed.path.strip("/") + segments = [segment for segment in path.split("/") if segment] + if len(segments) >= 3 and segments[0] == "lists": + username = segments[1] + slug = segments[2] + refs.append( + MdblistRef( + username=username, + slug=slug, + url=cleaned, + external_id=f"mdblist:{username}:{slug}", + name=slug, + ) + ) + return refs diff --git a/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py b/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py new file mode 100644 index 0000000..3a09fa9 --- /dev/null +++ b/backend/src/librarysync/db/migrations/versions/d2b7c8f9a1e0_add_stremio_external_catalogs.py @@ -0,0 +1,151 @@ +"""add stremio external catalogs + +Revision ID: d2b7c8f9a1e0 +Revises: a1d4e6f8b9c0 +Create Date: 2026-04-17 14:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "d2b7c8f9a1e0" +down_revision = "a1d4e6f8b9c0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "stremio_external_catalogs", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("user_id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=64), nullable=False), + sa.Column("addon_name", sa.String(length=255), nullable=True), + sa.Column("manifest_url", sa.String(length=500), nullable=False), + sa.Column("source_catalog_id", sa.String(length=255), nullable=False), + sa.Column("source_catalog_type", sa.String(length=32), nullable=False), + sa.Column("media_type", sa.String(length=32), nullable=False, server_default="movie"), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("filters", sa.JSON(), nullable=True), + sa.Column("order_by", sa.String(length=32), nullable=False, server_default="source"), + sa.Column("order_dir", sa.String(length=8), nullable=False, server_default="asc"), + sa.Column("page_size", sa.Integer(), nullable=False, server_default="30"), + sa.Column("show_in_home", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("last_refreshed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_refresh_error", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "slug", name="uq_stremio_external_catalogs_user_slug"), + ) + op.create_index( + op.f("ix_stremio_external_catalogs_user_id"), + "stremio_external_catalogs", + ["user_id"], + ) + + op.create_table( + "stremio_external_catalog_items", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("catalog_id", sa.String(length=36), nullable=False), + sa.Column("media_item_id", sa.String(length=36), nullable=True), + sa.Column("stremio_id", sa.String(length=255), nullable=False), + sa.Column("stremio_type", sa.String(length=32), nullable=False), + sa.Column("title", sa.String(length=255), nullable=True), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column("poster_url", sa.String(length=500), nullable=True), + sa.Column("imdb_id", sa.String(length=32), nullable=True), + sa.Column("position", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "fetched_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.ForeignKeyConstraint( + ["catalog_id"], ["stremio_external_catalogs.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["media_item_id"], ["media_items.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "catalog_id", + "position", + name="uq_stremio_external_catalog_items_catalog_position", + ), + sa.UniqueConstraint( + "catalog_id", + "stremio_id", + name="uq_stremio_external_catalog_items_catalog_stremio", + ), + ) + op.create_index( + "ix_stremio_external_catalog_items_catalog_position", + "stremio_external_catalog_items", + ["catalog_id", "position"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_catalog_id"), + "stremio_external_catalog_items", + ["catalog_id"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_media_item_id"), + "stremio_external_catalog_items", + ["media_item_id"], + ) + op.create_index( + op.f("ix_stremio_external_catalog_items_imdb_id"), + "stremio_external_catalog_items", + ["imdb_id"], + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_stremio_external_catalog_items_imdb_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + op.f("ix_stremio_external_catalog_items_media_item_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + op.f("ix_stremio_external_catalog_items_catalog_id"), + table_name="stremio_external_catalog_items", + ) + op.drop_index( + "ix_stremio_external_catalog_items_catalog_position", + table_name="stremio_external_catalog_items", + ) + op.drop_table("stremio_external_catalog_items") + op.drop_index( + op.f("ix_stremio_external_catalogs_user_id"), + table_name="stremio_external_catalogs", + ) + op.drop_table("stremio_external_catalogs") diff --git a/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py b/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py new file mode 100644 index 0000000..af0a231 --- /dev/null +++ b/backend/src/librarysync/db/migrations/versions/e4f6a8b1c2d3_add_external_catalog_source_fields.py @@ -0,0 +1,32 @@ +"""add external catalog source fields + +Revision ID: e4f6a8b1c2d3 +Revises: d2b7c8f9a1e0 +Create Date: 2026-04-17 16:30:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "e4f6a8b1c2d3" +down_revision = "d2b7c8f9a1e0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "stremio_external_catalogs", + sa.Column("source_kind", sa.String(length=32), nullable=False, server_default="manifest"), + ) + op.add_column( + "stremio_external_catalogs", + sa.Column("source_provider", sa.String(length=32), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("stremio_external_catalogs", "source_provider") + op.drop_column("stremio_external_catalogs", "source_kind") diff --git a/backend/src/librarysync/db/models.py b/backend/src/librarysync/db/models.py index 54eddef..264a5ba 100644 --- a/backend/src/librarysync/db/models.py +++ b/backend/src/librarysync/db/models.py @@ -465,6 +465,99 @@ class StremioCustomCatalogItem(Base): ) +class StremioExternalCatalog(Base): + __tablename__ = "stremio_external_catalogs" + __table_args__ = ( + UniqueConstraint( + "user_id", + "slug", + name="uq_stremio_external_catalogs_user_slug", + ), + Index("ix_stremio_external_catalogs_user_id", "user_id"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + user_id: Mapped[str] = mapped_column( + String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + name: Mapped[str] = mapped_column(String(255)) + slug: Mapped[str] = mapped_column(String(64)) + source_kind: Mapped[str] = mapped_column(String(32), default="manifest") + source_provider: Mapped[str | None] = mapped_column(String(32), nullable=True) + addon_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + manifest_url: Mapped[str] = mapped_column(String(500)) + source_catalog_id: Mapped[str] = mapped_column(String(255)) + source_catalog_type: Mapped[str] = mapped_column(String(32)) + media_type: Mapped[str] = mapped_column(String(32), default="movie") + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + filters: Mapped[dict | None] = mapped_column(JSON, nullable=True) + order_by: Mapped[str] = mapped_column(String(32), default="source") + order_dir: Mapped[str] = mapped_column(String(8), default="asc") + page_size: Mapped[int] = mapped_column(Integer, default=30) + show_in_home: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + last_refreshed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + last_refresh_error: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + + +class StremioExternalCatalogItem(Base): + __tablename__ = "stremio_external_catalog_items" + __table_args__ = ( + UniqueConstraint( + "catalog_id", + "position", + name="uq_stremio_external_catalog_items_catalog_position", + ), + UniqueConstraint( + "catalog_id", + "stremio_id", + name="uq_stremio_external_catalog_items_catalog_stremio", + ), + Index( + "ix_stremio_external_catalog_items_catalog_position", + "catalog_id", + "position", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + catalog_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("stremio_external_catalogs.id", ondelete="CASCADE"), + index=True, + ) + media_item_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("media_items.id", ondelete="SET NULL"), index=True, nullable=True + ) + stremio_id: Mapped[str] = mapped_column(String(255)) + stremio_type: Mapped[str] = mapped_column(String(32)) + title: Mapped[str | None] = mapped_column(String(255), nullable=True) + year: Mapped[int | None] = mapped_column(Integer, nullable=True) + poster_url: Mapped[str | None] = mapped_column(String(500), nullable=True) + imdb_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) + position: Mapped[int] = mapped_column(Integer, default=0) + fetched_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + + class WatchlistItem(Base): __tablename__ = "watchlist_items" __table_args__ = ( diff --git a/backend/src/librarysync/jobs/external_catalog_refresh.py b/backend/src/librarysync/jobs/external_catalog_refresh.py new file mode 100644 index 0000000..d128636 --- /dev/null +++ b/backend/src/librarysync/jobs/external_catalog_refresh.py @@ -0,0 +1,73 @@ +import logging +from datetime import timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from librarysync.config import settings +from librarysync.core.external_catalog import ( + mark_external_catalog_refresh_failed, + refresh_external_catalog, +) +from librarysync.core.scheduler import ( + claim_scheduled_job, + complete_scheduled_job, + extend_scheduled_job, + release_scheduled_job, +) +from librarysync.db.models import ScheduledJob, StremioExternalCatalog +from librarysync.db.session import SessionLocal, init_session_factory + +logger = logging.getLogger(__name__) + +EXTERNAL_CATALOG_REFRESH_JOB = "external_catalog_refresh" +EXTERNAL_CATALOG_REFRESH_INTERVAL = timedelta(hours=settings.external_catalog_refresh_hours) +EXTERNAL_CATALOG_REFRESH_LEASE = timedelta(minutes=30) +EXTERNAL_CATALOG_REFRESH_RETRY_DELAY = timedelta(minutes=5) + + +async def process_external_catalog_refresh_once() -> int: + init_session_factory() + async with SessionLocal() as db: + job = await claim_scheduled_job( + db, + EXTERNAL_CATALOG_REFRESH_JOB, + EXTERNAL_CATALOG_REFRESH_INTERVAL, + EXTERNAL_CATALOG_REFRESH_LEASE, + ) + if not job: + return 0 + try: + refreshed = await run_external_catalog_refresh(db, job) + except Exception: + logger.exception("External catalog refresh failed") + await release_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_RETRY_DELAY) + return 0 + await complete_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_INTERVAL) + return refreshed + + +async def run_external_catalog_refresh(db: AsyncSession, job: ScheduledJob) -> int: + logger.info("Starting external catalog refresh") + result = await db.execute( + select(StremioExternalCatalog) + .where(StremioExternalCatalog.enabled.is_(True)) + .order_by(StremioExternalCatalog.updated_at.asc()) + ) + catalogs = result.scalars().all() + refreshed = 0 + + for catalog in catalogs: + try: + await refresh_external_catalog(db, catalog) + await db.commit() + refreshed += 1 + except Exception as exc: + await db.rollback() + await mark_external_catalog_refresh_failed(db, catalog, exc) + await db.commit() + logger.warning("External catalog refresh failed for %s: %s", catalog.slug, exc) + await extend_scheduled_job(db, job, EXTERNAL_CATALOG_REFRESH_LEASE) + + logger.info("Finished external catalog refresh") + return refreshed diff --git a/backend/src/librarysync/jobs/letterboxd_import.py b/backend/src/librarysync/jobs/letterboxd_import.py index c7d38ef..ee51631 100644 --- a/backend/src/librarysync/jobs/letterboxd_import.py +++ b/backend/src/librarysync/jobs/letterboxd_import.py @@ -494,6 +494,14 @@ def _build_watchlist_candidate( ) +def build_letterboxd_list_candidate( + entry: dict[str, Any], + *, + list_context: dict[str, Any] | None = None, +) -> WatchlistCandidate | None: + return _build_watchlist_candidate(entry, list_context=list_context) + + async def _get_or_create_media_item( db: AsyncSession, film: FilmSummary ) -> MediaItem | None: diff --git a/backend/src/librarysync/static/page-stremio-addon.js b/backend/src/librarysync/static/page-stremio-addon.js index 3dd773b..9c51b55 100644 --- a/backend/src/librarysync/static/page-stremio-addon.js +++ b/backend/src/librarysync/static/page-stremio-addon.js @@ -1,7 +1,13 @@ const addonState = { config: null, catalogs: [], + externalCatalogs: [], customCatalogs: [], + discoveredExternalCatalogs: [], + discoveredExternalSourceUrl: "", + discoveredExternalSourceKind: "manifest", + discoveredExternalSourceProvider: "", + discoveredExternalAddonName: "", selectedCatalogId: null, items: [], candidates: [], @@ -61,6 +67,16 @@ const CUSTOM_ORDER_OPTIONS = [ ...ORDER_OPTIONS, ]; +const EXTERNAL_ORDER_OPTIONS = [ + { value: "source", label: "Source order" }, + ...ORDER_OPTIONS, +]; + +const EXTERNAL_TYPE_LABELS = { + movie: "Movie", + series: "Series", +}; + function setAddonMessage(message, isError = false) { setMessage("stremio-addon-message", message, isError); } @@ -338,6 +354,242 @@ function renderBuiltInCatalogs() { container.appendChild(fragment); } +function getDiscoveredExternalCatalog(key) { + return ( + addonState.discoveredExternalCatalogs.find( + (catalog) => `${catalog.type}::${catalog.id}` === key + ) || null + ); +} + +function renderExternalCatalogDiscovery() { + const form = document.getElementById("external-catalog-form"); + if (!form) { + return; + } + const select = form.querySelector('[name="catalog_key"]'); + if (!select) { + return; + } + const currentValue = select.value; + select.innerHTML = ""; + if (!addonState.discoveredExternalCatalogs.length) { + const option = document.createElement("option"); + option.value = ""; + option.textContent = "Discover a source first"; + select.appendChild(option); + select.disabled = true; + return; + } + const placeholder = document.createElement("option"); + placeholder.value = ""; + placeholder.textContent = "Choose a catalog"; + select.appendChild(placeholder); + addonState.discoveredExternalCatalogs.forEach((catalog) => { + const option = document.createElement("option"); + option.value = `${catalog.type}::${catalog.id}`; + option.textContent = `${catalog.name} · ${EXTERNAL_TYPE_LABELS[catalog.type] || catalog.type}`; + if (option.value === currentValue) { + option.selected = true; + } + select.appendChild(option); + }); + select.disabled = false; +} + +function renderExternalCatalogs() { + const container = document.getElementById("external-catalog-list"); + if (!container) { + return; + } + container.innerHTML = ""; + if (!addonState.externalCatalogs.length) { + const empty = document.createElement("p"); + empty.className = "empty-state"; + empty.textContent = "No external catalogs yet. Discover an addon above."; + container.appendChild(empty); + return; + } + + const fragment = document.createDocumentFragment(); + addonState.externalCatalogs.forEach((catalog) => { + const card = document.createElement("div"); + card.className = "rounded-2xl border border-line/60 bg-surface/80 p-5"; + card.dataset.externalCatalogId = catalog.id; + + const header = document.createElement("div"); + header.className = "flex flex-wrap items-start justify-between gap-3"; + const info = document.createElement("div"); + const title = document.createElement("h3"); + title.className = "font-display text-base font-semibold text-ink"; + title.textContent = catalog.name; + const subtitle = document.createElement("p"); + subtitle.className = "text-xs text-muted"; + const addonName = catalog.addon_name || "External source"; + const sourceLabel = + catalog.source_kind === "list" + ? `${(catalog.source_provider || "list").toUpperCase()} list` + : catalog.source_catalog_id; + subtitle.textContent = `${addonName} · ${sourceLabel} · ${EXTERNAL_TYPE_LABELS[catalog.source_catalog_type] || catalog.source_catalog_type}`; + info.appendChild(title); + info.appendChild(subtitle); + + const actions = document.createElement("div"); + actions.className = "flex flex-wrap items-center gap-2"; + const refreshButton = document.createElement("button"); + refreshButton.type = "button"; + refreshButton.className = "btn btn-secondary btn-sm"; + refreshButton.dataset.externalCatalogRefresh = catalog.id; + refreshButton.textContent = "Refresh"; + const deleteButton = document.createElement("button"); + deleteButton.type = "button"; + deleteButton.className = "btn btn-ghost btn-sm"; + deleteButton.dataset.externalCatalogDelete = catalog.id; + deleteButton.textContent = "Delete"; + actions.appendChild(refreshButton); + actions.appendChild(deleteButton); + + header.appendChild(info); + header.appendChild(actions); + + const body = document.createElement("div"); + body.className = "mt-4 grid gap-4 lg:grid-cols-[minmax(0,2fr)_minmax(0,1fr)]"; + + const visibilityBlock = document.createElement("div"); + const visibilityLabel = document.createElement("p"); + visibilityLabel.className = "text-xs font-semibold uppercase tracking-[0.2em] text-muted"; + visibilityLabel.textContent = "Visibility"; + const visibilityGroup = document.createElement("div"); + visibilityGroup.className = "mt-2 space-y-2"; + + const enabledLabel = document.createElement("label"); + enabledLabel.className = "inline-control"; + const enabledInput = document.createElement("input"); + enabledInput.type = "checkbox"; + enabledInput.checked = catalog.enabled !== false; + enabledInput.dataset.externalCatalogEnabled = "true"; + const enabledText = document.createElement("span"); + enabledText.textContent = "Enabled"; + enabledLabel.appendChild(enabledInput); + enabledLabel.appendChild(enabledText); + + const watchedLabel = document.createElement("label"); + watchedLabel.className = "inline-control"; + const watchedInput = document.createElement("input"); + watchedInput.type = "checkbox"; + watchedInput.checked = normalizeShowWatched(catalog); + watchedInput.dataset.externalCatalogShowWatched = "true"; + const watchedText = document.createElement("span"); + watchedText.textContent = "Show watched"; + watchedLabel.appendChild(watchedInput); + watchedLabel.appendChild(watchedText); + + const homeLabel = document.createElement("label"); + homeLabel.className = "inline-control"; + const homeInput = document.createElement("input"); + homeInput.type = "checkbox"; + homeInput.checked = !!catalog.show_in_home; + homeInput.dataset.externalCatalogShowInHome = "true"; + const homeText = document.createElement("span"); + homeText.textContent = "Show in home"; + homeLabel.appendChild(homeInput); + homeLabel.appendChild(homeText); + + visibilityGroup.appendChild(enabledLabel); + visibilityGroup.appendChild(watchedLabel); + visibilityGroup.appendChild(homeLabel); + visibilityBlock.appendChild(visibilityLabel); + visibilityBlock.appendChild(visibilityGroup); + + const configBlock = document.createElement("div"); + configBlock.className = "space-y-3"; + + const nameField = document.createElement("label"); + nameField.className = "field"; + const nameLabel = document.createElement("span"); + nameLabel.textContent = "Name"; + const nameInput = document.createElement("input"); + nameInput.className = "input"; + nameInput.type = "text"; + nameInput.value = catalog.name || ""; + nameInput.dataset.externalCatalogName = "true"; + nameField.appendChild(nameLabel); + nameField.appendChild(nameInput); + + const orderField = document.createElement("label"); + orderField.className = "field"; + const orderLabel = document.createElement("span"); + orderLabel.textContent = "Order"; + const orderSelect = buildSelect(EXTERNAL_ORDER_OPTIONS, catalog.order_by || "source"); + orderSelect.dataset.externalCatalogOrderBy = "true"; + orderField.appendChild(orderLabel); + orderField.appendChild(orderSelect); + + const dirField = document.createElement("label"); + dirField.className = "field"; + const dirLabel = document.createElement("span"); + dirLabel.textContent = "Direction"; + const dirSelect = buildSelect( + [ + { value: "asc", label: "Ascending" }, + { value: "desc", label: "Descending" }, + ], + catalog.order_dir || "asc" + ); + dirSelect.dataset.externalCatalogOrderDir = "true"; + dirField.appendChild(dirLabel); + dirField.appendChild(dirSelect); + + const sizeField = document.createElement("label"); + sizeField.className = "field"; + const sizeLabel = document.createElement("span"); + sizeLabel.textContent = "Page size"; + const sizeInput = document.createElement("input"); + sizeInput.className = "input"; + sizeInput.type = "number"; + sizeInput.min = "1"; + sizeInput.max = "100"; + sizeInput.value = catalog.page_size || 30; + sizeInput.dataset.externalCatalogPageSize = "true"; + sizeField.appendChild(sizeLabel); + sizeField.appendChild(sizeInput); + + configBlock.appendChild(nameField); + configBlock.appendChild(orderField); + configBlock.appendChild(dirField); + configBlock.appendChild(sizeField); + + body.appendChild(visibilityBlock); + body.appendChild(configBlock); + + const footer = document.createElement("div"); + footer.className = "mt-4 flex flex-wrap items-center justify-between gap-3"; + const status = document.createElement("span"); + status.className = "text-xs text-muted"; + if (catalog.last_refresh_error) { + status.textContent = `Last refresh failed: ${catalog.last_refresh_error}`; + } else if (catalog.last_refreshed_at) { + status.textContent = `Refreshed ${formatMetadataDate(catalog.last_refreshed_at)}`; + } else { + status.textContent = "Never refreshed"; + } + const saveButton = document.createElement("button"); + saveButton.type = "button"; + saveButton.className = "btn btn-secondary btn-sm"; + saveButton.dataset.externalCatalogSave = catalog.id; + saveButton.textContent = "Save changes"; + footer.appendChild(status); + footer.appendChild(saveButton); + + card.appendChild(header); + card.appendChild(body); + card.appendChild(footer); + fragment.appendChild(card); + }); + + container.appendChild(fragment); +} + function renderCustomCatalogs() { const container = document.getElementById("custom-catalog-list"); if (!container) { @@ -845,6 +1097,175 @@ async function handleCustomItemMove(mediaItemId, direction) { } } +async function handleExternalCatalogDiscover() { + const form = document.getElementById("external-catalog-form"); + if (!form) { + return; + } + const manifestInput = form.querySelector('[name="manifest_url"]'); + const nameInput = form.querySelector('[name="name"]'); + const sourceUrl = manifestInput ? manifestInput.value.trim() : ""; + if (!sourceUrl) { + setMessage("external-catalog-message", "Manifest or list URL is required.", true); + return; + } + try { + setMessage("external-catalog-message", "Discovering source..."); + const response = await requestJSON("/api/stremio-addon/external-catalogs/discover", { + method: "POST", + body: JSON.stringify({ source_url: sourceUrl }), + }); + addonState.discoveredExternalCatalogs = Array.isArray(response.catalogs) + ? response.catalogs + : []; + addonState.discoveredExternalSourceUrl = response.source_url || response.manifest_url || sourceUrl; + addonState.discoveredExternalSourceKind = response.source_kind || "manifest"; + addonState.discoveredExternalSourceProvider = response.source_provider || ""; + addonState.discoveredExternalAddonName = response.addon_name || ""; + renderExternalCatalogDiscovery(); + if (nameInput && !nameInput.value.trim() && addonState.discoveredExternalCatalogs.length) { + nameInput.value = addonState.discoveredExternalCatalogs[0].name || ""; + } + setMessage( + "external-catalog-message", + addonState.discoveredExternalCatalogs.length + ? `Found ${addonState.discoveredExternalCatalogs.length} catalog(s).` + : "No supported catalogs found." + ); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogCreate(data, form) { + const catalogKey = data.get("catalog_key") || ""; + const selectedCatalog = getDiscoveredExternalCatalog(String(catalogKey)); + const name = (data.get("name") || "").trim(); + if (!addonState.discoveredExternalSourceUrl) { + setMessage("external-catalog-message", "Discover a source first.", true); + return; + } + if (!selectedCatalog) { + setMessage("external-catalog-message", "Choose a discovered catalog.", true); + return; + } + if (!name) { + setMessage("external-catalog-message", "Name is required.", true); + return; + } + try { + const response = await requestJSON("/api/stremio-addon/external-catalogs", { + method: "POST", + body: JSON.stringify({ + name, + manifest_url: + addonState.discoveredExternalSourceKind === "manifest" + ? addonState.discoveredExternalSourceUrl + : null, + source_url: addonState.discoveredExternalSourceUrl, + source_kind: addonState.discoveredExternalSourceKind, + source_provider: addonState.discoveredExternalSourceProvider || null, + addon_name: addonState.discoveredExternalAddonName || null, + source_catalog_id: selectedCatalog.id, + source_catalog_type: selectedCatalog.type, + }), + }); + addonState.externalCatalogs = [...addonState.externalCatalogs, response]; + renderExternalCatalogs(); + if (form) { + form.reset(); + } + addonState.discoveredExternalCatalogs = []; + addonState.discoveredExternalSourceUrl = ""; + addonState.discoveredExternalSourceKind = "manifest"; + addonState.discoveredExternalSourceProvider = ""; + addonState.discoveredExternalAddonName = ""; + renderExternalCatalogDiscovery(); + setMessage("external-catalog-message", "External catalog added."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogUpdate(catalogId, card) { + const nameInput = card.querySelector("[data-external-catalog-name]"); + const enabledInput = card.querySelector("[data-external-catalog-enabled]"); + const showWatchedInput = card.querySelector("[data-external-catalog-show-watched]"); + const showInHomeInput = card.querySelector("[data-external-catalog-show-in-home]"); + const orderBySelect = card.querySelector("[data-external-catalog-order-by]"); + const orderDirSelect = card.querySelector("[data-external-catalog-order-dir]"); + const pageSizeInput = card.querySelector("[data-external-catalog-page-size]"); + const payload = { + name: nameInput ? nameInput.value.trim() : "", + enabled: enabledInput ? enabledInput.checked : true, + filters: { + show_watched: showWatchedInput ? showWatchedInput.checked : false, + statuses: [], + }, + order_by: orderBySelect ? orderBySelect.value : "source", + order_dir: orderDirSelect ? orderDirSelect.value : "asc", + page_size: pageSizeInput ? Number(pageSizeInput.value || 30) : 30, + show_in_home: showInHomeInput ? showInHomeInput.checked : true, + }; + if (!payload.name) { + setMessage("external-catalog-message", "Name is required.", true); + return; + } + try { + const response = await requestJSON(`/api/stremio-addon/external-catalogs/${catalogId}`, { + method: "PATCH", + body: JSON.stringify(payload), + }); + addonState.externalCatalogs = addonState.externalCatalogs.map((entry) => + entry.id === catalogId ? response : entry + ); + renderExternalCatalogs(); + setMessage("external-catalog-message", "External catalog updated."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogDelete(catalogId) { + const confirmDelete = window.confirm( + "Delete this external catalog? Cached items will be removed from librarySync." + ); + if (!confirmDelete) { + return; + } + try { + await requestJSON(`/api/stremio-addon/external-catalogs/${catalogId}`, { + method: "DELETE", + }); + addonState.externalCatalogs = addonState.externalCatalogs.filter( + (entry) => entry.id !== catalogId + ); + renderExternalCatalogs(); + setMessage("external-catalog-message", "External catalog deleted."); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + +async function handleExternalCatalogRefresh(catalogId) { + try { + const response = await requestJSON( + `/api/stremio-addon/external-catalogs/${catalogId}/refresh`, + { method: "POST" } + ); + addonState.externalCatalogs = addonState.externalCatalogs.map((entry) => + entry.id === catalogId ? response.catalog : entry + ); + renderExternalCatalogs(); + setMessage( + "external-catalog-message", + `External catalog refreshed (${response.item_count || 0} items).` + ); + } catch (error) { + setMessage("external-catalog-message", error.message, true); + } +} + async function handleCustomCatalogCreate(data, form) { setMessage("custom-catalog-message", ""); const payload = { @@ -1009,11 +1430,16 @@ async function loadAddonConfig() { const data = await requestJSON("/api/stremio-addon/config"); addonState.config = data; addonState.catalogs = Array.isArray(data.catalogs) ? data.catalogs : []; + addonState.externalCatalogs = Array.isArray(data.external_catalogs) + ? data.external_catalogs + : []; addonState.customCatalogs = Array.isArray(data.custom_catalogs) ? data.custom_catalogs : []; updateInstallLinks(data); renderInstallSection(); renderControlSection(); renderBuiltInCatalogs(); + renderExternalCatalogDiscovery(); + renderExternalCatalogs(); renderCustomCatalogs(); } catch (error) { setAddonMessage(error.message, true); @@ -1075,6 +1501,34 @@ function bindAddonActions() { }); } + const externalDiscover = document.getElementById("external-catalog-discover"); + if (externalDiscover) { + externalDiscover.addEventListener("click", handleExternalCatalogDiscover); + } + + const externalList = document.getElementById("external-catalog-list"); + if (externalList) { + externalList.addEventListener("click", (event) => { + const update = event.target.closest("[data-external-catalog-save]"); + if (update) { + const card = update.closest("[data-external-catalog-id]"); + if (card) { + handleExternalCatalogUpdate(update.dataset.externalCatalogSave, card); + } + return; + } + const refresh = event.target.closest("[data-external-catalog-refresh]"); + if (refresh) { + handleExternalCatalogRefresh(refresh.dataset.externalCatalogRefresh); + return; + } + const remove = event.target.closest("[data-external-catalog-delete]"); + if (remove) { + handleExternalCatalogDelete(remove.dataset.externalCatalogDelete); + } + }); + } + const customList = document.getElementById("custom-catalog-list"); if (customList) { customList.addEventListener("click", (event) => { @@ -1149,6 +1603,7 @@ window.librarysyncPageInit = async ({ user }) => { if (!user) { return; } + bindForm("external-catalog-form", handleExternalCatalogCreate); bindForm("custom-catalog-form", handleCustomCatalogCreate); bindForm("custom-catalog-items-lookup-form", handleCustomLookupSubmit); bindAddonActions(); diff --git a/backend/src/librarysync/templates/stremio-addon.html b/backend/src/librarysync/templates/stremio-addon.html index b2ef12f..c1d0464 100644 --- a/backend/src/librarysync/templates/stremio-addon.html +++ b/backend/src/librarysync/templates/stremio-addon.html @@ -7,7 +7,7 @@

Publish your watchlists.

- Turn librarySync lists into Stremio catalogs with per-catalog filters and custom picks. + Turn librarySync lists and external addon feeds into Stremio catalogs with per-catalog filters.

@@ -72,6 +72,45 @@

Built-in catalogs

Loading...
+
+
+
+

External catalogs

+

Pull catalogs from Stremio manifests or supported list URLs and filter out watched items.

+
+
+
+ + + +
+ + + +
+
+
Loading...
+
+
diff --git a/backend/src/librarysync/worker.py b/backend/src/librarysync/worker.py index 09c1360..8a3e2b7 100644 --- a/backend/src/librarysync/worker.py +++ b/backend/src/librarysync/worker.py @@ -5,6 +5,7 @@ from typing import Awaitable, Callable from librarysync.config import settings +from librarysync.jobs.external_catalog_refresh import process_external_catalog_refresh_once from librarysync.jobs.imports import process_import_all_once, process_quick_import_once from librarysync.jobs.merge_history import ( process_merge_all_history_once, @@ -34,6 +35,9 @@ class ModeConfig: "metadata_cache": ModeConfig("metadata_cache", process_metadata_cache_refresh_once, 60.0, 5.0), "quick_import": ModeConfig("quick_import", process_quick_import_once, 2.0, 0.5), "import_all": ModeConfig("import_all", process_import_all_once, 2.0, 0.5), + "external_catalog_refresh": ModeConfig( + "external_catalog_refresh", process_external_catalog_refresh_once, 60.0, 10.0 + ), "watchlist": ModeConfig("watchlist", process_watchlist_refresh_once, 60.0, 10.0), "merge_history": ModeConfig("merge_history", process_merge_history_once, 60.0, 10.0), "merge_all_history": ModeConfig( diff --git a/backend/tests/test_stremio_addon_catalogs.py b/backend/tests/test_stremio_addon_catalogs.py index a8cf25a..4a5f487 100644 --- a/backend/tests/test_stremio_addon_catalogs.py +++ b/backend/tests/test_stremio_addon_catalogs.py @@ -2,14 +2,32 @@ import copy import unittest from types import SimpleNamespace +from unittest.mock import AsyncMock, patch from fastapi import HTTPException from librarysync.api import ( routes_stremio_addon, routes_stremio_addon_public, ) +from librarysync.core.external_catalog import ( + ExternalCatalogListItem, + ExternalCatalogProviderError, + _build_external_manifest_catalog_url, + _dedupe_external_list_items, + _dedupe_external_metas, + _discover_tmdb_chart_catalogs, + _prefetch_external_media_item_ids, + normalize_external_manifest_url, +) from librarysync.core.stremio_addon import build_default_catalogs -from librarysync.db.models import MediaItem +from librarysync.core.watchlist_links import ( + parse_imdb_chart_urls, + parse_mdblist_urls, + parse_tmdb_chart_urls, + parse_tmdb_list_urls, + parse_tvdb_list_urls, +) +from librarysync.db.models import MediaItem, StremioExternalCatalogItem class TestStremioAddonCatalogs(unittest.TestCase): @@ -30,21 +48,63 @@ def test_merge_catalog_updates_does_not_mutate_existing(self) -> None: def test_build_manifest_includes_custom_and_enabled_catalogs(self) -> None: catalogs = build_default_catalogs() + external_catalog = SimpleNamespace( + name="Best Movies", + slug="best-movies", + enabled=True, + source_catalog_type="movie", + page_size=30, + show_in_home=True, + ) custom_catalog = SimpleNamespace( name="Curated Picks", slug="curated_picks", media_type="movie" ) - manifest = routes_stremio_addon_public._build_manifest(catalogs, [custom_catalog]) + manifest = routes_stremio_addon_public._build_manifest( + catalogs, + [external_catalog], + [custom_catalog], + ) - catalog_ids = {catalog["id"] for catalog in manifest["catalogs"]} + catalog_ids = [catalog["id"] for catalog in manifest["catalogs"]] self.assertIn("watchlist_movies", catalog_ids) self.assertIn("watchlist_shows", catalog_ids) self.assertIn("in_progress_shows", catalog_ids) + self.assertIn("best-movies", catalog_ids) self.assertIn("curated_picks", catalog_ids) self.assertNotIn("watchlist_anime", catalog_ids) + self.assertLess(catalog_ids.index("best-movies"), catalog_ids.index("curated_picks")) for catalog in manifest["catalogs"]: self.assertIn("extraSupported", catalog) + def test_external_catalog_query_hides_watched_items_by_default(self) -> None: + catalog = SimpleNamespace(id="external-1", filters={"show_watched": False}) + query = asyncio.run( + routes_stremio_addon_public._build_external_catalog_query( + "user-id", + catalog, + None, + ) + ) + + compiled = str(query.compile(compile_kwargs={"literal_binds": True})).lower() + self.assertIn("stremio_external_catalog_items", compiled) + self.assertIn("exists (select watched_items.id", compiled) + + def test_external_catalog_query_can_include_watched_items(self) -> None: + catalog = SimpleNamespace(id="external-1", filters={"show_watched": True}) + query = asyncio.run( + routes_stremio_addon_public._build_external_catalog_query( + "user-id", + catalog, + None, + ) + ) + + compiled = str(query.compile(compile_kwargs={"literal_binds": True})).lower() + self.assertIn("stremio_external_catalog_items", compiled) + self.assertNotIn("not (exists", compiled) + def test_build_meta_prefers_stremio_id(self) -> None: media_item = MediaItem( title="Example Movie", @@ -96,6 +156,316 @@ def test_reorder_map_requires_all_items(self) -> None: with self.assertRaises(HTTPException): routes_stremio_addon._build_reorder_map(["a", "b"], ["a"]) + def test_build_external_item_meta_uses_cached_fields_when_media_missing(self) -> None: + item = StremioExternalCatalogItem( + stremio_id="tt1234567", + stremio_type="movie", + title="Cached Movie", + year=1999, + poster_url="https://image.example/poster.jpg", + ) + meta = routes_stremio_addon_public._build_external_item_meta(item, None, "movie") + + self.assertEqual( + meta, + { + "id": "tt1234567", + "type": "movie", + "name": "Cached Movie", + "year": 1999, + "poster": "https://image.example/poster.jpg", + }, + ) + + def test_parse_tmdb_list_url(self) -> None: + refs = parse_tmdb_list_urls(["https://www.themoviedb.org/list/12345-best-movies"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].list_id, "12345") + self.assertEqual(refs[0].external_id, "tmdb:12345") + + def test_parse_tvdb_list_url(self) -> None: + refs = parse_tvdb_list_urls(["https://thetvdb.com/lists/top-shows"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].list_id, "top-shows") + self.assertEqual(refs[0].external_id, "tvdb:top-shows") + + def test_parse_tmdb_chart_url(self) -> None: + refs = parse_tmdb_chart_urls(["https://www.themoviedb.org/movie/top-rated"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].media_type, "movie") + self.assertEqual(refs[0].chart_slug, "top-rated") + self.assertEqual(refs[0].external_id, "tmdb-chart:movie:top-rated") + + def test_discover_tmdb_chart_catalog(self) -> None: + ref = parse_tmdb_chart_urls(["https://www.themoviedb.org/movie/top-rated"])[0] + + payload = asyncio.run(_discover_tmdb_chart_catalogs(ref)) + + self.assertEqual(payload["source_provider"], "tmdb") + self.assertEqual(payload["catalogs"][0]["id"], "tmdb-chart:movie:top-rated") + self.assertEqual(payload["catalogs"][0]["type"], "movie") + + def test_parse_imdb_chart_url_with_locale(self) -> None: + refs = parse_imdb_chart_urls(["https://www.imdb.com/de/chart/top/"]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].chart_slug, "top") + + def test_discover_payload_accepts_legacy_manifest_url_field(self) -> None: + payload = routes_stremio_addon.StremioExternalCatalogDiscoverPayload( + manifest_url="https://www.themoviedb.org/movie/top-rated" + ) + + self.assertEqual(payload.manifest_url, "https://www.themoviedb.org/movie/top-rated") + self.assertIsNone(payload.source_url) + + def test_parse_mdblist_url(self) -> None: + refs = parse_mdblist_urls([ + "https://mdblist.com/lists/cb2131/emby-imdb-top-rated-movies" + ]) + + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0].username, "cb2131") + self.assertEqual(refs[0].slug, "emby-imdb-top-rated-movies") + self.assertEqual(refs[0].external_id, "mdblist:cb2131:emby-imdb-top-rated-movies") + + def test_dedupe_external_metas_keeps_first_stremio_id(self) -> None: + metas = [ + {"id": "tt1", "name": "First"}, + {"id": "tt1", "name": "Duplicate"}, + {"id": "tt2", "name": "Second"}, + ] + + deduped = _dedupe_external_metas(metas) + + self.assertEqual(deduped, [{"id": "tt1", "name": "First"}, {"id": "tt2", "name": "Second"}]) + + def test_dedupe_external_list_items_keeps_first_stremio_id(self) -> None: + items = [ + ExternalCatalogListItem("tt1", "movie", "First", 2000, None), + ExternalCatalogListItem("tt1", "movie", "Duplicate", 2001, None), + ExternalCatalogListItem("tt2", "movie", "Second", 2002, None), + ] + + deduped = _dedupe_external_list_items(items) + + self.assertEqual([item.title for item in deduped], ["First", "Second"]) + + def test_normalize_external_manifest_url_preserves_query(self) -> None: + self.assertEqual( + asyncio.run(normalize_external_manifest_url("https://addon.example/path?foo=bar")), + "https://addon.example/path/manifest.json?foo=bar", + ) + + def test_normalize_external_manifest_url_rejects_disallowed_host(self) -> None: + with self.assertRaises(ValueError): + asyncio.run(normalize_external_manifest_url("http://127.0.0.1/manifest.json")) + + def test_build_external_manifest_catalog_url_preserves_query(self) -> None: + self.assertEqual( + _build_external_manifest_catalog_url( + "https://addon.example/path/manifest.json?foo=bar", + "movie", + "top rated", + skip=100, + limit=50, + ), + "https://addon.example/path/catalog/movie/top%20rated/skip=100&limit=50.json?foo=bar", + ) + + def test_prefetch_external_media_item_ids_uses_raw_stremio_id_and_series_type(self) -> None: + class _FakeScalarResult: + def __init__(self, values): + self._values = values + + def all(self): + return self._values + + class _FakeExecuteResult: + def __init__(self, values): + self._values = values + + def scalars(self): + return _FakeScalarResult(self._values) + + class _FakeDb: + async def execute(self, query): + return _FakeExecuteResult( + [ + MediaItem( + id="media-1", + media_type="tv", + title="Example Show", + imdb_id="tt1234567", + raw={"stremio_id": "stremio:series:1"}, + ) + ] + ) + + lookup = asyncio.run( + _prefetch_external_media_item_ids( + _FakeDb(), + [{"id": "stremio:series:1", "type": "series", "imdb_id": "tt1234567"}], + "series", + ) + ) + + self.assertEqual(lookup.by_stremio_id[("series", "stremio:series:1")], "media-1") + self.assertEqual(lookup.by_imdb_id[("series", "tt1234567")], "media-1") + + def test_refresh_external_catalog_or_502_maps_provider_errors(self) -> None: + class _FakeDb: + def __init__(self): + self.rollback_calls = 0 + self.commit_calls = 0 + + async def rollback(self): + self.rollback_calls += 1 + + async def commit(self): + self.commit_calls += 1 + + async def refresh(self, catalog): + raise AssertionError("refresh should not be called") + + db = _FakeDb() + catalog = SimpleNamespace() + + with ( + patch.object( + routes_stremio_addon, + "refresh_external_catalog", + AsyncMock(side_effect=ExternalCatalogProviderError("bad payload")), + ), + patch.object( + routes_stremio_addon, + "mark_external_catalog_refresh_failed", + AsyncMock(), + ) as mark_failed, + ): + with self.assertRaises(HTTPException) as ctx: + asyncio.run(routes_stremio_addon._refresh_external_catalog_or_502(db, catalog)) + + self.assertEqual(ctx.exception.status_code, 502) + self.assertEqual(ctx.exception.detail, "External catalog refresh failed") + self.assertEqual(db.rollback_calls, 1) + self.assertEqual(db.commit_calls, 1) + mark_failed.assert_awaited_once() + + def test_update_external_catalog_rejects_null_enabled(self) -> None: + class _FakeDb: + async def commit(self): + raise AssertionError("commit should not be called") + + async def refresh(self, catalog): + raise AssertionError("refresh should not be called") + + def add(self, catalog): + raise AssertionError("add should not be called") + + payload = routes_stremio_addon.StremioExternalCatalogUpdate(enabled=None) + catalog = SimpleNamespace(id="catalog-1", name="Catalog") + user = SimpleNamespace(id="user-1") + + with patch.object( + routes_stremio_addon, + "_load_external_catalog", + AsyncMock(return_value=catalog), + ): + with self.assertRaises(HTTPException) as ctx: + asyncio.run( + routes_stremio_addon.update_external_catalog( + "catalog-1", + payload, + user, + _FakeDb(), + ) + ) + + self.assertEqual(ctx.exception.status_code, 400) + self.assertEqual(ctx.exception.detail, "Enabled must be true or false") + + def test_serve_catalog_supports_external_catalog_without_builtin_catalog(self) -> None: + class _FakeScalarResult: + def __init__(self, value): + self._value = value + + def first(self): + return self._value + + class _FakeExecuteResult: + def __init__(self, *, scalar=None, rows=None): + self._scalar = scalar + self._rows = rows or [] + + def scalars(self): + return _FakeScalarResult(self._scalar) + + def all(self): + return self._rows + + class _FakeQuery: + def order_by(self, *args): + return self + + def offset(self, value): + return self + + def limit(self, value): + return self + + class _FakeDb: + def __init__(self): + self._results = [ + _FakeExecuteResult( + scalar=SimpleNamespace( + enabled=True, + source_catalog_type="movie", + order_by="source", + order_dir="asc", + ) + ), + _FakeExecuteResult(rows=[]), + ] + + async def execute(self, query): + return self._results.pop(0) + + request = SimpleNamespace(query_params={}) + config = SimpleNamespace( + is_enabled=True, + user_id="user-1", + default_catalogs=build_default_catalogs(), + ) + + with ( + patch.object( + routes_stremio_addon_public, + "get_addon_config_by_id", + AsyncMock(return_value=config), + ), + patch.object( + routes_stremio_addon_public, + "_build_external_catalog_query", + AsyncMock(return_value=_FakeQuery()), + ), + ): + payload = asyncio.run( + routes_stremio_addon_public._serve_catalog( + "addon-1", + "movie", + "top-rated-movies", + request, + _FakeDb(), + None, + ) + ) + + self.assertEqual(payload, {"metas": []}) + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 0e7cb5f..5e29f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ members = ["backend"] [tool.ruff] -line-length = 100 +line-length = 120 extend-exclude = ["**/db/migrations/**"] [tool.ruff.lint]