From 5ee75a6bb7eb59dec262adc0676fbebb73a14426 Mon Sep 17 00:00:00 2001 From: steadyfetch Date: Fri, 11 Sep 2026 20:27:50 +0300 Subject: [PATCH] feat: add a transcription tool family (Facebook Ads, YouTube, media) The package covers what a page or profile shows; it has no tool for what is said inside a video or audio file. An agent asked for the script of an ad, the words in a YouTube video or the text of a podcast episode has to fall back to ApifyActorsTool and know an Actor ID. Three classes, shaped on the existing third-party wrappers: - ApifyFacebookAdsTranscriptTool -> steadyfetch/facebook-ads-transcript-scraper - ApifyYouTubeTranscriptTool -> steadyfetch/youtube-transcript-scraper - ApifyMediaTranscriberTool -> steadyfetch/media-transcriber They go through ApifyToolsClient, return the standard {"run", "items"} envelope, clamp max_results through the base class, and are exported as APIFY_TRANSCRIPT_TOOLS. The three Actors are pay-per-event and published by steadyfetch; no price is named in the code, since a released package cannot be edited when one changes. --- README.md | 24 +- langchain_apify/__init__.py | 9 + langchain_apify/_client.py | 117 ++++++++ langchain_apify/_constants.py | 4 + langchain_apify/_error_messages.py | 2 + langchain_apify/tools/__init__.py | 16 + langchain_apify/tools/transcripts.py | 279 ++++++++++++++++++ tests/unit_tests/test_merge_surface.py | 13 +- tests/unit_tests/test_transcript_tools.py | 344 ++++++++++++++++++++++ 9 files changed, 803 insertions(+), 5 deletions(-) create mode 100644 langchain_apify/tools/transcripts.py create mode 100644 tests/unit_tests/test_transcript_tools.py diff --git a/README.md b/README.md index 09b108b..d7e86ec 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Get your API token from [Apify Console](https://console.apify.com/settings/integ ## Tools -The package ships dedicated tools across three families plus a generic "wrap any Actor by ID" tool for everything else. All return a uniform `{"run": {...}, "items": [...]}` JSON envelope (parse with `json.loads`). +The package ships dedicated tools across four families plus a generic "wrap any Actor by ID" tool for everything else. All return a uniform `{"run": {...}, "items": [...]}` JSON envelope (parse with `json.loads`). ### Core tools @@ -122,6 +122,28 @@ result = ApifyInstagramScraperTool().invoke({ print(json.loads(result)) ``` +### Transcription tools + +Spoken words out of video and audio, rather than what a page shows. Available as `APIFY_TRANSCRIPT_TOOLS`: + +- `ApifyFacebookAdsTranscriptTool`: transcripts, hooks and CTAs of the Facebook Ad Library ads running now +- `ApifyYouTubeTranscriptTool`: transcripts of specific YouTube videos +- `ApifyMediaTranscriberTool`: transcripts of audio/video file links and supported podcast or video pages + +These three wrap pay-per-event Actors published by `steadyfetch`; a run is charged to your Apify account at the price on each Actor's store page. + +```python +import os, json +from langchain_apify import ApifyYouTubeTranscriptTool + +os.environ["APIFY_TOKEN"] = "YOUR_APIFY_TOKEN" + +result = ApifyYouTubeTranscriptTool().invoke({ + "video_urls": ["https://www.youtube.com/watch?v=jNQXAC9IVRw"], +}) +print(json.loads(result)) +``` + ### Using tools with an agent Each convenience list lets you bind a whole tool family to an agent in one line. Don't bind all tools at once. Most LLMs lose routing accuracy past ~8 tools, so pick the family the agent actually needs. diff --git a/langchain_apify/__init__.py b/langchain_apify/__init__.py index 00c0a13..031bddf 100644 --- a/langchain_apify/__init__.py +++ b/langchain_apify/__init__.py @@ -9,8 +9,10 @@ APIFY_CORE_TOOLS, APIFY_SEARCH_TOOLS, APIFY_SOCIAL_TOOLS, + APIFY_TRANSCRIPT_TOOLS, ApifyActorsTool, ApifyEcommerceScraperTool, + ApifyFacebookAdsTranscriptTool, ApifyFacebookPostsScraperTool, ApifyGetDatasetItemsTool, ApifyGoogleMapsTool, @@ -19,6 +21,7 @@ ApifyLinkedInProfileDetailTool, ApifyLinkedInProfilePostsTool, ApifyLinkedInProfileSearchTool, + ApifyMediaTranscriberTool, ApifyRAGWebBrowserTool, ApifyRunActorAndGetDatasetTool, ApifyRunActorTool, @@ -29,6 +32,7 @@ ApifyTwitterScraperTool, ApifyWebCrawlerTool, ApifyYouTubeScraperTool, + ApifyYouTubeTranscriptTool, ) from langchain_apify.wrappers import ApifyWrapper @@ -70,10 +74,15 @@ 'ApifyGoogleMapsTool', 'ApifyYouTubeScraperTool', 'ApifyEcommerceScraperTool', + # Transcription Actor tools + 'ApifyFacebookAdsTranscriptTool', + 'ApifyYouTubeTranscriptTool', + 'ApifyMediaTranscriberTool', # Tool group lists 'APIFY_CORE_TOOLS', 'APIFY_SOCIAL_TOOLS', 'APIFY_SEARCH_TOOLS', + 'APIFY_TRANSCRIPT_TOOLS', # Meta '__version__', ] diff --git a/langchain_apify/_client.py b/langchain_apify/_client.py index 49b864c..2bf3a5b 100644 --- a/langchain_apify/_client.py +++ b/langchain_apify/_client.py @@ -19,11 +19,14 @@ _DEFAULT_SCRAPE_TIMEOUT_SECS, _DEFAULT_SOCIAL_RESULTS_LIMIT, _DEFAULT_SOCIAL_TIMEOUT_SECS, + _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + _DEFAULT_TRANSCRIPT_TIMEOUT_SECS, _DEFAULT_YOUTUBE_MAX_RESULTS, ) from langchain_apify._error_messages import ( _ERROR_ACTOR_RUN_FAILED, _ERROR_APIFY_TOKEN_ENV_VAR_NOT_SET, + _ERROR_EMPTY_INPUT_LIST, _ERROR_SCRAPE_EMPTY, ) from langchain_apify._types import CrawlerType # noqa: TCH001 # runtime-needed: pydantic-free annotation @@ -57,6 +60,11 @@ _TIKTOK_ACTOR_ID = 'clockworks/tiktok-scraper' _FACEBOOK_ACTOR_ID = 'apify/facebook-posts-scraper' +# Actor IDs - transcripts. +_FACEBOOK_ADS_TRANSCRIPT_ACTOR_ID = 'steadyfetch/facebook-ads-transcript-scraper' +_YOUTUBE_TRANSCRIPT_ACTOR_ID = 'steadyfetch/youtube-transcript-scraper' +_MEDIA_TRANSCRIBER_ACTOR_ID = 'steadyfetch/media-transcriber' + # Accepted parameter values validated client-side before a run. _YOUTUBE_SEARCH_TYPES = ('search', 'video', 'channel') _ECOMMERCE_URL_TYPES = ('product', 'category') @@ -850,6 +858,115 @@ def crawl_website( dataset_items_limit=max_crawl_pages, ) + def facebook_ads_transcript_scrape( + self, + search_queries: list[str], + country: str = 'US', + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + timeout_secs: int = _DEFAULT_TRANSCRIPT_TIMEOUT_SECS, + ) -> tuple[dict, list[dict]]: + """Transcribe Facebook Ad Library ads via ``steadyfetch/facebook-ads-transcript-scraper``. + + Args: + search_queries: Keywords or advertiser page names to search the Ad Library for. + country: Two-letter country code the ads are served in. + max_results: Maximum number of ad creatives to return. + timeout_secs: Maximum time to wait for the run to finish. + + Returns: + A ``(run_details, items)`` tuple. + + Raises: + ValueError: If ``search_queries`` is empty. + RuntimeError: If the Actor run does not succeed. + """ + self._require_non_empty(search_queries, 'search_queries') + run_input: dict = { + 'searchQueries': list(search_queries), + 'country': country, + 'searchMaxAds': max_results, + 'maxAds': max_results, + } + return self.run_actor_and_get_items( + _FACEBOOK_ADS_TRANSCRIPT_ACTOR_ID, + run_input=run_input, + timeout_secs=timeout_secs, + dataset_items_limit=max_results, + ) + + def youtube_transcript_scrape( + self, + video_urls: list[str], + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + timeout_secs: int = _DEFAULT_TRANSCRIPT_TIMEOUT_SECS, + ) -> tuple[dict, list[dict]]: + """Transcribe YouTube videos via ``steadyfetch/youtube-transcript-scraper``. + + Args: + video_urls: Watch, youtu.be, ``/shorts/`` or ``/live/`` links, or bare video IDs. + max_results: Maximum number of transcripts to return. + timeout_secs: Maximum time to wait for the run to finish. + + Returns: + A ``(run_details, items)`` tuple. + + Raises: + ValueError: If ``video_urls`` is empty. + RuntimeError: If the Actor run does not succeed. + """ + self._require_non_empty(video_urls, 'video_urls') + run_input: dict = {'videoUrls': list(video_urls), 'maxItems': max_results} + return self.run_actor_and_get_items( + _YOUTUBE_TRANSCRIPT_ACTOR_ID, + run_input=run_input, + timeout_secs=timeout_secs, + dataset_items_limit=max_results, + ) + + def media_transcribe( + self, + urls: list[str], + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + timeout_secs: int = _DEFAULT_TRANSCRIPT_TIMEOUT_SECS, + ) -> tuple[dict, list[dict]]: + """Transcribe audio or video files via ``steadyfetch/media-transcriber``. + + Args: + urls: Direct audio/video file links, or page links on a supported host. + max_results: Maximum number of transcripts to return. + timeout_secs: Maximum time to wait for the run to finish. + + Returns: + A ``(run_details, items)`` tuple. + + Raises: + ValueError: If ``urls`` is empty. + RuntimeError: If the Actor run does not succeed. + """ + self._require_non_empty(urls, 'urls') + run_input: dict = {'urls': list(urls)} + return self.run_actor_and_get_items( + _MEDIA_TRANSCRIBER_ACTOR_ID, + run_input=run_input, + timeout_secs=timeout_secs, + dataset_items_limit=max_results, + ) + + @staticmethod + def _require_non_empty(values: list[str], field: str) -> None: + """Raise if a required list input is empty. + + Args: + values: The list supplied by the caller. + field: Name of the field, used in the error message. + + Raises: + ValueError: If ``values`` is empty. + """ + if not values: + msg = _ERROR_EMPTY_INPUT_LIST.format(field=field) + raise ValueError(msg) + @staticmethod def _check_run_status(run: dict) -> None: """Raise if the run did not succeed.""" diff --git a/langchain_apify/_constants.py b/langchain_apify/_constants.py index 9615854..7bb68a8 100644 --- a/langchain_apify/_constants.py +++ b/langchain_apify/_constants.py @@ -15,6 +15,7 @@ _DEFAULT_RUN_TIMEOUT_SECS = 300 _DEFAULT_SCRAPE_TIMEOUT_SECS = 120 _DEFAULT_SOCIAL_TIMEOUT_SECS = 600 +_DEFAULT_TRANSCRIPT_TIMEOUT_SECS = 600 # Default result / page limits, by operation. _DEFAULT_DATASET_ITEMS_LIMIT = 100 @@ -27,6 +28,9 @@ _DEFAULT_GOOGLE_MAPS_MAX_RESULTS = 10 _DEFAULT_YOUTUBE_MAX_RESULTS = 10 _DEFAULT_ECOMMERCE_MAX_RESULTS = 20 +# Transcription Actors charge per delivered item, so the family defaults to a +# smaller batch than the scraping families. +_DEFAULT_TRANSCRIPT_RESULTS_LIMIT = 10 # Upper-bound clamp ceilings applied by _ApifyGenericTool to LLM-supplied # values. The Pydantic Field defaults on the base class reference these so diff --git a/langchain_apify/_error_messages.py b/langchain_apify/_error_messages.py index 95c1aa0..f977554 100644 --- a/langchain_apify/_error_messages.py +++ b/langchain_apify/_error_messages.py @@ -11,6 +11,8 @@ _ERROR_SCRAPE_EMPTY = 'No content extracted from {url}.' +_ERROR_EMPTY_INPUT_LIST = '{field} must contain at least one value.' + _NOTICE_TWITTER_DEMO = ( 'The Twitter/X Actor returned demo placeholder data instead of real tweets.' ' This happens on the Apify free plan; a paid Apify plan is required to scrape real tweets.' diff --git a/langchain_apify/tools/__init__.py b/langchain_apify/tools/__init__.py index c0ff4c6..451dd25 100644 --- a/langchain_apify/tools/__init__.py +++ b/langchain_apify/tools/__init__.py @@ -58,14 +58,26 @@ TwitterSearchMode, TwitterSort, ) +from langchain_apify.tools.transcripts import ( + APIFY_TRANSCRIPT_TOOLS, + ApifyFacebookAdsTranscriptInput, + ApifyFacebookAdsTranscriptTool, + ApifyMediaTranscriberInput, + ApifyMediaTranscriberTool, + ApifyYouTubeTranscriptInput, + ApifyYouTubeTranscriptTool, +) __all__ = [ 'APIFY_CORE_TOOLS', 'APIFY_SEARCH_TOOLS', 'APIFY_SOCIAL_TOOLS', + 'APIFY_TRANSCRIPT_TOOLS', 'ApifyActorsTool', 'ApifyEcommerceScraperInput', 'ApifyEcommerceScraperTool', + 'ApifyFacebookAdsTranscriptInput', + 'ApifyFacebookAdsTranscriptTool', 'ApifyFacebookPostsScraperInput', 'ApifyFacebookPostsScraperTool', 'ApifyGetDatasetItemsInput', @@ -82,6 +94,8 @@ 'ApifyLinkedInProfilePostsTool', 'ApifyLinkedInProfileSearchInput', 'ApifyLinkedInProfileSearchTool', + 'ApifyMediaTranscriberInput', + 'ApifyMediaTranscriberTool', 'ApifyRAGWebBrowserInput', 'ApifyRAGWebBrowserTool', 'ApifyRunActorAndGetDatasetInput', @@ -102,6 +116,8 @@ 'ApifyWebCrawlerTool', 'ApifyYouTubeScraperInput', 'ApifyYouTubeScraperTool', + 'ApifyYouTubeTranscriptInput', + 'ApifyYouTubeTranscriptTool', 'InstagramSearchType', 'TikTokSearchType', 'TwitterSearchMode', diff --git a/langchain_apify/tools/transcripts.py b/langchain_apify/tools/transcripts.py new file mode 100644 index 0000000..3a5f7c8 --- /dev/null +++ b/langchain_apify/tools/transcripts.py @@ -0,0 +1,279 @@ +"""Transcription Actor tools and their input schemas. + +Each tool wraps a single Apify transcription Actor behind a simplified, +LLM-friendly interface returning the standard JSON envelope. The other tool +families return what a page or profile *shows*; these return the spoken words +inside a video or audio file. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from langchain_core.tools import ArgsSchema, ToolException +from pydantic import BaseModel, Field + +from langchain_apify._constants import ( + _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + _MAX_ITEMS_CAP, +) +from langchain_apify.tools.base import _TOOL_RUN_ERRORS, _ApifyGenericTool + +if TYPE_CHECKING: + from langchain_core.callbacks import CallbackManagerForToolRun + from langchain_core.tools import BaseTool + + +# --------------------------------------------------------------------------- +# Input schemas +# --------------------------------------------------------------------------- + + +class ApifyFacebookAdsTranscriptInput(BaseModel): + """Input schema for :class:`ApifyFacebookAdsTranscriptTool`.""" + + search_queries: list[str] = Field( + description=( + 'Keywords or advertiser page names to search the Facebook Ad Library ' + 'for (e.g. ["fitness app"]). At least one value is required.' + ), + ) + country: str = Field( + default='US', + description='Two-letter country code the ads are served in (e.g. "US", "GB", "DE").', + ) + max_results: int = Field( + default=_DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + description=f'Maximum number of ad creatives to return (clamped to {_MAX_ITEMS_CAP} max).', + ) + + +class ApifyYouTubeTranscriptInput(BaseModel): + """Input schema for :class:`ApifyYouTubeTranscriptTool`.""" + + video_urls: list[str] = Field( + description=( + 'YouTube watch, youtu.be, /shorts/ or /live/ links, or bare 11-character ' + 'video IDs (e.g. ["https://www.youtube.com/watch?v=jNQXAC9IVRw"]). ' + 'At least one value is required.' + ), + ) + max_results: int = Field( + default=_DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + description=f'Maximum number of transcripts to return (clamped to {_MAX_ITEMS_CAP} max).', + ) + + +class ApifyMediaTranscriberInput(BaseModel): + """Input schema for :class:`ApifyMediaTranscriberTool`.""" + + urls: list[str] = Field( + description=( + 'Direct audio or video file links on any host, or page links on a supported ' + 'podcast or video host (e.g. ["https://example.com/episode.mp3"]). ' + 'At least one value is required.' + ), + ) + max_results: int = Field( + default=_DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + description=f'Maximum number of transcripts to return (clamped to {_MAX_ITEMS_CAP} max).', + ) + + +# --------------------------------------------------------------------------- +# Tools +# --------------------------------------------------------------------------- + + +class ApifyFacebookAdsTranscriptTool(_ApifyGenericTool): # type: ignore[override] + """Transcribe the Facebook Ad Library ads running now for a keyword or advertiser. + + Uses the ``steadyfetch/facebook-ads-transcript-scraper`` Actor under the + hood: each matching ad creative comes back with its video transcript, the + opening hook, the call to action and the ad's metadata (on-image text for + image ads). + + Args: + apify_token: Apify API token. Falls back to the ``APIFY_TOKEN`` + environment variable when *None*. + + Returns: + JSON string with two keys: ``run`` (dict with ``run_id``, ``status``, + ``dataset_id``, ``started_at``, ``finished_at``) and ``items`` (list + of ad dicts). + + Example: + .. code-block:: python + + import os + os.environ["APIFY_TOKEN"] = "your-apify-token" + + from langchain_apify import ApifyFacebookAdsTranscriptTool + + tool = ApifyFacebookAdsTranscriptTool() + result = tool.invoke({ + "search_queries": ["fitness app"], + "country": "US", + "max_results": 10, + }) + """ + + name: str = 'apify_facebook_ads_transcript' + description: str = ( + 'Transcribe the Facebook Ad Library ads running now for a keyword or advertiser page name' + ' and return the transcript, opening hook, call to action and ad metadata as JSON.' + ' Required: search_queries (list of str - keywords or advertiser page names, e.g. ["fitness app"]).' + ' Optional: country (str - two-letter code, default "US"),' + f' max_results (int, default {_DEFAULT_TRANSCRIPT_RESULTS_LIMIT}).' + ' Returns JSON with keys: run (run_id, status, dataset_id, started_at, finished_at) and items.' + ' Use only the data returned; do not hallucinate missing fields.' + ) + args_schema: ArgsSchema | None = ApifyFacebookAdsTranscriptInput + + def _run( + self, + search_queries: list[str], + country: str = 'US', + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + _run_manager: CallbackManagerForToolRun | None = None, + ) -> str: + try: + run, items = self._client.facebook_ads_transcript_scrape( + search_queries=search_queries, + country=country, + max_results=self._clamp_items(max_results), + timeout_secs=self.max_timeout_secs, + ) + except _TOOL_RUN_ERRORS as exc: + raise ToolException(str(exc)) from exc + return self._envelope(run, items) + + +class ApifyYouTubeTranscriptTool(_ApifyGenericTool): # type: ignore[override] + """Get the transcript of specific YouTube videos. + + Uses the ``steadyfetch/youtube-transcript-scraper`` Actor under the hood: + published captions are used where they exist, and speech-to-text fills in + the videos that have none. + + Args: + apify_token: Apify API token. Falls back to the ``APIFY_TOKEN`` + environment variable when *None*. + + Returns: + JSON string with two keys: ``run`` (dict with ``run_id``, ``status``, + ``dataset_id``, ``started_at``, ``finished_at``) and ``items`` (list + of transcript dicts). + + Example: + .. code-block:: python + + import os + os.environ["APIFY_TOKEN"] = "your-apify-token" + + from langchain_apify import ApifyYouTubeTranscriptTool + + tool = ApifyYouTubeTranscriptTool() + result = tool.invoke({ + "video_urls": ["https://www.youtube.com/watch?v=jNQXAC9IVRw"], + }) + """ + + name: str = 'apify_youtube_transcript' + description: str = ( + 'Get the transcript of specific YouTube videos and return them as JSON.' + ' Published captions are used where they exist; videos without captions are transcribed by speech-to-text.' + ' Required: video_urls (list of str - watch, youtu.be, /shorts/ or /live/ links,' + ' or bare 11-character video IDs).' + f' Optional: max_results (int, default {_DEFAULT_TRANSCRIPT_RESULTS_LIMIT}).' + ' Returns JSON with keys: run (run_id, status, dataset_id, started_at, finished_at) and items.' + ' Use only the data returned; do not hallucinate missing fields.' + ) + args_schema: ArgsSchema | None = ApifyYouTubeTranscriptInput + + def _run( + self, + video_urls: list[str], + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + _run_manager: CallbackManagerForToolRun | None = None, + ) -> str: + try: + run, items = self._client.youtube_transcript_scrape( + video_urls=video_urls, + max_results=self._clamp_items(max_results), + timeout_secs=self.max_timeout_secs, + ) + except _TOOL_RUN_ERRORS as exc: + raise ToolException(str(exc)) from exc + return self._envelope(run, items) + + +class ApifyMediaTranscriberTool(_ApifyGenericTool): # type: ignore[override] + """Transcribe audio or video from a file link or a supported media page. + + Uses the ``steadyfetch/media-transcriber`` Actor under the hood: a direct + audio or video file link on any host, or a page link on a supported + podcast or video host (Libsyn, Megaphone, Buzzsprout, Acast, Apple + Podcasts, Spotify for Creators, Archive.org, SoundCloud, Loom, Twitch + VODs, Wistia, Facebook and TikTok video links). + + Args: + apify_token: Apify API token. Falls back to the ``APIFY_TOKEN`` + environment variable when *None*. + + Returns: + JSON string with two keys: ``run`` (dict with ``run_id``, ``status``, + ``dataset_id``, ``started_at``, ``finished_at``) and ``items`` (list + of transcript dicts). + + Example: + .. code-block:: python + + import os + os.environ["APIFY_TOKEN"] = "your-apify-token" + + from langchain_apify import ApifyMediaTranscriberTool + + tool = ApifyMediaTranscriberTool() + result = tool.invoke({ + "urls": ["https://example.com/episode.mp3"], + }) + """ + + name: str = 'apify_media_transcriber' + description: str = ( + 'Transcribe audio or video and return the text as JSON.' + ' Accepts a direct audio or video file link on any host, or a page link on a supported podcast' + ' or video host (Libsyn, Megaphone, Buzzsprout, Acast, Apple Podcasts, Spotify for Creators,' + ' Archive.org, SoundCloud, Loom, Twitch VODs, Wistia, Facebook and TikTok video links).' + ' Required: urls (list of str).' + f' Optional: max_results (int, default {_DEFAULT_TRANSCRIPT_RESULTS_LIMIT}).' + ' Returns JSON with keys: run (run_id, status, dataset_id, started_at, finished_at) and items.' + ' For YouTube links use apify_youtube_transcript instead.' + ' Use only the data returned; do not hallucinate missing fields.' + ) + args_schema: ArgsSchema | None = ApifyMediaTranscriberInput + + def _run( + self, + urls: list[str], + max_results: int = _DEFAULT_TRANSCRIPT_RESULTS_LIMIT, + _run_manager: CallbackManagerForToolRun | None = None, + ) -> str: + try: + run, items = self._client.media_transcribe( + urls=urls, + max_results=self._clamp_items(max_results), + timeout_secs=self.max_timeout_secs, + ) + except _TOOL_RUN_ERRORS as exc: + raise ToolException(str(exc)) from exc + return self._envelope(run, items) + + +# Convenience tool-class list for selective agent binding. +APIFY_TRANSCRIPT_TOOLS: list[type[BaseTool]] = [ + ApifyFacebookAdsTranscriptTool, + ApifyYouTubeTranscriptTool, + ApifyMediaTranscriberTool, +] diff --git a/tests/unit_tests/test_merge_surface.py b/tests/unit_tests/test_merge_surface.py index c9c6b52..b4e72b5 100644 --- a/tests/unit_tests/test_merge_surface.py +++ b/tests/unit_tests/test_merge_surface.py @@ -1,9 +1,9 @@ """Package-surface invariants for the merged tool families. Ported from the offline ``test_merge_surface`` playground check: the core, -social, and search tool groups must keep their expected sizes, stay disjoint, -expose globally-unique tool ``name`` values (a name collision would surface -here), and keep ``_run_meta`` in a single canonical home. +social, search, and transcript tool groups must keep their expected sizes, +stay disjoint, expose globally-unique tool ``name`` values (a name collision +would surface here), and keep ``_run_meta`` in a single canonical home. """ from __future__ import annotations @@ -14,6 +14,7 @@ APIFY_CORE_TOOLS, APIFY_SEARCH_TOOLS, APIFY_SOCIAL_TOOLS, + APIFY_TRANSCRIPT_TOOLS, ) @@ -23,6 +24,7 @@ (APIFY_CORE_TOOLS, 6), (APIFY_SOCIAL_TOOLS, 7), (APIFY_SEARCH_TOOLS, 6), + (APIFY_TRANSCRIPT_TOOLS, 3), ], ) def test_tool_group_sizes(group: list[type], expected: int) -> None: @@ -33,10 +35,13 @@ def test_tool_groups_are_disjoint() -> None: assert not set(APIFY_SOCIAL_TOOLS) & set(APIFY_SEARCH_TOOLS) assert not set(APIFY_CORE_TOOLS) & set(APIFY_SOCIAL_TOOLS) assert not set(APIFY_CORE_TOOLS) & set(APIFY_SEARCH_TOOLS) + assert not set(APIFY_TRANSCRIPT_TOOLS) & set(APIFY_CORE_TOOLS) + assert not set(APIFY_TRANSCRIPT_TOOLS) & set(APIFY_SOCIAL_TOOLS) + assert not set(APIFY_TRANSCRIPT_TOOLS) & set(APIFY_SEARCH_TOOLS) def test_tool_names_are_globally_unique() -> None: - all_tools = [*APIFY_CORE_TOOLS, *APIFY_SOCIAL_TOOLS, *APIFY_SEARCH_TOOLS] + all_tools = [*APIFY_CORE_TOOLS, *APIFY_SOCIAL_TOOLS, *APIFY_SEARCH_TOOLS, *APIFY_TRANSCRIPT_TOOLS] names = [tool_cls.model_fields['name'].default for tool_cls in all_tools] dupes = sorted({n for n in names if names.count(n) > 1}) assert not dupes, f'duplicate tool names: {dupes}' diff --git a/tests/unit_tests/test_transcript_tools.py b/tests/unit_tests/test_transcript_tools.py new file mode 100644 index 0000000..1fcad22 --- /dev/null +++ b/tests/unit_tests/test_transcript_tools.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from langchain_core.tools import ToolException + +from langchain_apify import ( + APIFY_TRANSCRIPT_TOOLS, + ApifyFacebookAdsTranscriptTool, + ApifyMediaTranscriberTool, + ApifyYouTubeTranscriptTool, +) +from tests.unit_tests.conftest import SAMPLE_ITEMS, SUCCEEDED_RUN, make_tool + +if TYPE_CHECKING: + from langchain_apify._client import ApifyToolsClient + +EXPECTED_RUN_META: dict = { + 'run_id': 'run-abc', + 'status': 'SUCCEEDED', + 'dataset_id': 'dataset-xyz', + 'started_at': '2025-01-01T00:00:00.000Z', + 'finished_at': '2025-01-01T00:01:00.000Z', +} + + +def _setup_run_and_items(mock_apify_client: MagicMock, items: list[dict] | None = None) -> None: + mock_apify_client.actor.return_value.call.return_value = SUCCEEDED_RUN + mock_apify_client.dataset.return_value.list_items.return_value.items = items or SAMPLE_ITEMS + + +# --------------------------------------------------------------------------- +# Client - facebook_ads_transcript_scrape +# --------------------------------------------------------------------------- + + +def test_facebook_ads_transcript_scrape_maps_input(client: ApifyToolsClient, mock_apify_client: MagicMock) -> None: + _setup_run_and_items(mock_apify_client) + + run, items = client.facebook_ads_transcript_scrape(['fitness app'], max_results=12) + + mock_apify_client.actor.assert_called_once_with('steadyfetch/facebook-ads-transcript-scraper') + call_kwargs = mock_apify_client.actor.return_value.call.call_args.kwargs + assert call_kwargs['run_input'] == { + 'searchQueries': ['fitness app'], + 'country': 'US', + 'searchMaxAds': 12, + 'maxAds': 12, + } + assert run == SUCCEEDED_RUN + assert items == SAMPLE_ITEMS + + +def test_facebook_ads_transcript_scrape_passes_country(client: ApifyToolsClient, mock_apify_client: MagicMock) -> None: + _setup_run_and_items(mock_apify_client) + + client.facebook_ads_transcript_scrape(['fitness app'], country='GB') + + call_kwargs = mock_apify_client.actor.return_value.call.call_args.kwargs + assert call_kwargs['run_input']['country'] == 'GB' + + +def test_facebook_ads_transcript_scrape_bounds_dataset_items( + client: ApifyToolsClient, mock_apify_client: MagicMock +) -> None: + _setup_run_and_items(mock_apify_client) + + client.facebook_ads_transcript_scrape(['fitness app'], max_results=7) + + assert mock_apify_client.dataset.return_value.list_items.call_args.kwargs['limit'] == 7 + + +def test_facebook_ads_transcript_scrape_empty_queries_raises(client: ApifyToolsClient) -> None: + with pytest.raises(ValueError, match='search_queries'): + client.facebook_ads_transcript_scrape([]) + + +# --------------------------------------------------------------------------- +# Client - youtube_transcript_scrape +# --------------------------------------------------------------------------- + + +def test_youtube_transcript_scrape_maps_input(client: ApifyToolsClient, mock_apify_client: MagicMock) -> None: + _setup_run_and_items(mock_apify_client) + + run, items = client.youtube_transcript_scrape( + ['https://www.youtube.com/watch?v=jNQXAC9IVRw'], + max_results=5, + ) + + mock_apify_client.actor.assert_called_once_with('steadyfetch/youtube-transcript-scraper') + call_kwargs = mock_apify_client.actor.return_value.call.call_args.kwargs + assert call_kwargs['run_input'] == { + 'videoUrls': ['https://www.youtube.com/watch?v=jNQXAC9IVRw'], + 'maxItems': 5, + } + assert run == SUCCEEDED_RUN + assert items == SAMPLE_ITEMS + + +def test_youtube_transcript_scrape_empty_urls_raises(client: ApifyToolsClient) -> None: + with pytest.raises(ValueError, match='video_urls'): + client.youtube_transcript_scrape([]) + + +# --------------------------------------------------------------------------- +# Client - media_transcribe +# --------------------------------------------------------------------------- + + +def test_media_transcribe_maps_input(client: ApifyToolsClient, mock_apify_client: MagicMock) -> None: + _setup_run_and_items(mock_apify_client) + + run, items = client.media_transcribe(['https://example.com/episode.mp3'], max_results=3) + + mock_apify_client.actor.assert_called_once_with('steadyfetch/media-transcriber') + call_kwargs = mock_apify_client.actor.return_value.call.call_args.kwargs + assert call_kwargs['run_input'] == {'urls': ['https://example.com/episode.mp3']} + assert mock_apify_client.dataset.return_value.list_items.call_args.kwargs['limit'] == 3 + assert run == SUCCEEDED_RUN + assert items == SAMPLE_ITEMS + + +def test_media_transcribe_empty_urls_raises(client: ApifyToolsClient) -> None: + with pytest.raises(ValueError, match='urls'): + client.media_transcribe([]) + + +# --------------------------------------------------------------------------- +# ApifyFacebookAdsTranscriptTool +# --------------------------------------------------------------------------- + + +def test_facebook_ads_transcript_tool_happy_path(mock_tools_client: MagicMock) -> None: + mock_tools_client.facebook_ads_transcript_scrape.return_value = (SUCCEEDED_RUN, SAMPLE_ITEMS) + tool = make_tool(ApifyFacebookAdsTranscriptTool, mock_tools_client) + + result = tool._run(search_queries=['fitness app'], country='GB', max_results=12) + parsed = json.loads(result) + + assert parsed['run'] == EXPECTED_RUN_META + assert parsed['items'] == SAMPLE_ITEMS + mock_tools_client.facebook_ads_transcript_scrape.assert_called_once_with( + search_queries=['fitness app'], + country='GB', + max_results=12, + timeout_secs=600, + ) + + +def test_facebook_ads_transcript_tool_defaults_to_us(mock_tools_client: MagicMock) -> None: + mock_tools_client.facebook_ads_transcript_scrape.return_value = (SUCCEEDED_RUN, []) + tool = make_tool(ApifyFacebookAdsTranscriptTool, mock_tools_client) + + tool._run(search_queries=['fitness app']) + + assert mock_tools_client.facebook_ads_transcript_scrape.call_args.kwargs['country'] == 'US' + + +def test_facebook_ads_transcript_tool_clamps_max_results(mock_tools_client: MagicMock) -> None: + mock_tools_client.facebook_ads_transcript_scrape.return_value = (SUCCEEDED_RUN, []) + tool = make_tool(ApifyFacebookAdsTranscriptTool, mock_tools_client, max_items=4) + + tool._run(search_queries=['fitness app'], max_results=500) + + assert mock_tools_client.facebook_ads_transcript_scrape.call_args.kwargs['max_results'] == 4 + + +# --------------------------------------------------------------------------- +# ApifyYouTubeTranscriptTool +# --------------------------------------------------------------------------- + + +def test_youtube_transcript_tool_happy_path(mock_tools_client: MagicMock) -> None: + mock_tools_client.youtube_transcript_scrape.return_value = (SUCCEEDED_RUN, SAMPLE_ITEMS) + tool = make_tool(ApifyYouTubeTranscriptTool, mock_tools_client) + + result = tool._run(video_urls=['https://www.youtube.com/watch?v=jNQXAC9IVRw'], max_results=5) + parsed = json.loads(result) + + assert parsed['run'] == EXPECTED_RUN_META + assert parsed['items'] == SAMPLE_ITEMS + mock_tools_client.youtube_transcript_scrape.assert_called_once_with( + video_urls=['https://www.youtube.com/watch?v=jNQXAC9IVRw'], + max_results=5, + timeout_secs=600, + ) + + +def test_youtube_transcript_tool_clamps_max_results(mock_tools_client: MagicMock) -> None: + mock_tools_client.youtube_transcript_scrape.return_value = (SUCCEEDED_RUN, []) + tool = make_tool(ApifyYouTubeTranscriptTool, mock_tools_client, max_items=2) + + tool._run(video_urls=['jNQXAC9IVRw'], max_results=900) + + assert mock_tools_client.youtube_transcript_scrape.call_args.kwargs['max_results'] == 2 + + +# --------------------------------------------------------------------------- +# ApifyMediaTranscriberTool +# --------------------------------------------------------------------------- + + +def test_media_transcriber_tool_happy_path(mock_tools_client: MagicMock) -> None: + mock_tools_client.media_transcribe.return_value = (SUCCEEDED_RUN, SAMPLE_ITEMS) + tool = make_tool(ApifyMediaTranscriberTool, mock_tools_client) + + result = tool._run(urls=['https://example.com/episode.mp3'], max_results=3) + parsed = json.loads(result) + + assert parsed['run'] == EXPECTED_RUN_META + assert parsed['items'] == SAMPLE_ITEMS + mock_tools_client.media_transcribe.assert_called_once_with( + urls=['https://example.com/episode.mp3'], + max_results=3, + timeout_secs=600, + ) + + +def test_media_transcriber_tool_clamps_max_results(mock_tools_client: MagicMock) -> None: + mock_tools_client.media_transcribe.return_value = (SUCCEEDED_RUN, []) + tool = make_tool(ApifyMediaTranscriberTool, mock_tools_client, max_items=1) + + tool._run(urls=['https://example.com/episode.mp3'], max_results=50) + + assert mock_tools_client.media_transcribe.call_args.kwargs['max_results'] == 1 + + +# --------------------------------------------------------------------------- +# Shared behaviour across the transcript family +# --------------------------------------------------------------------------- + +# (tool_cls, client_method_name, _run kwargs) +_TRANSCRIPT_TOOL_INVOCATIONS: list[tuple[type, str, dict]] = [ + (ApifyFacebookAdsTranscriptTool, 'facebook_ads_transcript_scrape', {'search_queries': ['fitness app']}), + (ApifyYouTubeTranscriptTool, 'youtube_transcript_scrape', {'video_urls': ['jNQXAC9IVRw']}), + (ApifyMediaTranscriberTool, 'media_transcribe', {'urls': ['https://example.com/episode.mp3']}), +] + + +@pytest.mark.parametrize('tool_cls', APIFY_TRANSCRIPT_TOOLS) +def test_transcript_tool_handle_tool_error_enabled(tool_cls: type, mock_tools_client: MagicMock) -> None: + tool = make_tool(tool_cls, mock_tools_client) + assert tool.handle_tool_error is True + + +@pytest.mark.parametrize(('tool_cls', 'method_name', 'run_kwargs'), _TRANSCRIPT_TOOL_INVOCATIONS) +def test_transcript_tool_runtime_error_raises_tool_exception( + tool_cls: type, + method_name: str, + run_kwargs: dict, + mock_tools_client: MagicMock, +) -> None: + getattr(mock_tools_client, method_name).side_effect = RuntimeError( + 'Actor run run-XYZ ended with status FAILED.', + ) + tool = make_tool(tool_cls, mock_tools_client) + + with pytest.raises(ToolException, match='run-XYZ'): + tool._run(**run_kwargs) + + +@pytest.mark.parametrize(('tool_cls', 'method_name', 'run_kwargs'), _TRANSCRIPT_TOOL_INVOCATIONS) +def test_transcript_tool_value_error_raises_tool_exception( + tool_cls: type, + method_name: str, + run_kwargs: dict, + mock_tools_client: MagicMock, +) -> None: + getattr(mock_tools_client, method_name).side_effect = ValueError('At least one value is required.') + tool = make_tool(tool_cls, mock_tools_client) + + with pytest.raises(ToolException, match='At least one value is required'): + tool._run(**run_kwargs) + + +@pytest.mark.parametrize(('tool_cls', 'method_name', 'run_kwargs'), _TRANSCRIPT_TOOL_INVOCATIONS) +def test_transcript_tool_returns_valid_json_for_empty_items( + tool_cls: type, + method_name: str, + run_kwargs: dict, + mock_tools_client: MagicMock, +) -> None: + getattr(mock_tools_client, method_name).return_value = (SUCCEEDED_RUN, []) + tool = make_tool(tool_cls, mock_tools_client) + + result = tool._run(**run_kwargs) + parsed = json.loads(result) + + assert parsed['items'] == [] + assert parsed['run'] == EXPECTED_RUN_META + + +@pytest.mark.parametrize(('tool_cls', 'method_name', 'run_kwargs'), _TRANSCRIPT_TOOL_INVOCATIONS) +def test_transcript_tool_serialises_datetime_in_items( + tool_cls: type, + method_name: str, + run_kwargs: dict, + mock_tools_client: MagicMock, +) -> None: + timestamp = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + items_with_datetime = [{'id': 'item-1', 'timestamp': timestamp, 'text': 'hello'}] + getattr(mock_tools_client, method_name).return_value = (SUCCEEDED_RUN, items_with_datetime) + tool = make_tool(tool_cls, mock_tools_client) + + result = tool._run(**run_kwargs) + parsed = json.loads(result) + + assert isinstance(parsed['items'][0]['timestamp'], str) + assert '2026-01-02' in parsed['items'][0]['timestamp'] + + +@pytest.mark.parametrize(('tool_cls', 'method_name', 'run_kwargs'), _TRANSCRIPT_TOOL_INVOCATIONS) +def test_transcript_tool_envelope_shape( + tool_cls: type, + method_name: str, + run_kwargs: dict, + mock_tools_client: MagicMock, +) -> None: + getattr(mock_tools_client, method_name).return_value = (SUCCEEDED_RUN, SAMPLE_ITEMS) + tool = make_tool(tool_cls, mock_tools_client) + + parsed = json.loads(tool._run(**run_kwargs)) + + assert set(parsed) == {'run', 'items'} + assert isinstance(parsed['items'], list) + assert isinstance(parsed['run'], dict) + + +def test_transcript_tool_group_membership() -> None: + expected = [ApifyFacebookAdsTranscriptTool, ApifyYouTubeTranscriptTool, ApifyMediaTranscriberTool] + assert expected == APIFY_TRANSCRIPT_TOOLS + + +@pytest.mark.parametrize('tool_cls', APIFY_TRANSCRIPT_TOOLS) +def test_transcript_tool_description_carries_no_price(tool_cls: type) -> None: + """Prices go stale and cannot be edited after a release, so they stay out of the package.""" + description = tool_cls.model_fields['description'].default + assert '$' not in description + assert 'price' not in description.lower()