From 632727953b755c2be41c2e6d7e370c232905ebea Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Tue, 13 Oct 2026 15:01:52 +0300 Subject: [PATCH] use external endpoint --- .../decodo/langchain_decodo/_version.py | 2 +- .../langchain_decodo/document_loaders.py | 39 ++++---- .../partners/decodo/langchain_decodo/tools.py | 88 +++++++++++-------- libs/partners/decodo/pyproject.toml | 2 +- .../tests/unit_tests/test_document_loaders.py | 26 ++++++ .../decodo/tests/unit_tests/test_tools.py | 75 ++++++++++------ libs/partners/decodo/uv.lock | 2 +- 7 files changed, 151 insertions(+), 83 deletions(-) diff --git a/libs/partners/decodo/langchain_decodo/_version.py b/libs/partners/decodo/langchain_decodo/_version.py index b3f475621..bbab0242f 100644 --- a/libs/partners/decodo/langchain_decodo/_version.py +++ b/libs/partners/decodo/langchain_decodo/_version.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.4" diff --git a/libs/partners/decodo/langchain_decodo/document_loaders.py b/libs/partners/decodo/langchain_decodo/document_loaders.py index e8e6f6fa8..b7fb224c1 100644 --- a/libs/partners/decodo/langchain_decodo/document_loaders.py +++ b/libs/partners/decodo/langchain_decodo/document_loaders.py @@ -10,8 +10,10 @@ Two authentication modes are supported via ``auth_mode``: * ``"basic"`` *(default)* — username:password credentials encoded as a Basic - auth token. Uses the ``/v2/scrape`` endpoint. -* ``"token"`` — an API key. Uses the ``/unified/v1/scrape`` endpoint. + auth token, sent as ``Authorization: Basic `` to + ``https://scraper-api.decodo.com/v2/scrape``. +* ``"token"`` — an API key, sent as ``Authorization: Bearer `` to + ``https://data.decodo.com/v1/scrape``. Typical usage — basic auth (default):: @@ -45,37 +47,40 @@ from langchain_core.documents import Document from pydantic import SecretStr -_API_BASE = "https://scraper-api.decodo.com" -_V2_SCRAPE_PATH = "/v2/scrape" -_UNIFIED_SCRAPE_PATH = "/unified/v1/scrape" +# API keys and basic auth tokens are only accepted on their own host. +_BASIC_SCRAPE_ENDPOINT = "https://scraper-api.decodo.com/v2/scrape" +_TOKEN_SCRAPE_ENDPOINT = "https://data.decodo.com/v1/scrape" _DEFAULT_TIMEOUT = 180.0 -def _scrape_path(auth_mode: str) -> str: - """Return the API path for the given auth mode. +def _scrape_endpoint(auth_mode: str) -> str: + """Return the scrape endpoint URL for the given auth mode. Args: auth_mode: Either ``"basic"`` or ``"token"``. Returns: - The URL path string for the scrape endpoint. + The scrape endpoint URL. """ - return _UNIFIED_SCRAPE_PATH if auth_mode == "token" else _V2_SCRAPE_PATH + return _TOKEN_SCRAPE_ENDPOINT if auth_mode == "token" else _BASIC_SCRAPE_ENDPOINT -def _build_headers(token: str) -> dict[str, str]: +def _build_headers(token: str, auth_mode: str) -> dict[str, str]: """Build HTTP headers for a Decodo API request. Args: token: Raw token string. For ``"basic"`` auth this is the base64-encoded ``username:password`` value; for ``"token"`` auth this is the API key. + auth_mode: Either ``"basic"`` or ``"token"``; selects the + ``Authorization`` scheme. Returns: Dictionary of HTTP headers including ``Authorization``. """ + scheme = "Bearer" if auth_mode == "token" else "Basic" return { - "Authorization": f"Basic {token}", + "Authorization": f"{scheme} {token}", "Content-Type": "application/json", "Accept": "application/json", "x-integration": "langchain-python", @@ -94,8 +99,8 @@ def _scrape_url( token: Raw token string (format depends on ``auth_mode``). url: Full URL to scrape (must include scheme). timeout: Request timeout in seconds. - auth_mode: ``"basic"`` (username:password, ``/v2/scrape``) or - ``"token"`` (API key, ``/unified/v1/scrape``). + auth_mode: ``"basic"`` (username:password, ``scraper-api.decodo.com``) + or ``"token"`` (API key, ``data.decodo.com``). Returns: Parsed JSON response from the Decodo API. @@ -103,7 +108,7 @@ def _scrape_url( Raises: RuntimeError: On timeout, network error, or non-2xx HTTP response. """ - endpoint = f"{_API_BASE}{_scrape_path(auth_mode)}" + endpoint = _scrape_endpoint(auth_mode) payload: dict[str, Any] = { "target": "universal", "url": url, @@ -113,7 +118,7 @@ def _scrape_url( try: response = httpx.post( endpoint, - headers=_build_headers(token), + headers=_build_headers(token, auth_mode), json=payload, timeout=timeout, ) @@ -175,8 +180,8 @@ class DecodoLoader(BaseLoader): auth_mode: Authentication mode. * ``"basic"`` *(default)* — username:password credentials, - uses ``/v2/scrape``. - * ``"token"`` — API key, uses ``/unified/v1/scrape``. + uses ``https://scraper-api.decodo.com/v2/scrape``. + * ``"token"`` — API key, uses ``https://data.decodo.com/v1/scrape``. timeout: HTTP request timeout in seconds (default 180). continue_on_error: When ``True`` (default), skip URLs that fail and diff --git a/libs/partners/decodo/langchain_decodo/tools.py b/libs/partners/decodo/langchain_decodo/tools.py index 8a8655b09..4d3458b49 100644 --- a/libs/partners/decodo/langchain_decodo/tools.py +++ b/libs/partners/decodo/langchain_decodo/tools.py @@ -10,11 +10,10 @@ Both tools support two authentication modes selected via ``auth_mode``: * ``"basic"`` *(default)* — username:password credentials encoded as a Basic - auth token. Uses the ``/v2/scrape`` endpoint. -* ``"token"`` — an API key. Uses the ``/unified/v1/scrape`` endpoint. - -In both modes the ``Authorization: Basic `` header is used; only the -value and the target endpoint differ. + auth token, sent as ``Authorization: Basic `` to + ``https://scraper-api.decodo.com/v2/scrape``. +* ``"token"`` — an API key, sent as ``Authorization: Bearer `` to + ``https://data.decodo.com/v1/scrape``. """ from __future__ import annotations @@ -28,11 +27,11 @@ from langchain_core.tools import BaseTool from pydantic import BaseModel, Field, SecretStr, model_validator -_DEFAULT_BASE_URL = "https://scraper-api.decodo.com" - -# Endpoint selected by auth_mode -_V2_SCRAPE_PATH = "/v2/scrape" -_UNIFIED_SCRAPE_PATH = "/unified/v1/scrape" +# API keys and basic auth tokens are only accepted on their own host. +_BASIC_BASE_URL = "https://scraper-api.decodo.com" +_BASIC_SCRAPE_PATH = "/v2/scrape" +_TOKEN_BASE_URL = "https://data.decodo.com" +_TOKEN_SCRAPE_PATH = "/v1/scrape" _DEFAULT_TIMEOUT = 180.0 # seconds @@ -45,31 +44,37 @@ _REDDIT_SITE_FILTER = "site:reddit.com" -def _scrape_path(auth_mode: str) -> str: - """Return the API path for the given auth mode. +def _scrape_url(auth_mode: str, base_url: str | None = None) -> str: + """Return the full scrape endpoint URL for the given auth mode. Args: auth_mode: Either ``"basic"`` or ``"token"``. + base_url: Overrides the host chosen by ``auth_mode`` when set. Returns: - The URL path string for the scrape endpoint. + The scrape endpoint URL. """ - return _UNIFIED_SCRAPE_PATH if auth_mode == "token" else _V2_SCRAPE_PATH + if auth_mode == "token": + return f"{base_url or _TOKEN_BASE_URL}{_TOKEN_SCRAPE_PATH}" + return f"{base_url or _BASIC_BASE_URL}{_BASIC_SCRAPE_PATH}" -def _build_headers(token: str) -> dict[str, str]: +def _build_headers(token: str, auth_mode: str) -> dict[str, str]: """Build HTTP headers for a Decodo API request. Args: token: Raw token string. For ``"basic"`` auth this is the base64-encoded ``username:password`` value; for ``"token"`` auth this is the API key. + auth_mode: Either ``"basic"`` or ``"token"``; selects the + ``Authorization`` scheme. Returns: Dictionary of HTTP headers including ``Authorization``. """ + scheme = "Bearer" if auth_mode == "token" else "Basic" return { - "Authorization": f"Basic {token}", + "Authorization": f"{scheme} {token}", "Content-Type": "application/json", "Accept": "application/json", "x-integration": "langchain-python", @@ -78,7 +83,7 @@ def _build_headers(token: str) -> dict[str, str]: def _do_scrape( token: str, - base_url: str, + base_url: str | None, payload: dict[str, Any], auth_mode: str = "basic", timeout: float = _DEFAULT_TIMEOUT, @@ -87,10 +92,11 @@ def _do_scrape( Args: token: Raw token string (format depends on ``auth_mode``). - base_url: Base URL of the Decodo API. + base_url: Base URL of the Decodo API. ``None`` uses the host that + matches ``auth_mode``. payload: Request body to send as JSON. - auth_mode: ``"basic"`` (username:password, ``/v2/scrape``) or - ``"token"`` (API key, ``/unified/v1/scrape``). + auth_mode: ``"basic"`` (username:password, ``scraper-api.decodo.com``) + or ``"token"`` (API key, ``data.decodo.com``). timeout: Request timeout in seconds. Returns: @@ -99,8 +105,8 @@ def _do_scrape( Raises: RuntimeError: On timeout, network error, or non-2xx HTTP response. """ - url = f"{base_url}{_scrape_path(auth_mode)}" - headers = _build_headers(token) + url = _scrape_url(auth_mode, base_url) + headers = _build_headers(token, auth_mode) try: response = httpx.post(url, headers=headers, json=payload, timeout=timeout) @@ -190,12 +196,11 @@ class DecodoWebScrapeTool(BaseTool): * ``"basic"`` *(default)* — supply the base64-encoded ``username:password`` string as ``decodo_api_token`` (or set - ``DECODO_API_TOKEN``). Requests go to ``/v2/scrape``. + ``DECODO_API_TOKEN``). Requests go to + ``https://scraper-api.decodo.com/v2/scrape`` with ``Basic`` auth. * ``"token"`` — supply an API key as ``decodo_api_token`` (or set - ``DECODO_API_TOKEN``). Requests go to ``/unified/v1/scrape``. - - Both modes send ``Authorization: Basic ``; the value and endpoint - differ. + ``DECODO_API_TOKEN``). Requests go to + ``https://data.decodo.com/v1/scrape`` with ``Bearer`` auth. Attributes: decodo_api_token: Credential value. For ``"basic"`` mode this is the @@ -203,7 +208,8 @@ class DecodoWebScrapeTool(BaseTool): the API key. Falls back to the ``DECODO_API_TOKEN`` environment variable when not provided explicitly. auth_mode: Authentication mode — ``"basic"`` (default) or ``"token"``. - base_url: Base URL of the Decodo Scraper API. + base_url: Base URL override. Defaults to the host that matches + ``auth_mode``. Example — basic auth (default):: @@ -243,13 +249,16 @@ class DecodoWebScrapeTool(BaseTool): default="basic", description=( "Authentication mode. " - "'basic' (default): username:password credentials, uses /v2/scrape. " - "'token': API key, uses /unified/v1/scrape." + "'basic' (default): username:password credentials, " + "uses scraper-api.decodo.com/v2/scrape. " + "'token': API key, uses data.decodo.com/v1/scrape." ), ) - base_url: str = Field( - default=_DEFAULT_BASE_URL, - description="Base URL of the Decodo Scraper API.", + base_url: str | None = Field( + default=None, + description=( + "Base URL override for the Decodo API. Defaults to the host that matches auth_mode." + ), ) @model_validator(mode="before") @@ -316,9 +325,11 @@ class DecodoSearchTool(BaseTool): * ``"basic"`` *(default)* — supply the base64-encoded ``username:password`` string as ``decodo_api_token`` (or set - ``DECODO_API_TOKEN``). Requests go to ``/v2/scrape``. + ``DECODO_API_TOKEN``). Requests go to + ``https://scraper-api.decodo.com/v2/scrape`` with ``Basic`` auth. * ``"token"`` — supply an API key as ``decodo_api_token`` (or set - ``DECODO_API_TOKEN``). Requests go to ``/unified/v1/scrape``. + ``DECODO_API_TOKEN``). Requests go to + ``https://data.decodo.com/v1/scrape`` with ``Bearer`` auth. Attributes: decodo_api_token: Credential value. Falls back to ``DECODO_API_TOKEN``. @@ -366,8 +377,9 @@ class DecodoSearchTool(BaseTool): default="basic", description=( "Authentication mode. " - "'basic' (default): username:password credentials, uses /v2/scrape. " - "'token': API key, uses /unified/v1/scrape." + "'basic' (default): username:password credentials, " + "uses scraper-api.decodo.com/v2/scrape. " + "'token': API key, uses data.decodo.com/v1/scrape." ), ) engine: str = Field( @@ -434,7 +446,7 @@ def _run( "parse": True, } - response = _do_scrape(token, _DEFAULT_BASE_URL, payload, auth_mode=self.auth_mode) + response = _do_scrape(token, None, payload, auth_mode=self.auth_mode) results = response.get("results", []) # A failed scrape comes back as HTTP 200, either with a failed status diff --git a/libs/partners/decodo/pyproject.toml b/libs/partners/decodo/pyproject.toml index 4e836b3d7..fcac80c91 100644 --- a/libs/partners/decodo/pyproject.toml +++ b/libs/partners/decodo/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "langchain-decodo" -version = "0.1.3" +version = "0.1.4" description = "LangChain integration for the Decodo web scraping API" readme = "README.md" license = { text = "MIT" } diff --git a/libs/partners/decodo/tests/unit_tests/test_document_loaders.py b/libs/partners/decodo/tests/unit_tests/test_document_loaders.py index 88f8d2e31..a3e69844e 100644 --- a/libs/partners/decodo/tests/unit_tests/test_document_loaders.py +++ b/libs/partners/decodo/tests/unit_tests/test_document_loaders.py @@ -18,3 +18,29 @@ def test_loader_requests_markdown(mock_post: MagicMock) -> None: assert mock_post.call_args[1]["json"]["markdown"] is True assert docs[0].page_content == "# Title" + + +@patch("langchain_decodo.document_loaders.httpx.post") +def test_loader_basic_mode_uses_scraper_api(mock_post: MagicMock) -> None: + response = MagicMock() + response.is_success = True + response.json.return_value = {"results": [{"content": "x", "status_code": 200}]} + mock_post.return_value = response + + DecodoLoader(urls="https://example.com", api_token="tok").load() + + assert mock_post.call_args[0][0] == "https://scraper-api.decodo.com/v2/scrape" + assert mock_post.call_args[1]["headers"]["Authorization"] == "Basic tok" + + +@patch("langchain_decodo.document_loaders.httpx.post") +def test_loader_token_mode_uses_data_api(mock_post: MagicMock) -> None: + response = MagicMock() + response.is_success = True + response.json.return_value = {"results": [{"content": "x", "status_code": 200}]} + mock_post.return_value = response + + DecodoLoader(urls="https://example.com", api_token="key", auth_mode="token").load() + + assert mock_post.call_args[0][0] == "https://data.decodo.com/v1/scrape" + assert mock_post.call_args[1]["headers"]["Authorization"] == "Bearer key" diff --git a/libs/partners/decodo/tests/unit_tests/test_tools.py b/libs/partners/decodo/tests/unit_tests/test_tools.py index eed49da69..c32d425da 100644 --- a/libs/partners/decodo/tests/unit_tests/test_tools.py +++ b/libs/partners/decodo/tests/unit_tests/test_tools.py @@ -7,7 +7,7 @@ from __future__ import annotations import json -from typing import Any +from typing import Any, Literal from unittest.mock import MagicMock, patch import pytest @@ -68,7 +68,7 @@ def test_name_and_description(self) -> None: def test_default_base_url(self) -> None: tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("x")) - assert tool.base_url == "https://scraper-api.decodo.com" + assert tool.base_url is None def test_custom_base_url(self) -> None: tool = DecodoWebScrapeTool( @@ -350,41 +350,56 @@ class TestAuthMode: """Verify that auth_mode controls which endpoint is called.""" @patch("langchain_decodo.tools.httpx.post") - def test_scrape_basic_mode_uses_v2_endpoint(self, mock_post: MagicMock) -> None: + def test_scrape_basic_mode_uses_scraper_api(self, mock_post: MagicMock) -> None: mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("tok"), auth_mode="basic") tool._run("https://example.com") - url_called = mock_post.call_args[0][0] - assert "/v2/scrape" in url_called - assert "/unified/" not in url_called + assert mock_post.call_args[0][0] == "https://scraper-api.decodo.com/v2/scrape" @patch("langchain_decodo.tools.httpx.post") - def test_scrape_token_mode_uses_unified_endpoint(self, mock_post: MagicMock) -> None: + def test_scrape_token_mode_uses_data_api(self, mock_post: MagicMock) -> None: mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("tok"), auth_mode="token") tool._run("https://example.com") - url_called = mock_post.call_args[0][0] - assert "/unified/v1/scrape" in url_called + assert mock_post.call_args[0][0] == "https://data.decodo.com/v1/scrape" + + @pytest.mark.parametrize( + ("mode", "expected"), + [ + ("basic", "https://custom.example.com/v2/scrape"), + ("token", "https://custom.example.com/v1/scrape"), + ], + ) + @patch("langchain_decodo.tools.httpx.post") + def test_scrape_custom_base_url_keeps_mode_path( + self, mock_post: MagicMock, mode: Literal["basic", "token"], expected: str + ) -> None: + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) + tool = DecodoWebScrapeTool( + decodo_api_token=SecretStr("tok"), + auth_mode=mode, + base_url="https://custom.example.com", + ) + tool._run("https://example.com") + assert mock_post.call_args[0][0] == expected @patch("langchain_decodo.tools.httpx.post") - def test_search_basic_mode_uses_v2_endpoint(self, mock_post: MagicMock) -> None: + def test_search_basic_mode_uses_scraper_api(self, mock_post: MagicMock) -> None: mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok"), auth_mode="basic") tool._run("query") - url_called = mock_post.call_args[0][0] - assert "/v2/scrape" in url_called + assert mock_post.call_args[0][0] == "https://scraper-api.decodo.com/v2/scrape" @patch("langchain_decodo.tools.httpx.post") - def test_search_token_mode_uses_unified_endpoint(self, mock_post: MagicMock) -> None: + def test_search_token_mode_uses_data_api(self, mock_post: MagicMock) -> None: mock_post.return_value = _make_mock_httpx_response(_make_api_response()) tool = DecodoSearchTool(decodo_api_token=SecretStr("tok"), auth_mode="token") tool._run("query") - url_called = mock_post.call_args[0][0] - assert "/unified/v1/scrape" in url_called + assert mock_post.call_args[0][0] == "https://data.decodo.com/v1/scrape" def test_default_auth_mode_is_basic(self) -> None: tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("tok")) @@ -394,15 +409,25 @@ def test_token_auth_mode_accepted(self) -> None: tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("tok"), auth_mode="token") assert tool.auth_mode == "token" + @pytest.mark.parametrize( + ("mode", "expected"), + [("basic", "Basic mytoken"), ("token", "Bearer mytoken")], + ) @patch("langchain_decodo.tools.httpx.post") - def test_auth_header_same_regardless_of_mode(self, mock_post: MagicMock) -> None: - """Authorization header format is identical for both modes.""" + def test_auth_header_scheme_follows_mode( + self, mock_post: MagicMock, mode: Literal["basic", "token"], expected: str + ) -> None: mock_post.return_value = _make_mock_httpx_response(_make_api_response()) - for mode in ("basic", "token"): - tool = DecodoWebScrapeTool( - decodo_api_token=SecretStr("mytoken"), - auth_mode=mode, - ) - tool._run("https://example.com") - _, kwargs = mock_post.call_args - assert kwargs["headers"]["Authorization"] == "Basic mytoken" + tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("mytoken"), auth_mode=mode) + tool._run("https://example.com") + _, kwargs = mock_post.call_args + assert kwargs["headers"]["Authorization"] == expected + + @patch("langchain_decodo.tools.httpx.post") + def test_search_token_mode_sends_bearer(self, mock_post: MagicMock) -> None: + mock_post.return_value = _make_mock_httpx_response(_make_api_response()) + tool = DecodoSearchTool(decodo_api_token=SecretStr("key"), auth_mode="token") + tool._run("query") + + _, kwargs = mock_post.call_args + assert kwargs["headers"]["Authorization"] == "Bearer key" diff --git a/libs/partners/decodo/uv.lock b/libs/partners/decodo/uv.lock index a844794a8..57534bfda 100644 --- a/libs/partners/decodo/uv.lock +++ b/libs/partners/decodo/uv.lock @@ -342,7 +342,7 @@ wheels = [ [[package]] name = "langchain-decodo" -version = "0.1.2" +version = "0.1.4" source = { editable = "." } dependencies = [ { name = "httpx" },