diff --git a/libs/partners/decodo/Makefile b/libs/partners/decodo/Makefile index 87a15cc58..39bdabfa2 100644 --- a/libs/partners/decodo/Makefile +++ b/libs/partners/decodo/Makefile @@ -43,4 +43,4 @@ help: @echo 'lint - run linters' @echo 'format - run formatters' @echo 'tests - run unit tests' - @echo 'integration_tests - run integration tests (requires DECODO_API_TOKEN)' + @echo 'integration_tests - run integration tests (requires an API key in DECODO_API_TOKEN)' diff --git a/libs/partners/decodo/README.md b/libs/partners/decodo/README.md index 915192a20..70314e893 100644 --- a/libs/partners/decodo/README.md +++ b/libs/partners/decodo/README.md @@ -14,23 +14,21 @@ pip install langchain-decodo ## Authentication -All classes read your Decodo API token from the `DECODO_API_TOKEN` environment -variable, or you can pass it explicitly. +Use the Web Data API key from your Web Data API subscription on the +[Decodo Dashboard](https://dashboard.decodo.com/web-data/playground). All classes read it from the `DECODO_API_TOKEN` environment variable, or you can pass it explicitly. Pass `auth_mode="token"` when using the key: ```bash -export DECODO_API_TOKEN="your-decodo-api-token" +export DECODO_API_TOKEN="your-decodo-api-key" ``` -Get a token from the [Decodo Dashboard](https://app.decodo.com). - -By default every class expects basic credentials (base64-encoded -`username:password`). If your token is a plain API token, pass -`auth_mode="token"`.: - ```python tool = DecodoWebScrapeTool(auth_mode="token") ``` +Older plans only have a basic authentication token (base64-encoded +`username:password`). That is the default `auth_mode="basic"`, so omit +`auth_mode` and export the basic token as `DECODO_API_TOKEN`. + ## Components ### `DecodoWebScrapeTool` @@ -41,7 +39,7 @@ Handles JavaScript rendering, CAPTCHAs, and geo-blocking automatically. ```python from langchain_decodo import DecodoWebScrapeTool -tool = DecodoWebScrapeTool() # reads DECODO_API_TOKEN from env +tool = DecodoWebScrapeTool(auth_mode="token") # reads DECODO_API_TOKEN from env content = tool.run("https://example.com") print(content) ``` @@ -52,7 +50,10 @@ Pass explicitly: from pydantic import SecretStr from langchain_decodo import DecodoWebScrapeTool -tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("YOUR_TOKEN")) +tool = DecodoWebScrapeTool( + decodo_api_token=SecretStr("YOUR_API_KEY"), + auth_mode="token", +) ``` ### `DecodoSearchTool` @@ -62,7 +63,7 @@ Search Google, Amazon, or Reddit and return structured JSON results. ```python from langchain_decodo import DecodoSearchTool -tool = DecodoSearchTool() +tool = DecodoSearchTool(auth_mode="token") # Google search (default) results = tool.run({"query": "LangChain latest release", "engine": "google"}) @@ -97,6 +98,7 @@ loader = DecodoLoader( "https://python.org/about/", "https://docs.python.org/3/whatsnew/3.12.html", ], + auth_mode="token", ) docs = loader.load() @@ -123,7 +125,10 @@ from langchain_decodo import DecodoSearchTool, DecodoWebScrapeTool agent = create_agent( model="openai:gpt-4o-mini", - tools=[DecodoWebScrapeTool(), DecodoSearchTool()], + tools=[ + DecodoWebScrapeTool(auth_mode="token"), + DecodoSearchTool(auth_mode="token"), + ], ) result = agent.invoke( @@ -151,7 +156,7 @@ from langchain_openai import ChatOpenAI, OpenAIEmbeddings from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_decodo import DecodoLoader -docs = DecodoLoader(urls=["https://python.org/about/"]).load() +docs = DecodoLoader(urls=["https://python.org/about/"], auth_mode="token").load() splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) chunks = splitter.split_documents(docs) @@ -172,5 +177,5 @@ print(answer.content) - [Decodo website](https://decodo.com) - [Decodo API documentation](https://help.decodo.com) -- [Decodo Dashboard](https://app.decodo.com) +- [Decodo Dashboard](https://dashboard.decodo.com/) - [LangChain documentation](https://python.langchain.com) diff --git a/libs/partners/decodo/langchain_decodo/_version.py b/libs/partners/decodo/langchain_decodo/_version.py index 485f44ac2..b3f475621 100644 --- a/libs/partners/decodo/langchain_decodo/_version.py +++ b/libs/partners/decodo/langchain_decodo/_version.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/libs/partners/decodo/langchain_decodo/document_loaders.py b/libs/partners/decodo/langchain_decodo/document_loaders.py index 51ad982b0..f43afcd5e 100644 --- a/libs/partners/decodo/langchain_decodo/document_loaders.py +++ b/libs/partners/decodo/langchain_decodo/document_loaders.py @@ -11,7 +11,7 @@ * ``"basic"`` *(default)* — username:password credentials encoded as a Basic auth token. Uses the ``/v2/scrape`` endpoint. -* ``"token"`` — a plain API token. Uses the ``/unified/v1/scrape`` endpoint. +* ``"token"`` — an API key. Uses the ``/unified/v1/scrape`` endpoint. Typical usage — basic auth (default):: @@ -27,7 +27,7 @@ loader = DecodoLoader( urls=["https://example.com"], - api_token="your-api-token", + api_token="your-api-key", auth_mode="token", ) docs = loader.load() @@ -69,7 +69,7 @@ def _build_headers(token: str) -> dict[str, str]: Args: token: Raw token string. For ``"basic"`` auth this is the base64-encoded ``username:password`` value; for ``"token"`` auth - this is the plain API token. + this is the API key. Returns: Dictionary of HTTP headers including ``Authorization``. @@ -95,7 +95,7 @@ def _scrape_url( url: Full URL to scrape (must include scheme). timeout: Request timeout in seconds. auth_mode: ``"basic"`` (username:password, ``/v2/scrape``) or - ``"token"`` (API token, ``/unified/v1/scrape``). + ``"token"`` (API key, ``/unified/v1/scrape``). Returns: Parsed JSON response from the Decodo API. @@ -170,13 +170,13 @@ class DecodoLoader(BaseLoader): urls: A single URL string or a list of URL strings to scrape. api_token: Credential value. For ``"basic"`` auth (default) this is the base64-encoded ``username:password`` string; for ``"token"`` - auth this is the plain API token. Falls back to the + auth this is the API key. Falls back to the ``DECODO_API_TOKEN`` environment variable when omitted. auth_mode: Authentication mode. * ``"basic"`` *(default)* — username:password credentials, uses ``/v2/scrape``. - * ``"token"`` — plain API token, uses ``/unified/v1/scrape``. + * ``"token"`` — API key, uses ``/unified/v1/scrape``. timeout: HTTP request timeout in seconds (default 180). continue_on_error: When ``True`` (default), skip URLs that fail and @@ -191,11 +191,11 @@ class DecodoLoader(BaseLoader): ) docs = loader.load() - Example — token auth:: + Example — API key auth:: loader = DecodoLoader( urls=["https://example.com", "https://example.org"], - api_token="your-api-token", + api_token="your-api-key", auth_mode="token", ) docs = loader.load() @@ -217,7 +217,7 @@ def __init__( raw_token = api_token or os.environ.get("DECODO_API_TOKEN", "") if not raw_token: raise ValueError( - "Decodo API token is required. Provide it via the ``api_token`` " + "Decodo API key or token is required. Provide it via the ``api_token`` " "argument or the ``DECODO_API_TOKEN`` environment variable." ) self._token: SecretStr = SecretStr(raw_token) diff --git a/libs/partners/decodo/langchain_decodo/tools.py b/libs/partners/decodo/langchain_decodo/tools.py index 5aa858c98..53caf2f0c 100644 --- a/libs/partners/decodo/langchain_decodo/tools.py +++ b/libs/partners/decodo/langchain_decodo/tools.py @@ -11,7 +11,7 @@ * ``"basic"`` *(default)* — username:password credentials encoded as a Basic auth token. Uses the ``/v2/scrape`` endpoint. -* ``"token"`` — a plain API token. Uses the ``/unified/v1/scrape`` endpoint. +* ``"token"`` — an API key. Uses the ``/unified/v1/scrape`` endpoint. In both modes the ``Authorization: Basic `` header is used; only the value and the target endpoint differ. @@ -63,7 +63,7 @@ def _build_headers(token: str) -> dict[str, str]: Args: token: Raw token string. For ``"basic"`` auth this is the base64-encoded ``username:password`` value; for ``"token"`` auth - this is the plain API token. + this is the API key. Returns: Dictionary of HTTP headers including ``Authorization``. @@ -90,7 +90,7 @@ def _do_scrape( base_url: Base URL of the Decodo API. payload: Request body to send as JSON. auth_mode: ``"basic"`` (username:password, ``/v2/scrape``) or - ``"token"`` (API token, ``/unified/v1/scrape``). + ``"token"`` (API key, ``/unified/v1/scrape``). timeout: Request timeout in seconds. Returns: @@ -191,7 +191,7 @@ class DecodoWebScrapeTool(BaseTool): * ``"basic"`` *(default)* — supply the base64-encoded ``username:password`` string as ``decodo_api_token`` (or set ``DECODO_API_TOKEN``). Requests go to ``/v2/scrape``. - * ``"token"`` — supply a plain API token as ``decodo_api_token`` (or set + * ``"token"`` — supply an API key as ``decodo_api_token`` (or set ``DECODO_API_TOKEN``). Requests go to ``/unified/v1/scrape``. Both modes send ``Authorization: Basic ``; the value and endpoint @@ -200,7 +200,7 @@ class DecodoWebScrapeTool(BaseTool): Attributes: decodo_api_token: Credential value. For ``"basic"`` mode this is the base64-encoded ``username:password``; for ``"token"`` mode this is - the plain API token. Falls back to the ``DECODO_API_TOKEN`` + the API key. Falls back to the ``DECODO_API_TOKEN`` environment variable when not provided explicitly. auth_mode: Authentication mode — ``"basic"`` (default) or ``"token"``. base_url: Base URL of the Decodo Scraper API. @@ -212,10 +212,10 @@ class DecodoWebScrapeTool(BaseTool): tool = DecodoWebScrapeTool(decodo_api_token="base64(user:pass)") result = tool.run("https://example.com") - Example — token auth:: + Example — API key auth:: tool = DecodoWebScrapeTool( - decodo_api_token="your-api-token", + decodo_api_token="your-api-key", auth_mode="token", ) result = tool.run("https://example.com") @@ -235,7 +235,7 @@ class DecodoWebScrapeTool(BaseTool): default=SecretStr(""), description=( "Credential value. For 'basic' auth: base64-encoded username:password. " - "For 'token' auth: plain API token. " + "For 'token' auth: API key. " "Falls back to the DECODO_API_TOKEN environment variable." ), ) @@ -244,7 +244,7 @@ class DecodoWebScrapeTool(BaseTool): description=( "Authentication mode. " "'basic' (default): username:password credentials, uses /v2/scrape. " - "'token': plain API token, uses /unified/v1/scrape." + "'token': API key, uses /unified/v1/scrape." ), ) base_url: str = Field( @@ -283,13 +283,13 @@ def _run( Page content as a string (markdown or plain text). Raises: - ValueError: If no API token is configured. + ValueError: If no API key or token is configured. RuntimeError: On API errors or network failures. """ token = self.decodo_api_token.get_secret_value() if not token: raise ValueError( - "Decodo API token is required. Set it via the ``decodo_api_token`` " + "Decodo API key or token is required. Set it via the ``decodo_api_token`` " "field or the ``DECODO_API_TOKEN`` environment variable." ) @@ -317,7 +317,7 @@ class DecodoSearchTool(BaseTool): * ``"basic"`` *(default)* — supply the base64-encoded ``username:password`` string as ``decodo_api_token`` (or set ``DECODO_API_TOKEN``). Requests go to ``/v2/scrape``. - * ``"token"`` — supply a plain API token as ``decodo_api_token`` (or set + * ``"token"`` — supply an API key as ``decodo_api_token`` (or set ``DECODO_API_TOKEN``). Requests go to ``/unified/v1/scrape``. Attributes: @@ -332,10 +332,10 @@ class DecodoSearchTool(BaseTool): tool = DecodoSearchTool(decodo_api_token="base64(user:pass)") results = tool.run({"query": "best Python web scraping libraries"}) - Example — token auth:: + Example — API key auth:: tool = DecodoSearchTool( - decodo_api_token="your-api-token", + decodo_api_token="your-api-key", auth_mode="token", ) results = tool.run({"query": "best Python web scraping libraries"}) @@ -358,7 +358,7 @@ class DecodoSearchTool(BaseTool): default=SecretStr(""), description=( "Credential value. For 'basic' auth: base64-encoded username:password. " - "For 'token' auth: plain API token. " + "For 'token' auth: API key. " "Falls back to the DECODO_API_TOKEN environment variable." ), ) @@ -367,7 +367,7 @@ class DecodoSearchTool(BaseTool): description=( "Authentication mode. " "'basic' (default): username:password credentials, uses /v2/scrape. " - "'token': plain API token, uses /unified/v1/scrape." + "'token': API key, uses /unified/v1/scrape." ), ) engine: str = Field( @@ -411,13 +411,13 @@ def _run( ``status_code``, and ``url`` fields. Raises: - ValueError: If no API token is configured. + ValueError: If no API key or token is configured. RuntimeError: On API errors or network failures. """ token = self.decodo_api_token.get_secret_value() if not token: raise ValueError( - "Decodo API token is required. Set it via the ``decodo_api_token`` " + "Decodo API key or token is required. Set it via the ``decodo_api_token`` " "field or the ``DECODO_API_TOKEN`` environment variable." ) diff --git a/libs/partners/decodo/pyproject.toml b/libs/partners/decodo/pyproject.toml index c2e159693..9bc85d8ed 100644 --- a/libs/partners/decodo/pyproject.toml +++ b/libs/partners/decodo/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "langchain-decodo" -version = "0.1.1" +version = "0.1.2" description = "LangChain integration for the Decodo web scraping API" readme = "README.md" license = { text = "MIT" } diff --git a/libs/partners/decodo/tests/integration_tests/test_tools.py b/libs/partners/decodo/tests/integration_tests/test_tools.py index 073007d79..be58f907c 100644 --- a/libs/partners/decodo/tests/integration_tests/test_tools.py +++ b/libs/partners/decodo/tests/integration_tests/test_tools.py @@ -1,8 +1,9 @@ """Integration tests for langchain_decodo tools. -Run with a real DECODO_API_TOKEN set: +Run with a real Decodo API key in DECODO_API_TOKEN (a basic auth token from an older +plan needs `auth_mode="basic"` in the constructors below): - DECODO_API_TOKEN=your_token pytest tests/integration_tests/ + DECODO_API_TOKEN=your_api_key pytest tests/integration_tests/ All tests are skipped automatically when ``DECODO_API_TOKEN`` is not set in the environment, so this file is safe to include in CI without secrets. @@ -16,10 +17,10 @@ import pytest -# Skip the entire module if no token is configured. +# Skip the entire module if no credential is configured. pytestmark = pytest.mark.skipif( not os.environ.get("DECODO_API_TOKEN"), - reason="DECODO_API_TOKEN environment variable is not set", + reason="DECODO_API_TOKEN environment variable (API key) is not set", ) @@ -27,21 +28,21 @@ def scrape_tool() -> Any: from langchain_decodo.tools import DecodoWebScrapeTool - return DecodoWebScrapeTool() + return DecodoWebScrapeTool(auth_mode="token") @pytest.fixture() def search_tool() -> Any: from langchain_decodo.tools import DecodoSearchTool - return DecodoSearchTool() + return DecodoSearchTool(auth_mode="token") @pytest.fixture() def loader() -> Any: from langchain_decodo import DecodoLoader - return DecodoLoader(urls=["https://example.com"]) + return DecodoLoader(urls=["https://example.com"], auth_mode="token") # --------------------------------------------------------------------------- @@ -115,7 +116,7 @@ def test_load_single_url(self) -> None: from langchain_decodo import DecodoLoader - loader = DecodoLoader(urls="https://example.com") + loader = DecodoLoader(urls="https://example.com", auth_mode="token") docs = loader.load() assert isinstance(docs, list) assert len(docs) > 0 @@ -129,7 +130,7 @@ def test_lazy_load_yields_documents(self) -> None: from langchain_decodo import DecodoLoader - loader = DecodoLoader(urls=["https://example.com"]) + loader = DecodoLoader(urls=["https://example.com"], auth_mode="token") docs = list(loader.lazy_load()) assert all(isinstance(d, Document) for d in docs) @@ -138,6 +139,7 @@ def test_continue_on_error_skips_bad_url(self) -> None: loader = DecodoLoader( urls=["https://this-domain-definitely-does-not-exist-xyz123.com"], + auth_mode="token", continue_on_error=True, ) docs = loader.load() diff --git a/libs/partners/decodo/tests/unit_tests/test_tools.py b/libs/partners/decodo/tests/unit_tests/test_tools.py index 03f3108d4..eed49da69 100644 --- a/libs/partners/decodo/tests/unit_tests/test_tools.py +++ b/libs/partners/decodo/tests/unit_tests/test_tools.py @@ -80,7 +80,7 @@ def test_custom_base_url(self) -> None: def test_missing_token_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("DECODO_API_TOKEN", raising=False) tool = DecodoWebScrapeTool() - with pytest.raises(ValueError, match="Decodo API token"): + with pytest.raises(ValueError, match="Decodo API key or token"): tool._run("https://example.com") @@ -184,7 +184,7 @@ def test_default_engine(self) -> None: def test_missing_token_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("DECODO_API_TOKEN", raising=False) tool = DecodoSearchTool() - with pytest.raises(ValueError, match="Decodo API token"): + with pytest.raises(ValueError, match="Decodo API key or token"): tool._run("python web scraping")