From 0ccf5a2f2d188507c1f5def0458b83f12dafcf6f Mon Sep 17 00:00:00 2001 From: "aiste.grigaliunaite@decodo.com" Date: Mon, 21 Sep 2026 16:29:35 +0300 Subject: [PATCH] support api key --- pyproject.toml | 2 +- src/decodo/__init__.py | 3 +- src/decodo/api/web_scraping_api.py | 29 ++++++++--- src/decodo/client.py | 57 ++++++++++++++++++--- src/decodo/http.py | 2 +- tests/test_client.py | 79 ++++++++++++++++++++++++++++++ tests/test_http.py | 32 +++++++++++- 7 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 tests/test_client.py diff --git a/pyproject.toml b/pyproject.toml index b6a7582..6235c8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decodo-sdk" -version = "2.2.0" +version = "2.3.0" description = "Official Python SDK for the Decodo APIs" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index dc042a4..ba0ec2e 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -1,4 +1,4 @@ -from .api.web_scraping_api import WebScrapingApi +from .api.web_scraping_api import WebScrapingApi, WebScrapingApiRoutes from .client import DecodoClient, DecodoConfig, WebScrapingApiConfig from .errors import ( AuthenticationError, @@ -268,6 +268,7 @@ def __getattr__(name: str) -> object: "DecodoConfig", "WebScrapingApiConfig", "WebScrapingApi", + "WebScrapingApiRoutes", "BundledSchema", "RemoteSchema", "DecodoSchema", diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index ef6ca5a..aca5786 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast import jsonschema @@ -22,6 +23,16 @@ ) +@dataclass(frozen=True) +class WebScrapingApiRoutes: + scrape: str + task: str + + +SCRAPER_API_ROUTES = WebScrapingApiRoutes(scrape="/v2/scrape", task="/v3/task") +DATA_API_ROUTES = WebScrapingApiRoutes(scrape="/v1/scrape", task="/v1/task") + + def _to_payload(params: ScrapeRequest | BatchRequest | Mapping[str, Any]) -> dict[str, Any]: if isinstance(params, BaseModel): return params.model_dump(by_alias=True, exclude_none=True, mode="json") @@ -29,9 +40,15 @@ def _to_payload(params: ScrapeRequest | BatchRequest | Mapping[str, Any]) -> dic class WebScrapingApi: - def __init__(self, http: HttpClient, schema: DecodoSchema = BundledSchema.shared) -> None: + def __init__( + self, + http: HttpClient, + schema: DecodoSchema = BundledSchema.shared, + routes: WebScrapingApiRoutes = SCRAPER_API_ROUTES, + ) -> None: self._http = http self._schema = schema + self._routes = routes def _validate(self, payload: dict[str, Any]) -> None: if self._schema is None: @@ -55,20 +72,20 @@ def _validate(self, payload: dict[str, Any]) -> None: def scrape(self, params: ScrapeRequest | Mapping[str, Any]) -> SyncResponse: payload = _to_payload(params) self._validate(payload) - return cast(SyncResponse, self._http.post("/v2/scrape", payload)) + return cast(SyncResponse, self._http.post(self._routes.scrape, payload)) def scrape_async(self, params: ScrapeRequest | Mapping[str, Any]) -> AsyncTaskResponse: payload = _to_payload(params) self._validate(payload) - return cast(AsyncTaskResponse, self._http.post("/v3/task", payload)) + return cast(AsyncTaskResponse, self._http.post(self._routes.task, payload)) def scrape_batch(self, params: BatchRequest | Mapping[str, Any]) -> BatchResponse: payload = _to_payload(params) - return cast(BatchResponse, self._http.post("/v3/task/batch", payload)) + return cast(BatchResponse, self._http.post(f"{self._routes.task}/batch", payload)) def get_status(self, task_id: str) -> TaskMetadata: - return cast(TaskMetadata, self._http.get(f"/v3/task/{task_id}")) + return cast(TaskMetadata, self._http.get(f"{self._routes.task}/{task_id}")) def get_results(self, task_id: str) -> TaskResultsResponse | None: - result = self._http.get(f"/v3/task/{task_id}/results") + result = self._http.get(f"{self._routes.task}/{task_id}/results") return cast(TaskResultsResponse, result) if result is not None else None diff --git a/src/decodo/client.py b/src/decodo/client.py index 08744a3..bc381bf 100644 --- a/src/decodo/client.py +++ b/src/decodo/client.py @@ -3,18 +3,25 @@ from dataclasses import dataclass from typing import Any -from .api.web_scraping_api import WebScrapingApi -from .http import BasicAuth, HttpClient, HttpClientConfig +from .api.web_scraping_api import ( + DATA_API_ROUTES, + SCRAPER_API_ROUTES, + WebScrapingApi, + WebScrapingApiRoutes, +) +from .http import ApiKeyAuth, BasicAuth, HttpClient, HttpClientConfig from .schema.bundled_schema import BundledSchema from .schema.types import DecodoSchema WEB_API_BASE_URL = "https://scraper-api.decodo.com" +DATA_API_BASE_URL = "https://data.decodo.com" DEFAULT_TIMEOUT_MS = 180_000 @dataclass class WebScrapingApiConfig: - token: str + token: str | None = None + api_key: str | None = None integration_header: str | None = None @@ -25,6 +32,43 @@ class DecodoConfig: schema: DecodoSchema | None = None +@dataclass(frozen=True) +class _Transport: + base_url: str + auth: BasicAuth | ApiKeyAuth + routes: WebScrapingApiRoutes + + +def _present(value: str | None) -> str | None: + return value if value is not None and value.strip() else None + + +def _resolve_transport(config: WebScrapingApiConfig) -> _Transport: + token = _present(config.token) + api_key = _present(config.api_key) + + if token is not None and api_key is not None: + raise ValueError( + "web_scraping_api accepts either token or api_key, not both. Provide only one." + ) + + if api_key is not None: + return _Transport( + base_url=DATA_API_BASE_URL, + auth=ApiKeyAuth(type="apiKey", api_key=api_key), + routes=DATA_API_ROUTES, + ) + + if token is not None: + return _Transport( + base_url=WEB_API_BASE_URL, + auth=BasicAuth(type="basic", token=token), + routes=SCRAPER_API_ROUTES, + ) + + raise ValueError("web_scraping_api requires token in DecodoConfig.") + + def _not_configured(namespace: str, hint: str) -> Any: raise RuntimeError(f"{namespace} is not configured. {hint}") @@ -45,14 +89,15 @@ def __init__(self, config: DecodoConfig) -> None: schema: DecodoSchema = config.schema if config.schema is not None else BundledSchema.shared if config.web_scraping_api is not None: + transport = _resolve_transport(config.web_scraping_api) http = HttpClient( HttpClientConfig( - base_url=WEB_API_BASE_URL, - auth=BasicAuth(type="basic", token=config.web_scraping_api.token), + base_url=transport.base_url, + auth=transport.auth, timeout_ms=timeout_ms, integration_header=config.web_scraping_api.integration_header, ) ) - self.web_scraping_api = WebScrapingApi(http, schema) + self.web_scraping_api = WebScrapingApi(http, schema, transport.routes) else: self.web_scraping_api = _UnconfiguredWebScrapingApi() # type: ignore[assignment] diff --git a/src/decodo/http.py b/src/decodo/http.py index 3c76f35..cf6afef 100644 --- a/src/decodo/http.py +++ b/src/decodo/http.py @@ -46,7 +46,7 @@ def __init__(self, config: HttpClientConfig) -> None: self._auth_header = f"Basic {config.auth.token}" else: assert isinstance(config.auth, ApiKeyAuth) - self._auth_header = config.auth.api_key + self._auth_header = f"Bearer {config.auth.api_key}" def request(self, method: str, path: str, body: Any = None) -> Any: url = f"{self._base_url}{path}" diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..8e80cff --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from decodo.client import DecodoClient, DecodoConfig, WebScrapingApiConfig + +SCRAPE_PARAMS = {"target": "universal", "url": "https://example.com"} + + +def _make_response(body: Any, status: int = 200) -> MagicMock: + mock = MagicMock(spec=httpx.Response) + mock.status_code = status + mock.is_success = 200 <= status < 300 + mock.json.return_value = body + return mock + + +class TestTransportSelection: + def _collect_urls(self, config: WebScrapingApiConfig) -> list[str]: + urls: list[str] = [] + + def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock: + urls.append(url) + return _make_response({"results": []}) + + with patch("httpx.request", side_effect=fake_request): + client = DecodoClient(DecodoConfig(web_scraping_api=config)) + client.web_scraping_api.scrape(SCRAPE_PARAMS) + client.web_scraping_api.get_status("task-1") + + return urls + + def test_routes_a_token_to_scraper_api_over_v2_v3(self) -> None: + assert self._collect_urls(WebScrapingApiConfig(token="test-token")) == [ + "https://scraper-api.decodo.com/v2/scrape", + "https://scraper-api.decodo.com/v3/task/task-1", + ] + + def test_routes_an_api_key_to_data_decodo_com_over_v1(self) -> None: + assert self._collect_urls(WebScrapingApiConfig(api_key="test-key")) == [ + "https://data.decodo.com/v1/scrape", + "https://data.decodo.com/v1/task/task-1", + ] + + def test_raises_when_both_token_and_api_key_are_provided(self) -> None: + with pytest.raises(ValueError, match="either token or api_key"): + DecodoClient( + DecodoConfig( + web_scraping_api=WebScrapingApiConfig( + token="test-token", api_key="test-key" + ) + ) + ) + + def test_raises_when_neither_token_nor_api_key_is_provided(self) -> None: + with pytest.raises(ValueError, match="web_scraping_api requires"): + DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig())) + + @pytest.mark.parametrize("blank", ["", " "]) + def test_raises_when_the_only_credential_is_blank(self, blank: str) -> None: + with pytest.raises(ValueError, match="web_scraping_api requires"): + DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig(token=blank))) + with pytest.raises(ValueError, match="web_scraping_api requires"): + DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig(api_key=blank))) + + @pytest.mark.parametrize("blank", ["", " "]) + def test_blank_credential_does_not_shadow_the_other_one(self, blank: str) -> None: + assert self._collect_urls(WebScrapingApiConfig(token=blank, api_key="test-key")) == [ + "https://data.decodo.com/v1/scrape", + "https://data.decodo.com/v1/task/task-1", + ] + assert self._collect_urls(WebScrapingApiConfig(token="test-token", api_key=blank)) == [ + "https://scraper-api.decodo.com/v2/scrape", + "https://scraper-api.decodo.com/v3/task/task-1", + ] diff --git a/tests/test_http.py b/tests/test_http.py index ea75c6d..2323819 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -5,7 +5,7 @@ import httpx -from decodo.http import BasicAuth, HttpClient, HttpClientConfig +from decodo.http import ApiKeyAuth, BasicAuth, HttpClient, HttpClientConfig BASE_CONFIG = HttpClientConfig( base_url="https://api.test", @@ -55,3 +55,33 @@ def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock: client.get("/v3/task/1") assert captured["headers"]["x-integration"] == "cli" + + +class TestHttpClientAuthHeader: + def _captured_headers(self, config: HttpClientConfig) -> dict[str, str]: + captured: dict[str, dict[str, str]] = {} + + def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock: + captured["headers"] = kwargs.get("headers", {}) + return _make_response({"ok": True}) + + with patch("httpx.request", side_effect=fake_request): + HttpClient(config).get("/v3/task/1") + + return captured["headers"] + + def test_sends_basic_auth_for_a_token(self) -> None: + headers = self._captured_headers(BASE_CONFIG) + + assert headers["Authorization"] == "Basic test-token" + + def test_sends_bearer_auth_for_an_api_key(self) -> None: + headers = self._captured_headers( + HttpClientConfig( + base_url="https://api.test", + auth=ApiKeyAuth(type="apiKey", api_key="test-key"), + timeout_ms=5000, + ) + ) + + assert headers["Authorization"] == "Bearer test-key"