diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f5eaf..9737300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to PyScrappy are documented here. The format is based on ## [Unreleased] +## [1.6.3] - 2026-09-01 + +### Fixed +- **The browser backend now honors `config.user_agent`.** With `render_js=True`, a configured single `user_agent` was silently ignored (the browser always sent `user_agents[0]`), and `screenshot()` sent no User-Agent at all (Playwright's headless-Chrome default). Both now use the same UA contract as the HTTP backend via `ScraperConfig.pick_user_agent()` — a configured `user_agent` overrides, otherwise the `user_agents` list is rotated. +- **Crypto: a specific-coin query that resolves to nothing now errors instead of returning the top market.** `get_crypto(query=...)` where none of the terms resolved to a CoinGecko id (a typo, an unknown coin, or a failing search) silently dropped the `ids` filter and returned the unfiltered top coins by market cap. It now returns an empty `ScrapeResult` with a "No coins matched query" error, so a caller can tell the lookup failed. + +### Added +- **Stock: unknown `mode` values are rejected.** `scrape_stock(mode=...)` with an unrecognized mode (e.g. a `"quotes"` typo) previously fell through to a quote silently; it now returns a `ScrapeError` naming the allowed modes (`quote`, `history`, `profile`). + ## [1.6.2] - 2026-09-01 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 6a6e210..c53284f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pyscrappy" -version = "1.6.2" +version = "1.6.3" description = "A robust, all-in-one Python web scraping toolkit" readme = "README.md" license = "MIT" diff --git a/server.json b/server.json index 8b4ad09..9f5dd20 100644 --- a/server.json +++ b/server.json @@ -7,13 +7,13 @@ "url": "https://github.com/mldsveda/PyScrappy", "source": "github" }, - "version": "1.6.2", + "version": "1.6.3", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "pyscrappy", - "version": "1.6.2", + "version": "1.6.3", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/pyscrappy/__init__.py b/src/pyscrappy/__init__.py index 8f8b4cf..2b6e3b0 100644 --- a/src/pyscrappy/__init__.py +++ b/src/pyscrappy/__init__.py @@ -103,7 +103,7 @@ register(_cls.name, _cls) del _cls -__version__ = "1.6.2" +__version__ = "1.6.3" __all__ = [ # Core diff --git a/src/pyscrappy/core/async_http.py b/src/pyscrappy/core/async_http.py index 5f2e527..d0b0633 100644 --- a/src/pyscrappy/core/async_http.py +++ b/src/pyscrappy/core/async_http.py @@ -12,7 +12,6 @@ import asyncio import logging -import random import time from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlparse @@ -244,16 +243,19 @@ def _ensure_client(self) -> httpx.AsyncClient | AsyncStealthClient: self._client = self._build_client() return self._client - def _pick_ua(self) -> str: - if self.config.user_agent: - return self.config.user_agent - return random.choice(self.config.user_agents) + def _pick_ua(self) -> str | None: + # Shared contract with the sync client and browser backend via + # ScraperConfig.pick_user_agent(); None only if no UA is configured at all. + return self.config.pick_user_agent() def _merge_headers( self, extra: dict[str, str], user_agent: str | None = None ) -> dict[str, str]: ua = user_agent or self._pick_ua() - return {**self.config.headers, "User-Agent": ua, **extra} + headers = {**self.config.headers} + if ua: # omit the header entirely if no UA is configured (user_agents=[]) + headers["User-Agent"] = ua + return {**headers, **extra} # Cache is shared with the sync client (same module-level store + lock). diff --git a/src/pyscrappy/core/browser.py b/src/pyscrappy/core/browser.py index 5c25970..abbc8aa 100644 --- a/src/pyscrappy/core/browser.py +++ b/src/pyscrappy/core/browser.py @@ -82,7 +82,9 @@ def get_html( self._start() timeout_ms = int((wait_timeout or self.config.timeout) * 1000) - ua = self.config.user_agents[0] if self.config.user_agents else None + # Honor the same UA contract as the HTTP backend: a configured + # user_agent overrides, otherwise rotate through user_agents. + ua = self.config.pick_user_agent() context = self._browser.new_context(user_agent=ua) page = context.new_page() @@ -103,7 +105,7 @@ def screenshot(self, url: str, path: str, full_page: bool = True) -> None: if not self._browser: self._start() - context = self._browser.new_context() + context = self._browser.new_context(user_agent=self.config.pick_user_agent()) page = context.new_page() try: page.goto(url, wait_until="networkidle", timeout=int(self.config.timeout * 1000)) diff --git a/src/pyscrappy/core/config.py b/src/pyscrappy/core/config.py index d676020..7818214 100644 --- a/src/pyscrappy/core/config.py +++ b/src/pyscrappy/core/config.py @@ -135,3 +135,14 @@ def pick_proxy(self, exclude: str | None = None) -> str | None: if not choices: return random.choice(self.proxy) if self.proxy else None return random.choice(choices) + + def pick_user_agent(self) -> str | None: + """Return the User-Agent to send: a configured single ``user_agent`` + overrides ``user_agents`` rotation, otherwise one is chosen at random from + ``user_agents`` (or None if that list is empty). Shared by the HTTP and + browser backends so both honor the same contract.""" + import random + + if self.user_agent: + return self.user_agent + return random.choice(self.user_agents) if self.user_agents else None diff --git a/src/pyscrappy/core/http.py b/src/pyscrappy/core/http.py index 4945239..157a474 100644 --- a/src/pyscrappy/core/http.py +++ b/src/pyscrappy/core/http.py @@ -524,11 +524,11 @@ def _ensure_client(self) -> httpx.Client | StealthClient: self._client = self._build_client() return self._client - def _pick_ua(self) -> str: - # A configured single user_agent overrides rotation. - if self.config.user_agent: - return self.config.user_agent - return random.choice(self.config.user_agents) + def _pick_ua(self) -> str | None: + # A configured single user_agent overrides rotation (shared with the + # browser backend via ScraperConfig.pick_user_agent). Returns None only + # if both user_agent and user_agents are empty. + return self.config.pick_user_agent() def _merge_headers( self, extra: dict[str, str], user_agent: str | None = None @@ -536,7 +536,10 @@ def _merge_headers( """Build the request headers: config.headers (lowest priority), then the chosen User-Agent, then per-call headers (highest priority).""" ua = user_agent or self._pick_ua() - return {**self.config.headers, "User-Agent": ua, **extra} + headers = {**self.config.headers} + if ua: # omit the header entirely if no UA is configured (user_agents=[]) + headers["User-Agent"] = ua + return {**headers, **extra} def _backoff_delay(self, attempt: int) -> float: """Retry delay for this client's config (see module-level backoff_delay).""" diff --git a/src/pyscrappy/scrapers/crypto.py b/src/pyscrappy/scrapers/crypto.py index 9d68d21..a513457 100644 --- a/src/pyscrappy/scrapers/crypto.py +++ b/src/pyscrappy/scrapers/crypto.py @@ -56,6 +56,10 @@ def scrape( # type: ignore[override] ScrapeResult with coin data (name, symbol, price, market cap, …). """ ids = self._resolve_ids(query) if query else None + if query and not ids: + # A specific-coin query that resolved to nothing must not silently fall + # through to the unfiltered top-market list — report it instead. + return self._err(_MARKETS, f"No coins matched query {query!r}.") url = self._build_markets_url(vs_currency, max_results, ids) try: @@ -73,6 +77,9 @@ async def scrape_async( # type: ignore[override] ) -> ScrapeResult: """Async counterpart to :meth:`scrape` (same args/returns).""" ids = await self._resolve_ids_async(query) if query else None + if query and not ids: + # See scrape(): don't fall through to the unfiltered top-market list. + return self._err(_MARKETS, f"No coins matched query {query!r}.") url = self._build_markets_url(vs_currency, max_results, ids) try: diff --git a/src/pyscrappy/scrapers/stock.py b/src/pyscrappy/scrapers/stock.py index c84cc06..ed7eff4 100644 --- a/src/pyscrappy/scrapers/stock.py +++ b/src/pyscrappy/scrapers/stock.py @@ -8,7 +8,23 @@ from pyscrappy.core.base import BaseScraper from pyscrappy.core.config import ScraperConfig from pyscrappy.core.exceptions import NetworkError -from pyscrappy.core.models import ScrapeMetadata, ScrapeResult +from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult + +_MODES = ("quote", "history", "profile") + + +def _invalid_mode_result(mode: str, scraper_name: str) -> ScrapeResult: + return ScrapeResult( + data=[], + metadata=ScrapeMetadata(scraper=scraper_name), + errors=[ + ScrapeError( + url=_YF_BASE, + message=f"Unknown mode {mode!r}. Use one of: {', '.join(_MODES)}.", + ) + ], + ) + _YF_BASE = "https://query1.finance.yahoo.com" @@ -61,6 +77,8 @@ def scrape( # type: ignore[override] ScrapeResult with stock data. """ symbol = symbol.upper().strip() + if mode not in _MODES: + return _invalid_mode_result(mode, self.name) if mode == "history": return self._scrape_history(symbol, period, interval) @@ -78,6 +96,8 @@ async def scrape_async( # type: ignore[override] ) -> ScrapeResult: """Async counterpart to :meth:`scrape` (same args/returns).""" symbol = symbol.upper().strip() + if mode not in _MODES: + return _invalid_mode_result(mode, self.name) if mode == "history": url = f"{_YF_BASE}/v8/finance/chart/{symbol}?range={period}&interval={interval}" diff --git a/tests/test_core/test_browser.py b/tests/test_core/test_browser.py index fb097fb..b79efee 100644 --- a/tests/test_core/test_browser.py +++ b/tests/test_core/test_browser.py @@ -109,6 +109,22 @@ def test_get_html_with_scroll(self): assert mock_page.evaluate.call_count == 3 assert mock_page.wait_for_timeout.call_count == 3 + def test_get_html_honors_configured_user_agent(self): + # A configured single user_agent must be passed to the browser context, + # not silently ignored in favor of user_agents[0]. + bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0")) + mock_page = MagicMock() + mock_page.content.return_value = "" + mock_context = MagicMock() + mock_context.new_page.return_value = mock_page + mock_browser = MagicMock() + mock_browser.new_context.return_value = mock_context + bm._browser = mock_browser + + bm.get_html("https://example.com") + + mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0") + def test_get_html_cleans_up_on_error(self): bm = BrowserManager() mock_page = MagicMock() @@ -143,6 +159,21 @@ def test_screenshot_calls_browser(self): mock_page.close.assert_called_once() mock_context.close.assert_called_once() + def test_screenshot_honors_configured_user_agent(self): + # screenshot() previously created a context with no UA (headless default); + # it must now pass the configured user_agent like get_html. + bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0")) + mock_page = MagicMock() + mock_context = MagicMock() + mock_context.new_page.return_value = mock_page + mock_browser = MagicMock() + mock_browser.new_context.return_value = mock_context + bm._browser = mock_browser + + bm.screenshot("https://example.com", "/tmp/test.png") + + mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0") + def test_screenshot_partial_page(self): bm = BrowserManager() mock_page = MagicMock() diff --git a/tests/test_core/test_config.py b/tests/test_core/test_config.py index f1d2d74..27750a4 100644 --- a/tests/test_core/test_config.py +++ b/tests/test_core/test_config.py @@ -54,6 +54,19 @@ def test_custom_user_agents(self): config = ScraperConfig(user_agents=agents) assert config.user_agents == agents + def test_pick_user_agent_override_wins(self): + # A single configured user_agent overrides the rotation list. + config = ScraperConfig(user_agent="MyBot/1.0", user_agents=["A/1", "B/2"]) + assert config.pick_user_agent() == "MyBot/1.0" + + def test_pick_user_agent_rotates_when_no_override(self): + config = ScraperConfig(user_agents=["A/1", "B/2"]) + assert config.pick_user_agent() in {"A/1", "B/2"} + + def test_pick_user_agent_none_when_no_agents(self): + config = ScraperConfig(user_agent=None, user_agents=[]) + assert config.pick_user_agent() is None + def test_render_js_auto(self): config = ScraperConfig(render_js="auto") assert config.render_js == "auto" diff --git a/tests/test_core/test_http.py b/tests/test_core/test_http.py index 4d0208a..44e8f60 100644 --- a/tests/test_core/test_http.py +++ b/tests/test_core/test_http.py @@ -430,6 +430,14 @@ def test_config_user_agent_overrides_rotation(self): client = HttpClient(config) assert {client._pick_ua() for _ in range(20)} == {"Custom/9.9"} + def test_no_user_agents_does_not_crash_and_omits_header(self): + # user_agents=[] with no override must not raise (previously IndexError via + # random.choice([])); _merge_headers then omits the User-Agent entirely. + config = ScraperConfig(user_agent=None, user_agents=[]) + client = HttpClient(config) + assert client._pick_ua() is None + assert "User-Agent" not in client._merge_headers({}) + class TestHttpClientCustomHeaders: def test_config_headers_are_sent(self): diff --git a/tests/test_scrapers/test_data_apis.py b/tests/test_scrapers/test_data_apis.py index 61734de..d681720 100644 --- a/tests/test_scrapers/test_data_apis.py +++ b/tests/test_scrapers/test_data_apis.py @@ -53,6 +53,17 @@ def test_query_resolves_ids(self): # the markets URL should carry the resolved id assert "ids=ethereum" in s._http.get_html.call_args[0][0] + def test_unresolvable_query_errors_not_falls_back_to_top_market(self): + # A specific query that resolves to nothing (typo / unknown coin) must + # error, not silently return the unfiltered top-market list. + search = json.dumps({"coins": []}) # /search finds nothing + s = _mock(CryptoScraper(), search) + r = s.scrape(query="bitcon") # typo + assert r.data == [] + assert r.errors and "No coins matched" in r.errors[0].message + # It must NOT have fetched the markets endpoint after the empty search. + assert s._http.get_html.call_count == 1 + class TestCurrency: def test_rates_and_conversion(self): diff --git a/tests/test_scrapers/test_stock.py b/tests/test_scrapers/test_stock.py index ad9e0f0..7505986 100644 --- a/tests/test_scrapers/test_stock.py +++ b/tests/test_scrapers/test_stock.py @@ -115,6 +115,14 @@ def test_symbol_uppercased(self): assert "AAPL" in call_url scraper.close() + def test_unknown_mode_errors_instead_of_silently_quoting(self): + # A typo'd mode previously fell through to a quote; it must now error. + scraper = StockScraper() + result = scraper.scrape(symbol="AAPL", mode="quotes") # typo + assert result.data == [] + assert result.errors and "Unknown mode" in result.errors[0].message + scraper.close() + class TestStockScraperHistory: def test_scrape_history(self):