From e027ec3ee8e2d162d8f0eb26c20b2ece7b2e163e Mon Sep 17 00:00:00 2001 From: Vedaant Singh Date: Tue, 8 Sep 2026 08:36:28 +0100 Subject: [PATCH 1/2] fix: browser honors user_agent; crypto errors on unresolved query; stock rejects bad mode (1.6.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes found sweeping the scrapers and browser backend: - Browser backend ignored config.user_agent: with render_js=True it always sent user_agents[0], and screenshot() sent no UA at all. Both now share the HTTP backend's contract via a new ScraperConfig.pick_user_agent() (override wins, else rotate). Fixes a real config divergence between the two backends. - CryptoScraper: a query whose terms all failed to resolve to a CoinGecko id dropped the ids filter and returned the unfiltered top market — wrong data, no error. It now returns an error result instead (sync + async). - StockScraper: an unrecognized mode silently coerced to a quote; it now returns a ScrapeError naming the allowed modes (sync + async), mirroring IMDB. Regression-proven tests for each; 597 passed, ruff clean. Bump to 1.6.3. --- CHANGELOG.md | 9 ++++++++ pyproject.toml | 2 +- server.json | 4 ++-- src/pyscrappy/__init__.py | 2 +- src/pyscrappy/core/browser.py | 6 ++++-- src/pyscrappy/core/config.py | 11 ++++++++++ src/pyscrappy/core/http.py | 7 +++--- src/pyscrappy/scrapers/crypto.py | 7 ++++++ src/pyscrappy/scrapers/stock.py | 22 ++++++++++++++++++- tests/test_core/test_browser.py | 31 +++++++++++++++++++++++++++ tests/test_core/test_config.py | 13 +++++++++++ tests/test_scrapers/test_data_apis.py | 11 ++++++++++ tests/test_scrapers/test_stock.py | 8 +++++++ 13 files changed, 122 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f5eaf..9737300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ All notable changes to PyScrappy are documented here. The format is based on ## [Unreleased] +## [1.6.3] - 2026-09-01 + +### Fixed +- **The browser backend now honors `config.user_agent`.** With `render_js=True`, a configured single `user_agent` was silently ignored (the browser always sent `user_agents[0]`), and `screenshot()` sent no User-Agent at all (Playwright's headless-Chrome default). Both now use the same UA contract as the HTTP backend via `ScraperConfig.pick_user_agent()` — a configured `user_agent` overrides, otherwise the `user_agents` list is rotated. +- **Crypto: a specific-coin query that resolves to nothing now errors instead of returning the top market.** `get_crypto(query=...)` where none of the terms resolved to a CoinGecko id (a typo, an unknown coin, or a failing search) silently dropped the `ids` filter and returned the unfiltered top coins by market cap. It now returns an empty `ScrapeResult` with a "No coins matched query" error, so a caller can tell the lookup failed. + +### Added +- **Stock: unknown `mode` values are rejected.** `scrape_stock(mode=...)` with an unrecognized mode (e.g. a `"quotes"` typo) previously fell through to a quote silently; it now returns a `ScrapeError` naming the allowed modes (`quote`, `history`, `profile`). + ## [1.6.2] - 2026-09-01 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 6a6e210..c53284f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pyscrappy" -version = "1.6.2" +version = "1.6.3" description = "A robust, all-in-one Python web scraping toolkit" readme = "README.md" license = "MIT" diff --git a/server.json b/server.json index 8b4ad09..9f5dd20 100644 --- a/server.json +++ b/server.json @@ -7,13 +7,13 @@ "url": "https://github.com/mldsveda/PyScrappy", "source": "github" }, - "version": "1.6.2", + "version": "1.6.3", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "pyscrappy", - "version": "1.6.2", + "version": "1.6.3", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/pyscrappy/__init__.py b/src/pyscrappy/__init__.py index 8f8b4cf..2b6e3b0 100644 --- a/src/pyscrappy/__init__.py +++ b/src/pyscrappy/__init__.py @@ -103,7 +103,7 @@ register(_cls.name, _cls) del _cls -__version__ = "1.6.2" +__version__ = "1.6.3" __all__ = [ # Core diff --git a/src/pyscrappy/core/browser.py b/src/pyscrappy/core/browser.py index 5c25970..abbc8aa 100644 --- a/src/pyscrappy/core/browser.py +++ b/src/pyscrappy/core/browser.py @@ -82,7 +82,9 @@ def get_html( self._start() timeout_ms = int((wait_timeout or self.config.timeout) * 1000) - ua = self.config.user_agents[0] if self.config.user_agents else None + # Honor the same UA contract as the HTTP backend: a configured + # user_agent overrides, otherwise rotate through user_agents. + ua = self.config.pick_user_agent() context = self._browser.new_context(user_agent=ua) page = context.new_page() @@ -103,7 +105,7 @@ def screenshot(self, url: str, path: str, full_page: bool = True) -> None: if not self._browser: self._start() - context = self._browser.new_context() + context = self._browser.new_context(user_agent=self.config.pick_user_agent()) page = context.new_page() try: page.goto(url, wait_until="networkidle", timeout=int(self.config.timeout * 1000)) diff --git a/src/pyscrappy/core/config.py b/src/pyscrappy/core/config.py index d676020..7818214 100644 --- a/src/pyscrappy/core/config.py +++ b/src/pyscrappy/core/config.py @@ -135,3 +135,14 @@ def pick_proxy(self, exclude: str | None = None) -> str | None: if not choices: return random.choice(self.proxy) if self.proxy else None return random.choice(choices) + + def pick_user_agent(self) -> str | None: + """Return the User-Agent to send: a configured single ``user_agent`` + overrides ``user_agents`` rotation, otherwise one is chosen at random from + ``user_agents`` (or None if that list is empty). Shared by the HTTP and + browser backends so both honor the same contract.""" + import random + + if self.user_agent: + return self.user_agent + return random.choice(self.user_agents) if self.user_agents else None diff --git a/src/pyscrappy/core/http.py b/src/pyscrappy/core/http.py index 4945239..e8b6d19 100644 --- a/src/pyscrappy/core/http.py +++ b/src/pyscrappy/core/http.py @@ -525,10 +525,9 @@ def _ensure_client(self) -> httpx.Client | StealthClient: return self._client def _pick_ua(self) -> str: - # A configured single user_agent overrides rotation. - if self.config.user_agent: - return self.config.user_agent - return random.choice(self.config.user_agents) + # A configured single user_agent overrides rotation (shared with the + # browser backend via ScraperConfig.pick_user_agent). + return self.config.pick_user_agent() or random.choice(self.config.user_agents) def _merge_headers( self, extra: dict[str, str], user_agent: str | None = None diff --git a/src/pyscrappy/scrapers/crypto.py b/src/pyscrappy/scrapers/crypto.py index 9d68d21..a513457 100644 --- a/src/pyscrappy/scrapers/crypto.py +++ b/src/pyscrappy/scrapers/crypto.py @@ -56,6 +56,10 @@ def scrape( # type: ignore[override] ScrapeResult with coin data (name, symbol, price, market cap, …). """ ids = self._resolve_ids(query) if query else None + if query and not ids: + # A specific-coin query that resolved to nothing must not silently fall + # through to the unfiltered top-market list — report it instead. + return self._err(_MARKETS, f"No coins matched query {query!r}.") url = self._build_markets_url(vs_currency, max_results, ids) try: @@ -73,6 +77,9 @@ async def scrape_async( # type: ignore[override] ) -> ScrapeResult: """Async counterpart to :meth:`scrape` (same args/returns).""" ids = await self._resolve_ids_async(query) if query else None + if query and not ids: + # See scrape(): don't fall through to the unfiltered top-market list. + return self._err(_MARKETS, f"No coins matched query {query!r}.") url = self._build_markets_url(vs_currency, max_results, ids) try: diff --git a/src/pyscrappy/scrapers/stock.py b/src/pyscrappy/scrapers/stock.py index c84cc06..ed7eff4 100644 --- a/src/pyscrappy/scrapers/stock.py +++ b/src/pyscrappy/scrapers/stock.py @@ -8,7 +8,23 @@ from pyscrappy.core.base import BaseScraper from pyscrappy.core.config import ScraperConfig from pyscrappy.core.exceptions import NetworkError -from pyscrappy.core.models import ScrapeMetadata, ScrapeResult +from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult + +_MODES = ("quote", "history", "profile") + + +def _invalid_mode_result(mode: str, scraper_name: str) -> ScrapeResult: + return ScrapeResult( + data=[], + metadata=ScrapeMetadata(scraper=scraper_name), + errors=[ + ScrapeError( + url=_YF_BASE, + message=f"Unknown mode {mode!r}. Use one of: {', '.join(_MODES)}.", + ) + ], + ) + _YF_BASE = "https://query1.finance.yahoo.com" @@ -61,6 +77,8 @@ def scrape( # type: ignore[override] ScrapeResult with stock data. """ symbol = symbol.upper().strip() + if mode not in _MODES: + return _invalid_mode_result(mode, self.name) if mode == "history": return self._scrape_history(symbol, period, interval) @@ -78,6 +96,8 @@ async def scrape_async( # type: ignore[override] ) -> ScrapeResult: """Async counterpart to :meth:`scrape` (same args/returns).""" symbol = symbol.upper().strip() + if mode not in _MODES: + return _invalid_mode_result(mode, self.name) if mode == "history": url = f"{_YF_BASE}/v8/finance/chart/{symbol}?range={period}&interval={interval}" diff --git a/tests/test_core/test_browser.py b/tests/test_core/test_browser.py index fb097fb..b79efee 100644 --- a/tests/test_core/test_browser.py +++ b/tests/test_core/test_browser.py @@ -109,6 +109,22 @@ def test_get_html_with_scroll(self): assert mock_page.evaluate.call_count == 3 assert mock_page.wait_for_timeout.call_count == 3 + def test_get_html_honors_configured_user_agent(self): + # A configured single user_agent must be passed to the browser context, + # not silently ignored in favor of user_agents[0]. + bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0")) + mock_page = MagicMock() + mock_page.content.return_value = "" + mock_context = MagicMock() + mock_context.new_page.return_value = mock_page + mock_browser = MagicMock() + mock_browser.new_context.return_value = mock_context + bm._browser = mock_browser + + bm.get_html("https://example.com") + + mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0") + def test_get_html_cleans_up_on_error(self): bm = BrowserManager() mock_page = MagicMock() @@ -143,6 +159,21 @@ def test_screenshot_calls_browser(self): mock_page.close.assert_called_once() mock_context.close.assert_called_once() + def test_screenshot_honors_configured_user_agent(self): + # screenshot() previously created a context with no UA (headless default); + # it must now pass the configured user_agent like get_html. + bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0")) + mock_page = MagicMock() + mock_context = MagicMock() + mock_context.new_page.return_value = mock_page + mock_browser = MagicMock() + mock_browser.new_context.return_value = mock_context + bm._browser = mock_browser + + bm.screenshot("https://example.com", "/tmp/test.png") + + mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0") + def test_screenshot_partial_page(self): bm = BrowserManager() mock_page = MagicMock() diff --git a/tests/test_core/test_config.py b/tests/test_core/test_config.py index f1d2d74..27750a4 100644 --- a/tests/test_core/test_config.py +++ b/tests/test_core/test_config.py @@ -54,6 +54,19 @@ def test_custom_user_agents(self): config = ScraperConfig(user_agents=agents) assert config.user_agents == agents + def test_pick_user_agent_override_wins(self): + # A single configured user_agent overrides the rotation list. + config = ScraperConfig(user_agent="MyBot/1.0", user_agents=["A/1", "B/2"]) + assert config.pick_user_agent() == "MyBot/1.0" + + def test_pick_user_agent_rotates_when_no_override(self): + config = ScraperConfig(user_agents=["A/1", "B/2"]) + assert config.pick_user_agent() in {"A/1", "B/2"} + + def test_pick_user_agent_none_when_no_agents(self): + config = ScraperConfig(user_agent=None, user_agents=[]) + assert config.pick_user_agent() is None + def test_render_js_auto(self): config = ScraperConfig(render_js="auto") assert config.render_js == "auto" diff --git a/tests/test_scrapers/test_data_apis.py b/tests/test_scrapers/test_data_apis.py index 61734de..d681720 100644 --- a/tests/test_scrapers/test_data_apis.py +++ b/tests/test_scrapers/test_data_apis.py @@ -53,6 +53,17 @@ def test_query_resolves_ids(self): # the markets URL should carry the resolved id assert "ids=ethereum" in s._http.get_html.call_args[0][0] + def test_unresolvable_query_errors_not_falls_back_to_top_market(self): + # A specific query that resolves to nothing (typo / unknown coin) must + # error, not silently return the unfiltered top-market list. + search = json.dumps({"coins": []}) # /search finds nothing + s = _mock(CryptoScraper(), search) + r = s.scrape(query="bitcon") # typo + assert r.data == [] + assert r.errors and "No coins matched" in r.errors[0].message + # It must NOT have fetched the markets endpoint after the empty search. + assert s._http.get_html.call_count == 1 + class TestCurrency: def test_rates_and_conversion(self): diff --git a/tests/test_scrapers/test_stock.py b/tests/test_scrapers/test_stock.py index ad9e0f0..7505986 100644 --- a/tests/test_scrapers/test_stock.py +++ b/tests/test_scrapers/test_stock.py @@ -115,6 +115,14 @@ def test_symbol_uppercased(self): assert "AAPL" in call_url scraper.close() + def test_unknown_mode_errors_instead_of_silently_quoting(self): + # A typo'd mode previously fell through to a quote; it must now error. + scraper = StockScraper() + result = scraper.scrape(symbol="AAPL", mode="quotes") # typo + assert result.data == [] + assert result.errors and "Unknown mode" in result.errors[0].message + scraper.close() + class TestStockScraperHistory: def test_scrape_history(self): From fca798bdbc6ec534ab65c5af9eb13d997604fefd Mon Sep 17 00:00:00 2001 From: Vedaant Singh Date: Tue, 8 Sep 2026 08:56:55 +0100 Subject: [PATCH 2/2] http: _pick_ua no longer crashes on empty user_agents; omit UA header instead Address review of #183: _pick_ua's fallback did random.choice(user_agents), which raises IndexError when ScraperConfig(user_agent=None, user_agents=[]). Delegate fully to ScraperConfig.pick_user_agent() (no duplicated random.choice) and have _merge_headers omit the User-Agent header entirely when none is configured, rather than setting it to None. Bring the async client onto the same contract (it had the identical latent crash + duplication) and drop its now-unused random import. --- src/pyscrappy/core/async_http.py | 14 ++++++++------ src/pyscrappy/core/http.py | 12 ++++++++---- tests/test_core/test_http.py | 8 ++++++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/pyscrappy/core/async_http.py b/src/pyscrappy/core/async_http.py index 5f2e527..d0b0633 100644 --- a/src/pyscrappy/core/async_http.py +++ b/src/pyscrappy/core/async_http.py @@ -12,7 +12,6 @@ import asyncio import logging -import random import time from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlparse @@ -244,16 +243,19 @@ def _ensure_client(self) -> httpx.AsyncClient | AsyncStealthClient: self._client = self._build_client() return self._client - def _pick_ua(self) -> str: - if self.config.user_agent: - return self.config.user_agent - return random.choice(self.config.user_agents) + def _pick_ua(self) -> str | None: + # Shared contract with the sync client and browser backend via + # ScraperConfig.pick_user_agent(); None only if no UA is configured at all. + return self.config.pick_user_agent() def _merge_headers( self, extra: dict[str, str], user_agent: str | None = None ) -> dict[str, str]: ua = user_agent or self._pick_ua() - return {**self.config.headers, "User-Agent": ua, **extra} + headers = {**self.config.headers} + if ua: # omit the header entirely if no UA is configured (user_agents=[]) + headers["User-Agent"] = ua + return {**headers, **extra} # Cache is shared with the sync client (same module-level store + lock). diff --git a/src/pyscrappy/core/http.py b/src/pyscrappy/core/http.py index e8b6d19..157a474 100644 --- a/src/pyscrappy/core/http.py +++ b/src/pyscrappy/core/http.py @@ -524,10 +524,11 @@ def _ensure_client(self) -> httpx.Client | StealthClient: self._client = self._build_client() return self._client - def _pick_ua(self) -> str: + def _pick_ua(self) -> str | None: # A configured single user_agent overrides rotation (shared with the - # browser backend via ScraperConfig.pick_user_agent). - return self.config.pick_user_agent() or random.choice(self.config.user_agents) + # browser backend via ScraperConfig.pick_user_agent). Returns None only + # if both user_agent and user_agents are empty. + return self.config.pick_user_agent() def _merge_headers( self, extra: dict[str, str], user_agent: str | None = None @@ -535,7 +536,10 @@ def _merge_headers( """Build the request headers: config.headers (lowest priority), then the chosen User-Agent, then per-call headers (highest priority).""" ua = user_agent or self._pick_ua() - return {**self.config.headers, "User-Agent": ua, **extra} + headers = {**self.config.headers} + if ua: # omit the header entirely if no UA is configured (user_agents=[]) + headers["User-Agent"] = ua + return {**headers, **extra} def _backoff_delay(self, attempt: int) -> float: """Retry delay for this client's config (see module-level backoff_delay).""" diff --git a/tests/test_core/test_http.py b/tests/test_core/test_http.py index 4d0208a..44e8f60 100644 --- a/tests/test_core/test_http.py +++ b/tests/test_core/test_http.py @@ -430,6 +430,14 @@ def test_config_user_agent_overrides_rotation(self): client = HttpClient(config) assert {client._pick_ua() for _ in range(20)} == {"Custom/9.9"} + def test_no_user_agents_does_not_crash_and_omits_header(self): + # user_agents=[] with no override must not raise (previously IndexError via + # random.choice([])); _merge_headers then omits the User-Agent entirely. + config = ScraperConfig(user_agent=None, user_agents=[]) + client = HttpClient(config) + assert client._pick_ua() is None + assert "User-Agent" not in client._merge_headers({}) + class TestHttpClientCustomHeaders: def test_config_headers_are_sent(self):