fix: browser UA, crypto unresolved-query, stock bad-mode (1.6.3)

CHANGELOG.md

@@ -6,6 +6,15 @@ All notable changes to PyScrappy are documented here. The format is based on
 
 ## [Unreleased]
 
+## [1.6.3] - 2026-09-01
+
+### Fixed
+- **The browser backend now honors `config.user_agent`.** With `render_js=True`, a configured single `user_agent` was silently ignored (the browser always sent `user_agents[0]`), and `screenshot()` sent no User-Agent at all (Playwright's headless-Chrome default). Both now use the same UA contract as the HTTP backend via `ScraperConfig.pick_user_agent()` — a configured `user_agent` overrides, otherwise the `user_agents` list is rotated.
+- **Crypto: a specific-coin query that resolves to nothing now errors instead of returning the top market.** `get_crypto(query=...)` where none of the terms resolved to a CoinGecko id (a typo, an unknown coin, or a failing search) silently dropped the `ids` filter and returned the unfiltered top coins by market cap. It now returns an empty `ScrapeResult` with a "No coins matched query" error, so a caller can tell the lookup failed.
+
+### Added
+- **Stock: unknown `mode` values are rejected.** `scrape_stock(mode=...)` with an unrecognized mode (e.g. a `"quotes"` typo) previously fell through to a quote silently; it now returns a `ScrapeError` naming the allowed modes (`quote`, `history`, `profile`).
+
 ## [1.6.2] - 2026-09-01
 
 ### Fixed

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pyscrappy"
-version = "1.6.2"
+version = "1.6.3"
 description = "A robust, all-in-one Python web scraping toolkit"
 readme = "README.md"
 license = "MIT"

server.json

@@ -7,13 +7,13 @@
     "url": "https://github.com/mldsveda/PyScrappy",
     "source": "github"
   },
-  "version": "1.6.2",
+  "version": "1.6.3",
   "packages": [
     {
       "registryType": "pypi",
       "registryBaseUrl": "https://pypi.org",
       "identifier": "pyscrappy",
-      "version": "1.6.2",
+      "version": "1.6.3",
       "runtimeHint": "uvx",
       "transport": {
         "type": "stdio"

src/pyscrappy/__init__.py

@@ -103,7 +103,7 @@
     register(_cls.name, _cls)
 del _cls
 
-__version__ = "1.6.2"
+__version__ = "1.6.3"
 
 __all__ = [
     # Core

src/pyscrappy/core/async_http.py

@@ -12,7 +12,6 @@
 
 import asyncio
 import logging
-import random
 import time
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode, urlparse
@@ -244,16 +243,19 @@ def _ensure_client(self) -> httpx.AsyncClient | AsyncStealthClient:
             self._client = self._build_client()
         return self._client
 
-    def _pick_ua(self) -> str:
-        if self.config.user_agent:
-            return self.config.user_agent
-        return random.choice(self.config.user_agents)
+    def _pick_ua(self) -> str | None:
+        # Shared contract with the sync client and browser backend via
+        # ScraperConfig.pick_user_agent(); None only if no UA is configured at all.
+        return self.config.pick_user_agent()
 
     def _merge_headers(
         self, extra: dict[str, str], user_agent: str | None = None
     ) -> dict[str, str]:
         ua = user_agent or self._pick_ua()
-        return {**self.config.headers, "User-Agent": ua, **extra}
+        headers = {**self.config.headers}
+        if ua:  # omit the header entirely if no UA is configured (user_agents=[])
+            headers["User-Agent"] = ua
+        return {**headers, **extra}
 
     # Cache is shared with the sync client (same module-level store + lock).
 

src/pyscrappy/core/browser.py

@@ -82,7 +82,9 @@ def get_html(
             self._start()
 
         timeout_ms = int((wait_timeout or self.config.timeout) * 1000)
-        ua = self.config.user_agents[0] if self.config.user_agents else None
+        # Honor the same UA contract as the HTTP backend: a configured
+        # user_agent overrides, otherwise rotate through user_agents.
+        ua = self.config.pick_user_agent()
 
         context = self._browser.new_context(user_agent=ua)
         page = context.new_page()
@@ -103,7 +105,7 @@ def screenshot(self, url: str, path: str, full_page: bool = True) -> None:
         if not self._browser:
             self._start()
 
-        context = self._browser.new_context()
+        context = self._browser.new_context(user_agent=self.config.pick_user_agent())
         page = context.new_page()
         try:
             page.goto(url, wait_until="networkidle", timeout=int(self.config.timeout * 1000))

src/pyscrappy/core/config.py

@@ -135,3 +135,14 @@ def pick_proxy(self, exclude: str | None = None) -> str | None:
         if not choices:
             return random.choice(self.proxy) if self.proxy else None
         return random.choice(choices)
+
+    def pick_user_agent(self) -> str | None:
+        """Return the User-Agent to send: a configured single ``user_agent``
+        overrides ``user_agents`` rotation, otherwise one is chosen at random from
+        ``user_agents`` (or None if that list is empty). Shared by the HTTP and
+        browser backends so both honor the same contract."""
+        import random
+
+        if self.user_agent:
+            return self.user_agent
+        return random.choice(self.user_agents) if self.user_agents else None

src/pyscrappy/core/http.py

@@ -524,19 +524,22 @@ def _ensure_client(self) -> httpx.Client | StealthClient:
             self._client = self._build_client()
         return self._client
 
-    def _pick_ua(self) -> str:
-        # A configured single user_agent overrides rotation.
-        if self.config.user_agent:
-            return self.config.user_agent
-        return random.choice(self.config.user_agents)
+    def _pick_ua(self) -> str | None:
+        # A configured single user_agent overrides rotation (shared with the
+        # browser backend via ScraperConfig.pick_user_agent). Returns None only
+        # if both user_agent and user_agents are empty.
+        return self.config.pick_user_agent()
 
     def _merge_headers(
         self, extra: dict[str, str], user_agent: str | None = None
     ) -> dict[str, str]:
         """Build the request headers: config.headers (lowest priority), then the
         chosen User-Agent, then per-call headers (highest priority)."""
         ua = user_agent or self._pick_ua()
-        return {**self.config.headers, "User-Agent": ua, **extra}
+        headers = {**self.config.headers}
+        if ua:  # omit the header entirely if no UA is configured (user_agents=[])
+            headers["User-Agent"] = ua
+        return {**headers, **extra}
 
     def _backoff_delay(self, attempt: int) -> float:
         """Retry delay for this client's config (see module-level backoff_delay)."""

src/pyscrappy/scrapers/crypto.py

@@ -56,6 +56,10 @@ def scrape(  # type: ignore[override]
             ScrapeResult with coin data (name, symbol, price, market cap, …).
         """
         ids = self._resolve_ids(query) if query else None
+        if query and not ids:
+            # A specific-coin query that resolved to nothing must not silently fall
+            # through to the unfiltered top-market list — report it instead.
+            return self._err(_MARKETS, f"No coins matched query {query!r}.")
         url = self._build_markets_url(vs_currency, max_results, ids)
 
         try:
@@ -73,6 +77,9 @@ async def scrape_async(  # type: ignore[override]
     ) -> ScrapeResult:
         """Async counterpart to :meth:`scrape` (same args/returns)."""
         ids = await self._resolve_ids_async(query) if query else None
+        if query and not ids:
+            # See scrape(): don't fall through to the unfiltered top-market list.
+            return self._err(_MARKETS, f"No coins matched query {query!r}.")
         url = self._build_markets_url(vs_currency, max_results, ids)
 
         try:

src/pyscrappy/scrapers/stock.py

@@ -8,7 +8,23 @@
 from pyscrappy.core.base import BaseScraper
 from pyscrappy.core.config import ScraperConfig
 from pyscrappy.core.exceptions import NetworkError
-from pyscrappy.core.models import ScrapeMetadata, ScrapeResult
+from pyscrappy.core.models import ScrapeError, ScrapeMetadata, ScrapeResult
+
+_MODES = ("quote", "history", "profile")
+
+
+def _invalid_mode_result(mode: str, scraper_name: str) -> ScrapeResult:
+    return ScrapeResult(
+        data=[],
+        metadata=ScrapeMetadata(scraper=scraper_name),
+        errors=[
+            ScrapeError(
+                url=_YF_BASE,
+                message=f"Unknown mode {mode!r}. Use one of: {', '.join(_MODES)}.",
+            )
+        ],
+    )
+
 
 _YF_BASE = "https://query1.finance.yahoo.com"
 
@@ -61,6 +77,8 @@ def scrape(  # type: ignore[override]
             ScrapeResult with stock data.
         """
         symbol = symbol.upper().strip()
+        if mode not in _MODES:
+            return _invalid_mode_result(mode, self.name)
 
         if mode == "history":
             return self._scrape_history(symbol, period, interval)
@@ -78,6 +96,8 @@ async def scrape_async(  # type: ignore[override]
     ) -> ScrapeResult:
         """Async counterpart to :meth:`scrape` (same args/returns)."""
         symbol = symbol.upper().strip()
+        if mode not in _MODES:
+            return _invalid_mode_result(mode, self.name)
 
         if mode == "history":
             url = f"{_YF_BASE}/v8/finance/chart/{symbol}?range={period}&interval={interval}"

tests/test_core/test_browser.py

@@ -109,6 +109,22 @@ def test_get_html_with_scroll(self):
         assert mock_page.evaluate.call_count == 3
         assert mock_page.wait_for_timeout.call_count == 3
 
+    def test_get_html_honors_configured_user_agent(self):
+        # A configured single user_agent must be passed to the browser context,
+        # not silently ignored in favor of user_agents[0].
+        bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0"))
+        mock_page = MagicMock()
+        mock_page.content.return_value = "<html></html>"
+        mock_context = MagicMock()
+        mock_context.new_page.return_value = mock_page
+        mock_browser = MagicMock()
+        mock_browser.new_context.return_value = mock_context
+        bm._browser = mock_browser
+
+        bm.get_html("https://example.com")
+
+        mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0")
+
     def test_get_html_cleans_up_on_error(self):
         bm = BrowserManager()
         mock_page = MagicMock()
@@ -143,6 +159,21 @@ def test_screenshot_calls_browser(self):
         mock_page.close.assert_called_once()
         mock_context.close.assert_called_once()
 
+    def test_screenshot_honors_configured_user_agent(self):
+        # screenshot() previously created a context with no UA (headless default);
+        # it must now pass the configured user_agent like get_html.
+        bm = BrowserManager(ScraperConfig(user_agent="MyBot/1.0"))
+        mock_page = MagicMock()
+        mock_context = MagicMock()
+        mock_context.new_page.return_value = mock_page
+        mock_browser = MagicMock()
+        mock_browser.new_context.return_value = mock_context
+        bm._browser = mock_browser
+
+        bm.screenshot("https://example.com", "/tmp/test.png")
+
+        mock_browser.new_context.assert_called_once_with(user_agent="MyBot/1.0")
+
     def test_screenshot_partial_page(self):
         bm = BrowserManager()
         mock_page = MagicMock()

tests/test_core/test_config.py

@@ -54,6 +54,19 @@ def test_custom_user_agents(self):
         config = ScraperConfig(user_agents=agents)
         assert config.user_agents == agents
 
+    def test_pick_user_agent_override_wins(self):
+        # A single configured user_agent overrides the rotation list.
+        config = ScraperConfig(user_agent="MyBot/1.0", user_agents=["A/1", "B/2"])
+        assert config.pick_user_agent() == "MyBot/1.0"
+
+    def test_pick_user_agent_rotates_when_no_override(self):
+        config = ScraperConfig(user_agents=["A/1", "B/2"])
+        assert config.pick_user_agent() in {"A/1", "B/2"}
+
+    def test_pick_user_agent_none_when_no_agents(self):
+        config = ScraperConfig(user_agent=None, user_agents=[])
+        assert config.pick_user_agent() is None
+
     def test_render_js_auto(self):
         config = ScraperConfig(render_js="auto")
         assert config.render_js == "auto"

tests/test_core/test_http.py

@@ -430,6 +430,14 @@ def test_config_user_agent_overrides_rotation(self):
         client = HttpClient(config)
         assert {client._pick_ua() for _ in range(20)} == {"Custom/9.9"}
 
+    def test_no_user_agents_does_not_crash_and_omits_header(self):
+        # user_agents=[] with no override must not raise (previously IndexError via
+        # random.choice([])); _merge_headers then omits the User-Agent entirely.
+        config = ScraperConfig(user_agent=None, user_agents=[])
+        client = HttpClient(config)
+        assert client._pick_ua() is None
+        assert "User-Agent" not in client._merge_headers({})
+
 
 class TestHttpClientCustomHeaders:
     def test_config_headers_are_sent(self):

tests/test_scrapers/test_data_apis.py

@@ -53,6 +53,17 @@ def test_query_resolves_ids(self):
         # the markets URL should carry the resolved id
         assert "ids=ethereum" in s._http.get_html.call_args[0][0]
 
+    def test_unresolvable_query_errors_not_falls_back_to_top_market(self):
+        # A specific query that resolves to nothing (typo / unknown coin) must
+        # error, not silently return the unfiltered top-market list.
+        search = json.dumps({"coins": []})  # /search finds nothing
+        s = _mock(CryptoScraper(), search)
+        r = s.scrape(query="bitcon")  # typo
+        assert r.data == []
+        assert r.errors and "No coins matched" in r.errors[0].message
+        # It must NOT have fetched the markets endpoint after the empty search.
+        assert s._http.get_html.call_count == 1
+
 
 class TestCurrency:
     def test_rates_and_conversion(self):

tests/test_scrapers/test_stock.py

@@ -115,6 +115,14 @@ def test_symbol_uppercased(self):
         assert "AAPL" in call_url
         scraper.close()
 
+    def test_unknown_mode_errors_instead_of_silently_quoting(self):
+        # A typo'd mode previously fell through to a quote; it must now error.
+        scraper = StockScraper()
+        result = scraper.scrape(symbol="AAPL", mode="quotes")  # typo
+        assert result.data == []
+        assert result.errors and "Unknown mode" in result.errors[0].message
+        scraper.close()
+
 
 class TestStockScraperHistory:
     def test_scrape_history(self):