diff --git a/CHANGELOG.md b/CHANGELOG.md index b244459..28f5eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to PyScrappy are documented here. The format is based on ## [Unreleased] +## [1.6.2] - 2026-09-01 + +### Fixed +- **`on_retry` reports the target URL on a 429 under `scraper_api`.** The sync client's rate-limit (429) retry branch passed the rewritten provider endpoint (e.g. `api.scraperapi.com`) to the `on_retry` hook instead of the logical target URL, unlike every other hook call and unlike the async client. Retry callbacks/metrics now attribute a rate-limited retry to the site being scraped, consistent with #171. + ## [1.6.1] - 2026-08-27 ### Added diff --git a/pyproject.toml b/pyproject.toml index 0b55c85..6a6e210 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pyscrappy" -version = "1.6.1" +version = "1.6.2" description = "A robust, all-in-one Python web scraping toolkit" readme = "README.md" license = "MIT" diff --git a/server.json b/server.json index 5421579..8b4ad09 100644 --- a/server.json +++ b/server.json @@ -7,13 +7,13 @@ "url": "https://github.com/mldsveda/PyScrappy", "source": "github" }, - "version": "1.6.1", + "version": "1.6.2", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "pyscrappy", - "version": "1.6.1", + "version": "1.6.2", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/pyscrappy/__init__.py b/src/pyscrappy/__init__.py index 8747f55..8f8b4cf 100644 --- a/src/pyscrappy/__init__.py +++ b/src/pyscrappy/__init__.py @@ -103,7 +103,7 @@ register(_cls.name, _cls) del _cls -__version__ = "1.6.1" +__version__ = "1.6.2" __all__ = [ # Core diff --git a/src/pyscrappy/core/http.py b/src/pyscrappy/core/http.py index 246f0c8..4945239 100644 --- a/src/pyscrappy/core/http.py +++ b/src/pyscrappy/core/http.py @@ -391,7 +391,11 @@ def get(self, url: str, skip_robots_check: bool = False, **kwargs: Any) -> httpx if attempt < self.config.max_retries: logger.warning("Rate-limited on %s, retrying in %.1fs", url, retry_after) _fire( - self.config.on_retry, url, attempt, retry_after, "429 Too Many Requests" + self.config.on_retry, + target_url, + attempt, + retry_after, + "429 Too Many Requests", ) time.sleep(retry_after) continue diff --git a/tests/test_core/test_http.py b/tests/test_core/test_http.py index a65cbc3..4d0208a 100644 --- a/tests/test_core/test_http.py +++ b/tests/test_core/test_http.py @@ -888,3 +888,32 @@ def test_on_request_reports_target_url_not_scraper_api_endpoint(self): client.get("https://target.example.com/page") assert seen == ["https://target.example.com/page"] # not api.scraperapi.com client.close() + + def test_on_retry_429_reports_target_url_not_scraper_api_endpoint(self): + # A 429 retry under scraper_api routing must report the logical target + # URL, like every other hook (#171) — the async path already does, the + # sync 429 branch previously leaked the provider endpoint. + seen = [] + config = ScraperConfig( + rate_limit=0, + retry_delay=0, + max_retries=3, + scraper_api={"provider": "scraperapi", "api_key": "KEY"}, + on_retry=lambda url, attempt, delay, error: seen.append(url), + ) + client = HttpClient(config) + + rl = MagicMock(spec=httpx.Response) + rl.status_code = 429 + rl.headers = {} + ok = MagicMock(spec=httpx.Response) + ok.status_code = 200 + ok.headers = {} + ok.raise_for_status = MagicMock() + mock = MagicMock() + mock.get.side_effect = [rl, ok] # 429 then success + client._client = mock + + client.get("https://target.example.com/page") + assert seen == ["https://target.example.com/page"] # not api.scraperapi.com + client.close()