diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bd252..00385aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,19 @@ that is a fix, and the release notes lead with it. the note in the workflow says to raise it only when a three-page run has been measured, naming that run. -## [Unreleased] +### Fixed + +- **Scraper API: `waitFor` is now sent as a JSON object.** It went out as a + JSON-encoded string, following a docstring that said the API required one. + Measured 2026-09-23 against `/tasks/sync`: the string form is refused with + HTTP 422 ("params.waitFor must be an object") and still billed ($0.0005); + the object form answers HTTP 200. Every `--wait-*` run was therefore + a paid exit 5. +- **Scraper API: the target's HTTP status is read from `http_code`.** The + client read `status`, which is the API's own verdict string (`"success"`), + so the page classifier never saw a target 403/503 — a live rakuten run + logged "Upstream page status success". `status` is now a fallback only + when it is an integer. ## [0.1.0] — 2026-09-14 diff --git a/scraper_api_client.py b/scraper_api_client.py index b506b99..db569d1 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -122,9 +122,15 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is sent as a JSON OBJECT. + + This docstring used to say the opposite -- that the API wanted it as a + double-encoded string -- and the client built it that way. Measured + 2026-09-23 against /tasks/sync: the string form is refused with HTTP + 422 ("params.waitFor must be an object") and the call is STILL BILLED + ($0.0005); the same request with an object answers HTTP 200. So every + --wait-* flag was a paid exit 5. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -132,11 +138,11 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} return None @@ -145,14 +151,14 @@ def fetch_html(args) -> str: "task_type": "scrape", "url": args.url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # {"status": verdict, "http_code", "headers", "body"} "timeout": min(args.timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for, ensure_ascii=False)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -187,8 +193,16 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + # The TARGET's HTTP code is `http_code`. `status` is the API's own + # verdict STRING ("success") -- measured 2026-09-23 -- and reading it + # here handed "success" to detect_page_state, so a target 403/503 was + # never seen. `status` is kept as a fallback only if it is an int. + upstream_status = body.get("http_code") + if not isinstance(upstream_status, int): + legacy = body.get("status") + upstream_status = legacy if isinstance(legacy, int) else None + logger.info("Upstream page status %s (target HTTP code, from http_code), %d bytes of HTML.", + upstream_status, len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction. On this # site a refusal carries no markup at all — nothing a challenge check diff --git a/smoke_test.py b/smoke_test.py index e4781c5..76b07de 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1498,6 +1498,57 @@ def test_x_debug_header_is_redacted(): return ok +def test_scraper_api_waitfor_is_an_object(): + """Both Scraper API defects measured 2026-09-23, through the real + parse_args() and fetch_html(), with requests.post captured (no network). + + waitFor went out as a JSON-encoded STRING, which the live API refuses + with HTTP 422 and still bills; and the target's status was read from + `status`, which is the API's own verdict string ("success"), so a + target 403/503 never reached detect_page_state. The real field is + `http_code`.""" + group("Scraper API: waitFor is an object, the target status is http_code") + try: + import scraper_api_client as sac + except ImportError as e: + return check("scraper_api_client imports (%s)" % e, False) + + captured = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, "headers": {}, + "body": ""} + + def _post(url, **kw): + captured.update(kw) + return _Resp() + + real_post, real_argv = sac.requests.post, sys.argv + sac.requests.post = _post + sys.argv = ["scraper_api_client.py", "--key", "k" * 8, + "--url", "https://www.vrbo.com/search?destination=Orlando,%20Florida,%20United%20States%20of%20America", "--wait-text", "Orlando"] + status = None + try: + args = sac.parse_args() + got = sac.fetch_html(args) + status = got[1] if isinstance(got, tuple) else None + finally: + sac.requests.post, sys.argv = real_post, real_argv + wf = (captured.get("json") or {}).get("waitFor") + ok = check("Scraper API: --wait-text sends waitFor as an OBJECT, not a " + "JSON-encoded string (HTTP 422 and still billed, 2026-09-23)", + isinstance(wf, dict) and wf.get("text") == "Orlando") + ok &= check("Scraper API: the status handed onward is the target's " + "http_code (403, an int), not the API's verdict 'success'", + isinstance(status, int) and status == 403) + return ok + + def main() -> int: ok = True skips = [] @@ -1527,6 +1578,7 @@ def main() -> int: ok &= test_required_files_are_committed() ok &= test_readme_claims() ok &= test_x_debug_header_is_redacted() + ok &= test_scraper_api_waitfor_is_an_object() passed = _total_checks - len(_failures) if passed < CLAIMED_CHECK_FLOOR: