Scraper API: send waitFor as an object, read the target status from http_code

CHANGELOG.md

@@ -22,7 +22,19 @@ that is a fix, and the release notes lead with it.
   the note in the workflow says to raise it only when a three-page run has
   been measured, naming that run.
 
-## [Unreleased]
+### Fixed
+
+- **Scraper API: `waitFor` is now sent as a JSON object.** It went out as a
+  JSON-encoded string, following a docstring that said the API required one.
+  Measured 2026-09-23 against `/tasks/sync`: the string form is refused with
+  HTTP 422 ("params.waitFor must be an object") and still billed ($0.0005);
+  the object form answers HTTP 200. Every `--wait-*` run was therefore
+  a paid exit 5.
+- **Scraper API: the target's HTTP status is read from `http_code`.** The
+  client read `status`, which is the API's own verdict string (`"success"`),
+  so the page classifier never saw a target 403/503 — a live rakuten run
+  logged "Upstream page status success". `status` is now a fallback only
+  when it is an integer.
 
 ## [0.1.0] — 2026-09-14
 

scraper_api_client.py

@@ -122,21 +122,27 @@ def _redact_debug_header(value: str) -> str:
                                _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value))
 
 
-def _build_wait_for(args) -> Optional[str]:
-    """`waitFor` must be a JSON STRING (double-encoded), per the API docs.
-    Passing a nested object is silently wrong.
+def _build_wait_for(args) -> Optional[dict]:
+    """`waitFor` is sent as a JSON OBJECT.
+
+    This docstring used to say the opposite -- that the API wanted it as a
+    double-encoded string -- and the client built it that way. Measured
+    2026-09-23 against /tasks/sync: the string form is refused with HTTP
+    422 ("params.waitFor must be an object") and the call is STILL BILLED
+    ($0.0005); the same request with an object answers HTTP 200. So every
+    --wait-* flag was a paid exit 5.
 
     Default (no flag): wait for the DOM. On a challenge-protected page
     that resolves instantly against the challenge page itself — which is
     exactly the trap documented in this module's docstring, so
     --wait-text/--wait-element exist to wait on something only the real
     page can contain."""
     if args.wait_text:
-        return json.dumps({"text": args.wait_text})
+        return {"text": args.wait_text}
     if args.wait_element:
-        return json.dumps({"element": args.wait_element, "checkVisible": True})
+        return {"element": args.wait_element, "checkVisible": True}
     if args.wait_state:
-        return json.dumps({"state": args.wait_state})
+        return {"state": args.wait_state}
     return None
 
 
@@ -145,14 +151,14 @@ def fetch_html(args) -> str:
         "task_type": "scrape",
         "url": args.url,
         "data_format": "raw",   # we want HTML; product_parser does the rest
-        "format": "json",       # so we get {"status", "headers", "body"}
+        "format": "json",       # {"status": verdict, "http_code", "headers", "body"}
         "timeout": min(args.timeout, MAX_API_TIMEOUT),
     }
 
     wait_for = _build_wait_for(args)
     if wait_for:
         payload["waitFor"] = wait_for
-        logger.info("waitFor: %s", wait_for)
+        logger.info("waitFor: %s", json.dumps(wait_for, ensure_ascii=False))
 
     if args.cdp_url:
         payload["cdpurl"] = args.cdp_url
@@ -187,8 +193,16 @@ def fetch_html(args) -> str:
 
     body = resp.json()
     html = body.get("body") or ""
-    upstream_status = body.get("status")
-    logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html))
+    # The TARGET's HTTP code is `http_code`. `status` is the API's own
+    # verdict STRING ("success") -- measured 2026-09-23 -- and reading it
+    # here handed "success" to detect_page_state, so a target 403/503 was
+    # never seen. `status` is kept as a fallback only if it is an int.
+    upstream_status = body.get("http_code")
+    if not isinstance(upstream_status, int):
+        legacy = body.get("status")
+        upstream_status = legacy if isinstance(legacy, int) else None
+    logger.info("Upstream page status %s (target HTTP code, from http_code), %d bytes of HTML.",
+                upstream_status, len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
     # be, and that cost this engine the family's central distinction. On this
     # site a refusal carries no markup at all — nothing a challenge check

smoke_test.py

@@ -1498,6 +1498,57 @@ def test_x_debug_header_is_redacted():
     return ok
 
 
+def test_scraper_api_waitfor_is_an_object():
+    """Both Scraper API defects measured 2026-09-23, through the real
+    parse_args() and fetch_html(), with requests.post captured (no network).
+
+    waitFor went out as a JSON-encoded STRING, which the live API refuses
+    with HTTP 422 and still bills; and the target's status was read from
+    `status`, which is the API's own verdict string ("success"), so a
+    target 403/503 never reached detect_page_state. The real field is
+    `http_code`."""
+    group("Scraper API: waitFor is an object, the target status is http_code")
+    try:
+        import scraper_api_client as sac
+    except ImportError as e:
+        return check("scraper_api_client imports (%s)" % e, False)
+
+    captured = {}
+
+    class _Resp:
+        status_code = 200
+        headers = {}
+        text = ""
+
+        def json(self):
+            return {"status": "success", "http_code": 403, "headers": {},
+                    "body": "<html></html>"}
+
+    def _post(url, **kw):
+        captured.update(kw)
+        return _Resp()
+
+    real_post, real_argv = sac.requests.post, sys.argv
+    sac.requests.post = _post
+    sys.argv = ["scraper_api_client.py", "--key", "k" * 8,
+                "--url", "https://www.vrbo.com/search?destination=Orlando,%20Florida,%20United%20States%20of%20America", "--wait-text", "Orlando"]
+    status = None
+    try:
+        args = sac.parse_args()
+        got = sac.fetch_html(args)
+        status = got[1] if isinstance(got, tuple) else None
+    finally:
+        sac.requests.post, sys.argv = real_post, real_argv
+    wf = (captured.get("json") or {}).get("waitFor")
+    ok = check("Scraper API: --wait-text sends waitFor as an OBJECT, not a "
+               "JSON-encoded string (HTTP 422 and still billed, 2026-09-23)",
+               isinstance(wf, dict) and wf.get("text") == "Orlando")
+    ok &= check("Scraper API: the status handed onward is the target's "
+                "http_code (403, an int), not the API's verdict 'success'",
+                isinstance(status, int) and status == 403)
+    return ok
+
+
 def main() -> int:
     ok = True
     skips = []
@@ -1527,6 +1578,7 @@ def main() -> int:
     ok &= test_required_files_are_committed()
     ok &= test_readme_claims()
     ok &= test_x_debug_header_is_redacted()
+    ok &= test_scraper_api_waitfor_is_an_object()
 
     passed = _total_checks - len(_failures)
     if passed < CLAIMED_CHECK_FLOOR: