From fd0434e704fd0f09b775fd7799802ca23816488a Mon Sep 17 00:00:00 2001 From: aleekaz Date: Wed, 9 Sep 2026 16:35:00 +0300 Subject: [PATCH 1/9] Add check() and the dashboard API client, and re-measure the gateway This is one commit because it does not split into ones that build: the README is tested by tests/test_readme.py, and the same probe runs that produced api.py also moved values in the provider definition that proxy.py and errors.py read. The library stopped being a pure string builder. Two network-touching halves arrive: - check() opens one CONNECT, by name and never on construction, and reports what the gateway said about it. - Client wraps the dashboard account API against the vendor's OpenAPI 3.0.3 document, fetched 2026-09-09, 24 paths. The specification is documentation and gets no more authority than a vendor's client did. Every path here was called from a real account on 2026-09-08 and 2026-09-09, and the document is wrong in four measured places, all recorded in the docstring of the method affected: - statistics dates are dd-mm-yyyy, while `format: date` in the type is a string the server cannot parse - three paging conventions, not one, and both page-numbered endpoints are 1-based - sub-users/ ends a walk with an empty payload, whitelist/ips with 404 - whitelist/ip/upsert refuses the two fields its own schema marks required; protocol is what it lacks, and the server does not apply the default: HTTP it declares, so this client sends it The gateway's status codes were re-measured by raw CONNECT and most of the table moved. A bad country is 407 and not the 406 four artifacts had carried for four weeks, which is the case for client-side validation rather than against it. Nothing is released. __version__ stays 0.1.2 and the CHANGELOG entry is under Unreleased, so the PyPI page is unchanged until a version bump. --- CHANGELOG.md | 779 +++++++++ README.md | 840 +++++++-- pyproject.toml | 15 +- src/nodemaven/__init__.py | 25 + src/nodemaven/api.py | 1720 +++++++++++++++++++ src/nodemaven/check.py | 484 ++++++ src/nodemaven/data/providers/nodemaven.toml | 273 ++- src/nodemaven/errors.py | 99 +- src/nodemaven/providers.py | 128 +- src/nodemaven/proxy.py | 230 ++- tests/test_api.py | 1180 +++++++++++++ tests/test_check.py | 804 +++++++++ tests/test_proxy.py | 374 +++- tests/test_readme.py | 453 +++++ 14 files changed, 7238 insertions(+), 166 deletions(-) create mode 100644 src/nodemaven/api.py create mode 100644 src/nodemaven/check.py create mode 100644 tests/test_api.py create mode 100644 tests/test_check.py create mode 100644 tests/test_readme.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c75407..69f1e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,495 @@ says so" is not one of those, and an entry that rests on it says so outright. ## Unreleased +### The library opens sockets now, and the README said it did not + +Two calls reach the network and neither of them runs on import: `proxy.check()` +opens one CONNECT to the gateway, and `nodemaven.Client()` talks to the +dashboard API. Everything that was in 0.1.2 still opens nothing - `Proxy` builds +a string and hands it to whatever client you already have. + +- **`proxy.check()`, and `Check`, `CheckError`.** One CONNECT to the gateway, + the reply read and handed back whole: `status`, the reason phrase verbatim, + every header, the elapsed seconds, the exit address when the gateway sends + one, and the gateway's own explanation of that status. `ok` is `status == 200` + and nothing more. + + **A refusal is a return value, not an exception.** A 407 is the gateway + answering the question, so it comes back as a `Check`. `CheckError` is raised + only when *nothing came back* - DNS, a refused connection, a timeout, or a + first line that is not a status line. + + The CONNECT is hand-rolled rather than delegated, and the reason is what an + HTTP library does to the diagnosis. `requests` reports a failed tunnel as + `ProxyError('Unable to connect to proxy', OSError('Tunnel connection failed: + 407 Proxy Authentication Required'))`: the status survives as text inside a + nested exception, and the reason phrase and every header are gone. This + package exists to tell a caller which of seven inputs was wrong, and it cannot + do that from a string it has to parse back out of an error message. + + Three details that are decisions rather than defaults. The response head is + decoded **latin-1, never utf-8**, because a header value is bytes by + specification and a gateway may put anything in a reason phrase - utf-8 would + raise on a byte a proxy is entitled to send, turning a readable diagnosis into + a decode error. `timeout` defaults to **15 seconds** because one of the seven + measured reactions is *no reply for about 20 s*, and a 5 s default would + report it as a network problem. And there is no null CONNECT, so the target is + a parameter on every entry point rather than a constant: whatever is named + sees a real TCP connection from the exit address. It defaults to + `api.ipify.org:443`, port 443 because a gateway may treat plaintext + differently. + + **`ok` does not mean your parameters were applied**, and `check()` does not + pretend otherwise. An unrecognised parameter name is answered 200 and dropped, + which is why this package refuses unknown names before sending; no reply can + recover that afterwards. + +- **Three uncaught exceptions in `check()` and one cross-language divergence, + all found by porting the module to Rust.** None is reachable from a correct + call and all four are reachable from a typo, which is the population this + module exists for. + + `str.isdigit()` gated both the status code and the port, and it is not a test + for "a number I can hand to C". It is True for the latin-1 superscripts + U+00B9, U+00B2 and U+00B3, where `int()` raises `ValueError` - so + `check(server="127.0.0.1:\xb2")` raised `ValueError` instead of the + `CheckError` this module promises. It is also True for a digit string of any + length, where `int()` succeeds and the socket layer raises `OverflowError` - + which derives from `ArithmeticError` and **not** `OSError`, so it walks past + the `except OSError` whose entire job is turning socket-layer failures into + `CheckError`. Both gates are now a spelled-out ASCII-digit comparison, the + port is length-bounded before `int()` sees it, and **port 0 is refused by + name** rather than left to a platform-specific errno - Windows answers + WinError 10049 and Linux ECONNREFUSED for the same input, and neither says + what is wrong. + + Response header names were folded with `str.lower()`, which is Unicode-aware. + The head is decoded latin-1 on purpose, so byte `0xC0` arrives as U+00C0; + Python folds it to U+00E0 and Rust's `to_ascii_lowercase` does not, and the + two SDKs would key one response two ways. RFC 9110 makes a field name a token + and a token is ASCII, so ASCII-only is the correct fold rather than merely the + portable one. Not reachable through this gateway, which sends ASCII header + names; reachable through one that does not. + + Recorded as a method and not as three bugs: **writing the port is the review.** + Four of the defects fixed in this release were found by reimplementing code + that already had a passing test suite, because a second language does not + share the first one's assumptions about what its own standard library means. + +- **`check()` could not report a single refusal from the live gateway, because + it required CRLF framing.** Measured 2026-09-08 against + `gate.nodemaven.com:8080` with a raw byte dump: this gateway frames a 200 with + CRLF and frames **every refusal - 406, 407, 500 - with a bare LF**, and each + refusal carries `Connection: close`. A reader that ends a head only at + `\r\n\r\n` therefore saw the peer hang up with no blank line, called it a + truncated head, and raised - so the 407 and 406 a caller most needs to see were + the two statuses this module could not return. + + A head now ends at any of the four spellings of a blank line - `\r\n\r\n`, + `\n\n`, `\r\n\n`, `\n\r\n` - and lines are split on LF with one optional + preceding CR stripped. RFC 9112 section 2.2 permits exactly this: a recipient + may recognise a single LF as a line terminator and ignore any preceding CR. A + CR anywhere else in a line stays in the value. The request this client *sends* + is unchanged and still CRLF. + + **The truncation guard is unchanged**: a head that stops without its blank + line, LF-framed or not, is still refused rather than reported as an answer. + Both halves are pinned by tests replaying the gateway's exact bytes. + +- **`connect_reactions` in the provider schema** - the gateway's own reading of + its status codes, as data. A status code on this gateway is not a diagnosis. + Measured 2026-09-08 by raw CONNECT: a value the gateway will not take on + `country`, `filter`, `ttl`, `type` or `speed` is answered **407**, which sends + the caller to check credentials that are correct, and so is a wrong password. + `check()` prints the gateway's own sentence next to the code. The translation + is per-gateway dialect, so it belongs in the TOML beside the separators and not + in Python. + + Five statuses are described - 200, 406, 407, 410, 500 - and the table is + exactly as long as the measurements. **Three of the entries report an ambiguity + rather than a diagnosis, on purpose.** `406` answers a bad `region`, a bad + `isp` and `charter` - a real ISP - alike, so it says neither which of the two + parameters was refused nor whether the name is unknown or merely unavailable. + `410` came from one value, `comcast`, and reads as a name the gateway knows + and a pool the account cannot reach, so the entry states its sample rather + than a rule. An entry that named a cause where none was measured would be this + package inventing the diagnosis it exists to stop the gateway inventing. A + regression test pins each of them so they cannot be tidied into confident + sentences later. + + **The `city` entry went through two corrections in one day and the second one + reversed it.** It first said `500` meant no `city` was reachable at all on the + account, on seven values - six real US cities and a junk one. That was + narrowed the same morning to "every `city` tried", because `locations/cities` + answers normally on the same account, so the values existed and were not + misspelt, and a `500` is the server reporting that *it* broke rather than that + the input was wrong. The narrowing named the run that would settle it: a + catalogue city sent with its own country and region. + + That run happened, and `city` **works**. `probe_gateway_city_from_catalogue.py`, + 2026-09-08, six CONNECTs holding the login, the password, the target, the + gateway host and port and the parameter order fixed: + `us`/`louisiana`/`abbeville` answers `200`, a second city in a second region + answers `200`, the same city with the `region` removed answers `500`, and an + invented name with a real region answers `406`. So `500` is `city` without + `region`, `406` is a city the gateway does not hold, and the gateway does look + the name up. + + What the mistake looked like from the inside: all seven earlier values were + sent as `country` + `city`, with no `region` - one line in one probe, held + fixed across every arm, and therefore invisible in the comparison between + them. The arms varied the city and agreed, which reads as evidence about + cities. **A control names what it varied and what it silently held fixed**, and + the thing held fixed was the cause. The first wording was wrong about the + gateway; the second was honest about its own limits and still could not see + this, because narrowing a claim does not find a confound. + + Keys are the status as a **string**, because TOML has no integer keys and + neither does JSON, and the golden vectors are JSON. + + This is the **third key added to the schema before the vectors freeze**, after + `values` and `normalize`, and the arithmetic is the same one this file already + records: one edit now, a migration in four repositories later. + +- **Four places said a bad `country` gives 406. It gives 407.** Measured + 2026-09-08. Three were in `api.py` - a comment, a docstring and a live error + message - and one was in the README's status list, so + `Client.validate(proxy)` told a caller to expect a status the gateway does not + send for that input - and the status it does send is the one the same page + calls actively misleading. Corrected everywhere, and `ttl` case-sensitivity is + now documented with it: `ttl-10m` opens the tunnel and `ttl-10M` is answered + 407, which is the one value on this gateway whose case matters. + + A dated mapping from reason phrases to causes has been **removed rather than + annotated**, because a later raw dump did not reproduce it. The README's + exit-address paragraph now says what is true instead: more than one + implementation answers behind this hostname, which one you reach is decided by + your username, and they disagree about the header name - so `Check.exit_ip` + being `None` is normal rather than an error. + +- **The session key is the parsed set, and parameter order does not reach it.** + Measured 2026-09-08, 20 rounds a side: the canonical parameter order and a + shuffled one each drew the same single exit 20 times, while a control + differing by one parameter *value* drew a different exit 20 times. Until now + the shipped note said the key is the whole parameter set, measured 2026-08-10, + and said nothing about order - so emitting parameters in a fixed order was an + untested assumption underneath every username this library builds. It is now + measured, and the golden vectors may pin call order across the SDKs without + pinning a behaviour that could change which exit a caller gets. + + **The instrument was changed before the run and that is the load-bearing + part.** An earlier 4-round attempt read some arms from the CONNECT response's + `X-Proxy-Exit-IP` header and others from an echo service. More than one + implementation answers behind this hostname, which one you reach is decided by + the username, and probe arms differ by username *by construction* - so that + design compared two instruments and attributed the difference to the + parameter. Every round now reads the exit from an echo service. The cost is + named rather than hidden: the probe depends on a third-party host being up, + and that host's own 503s and timeouts appear in the rows looking like gateway + answers. Instrument agreement was 59 of 59 usable rounds. + +- **`Proxy.sessions(n)`** - `n` proxies differing only in session id. The ids are + hex from `secrets`, and both halves of that are load-bearing. Hex because a + value containing the gateway's separator is cut and every id sharing a prefix + collapses onto one exit, measured 2026-08-20 - `secrets.token_urlsafe` emits + `-` and `_`, `uuid4()` emits `-`, base64 emits `+` and `/`, and all three + would produce that silently. `secrets` and not `random` because `random` is + clock-seeded, so two workers starting in the same millisecond would draw the + same ids and share one exit while looking like two. + +- **`Client`: the dashboard REST API.** `me()`, the location catalogue + (`countries`, `regions`, `cities`, `isps`, `zip_codes`), `statistics`, + `domain_statistics`, sub-user CRUD, and the IP whitelist. `Page` and + `iterate()` for the paginated ones. + + **Four of the calls have now been exercised against the live API and the rest + have not.** Every path, filter name, header form and environment variable was + read out of the vendor's own client - `github.com/nodemavencom/proxy`, + `python/nodemaven/client.py`, read 2026-09-07 - and on 2026-09-08 `users/me`, + `countries`, `regions` and `cities` were sent to the live API from the user's + own connection, this machine routing through a VPN gateway. Everything else + here is still documented rather than measured, in exactly the sense the + provider schema's `status` field means. + + It is not the mistake `norotate` was, and the difference is categorical rather + than a matter of degree: a wrong **API path** is answered 404, loudly, and + corrects itself the first time anybody runs it. A wrong **gateway parameter** + is answered 200 and dropped, and nothing ever corrects it. Reading a vendor + file is an acceptable source for the first and was not for the second. + + Two details worth carrying. The auth header is `Authorization: x-api-key + `, which is unusual and is what the server wants - measured 2026-09-08, + that form answers 200 while `Authorization: Bearer ` and the conventional + `X-API-Key` header are both answered 403. And `me()` returns the raw dict: a + dataclass is a claim about field names, and the live response is why that + restraint paid. It answers six fields - `data`, `email`, `is_traffic_frozen`, + `proxy_password`, `proxy_username`, `subscription_status`. A dataclass written + a day earlier from the vendor's client would have declared `traffic_left`, + which does not exist, and typed `is_traffic_frozen` as a bool, where the server + sends a string. The raw dict was wrong about nothing, because it claimed + nothing. + + **The pagination shape was inference, it was marked as such in the code, and + the live API answered something else.** `limit`/`offset` with `country__code` + is Django REST Framework, whose `LimitOffsetPagination` answers + `{"count", "next", "previous", "results"}`; that was a strong inference from + the vendor's own parameter names, and `_page()` was written to accept a + paginated object, a bare list, a single object and an empty object rather than + to assume it. `page.count`, `.next` and `.previous` are all `None` on every + endpoint reached so far. + + This entry said until later the same day that `countries`, `regions` and + `cities` reply with a **bare JSON array**, and that the branch the inference + did not predict was the one that runs. **Both halves were wrong**, measured + 2026-09-08 by `probe_catalogue_paging.py`: all three answer with the + **envelope**, and it is the DRF-shaped branch that runs. What the mistake + looked like from the inside: the run that produced it printed `page.count` and + nothing else about the shape, and `count` is `None` for a bare array *and* for + an envelope that does not fill it. One reading fitted the single number that + was on the screen, and it was written down as a measurement in eight files. + The instrument, not the reasoning, was the fault - the second probe prints the + shape and the question closes in one line. + + The branch that really does run and was never predicted is the single-object + one: `locations/isps` answers 200 with an object carrying no `results` key at + all, so `_page()` wraps it as one row that is not a row. Its contents were not + printed and the endpoint is still unread. + + What survives the correction, and is the reason the code kept working through + it: `_page()` accepts four shapes because the inference was marked as an + inference, not because anything suggested a specific alternative. That habit + has now paid twice in opposite directions. + +- **`_list` sends `limit` and `offset` on every list call, and `iterate()` walks + by offset when there is no `next`.** Two defects, one measurement, + 2026-09-08. + + `isps()` was broken outright: `locations/isps` **refuses** a request that + omits `limit` and `offset`. The other three location endpoints answer such a + request with exactly 50 rows and no reported total - which is byte-for-byte + what a complete collection looks like. That is the worse of the two failures. + A refusal is visible on the first call; a silent truncation at 50 is a caller + quietly building on a fraction of the catalogue with no way to detect it from + the response. Both are fixed by always sending `DEFAULT_PAGE_SIZE` and + `offset = 0`, which the caller can override. + + **`DEFAULT_PAGE_SIZE` is 1000**, raised from 50 later the same day once the + server's own answer had been measured rather than guessed. 1000 is the largest + value all five list endpoints accept: `limit=10000` is refused with 400 by + `isps`, and on `cities` it returns exactly the same 1000 rows `limit=1000` + does. At 1000 the whole country catalogue (192 rows) and a country's regions + (51) arrive in one request each. + + That `cities` result was written up here, in `api.py` and in the README as a + **ceiling** - "cut back without being reported, the original failure one order + of magnitude further out". It was not evidence of one, and all three files + withdrew the word: a ceiling at 1000 and a US city collection exactly 1000 + rows long make arms C and D agree for different reasons, and both arms asked + for more than they got, so neither could see the difference. + + **Arm `limit=1000&offset=1000` was then run and the ceiling is real.** It + returns a further 965 rows, so US cities number 1965 and 1000 is a hard server + limit. The withdrawn sentence was true. It was still right to withdraw it: it + had been resting on two arms that could not carry it, and it came back only + because the third arm was run. **Being right by luck is not being justified**, + and the version that ships now names the arm rather than the guess. + + The consequence for callers is the one the withdrawn sentence predicted. + `cities(country__code="us")` returns 1000 of 1965 rows and nothing in the + answer says so - raising the default from 50 moved that failure one order of + magnitude out rather than removing it. `iterate()` is the call that returns + the collection, and `cities()` is documented as a page. + + `iterate()` previously followed `next` and stopped when there was none, so on + this API it yielded one page and called it the collection - the README said it + walked pages, and it did not. It now records the call a `Page` came from, in + `Page.request_path` and `Page.request_params`, and asks for `offset + limit` + until a page comes back shorter than the limit. + + **That walk is an inference and it carries a guard rather than a comment.** + The server was measured to *require* `limit` and `offset`, never to *honour* + them across pages, and those are different claims. A server that accepts + `offset` and ignores it would hand back the same rows a hundred times and + `iterate()` would call it a collection - so a page identical to the one before + it raises `ApiError` naming that cause. This is the pattern the `norotate` + correction in 0.1.2 was about, applied before the fact: where a parameter is + accepted with no observable effect, the code must be able to tell "honoured" + from "swallowed", and here it is the only thing standing between an inference + and a wrong collection. The inference has since been confirmed on `countries`, + `regions` and `cities` - `offset=50` returns rows disjoint from `offset=0` - + and never on `isps`, so the guard stays. + + **The first version of that walk carried a defect that a test caught, and the + fix for it was a second defect that only a measurement caught.** A `Page. + enveloped` flag was added on the reasoning that a `next` of `None` means two + opposite things: inside a paging envelope the server is saying the collection + ended, and in a bare array it is saying nothing, because there is no `next` + field to carry a value. That reasoning is sound and the flag was wrong anyway, + because the shape it was told apart from does not exist here - every location + endpoint is enveloped, so gating on the flag would have stopped every + catalogue read after one page. The exact truncation the whole entry is about, + reintroduced by the fix for it. + + Two things made it possible and both are worth naming. The test that + prompted the flag was written against a **hand-made** DRF envelope carrying a + real `count`; it was pinning an inferred shape, so it could only ever confirm + the inference. And the live envelope reports **no `count`** on a page of 50 + out of 194 - so its paging fields are not being filled, and an empty `next` + beside an empty `count` is absence rather than an answer. + + `Page.enveloped` is gone. The rule is now shape-independent: no `next` url to + follow means step by offset, and a page shorter than the limit asked for is + the end. Whether the envelope even carries a `next` key has not been printed, + and this rule is correct either way - which is the property that was missing + from the version that had to guess right. + +- **Zero required dependencies still, and no async.** The API client is + `urllib.request` from the standard library. An SDK that pulled in an HTTP + client would dictate one to a caller who already has one, and these calls + happen once at start-up. `Client(transport=...)` is the seam: four arguments, + no `timeout`, so async or your own session goes there if it is ever asked for. + + `ProxyHandler({})` in the default transport is load-bearing rather than + decorative. Left out, `urlopen` reads `http_proxy` and `https_proxy` from the + environment - set on precisely the machines that use proxies - and an API call + would route through a proxy nobody asked for. Pinned by a test that reads the + source. + +- **`Client.validate(proxy)` asks the live catalogue, and the catalogue is + deliberately not frozen into the `values` table.** The data that would fill + that table exists in the product, which is what the note under "Values are not + validated" was missing. It is still not shipped: a snapshot of the countries + list is a false refusal waiting for the day a country is added, and a false + refusal with our name on the error is worse than the gap. So the check is a + network call, which makes it the caller's decision, which is why it is a method + on `Client` and not something `Proxy` does behind your back. + + It also refuses a `city` sent without a `region`, which needs no network at + all. Measured 2026-09-08, that combination is answered `500` while the same + city with its region opens the tunnel - a shipped validation rule wearing the + status code of a server fault. It lives here rather than in `Proxy` for the + same reason as the rest: `Client.validate()` is the caller's call to make and + can be wrong without shipping the wrongness into a release. + +- **Five error classes, because the taxonomy crosses languages.** `CheckError`, + `ApiError`, `AuthError`, `NotFoundError`, `RateLimitError`, all under + `NodeMavenError`. A class exists only where a caller would plausibly write + different code for it: 401 means fix your key, 429 means wait, a 404 from CRUD + means the row is gone. A 400 means fix your program and there is nothing to + branch on, so it stays on the base `ApiError`. + + Two credentials, two classes, and they are not interchangeable. + `CredentialsError` is the **proxy** login and is raised before anything is + sent. `AuthError` is the **dashboard API key** and has been to the server. + +- **The README was wrong about the network in three places and is corrected in + place.** It opened with "This library opens no socket", which was true of every + version up to 0.1.2 and false the moment `check()` landed, and the Errors + section said "Nothing here is raised from a response, because nothing here + sends one". Both sentences are kept in HTML comments next to their + corrections, per the house rule, and a test asserts each is gone from the prose + *and* present in the file - the comment that records a fix is otherwise + indistinguishable from the bug. + + New sections: `Asking the gateway`, `Account API`, and `Many sessions at once`. + +- **A test per quotation, enforced.** `tests/test_readme.py` builds every output + the README quotes and compares it, whitespace collapsed and nothing else + forgiven. That includes the 407 block, whose text comes from the TOML - so the + README and the shipped definition have to agree word for word, and they are two + files of which one goes to PyPI. Both directions of the parameter table are + pinned too: every `known_params` entry has a row, and no row names a parameter + the package refuses, which is exactly what `norotate` was for five days. + +**Two bugs in the new API client, found by reading it before wiring it up rather +than by a test.** `_urllib_transport` hardcoded `timeout=30.0`, so +`Client(timeout=...)` was accepted and silently inert, and `_interpret` took a +`timeout` argument it never used. The timeout is now bound into the default +transport with `functools.partial` at construction. Worth recording because both +are the same shape: a parameter that is threaded through the signature, looks +configured at every call site, and is dropped at the one place it would have +taken effect. Nothing failing, nothing to notice. + +- **`type` is a known parameter**, and refusing it was blocking the paid mobile + tier. Probed from the VPS on 2026-08-26, the same run that removed `norotate` + and by the same method: a junk value discriminates where acceptance does not. + `type=zzqqx` is answered **407**, exactly as the positive control on `filter` + and unlike the negative control `zzqqx=1`, which is answered 200 and dropped. + So the gateway recognises the name. + + It selects a pool rather than filtering one. Five echo requests per arm, a + fresh `sid` and `country=us`: `type=mobile` drew AS21928 T-Mobile three times, + AS6167 Cellco and AS7018; `type=residential` and the parameter left off drew + Comcast, Charter, Windstream, Metronet, Fidium, Planet and AS701 wireline, + with no mobile ASN in either arm. + +- **`speed`'s comment in the shipped definition was corrected.** It said the + name was read off the vendor's proxy generator on 2026-08-12 and never + probed. It was probed on 2026-08-26 in the run above and is answered 407, so + the name is confirmed. What remains unmeasured is what it *does*; the comment + now says which half is which. No behaviour changed. + +- **A place name is written the way you would say it.** `region="District of + Columbia"` now builds `region-district_of_columbia`. Five parameters are + folded to their wire form before anything else looks at them - `country`, + `region`, `city`, `isp`, `type` - and the fold is three fixed steps: strip + ASCII whitespace from both ends, lower-case ASCII `A-Z` only, replace each + space with `_`. + + Until now a space passed straight through into the username and `country="US"` + was sent as `country-US`. Both are settings the gateway does not apply. + + This is added on **two independent sources**, which is the bar `norotate` set + and failed. The gateway itself emits `region-district_of_columbia` in a + username generated on 2026-09-07, and the vendor's own client builds the same + form in `nodemaven/utils.py::build_proxy_username`. It is also a different + kind of claim from `norotate`: that one asserted the gateway recognises a + *name*, which cannot be established from a generator, because an unrecognised + name is answered 200 and dropped. This one converts an input whose behaviour + is unknown into a form that is known to work, so there is no false refusal + available to be wrong about. + + Which parameters fold is declared in the provider TOML under a new + `normalize` key, as data, for the same reason `known_params` is: the public + API must never name a gateway's parameters in its own code, and the next + gateway will fold a different set or none at all. + + Four parameters are deliberately left out. `sid` is opaque and chosen by the + caller, and folding it would silently move a sticky session to a different + exit. `filter` and `ttl` are not folded by the vendor's client either, not + even to lower case, and the generated string says nothing about them because + its `medium` was already lowercase. `speed` does not appear in the vendor's + builder at all. + + The fold is ASCII-only and says so in the code, because `str.lower()` is + Unicode-aware: it maps characters like the Turkish dotted capital I in a way + Rust and Go do not reproduce, and a golden vector that depended on it would + fail in one SDK for reasons that have nothing to do with proxies. + +- **A value containing ASCII whitespace is refused**, for every parameter + nothing folds. A proxy username is a single token on the CONNECT line, so a + space either malforms the line or cuts the value short, and the object would + otherwise present three spellings of one value - raw from `username`, as + `%20` from `url()`, and a third to a browser driver taking the fields + separately - of which at most one can be right. The six characters treated as + whitespace are spelled out as `ASCII_WHITESPACE` rather than delegated to + `str.isspace`, which is Unicode-wide; a no-break space is not in the set and + passes, which is the honest answer for input nobody has asked the gateway + about. + +- **Two ways a provider definition can now be refused at load.** Normalizing a + parameter that is not in `known_params` is refused, because that parameter is + rejected by name and the fold could never run. Declaring `normalize` while + separating parameters with `_` is refused, because the fold *inserts* that + character: `city="New York"` would become `new_york` and then be cut in half + by the separator the fold had just produced, and the caller would be blamed + for input that was correct. + +- The unknown-parameter error message quoted in the README is now pinned by a + test that compares the whole string. It had drifted to nine parameter names + while the code produced ten - the Rust port carried that test from its first + commit and this one did not. + - `Source` and `Issues` added to the package metadata, and the README gained a CI badge, a link to its own LICENSE and a link to the benchmark harness the retry table is measured on. That is five things and they were held back for @@ -22,6 +511,296 @@ says so" is not one of those, and an entry that rests on it says so outright. was there and the README buried it in the last section. - A `Parameters` reference for the shipped gateway, and an `Errors` list. +- **`isps()` reads a different key, because that endpoint answers a different + envelope.** Measured 2026-09-08 against a live account: `locations/isps` + answers 200 with an object whose keys are `city`, `country`, `isps` and + `region`, and there is no `results` anywhere. The rows are the list under + `isps` - 358 of them for `country__code="us"` - and the other three fields are + strings, whose contents have still not been printed and which this package + therefore says nothing about and does not return. + + The fix is a `rows_key` threaded through `_list`, `_page` and `Page`, with + `isps()` the only caller that passes anything but `results`. It is recorded on + the `Page` so `iterate()` reads the second page the way it read the first - + without that, the walk would parse page one as ISPs and page two as a one-row + envelope and stop, which is a truncation wearing the shape of an ending. + + **What was rejected is the more interesting half.** The obvious alternative is + to have `_page()` return whichever value in the object happens to be a list. + That reads as robustness and is a guess: an envelope carrying two lists is + resolved by key order, silently, and differently the day the server adds a + field. Naming the key per endpoint is knowledge this package actually has. + + **This entry said until later the same day that the method did not work and + was documented rather than fixed**, because the first probe printed the key + *names* and not the values, so whether `body["isps"]` held a list had not been + observed - and unwrapping a key because its name reads right is the move that + put `norotate` in this package and `traffic_left` in its README. That was the + right call at the time and the record is worth keeping for what it cost: one + command. The probe was changed to print value types and lengths, it was run, + and the answer was the obvious one. **Being obvious is not what made it safe to + ship; being run is.** + + Nothing was released in the broken state - the whole `Client` is new in this + version - so what the day bought was a decision procedure rather than a + correction to a published artifact. + +- **`offset` is honoured on `isps`, and the arm that shows it is not the + obvious one.** `limit=50&offset=0` and `limit=50&offset=50` both answer 50 + rows out of 358, which is exactly what a server accepting `offset` and + dropping it would answer, so the two arms that look like a paging test cannot + separate the two cases. What separates them is `limit=1000&offset=1000` + answering **zero** rows where an ignored offset would have answered all 358. + + That is the weaker of the two claims, and the code says so. On `countries`, + `regions` and `cities` the rows at `offset=50` were read and are disjoint from + those at `offset=0`; on `isps` page-to-page disjointness has still never been + read. `iterate()`'s duplicate-page guard is what covers the gap, and it stays + for that reason rather than as a leftover. + +- **The `limit=1000` ceiling, withdrawn and then measured.** An earlier draft of + this section asserted 1000 was a server ceiling on the strength of arms C + (`limit=1000`) and D (`limit=10000`) returning the same 1000 rows. Those two + arms cannot separate a ceiling from a collection that is exactly 1000 rows + long, so the claim was withdrawn from `api.py`, `README.md` and from here. + Arm H - `limit=1000, offset=1000` - was then added and returned a further 965 + rows, so `cities(country__code="us")` spans 1965 rows and one call sees the + first 1000 of them with `count`, `next` and `previous` all `None`. The claim + is restored naming the arm that establishes it. **Being right by luck is not + the same as being justified**, and the withdrawal was correct when it was + made. + +- **Dates came out of the README, and out of the five `connect_reactions` + strings.** They are here instead, and the rule is audience rather than + subject: this file and the definition's `notes` are read to audit a claim, so + a date is the point; the README is read to use the library and + `connect_reactions` is printed to somebody whose tunnel just failed, where + *when we measured it* is not part of the answer they came for. Nothing was + softened - the README still says which behaviours were measured and which + were transcribed, and it now names this file as where the provenance lives. + Counted after the sweep rather than during it: `README.md` holds two dates and + both are the arguments in `statistics(start_date=..., end_date=...)`, the Rust + README holds none, all five `connect_reactions` hold none, and `notes` keeps + its six. + + This bullet first quoted those counts as *22, 15 and 4*, written from memory + while the edits were still open and wrong in two of three places - inside the + entry whose whole subject is moving claims to where they get audited. It is + worth recording because of how it happened rather than what it cost: the + numbers were a byproduct of the work, not the finding, and that is the class + of number nobody thinks to measure. `git diff` cannot answer it either, since + these files carry several sessions of uncommitted work and the counts get + attributed to whichever edit the diff algorithm pairs them with. Counting the + finished file is the only reading that means anything. + +- **A fabricated error message in the README, caught by the test written for + exactly that.** The new quickstart contrast between a hand-written URL and a + validated one quoted a `ParamError` reading *"'filtr' is not a parameter this + gateway knows. Did you mean 'filter'?"*. This library has never said that - + there is no "Did you mean" in it at all. What it says is measured and now + quoted verbatim. The test that caught it only did so by luck: it pinned + `quoting[0]`, the first README block quoting a `ParamError`, and the invented + one happened to land first in the file. It now checks **every** such block, + extracting the typo from each rather than hardcoding one, so a second + fabricated quotation lower down would fail too. + +### The account API was rewritten against the vendor's specification, 2026-09-09 + +The vendor publishes an OpenAPI 3.0.3 document, unauthenticated, 24 paths. It +was read on 2026-09-09 and **four of the paths this package sent did not +exist**. Nothing shipped in that state - the whole account API is unreleased, +0.1.2 carries none of it - so the corrections below cost renames and no +deprecations. + +- **The paths.** `locations/zip-codes/` is `locations/zipcodes/`, solid, no + separator. `statistics/` does not exist and was never one endpoint: it is + `statistics/data/`, `statistics/requests/` and `statistics/domains/`, each + requiring a `proxy_username`. `whitelist-ips/` is `whitelist/ips`, with no + trailing slash. `sub-users/` was right and was being read under the wrong key + - its rows are under `payload`, inside a `{success, description, errors, + payload}` envelope that is the specification's own declared 200 schema and not + a server defect. `update_sub_user` sent `PATCH` to a path that does not exist + at all; it is a `PUT` to the collection with the id in the body. + +- **What made this survive three weeks.** The section documenting these calls + argued they were safe to ship untested, because *a wrong path is answered 404 + on the first call and corrects itself*. **That is false on this host**, + measured 2026-09-09: three of the four wrong paths were answered `200`, + `text/html`, 6415 bytes - byte-identical to a path invented on the spot as a + negative control. The dashboard serves its front end for anything it does not + route. + + What the mistake looked like from the inside: the claim is true of nearly + every REST API, and it read as a property of HTTP rather than as an assumption + about one server. It was never checked, and the check is one extra request. + This is the same failure as `norotate` in 0.1.1 and `OptimizationHints` before + it - **a wrong name answered with success** - which the package was built to + defend against at the gateway and then trusted the dashboard not to do. + +- **Paging is three conventions, not one.** The nine location endpoints take + `limit`/`offset`; `sub-users/` takes `page`/`per_page`; `whitelist/ips` takes + `page`/`page_size`. An unknown query parameter is ignored rather than refused, + so the single hard-coded pair was silently doing nothing on two thirds of the + surface. `Page` now carries the convention it was fetched under. + +- **`iterate()` stops on an empty page, not on a short one.** The old rule + truncated wherever the server caps the page size below the request, and this + server does: `cities(limit=10000)` is answered with 1000 rows out of 1965, so + "shorter than asked" read those 1000 as the end. **The measurement that says + so was already written in this package's own docstrings** and the stop rule + was never checked against it. A measurement sitting in a docstring is not a + measurement anyone applied. + +- **Both page-numbered endpoints number their first page 1**, measured + 2026-09-09 by `lab/probes/probe_account_api.py --phase 9` against a live + account, asking each for pages 0, 1, 2 and 9999 at a size of one row. On + `sub-users/` the evidence is where the rows are: page 0 came back with none + and page 1 with the account's only sub-user, which a 0-based server cannot + produce. On `whitelist/ips` the collection is empty, so the reading is the + weaker one - page 1 is the only number answered `200` at all, while 0, 2 and + 9999 are answered `404`, and a 0-based server would have answered page 0. + + Written earlier the same day and retired before any release: `iterate()` + *refused* to walk a page-numbered call whose first `page` the caller had not + supplied, because nothing measured whether the server numbered from 0 or 1 and + the two guesses are not symmetric - assume 1 against a 0-based server and the + first page is dropped in silence. The refusal was the only reading that could + not be silently wrong while the question was open, and what retired it is the + measurement rather than a second opinion about what servers usually do. + `Paging.first_cursor` carries the answer and the refusal is gone from the code + rather than sitting behind a flag. + +- **The two page-numbered endpoints do not mark the end of a collection the same + way**, measured in the same run, and a single rule for both is a defect either + way round. Past the last page `sub-users/` answers `200` with an empty payload + and `whitelist/ips` answers `404`. `Paging.ends_with_not_found` is therefore + set on the whitelist alone, and `iterate()` treats a `404` as the end only + there - everywhere else a `404` stays an error, because on this host it is + also what a wrong path looks like. Without that field, walking the whitelist + past its last page raised `NotFoundError` instead of stopping. + + Half of it is inference and the shipped docstring says which half: the account + holds no whitelist entries, so what is measured is `404` on pages 0, 2 and + 9999 against a `200` on page 1, and this endpoint has never been seen to end + anywhere but at page 1. Put one address on the account and it becomes a + measurement. + +- **New methods**, all from the specification: `isp_regions`, `isp_cities`, + `zip_code_regions`, `zip_code_cities`, `statistics_data`, + `statistics_requests`, `whitelist_ip`, `reset_sub_user_usage`. + `add_whitelist_ip` became `upsert_whitelist_ip` because the endpoint replaces + an existing entry as well as creating one, and takes a required `ports_count`. + +- **The specification is documentation and gets no more authority than that.** + Two defects in it: `info.description` names host `api.nodemaven.com`, which + 404s at nginx while `dashboard.nodemaven.com` answers; and the statistics + endpoints describe their dates as "dd-mm-yyyy" in prose while typing the same + fields `format: date`, which is `yyyy-mm-dd`. One disagreement with a run: it + calls the availability field `effective_availability` where the 2026-09-08 + measurement read `availability`. The run wins, and nothing depends on it. + + **The date disagreement was settled live the same day and the typed half is + the wrong half**, `--phase 10`: `start=20-08-2026` answers `200` with 21 data + points, `start=2026-08-20` answers `400`, and that `400`'s body is + byte-identical to the body for `start=not-a-date`. So `format: date` is not a + rival spelling the server declines - it is a string the server cannot parse, + and every client generated from this document sends the one form that fails. + The docstrings on the statistics methods now say `dd-mm-yyyy` outright. + + The same run found that all three answer **500**, not 400, when `start`, `end` + and `period` are omitted together, though the document marks all three + optional. There is no "give me everything" call on this surface. + + This entry read "unresolved; nothing here validates a date, so nothing here + depends on the answer" until that run. The second clause was true, and it was + the reason not to look - which is backwards for a defect in a document other + people generate clients from. What this package does with a date is pass it + through, so the cost lands on a caller who read the spec instead of this file, + and it lands as a `400` whose body does not name which argument was refused. + +- **`sub_users()` returns every sub-user's `proxy_password` in clear text**, by + the specification's own required-field list, as does `me()`. This is now said + in the README and in both docstrings. It is written here because a probe in + the sibling `lab/` tree printed one to a console on 2026-09-09 and the + password had to be rotated. + + **`create_sub_user()` does too, measured the same day rather than read off a + schema.** The create response carries `proxy_password` alongside the id, so it + is a credential and not a receipt. + +- **All five write endpoints were called live on 2026-09-09** and four of them + behave. Each write was followed by a *separate read* rather than trusted, which + on this host is the only thing that distinguishes success from a `200` carrying + the dashboard's front page: `create_sub_user` answers **`201`** where its own + path declares only `200`; `update_sub_user`'s PUT-to-the-collection answered + `200` and a re-read of the row showed the changed field, so the shape is + measured and not transcribed; `reset_sub_user_usage` answers `200` with a + `payload` **array** where every other envelope here carries an object; + `delete_sub_user` with the id as a query parameter answers `200` with a body + where the document declares `204`. + + **Both deletes do remove the object, and the whitelist's uniqueness check + lags behind them.** This entry passed through two wrong versions the same day + and both are kept, because the second was written as a correction of the + first. + + It said "a re-read showed the row gone" for both deletes. That was withdrawn: + the whitelist delete answered `200` at 15:13 and its immediate re-read did not + show the address, and at 15:18 the server refused to whitelist that same + address with `400 IP is already whitelisted.`, before any write in that run + had succeeded. So the pair had been believed once and had been wrong once. + + It then said neither delete was confirmed to have removed anything, and named + a soft or uncommitted delete as one of two live candidates. At 16:18 the same + address was accepted `201`. An invisible surviving row would still have + blocked it, so what was stale is the uniqueness check and not the storage. The + sub-user was separately absent from a listing read at 16:15, an hour after its + delete rather than a second after it. + + Only the bounds of the lag are measured and they are loose: still refusing at + 5 minutes, over by 62. What a caller should take from it is unchanged by the + resolution - re-adding an address you have just deleted may be refused as a + duplicate for some minutes. + + **`upsert_whitelist_ip()` sends `protocol` on every call**, because the + specification marks it optional with `default: "HTTP"` and the server does not + apply that default. Measured 2026-09-09 at 16:18, one field at a time: `ip`, + `ports_count` and `name` is refused `400`, + `{"error": "Please enter a valid protocol(HTTP or SOCKS5)."}`; the same body + plus `protocol: "HTTP"` is accepted `201`, and a read of `whitelist/ips` finds + the address. A client generated from the document therefore sends the one body + that fails, which is why the default lives in this method rather than in the + caller's arguments. + + An earlier run had settled the likelier-looking reading first: `not-an-ip` as + the address draws that same message byte for byte, so `protocol` is validated + before `ip` is examined and the reserved test address was never the problem. + This entry then said which field was short remained the server's word rather + than a measurement, since the accepted body varied six at once. That was right, + and it was one call away from being settled. + + Untested and worth knowing: `name` was present in every rung of the ladder, so + whether it is required too is unmeasured, and no rung sent `ip`, `ports_count` + and `protocol` alone. + + **The success body is `{"ip_id", "message"}` and neither key is documented**; + the path binds both `200` and `201` to a schema whose only property is + `message`. The identifier is `ip_id` where the listing calls the same value + `id`. `delete_whitelist_ip` takes it and answers `200`. + +- **The API key is a JWT that lives 1800 seconds**, `exp - iat`, measured + 2026-09-09 by decoding a live one locally. No path in the vendor's 24 issues or + refreshes a token and the document does not mention an expiry at all, so a + `Client` held longer than half an hour begins raising `AuthError` for a reason + no response distinguishes from a wrong key or a wrong header form. The message + now names the clock as a third possibility. Nothing here reads `exp` or renews + on its own - that would mean decoding a credential to take a control-flow + decision, and one token's lifetime is not the policy. + +Five of the vendor's 24 paths are deliberately not wrapped: `locations/all-doc/`, +`notifications/`, `llm/submit/`, `llm/results/{id}/` and `llm/balance/`. The +omission is listed rather than left silent. + ## 0.1.2 - 2026-08-26 09:23 - **`values`**, a per-parameter list of legal values, added to the provider diff --git a/README.md b/README.md index 462c296..fb22a46 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,50 @@
- NodeMaven -# nodemaven +# NodeMaven Python SDK **Builds the proxy username a gateway expects, and refuses the input it would silently drop.** + release. The CI badge reads `.github/workflows/ci.yml` on `main` and needs + the repository to be public to resolve; unlike the other three it can go red + by itself, which is the point of having it. --> [![pypi](https://img.shields.io/pypi/v/nodemaven?style=flat-square)](https://pypi.org/project/nodemaven/) [![python](https://img.shields.io/pypi/pyversions/nodemaven?style=flat-square)](https://pypi.org/project/nodemaven/) [![ci](https://img.shields.io/github/actions/workflow/status/nodemaven/nodemaven-python/ci.yml?branch=main&style=flat-square&label=tests)](https://github.com/nodemaven/nodemaven-python/actions/workflows/ci.yml) - + [![license](https://img.shields.io/pypi/l/nodemaven?style=flat-square)](https://opensource.org/licenses/MIT) -[Quickstart](#quickstart) · [Sticky sessions](#sticky-sessions) · [Why the validation is the point](#why-the-validation-is-the-point) · [What it does not do](#what-this-library-does-not-do) · [Other gateways](#other-gateways) · [Docs](https://docs.nodemaven.com?utm_source=github&utm_medium=sdk_python&utm_campaign=readme) + + +[Quickstart](#quickstart) · [Reference](#reference) · [Parameters](#parameters) · [Errors](#errors) · [Sticky sessions](#sticky-sessions) · [Why the validation is the point](#why-the-validation-is-the-point) · [Asking the gateway](#asking-the-gateway) · [Account API](#account-api) · [What it does not do](#what-this-library-does-not-do) · [Other gateways](#other-gateways) · [Docs](https://docs.nodemaven.com?utm_source=github&utm_medium=sdk_python&utm_campaign=readme)
-Build and validate proxy connection strings. +`Proxy` opens no socket. It builds the username a proxy gateway expects, refuses +the input that gateway would mishandle, and hands the result to whatever HTTP +client you already use. + +Two calls do reach the network, both by name and neither on import: +[`proxy.check()`](#asking-the-gateway) opens one CONNECT and tells you what the +gateway said about it, and [`Client`](#account-api) talks to the dashboard API. -This library opens no socket. It builds the username a proxy gateway expects, -refuses the input that gateway would mishandle, and hands the result to whatever -HTTP client you already use. +**Works with** requests · httpx · aiohttp · Playwright · Patchright · Puppeteer · +curl - and anything else that takes a proxy URL, because that is all it hands +back. ``` pip install nodemaven @@ -67,11 +52,6 @@ pip install nodemaven ## Quickstart - - `login` and `password` are the **Proxy Username and Proxy Password** assigned under Proxy Setup in the [dashboard](https://dashboard.nodemaven.com) - a separate pair from the account you sign in with. The other option there is IP whitelisting, which needs no @@ -89,40 +69,38 @@ r = requests.get("https://api.ipify.org", proxies=proxy.requests()) print(r.text) ``` - +``` +203.0.113.42 +``` -**No account? Any proxy you already have works.** A gateway is a data -description, not a code path, so one that ships no definition here goes through -the same builder and the same validation: +**If that is not your own address, it worked.** If it is your own, the request +never went through the proxy. -```python -import requests -from nodemaven import Proxy, Provider +### Why not just write the URL yourself? -# An empty known_params is not a stub. It says nobody has established what this -# gateway recognises, so every parameter is refused rather than sent to be -# silently dropped - see "Why the validation is the point" below. -mine = Provider(id="mine", label="My proxy", known_params=frozenset()) +Because the gateway does not tell you when you get it wrong. A misspelt +parameter is not refused - the tunnel opens, the setting is dropped, and the +traffic you are paying to route through a medium-quality US pool goes out +wherever the gateway felt like: -proxy = Proxy(provider=mine, login="your-login", password="your-password", - host="proxy.example.com", port=8000) +```python +# by hand: a typo the gateway answers 200 to, and never mentions again +"http://user-country-us-filtr-medium:pass@gate.nodemaven.com:8080" -r = requests.get("https://api.ipify.org", proxies=proxy.requests()) -print(r.text) +# with this library: refused before anything is sent +Proxy(login="user", password="pass", country="us", filtr="medium") +``` + +``` +ParamError: NodeMaven does not know the parameter 'filtr': it is answered with +200 and dropped, so the connection would succeed and your setting would NOT be +applied. Known: ['city', 'country', 'filter', 'ipv4', 'isp', 'region', 'sid', +'speed', 'ttl', 'type'] ``` -Describe the parameters it does take and it validates those too - see -[Other gateways](#other-gateways). Everything from here to the end of that -section builds strings offline and opens no socket at all. +That is the whole reason the package exists, and +[why the validation is the point](#why-the-validation-is-the-point) has the +measurements behind it. Credentials can come from the environment instead, so nothing is in your source: @@ -152,15 +130,177 @@ with sync_playwright() as p: context = browser.new_context(proxy=proxy.playwright()) ``` +## Reference + + + +What every public name takes, returns and raises. Nothing in this section needs +the ones after it; those carry the measurements the refusals were built on. + +### `Proxy` + +```python +Proxy(*, login=None, password=None, host=None, port=None, provider=None, **params) +``` + +Keyword-only. Builds a username and opens nothing. + +| argument | falls back to | refused when | +|---|---|---| +| `login` | `NODEMAVEN_LOGIN` | missing → `CredentialsError` | +| `password` | `NODEMAVEN_PASSWORD` | missing → `CredentialsError` | +| `host` | `NODEMAVEN_HOST`, then the definition's own | - | +| `port` | `NODEMAVEN_PORT`, then the definition's own | not a whole number 1 to 65535 → `CredentialsError` | +| `provider` | the shipped `nodemaven` definition | - | +| `**params` | - | a name outside `known_params`, or a value that is empty, carries a separator, carries whitespace the definition does not fold, or is outside a list the definition declares → `ParamError` | + +The environment names come from the definition's id in upper case, so a gateway +of your own reads its own pair - see [Other gateways](#other-gateways). + +| attribute | is | +|---|---| +| `.username` | the username, in the gateway's dialect | +| `.server` | `host:port`, with no credentials in it | +| `.params` | the parameters as they will be sent, already folded. A copy | +| `.provider` | the `Provider` behind it | + +| call | returns | +|---|---| +| `.url(scheme="http")` | `http://user:pass@host:port`, both credentials percent-encoded | +| `.requests(scheme="http")` | `{"http": ..., "https": ...}` | +| `.httpx(scheme="http")` | `{"http://": ..., "https://": ...}` | +| `.playwright()` | `{"server": ..., "username": ..., "password": ...}`, credentials **not** encoded | +| `.session(session_id)` | a new `Proxy` pinned to that sticky session | +| `.sessions(count, *, length=6)` | a list of `count` new `Proxy` objects, ids distinct | +| `.replace(**changes)` | a new `Proxy`; a value of `None` removes that parameter | +| `.check(*, target="api.ipify.org:443", timeout=15.0)` | a `Check`. **The only call here that opens a socket** | + +`session()` raises `ParamError` on a definition that declares no session +parameter. `sessions()` ids are `2 * length` hexadecimal characters, so the +default holds 2\*\*48; it raises `ParamError` when `count` asks for more distinct +ids than `length` bytes can hold, rather than looping forever looking for them. + +### `Check` + +Returned by `proxy.check()`. Frozen, and a refusal arrives as one of these +rather than as an exception. + +| attribute | is | +|---|---| +| `.ok` | `True` only on 200 | +| `.status` | the CONNECT status, an `int` | +| `.reason` | the reason phrase, verbatim - it labels which back end answered | +| `.server` | the `host:port` that was asked | +| `.elapsed` | seconds, including DNS and the TCP handshake | +| `.headers` | every response header, keys lower-cased | +| `.exit_ip` | the exit address, or `None` when the gateway did not send one | +| `.meaning` | what this status means **on this gateway**, or `None` | + +`CheckError` is raised only when nothing usable came back at all: DNS failure, a +refused connection, a timeout, a response head that never ended, or a first line +that is not a status line. Refusals that happen before anything is sent are +`CheckError` too, and those messages end in *Nothing was sent.* + +### `Client` and `Page` + +```python +Client(api_key=None, *, base_url=None, timeout=30.0, transport=None) +``` + +`api_key` falls back to `NODEMAVEN_APIKEY` and raises `CredentialsError` when +neither is set. `transport` is `(method, url, headers, body) -> (status, bytes)` +and defaults to `urllib.request`. + +| call | returns | +|---|---| +| `.me()` | the account object, the server's own field names | +| `.countries()` `.regions()` `.cities()` | a `Page` | +| `.zip_codes()` `.zip_code_regions()` `.zip_code_cities()` | a `Page` | +| `.isps()` `.isp_regions()` `.isp_cities()` | a `Page` - `isps()` has an [envelope that is not the usual one](#the-isp-catalogue-answers-a-different-shape) | +| `.statistics_data(proxy_username, **filters)` / `.statistics_requests(...)` | a `dict` | +| `.domain_statistics(proxy_username, **filters)` | a `Page` that does not page - the whole answer is one object | +| `.sub_users()` `.whitelist_ips()` | a `Page`, numbered by page rather than by offset | +| `.whitelist_ip(id)` | a `dict` | +| `.create_sub_user(username, password, *, traffic_limit=None, is_traffic_limited=None, **extra)` | a `dict` | +| `.update_sub_user(id, **changes)` / `.delete_sub_user(id)` | a `dict` | +| `.reset_sub_user_usage(ids)` | a `dict`. Takes a list, not one id | +| `.upsert_whitelist_ip(ip, ports_count, *, name=None, protocol="HTTP", sticky=None, ttl=None, **extra)` | a `dict`. Creates **or replaces**. [`protocol` is always sent](#the-whitelist-needs-a-field-the-spec-marks-optional) | +| `.delete_whitelist_ip(id)` | a `dict` | +| `.iterate(page, *, max_pages=100)` | an iterator over the rest of the collection, page by page | +| `.validate(proxy)` | a list of problem strings, empty when the catalogue agrees | + +Every list call takes `**filters`, passed through as query parameters in the +server's own spelling - `country__code="us"`. + +**Paging differs by endpoint and the difference is not cosmetic.** The nine +location endpoints take `limit` and `offset`, both **always sent**: `limit` +defaults to `DEFAULT_PAGE_SIZE`, 1000, and `offset` to 0. `sub_users()` takes +`page` and `per_page`; `whitelist_ips()` takes `page` and `page_size`, whose +documented default is 5 and maximum 100. An unknown query parameter is ignored +rather than refused, so one hard-coded pair would silently do nothing on two +thirds of this surface. + +1000 is the largest limit the location endpoints accept: `limit=10000` is +refused outright by `isps`, and on `cities` it returns the same 1000 rows +`limit=1000` does. + +**That 1000 is a ceiling on the answer, not a count of the rows behind it.** +Asking `cities` for the next offset at the same limit returns a further 965, so +one call sees about half that collection and nothing in the reply says so - +`count`, `next` and `previous` all come back `None`. Use `iterate()` there. The +other catalogues do fit in one request. + +A `Page` iterates its own rows and has a `len()`. `.count`, `.next` and +`.previous` are the server's, and are `None` everywhere - no schema in the +vendor's own specification declares either field, so this is the API's shape and +not a gap in one endpoint. `.request_path`, `.request_params` and `.paging` +record the call the page came from, which is what lets `iterate()` ask for the +next one. + +`iterate()` raises `ApiError` rather than looping in four cases: more than +`max_pages` pages, a next-page url that has already been returned, a page +identical to the one before it - the server accepted the cursor and ignored it - +and a next-page url pointing at a different scheme, host or port than +`base_url`. The host check is there because the request that would follow the +url carries the API key in a header. + +**Both page-numbered endpoints start at page 1**, measured 2026-09-09 by +`lab/probes/probe_account_api.py --phase 9`. On `sub-users/` the evidence is +where the rows are - page 0 came back with none and page 1 with the account's +only sub-user, which a 0-based server cannot produce. On `whitelist/ips` the +collection is empty, so the reading is weaker: page 1 is the only number +answered `200` at all, while 0, 2 and 9999 are answered `404`. + +**The two endpoints do not end their collections the same way.** Past the last +page `sub-users/` answers `200` with an empty payload and `whitelist/ips` +answers `404`, so a `404` is treated as the end of the walk on the whitelist and +nowhere else - everywhere else it is still an error, because a `404` is also +what a wrong path looks like. + +### `Provider` and the module functions + +| call | returns | +|---|---| +| `load(provider_id="nodemaven")` | a shipped definition | +| `load_file(path, provider_id=None)` | a definition from a TOML file; the id is the filename unless named | +| `available()` | the ids of every shipped definition, as a list | + +A `Provider` is a frozen description of one gateway: `id`, `label`, +`known_params`, `status`, `prefix`, `separator`, `pair_separator`, +`session_param`, `host`, `port`, `aliases`, `values`, `normalize`, +`connect_reactions`, `exit_ip_header`, `source`, `source_read`, `notes`. Only +`id`, `label` and `known_params` are required. `.is_measured` is `True` when +`status` is `measured`. + ## Parameters - + What the shipped NodeMaven definition accepts. Every name here was confirmed against the gateway rather than transcribed: @@ -169,13 +309,29 @@ against the gateway rather than transcribed: |---|---|---| | `country` | country code, or `any` | `us`, `de`, ... | | `region` | area inside the country | a name | -| `city` | city inside the country | a name | +| `city` | city inside the country | a name, with a `region` beside it | | `isp` | the exit's ISP | a name | +| `type` | mobile or residential exits | `mobile`, `residential` | | `sid` | the sticky session - see below | any string with no `-` | -| `ttl` | how long that session is held | `10m`, `1m` | -| `filter` | IP quality | `low`, `medium`, `high` (2026-08-13) | -| `speed` | connection speed class | `fast`, `slow` (2026-08-12) | -| `ipv4` | force IPv4 | `True` / `False` | +| `ttl` | how long that session is held | `1m`, `10m`, `10h`, `24h` | +| `filter` | IP quality | `low`, `medium`, `high` | +| `speed` | claims a connection speed class - see below | `fast`, `slow` | +| `ipv4` | claims to force IPv4 - see below | `True` | + +`type` picks a different pool rather than a filter over one pool. Five requests +per arm with a fresh `sid` and `country=us`: +`type=mobile` drew T-Mobile and Verizon Wireless ASNs, while `type=residential` +and leaving it unset drew Comcast, Charter, Windstream and other wireline +carriers, with no mobile ASN among them. + +**`ipv4` and `speed` are confirmed names whose effects are unmeasured**, and the +table says `claims to` for that reason. For `ipv4` the name took a different +method to confirm: a junk value on it is answered `200`, so it cannot be told +apart from an unimplemented name that way. What tells them apart is the sticky +session, whose key is the parsed parameter set - over two +independent session ids with both controls holding, `ipv4=True` moves the exit +and an unknown name does not. `ipv4=False` lands on the same exit as leaving it +out, which is what a default would do and also what a dropped value would do. **Names are validated. Values, on this gateway, are not.** Passing a name that is not in this table raises before anything is sent, because the gateway answers an @@ -187,27 +343,78 @@ carry a per-parameter list of legal values and refuses anything outside it; the shipped definition leaves that list empty for every parameter, deliberately, and a definition you write yourself gets the check as soon as you fill it in. +### Case and spacing + +`country`, `region`, `city`, `isp` and `type` are folded before they are sent: +surrounding whitespace trimmed, ASCII `A-Z` lowered, each remaining space turned +into `_`. `country="US"` and `country="us"` are therefore the same request, and + +```python +Proxy(login="u", password="p", region="District of Columbia").username +# u-region-district_of_columbia +``` + +`region-district_of_columbia` is the form this gateway generates for itself - it +appears in a username the dashboard issued - and the vendor's own client applies +the same transformation. Without the fold a space reaches the username, which +cannot carry one: the CONNECT line is a single token, so the value is either +malformed or cut short. + +**`sid`, `filter`, `ttl` and `speed` are sent with their case unchanged.** `sid` +is yours, and folding an identifier a caller chose would rename their session, so +it is left alone whatever the gateway does with it. + +For the other three, **pass lower case**, and for `ttl` that is not advice: +`ttl-10m` opens the tunnel and `ttl-10M` is answered `407`, +which reads as a credentials problem and is not one. `filter` and `speed` were +not refused in either case, and this library still does not fold them - what the +gateway accepts today and what it will accept next month are different claims, +and the fold list is data in the gateway definition rather than a decision in +this package. + +Which parameters fold is declared in the gateway definition, as data, so a +gateway you describe yourself folds what you say it folds and nothing else. + +**Every other value is refused if it contains whitespace.** There is no spelling +of a space that works here - `username` would emit it raw, `url()` would +percent-encode it to `%20`, and `playwright()` would hand over a third thing - +so between the fold and the refusal, no value with whitespace in it can reach +the wire by any path. + Credentials come from `NODEMAVEN_LOGIN` and `NODEMAVEN_PASSWORD` when not passed in, and the gateway address from `NODEMAVEN_HOST` and `NODEMAVEN_PORT`. ## Errors -All four inherit from `NodeMavenError`, so one `except` catches everything this -library raises. +Everything inherits from `NodeMavenError`, so one `except` catches everything +this library raises. | exception | raised when | |---|---| -| `ParamError` | a parameter name is unknown, a value is empty, a value contains the gateway's separator, or a value is outside a list the definition declares | -| `CredentialsError` | no login, no password, or no gateway address, from arguments or environment | +| `ParamError` | a parameter name is unknown, a value is empty, a value contains whitespace or the gateway's separator, or a value is outside a list the definition declares | +| `CredentialsError` | no login, no password, no gateway address, or no API key, from arguments or environment | | `ProviderError` | a gateway definition is missing, unreadable, or internally inconsistent | +| `CheckError` | `check()` got no answer at all - DNS, a refused connection, a timeout, or something that is not a proxy on that port | +| `ApiError` | the account API refused a call, or answered a shape this library cannot read. Carries `.status` and `.body` | +| `AuthError` | the API key was rejected - `ApiError` with a 401 or 403 | +| `NotFoundError` | the row is gone - `ApiError` with a 404 | +| `RateLimitError` | too many calls - `ApiError` with a 429, and a `.retry_after` | | `NodeMavenError` | the base, never raised on its own | `ParamError` also covers the two structural cases: `session()` on a definition that declares no session parameter, and a definition whose parameter names collide with `login`, `password`, `host`, `port` or `provider`. -Nothing here is raised from a response, because nothing here sends one. Every -failure this library reports is a failure it found before a socket existed. +The first three are found **before a socket exists** - they are failures in what +you asked for, not in what happened. The next five have been to the network and +back. That split is why they are separate classes: retrying a `ParamError` can +only produce the same `ParamError`. + +Two credentials, two exceptions, and they are not interchangeable. +`CredentialsError` from `Proxy` is the **proxy password**, refused before +anything is sent; `AuthError` from `Client` is the **dashboard API key**, and it +has been to the server. A library that reported both as one would tell you to fix +the key when the password is wrong. ## Sticky sessions @@ -219,13 +426,19 @@ held = proxy.session("order4417") **A session id cannot contain the character the gateway separates parameters with**, which for this one is `-`, and passing one raises rather than connecting. -That is measured and not a precaution: on 2026-08-20 a probe opened tunnels with +That is measured and not a precaution: a probe opened tunnels with `sid-order8e3bf9-4417` and with `sid-order8e3bf9`, four rounds each, interleaved, and both landed on **one exit address** while a third arm spelled `sid-order8e3bf94417` held a different one throughout. The gateway cuts the value at the separator and reads the rest as something else, so every order id beginning `order` would quietly share one session and one exit. +**The same cut applies to every parameter, not just `sid`,** which is why a +separator in any value is refused. `isp-verizon` opens the +tunnel, a junk `isp` is answered `406`, and `isp-verizon-zzqqx-zzqqx` is answered +`200` - so the gateway took `verizon` as the ISP and read the tail as a parameter +name it does not know, which it drops silently. + **The session key is the whole parameter set, not the session id.** `country=us, sid=A` and `country=us, sid=A, filter=medium` are two different sessions on the gateway, so adding or removing any parameter moves you to a @@ -238,26 +451,89 @@ germany = proxy.replace(country="de") # a new identity, a new exit plain = proxy.replace(filter=None) # also a new identity ``` +**The set, not the order.** Measured over 20 rounds a side: the canonical +parameter order and a shuffled one drew the same exit 20 times each, while a +control differing by one parameter *value* drew a different exit 20 times. So +the order this library emits parameters in cannot change which exit you get. + +### Many sessions at once + +For a worker pool, one identity per worker: + +```python +for identity in proxy.sessions(50): + queue.put(identity) # each one a different exit +``` + +Each id is `2 * length` hexadecimal characters from `secrets`, `length=6` by +default, and the ids are distinct **within one call**. Asking for the whole +space or more raises `ParamError` - `sessions(256, length=1)` wants every one of +the 256 ids an eight-bit space holds, and drawing them without repeating is a +loop that either never finishes or leaves nothing for the next caller. + +Hex, and not the alphabets people reach for first, because the gateway cuts a +value at its separator and every id sharing a prefix then collapses onto one +exit - silently, since the connection still succeeds. `secrets.token_urlsafe` +emits `-` and `_`, `uuid4()` emits `-` four times, base64 emits `+` and `/`, and +each of those is a separator on some gateway. From `secrets` and not `random` +because `random` is seeded from the clock: two workers starting in the same +millisecond would draw the same ids. + ## Why the validation is the point -A gateway is bad at telling you that you got the username wrong. Measured -against this one on 2026-08-10, seven kinds of bad input produce seven -different reactions and not one of them names the cause: + + +Every gateway behaviour below was measured against the live gateway rather than +transcribed from documentation, and each one carries its date and the probe +behind it in [CHANGELOG.md](https://github.com/nodemaven/nodemaven-python/blob/main/CHANGELOG.md). + +A gateway is bad at telling you that you got the username wrong. One class of +mistake - a value it will not take - comes back five different ways, and not one +of them names the parameter. Read by raw CONNECT, one arm per row: | you sent | the gateway answers | |---|---| -| bad country | `406 Not Acceptable` | -| bad region | `406 Not Acceptable` | -| bad city | `500 Internal Server Error` | +| bad `country` value | `407 Proxy Authentication Required` | +| bad `region` value | `406 Not Acceptable` | +| bad `city` value | `406 Not Acceptable` | +| `city` sent without `region` | `500 Internal Server Error` | +| bad `isp` value | `406 Not Acceptable`, and `410 Gone` for `comcast` | | bad `filter` value | `407 Proxy Authentication Required` | | bad `ttl` value | `407 Proxy Authentication Required` | +| bad `type` or `speed` value | `407 Proxy Authentication Required` | | empty value | nothing, the connection hangs about 20 s | | **unknown parameter name** | **`200`, and the parameter is ignored** | -The two `407` replies send you to check credentials that are correct. The last -row is worse than any of them: the request succeeds, your code carries on, and -the setting you asked for was never applied. Nothing that comes back over the -wire can tell you. +Every `407` there sends you to check credentials that are correct, and the `406` +does not even say which of the two parameters it refused: a bad `region`, a bad +`isp` and `charter` - a real ISP - all answer it, so it separates neither the +parameter nor a misspelling from a pool you cannot have. `comcast` is the one +value measured to answer `410` instead, which reads as a name the gateway knows +and a pool this account cannot reach - one ISP, so read it that narrowly. + +**The two `city` rows are one rule: send `city` with its `region`.** Measured +over six CONNECTs holding the login, the password, the target, the +gateway host and port and the parameter order fixed. `country=us`, +`region=louisiana`, `city=abbeville` answers `200`, and so does a second city in +a second region. The same city with the region left out answers `500`, and an +invented name sent with a real region answers `406` - so the gateway does look +the name up, and the `500` is a request it could not resolve rather than a fault +on their side. `Client.validate()` refuses that combination before it goes out. + +The last row is worse than any of them: the request succeeds, your code carries +on, and the setting you asked for was never applied. Nothing that comes back +over the wire can tell you. + +`ttl` counts in minutes and hours - `1m`, `10m`, `10h` and `24h` connect, while +`10s`, `10d` and a bare `10` are answered `407`. It is also the one parameter +whose value case matters: `10M` is refused where `10m` is accepted. Parameter +*names* are case-insensitive at the gateway; this library folds values for +`country`, `region`, `city`, `isp` and `type` to the wire form anyway, so a space +or a capital in a place name is not your problem. So this library checks before anything is sent: @@ -266,9 +542,327 @@ So this library checks before anything is sent: ParamError: NodeMaven does not know the parameter 'contry': it is answered with 200 and dropped, so the connection would succeed and your setting would NOT be applied. Known: ['city', 'country', 'filter', 'ipv4', 'isp', 'region', 'sid', -'speed', 'ttl'] +'speed', 'ttl', 'type'] +``` + +## Asking the gateway + +Validation catches everything knowable without sending anything. For the rest - +a wrong password, a country the pool does not have, a value the gateway dislikes +- there is one call that opens a single CONNECT and reports what came back: + +```python +result = proxy.check() +``` + + + +``` +200 Connection established via gate.nodemaven.com:8080 in 0.42s, exit 203.0.113.7 +``` + +The exit address arrives **on the CONNECT reply itself**, on a header the gateway +definition names, so knowing where you came out costs one handshake and no +traffic through the tunnel. + +**Do not build anything on it being there.** More than one implementation +answers behind this hostname, which one you reach is decided by your username, +and they do not agree about the header: one measured `200` carried +`X-Exit-IP` where the shipped definition names `X-Proxy-Exit-IP`, and others +send no address at all. So `result.exit_ip` is `None` more often than the +definition suggests, and that is normal rather than an error. If you need the +address every time, read it through the tunnel from a service that echoes it. + +**A refusal is a return value, not an exception.** The status code is the thing +you came for, and raising would bury it in a traceback - which is what a general +HTTP client does. So a failed tunnel comes back as an object, carrying the +gateway's own reading of its own status code: + +``` +407 Proxy Authentication Required via gate.nodemaven.com:8080 in 0.19s +usually NOT your credentials, despite what the status says. A value the gateway +will not take on `country`, `filter`, `ttl`, `type` or `speed` answers 407, and +so does a wrong password. Check the values before the password - and check the +case of `ttl`, which is the one value that is case-sensitive: `10M` is refused +where `10m` is accepted. +``` + +That second paragraph is data in the gateway definition, not a string in this +library, because what a status code means is per-gateway. The fields a `Check` +carries are in the [reference](#check); `CheckError` and when it is raised are +there too. + +**`ok` does not mean your parameters were applied.** An unrecognised parameter +name is also answered `200` and dropped, which is the whole reason the section +above refuses unknown names before sending. No call can recover that after the +fact, and this one does not pretend to. + +`check()` names the host it tunnels to - `api.ipify.org:443` by default - and +whatever you name will see a TCP connection from your exit address. There is no +CONNECT to nowhere, so this is a parameter rather than a constant: + +```python +proxy.check(target="example.com:443", timeout=15.0) +``` + +The timeout defaults to 15 seconds rather than something brisk, because one of +this gateway's measured reactions is no reply for about 20 seconds. A 5-second +timeout would report that as a network problem. + +## Account API + +Quota, usage, sub-users and the location catalogue. Separately credentialled, +because the API key and the proxy password are different secrets from different +places. + +**Every path here now comes from the vendor's own OpenAPI specification**, read +2026-09-09. Five of the calls have also been sent against a live account and +answered: `users/me`, `countries`, `regions`, `cities` and `isps`, measured +2026-09-08. **The rest are still transcribed** - from the specification now +rather than from the vendor's client at `github.com/nodemavencom/proxy`, +`python/nodemaven/client.py`, where four paths were wrong. The five write calls +have still never been called, on purpose: probing one costs a real object on a +production account. + + + +**A wrong path is not answered `404` here**, measured 2026-09-09. +`locations/zip-codes/`, `statistics/` and `whitelist-ips/` - three paths this +package sent before the specification was read - are answered `200`, +`text/html`, 6415 bytes, byte-identical to a path invented on the spot as a +negative control. The dashboard serves its front end for anything it does not +route, so **calling a path cannot tell you whether the path exists.** If you +wrap a path of your own against this API, compare its response against a path +nobody could have implemented rather than against a status code. + +The real paths are `locations/zipcodes/` (solid, no separator), +`statistics/data/`, `statistics/requests/`, `statistics/domains/` and +`whitelist/ips`. `sub-users/` was real all along but was being read under +`results`, where the rows are under `payload`. + +Calling `isps` is how its [different envelope](#the-isp-catalogue-answers-a-different-shape) +was found; the other four came out of the specification. + +```python +from nodemaven import Client + +client = Client() # NODEMAVEN_APIKEY from the environment +me = client.me() +print(me["data"]) # traffic left +``` + +`me()` returns the server's own object with its own field names, unrenamed and +unmodelled. It answers six fields: + +| field | type | | +|---|---|---| +| `data` | `int` | traffic left. **Not** `traffic_left` | +| `email` | `str` | | +| `is_traffic_frozen` | `str` | **not a bool** - see below | +| `proxy_password` | `str` | the proxy password, not your API key | +| `proxy_username` | `str` | | +| `subscription_status` | `str` | | + +**`is_traffic_frozen` is a string, so `if me["is_traffic_frozen"]:` is true +whichever way it reads.** Compare it against the value rather than for truth. + +Not modelling this into a dataclass is deliberate, and the response above is the +argument for it. Written a day earlier from the vendor's client, a dataclass +would have declared `traffic_left`, which does not exist, and typed +`is_traffic_frozen` as a bool, which it is not. The raw dict was wrong about +nothing, because it claimed nothing. + +```python +client.countries() # the catalogue, paginated +client.regions(country__code="us") # Django's field lookup, the server's spelling +client.cities(country__code="us", region__code="dc") +client.isps(country__code="us") # a different envelope, see below +client.zip_codes(country__code="us") + +# Which regions and cities the ISP and zip-code catalogues actually cover. +# Separate endpoints, not filters on the two above. +client.isp_regions(country__code="us") +client.isp_cities(country__code="us", region__code="dc") +client.zip_code_regions(country__code="us") +client.zip_code_cities(country__code="us", region__code="dc") + +# Statistics are per proxy username, and the username is required. +client.statistics_data("acct-1", start_date="2026-09-01", end_date="2026-09-07") +client.statistics_requests("acct-1", start_date="2026-09-01") +client.domain_statistics("acct-1") + +client.sub_users(page=1) +client.create_sub_user("worker-1", "a-password", traffic_limit=1024) +client.update_sub_user(id, traffic_limit=2048) +client.delete_sub_user(id) +client.reset_sub_user_usage([id]) # a list, even for one + +client.whitelist_ips(page=1) +client.whitelist_ip(id) +client.upsert_whitelist_ip("203.0.113.7", 10, name="the office") +client.delete_whitelist_ip(id) +``` + +The dates go as `yyyy-mm-dd`. The vendor's documentation says `dd-mm-yyyy` in +its prose and types the same fields as ISO dates two lines below; the type is +what the server parses. + +**`sub_users()` returns each sub-user's `proxy_password` in clear text**, on +every row, by the specification's own required-field list. So does `me()`. Do +not print a row of either, and do not paste one into an issue. + +Five of the vendor's twenty-four documented paths are deliberately not wrapped: +`locations/all-doc/`, `notifications/`, `llm/submit/`, `llm/results/{id}/` and +`llm/balance/`. Listing them is the point - an omission nobody wrote down reads +the same as an oversight. + +A list endpoint returns a `Page`, which iterates **one page** and not the +collection. `iterate()` gets the rest: + +```python +page = client.countries() # one page, 1000 rows by default +for country in client.iterate(page): # all of them + ... ``` +**`page.count` is `None` everywhere, and that is the API's shape rather than a +gap here.** No schema in the vendor's specification declares a `count` or a +`next` at all - `PaginatedCountryList` is `{"results": [...]}` and nothing else - +and a page of 50 drawn from a catalogue of nearly 200 countries came back with +all three fields empty. A full page is therefore indistinguishable from a +complete collection by looking at it, so `iterate()` asks for the next cursor +and reads the answer. + +**It stops on an empty page, not on a short one.** That rule changed on +2026-09-09 and the one before it was unsafe against this server in particular: +it stopped at the first page shorter than the size it had asked for, which is +wrong wherever the server caps the size below the request. `cities(limit=10000)` +is answered with 1000 rows out of 1965, and "shorter than asked" reads those +1000 as the end. The measurement that says so is four paragraphs up in this same +file and was already there - a measurement sitting in a document is not a +measurement anyone applied. Stopping on empty costs one spare request per walk +and cannot truncate. + +`limit` and `offset` are sent on your behalf and are overridable: + +```python +client.countries(limit=50) # four requests instead of one +``` + +They are sent rather than omitted because `isps()` **refuses** a request without +them, while the others answer one with a silently partial list. `offset` is +honoured: `offset=50` returns rows disjoint from `offset=0`, and an offset past +the end answers `200` with zero rows. + +`sub_users()` and `whitelist_ips()` are numbered by page instead, and **both +start at page 1**, measured 2026-09-09 - see +[the reference](#client-and-page). They mark the end of a collection +differently, so `iterate()` cannot use one rule for both: past the last page +`sub-users/` answers `200` with an empty payload and `whitelist/ips` answers +`404`. A `404` therefore ends the walk on the whitelist and nowhere else. + +`iterate()` raises rather than looping in four cases: a next-page url it has +already returned, more than `max_pages` pages, a page identical to the +one before it - which means the server took the cursor and ignored it - and a +next-page url on another host. None is a tuning knob; each is the difference +between a bug you can see and one that surfaces as a rate limit or a short +answer. + +### The ISP catalogue answers a different shape + +`isps()` returns a `Page` like the others and most callers can stop here. The +rest of this section is why it took two runs to get there. + +Its `200` is an object keyed `city`, `country`, `isps` and `region`, with no +`results` anywhere. The rows are the list under `isps` - 358 of them for +`country__code="us"`; the other three fields are strings and are not rows. +`isps()` reads that key and the other four list methods read `results`. That is +per-endpoint knowledge in the client rather than a page builder that goes +looking for whichever value happens to be a list, which is a rule decided by key +order the day a second list appears. + +For a day this method returned a `Page` of exactly one row - the envelope itself +- whatever the account held. Nothing was released in that state; it was written +up as broken rather than fixed, because the run that found it printed the key +*names* and not the values, and a key unwrapped because its name reads right is +how a parameter the gateway ignores once got into this package and a field the +server does not send got into this README. The fix waited for one command, and +was the same edit either way. + +### The whitelist needs a field the spec marks optional + +`upsert_whitelist_ip()` always sends `protocol`, defaulting to `"HTTP"`. The +vendor's OpenAPI document marks `ip` and `ports_count` required and gives +`protocol` a `default: "HTTP"`; the server does not apply that default. Measured +2026-09-09 at 16:18, one field at a time: + +| body | answer | +|---|---| +| `ip`, `ports_count`, `name` | `400 "Please enter a valid protocol(HTTP or SOCKS5)."` | +| the same plus `protocol: "HTTP"` | `201`, `{"ip_id", "message"}`, and a read of `whitelist/ips` found the address | + +So a client written from the specification sends the one body that fails. The +default here is a compensation for that, not a convenience, which is why +`protocol` is a keyword with a value rather than another `None`. + +Two things that run did not settle: `name` was present in every rung, so whether +it is also required is untested, and no rung tried `ip`, `ports_count` and +`protocol` alone, so that trio is not known to be a complete body. + +An earlier version of this section said only that the required pair was refused +and that which field was short was the server's word. That was accurate and it +was one call away from being a measurement. + +### Checking a Proxy against the catalogue + +The gateway answers a country it does not have with `407`, which reads as a +credentials problem, and does not say which parameter was wrong. The catalogue +knows, so it can be asked: + +```python +problems = client.validate(proxy) +if problems: + raise SystemExit("\n".join(problems)) +``` + +This is a method on `Client` and not a check inside `Proxy`, for one reason: a +refusal that ships in a release can be wrong forever, and the catalogue moves. +Asking the live catalogue cannot go stale - and it costs a network call, so it +has to be your decision rather than a hidden one. + +Two things are checked. `country` is matched against the catalogue, and **a +`city` sent without a `region` is refused before the request goes out** - that +one needs no network. The gateway answers a city with no region with `500`, +which reads as a fault on their side and is not one; the same city with its own +region answers `200`. The values of `region`, `city` and +`isp` are not matched against the catalogue, because city codes repeat across +regions - `aberdeen`, `albany` and `alexandria` each appear twice in a single +page - so matching a bare code would report success and mean nothing. + +### No dependencies, and your own client if you want one + +The transport is `urllib.request` from the standard library, so this adds nothing +to your dependency tree. If you would rather it went through the client you +already have, that is one function: + +```python +def transport(method, url, headers, body): + r = requests.request(method, url, headers=headers, data=body) + return r.status_code, r.content + +client = Client(transport=transport) +``` + +Return the status rather than raising on it; mapping statuses to exceptions is +this library's job, and doing it in both places is how a `NotFoundError` becomes +somebody else's exception halfway up a stack. + ## What this library does not do **It does not retry.** That is deliberate, and it is the one design decision @@ -293,13 +887,6 @@ attempts per delivered page, against 1.7 in a healthy session. A library that shipped automatic retry as a default would be spending that on your behalf without telling you. - - Those 1464 attempts, and the cells they came from, are in [nodemaven/proxy-benchmark](https://github.com/nodemaven/proxy-benchmark) - the harness that measured them, open source, so the table above can be re-run rather @@ -310,11 +897,24 @@ yours, and they are better than anything a vendor SDK would bundle. ## Other gateways -Parameters are data, not hardcoded keywords. A gateway is its prefix, -separators, session parameter and the set of parameter names it actually -recognises - and a definition written by you goes through the same builder and -the same validation as the one shipped here. Either build it in place, as in the -[quickstart](#quickstart), or keep it in a TOML file: +**No account here? Any proxy you already have works.** Parameters are data, not +hardcoded keywords. A gateway is its prefix, separators, session parameter and +the set of parameter names it actually recognises - and a definition written by +you goes through the same builder and the same validation as the one shipped +here. + +Build it in place: + +```python +from nodemaven import Provider, Proxy + +mine = Provider(id="mine", label="My proxy", known_params=frozenset()) +proxy = Proxy(provider=mine, login="u", password="p", + host="proxy.example.com", port=8000) +``` + +Describe the parameters it does take and it validates those too. Or keep the +definition in a TOML file: ```toml # my-gateway.toml @@ -366,9 +966,7 @@ believed to accept. ## License - + [MIT](https://github.com/nodemaven/nodemaven-python/blob/main/LICENSE). diff --git a/pyproject.toml b/pyproject.toml index 571d0a9..011ff50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,13 +33,16 @@ dependencies = ["tomli>=1.1.0; python_version < '3.11'"] # # `Source` and `Issues` were held back while nodemaven/nodemaven-python was # internal, because an internal repository answers 404 to an anonymous visitor -# exactly as a repository that does not exist does. They go in with the commit -# that makes it public and not before. +# exactly as a repository that does not exist does. That condition is gone: +# `gh api repos/nodemaven/nodemaven-python --jq .visibility` answered `public` +# on 2026-09-08, so both links resolve for a logged-out reader and can stay. # -# **They will not appear on the package page that is up now.** PyPI bakes this -# table into each uploaded distribution and never re-reads the repository, so -# 0.1.2's sidebar is fixed whatever this file says. The links arrive with the -# next release; the visibility flip on its own does nothing to them. +# **They still will not appear on the package page that is up now.** PyPI bakes +# this table into each uploaded distribution and never re-reads the repository, +# so 0.1.2's sidebar is fixed whatever this file says - re-read on 2026-09-08, +# it carries Homepage and Documentation and no Source. The links arrive with the +# next release; the visibility flip on its own did nothing to them, which is the +# whole reason this note is worth keeping. Homepage = "https://nodemaven.com" Documentation = "https://docs.nodemaven.com" Source = "https://github.com/nodemaven/nodemaven-python" diff --git a/src/nodemaven/__init__.py b/src/nodemaven/__init__.py index b9ffdbf..bbcb81c 100644 --- a/src/nodemaven/__init__.py +++ b/src/nodemaven/__init__.py @@ -15,13 +15,30 @@ failures returned three pages - 98 attempts per delivered page against 1.7 in a healthy session. A library that hid that behind a default would be spending a shared pool's reputation on your behalf. + +Two things here do touch the network, and they are the only two. Both are +explicit calls and neither happens on import: + + >>> proxy.check() # one CONNECT, and what the gateway said + >>> Client().me() # the account API: quota, usage, sub-users + +``Proxy`` itself still opens nothing. Keeping those on separate objects is what +lets the whole string-building half of this package stay testable with no socket +and no account - see ``nodemaven.check`` and ``nodemaven.api``. """ +from .api import Client, Page +from .check import Check from .errors import ( + ApiError, + AuthError, + CheckError, CredentialsError, NodeMavenError, + NotFoundError, ParamError, ProviderError, + RateLimitError, ) from .providers import Provider, available, load, load_file from .proxy import Proxy @@ -31,6 +48,9 @@ __all__ = [ "Proxy", "Provider", + "Client", + "Page", + "Check", "load", "load_file", "available", @@ -38,5 +58,10 @@ "ParamError", "CredentialsError", "ProviderError", + "ApiError", + "AuthError", + "NotFoundError", + "RateLimitError", + "CheckError", "__version__", ] diff --git a/src/nodemaven/api.py b/src/nodemaven/api.py new file mode 100644 index 0000000..ed6e30b --- /dev/null +++ b/src/nodemaven/api.py @@ -0,0 +1,1720 @@ +"""The account API: quota, usage, sub-users, whitelisted addresses, locations. + +Separate from :class:`~nodemaven.Proxy` and separately credentialled, because +they are two different products with two different secrets. A proxy login and +an API key are different strings from different places, and a package that +blurred them would tell you to fix the key when the password is wrong. + +**Zero required dependencies, on purpose.** The transport is +:mod:`urllib.request` from the standard library. A proxy SDK that pulled in an +HTTP client would be dictating one to a caller who already has one - and these +calls are made once at start-up, so there is nothing here that pays for the +dependency. If you want your own client, pass ``transport=``; the signature is +one function and :class:`Client` never touches a socket itself. + +**No async, and that is a decision rather than an omission.** ``users/me`` is +called once per process. Shipping a second code path for it means two sets of +bugs to keep in agreement for a call nobody makes in a loop. When somebody +shows a use that needs it, the transport seam above is already the place it +goes. + +Where the paths come from +------------------------- + +**The vendor publishes an OpenAPI 3.0.3 document and it is unauthenticated.** +``https://dashboard.nodemaven.com/documentation/v2/``, fetched 2026-09-09, +179194 bytes, sha256 beginning ``cc958d79a68d58d0``, 24 paths. A copy is kept at +``lab/notes/nodemaven_openapi_v2_20260909.json`` so that every claim below has a +fixed thing to be checked against rather than a live URL that moves. + +This module's paths were **transcribed from the vendor's Python client** on +2026-09-07 until that fetch, and five of them were wrong. What the mistake +looked like from the inside: the transcription was treated as a reading of the +API because it came from the vendor, and three sweeps of the dashboard's own +JavaScript bundles were then written to discover routes - while +``/documentation`` sat in the sweep's own output, twice, and was probed as if it +were an API resource instead of being opened. The general form is the one this +tree keeps recording: **a source that looks authoritative was allowed to stand +in for a measurement, and then a hard method was chosen over an easy one because +nobody asked what the cheapest check was.** + +**The spec is documentation and is wrong in at least two places**, so it is not +promoted to the status the vendor's client wrongly held: + +* ``info.description`` says requests go to + ``https://api.nodemaven.com/v2/base//``. Measured + unauthenticated 2026-09-09: that host answers ``users/me`` with **404 + text/html, 146 bytes, from nginx**, byte-identical to a deliberately + nonexistent path on it, while ``https://dashboard.nodemaven.com/api/v2/base/ + users/me`` answers **403 application/json**, + ``{"detail":"Authentication credentials were not provided."}``. A 403 is + routing succeeding and authentication refusing, so the route is there. + :data:`DEFAULT_BASE_URL` is right and the spec's description is the defect. +* ``statistics/*`` document their dates as ``"dd-mm-yyyy"`` in prose while + typing them ``format: date``, which is ``yyyy-mm-dd``. **The machine-readable + half is the wrong half**, measured 2026-09-09 by ``--phase 10``: + ``start=20-08-2026`` answers **200 with 21 data points**, ``start=2026-08-20`` + answers **400**, and the body of that 400 is byte-identical to the body for + ``start=not-a-date``. So ``format: date`` is not a second spelling the server + declines - it is a string the server cannot parse at all, and every client + generated from this document sends the one form that fails. + + This line said "unresolved; nothing here validates a date, so nothing here + depends on the answer" until that run. The second half was true and the first + was a reason not to look. What it cost is small here and would not be small in + a generated client: this module passes a date through untouched, so a caller + who reads the spec rather than this file writes the ISO form and gets a 400 + whose body says nothing about which of its three arguments was refused. + +And in one place the spec disagrees with a live run, which the run wins: the +location schemas name a field ``effective_availability``, and the 2026-09-08 +run read ``availability`` off ``regions`` and ``cities``. Nothing in this module +reads either. + +What was measured, and when +--------------------------- + +Sent to the live API on **2026-09-08** by ``lab/probes/probe_account_api.py``, +from a connection that does not route through this tree's VPN gateway: +``users/me``, ``locations/countries``, ``.../regions``, ``.../cities`` and +``.../isps``. + +Sent on **2026-09-09**, same probe, phase 4: the five paths this module had +never called. **Four of the five did not exist.** + +* ``locations/zip-codes/``, ``statistics/`` and ``whitelist-ips/`` answered + **200, text/html, 6415 bytes, byte-identical to a deliberately nonexistent + path**. Those three paths were inventions. The real ones are + ``locations/zipcodes/`` - solid, no separator - ``statistics/data/``, + ``statistics/requests/``, ``statistics/domains/``, and ``whitelist/ips``. +* ``statistics/domains/`` is real and answered 404 JSON for a missing argument. +* ``sub-users/`` is real and answers an **envelope** - + ``{success, description, errors, payload}`` - with no ``count`` and no + ``next``, so the ``rows_key="results"`` this module used could not read it and + :meth:`Client.iterate` could not walk it. That envelope is the spec's declared + 200 schema and not a server defect, which is how it was first written down + here. + +**That host answers an unregistered dashboard path with 200 and the front +end's HTML.** So *calling a path cannot tell you whether the path exists*, and +every probe against it compares a digest against a negative control. This is the +same defect as the gateway answering 200 to an unknown username parameter and +dropping it, which is what put ``norotate`` in this package. + +What is still not measured +-------------------------- + +* **The six write endpoints have never been called** - creating, updating, + deleting and resetting a sub-user, upserting a whitelist address, deleting + one - because each costs a real object on a production account. Their paths, + methods and body fields are the spec's, cross-checked against nothing. +* **The page-number base.** ``sub-users/`` and ``whitelist/ips`` page by page + number rather than by row offset, and neither the spec nor any run says + whether the first page is 0 or 1. This module refuses to guess - see + :class:`Paging` and :meth:`Client.iterate`. + +Twenty-four paths are in the spec and this module wraps nineteen of them. Not +wrapped, deliberately: ``locations/all-doc/``, ``notifications/``, +``llm/submit/``, ``llm/results/{id}/`` and ``llm/balance/``. The first is a +documentation dump, the second is dashboard furniture, and the last three are a +different product that happens to share a host. Listed rather than left out +silently, so the omission is a decision somebody can disagree with. +""" + +from __future__ import annotations + +import functools +import json +import os +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass, replace +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple + +from .errors import ApiError, AuthError, CredentialsError, NotFoundError, RateLimitError + +__all__ = [ + "Client", + "Page", + "Paging", + "Transport", + "DEFAULT_BASE_URL", + "API_ROOT", + "DEFAULT_PAGE_SIZE", + "BY_OFFSET", + "BY_PAGE_NUMBER", + "WHITELIST_PAGING", +] + +#: The dashboard host the API lives on. Overridden by ``NODEMAVEN_BASE_URL``. +#: +#: **Not ``api.nodemaven.com``**, which the vendor's own OpenAPI description +#: names. Measured 2026-09-09: that host 404s ``users/me`` from nginx with the +#: same 146 bytes it gives a nonexistent path, and this one answers 403 JSON. +#: See the module docstring. +DEFAULT_BASE_URL = "https://dashboard.nodemaven.com" + +#: Every path below hangs off this. Kept as one constant so a version bump is +#: one edit rather than nineteen. +API_ROOT = "/api/v2/base" + +#: Rows asked for per request on the ``limit``/``offset`` endpoints. +#: +#: It is sent rather than left to the server for two measured reasons, both +#: 2026-09-08. ``locations/isps`` **refuses** a request without ``limit`` and +#: ``offset`` - ``{'limit': 'This field is required.'}`` - and the spec marks +#: both required on that path alone, so omitting them is not a working default, +#: it is a broken endpoint. And the three location endpoints that do answer +#: without them returned exactly 50 rows and reported no total, which is a page +#: presented as if it were the whole collection. +#: +#: **1000 rather than 50, and the number is the largest one measured accepted by +#: all four endpoints.** ``limit=1000`` answers 200 everywhere; ``limit=10000`` +#: is refused with 400 by ``isps``, and on ``cities`` it returns exactly the +#: same 1000 rows that ``limit=1000`` does. +#: +#: **1000 is a hard server ceiling on ``limit``, and it truncates ``cities``.** +#: ``limit=1000&offset=1000`` returns a further 965 rows, so the US city +#: collection is 1965 and a single default call sees the first 1000 of them with +#: nothing in the answer saying so. Raising the default from 50 did not remove +#: that failure, it moved it one order of magnitude further out. +#: :meth:`Client.iterate` is the only call that returns all 1965; a bare +#: ``cities()`` is a page and has to be read as one. +#: +#: This comment claimed the ceiling, then withdrew it, then measured it. The +#: withdrawal was right: arms C and D both asked for more than they got, so a +#: ceiling at 1000 and a 1000-row collection looked identical, and the sentence +#: was resting on a reading the data could not carry. **Being right by luck is +#: not the same as being justified** - the claim came back only because arm H +#: was run, and it is the arm, not the guess, that it now rests on. +#: +#: The value was 50 for one day, taken from the size the server chose for +#: itself. At 1000 the whole country catalogue (194 rows on 2026-09-08) and a +#: country's regions (51) arrive in one request instead of four and two. +#: ``cities`` does not fit and needs :meth:`Client.iterate`. Pass ``limit=`` to +#: override it. +DEFAULT_PAGE_SIZE = 1000 + +#: What a transport has to be: ``(method, url, headers, body) -> (status, bytes)``. +#: +#: Note that it returns the status rather than raising on it. Error mapping is +#: this module's job, so a caller who plugs in ``requests`` does not have to +#: reproduce it - and, more to the point, so that a caller who plugs in something +#: that raises on 4xx does not turn a :class:`~nodemaven.errors.NotFoundError` +#: into their own library's exception halfway up the stack. +#: +#: There is no ``timeout`` in the signature. The default transport gets the +#: client's timeout bound into it at construction, and a caller who supplies +#: their own transport already has a timeout configured on whatever client they +#: built it from - passing ours in as well would give them two, of which only one +#: would take effect, and nothing in the signature would say which. +Transport = Callable[[str, str, Dict[str, str], Optional[bytes]], Tuple[int, bytes]] + +_REDACTED = "***" +_USER_AGENT = "nodemaven-python" + + +@dataclass(frozen=True) +class Paging: + """How one endpoint numbers its pages. + + **This API uses three conventions and they are not interchangeable**, read + off the spec on 2026-09-09: ``limit``/``offset`` on the nine location + endpoints, ``page``/``per_page`` on ``sub-users/``, and ``page``/ + ``page_size`` on ``whitelist/ips``. A single hard-coded pair would silently + do nothing on two thirds of the surface, because an unknown query parameter + is ignored by every REST framework there is - the caller would get page one + back forever and no error. + + ``first_cursor`` is the number the first page carries. ``limit``/``offset`` + has an unambiguous origin at 0; a page *number* can start at 0 or at 1 with + nothing in the response telling you which, and the two readings are not + symmetric, because guessing 1 against a 0-based server drops the first page + in silence. + + **Both page-number endpoints are 1-based, measured 2026-09-09** by + ``lab/probes/probe_account_api.py --phase 9``, asking each for pages 0, 1, 2 + and 9999 at a size of one row. On ``sub-users/`` the evidence is where the + rows are: page 0 came back with **none** and page 1 with the account's single + sub-user, which a 0-based server cannot produce. On ``whitelist/ips`` the + collection is empty, so the reading is the weaker one - page 1 is the only + number answered ``200`` at all, while 0, 2 and 9999 are answered ``404``, and + a 0-based server would have answered page 0. + + This field was ``None`` on both until that run, and :meth:`Client.iterate` + refused to walk them rather than guess. The refusal was right for the day it + shipped and it is the measurement that retires it, not a second opinion about + what servers usually do. + + ``default_size`` is ``None`` where no size has been measured as safe to + send. It is set on ``whitelist/ips`` because the spec states the numbers + outright - default 5, maximum 100 - and a default of 5 is a truncation + nobody would notice. + + ``ends_with_not_found`` says the server marks the end of this collection by + refusing the page rather than answering it empty. **The two endpoints that + number their pages do not agree**, measured in the same run on 2026-09-09: + ``sub-users/`` answers ``200`` with an empty payload past the end, and + ``whitelist/ips`` answers ``404``. So one rule cannot cover both, and + :meth:`Client.iterate` treats a ``404`` as the end only where this is set - + everywhere else a ``404`` stays an error, because it is also what a wrong + path looks like. + + The measured half is that ``whitelist/ips`` answers ``404`` for pages 0, 2 + and 9999 while page 1 is answered ``200``. That the same holds *after a full + page of rows* is an inference: the account holds no whitelist entries, so + nothing here has ever seen this endpoint end anywhere but at page 1. Put one + address on the account and it becomes a measurement. + """ + + size_key: str + cursor_key: str + cursor_counts_rows: bool + default_size: Optional[int] = None + first_cursor: Optional[int] = None + ends_with_not_found: bool = False + + +#: The nine ``locations/*`` endpoints, and the only convention with a measured +#: origin: ``offset=0`` is the start of the collection by definition. +BY_OFFSET = Paging( + size_key="limit", + cursor_key="offset", + cursor_counts_rows=True, + default_size=DEFAULT_PAGE_SIZE, + first_cursor=0, +) + +#: ``sub-users/``. No size is sent: the spec gives ``per_page`` no default and +#: no maximum, so any number here would be this package inventing a limit and +#: then reading the server's refusal of it as an empty account. +BY_PAGE_NUMBER = Paging( + size_key="per_page", + cursor_key="page", + cursor_counts_rows=False, + first_cursor=1, +) + +#: ``whitelist/ips``. ``page_size`` is sent at the spec's stated maximum, +#: because its stated default is **5** and five addresses arriving where the +#: account has forty is the failure this module exists to avoid. +WHITELIST_PAGING = Paging( + size_key="page_size", + cursor_key="page", + cursor_counts_rows=False, + default_size=100, + first_cursor=1, + ends_with_not_found=True, +) + + +@dataclass(frozen=True) +class Page: + """One page of a list endpoint. + + Iterating a ``Page`` iterates its ``results``, so the common case reads as + ``for country in client.countries():``. **That is one page and not the + collection** - use :meth:`Client.iterate` for the rest. + + ``count`` is the total the server reports behind all pages, and it is + ``None`` when the server reported none. **Measured 2026-09-08 and confirmed + against the spec 2026-09-09: it is always ``None`` here.** Not one of the + three paging envelopes the spec declares has a ``count`` field - + ``PaginatedCountryList`` and its siblings are ``{"results": [...]}`` and + nothing else, and ``SubUserManyResponse`` is ``{success, description, + errors, payload}``. So ``len(page)`` is the only number there is, and a page + exactly as long as the size that was asked for is indistinguishable from a + complete collection by looking at it. That is why :meth:`Client.iterate` + asks for one more page rather than trusting a missing ``next``. + + ``next`` and ``previous`` are not in any declared schema either and have + never been seen populated. They are still read, because a server may send + more than its own document promises and following a link the server wrote is + better than arithmetic - but nothing here waits for them. + + ``request_path``, ``request_params`` and ``paging`` record the call this page + came from, so ``iterate`` can advance the cursor on a server that sends no + ``next`` url. They are ``None`` on a page that was built from following one. + + ``rows_key`` is the field the rows came out of. **Six** values are in use: + ``results`` on the paginated catalogue and the whitelist, ``isps`` on + ``locations/isps/``, ``regions`` on the two ``*/regions/`` groupings, + ``cities`` on the two ``*/cities/`` groupings, ``payload`` on ``sub-users/`` + and ``data`` on ``statistics/domains/``. It is recorded rather than + re-derived so ``iterate`` reads the second page the same way it read the + first. + + All six were measured against the server on 2026-09-09 and every one of them + named a populated list: ``isps`` 359 rows, ``regions`` 51, ``cities`` 1000, + ``results`` 1000 on ``locations/zipcodes/`` and 17 and 805 on its two + groupings, ``payload`` 1 on ``sub-users/``. Only ``isps`` had been measured + before that, on 2026-09-08; the rest were read off the spec's own response + schemas. ``data`` on ``statistics/domains/`` is still schema-only, because + that endpoint needs a ``proxy_username`` and a date window to answer at all. + + A wrong value here yields an empty page rather than raising, so that is the + failure mode to expect if a grouping endpoint reports no rows on an account + that has them. + """ + + results: List[Any] + count: Optional[int] = None + next: Optional[str] = None + previous: Optional[str] = None + request_path: Optional[str] = None + request_params: Optional[Dict[str, Any]] = None + rows_key: str = "results" + paging: Optional[Paging] = None + + def __iter__(self) -> Iterator[Any]: + return iter(self.results) + + def __len__(self) -> int: + return len(self.results) + + +class Client: + """Talks to the account API. One instance, one API key. + + :: + + from nodemaven import Client + + client = Client() # NODEMAVEN_APIKEY from the environment + me = client.me() + print(me["data"]) # traffic left, measured 2026-09-08 + + The key falls back to ``NODEMAVEN_APIKEY`` and the base url to + ``NODEMAVEN_BASE_URL``. Both names are the vendor's own, so a ``.env`` written + for their client works here unchanged - which is worth more than a name of + our choosing, because the alternative is a developer with two ``.env`` files + that disagree. + """ + + __slots__ = ("_key", "_base", "_timeout", "_transport") + + def __init__( + self, + api_key: Optional[str] = None, + *, + base_url: Optional[str] = None, + timeout: float = 30.0, + transport: Optional[Transport] = None, + ) -> None: + self._key = api_key if api_key is not None else os.environ.get("NODEMAVEN_APIKEY") + if not self._key: + raise CredentialsError( + "no API key: pass api_key= to Client() or set NODEMAVEN_APIKEY in " + "the environment. This is the dashboard API key and not the proxy " + "password - they are different secrets and the API will answer 401 " + "to the wrong one." + ) + base = base_url if base_url is not None else os.environ.get("NODEMAVEN_BASE_URL") + self._base = (base or DEFAULT_BASE_URL).rstrip("/") + self._timeout = timeout + # The timeout is bound here rather than passed per call, so that + # `Transport` stays a four-argument function - see the note on it above. + # `timeout=` therefore applies to the default transport only, and a + # caller who passes their own owns theirs. + self._transport: Transport = ( + transport + if transport is not None + else functools.partial(_urllib_transport, timeout=timeout) + ) + + def __repr__(self) -> str: + """Never carries the key. Same reasoning as ``Proxy.__repr__``. + + An API key in a repr reaches CI logs and pasted bug reports without + anyone choosing to print it, because tracebacks and ``pytest + --showlocals`` print reprs of locals. + """ + return f"Client(base_url={self._base!r}, api_key={_REDACTED!r})" + + # -- the account -------------------------------------------------------- + + def me(self) -> Dict[str, Any]: + """The account: traffic left, subscription, and the proxy credentials. + + Returns the server's own object, unmodelled. The six fields it answered + with on 2026-09-08, all six of them also the spec's required list for + ``UsersRetrieveResponse``: + + ============================ ======================================== + field what came back + ============================ ======================================== + ``data`` ``int``. Traffic left. Not ``traffic_left`` + ``email`` ``str`` + ``is_traffic_frozen`` ``str``, **not a bool** - see below + ``proxy_password`` ``str``. The proxy password, not the key + ``proxy_username`` ``str`` + ``subscription_status`` ``str`` + ============================ ======================================== + + **``is_traffic_frozen`` is a string.** So ``if me["is_traffic_frozen"]:`` + is true whichever way it reads, and a caller who treats it as a flag + gets a branch that never takes its other arm. Compare it against the + value, not for truth. This was written down here as a server defect on + 2026-09-08; the spec types it ``string`` on purpose, so it is a design + decision to work around rather than a bug to report. + + **Deliberately not parsed into a dataclass**, and this response is the + argument for that rather than against it. A dataclass is a claim about + field names and types, and written a day earlier from the vendor's + client it would have declared ``traffic_left``, which does not exist, + and typed ``is_traffic_frozen`` as ``bool``, which it is not. The raw + dict was wrong about nothing, because it claimed nothing. + + **This response carries the proxy password in clear text.** Do not log + it whole. That is not a rule about this method, it is a rule about the + endpoint - the same field arrives from :meth:`sub_users` for every + sub-user on the account. + """ + return self._get(f"{API_ROOT}/users/me") + + # -- where you can exit from -------------------------------------------- + # + # The catalogue. Two things it is for, and the second is the interesting one. + # + # It answers "what can I ask for", which is otherwise guesswork: a bad + # `country` is answered 407, a bad `region` 406, any `city` 500 and a bad + # `isp` 410, and not one of them names the parameter that was wrong. And it + # is the data that would fill the `values` table in the + # provider TOML, which ships empty because "the data to fill it with does not + # exist here" - true of this tree and not true of the product. + # + # The `city` row is the one this catalogue changed the reading of. Measured + # 2026-09-08: `cities()` returns real codes in the gateway's own wire form - + # `alexander_city`, `altamonte_springs` - so the 500 that every city value + # drew is not a spelling problem, which was one of the two live readings + # until then. What it is instead - the account, the pool, or the gateway - + # is not established, and the codes to settle it are now in hand. + # + # Also measured that day, and it constrains what a city check could look + # like: **city codes repeat across regions.** `aberdeen`, `albany` and + # `alexandria` each appeared twice in one 50-row page. So a city identifies + # a place only together with its region, and any future `validate()` city + # check has to match the triple rather than the code. + # + # It is deliberately **not** frozen into that TOML. A snapshot of a live + # catalogue is a false refusal waiting for the day a country is added: the + # SDK would reject a place the gateway serves, loudly, with our name on the + # error. `validate()` below checks against the catalogue as it is now + # instead, which cannot go stale and cannot ship a refusal. + # + # `country__code` is marked required by the spec on every one of these + # except `countries`. It is not enforced here: the server answers a missing + # one with a 400 naming the field, which is a better message than any this + # package would write, and a client-side copy of a server-side rule is a + # second thing to keep in agreement. + + def countries(self, **filters: Any) -> Page: + """Countries available for proxy connections. + + ``connection_type`` defaults to ``residential`` **on the server**, not + here, so pass ``connection_type="mobile"`` to see the mobile pool. That + default is the vendor client's and is repeated by every method below; + this package does not add one of its own, because a default sent is a + default that shows up in a support ticket as something the caller chose. + """ + return self._list(f"{API_ROOT}/locations/countries/", filters) + + def regions(self, **filters: Any) -> Page: + """Regions. Filter with ``country__code="us"`` - two underscores. + + The double underscore is Django's field-lookup separator and it is the + server's spelling, not a typo here. It is passed through untranslated: + renaming it to ``country_code`` would be one alias to maintain, one thing + for the docs to disagree about, and one more place a caller's filter can + be dropped silently, because an unknown query parameter is ignored by + every REST framework there is. + """ + return self._list(f"{API_ROOT}/locations/regions/", filters) + + def cities(self, **filters: Any) -> Page: + """Cities. Filter with ``country__code`` and ``region__code``. + + **This is the one list endpoint a single call does not finish.** The + server ceiling on ``limit`` is 1000 and ``country__code="us"`` has 1965 + rows, measured 2026-09-08, so a bare call here returns a page that looks + exactly like a complete answer and is not one. Use + :meth:`iterate` when you want the collection. + """ + return self._list(f"{API_ROOT}/locations/cities/", filters) + + def isps(self, **filters: Any) -> Page: + """ISPs. Filter with ``country__code``, ``region__code``, ``city__code``. + + **This endpoint uses a different envelope from the others and the rows + are under ``isps``.** Measured 2026-09-08 with ``country__code="us"``: + the 200 is an object keyed ``city``, ``country``, ``isps`` and + ``region``, with no ``results`` at all, and ``isps`` is a list - 50 rows + at ``limit=50``, 358 at ``limit=1000``, 0 at an offset past the end. The + spec declares exactly that as ``LocationsISPsResponse``, with the other + three fields typed ``string``: they are the query echoed back, not rows, + and this method does not return them. + + For a day this method was broken, returning a :class:`Page` of exactly + one row - the envelope itself - whatever the account held. No release + carried it. It was documented rather than fixed on purpose, because the + first probe printed the key *names* and not the values, and unwrapping a + key because its name looks right is the move that put ``norotate`` in + this package and ``traffic_left`` in its README. The fix waited for one + command. That is the whole difference between this and those two: the + same edit, made after the measurement instead of before it. + """ + return self._list(f"{API_ROOT}/locations/isps/", filters, rows_key="isps") + + def isp_regions(self, **filters: Any) -> Page: + """Regions that have ISPs, grouped. Needs ``country__code``. + + Rows are under ``regions`` and each carries its own ISP list. Spec + shape, never called. + """ + return self._list( + f"{API_ROOT}/locations/isps/regions/", filters, rows_key="regions" + ) + + def isp_cities(self, **filters: Any) -> Page: + """Cities that have ISPs, grouped. Needs ``country__code``. + + Rows are under ``cities``. Spec shape, never called. + """ + return self._list( + f"{API_ROOT}/locations/isps/cities/", filters, rows_key="cities" + ) + + def zip_codes(self, **filters: Any) -> Page: + """ZIP codes. Needs ``country__code``. + + **The path is ``locations/zipcodes/``, solid and with no separator.** + This method sent ``locations/zip-codes/`` until 2026-09-09, transcribed + from the vendor's client, and that path does not exist: it answered 200 + with 6415 bytes of the dashboard's HTML, byte-identical to a path nobody + ever registered. There is no naming rule to extrapolate from on this + API - ``sub-users`` is hyphenated, ``zipcodes`` is not, ``users/me`` + carries no trailing slash where everything around it does - so a + spelling that was not read off the spec or a run is a lottery ticket. + + Note what this is **not**: ``zip_code`` is not in any ``known_params`` + list in this package, so the gateway may or may not accept it as a + targeting parameter. The catalogue existing says the product knows about + ZIP codes; it does not say the proxy username carries them. That is a + name to probe and not a name to add. + """ + return self._list(f"{API_ROOT}/locations/zipcodes/", filters) + + def zip_code_regions(self, **filters: Any) -> Page: + """Regions that have ZIP codes, grouped. Needs ``country__code``. + + Rows are under ``regions``. Spec shape, never called. + """ + return self._list( + f"{API_ROOT}/locations/zipcodes/regions/", filters, rows_key="regions" + ) + + def zip_code_cities(self, **filters: Any) -> Page: + """Cities that have ZIP codes, grouped. Needs ``country__code``. + + Rows are under ``cities``. Spec shape, never called. + """ + return self._list( + f"{API_ROOT}/locations/zipcodes/cities/", filters, rows_key="cities" + ) + + # -- what you used ------------------------------------------------------ + # + # Three endpoints, not one. This module had a single `statistics()` on + # `/statistics/`, transcribed, and that path does not exist - it answered + # 200 with the dashboard's HTML on 2026-09-09, the same 6415 bytes a + # nonexistent path gets. + # + # All three require `proxy_username`, which is why it is a positional + # argument here rather than one more entry in `**filters`: it is the + # difference between a 400 at run time and a TypeError at the call site. + # The value is the sub-user's proxy login - `me()["proxy_username"]` for the + # account's own. + # + # The shared optional filters, from the spec: `timezone` (default "UTC"), + # `start` and `end` dates, `period` in {"today", "hours24"} and + # `request_source` in {"proxy", "browser"}. + # + # **Dates are `dd-mm-yyyy`**, measured 2026-09-09 by `--phase 10`: + # `start=20-08-2026` answers 200 with 21 data points, `start=2026-08-20` + # answers 400, and that 400's body is byte-identical to the one for + # `start=not-a-date`. The spec writes it both ways - "dd-mm-yyyy" in prose + # against `format: date` in the type - and the typed half is the wrong half, + # so anything generated from the document sends the form that fails. + # + # Nothing here validates or reformats a date, and that is deliberate rather + # than unfinished: the server checks them anyway, this module cannot tell a + # naive `date` from a string, and a client-side reformat would have to guess + # a caller's intent for `01-02-2026`. What the measurement buys is the + # docstrings below saying which form to write. + # + # All three answer **500**, not 400, when `start`, `end` and `period` are all + # omitted, though the document marks all three optional. So there is no + # "just give me everything" call here; send a `period` or a date range. + + def statistics_data(self, proxy_username: str, **filters: Any) -> Dict[str, Any]: + """Traffic over time. Returns ``{"labels": [...], "data": [...]}``. + + Two parallel arrays rather than a list of points, so ``labels[i]`` names + ``data[i]``. Not zipped here: the caller who wants a chart wants the + arrays, and the caller who wants pairs writes one ``zip``. + + Dates are ``dd-mm-yyyy`` and **not** ISO - ``start="20-08-2026"``, not + ``"2026-08-20"``, which is answered 400. Send a ``period`` or a range; + omitting ``start``, ``end`` and ``period`` together answers 500. Both + measured 2026-09-09, see the comment above this block. + """ + return self._get( + f"{API_ROOT}/statistics/data/", + dict(filters, proxy_username=proxy_username), + ) + + def statistics_requests(self, proxy_username: str, **filters: Any) -> Dict[str, Any]: + """Request counts over time. Same two-array shape as + :meth:`statistics_data`.""" + return self._get( + f"{API_ROOT}/statistics/requests/", + dict(filters, proxy_username=proxy_username), + ) + + def domain_statistics(self, proxy_username: str, **filters: Any) -> Page: + """Usage split by target domain. Rows carry ``domain_name``, + ``requests`` and ``data``. + + **Not paginated**, whatever its previous docstring here said. The + response is ``{"data": [...]}`` with no cursor of any kind, and ``limit`` + is a top-N cut rather than a page size - there is no ``offset`` in the + spec's parameter list. So the :class:`Page` this returns carries no + paging convention and :meth:`iterate` over it yields exactly these rows + and stops. It is a ``Page`` at all only so that iterating the result + reads the same as iterating the catalogue. + """ + return self._list( + f"{API_ROOT}/statistics/domains/", + dict(filters, proxy_username=proxy_username), + rows_key="data", + paging=None, + ) + + # -- sub-users ---------------------------------------------------------- + # + # How agencies and resellers actually use a proxy account: one plan, many + # credentials, a traffic cap on each. This is the part of the API with side + # effects, and the five methods below are the only ones in this package that + # change anything anywhere. **None of the five has ever been called** - each + # costs a real object on a production account - so their bodies are the + # spec's and nothing more. + # + # Every one of them answers with the same envelope: + # `{success, description, errors, payload}`. `payload` is a list on the + # collection reads and a single object on create and update. It is unwrapped + # here rather than handed to the caller, because a caller who has to know + # about `payload` has to know about it at nine call sites. + + def sub_users(self, **filters: Any) -> Page: + """The sub-users on this account. Pass ``id=`` for one of them. + + **Rows arrive under ``payload``, not ``results``.** Measured 2026-09-09 + and declared by the spec as ``SubUserManyResponse``: the 200 is + ``{success, description, errors, payload}``, with no ``count`` and no + ``next``. This module read ``results`` until that run, so this method + returned an empty page against a populated account and said nothing. + + **Every row carries ``proxy_password`` in clear text.** That is the + spec's own required field list, not an accident of one account. Anything + that logs, prints or serialises these rows whole is publishing working + credentials for every sub-user at once. A probe in this tree did exactly + that on 2026-09-09 and the account's password had to be rotated. + + Pages by ``page``/``per_page``, from **page 1**, measured 2026-09-09: + page 0 came back with no rows and page 1 with the account's single + sub-user. Past the end it answers ``200`` with an empty payload, which is + what :meth:`iterate` stops on. No ``per_page`` is sent - see + :class:`Paging`. + """ + return self._list( + f"{API_ROOT}/sub-users/", + filters, + rows_key="payload", + paging=BY_PAGE_NUMBER, + ) + + def create_sub_user( + self, + proxy_username: str, + proxy_password: str, + *, + traffic_limit: Optional[int] = None, + is_traffic_limited: Optional[bool] = None, + **extra: Any, + ) -> Any: + """Create a sub-user. Returns the created object. + + The two credentials are positional because there is no sensible default + for either and a keyword-only signature would let a caller create one + with an empty password by forgetting an argument. + + **The field names are ``proxy_username`` and ``proxy_password``.** They + were ``username`` and ``password`` here until 2026-09-09, transcribed; + the spec marks both of the real names required, so the old body would + have been refused with a 400 naming two fields that were in fact sent + under other names - a message that reads as "you forgot these" when what + happened is "we called them something else". + + ``**extra`` is passed through untouched, which is now a smaller promise + than it was: the spec's field list is closed at four, so an extra field + is a bet on the server accepting one it does not document. + + **Sent live 2026-09-09** with the two credentials and nothing else, and + the two field names are no longer transcription: the server answered + **201** - not the 200 the rest of this API answers with - and a read-back + of the collection found the username. The payload carried ``id``, + ``is_default_user``, ``is_traffic_limited``, ``proxy_password``, + ``proxy_username`` and ``traffic_limit``, which measures the thing the + credential rule was written from: **the create response hands back a live + proxy password**, so its body is a credential and not a receipt. + """ + body: Dict[str, Any] = { + "proxy_username": proxy_username, + "proxy_password": proxy_password, + } + if traffic_limit is not None: + body["traffic_limit"] = traffic_limit + if is_traffic_limited is not None: + body["is_traffic_limited"] = is_traffic_limited + body.update(extra) + return self._envelope("POST", f"{API_ROOT}/sub-users/", body=body) + + def update_sub_user(self, sub_user_id: Any, **changes: Any) -> Any: + """Change a sub-user. Returns the updated object. + + **PUT to the collection with the id in the body**, not PATCH to a path + segment. That is what the spec declares - there is no + ``sub-users/{id}/`` path at all - and it is what this method sent until + 2026-09-09. The old spelling would have been a 404 or, on a server that + answers unknown paths with its front end, a 200 carrying HTML. + + The body fields are ``proxy_username``, ``proxy_password``, + ``is_traffic_limited``, ``traffic_limit``, ``used_traffic`` and + ``traffic_limit_increment_bytes``. Only ``id`` is required, so this + behaves like a PATCH despite the verb: unlisted fields are left alone. + + Refuses an empty change set rather than sending a body of nothing but an + id, which a server can answer 200 to - and a call that reports success + while changing nothing is the failure mode this whole package is + organised against. + + **Sent live 2026-09-09** and it is the one write here where the status was + not the evidence. ``traffic_limit`` was set to 777000111, a number nothing + else on the account would produce, the server answered 200, and a + *separate read of the collection* came back with 777000111 in the row. So + PUT-to-the-collection both is accepted and takes effect. The response + payload also carries ``used_traffic``, which the create response does not, + so the two differ in shape and neither is a subset of the row the listing + returns. + """ + if not changes: + raise ApiError( + f"update_sub_user({sub_user_id!r}) was given nothing to change. " + f"A body carrying only an id can be answered 200, so this would " + f"look like it worked and do nothing." + ) + body = dict(changes) + body["id"] = sub_user_id + return self._envelope("PUT", f"{API_ROOT}/sub-users/", body=body) + + def delete_sub_user(self, sub_user_id: Any) -> Any: + """Delete a sub-user. Irreversible, and nothing here asks twice. + + **The id is a query parameter and not a path segment**, which is the + spec's shape and was not this module's until 2026-09-09. + + **Sent live 2026-09-09**: 200, with a JSON body where the document + declares 204. The query-parameter shape is therefore measured rather than + read off the document, which matters more here than on the other writes - + a DELETE aimed at a path this host does not serve is answered 200 with + the dashboard's HTML, so the wrong spelling would have looked exactly + like a successful delete that left the sub-user in place. + + **The row being gone rests on a later read, not on that run.** This + docstring said "a read-back of the collection no longer found the + username" and offered it as the proof; the whitelist delete in the same + phase produced the identical pair - 200, then a listing that did not show + the row - and five minutes later the server refused to re-create that + address as already whitelisted, so the pair had been believed once + already and had been wrong once already. + + What the mistake looked like from the inside: the read-back *was* the + control - it is what this file added precisely because a status code is + not evidence on this host - so once it agreed there was nothing left to + doubt. The gap is that it runs one second after the write and answers + "what does the listing say now", while the claim being made is "the + object no longer exists". + + The second question was asked at 16:15 by + ``lab/probes/probe_account_api.py --phase 12``, an hour after the write, + and no ``proxy_username`` on the account begins ``probe_delete_me_``. The + whitelist half of the same worry then resolved as a stale uniqueness + check rather than a surviving row - see :meth:`delete_whitelist_ip` - so + the mechanism that would have hidden a live sub-user from a listing is + the one that was ruled out. This is still a listing read and not a + guarantee about storage; it is one taken far enough after the write that + the failure mode above cannot account for it. + """ + return self._envelope( + "DELETE", f"{API_ROOT}/sub-users/", params={"id": sub_user_id} + ) + + def reset_sub_user_usage(self, ids: List[Any]) -> Any: + """Zero the recorded traffic of one or more sub-users. + + Takes a list because the endpoint does - ``ids`` is the spec's only + required field and it is an array. A single id still goes in a list, so + that the one-and-many cases cannot diverge. + + **Sent live 2026-09-09** with one id: 200, and the payload is a **list** + where every other envelope on this API carries an object. :meth:`_envelope` + returns it as it comes, so a caller indexing it by key gets a + ``TypeError`` rather than a ``KeyError``. What that list holds was not + read - the response body of an endpoint that touches usage records is not + something to print - so treat the return value as unknown-shaped. + """ + return self._envelope( + "POST", f"{API_ROOT}/sub-users/reset/usage", body={"ids": list(ids)} + ) + + # -- authorising by address instead of by password ---------------------- + + def whitelist_ips(self, **filters: Any) -> Page: + """The addresses allowed to use this account without a password. + + **The path is ``whitelist/ips`` with no trailing slash.** This module + sent ``whitelist-ips/`` until 2026-09-09; that path does not exist and + answered 200 with 6415 bytes of dashboard HTML, which :func:`_page` + would have refused as "neither a page nor a list" - the one of the four + invented paths that would have failed loudly rather than quietly. + + Pages by ``page``/``page_size``, from **page 1**. ``page_size`` is sent + at 100 because the spec's stated default is **5**. + + **This endpoint ends its collection with a ``404``**, not with an empty + page, measured 2026-09-09 on an account whose whitelist is empty: page 1 + answers ``200`` with no rows while pages 0, 2 and 9999 answer ``404``. + :meth:`iterate` therefore treats a ``404`` here - and only here - as the + end of the walk. See :class:`Paging` for which half of that is measured + and which is inference. + """ + return self._list( + f"{API_ROOT}/whitelist/ips", filters, paging=WHITELIST_PAGING + ) + + def whitelist_ip(self, ip_id: Any) -> Dict[str, Any]: + """One whitelisted address by id.""" + return self._get(f"{API_ROOT}/whitelist/ip/{_segment(ip_id)}") + + def upsert_whitelist_ip( + self, + ip: str, + ports_count: int, + *, + name: Optional[str] = None, + protocol: str = "HTTP", + sticky: Optional[bool] = None, + ttl: Optional[int] = None, + **extra: Any, + ) -> Dict[str, Any]: + """Add or update a whitelisted address. Returns ``{"ip_id", "message"}``. + + **Named for what it does.** It was ``add_whitelist_ip`` posting to + ``whitelist-ips/`` with a body of ``ip_address`` and ``description``; + the endpoint is ``whitelist/ip/upsert``, its required fields are ``ip`` + and ``ports_count``, and passing ``id=`` updates an existing row instead + of adding one. A method called ``add`` that silently updates is a worse + bug than a wrong path, because the wrong path fails. + + ``ports_count`` has no default here. It is required by the spec and it + decides how many proxy ports the address gets, which is not a number + this package can pick on a caller's behalf. + + ``protocol`` is ``HTTP`` or ``SOCKS5`` and is always sent - see below. + The other documented fields are ``id``, ``type`` (``residential`` or + ``mobile``), ``quality_filter_enabled``, ``country``, ``region``, + ``city`` and ``isp``; pass them through ``**extra``. + + **The two fields the spec marks required are not enough**, measured + 2026-09-09: a body of ``ip`` and ``ports_count`` alone is refused ``400`` + with ``{"error": "Please enter a valid protocol(HTTP or SOCKS5)."}``. A + body carrying every documented field at its documented default is accepted + ``201``, and a read of ``whitelist/ips`` afterwards found the address. So + the endpoint works and the document's ``required`` list is short. + + The same run killed the likelier of the two explanations. ``not-an-ip`` + sent as the address drew that message byte for byte, so ``protocol`` is + validated before ``ip`` is looked at, and "the reserved test address was + rejected" is dead - the accepted body used ``192.0.2.7``, which makes the + address positively fine rather than merely unexamined. + + **The missing field is ``protocol``, measured 2026-09-09 at 16:18** by a + one-field-at-a-time ladder in ``lab\\probes\\probe_account_api.py``. This + docstring said until then that the accepted body varied six fields at + once, so which one was short was the server's word rather than a + measurement; that was right to say and it was one call away from being + settled. Adding ``protocol: "HTTP"`` and nothing else flipped ``400`` to + ``201``, ``95`` bytes, and the read-back found the address. + + So the server does not apply the ``default: "HTTP"`` its own document + declares for that field, and ``protocol`` is sent here on every call + rather than added to the caller's burden - this default is a client-side + compensation for a server-side defect, not a convenience. + + What the ladder held fixed: every rung carried ``name``, including the + one the probe labels "the spec's required pair", which therefore was not + the pair. Whether ``name`` is also required is untested, and so is + whether ``ip``, ``ports_count`` and ``protocol`` are a *complete* body. + + **Returns ``{"ip_id": ..., "message": ...}``**, measured in the same run. + The identifier is ``ip_id`` and not ``id`` as everywhere else here, and + neither key is in the document: the spec binds both ``200`` and ``201`` on + this path to ``SuccessResponse``, whose only property is ``message``. Pass + ``ip_id`` to :meth:`delete_whitelist_ip`. + + Worth knowing what this changes about everything else in this package: + with an address whitelisted, the gateway accepts requests from it without + proxy credentials. :class:`~nodemaven.Proxy` still requires a login and + password to construct, because the username is where every targeting + parameter travels - the login half is load-bearing even when + authentication is not. + """ + body: Dict[str, Any] = { + "ip": ip, + "ports_count": ports_count, + "protocol": protocol, + } + if name is not None: + body["name"] = name + if sticky is not None: + body["sticky"] = sticky + if ttl is not None: + body["ttl"] = ttl + body.update(extra) + return self._get_object( + "POST", f"{API_ROOT}/whitelist/ip/upsert", body=body + ) + + def delete_whitelist_ip(self, ip_id: Any) -> Any: + """Remove an address from the whitelist. + + The id is the ``ip_id`` :meth:`upsert_whitelist_ip` returns, or the ``id`` + of a row from :meth:`whitelist_ips` - the create response and the listing + spell the same identifier differently. + + **Sent live 2026-09-09**: ``200``, ``{"message": ...}``. The document + declares ``200``, ``404`` and ``500`` here, so this one matches. + + **The delete is real and the uniqueness check lags behind it**, settled + 2026-09-09 at 16:18. Two earlier versions of this paragraph were wrong in + opposite directions and both are worth keeping. + + The first said "a read of ``whitelist/ips`` afterwards no longer found + the address", offered as proof the object was gone. At 15:18 the server + refused to whitelist that same ``192.0.2.7`` with + ``400 IP is already whitelisted.``, before anything in that run had been + written - so the delete at 15:13 plus a clean listing one second later + had proved nothing. + + The second read that refusal as evidence the delete might be soft or + uncommitted, and named an invisible surviving row as one of two live + candidates. At 16:18 the same address was accepted ``201``. A row the + listing cannot see and the uniqueness check can would still have blocked + it, so that candidate is dead: the object was removed, and what was stale + is the uniqueness check. + + What is measured about the lag is only its bounds, and they are loose. + It was still refusing at 5 minutes and was over by 62 minutes; nothing + here narrows that, because the only two writes against the address were + an hour apart. The rule for a caller is unchanged by the resolution: a + ``200`` from here means the row is gone from the listing, and re-adding + the same address within the hour may still be refused as a duplicate. + """ + return self._request( + "DELETE", f"{API_ROOT}/whitelist/ip/{_segment(ip_id)}" + ) + + # -- paging ------------------------------------------------------------- + + def iterate(self, page: Page, *, max_pages: int = 100) -> Iterator[Any]: + """Every item from ``page`` onward, across pages. + + Two ways forward, because this API might use both. A ``next`` url is + followed when the server sends one. When it does not - and no declared + schema in the spec has a ``next`` field, so it never does - the walk + continues by asking the same call again at the next cursor, which is + ``offset + limit`` on the catalogue and ``page + 1`` on the two + page-number endpoints. + + **The walk stops on an empty page and not on a short one.** That rule + changed on 2026-09-09 and the old one was unsafe against this server in + particular. It used to stop on a page shorter than the size that had + been asked for, which is wrong wherever the server caps the size below + the request: ``cities(limit=10000)`` is answered with 1000 rows out of + 1965, and "shorter than asked" reads those 1000 as the end of the + collection. The measurement that says so was already written in this + file - ``limit=10000`` returning the same 1000 rows as ``limit=1000`` - + and the stop rule was never checked against it. **A measurement sitting + in a docstring is not a measurement anyone applied.** + + Stopping on empty costs one extra request per walk and cannot truncate. + + **An empty ``next`` is not read as the end of the collection** either, + and that is the point of this method rather than a detail of it. A full + page with no total and no next url is byte-for-byte what a complete + answer looks like, so treating it as complete is a truncation the caller + cannot detect. + + The tempting exception is to trust an empty ``next`` when it arrives + *inside a paging envelope*, on the reasoning that there the server has + answered the question rather than stayed silent. This method deliberately + does not, and the reason is now stronger than the measurement it was + first written from: **no envelope the spec declares has a ``next`` field + at all**, so a ``None`` there is a key that was never going to be filled. + + That exception was in fact implemented for a few hours on 2026-09-08, + gated on a ``Page.enveloped`` flag, after a unit test built on a + hand-written Django REST Framework envelope failed against it. The test + was pinning an inferred shape, the flag then met the real one, and the + result would have been every catalogue read stopping after one page. + + **A page-number endpoint is walked from page 1**, measured 2026-09-09 - + see :class:`Paging`. Until that run this method *refused* to walk + ``sub-users/`` and ``whitelist/ips`` at all unless the caller named a + page, because guessing 1 against a 0-based server drops the first page in + silence and nothing said which this server was. What retired the refusal + is the measurement and not a second opinion about what servers usually + do; ``Paging.first_cursor`` carries it and the refusal is gone from the + code rather than left behind a flag. + + **A ``404`` ends the walk only on ``whitelist/ips``.** The two + page-number endpoints mark the end differently - ``sub-users/`` with an + empty page, that one with a refusal - so there is no single rule, and + treating ``404`` as an ending everywhere would swallow a wrong path. It + is not ambiguous where it is allowed: the same url answered this walk's + previous page, so the path exists and only the number changed. + + Three further refusals, none of them tuning knobs: + + * ``max_pages`` bounds the walk. A server returning a ``next`` that + points at the page you are on turns ``while next:`` into an infinite + loop of real requests, whose first symptom is a rate limit rather than + a hang. Reaching the bound raises rather than returning a truncated + list. + * A ``next`` already returned is a loop, and raises. + * A page identical to the one before it means the cursor was accepted + and ignored, and raises. **Cursor paging is an inference on every + endpoint here**: ``offset`` was measured honoured on ``countries``, + ``regions`` and ``cities`` on 2026-09-08 - ``offset=50`` returns rows + disjoint from ``offset=0`` - and on ``isps`` only the weaker half is + measured, that an offset past the end returns zero rows. On the two + page-number endpoints nothing at all is measured. Without this guard, + a server that ignores the cursor yields the same page a hundred times + and calls it a collection. + + **A ``next`` pointing at another host is refused rather than followed**, + in :func:`_is_same_origin`, because the request that follows it carries + the API key in a header. That guard and the loop guard answer different + questions and neither covers the other: one is about a server that + repeats itself, the other about a server - or something answering in its + place - that sends the reader somewhere else. + """ + pages = 0 + current: Optional[Page] = page + seen_urls = set() + while current is not None: + for item in current.results: + yield item + + url = current.next + step = None if url else _next_step(current) + if not url and step is None: + return + + pages += 1 + if pages >= max_pages: + raise ApiError( + f"stopped after {max_pages} pages, which is the bound in " + f"iterate(max_pages=). Raise it deliberately if the " + f"collection really is this large, rather than letting a " + f"paging bug run." + ) + + if url: + if url in seen_urls: + raise ApiError( + f"the API returned a next page url it had already " + f"returned ({url!r}), so following it is a loop. " + f"Stopped after {pages} pages." + ) + seen_urls.add(url) + current = _page(self._request("GET", url), current.rows_key) + continue + + path, asked = step # type: ignore[misc] + try: + body = self._request("GET", path, params=asked) + except NotFoundError: + if not current.paging.ends_with_not_found: # type: ignore[union-attr] + raise + return + following = _page(body, current.rows_key) + if following.results and following.results == current.results: + cursor = current.paging.cursor_key # type: ignore[union-attr] + raise ApiError( + f"asking {path} for {cursor}={asked[cursor]} returned the " + f"same {len(following.results)} rows as {cursor}=" + f"{current.request_params[cursor]}, so the server is " # type: ignore[index] + f"accepting `{cursor}` and ignoring it. Every further page " + f"would repeat these rows. Ask for the whole collection in " + f"one request instead." + ) + current = replace( + following, + request_path=path, + request_params=asked, + paging=current.paging, + ) + + # -- checking a Proxy against the live catalogue ------------------------ + + def validate(self, proxy: Any) -> List[str]: + """Check a :class:`~nodemaven.Proxy`'s location against the catalogue. + + Returns a list of complaints, empty if everything resolved. This is the + gap the ``values`` table in the provider TOML leaves open: the SDK + refuses a parameter *name* it does not know, and passes any *value* + through, so ``country="zz"`` builds a username and gets a 407 that reads + as a credentials problem and does not name the cause. + + It is a method on the client rather than a check inside ``Proxy``, and + the reason is the one recorded against ``values``: a refusal that ships + in a release is a refusal that can be wrong forever, and the catalogue + moves. Asking the live catalogue cannot go stale, and it costs a network + call, so it has to be the caller's decision to make and not a hidden one. + + Two things are checked today - the country against the catalogue, and + ``city`` without ``region``, which needs no network at all:: + + problems = client.validate(proxy) + if problems: + raise SystemExit("\\n".join(problems)) + + **``city`` requires ``region``.** Measured 2026-09-08 by + ``probe_gateway_city_from_catalogue.py``, six CONNECTs holding the login, + the password, the target, the gateway host and port and the parameter + order fixed: ``country=us, region=louisiana, city=abbeville`` answers + 200, and the same city with the region removed answers **500**. A second + city in a second region, ``maryland/aberdeen``, answers 200 as well, so + it is not one pool. This check is here rather than in ``Proxy`` for the + reason above - it is a shipped rule that can change, and it can be wrong + forever if a release refuses on it. + + ``region``, ``city`` and ``isp`` values are still **not** matched against + the catalogue, and the reason is no longer that the field names are + unknown. Measured 2026-09-08: ``regions`` answers ``availability, code, + country, name`` and ``cities`` answers ``availability, code, country, + name, region``, so a region check is now the same one line the country + check is. The spec calls that field ``effective_availability`` on all + three schemas, which is a disagreement nobody has resolved and which + nothing here depends on. + + What stops the city one is in that field list. City codes repeat across + regions - ``aberdeen``, ``albany`` and ``alexandria`` each twice in a + single page - so matching a bare code would pass a city that exists in + some other state, which is a check that reports success and means + nothing. It needs the ``region`` field beside it. That the gateway + resolves a city the same way is now known: an invented name sent with a + real region answers **406**, the code a bad ``region`` and a bad ``isp`` + also get, so the gateway does look the name up and does distinguish one + it holds from one it does not. + """ + problems: List[str] = [] + params = proxy.params + if params.get("city") and not params.get("region"): + problems.append( + f"city={params['city']!r} was sent without a region. The gateway " + f"answers that with 500 Internal Server Error, which reads as a " + f"fault on their side and is not one - the same city with its " + f"own region answers 200. Measured 2026-09-08." + ) + wanted = params.get("country") + if not wanted: + return problems + page = self.countries(connection_type=params.get("type") or "residential") + codes = { + str(item.get("code", "")).lower() + for item in self.iterate(page) + if isinstance(item, dict) + } + if not codes: + problems.append( + "the country catalogue came back with no readable codes, so " + "nothing was checked. This is a bug here rather than a problem " + "with your parameters - do not treat it as a pass." + ) + elif wanted != "any" and wanted.lower() not in codes: + problems.append( + f"country={wanted!r} is not in the catalogue for " + f"connection_type={params.get('type') or 'residential'!r}. The " + f"gateway answers this with 407 Proxy Authentication Required, " + f"which reads as a credentials problem and is not one." + ) + return problems + + # -- transport ---------------------------------------------------------- + + def _get(self, path: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return self._get_object("GET", path, params=params) + + def _get_object( + self, + method: str, + path: str, + *, + params: Optional[Dict[str, Any]] = None, + body: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + answer = self._request(method, path, params=params, body=body) + if not isinstance(answer, dict): + raise ApiError( + f"{method} {path} answered with {type(answer).__name__} where an " + f"object was expected. The API's shape has changed or something " + f"other than the API answered.", + body=answer, + ) + return answer + + def _envelope( + self, + method: str, + path: str, + *, + params: Optional[Dict[str, Any]] = None, + body: Optional[Dict[str, Any]] = None, + ) -> Any: + """One sub-user call, with ``{success, ..., payload}`` unwrapped. + + A 204 arrives as ``{}`` from :func:`_interpret` and is returned as is: + an empty body carries no envelope to unwrap and no success flag to + check, and inventing one would be this function claiming to know what + the server meant. + """ + answer = self._request(method, path, params=params, body=body) + if not isinstance(answer, dict) or "payload" not in answer: + return answer + if answer.get("success") is False: + raise ApiError( + f"{method} {path} answered 2xx with success=false: " + f"{_detail(answer) or 'no reason given'}. A success flag inside " + f"a 2xx is the server disagreeing with its own status line, and " + f"reading the status alone would report this as done.", + body=answer, + ) + return answer["payload"] + + def _list( + self, + path: str, + params: Optional[Dict[str, Any]] = None, + rows_key: str = "results", + paging: Optional[Paging] = BY_OFFSET, + ) -> Page: + """One list request, with paging stated rather than left implied. + + The size and the cursor go on every call whose convention has measured + values for them, and a caller's own values win - so + ``countries(limit=200)`` is one request for 200 and ``cities(offset=50)`` + starts halfway, both reachable because the names are the server's own. + + ``paging=None`` is for an endpoint that does not page at all; the only + one is :meth:`domain_statistics`. + """ + asked = dict(params or {}) + if paging is not None: + if paging.default_size is not None: + asked.setdefault(paging.size_key, paging.default_size) + if paging.first_cursor is not None: + asked.setdefault(paging.cursor_key, paging.first_cursor) + page = _page(self._request("GET", path, params=asked), rows_key) + return replace( + page, request_path=path, request_params=asked, paging=paging + ) + + def _request( + self, + method: str, + path: str, + *, + params: Optional[Dict[str, Any]] = None, + body: Optional[Dict[str, Any]] = None, + ) -> Any: + if path.startswith("http"): + # The only caller that passes an absolute url is `iterate`, and the + # url it passes is one the *server* wrote. The check belongs here + # rather than there because this is the function that attaches the + # key: a second caller added later would otherwise reopen the hole. + if not _is_same_origin(self._base, path): + raise ApiError( + f"refusing to send the API key to {path!r}, which is not " + f"{self._base}. This url came back from the API as a paging " + f"link, so either the account API is pointing somewhere else " + f"or something answered in its place - and the request that " + f"would follow it carries your key in a header." + ) + url = path + else: + url = f"{self._base}{path}" + if params: + # None means "not set" and is dropped, so `countries(name=None)` + # sends nothing rather than the string "None" - which a filter would + # match against zero rows and report as an empty catalogue. + clean = {k: v for k, v in params.items() if v is not None} + if clean: + joiner = "&" if "?" in url else "?" + url = f"{url}{joiner}{urllib.parse.urlencode(clean, doseq=True)}" + + headers = { + # `Authorization: x-api-key ` and not an `X-API-Key` header. + # Unusual, and it is what the server wants: measured 2026-09-08, + # this form answers 200 where `Bearer` and `Token` answer 403, and + # the spec's own securityScheme spells it out - an apiKey in the + # `Authorization` header, "in the following format - 'x-api-key + # '". Do not tidy it into the conventional spelling. + "Authorization": f"x-api-key {self._key}", + "Accept": "application/json", + "User-Agent": _USER_AGENT, + } + payload = None + if body is not None: + payload = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + status, raw = self._transport(method, url, headers, payload) + return _interpret(status, raw, method, url) + + +def _is_same_origin(base: str, url: str) -> bool: + """Whether ``url`` goes to the same scheme, host and port as ``base``. + + The rule an API key travels under, and it was missing until 2026-09-08. The + paging loop followed the ``next`` the server sent, verbatim, and + :meth:`Client._request` attaches ``Authorization: x-api-key `` to + whatever it is given: measured that day through the transport seam, a + ``next`` of ``https://evil.example/api/v2/base/x`` received the key in a + header. Nothing in the module was wrong about paging - the loop guard and + the page bound both worked - and the credential still left for a host + nobody configured. + + Scheme is compared and not just the host, because ``http://`` to the right + host is the same leak on a wire instead of to a stranger: the key is a + header and a header is plaintext. + + The default port is filled in on both sides rather than compared as text. + ``https://host`` and ``https://host:443`` are one origin, and a server that + builds paging links from its own absolute URI may well emit the explicit + form behind a proxy - so comparing ``netloc`` as a string would refuse a + legitimate page, which is the expensive direction to be wrong in for a check + that has never run against the live API. + + ``urlsplit(...).port`` raises ``ValueError`` on a port that is not a number, + so a hostile ``next`` could otherwise crash the paging loop from inside the + function meant to make it safe. That is caught here and answered as "not the + same origin", which is both true and the safe reading. + """ + try: + here = urllib.parse.urlsplit(base) + there = urllib.parse.urlsplit(url) + if there.scheme != here.scheme: + return False + if (there.hostname or "").lower() != (here.hostname or "").lower(): + return False + default = {"https": 443, "http": 80}.get(here.scheme) + return (there.port or default) == (here.port or default) + except ValueError: + return False + + +def _interpret(status: int, raw: bytes, method: str, url: str) -> Any: + """Turn a status and some bytes into a value or the right exception. + + ``url`` appears in messages and the key does not, because the key travels in + a header. That is not an accident of this function - it is why the key is a + header in the first place. + """ + text = raw.decode("utf-8", "replace").strip() + parsed: Any = None + if text: + try: + parsed = json.loads(text) + except ValueError: + parsed = text + + if 200 <= status < 300: + # 204 and an empty 200 are both real answers to a DELETE. An empty dict + # rather than None, so a caller can index the result of every method + # without branching on which one they called. + return parsed if text else {} + + detail = _detail(parsed) or f"HTTP {status}" + where = f"{method} {url}" + if status in (401, 403): + # The key this API takes is a JWT with an `exp` claim - measured + # 2026-09-09, a live one had 1723 seconds left mid-run - so a refusal has + # a third cause besides a wrong key and a wrong header form: a key that + # was valid when the `Client` was built and is not any more. All three + # look identical from here, which is why the clock is named in the + # message. Nothing in this module reads `exp` or renews on its own: that + # would mean decoding a credential to make a control-flow decision, and + # what the token's real lifetime is has not been measured, only how much + # of one instance was left. + raise AuthError( + f"{where} was refused: {detail}. This is the dashboard API key, not " + f"the proxy password - check NODEMAVEN_APIKEY. If the key worked " + f"earlier in the same process, check its expiry: the key is a JWT " + f"and an expired one is refused exactly like a wrong one.", + status=status, + body=parsed, + ) + if status == 404: + raise NotFoundError( + f"{where} found nothing: {detail}.", status=status, body=parsed + ) + if status == 429: + raise RateLimitError( + f"{where} was rate limited: {detail}. Nothing here retries - wait the " + f"server's own interval if it gave one, in retry_after.", + status=status, + body=parsed, + ) + if status >= 500: + raise ApiError( + f"{where} failed on the server: {detail}. Retrying immediately is " + f"the thing this package declines to do for you; a 5xx that repeats " + f"is worth reporting rather than hammering.", + status=status, + body=parsed, + ) + raise ApiError(f"{where} was refused: {detail}.", status=status, body=parsed) + + +def _detail(parsed: Any) -> str: + """The most useful sentence in an error body, whatever shape it arrived in. + + **This API speaks three error dialects and the spec names all three.** + ``ErrorResponse`` is ``{"detail": "..."}``, which is Django REST Framework's; + ``BadRequestErrorResponse`` is ``{"errors": {field: message}}``, a nested + object under a fixed key; and the sub-user envelope is ``{"success": false, + "description": "...", "errors": [...]}``. Only the first was handled until + 2026-09-09, so a 400 from any write endpoint came out as the literal text + ``errors: {'proxy_username': 'This field is required.'}`` - Python syntax + shown to somebody debugging an HTTP call. + + A bare list and a bare string are handled too, because they cost one line + each and a wrong guess about which shape arrives is what this function is + for. + """ + if isinstance(parsed, str): + return parsed + if isinstance(parsed, list): + return "; ".join(str(item) for item in parsed) + if not isinstance(parsed, dict): + return "" + + for key in ("detail", "message", "error", "description"): + value = parsed.get(key) + if isinstance(value, str) and value: + return value + + nested = parsed.get("errors") + if isinstance(nested, (dict, list)) and nested: + return _flatten(nested) + if isinstance(nested, str) and nested: + return nested + + return _flatten(parsed) + + +def _flatten(body: Any) -> str: + """``{field: message}`` and ``[message]`` as one readable line.""" + if isinstance(body, list): + return "; ".join(str(item) for item in body) + if isinstance(body, dict): + parts = [] + for key, value in body.items(): + if isinstance(value, (list, dict)): + parts.append(f"{key}: {_flatten(value)}") + else: + parts.append(f"{key}: {value}") + return ", ".join(parts) + return str(body) + + +def _next_step(page: Page) -> Optional[Tuple[str, Dict[str, Any]]]: + """The request that would follow ``page``, or ``None`` to stop. + + ``None`` on four conditions, and each is an ending rather than a guess: + + * the page carries no record of the call it came from, which is the case for + a page built by following a ``next`` url - and a server that sends ``next`` + sends it until the collection ends, so there is nothing to do by hand; + * the endpoint does not page at all, which is :meth:`Client.domain_statistics`; + * the page came back **empty**, which is the one end-of-collection signal + this server gives: measured 2026-09-08, an offset past the end answers 200 + with zero rows rather than repeating the last page; + * the cursor or the size in that call was not a usable whole number. + + **A short page is deliberately not an ending.** It was until 2026-09-09, and + that rule truncates against a server that caps the size below the request - + which this one does, at ``limit=1000`` on ``cities``, where the collection is + 1965. See :meth:`Client.iterate`. + """ + path = page.request_path + asked = page.request_params + paging = page.paging + if path is None or not asked or paging is None: + return None + if not page.results: + return None + + try: + cursor = int(asked[paging.cursor_key]) + except (KeyError, TypeError, ValueError): + return None + if cursor < 0: + return None + + following = dict(asked) + if paging.cursor_counts_rows: + try: + size = int(asked[paging.size_key]) + except (KeyError, TypeError, ValueError): + return None + if size <= 0: + return None + following[paging.cursor_key] = cursor + size + else: + following[paging.cursor_key] = cursor + 1 + return path, following + + +def _page(body: Any, rows_key: str = "results") -> Page: + """A ``Page`` from whichever shape the endpoint used. + + Accepts the paginated object and a bare list, and refuses anything else with + a message naming what came back. The alternative is ``body["results"]``, + which raises ``KeyError: 'results'`` - a message describing this package's + assumption rather than the server's answer. + + **Which branch runs was got wrong once, and the wrong answer was written + down as a measurement.** This docstring said "the bare-list branch is the one + that runs" for part of 2026-09-08, on the strength of a probe that printed + ``count`` and nothing else - and ``count`` is ``None`` for a bare array and + for an envelope that does not fill it, so the two readings were never + separated. ``probe_catalogue_paging.py`` prints the shape, and the answer is + the **envelope**: ``countries``, ``regions`` and ``cities`` all answer with + an object whose ``results`` is the list. + + The bare-list branch has therefore never run against this server. It stays, + because it was written for the reason "this is an inference and inferences + are wrong" rather than because anything suggested that shape - which is the + same reason it is worth keeping now that the inference has been wrong twice + in opposite directions. + + ``rows_key`` is named by the caller and never guessed. Six values are in + use and they are per-endpoint knowledge: ``results``, ``isps``, ``regions``, + ``cities``, ``payload``, ``data``. The alternative was to look for whichever + value in the object + happens to be a list, which reads as robustness and is a guess - an envelope + carrying two lists would be resolved by dict order, silently and differently + per server version, and ``LocationsISPsResponse`` and ``SubUserManyResponse`` + both carry other fields beside their rows. + + The single-object branch is for a filter that matches one row. Whether this + server ever takes it is unknown. + """ + if isinstance(body, list): + return Page(results=body, rows_key=rows_key) + if isinstance(body, dict): + if isinstance(body.get(rows_key), list): + return Page( + results=body[rows_key], + count=body.get("count"), + next=body.get("next"), + previous=body.get("previous"), + rows_key=rows_key, + ) + # A single object where a list was expected is a real thing servers do + # for a filter that matches one row. Wrapping it is friendlier than + # refusing, and it is visible in `count is None`. + if body: + return Page(results=[body], rows_key=rows_key) + return Page(results=[], rows_key=rows_key) + raise ApiError( + f"a list endpoint answered with {type(body).__name__}, which is neither a " + f"page nor a list. Either the API changed shape or something other than " + f"the API answered - a captive portal and a corporate proxy both do this, " + f"and so does this API's own host, which answers an unregistered path " + f"with 200 and the dashboard's HTML.", + body=body, + ) + + +def _segment(value: Any) -> str: + """One path segment, escaped. + + ``quote`` with an empty ``safe`` set, so an id containing ``/`` or ``?`` + cannot rewrite the path into a different endpoint. Ids come from the server, + which makes this look unnecessary - and the day one comes from a config file + or a command line argument instead, it is the difference between a 404 and a + DELETE against something else. + """ + return urllib.parse.quote(str(value), safe="") + + +def _urllib_transport( + method: str, + url: str, + headers: Dict[str, str], + body: Optional[bytes], + *, + timeout: float = 30.0, +) -> Tuple[int, bytes]: + """The default transport: the standard library, and no third-party anything. + + ``urllib.request.urlopen`` raises ``HTTPError`` on 4xx and 5xx, and that + object *is* the response - it has a status and a readable body. Catching it + and returning the pair is what lets :func:`_interpret` see the server's own + error message, which is the difference between "HTTP 400" and + "proxy_username: This field is required." + + ``ProxyHandler({})`` is passed explicitly, and it is the load-bearing part of + this function. Left out, ``urlopen`` reads ``http_proxy`` and ``https_proxy`` + from the environment - so on any machine where those are set, and they are + set on exactly the machines that use proxies, an API call would be routed + through a proxy nobody asked to route it through. The empty dict disables + that. A caller who does want it can pass their own transport. + """ + request = urllib.request.Request(url, data=body, headers=headers, method=method) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(request, timeout=timeout) as response: + return int(response.status), response.read() + except urllib.error.HTTPError as exc: # a response, not a failure + return int(exc.code), exc.read() + except urllib.error.URLError as exc: + raise ApiError( + f"{method} {url} never reached the API: {exc.reason}. No status came " + f"back, so this says nothing about the API key." + ) from exc diff --git a/src/nodemaven/check.py b/src/nodemaven/check.py new file mode 100644 index 0000000..e9b5c02 --- /dev/null +++ b/src/nodemaven/check.py @@ -0,0 +1,484 @@ +"""One CONNECT, and what the gateway said about it. + +This is the only module in the package that opens a socket, and it does it in +the smallest form there is: a raw ``CONNECT`` and the status line that comes +back. No TLS, no HTTP client, no target traffic, no dependency. + +The reason it is written by hand rather than delegated to an HTTP library is +that **the diagnosis is in the status line and libraries throw it away.** +``requests`` reports a failed tunnel as +``ProxyError('Unable to connect to proxy', OSError('Tunnel connection failed: +407 Proxy Authentication Required'))`` - the code survives as text inside a +nested exception, and the reason phrase and every response header do not. On +this gateway the reason phrase identifies which back end answered and one of +the headers carries the exit address, so both are worth more than the +convenience of not writing this file. + +There is also a failure mode this shape makes impossible. The vendor's own +client has a ``get_current_ip(proxies=...)`` whose fallback path - taken +whenever ``requests`` is not installed - builds a plain +``urllib.request.Request`` and never installs a ``ProxyHandler``, so the +``proxies`` argument is silently discarded and the function returns **your own +address** while reporting it as the proxy's exit. Read in +``nodemavencom/proxy``, ``python/nodemaven/utils.py``, on 2026-09-07. Here +there is no code path that does not go through the proxy socket, because the +socket is the whole implementation. +""" + +from __future__ import annotations + +import base64 +import socket +import time +from dataclasses import dataclass +from typing import Dict, Mapping, Optional + +from .errors import CheckError + +__all__ = ["Check", "connect"] + +#: What a CONNECT is opened *to*. The gateway has to be asked for some target, +#: there is no null CONNECT, so this is a real host that will see a TCP +#: connection from the exit address. It is a parameter on every entry point and +#: this is only the default. +#: +#: Port 443 and not 80 because a gateway may treat plaintext differently, and +#: because 443 is what a caller's real traffic will use. +DEFAULT_TARGET = "api.ipify.org:443" + +#: What this client *sends*. Reading is deliberately laxer - see +#: :func:`_head_end` - because being strict about what you send and liberal +#: about what you accept are the same rule, not opposite ones. +_CRLF = "\r\n" +_MAX_HEAD = 16384 + + +@dataclass(frozen=True) +class Check: + """What one CONNECT produced. + + A refusal is a **result and not an exception**: the status code is the thing + the caller came for, and raising would push the useful part into a traceback. + :class:`~nodemaven.errors.CheckError` is reserved for the cases where nothing + came back at all. + """ + + #: The CONNECT status, e.g. 200 or 407. + status: int + #: The reason phrase, verbatim and not normalised. On the shipped gateway + #: this identifies which back end answered: measured 2026-08-13, a 200 + #: carrying ``X-Proxy-Exit-IP`` arrives as ``Connection established``, while + #: the ones that arrive as ``OK`` or ``Connection Established`` do not carry + #: it. Any per-implementation number has to be split on this rather than + #: pooled, so it is preserved byte for byte. + reason: str + #: ``host:port`` of the gateway, with no credentials in it. + server: str + #: Wall-clock seconds from the first byte sent to the status line parsed. + #: Includes DNS and the TCP handshake, and is not comparable against a + #: number measured on a different network path. + elapsed: float + #: The response headers, lower-cased keys, in the order they arrived. + headers: Dict[str, str] + #: The exit address, when the gateway sent one on the header its provider + #: definition declares. ``None`` is normal rather than an error - on the + #: shipped gateway only one of at least three back ends sends it. + exit_ip: Optional[str] = None + #: What this status means on this gateway, from the provider definition, or + #: ``None`` if that gateway has no entry for it. This is where a 407 gets + #: told not to go and check its password. + meaning: Optional[str] = None + + @property + def ok(self) -> bool: + """Whether the tunnel opened. + + True means the gateway accepted the credentials and every parameter it + recognised. It does **not** mean every parameter was applied: an + unrecognised name is answered 200 and dropped. That is why this package + refuses unknown names before sending, and why ``ok`` cannot be the whole + answer on its own. + """ + return self.status == 200 + + def __str__(self) -> str: + head = f"{self.status} {self.reason} via {self.server} in {self.elapsed:.2f}s" + if self.exit_ip: + head += f", exit {self.exit_ip}" + if self.meaning and not self.ok: + head += f"\n{self.meaning}" + return head + + +def connect( + server: str, + username: str, + password: str, + *, + target: str = DEFAULT_TARGET, + timeout: float = 15.0, + exit_ip_header: Optional[str] = None, + reactions: Optional[Mapping[str, str]] = None, +) -> Check: + """Open one CONNECT through ``server`` and report what came back. + + ``username`` and ``password`` go into a ``Proxy-Authorization`` header. + **That header is never logged, never put in an exception message and never + returned**: it is base64 and not encryption, so anything that prints one has + put a working credential into a terminal history, a CI log and whatever bug + report gets pasted next. + + ``timeout`` defaults to 15 s rather than to something small, because one of + the gateway's documented reactions is *no reply at all* - an empty parameter + value hangs the connection for about 20 s. A 5 s timeout would report that + as a network problem. This package refuses empty values before sending, so + the case should be unreachable through :class:`~nodemaven.Proxy`; the + default is set for the caller who assembles a username by hand. + + ``reactions`` is the provider's ``connect_reactions`` table, keyed by the + status code as a string. It is passed in as plain data rather than looked up + through a ``Provider``, so this module needs nothing from the rest of the + package except one exception class - which is what makes it testable against + a socket on loopback and reusable by anyone assembling a username by hand. + + Raises :class:`~nodemaven.errors.CheckError` in two situations. Before + anything is sent, when ``server`` or ``target`` cannot make a well-formed + request line - those messages end in *Nothing was sent*. After sending, when + there was no usable answer: DNS failure, refused connection, timeout, a head + that never ended, or a status line that is not one. + """ + host, _, port_text = server.rpartition(":") + port = _port_number(port_text) + if not host or port is None: + raise CheckError( + f"{server!r} is not a gateway address: it has to be host:port, with " + f"a port from 1 to 65535. Nothing was sent." + ) + if not _is_request_target(target): + raise CheckError( + f"{target!r} cannot go in a request line: a target is visible ASCII " + f"with no spaces, so that it lands in the CONNECT as one token. " + f"Nothing was sent." + ) + + token = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + request = ( + f"CONNECT {target} HTTP/1.1{_CRLF}" + f"Host: {target}{_CRLF}" + f"Proxy-Authorization: Basic {token}{_CRLF}" + f"Proxy-Connection: close{_CRLF}" + f"{_CRLF}" + ).encode("utf-8") + + started = time.monotonic() + sock = None + try: + sock = socket.create_connection((host, port), timeout=timeout) + sock.sendall(request) + head = _read_head(sock, timeout) + except OSError as exc: + # Deliberately not chained into the message with the request bytes in + # scope. `request` holds the credential and an f-string that happened to + # include it would leak it into every traceback. + raise CheckError( + f"no answer from {server}: {exc}. Nothing here can tell you whether " + f"the credentials are right, because the gateway was never reached." + ) from exc + finally: + if sock is not None: + sock.close() + elapsed = time.monotonic() - started + + status, reason, headers = _parse_head(head, server) + exit_ip = headers.get(_ascii_lower(exit_ip_header)) if exit_ip_header else None + return Check( + status=status, + reason=reason, + server=server, + elapsed=elapsed, + headers=headers, + exit_ip=exit_ip, + meaning=(reactions or {}).get(str(status)), + ) + + +def _head_end(buffer: bytes) -> int: + """Index just past the blank line that ends a response head, or ``-1``. + + A blank line has four spellings once a bare LF is allowed as a terminator - + ``\\r\\n\\r\\n``, ``\\n\\n``, ``\\r\\n\\n`` and ``\\n\\r\\n`` - and searching + for the earlier of ``\\n\\n`` and ``\\n\\r\\n`` covers all four, because + every one of them ends in one of those two. + + LF is accepted because RFC 9112 section 2.2 says a recipient may recognise a + single LF as a line terminator and ignore any preceding CR, and because this + gateway needs it: a 200 is framed CRLF and every refusal - 406, 407, 500 - + is framed with bare LF. A CRLF-only reader cannot report any refusal code + from it, which is the one thing :func:`check` exists to do. + """ + end = -1 + for terminator in (b"\n\n", b"\n\r\n"): + found = buffer.find(terminator) + if found >= 0 and (end < 0 or found + len(terminator) < end): + end = found + len(terminator) + return end + + +def _read_head(sock: socket.socket, timeout: float) -> bytes: + """Read up to and including the blank line that ends the response head. + + Reads no further, so nothing is consumed from the tunnel body even when the + tunnel opened, and any body bytes that arrived in the same segment are cut + off rather than parsed as headers. Bounded at 16 KiB: a response head is a + few hundred bytes, so an unbounded read here is a memory exhaustion bug + waiting for a gateway that answers with a stream. + + **An unterminated head is refused rather than returned.** A peer that sends + ``HTTP/1.1 200 OK\\r\\nX-Proxy-Exit-IP: 1.2.3.4`` and then hangs up mid-head + would otherwise parse as ``status=200``, ``ok=True``, ``headers={}``: the + caller is told the tunnel opened and the one field it would have read off + the reply has silently gone missing. The message names which of the two + truncations happened, because they have different causes and different + fixes. + + An immediate close with nothing at all is not truncation - it is a + documented reaction of this gateway - so it returns empty and + :func:`_parse_head` has the sentence for it. + """ + sock.settimeout(timeout) + buffer = bytearray() + while True: + end = _head_end(buffer) + if end >= 0: + return bytes(buffer[:end]) + if len(buffer) > _MAX_HEAD: + raise CheckError( + f"{len(buffer)} bytes from the gateway with no end to the " + f"response head. A head is a few hundred bytes, so this is a " + f"stream and not a reply, and reading further is how a client " + f"runs out of memory." + ) + chunk = sock.recv(4096) + if not chunk: + if not buffer: + return b"" + raise CheckError( + f"the connection closed after {len(buffer)} bytes, part-way " + f"through the response head. Whatever came before the cut is " + f"not an answer - a status line without its blank line may be " + f"missing headers that had not arrived yet." + ) + buffer.extend(chunk) + + +def _is_ascii_digits(text: str) -> bool: + """Whether ``text`` is one or more of ``0``-``9`` and nothing else. + + Spelled out rather than ``str.isdigit()``, which is the thing this module + got wrong twice. ``isdigit()`` answers a question about Unicode categories + and every caller here is asking a question about ``int()``: the two disagree + on ``'\\xb2'``, where ``isdigit()`` is True and ``int()`` raises + ``ValueError``. ``str.isascii() and str.isdigit()`` would also work and is + 3.7+; the comparison is written out because this is a rule four SDKs have to + hold identically, and a comparison ports where a standard-library predicate + does not. + """ + return bool(text) and all("0" <= character <= "9" for character in text) + + +def _is_status_code(text: str) -> bool: + """Whether ``text`` is an HTTP status code: exactly three ASCII digits. + + ``str.isdigit()`` was the obvious spelling and it was a bug, found on + 2026-09-07 while porting this module to Rust. It is True for characters + ``int()`` refuses - ``'\\xb2'.isdigit()`` is True and ``int('\\xb2')`` raises + ``ValueError`` - and this function is reachable with exactly those + characters, because the head above is decoded **latin-1 on purpose** so that + any byte a proxy is entitled to send is accepted. So the deliberate widening + of the input alphabet is what made the narrow check unsound: bytes 0xB9, + 0xB2 and 0xB3 in a status line produced an uncaught ``ValueError``, past a + docstring promising that only :class:`~nodemaven.errors.CheckError` comes + out of here. + + Three digits and not "one or more" because that is what the specification + says - RFC 9110 calls the status code a three-digit integer - and because it + is the rule the Rust port can hold in a ``u16`` without diverging. A gateway + answering anything else is the case the caller's error message already + describes: something other than a proxy is listening, or a middlebox + answered instead. + """ + return len(text) == 3 and _is_ascii_digits(text) + + +def _is_http_version(text: str) -> bool: + """Whether ``text`` is an HTTP version token: ``HTTP/`` and two digits. + + RFC 9112 section 2.3 spells it ``HTTP-name "/" DIGIT "." DIGIT`` and makes + ``HTTP`` case-sensitive, so this is the grammar and not a house rule. Eight + characters exactly, because a CONNECT answered over a TCP socket this module + opened itself is HTTP/1.x by construction - there is no version negotiation + to be liberal about. + + It exists because it was missing, found 2026-09-08 in an external review and + reproduced the same day on a loopback socket. ``_parse_head`` split the + status line and looked only at the *second* token, so the first was accepted + whatever it was: ``garbage 200 OK`` came back as ``status=200``, ``ok=True`` + - and with ``exit=1.2.3.4`` when the same non-proxy also sent the exit + header. That is the worst available failure, because ``ok`` is what a caller + branches on and the exit address is what it then reports as its own. + + What the mistake looked like from the inside: the rules this module states in + words all ported to Rust exactly - three digits, ASCII digits, ASCII + lower-casing are each a paragraph here and each landed there intact. This + rule was never written down anywhere, so ``check.rs`` reproduced the hole + line for line: ``let _version = parts.next();``, discarded on purpose, + reviewed by nobody. **A cross-language contract is only the part that was + written down**; whatever is left implicit is re-implemented by hand in every + port, and re-implemented the same way, because the same reading produced it. + """ + return ( + len(text) == 8 + and text.startswith("HTTP/") + and _is_ascii_digits(text[5]) + and text[6] == "." + and _is_ascii_digits(text[7]) + ) + + +def _is_request_target(text: str) -> bool: + """Whether ``text`` can go into a request line as a single token. + + Visible ASCII with no space: bytes 0x21 to 0x7E. It is deliberately narrow + rather than a check for the specific characters that hurt, because the + request line is assembled by string interpolation and the set of characters + that change its shape is not something to enumerate from memory. + + The case that motivated it, measured 2026-09-08 on a loopback socket: + ``target="example.com:443\\r\\nX-Injected: yes"`` was interpolated straight + into ``f"CONNECT {target} HTTP/1.1"``, so the gateway received a request line + of ``CONNECT example.com:443`` with **no version token at all**, and + ``X-Injected: yes HTTP/1.1`` as a header of our own request. Header + injection is the obvious half; the request line losing its version to a + caller-supplied string is the half that is easy to miss. + + Two things this deliberately does not do, worth stating so the next port does + not add them by guesswork. It does not check that ``target`` is + ``host:port`` - the gateway is entitled to its own opinion about what it will + tunnel to, and a client that refuses a target the server would have accepted + is a client that has to be worked around. And it does not touch ``username`` + or ``password``, which cannot inject anything at all: they go through + ``base64`` two lines below, whose output alphabet is ``A-Za-z0-9+/=`` and + contains neither CR nor LF. The package validating what gets base64-encoded + while leaving the one field that lands in the clear unvalidated was the + actual shape of this defect. + """ + return bool(text) and all(" " < character <= "~" for character in text) + + +def _port_number(text: str) -> Optional[int]: + """``text`` as a TCP port, or ``None`` if it is not one. + + The same defect as ``_is_status_code`` above, in the other half of the + module and found the same day: the gate here was ``port_text.isdigit()``, + and ``connect("127.0.0.1:\\xb2", ...)`` raised an uncaught ``ValueError`` + from the ``int()`` two lines below it. The first fix caught one of the two + occurrences, which is the ordinary shape of this mistake - a predicate is + corrected where it was noticed rather than everywhere it is used. + + The length guard is not the same rule twice. ``isdigit()`` also accepts a + digit string of any length, so ``'99999999999999999999'`` reached ``int()``, + succeeded there, and raised ``OverflowError`` inside ``create_connection`` + - and ``OverflowError`` derives from ``ArithmeticError``, not ``OSError``, + so it walked straight past the handler that exists to turn everything from + the socket layer into a ``CheckError``. Refusing more than five characters + means ``int()`` is only ever called on something that fits, which also + sidesteps CPython 3.11+ refusing ``int()`` on a string of over 4300 digits. + + Port 0 is refused. It is legal in ``bind`` and means "any free port", and it + is meaningless in ``connect``: on Windows it fails with WinError 10049 and + on Linux it is answered ``ECONNREFUSED``, so refusing it here replaces a + platform-specific errno with the sentence that says what to fix. + """ + if not _is_ascii_digits(text) or len(text) > 5: + return None + port = int(text) + return port if 1 <= port <= 65535 else None + + +def _parse_head(head: bytes, server: str) -> "tuple": + """Split a response head into status, reason phrase and headers. + + A status line is accepted only when its **first two tokens** are an HTTP + version and a three-digit status code, per :func:`_is_http_version` and + :func:`_is_status_code`. Only the second was checked until 2026-09-08, which + is how ``garbage 200 OK`` parsed as a 200. + + Lines are split on LF with one optional preceding CR stripped, which accepts + a head framed either way for the reason given in :func:`_head_end`. A CR + anywhere else in a line is left alone: it is part of the value, and this + function does not repair a malformed reply. + + Decoded as latin-1 and never as utf-8. A header value is bytes by + specification and a gateway is free to put anything in a reason phrase; utf-8 + would raise on a byte a proxy is entitled to send, turning a readable + diagnosis into a decode error. latin-1 cannot fail, and the reason phrase is + something a human reads rather than something this package matches on. + """ + if not head: + raise CheckError( + f"{server} accepted the connection and then closed it without " + f"answering. That is not one of the reactions this gateway is known " + f"to have, so it is worth reporting with the parameters that produced " + f"it." + ) + text = head.decode("latin-1") + lines = [ + line[:-1] if line.endswith("\r") else line for line in text.split("\n") + ] + parts = lines[0].split(" ", 2) + if ( + len(parts) < 2 + or not _is_http_version(parts[0]) + or not _is_status_code(parts[1]) + ): + raise CheckError( + f"{server} answered {lines[0]!r}, which is not an HTTP status line. " + f"Either something other than a proxy is listening on that port, or " + f"a middlebox answered instead of the gateway." + ) + status = int(parts[1]) + reason = parts[2] if len(parts) > 2 else "" + + headers: Dict[str, str] = {} + for line in lines[1:]: + if not line: + break + name, sep, value = line.partition(":") + if sep: + headers[_ascii_lower(name.strip())] = value.strip() + return status, reason, headers + + +def _ascii_lower(text: str) -> str: + """Lower-case ``A``-``Z`` and leave every other character alone. + + ``str.lower()`` was here and it is a portability trap rather than a bug + today, found 2026-09-07 while writing the Rust port. Both languages have a + Unicode-aware lower-casing and an ASCII-only one, and they do not agree: + this head is decoded latin-1 on purpose, so a field name containing byte + ``0xC0`` reaches here as ``'\\xc0'``, Python's ``.lower()`` makes it + ``'\\xe0'`` and Rust's ``to_ascii_lowercase`` leaves it as ``'\\xc0'``. Two + SDKs would then key the same response header two ways. + + ASCII-only is not merely the portable choice, it is the correct one: RFC 9110 + says a field name is a token and a token is ASCII, so a non-ASCII byte in one + is already malformed and nothing is served by folding it. This is the same + decision, for the same reason, as the ASCII-only fold in + ``Provider.normalized`` - and that one is pinned by a test in both SDKs + because ``str.lower()`` and Rust's ``to_lowercase`` disagree with each other + on the Turkish dotted capital I as well. + """ + return "".join( + chr(ord(character) + 32) if "A" <= character <= "Z" else character + for character in text + ) diff --git a/src/nodemaven/data/providers/nodemaven.toml b/src/nodemaven/data/providers/nodemaven.toml index 14c38c8..9111f7d 100644 --- a/src/nodemaven/data/providers/nodemaven.toml +++ b/src/nodemaven/data/providers/nodemaven.toml @@ -23,6 +23,14 @@ port = 8080 # target traffic. Not guaranteed: more than one implementation answers on this # name, and only some of them send it, so handle its absence rather than # treating the first miss as an error. +# +# One name and more than one spelling of it, which this key cannot express. A +# 200 read on 2026-09-08 carried `X-Exit-IP` beside `X-Exit-Country`, +# `X-Exit-Timezone` and `X-Exit-ASN`, and no `X-Proxy-Exit-IP` at all, so an SDK +# reading only the name below gets no address from that back end. Which back end +# answers is decided by the username and cannot be asked for, so the name set is +# sampled rather than looked up: `lab\probes\probe_gateway_parameter_case.py`, +# phase 0. Widening this to a list is a schema change and waits for that sample. exit_ip_header = "X-Proxy-Exit-IP" # Confirmed to be recognised. Anything outside this set is refused before a @@ -54,11 +62,124 @@ exit_ip_header = "X-Proxy-Exit-IP" # 2026-08-21. Rotation is what `norotate` claims to change, and it does not # change it; that is a measurement of the effect, not of the name. # -# `speed` sits on the same provenance and has never been probed for its effect. -# It stays for now because removing it on inference would be the same error in -# the other direction. It is a probe waiting to be run, not a confirmed entry. +# The same 2026-08-26 probe settled two more names, and this file is the first +# shipped definition to carry them. It asked the gateway which names it knows by +# sending each one a junk value, which discriminates where acceptance does not: +# an unrecognised name is answered 200 and dropped, a recognised one refuses the +# value with 407. Negative control `zzqqx`, a name nobody implemented, 200; +# positive control `filter`, recognised since 2026-08-10, 407. +# +# type 407 recognised - an unknown name cannot produce this +# speed 407 recognised, confirming the 2026-08-12 generator read +# norotate 200 indistinguishable from the unknown name, hence removed +# +# `type` is the expensive one, because refusing it refused a product tier. It +# selects the network rather than labelling it: 5 echo requests per arm with a +# fresh sid and `country=us`, `type=mobile` drew AS21928 T-Mobile three times, +# one AS6167 - Cellco, the wireless side of Verizon - and one AS7018, while +# `type=residential` and the unset arm drew Comcast, Charter, Windstream, +# Metronet, Fidium, Planet and AS701 wireline with no mobile ASN between them. +# So it moves you to a different pool of addresses, not to a quality filter over +# one pool. It surfaced by reading the vendor's own public SDK at +# `nodemavencom/proxy` and was then verified the two ways above rather than +# taken from it, which is the rule `norotate` was written to enforce. +# +# `speed` is therefore no longer "a probe waiting to be run". Its name is +# confirmed; its effect is still unmeasured, and those are different claims. +# +# `ipv4` is the one name here a junk value cannot settle, and it was settled a +# different way on 2026-09-08. `ipv4` = true, TRUE, True and zzqqx are all +# answered 200, so the discriminator above is unavailable for it and its evidence +# was identical to a name nobody implemented - the position `norotate` was in. +# +# The handle that works is the sticky session, whose key is the *parsed* +# parameter set: a name the gateway parses joins the key and a name it drops +# cannot. Two independent session ids, two rounds each, five interleaved arms, +# with both controls holding in both blocks - the unknown name landed on the +# baseline exit every time and `filter` = medium moved off it every time: +# +# ipv4 = true a third exit, distinct from baseline and from `filter`, 4 of 4 +# ipv4 = false the baseline exit, 4 of 4 - indistinguishable from the +# unknown name +# +# So the name is parsed, which is what separates it from `norotate` at last. What +# `false` does is not settled: it is the default and so resolves to the +# baseline's parameter set, or that one value is dropped, and nothing measured +# separates those. +# +# The effect is a third claim and is also unsettled. Six draws against a +# dual-stacked echo returned one IPv6 exit, under `ipv4` unset, and five IPv4. +# That row is worth more than it looks - it proves the pool holds an IPv6-capable +# exit and that the instrument can see one - and at two draws per arm it says +# nothing about direction. So `ipv4` is a confirmed name whose effect is +# unmeasured, exactly like `speed`. known_params = ["country", "region", "city", "isp", "sid", "ttl", "filter", - "ipv4", "speed"] + "ipv4", "speed", "type"] + +# Values folded to their wire form before anything else looks at them: trim ASCII +# whitespace, lower-case A-Z, then each space to `_`. ASCII and not Unicode at +# every step, because `to_lowercase` and `.lower()` disagree between languages on +# characters like `I` with a dot, and a schema four SDKs share cannot depend on +# which one is running it. +# +# Added 2026-09-07 on two independent sources, which is the bar `norotate` set +# and failed. A username this gateway generated for a real account contains +# `region-district_of_columbia` - lower case, spaces as underscores - so that +# form is not a guess about what the gateway accepts, it is a copy of what the +# gateway itself emitted. Independently, the vendor's own client at +# `nodemavencom/proxy`, in `python/nodemaven/utils.py`, applies exactly this +# transformation before building the username. +# +# The distinction from `norotate` is the whole reason this is allowed to ship +# unprobed. `norotate` asserted the gateway *recognises a name*, which its own +# generator cannot establish, since an unrecognised name is answered 200 and +# dropped. This converts an input of unknown behaviour - `region="District of +# Columbia"`, which cannot be right, a space does not fit in a username - into +# one that is known to work. There is no false refusal available here to be +# wrong about. +# +# Five parameters and not more, and the omissions are each for a reason: +# +# sid opaque and caller-chosen. Folding it would rename an identity the +# caller picked, so it is left alone whatever the gateway does. +# +# This entry said until 2026-09-08 that "`sid=Order4417` and +# `sid=order4417` are two sessions on this gateway and the caller +# picked one of them", written as a fact about the gateway. Nothing +# ever probed it, in a block that is otherwise explicit about how +# many sources each claim has. +# +# What the mistake looked like from the inside: the decision is +# right either way - an SDK must not rewrite an identifier its +# caller chose - so the sentence read as the explanation of a +# settled choice rather than as a claim that could be false. Same +# shape as `norotate` two blocks up, with the damage somewhere +# else. `norotate` put a dead parameter in the library; this put a +# sentence in the published README telling a reader they hold two +# independent sessions when they may hold one - which is the +# failure `secrets`-not-`random` in `proxy.py` exists to prevent, +# arriving by another route. +# +# Acceptance cannot settle it, because every `sid` value is +# accepted, so the discriminator has to be the exit address: +# `lab\probes\probe_gateway_parameter_case.py`, phase 3. +# filter the vendor's client does not fold it, not even to lower case, and +# the generated username above says nothing either way because +# `medium` was already lower case. +# ttl the one value on this gateway whose case matters: `10M` is refused +# where `10m` is accepted. It is still not folded, and that is the +# decision rather than an oversight - folding here would encode +# which spellings the gateway takes today into a library that +# cannot re-measure them. The caller is told instead, in the 407 +# reaction below and in the README. +# speed absent from their builder entirely - see the note above, its name +# is confirmed and nothing else about it is. +# ipv4 a bool, written by the SDK and never by the caller. +# +# `country` and `type` are folded with the same rule as the three location names +# even though their values contain no spaces, because one rule in four languages +# is cheaper to keep true than two. +normalize = ["country", "region", "city", "isp", "type"] # Legal values per parameter, checked before anything is sent. # @@ -73,6 +194,12 @@ known_params = ["country", "region", "city", "isp", "sid", "ttl", "filter", # The generator is not a source of truth about the gateway at all. It also emits # `norotate`, and the gateway drops it - see the note on `known_params`. # +# `ttl` is the closest call and is still not listed. Four values are measured +# accepted and three measured refused - see `notes` - which is more than any +# other parameter has, and it is still a sample rather than a set: a list of four +# would refuse `2h` and `30m` if the gateway takes them, loudly, with our name on +# the error. The unit rule belongs in prose until somebody enumerates the range. +# # The key exists anyway, and that is the point: four language SDKs read this # schema. Filling this in later is an edit to a data file. Adding the key # later would be an edit to four parsers. @@ -84,22 +211,130 @@ known_params = ["country", "region", "city", "isp", "sid", "ttl", "filter", values = {} notes = """ -Measured 2026-08-10 by raw CONNECT probes. - Sticky sessions key on the whole recognised parameter set, not on `sid` alone. `country=us, sid=A` and `country=us, sid=A, filter=medium` are two different sessions; `ttl` does not participate and `filter` does. Adding or removing any -parameter moves you to another exit without saying so. - -Seven inputs, seven reactions, none of which names the cause: - bad country 406 Not Acceptable - bad region 406 Not Acceptable - bad city 500 Internal Server Error - bad filter value 407 Proxy Authentication Required - bad ttl value 407 Proxy Authentication Required - empty value no reply, the connection hangs about 20 s - unknown parameter 200, the parameter is ignored - -The two 407 cases are actively misleading - they send you to check credentials -that are correct. With no `country` the default country is not stable. +parameter moves you to another exit without saying so. Measured 2026-08-10. + +The key is the parsed SET and parameter *order* does not reach it. Measured +2026-09-08, 20 rounds a side: the canonical order and a shuffled one drew the +same exit 20 times each, while a control differing by one parameter value drew a +different exit 20 times. The exit was read from an echo service on every round +rather than from a proxy response header, because the header is sent by some +back ends and not others and the back end is chosen by the username - reading +two arms with two instruments would have compared the instruments. This is why +both SDKs may emit parameters in a fixed order without changing which exit you +get. + +A value the gateway will not take is answered several different ways and not one +of them names the parameter. Read by raw CONNECT on 2026-09-08 unless another +date is given: + + bad `country` value 407 Proxy Authentication Required + bad `region` value 406 Not Acceptable + bad `city` value 406 Not Acceptable, the same code a bad `region` and + a bad `isp` give + `city` without `region` 500 Internal Server Error. `city` works, and it needs + its own `region` beside it: `us`/`louisiana`/ + `abbeville` answers 200, and so does a second city in + a second region, while the same city with the region + removed answers 500. So the 500 is a request the + gateway could not resolve and not a fault on their + side. The Python SDK's `Client.validate()` refuses + the combination before sending; the Rust SDK has no + account API yet and so cannot + bad `isp` value 406 Not Acceptable, the same code a bad `region` + gives, so 406 does not say which of the two was + refused. `charter`, a real ISP, also answers 406, + while `comcast` answers 410 - so a real name is not + reliably told from a junk one either + `isp` = comcast 410 Gone, and the only value measured to produce it + bad `filter` value 407 Proxy Authentication Required + bad `ttl` value 407 Proxy Authentication Required + bad `type` value 407 Proxy Authentication Required (2026-08-26) + bad `speed` value 407 Proxy Authentication Required (2026-08-26) + empty value no reply, the connection hangs about 20 s + (2026-08-10) + unknown parameter name 200, and the parameter is ignored + +Every 407 in that list sends you to check credentials that are correct. With no +`country` the default country is not stable. + +`ttl` counts in minutes and hours. `1m`, `10m`, `10h` and `24h` open the tunnel; +`10s`, `10d` and a bare `10` are answered 407. It is also the one parameter whose +value case matters: `10M` is refused where `10m` is accepted. Nothing else +measured cares - `us` and `US`, `medium` and `MEDIUM`, `residential` and +`RESIDENTIAL`, `district_of_columbia` in either case are all accepted. Parameter +*names* are folded by the gateway: `COUNTRY` and `Ttl` refuse a junk value +exactly as their lower-case spellings do, so a name is recognised in any case. + +A value containing the separator is cut at it and the tail is read as a +parameter name, for every parameter and not only for `sid`. `isp` = verizon +opens the tunnel and a junk `isp` answers 406, while `isp` = verizon-zzqqx-zzqqx +answers 200 - so the gateway took `verizon` as the ISP and dropped the rest as an +unrecognised name. Both SDKs refuse such a value before sending, because the +alternative is a request that succeeds with settings nobody asked for. + +The response head is not always CRLF framed. A 200 is; a 406 and a 407 arrive +with bare LF line endings and `Connection: close`. A reader that ends a head at +CRLF CRLF and nowhere else sees every refusal as a truncated head and reports a +transport failure instead of the status code. + +An exit address arrives on the CONNECT reply itself, so it costs one handshake +and no target traffic. It is not guaranteed and the header name varies by back +end - see `exit_ip_header` above. Treat a reply with no address as normal. """ + +# What each CONNECT status means here. This is the machine-readable half of the +# table in `notes` above, and it exists because a status code on this gateway is +# not a diagnosis: five parameters answer a bad value with 407, which reads as a +# credentials problem and is not one. `check()` shows the caller this sentence +# next to the code. +# +# Both halves are kept. The prose carries the shape of the finding - one class of +# mistake answered five different ways, none naming the parameter - and no format +# renders that as a lookup table. These entries carry the answer to "I got a 407, +# now what". +# +# One entry per status actually measured. A default sentence would be this SDK +# inventing a diagnosis, which is the thing it exists to stop the gateway doing. +# +# No dates in these five strings, unlike everywhere else in this file. They are +# printed to a caller whose tunnel has just failed and who wants to know what to +# check; when we measured it is not part of that answer. The date and the probe +# behind each sentence are in `notes` above and in each SDK's CHANGELOG, which +# is where a reader asking whether this is still true will go. +# +# Last in the file on purpose: a `[table]` header in TOML swallows every key +# after it, so putting this above `notes` would make `notes` a member of it. +[connect_reactions] +200 = """\ +the gateway accepted the request and opened the tunnel. Note what this does not \ +say: an unrecognised parameter name is also answered with 200 and silently \ +dropped, so a 200 means the parameters were accepted and not that they were all \ +applied. This SDK refuses unknown names before sending for exactly that reason.""" +406 = """\ +`region`, `city` or `isp` names something this account cannot have. A junk \ +`region`, a junk `city`, a junk `isp` and `charter` - a real ISP - all answer \ +406, so this code says neither which of the three parameters was refused nor \ +whether the name is unknown or merely unavailable. Check all three values, and \ +see the 410 entry for the one ISP that is answered differently.""" +407 = """\ +usually NOT your credentials, despite what the status says. A value the gateway \ +will not take on `country`, `filter`, `ttl`, `type` or `speed` answers 407, and \ +so does a wrong password. Check the values before the password - and check the \ +case of `ttl`, which is the one value that is case-sensitive: `10M` is refused \ +where `10m` is accepted.""" +410 = """\ +`isp` names a network the gateway will not give you. Measured on one value: \ +`comcast` answers 410 where a junk `isp` and `charter` both answer 406, so 410 \ +looks like a name the gateway knows and a pool this account cannot reach. That \ +is one ISP, so read it as the narrower fact and not as a rule. The account \ +API's ISP catalogue, filtered by country, is what says which names exist.""" +500 = """\ +`city` was sent without a `region`. `country`+`region`+`city` opens the tunnel \ +on two different cities in two different regions, and dropping the `region` \ +from either of them turns the same request into a 500 - so this is an \ +incomplete request answered as a server fault, and not a fault. Add the region \ +the account API's city catalogue files that city under. A city name the \ +catalogue does not hold is answered 406 instead.""" diff --git a/src/nodemaven/errors.py b/src/nodemaven/errors.py index 01c95a6..c7e982a 100644 --- a/src/nodemaven/errors.py +++ b/src/nodemaven/errors.py @@ -3,11 +3,32 @@ Every message says what will happen to the caller, not that a value is invalid. A gateway parameter that is wrong is not a style problem: the request usually still succeeds, on settings nobody asked for. + +The classes here are part of the cross-language contract, not an implementation +detail. A golden vector says which error an invalid input must produce, so the +taxonomy has to keep agreeing across four SDKs even where the shape does not - +Rust has one enum where this has a class tree. The rule used to decide whether +something earns its own class is narrow on purpose: **a caller has to plausibly +write different code for it.** 401 means fix your key, 429 means wait, 404 in a +CRUD call means the row is gone; a 400 means fix your program and there is +nothing to branch on, so it stays on the base class. """ from __future__ import annotations -__all__ = ["NodeMavenError", "ParamError", "CredentialsError", "ProviderError"] +from typing import Any, Optional + +__all__ = [ + "NodeMavenError", + "ParamError", + "CredentialsError", + "ProviderError", + "ApiError", + "AuthError", + "NotFoundError", + "RateLimitError", + "CheckError", +] class NodeMavenError(Exception): @@ -24,3 +45,79 @@ class CredentialsError(NodeMavenError, ValueError): class ProviderError(NodeMavenError, ValueError): """A provider definition is missing or does not describe a gateway.""" + + +class ApiError(NodeMavenError): + """The account API answered with an error, or answered something unreadable. + + ``status`` is the HTTP status, or ``None`` when the request never got an + answer at all. ``body`` is whatever came back, decoded if it was JSON and + left as text if it was not - an HTML error page from a proxy or a load + balancer in front of the API is a real answer and throwing it away is how + "the API is broken" gets reported for a captive portal. + + The API key is **never** in here. It travels in a header and not in the URL + precisely so that an exception carrying the URL cannot carry the credential, + and nothing in this module formats it. + """ + + def __init__( + self, + message: str, + *, + status: Optional[int] = None, + body: Any = None, + ) -> None: + super().__init__(message) + self.status = status + self.body = body + + +class AuthError(ApiError): + """401 or 403: the API key is missing, wrong, or not allowed to do this. + + Separate from :class:`CredentialsError`, which is about the *proxy* login + and is raised before anything is sent. This one has been to the server. + Worth keeping apart in your own code as well: the two credentials are + different strings from different places, and a program that treats them as + one will tell you to fix the key when the password is the problem. + """ + + +class NotFoundError(ApiError): + """404: the thing addressed does not exist, or never did.""" + + +class RateLimitError(ApiError): + """429: too many requests. + + ``retry_after`` is the server's own number in seconds when it sent one, and + ``None`` when it did not. It is exposed rather than slept on, because this + package does not retry - see the note in ``__init__.py`` for the measurement + behind that, which is about the proxy pool rather than about this API, and + for the same reason applies less here. Waiting the number the server gave + you is not the behaviour that measurement warns about; a loop that ignores + it is. + """ + + def __init__( + self, + message: str, + *, + status: Optional[int] = None, + body: Any = None, + retry_after: Optional[float] = None, + ) -> None: + super().__init__(message, status=status, body=body) + self.retry_after = retry_after + + +class CheckError(NodeMavenError): + """:meth:`Proxy.check` could not reach the gateway at all. + + Deliberately not raised for a gateway that answered and refused. A 407 is an + answer and is reported as one in the returned :class:`~nodemaven.check.Check`, + because the status code is the diagnostic the caller came for. This error is + for the cases where nothing came back: DNS, a refused connection, a timeout, + a truncated status line. + """ diff --git a/src/nodemaven/providers.py b/src/nodemaven/providers.py index f5f4356..76d4e8f 100644 --- a/src/nodemaven/providers.py +++ b/src/nodemaven/providers.py @@ -29,12 +29,31 @@ else: # pragma: no cover - version dependent import tomli as tomllib -__all__ = ["Provider", "load", "load_file", "available"] +__all__ = ["Provider", "load", "load_file", "available", "ASCII_WHITESPACE"] DEFAULT_PROVIDER = "nodemaven" _REQUIRED = ("label", "known_params") +#: The six characters treated as whitespace in a parameter value. +#: +#: Spelled out rather than delegated to ``str.isspace`` or ``str.strip()`` with +#: no argument, because neither means the same thing in four languages: +#: ``str.isspace`` is Unicode-wide and also true of a no-break space, while +#: Rust's ``is_ascii_whitespace`` excludes the vertical tab that Python's +#: includes. A value carrying any of these is refused, so the set is part of the +#: cross-language contract and has to be a list somebody can copy. +ASCII_WHITESPACE = " \t\n\r\v\f" + +# ASCII case folding and nothing wider. ``str.lower()`` is Unicode-aware, so it +# maps characters like the Turkish dotted capital I in a way Rust and Go do not +# reproduce, and a golden vector that depended on it would fail in one SDK for +# reasons that have nothing to do with proxies. +_ASCII_LOWER = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "abcdefghijklmnopqrstuvwxyz", +) + @dataclass(frozen=True, eq=False) class Provider: @@ -62,6 +81,8 @@ class Provider: port: Optional[int] = None aliases: Dict[str, str] = field(default_factory=dict) values: Dict[str, Tuple[str, ...]] = field(default_factory=dict) + normalize: FrozenSet[str] = frozenset() + connect_reactions: Dict[str, str] = field(default_factory=dict) exit_ip_header: Optional[str] = None source: str = "" source_read: str = "" @@ -71,6 +92,18 @@ def spell(self, name: str) -> str: """The name this gateway uses on the wire for a canonical parameter.""" return self.aliases.get(name, name) + def reaction(self, status: int) -> Optional[str]: + """What a CONNECT status means **on this gateway**, or None if unrecorded. + + This is dialect and not HTTP. The shipped gateway answers a bad ``filter`` + value with 407 Proxy Authentication Required, which is a lie about the + cause in the most expensive direction available: it sends the caller to + check credentials that are correct. A status code alone is therefore not + a diagnosis here, and the translation is per-gateway, so it lives in the + TOML beside the separators rather than in a dict in this module. + """ + return self.connect_reactions.get(str(status)) + def allowed(self, name: str) -> Optional[Tuple[str, ...]]: """The legal values for a parameter, or None if they are not known. @@ -85,6 +118,36 @@ def allowed(self, name: str) -> Optional[Tuple[str, ...]]: """ return self.values.get(name) + def normalizes(self, name: str) -> bool: + """Whether this gateway wants the value of ``name`` folded. + + The parameters that say yes carry a human place name - a region, a city, + an ISP - and the gateway wants ``district_of_columbia`` where a caller + naturally writes ``District of Columbia``. See ``normalized()``. + """ + return name in self.normalize + + def normalized(self, name: str, value: str) -> str: + """The wire form of a value, or the value unchanged if it is not folded. + + The fold is three steps, in this order, and **it is part of the + cross-language contract** - a golden vector pins the username a set of + parameters produces, so four SDKs have to agree on it character for + character: + + 1. strip leading and trailing ``ASCII_WHITESPACE`` + 2. lower-case ASCII ``A-Z`` only + 3. replace each space with ``_`` + + Which parameters this applies to is declared in the provider TOML, as + data, for the same reason ``known_params`` is: the public API must never + name a gateway's parameters in its own code, and the next gateway will + fold a different set or none at all. + """ + if name not in self.normalize: + return value + return value.strip(ASCII_WHITESPACE).translate(_ASCII_LOWER).replace(" ", "_") + @property def is_measured(self) -> bool: """Whether traffic has actually gone through this gateway. @@ -184,6 +247,63 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: ) values[name] = tuple(str(item) for item in allowed) + # Which parameters are folded to their wire form before anything else looks + # at them. One list and one rule, deliberately: the vendor's own client + # applies two - lower-case for `country` and `type`, lower-case plus + # space-to-underscore for `region`, `city` and `isp` - and the difference + # cannot be observed, because a country code and a pool name have no spaces + # in them to convert. One rule that four languages have to agree on beats + # two. + normalize = frozenset(str(name) for name in (raw.get("normalize") or ())) + unknown_normalize = sorted(normalize - known) + if unknown_normalize: + raise ProviderError( + f"{path} normalizes {unknown_normalize} which are not in " + f"known_params, so those parameters are refused by name and the " + f"fold can never run." + ) + + # What each CONNECT status means on this gateway. Keys are the status code as + # a string, because TOML has no integer keys and JSON has none either - and + # the golden vectors are JSON, so a schema that used integers here would + # already have to be stringified to be shared. + # + # Any status may be described and none has to be: an entry is a sentence a + # human wrote after watching the gateway do it, so a gateway nobody has + # probed simply has none and `check()` reports the bare status. There is no + # validation to do beyond "it is a table of strings" - unlike `values`, an + # entry here cannot refuse anything, so a wrong one is a misleading sentence + # and not a blocked request. + connect_reactions: Dict[str, str] = {} + for status, meaning in (raw.get("connect_reactions") or {}).items(): + if not isinstance(meaning, str) or not meaning: + raise ProviderError( + f"{path} describes CONNECT status {status!r} as {meaning!r}. It has " + f"to be a non-empty string: this text is shown to a caller as the " + f"reason their connection was refused." + ) + connect_reactions[str(status)] = meaning + + separator = str(raw.get("separator", "-")) + pair_separator = str(raw.get("pair_separator", "-")) + + # A normalized value can never contain the separator, because the fold does + # not introduce one and a value carrying it is refused either way. But a + # separator of "_" would make the fold *produce* one - `city="New York"` + # becoming `new_york` and then being cut in half - so the definition that + # declares both is refused here rather than at the call site, where the + # caller would be blamed for input that is correct. + if normalize: + for candidate in (separator, pair_separator): + if candidate == "_": + raise ProviderError( + f"{path} separates parameters with {candidate!r} and also " + f"normalizes {sorted(normalize)}, and the fold turns a " + f"space into {candidate!r}. A value with a space in it " + f"would be cut at the separator the fold had just " + f"inserted. Pick one." + ) + port = raw.get("port") return Provider( id=provider_id or path.stem, @@ -191,13 +311,15 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: known_params=known, status=str(raw.get("status", "documented")), prefix=str(raw.get("prefix", "{login}")), - separator=str(raw.get("separator", "-")), - pair_separator=str(raw.get("pair_separator", "-")), + separator=separator, + pair_separator=pair_separator, session_param=session_param, host=raw.get("host"), port=int(port) if port is not None else None, aliases=aliases, values=values, + normalize=normalize, + connect_reactions=connect_reactions, exit_ip_header=raw.get("exit_ip_header"), source=str(raw.get("source", "")), source_read=str(raw.get("source_read", "")), diff --git a/src/nodemaven/proxy.py b/src/nodemaven/proxy.py index bf5c7f6..21d85bb 100644 --- a/src/nodemaven/proxy.py +++ b/src/nodemaven/proxy.py @@ -8,11 +8,15 @@ from __future__ import annotations import os -from typing import Any, Dict, Mapping, Optional +import secrets +from typing import Any, Dict, List, Mapping, Optional from urllib.parse import quote +from .check import DEFAULT_TARGET, Check +from .check import _port_number +from .check import connect as _connect from .errors import CredentialsError, ParamError -from .providers import Provider, load +from .providers import ASCII_WHITESPACE, Provider, load __all__ = ["Proxy"] @@ -78,7 +82,34 @@ def __init__( raw_port = port if port is not None else os.environ.get(f"{env}_PORT") self._host = raw_host or self._provider.host - self._port = int(raw_port) if raw_port not in (None, "") else self._provider.port + # Through `check._port_number` and not `int()`, and the import across + # modules is the point rather than a shortcut. That function already + # holds the rule - ASCII digits only, at most five of them, 1 to 65535 - + # with the three defects that produced it written down beside it. A + # second `int()` here is a second implementation of the same rule, and a + # rule with two implementations is the thing that let `check.rs` ship + # the missing `HTTP/` check that `check.py` also had. + # + # Measured 2026-09-08, both from an external review: `port='abc'` raised + # `ValueError: invalid literal for int() with base 10: 'abc'`, which is + # neither of this package's exception types and mentions nothing a + # caller can act on; and `port='0'` raised `CredentialsError` saying + # "pass host= and port=", although the port *was* passed. The first was + # the wrong class, the second the wrong sentence - both from `int()` + # accepting more than a port is and then a truthiness test standing in + # for a range check. + if raw_port in (None, ""): + self._port = self._provider.port + else: + self._port = _port_number(str(raw_port)) + if self._port is None: + raise CredentialsError( + f"port={raw_port!r} is not a TCP port: it has to be a whole " + f"number from 1 to 65535. Nothing was built. The gateway's " + f"own ports are {self._provider.port} and the ones its " + f"documentation lists; 0 is not one of them - it means 'any " + f"free port' when binding and is meaningless when connecting." + ) if not self._login or not self._password: missing = [ @@ -218,6 +249,145 @@ def session(self, session_id: str) -> "Proxy": ) return self.replace(**{self._provider.session_param: session_id}) + def sessions(self, count: int, *, length: int = 6) -> List["Proxy"]: + """``count`` identities with the same parameters and distinct session ids. + + The thing everybody writes by hand, and the two details that are easy to + get wrong when writing it by hand are why it is here. + + The ids are **hexadecimal**, from :mod:`secrets`. Hex because a session id + must not contain the gateway's separator - a value carrying one is cut and + every id sharing a prefix collapses onto one exit, measured 2026-08-20 - + and the alphabets people reach for first do not have that property: + :func:`secrets.token_urlsafe` emits ``-`` and ``_``, ``uuid4()`` emits + ``-`` four times, and base64 emits ``+`` and ``/``. Every one of those is + a separator on some gateway. The constructor would refuse them, loudly, + which is the safe failure - but only after the caller had written the + code. + + They come from :mod:`secrets` and not :mod:`random` because + :func:`random.random` is seeded from the clock and its stream is + reproducible: two processes started in the same millisecond would get the + same ids, so two workers meant to hold two exits would share one. + + ``length`` is in bytes, so the default is 12 hex characters and 2**48 + possible ids. + + **This paragraph used to end "so a very short ``length`` costs time and + not correctness", and that was wrong in two ways.** It was written about + the rejection loop below, which does guarantee distinctness within one + call, and it read the guarantee as free. + + The first way is a hang. Rejection sampling cannot produce more distinct + values than exist, so ``count`` above the size of the space is a loop + with no exit - measured 2026-09-08 in an external review and reproduced + here the same day: ``sessions(257, length=1)`` did not return in 4 s, + while ``sessions(200, length=1)`` built 200 immediately. The loop is not + slow there, it never finishes, and it does it while holding the CPU. So + ``count`` is now checked against the space, and the bound is strict: + asking for the whole space would draw every value that exists and leave + none for the next caller. + + The second way is not fixed by any check here and is the reason the + sentence was worth correcting rather than deleting. ``seen`` is local to + one call, so distinctness holds **inside a call and nowhere else**. Two + processes drawing 8-bit ids collide with each other about as often as + they do not, and a collision does not raise - it hands two workers one + exit and looks like a working program. That is the failure the default + length is set to make impossible rather than unlikely. + """ + if count < 1: + raise ParamError( + f"sessions({count!r}) asks for no identities. Nothing would be " + f"returned and the call is a mistake somewhere upstream." + ) + if length < 1: + raise ParamError(f"length={length!r} would produce an empty session id.") + # Compared in bits and not by computing 16**(2*length), which is a + # bignum for a large `length` in Python and an overflow in three of the + # four languages this has to hold in. `count.bit_length() > bits` is + # exactly `count >= 2**bits`, so the whole space is refused along with + # everything past it - one comparison, same answer everywhere. + bits = 8 * length + if count.bit_length() > bits: + raise ParamError( + f"sessions({count!r}, length={length!r}) asks for at least the " + f"whole space: {2 * length} hex characters make 2**{bits} " + f"distinct ids, and drawing without repeating is what this does. " + f"Raise length= rather than count=, and note that ids are only " + f"unique within one call - the space has to be large enough for " + f"every process that draws from it, not just for this one." + ) + + # Rejection rather than trust. 12 hex characters make a collision within + # a handful of draws vanishingly unlikely, and "vanishingly unlikely" is + # the wrong standard here: a collision does not raise, it hands two + # workers one exit and looks like a working program. Since a duplicate is + # detectable in one line, it is detected. + seen = set() + out: List["Proxy"] = [] + while len(out) < count: + session_id = secrets.token_hex(length) + if session_id in seen: + continue + seen.add(session_id) + out.append(self.session(session_id)) + return out + + # -- asking the gateway ------------------------------------------------- + + def check( + self, + *, + target: str = DEFAULT_TARGET, + timeout: float = 15.0, + ) -> Check: + """Open one CONNECT with these parameters and report what the gateway said. + + The one method here that touches the network, and it is a deliberate + exception to "this package opens no socket" rather than a retreat from + it. That rule is about **transport** - connection pools, timeouts, retry + semantics, four sets of bugs in four languages - and this is one socket, + opened when asked, closed before returning, holding no state. + + Returns a :class:`~nodemaven.check.Check`. A refused connection is a + return value and not an exception, because the status code is the reason + anyone calls this, and it arrives carrying the provider's own reading of + that code:: + + >>> result = proxy.check() # doctest: +SKIP + >>> result.ok, result.status # doctest: +SKIP + (False, 407) + >>> print(result.meaning) # doctest: +SKIP + usually NOT your credentials, despite what the status says... + + That last line is the whole point. 407 Proxy Authentication Required is + what this gateway answers to a bad ``filter`` value and to a bad ``ttl`` + value as well as to a wrong password, measured 2026-08-10, so the status + alone sends people to re-check credentials that are correct. + + **``ok`` is not "my settings were applied."** It means the gateway + accepted the request. An unrecognised parameter name is also answered + with 200 and dropped, which is why this package refuses unknown names + before sending; ``check()`` cannot recover that for you and does not + pretend to. + + ``target`` is the host the tunnel is opened to. It is a real third party + that sees a TCP connection from the exit address - there is no null + CONNECT - so it is a parameter and not a constant. Nothing is sent + through the tunnel: the exit address, when it arrives, comes back on the + CONNECT reply itself, so this costs one handshake and no target traffic. + """ + return _connect( + self.server, + self.username, + self._password, + target=target, + timeout=timeout, + exit_ip_header=self._provider.exit_ip_header, + reactions=self._provider.connect_reactions, + ) + # -- output that is not a credential ------------------------------------ def __repr__(self) -> str: @@ -240,14 +410,16 @@ def _validate(provider: Provider, params: Mapping[str, Any]) -> Dict[str, str]: """Refuse client-side what the gateway will not report. This is not politeness, it is the only check available. The gateway this - package ships a definition for answers seven kinds of bad input seven - different ways and none of them names the cause: a bad country or region - gives 406, a bad city 500, a bad filter or ttl value gives 407 - which sends - you to check credentials that are fine - an empty value hangs the connection - for about twenty seconds, and an unknown parameter name is answered with - **200 and the setting silently dropped**. That last one is why this function - exists: the request succeeds, and nothing that comes back can tell you the - setting was never applied. + package ships a definition for answers a value it will not take five + different ways and none of them names the parameter: a bad region gives 406, + a bad city 500, a bad isp 410, and a bad country, filter, ttl, type or speed + value gives 407 - which sends you to check credentials that are fine. An + empty value hangs the connection for about twenty seconds, and an unknown + parameter name is answered with **200 and the setting silently dropped**. + That last one is why this function exists: the request succeeds, and nothing + that comes back can tell you the setting was never applied. See + ``connect_reactions`` in the gateway definition for the sentence a caller is + shown next to each code. """ out: Dict[str, str] = {} for key, value in params.items(): @@ -258,15 +430,43 @@ def _validate(provider: Provider, params: Mapping[str, Any]) -> Dict[str, str]: f"and your setting would NOT be applied. " f"Known: {sorted(provider.known_params)}" ) - if value is None or value == "": + if isinstance(value, bool): + value = "true" if value else "false" + text = "" if value is None else str(value) + + # Fold before every remaining check, so that a refusal quotes the string + # that would actually have gone on the wire rather than the one that was + # typed. The folded value is what gets stored, so ``params``, + # ``username`` and the sticky-session identity all agree - a Proxy that + # reported ``New York`` while sending ``new_york`` would make two callers + # with the same visible configuration land on different exits. + text = provider.normalized(key, text) + + if not text: raise ParamError( f"empty value for {key!r}: the gateway does not reply to this, " f"the connection hangs for about 20 s and then fails. Drop the " f"parameter instead of passing an empty value." ) - if isinstance(value, bool): - value = "true" if value else "false" - text = str(value) + + # Whitespace inside a value, for a parameter nothing folds. There is no + # form of this that can be right: a username is one token on the CONNECT + # line, so the space either malforms the line or cuts the value short, + # and ``url()`` would percent-encode it to ``%20`` while a browser driver + # taking the fields separately would not - three spellings of one value, + # at most one of which any gateway accepts. Refusing it is loud, and loud + # beats a connection that succeeds with the setting quietly wrong. + whitespace = [c for c in text if c in ASCII_WHITESPACE] + if whitespace: + raise ParamError( + f"the value of {key!r} is {text!r} and contains whitespace " + f"({whitespace[0]!r}), which cannot be sent: a proxy username " + f"is a single token, so the value would be malformed or cut " + f"short. {provider.label} folds a space to an underscore for " + f"{sorted(provider.normalize)} and for nothing else, so pass " + f"{key!r} without whitespace." + ) + separators = sorted({provider.separator, provider.pair_separator} - {""}) bad = sorted({c for c in separators if c in text}) if bad: diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..85b4009 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,1180 @@ +"""The account API, driven through the transport seam. No socket, no key. + +Every case here supplies its own ``transport=``, which is the reason that seam +exists: the whole error mapping, the whole paging loop and every response shape +this module *assumes* can be exercised without an account and without a network. +The shapes are the interesting part - the module docstring in ``nodemaven.api`` +records that the paginated envelope is inferred from the vendor's own query +parameter names and has never been seen from this machine, so the cases below +include the shapes it might turn out to be instead. +""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlsplit + +import pytest + +from nodemaven import ( + ApiError, + AuthError, + Client, + CredentialsError, + NotFoundError, + Page, + Proxy, + RateLimitError, +) +from nodemaven.api import API_ROOT, DEFAULT_BASE_URL, DEFAULT_PAGE_SIZE + + +@pytest.fixture(autouse=True) +def no_api_env(monkeypatch): + """A developer's own ``.env`` must not reach into the suite. + + Both names are the vendor's, so they are plausibly already set on this + machine - and a test that silently picked up a real key would be a test that + passes here and fails in CI, or worse, one that spends a real call. + """ + monkeypatch.delenv("NODEMAVEN_APIKEY", raising=False) + monkeypatch.delenv("NODEMAVEN_BASE_URL", raising=False) + + +class Fake: + """A scripted transport that records what it was handed. + + ``responses`` is a list of ``(status, body)``, consumed in order; the last + one repeats, so a paging test does not have to count its own requests. A + body that is not ``bytes`` is JSON-encoded, because most cases care about the + shape rather than the encoding. + """ + + def __init__(self, *responses): + self.responses = list(responses) or [(200, {})] + self.calls = [] + + def __call__(self, method, url, headers, body): + self.calls.append( + { + "method": method, + "url": url, + "headers": headers, + "body": json.loads(body.decode()) if body else None, + } + ) + status, payload = ( + self.responses.pop(0) if len(self.responses) > 1 else self.responses[0] + ) + if isinstance(payload, bytes): + return status, payload + return status, json.dumps(payload).encode() + + @property + def url(self): + return self.calls[-1]["url"] + + @property + def headers(self): + return self.calls[-1]["headers"] + + +def client(*responses, **kwargs): + fake = Fake(*responses) + return Client(api_key="k3y", transport=fake, **kwargs), fake + + +class TestCredentialsAndTheirSecrecy: + def test_no_key_anywhere_is_refused_at_construction(self): + with pytest.raises(CredentialsError, match="NODEMAVEN_APIKEY"): + Client() + + def test_the_message_distinguishes_the_two_secrets(self): + # There are two credentials on this account and they come from different + # places. A program that treats them as one tells you to fix the key + # when the password is wrong. + with pytest.raises(CredentialsError, match="not the proxy password"): + Client() + + def test_the_key_is_read_from_the_vendors_own_variable(self, monkeypatch): + # Their name and not one of ours, so a `.env` written for their client + # works here unchanged. Two .env files that disagree is worse than a + # name we did not choose. + monkeypatch.setenv("NODEMAVEN_APIKEY", "from-the-environment") + fake = Fake((200, {})) + Client(transport=fake).me() + assert fake.headers["Authorization"] == "x-api-key from-the-environment" + + def test_the_repr_never_carries_the_key(self): + api, _ = client() + assert "k3y" not in repr(api) + assert "***" in repr(api) + + def test_the_key_travels_in_a_header_and_never_in_a_url(self): + api, fake = client((200, {})) + api.countries(country__code="us") + assert "k3y" not in fake.url + assert fake.headers["Authorization"] == "x-api-key k3y" + + def test_the_header_form_is_the_unusual_one_the_server_wants(self): + # `Authorization: x-api-key `, not an `X-API-Key` header. The + # conventional spelling would be answered 401, which reads as a bad key + # and sends the caller to regenerate a key that was fine. + api, fake = client((200, {})) + api.me() + assert "X-API-Key" not in fake.headers + assert fake.headers["Authorization"].startswith("x-api-key ") + + +class TestTheAccount: + def test_me_hits_the_documented_path(self): + api, fake = client((200, {"data": 123})) + api.me() + assert fake.url == f"{DEFAULT_BASE_URL}{API_ROOT}/users/me" + + def test_me_returns_the_servers_own_object_unmodelled(self): + # Deliberately not a dataclass, and the live response is the argument + # for it. Measured 2026-09-08, `users/me` answers the six fields below: + # traffic left is `data` and not `traffic_left`, and `is_traffic_frozen` + # is a string and not a bool. A dataclass written the day before from + # the vendor's client would have got both wrong - the first as an + # AttributeError on a working answer, the second as a value that is + # truthy whichever way it reads. + body = { + "data": 54316316722, + "email": "someone@example.test", + "is_traffic_frozen": "no", + "proxy_password": "p", + "proxy_username": "u", + "subscription_status": "active", + } + api, _ = client((200, body)) + assert api.me() == body + + def test_a_field_nobody_here_predicted_survives(self): + # The other half of not modelling it: an added field reaches the caller + # instead of being dropped by a schema written before it existed. + body = {"data": 1, "a_field_nobody_here_predicted": 2} + api, _ = client((200, body)) + assert api.me() == body + + def test_a_list_where_an_object_was_expected_is_refused_loudly(self): + api, _ = client((200, [1, 2, 3])) + with pytest.raises(ApiError, match="where an object was expected"): + api.me() + + def test_the_base_url_is_overridable_and_its_trailing_slash_is_dropped(self): + api, fake = client((200, {}), base_url="https://staging.example.test/") + api.me() + assert fake.url == f"https://staging.example.test{API_ROOT}/users/me" + + def test_the_base_url_comes_from_the_environment_too(self, monkeypatch): + monkeypatch.setenv("NODEMAVEN_BASE_URL", "https://env.example.test") + fake = Fake((200, {})) + Client(api_key="k", transport=fake).me() + assert fake.url.startswith("https://env.example.test") + + +class TestTheCatalogue: + @pytest.mark.parametrize( + "method,path", + [ + ("countries", "/locations/countries/"), + ("regions", "/locations/regions/"), + ("cities", "/locations/cities/"), + ("isps", "/locations/isps/"), + ("isp_regions", "/locations/isps/regions/"), + ("isp_cities", "/locations/isps/cities/"), + ("zip_codes", "/locations/zipcodes/"), + ("zip_code_regions", "/locations/zipcodes/regions/"), + ("zip_code_cities", "/locations/zipcodes/cities/"), + ], + ) + def test_each_endpoint_is_where_the_spec_says_it_is(self, method, path): + # `zip_codes` said `/locations/zip-codes/` here until 2026-09-09, + # transcribed from the vendor's client, and this test passed the whole + # time - because it asserted the transcription against itself. The real + # path is solid, `zipcodes`, and there is no naming rule on this API to + # have derived it from: `sub-users` is hyphenated and `users/me` carries + # no trailing slash where its neighbours do. A test that pins a path + # can only ever pin where the path came from. + api, fake = client((200, {"count": 0, "results": []})) + getattr(api, method)() + assert fake.url.split("?")[0] == f"{DEFAULT_BASE_URL}{API_ROOT}{path}" + + @pytest.mark.parametrize( + "method", + [ + "countries", + "regions", + "cities", + "isps", + "isp_regions", + "isp_cities", + "zip_codes", + "zip_code_regions", + "zip_code_cities", + ], + ) + def test_every_list_call_sends_limit_and_offset(self, method): + # Not a default, a requirement. Measured 2026-09-08 against the live + # API: `locations/isps` refuses a request that omits them, and the other + # three location endpoints answer such a request with exactly 50 rows + # and no reported total - byte-for-byte what a complete collection looks + # like. The refusal is visible on the first call; the truncation is not + # visible ever, which is why these are sent rather than left out. + api, fake = client((200, {"count": 0, "results": []})) + getattr(api, method)() + assert f"limit={DEFAULT_PAGE_SIZE}" in fake.url + assert "offset=0" in fake.url + + def test_a_callers_limit_and_offset_win_over_the_defaults(self): + # Parsed rather than matched as a substring: `limit=50` is a substring + # of `limit=500`, so the obvious `not in` assertion passes for the + # wrong reason and fails for the right one. It did, once. + api, fake = client((200, {"count": 0, "results": []})) + api.countries(limit=500, offset=50) + query = parse_qs(urlsplit(fake.url).query) + assert query["limit"] == ["500"] + assert query["offset"] == ["50"] + + def test_djangos_double_underscore_filter_is_passed_through_untranslated(self): + # The server's spelling, not a typo. Renaming it to `country_code` would + # be one alias to maintain and one more place a caller's filter is + # dropped in silence, because an unknown query parameter is ignored by + # every REST framework there is. + api, fake = client((200, {"results": []})) + api.cities(country__code="us", region__code="dc") + assert "country__code=us" in fake.url + assert "region__code=dc" in fake.url + + def test_a_none_filter_is_dropped_rather_than_sent_as_the_word_none(self): + # `countries(name=None)` must send nothing. Sent, the string "None" + # matches zero rows and comes back as an empty catalogue, which reads as + # "the product has no countries". + api, fake = client((200, {"results": []})) + api.countries(name=None, country__code="us") + assert "name=" not in fake.url + assert "country__code=us" in fake.url + + def test_no_connection_type_default_is_added_here(self): + # The server defaults it to residential. Sending a default of our own + # would show up in a support ticket as something the caller chose. + api, fake = client((200, {"results": []})) + api.countries() + assert "connection_type" not in fake.url + + +class TestThePageShapeIsInferredSoEveryShapeIsHandled: + def test_the_paginated_envelope(self): + api, _ = client( + (200, {"count": 250, "next": "u", "previous": None, "results": [1, 2]}) + ) + page = api.countries() + assert isinstance(page, Page) + assert page.count == 250 + assert page.next == "u" + assert list(page) == [1, 2] + assert len(page) == 2 + + def test_a_bare_list(self): + # What the endpoint answers if it is not paginated after all. + api, _ = client((200, [{"code": "us"}, {"code": "de"}])) + page = api.countries() + assert len(page) == 2 + assert page.count is None + + def test_a_single_object_is_wrapped_and_the_wrapping_is_visible(self): + # Real servers do this for a filter matching one row. `count is None` is + # how a caller can tell it was wrapped rather than counted. + api, _ = client((200, {"code": "us", "name": "United States"})) + page = api.countries() + assert len(page) == 1 + assert page.results[0]["code"] == "us" + assert page.count is None + + def test_an_empty_object_is_an_empty_page_and_not_a_page_of_one(self): + api, _ = client((200, {})) + assert len(api.countries()) == 0 + + def test_something_that_is_neither_names_what_came_back(self): + # The alternative is `body["results"]`, whose KeyError describes this + # package's assumption instead of the server's answer - on the one code + # path where the assumption is explicitly untested. + api, _ = client((200, b'"a captive portal wrote this"')) + with pytest.raises(ApiError, match="neither a page nor a list"): + api.countries() + + def test_the_refusal_mentions_the_two_things_that_actually_do_this(self): + api, _ = client((200, b"42")) + with pytest.raises(ApiError, match="captive portal"): + api.countries() + + +class TestIspsPutsItsRowsUnderADifferentKey: + """``locations/isps`` answers a different envelope from the other four. + + Measured 2026-09-08 with ``country__code="us"``: the object is keyed + ``city``, ``country``, ``isps``, ``region``, there is no ``results``, and + ``isps`` is a list of 358. The bodies below are that shape. + + For one day this method returned a page of exactly one row - the envelope + itself - and was documented as broken rather than fixed, because the first + probe printed key *names* and not values. These cases are what the second + probe bought. + """ + + ENVELOPE = { + "city": "", + "country": "us", + "region": "", + "isps": [{"id": 1, "name": "Comcast"}, {"id": 2, "name": "Charter"}], + } + + def test_the_rows_are_the_isps_list_and_not_the_envelope(self): + api, _ = client((200, self.ENVELOPE)) + page = api.isps(country__code="us") + assert len(page) == 2 + assert [row["name"] for row in page] == ["Comcast", "Charter"] + + def test_the_three_string_fields_are_not_rows(self): + # They are strings whose contents have never been printed, so this + # package says nothing about them - including by handing them back + # inside a list of ISPs. + api, _ = client((200, self.ENVELOPE)) + assert all(isinstance(row, dict) for row in api.isps()) + + def test_the_key_is_per_endpoint_and_not_a_search_for_any_list(self): + # The control for the fix. An envelope carrying two lists would be + # resolved by dict order if `_page` went looking for "whichever value + # is a list", silently and differently per server version. So the same + # body through `countries()` takes the single-object branch, because + # `countries` reads `results` and this body has none. + api, _ = client((200, self.ENVELOPE)) + page = api.countries() + assert len(page) == 1 + assert page.results[0] == self.ENVELOPE + + def test_a_body_with_no_isps_key_still_falls_through_to_the_old_branches(self): + api, _ = client((200, {"detail": "one row"})) + assert len(api.isps()) == 1 + + def test_the_walk_reads_the_second_page_the_same_way_as_the_first(self): + # `iterate` reparses each page it fetches, so the key has to travel on + # the Page. Without that it would read page one as ISPs and page two as + # a one-row envelope, and stop - a truncation wearing the shape of an + # ending. + api, fake = client( + (200, {"isps": list(range(50))}), + (200, {"isps": list(range(50, 60))}), + (200, {"isps": []}), + ) + assert list(api.iterate(api.isps(limit=50))) == list(range(60)) + assert len(fake.calls) == 3 + + def test_an_offset_past_the_end_ends_the_walk(self): + # Arm E and arm H of the live run, 2026-09-08: `offset=1000000` and + # `offset=1000` both answer 200 with `isps` an empty list rather than + # the last page. That is the ending `iterate` stops on, and it is also + # what rules out the server accepting `offset` and dropping it - an + # ignored offset would have answered all 358 rows. + api, _ = client( + (200, {"isps": list(range(50))}), + (200, {"isps": []}), + ) + assert list(api.iterate(api.isps(limit=50))) == list(range(50)) + + +class TestErrorsAreClassesACallerWouldBranchOn: + def test_401_is_an_auth_error_naming_which_secret(self): + api, _ = client((401, {"detail": "Invalid token."})) + with pytest.raises(AuthError, match="NODEMAVEN_APIKEY") as caught: + api.me() + assert caught.value.status == 401 + assert caught.value.body == {"detail": "Invalid token."} + + def test_403_is_the_same_class(self): + api, _ = client((403, {"detail": "no"})) + with pytest.raises(AuthError): + api.me() + + def test_404_is_its_own_class_because_crud_branches_on_it(self): + api, _ = client((404, {"detail": "Not found."})) + with pytest.raises(NotFoundError) as caught: + api.delete_sub_user(7) + assert caught.value.status == 404 + + def test_429_carries_the_servers_own_interval_and_retries_nothing(self): + api, _ = client((429, {"detail": "Request was throttled."})) + with pytest.raises(RateLimitError, match="Nothing here retries") as caught: + api.me() + assert caught.value.retry_after is None + + def test_5xx_says_that_hammering_it_is_the_thing_we_decline_to_do(self): + api, _ = client((503, {"detail": "upstream down"})) + with pytest.raises(ApiError) as caught: + api.me() + assert caught.value.status == 503 + assert not isinstance(caught.value, (AuthError, NotFoundError, RateLimitError)) + + def test_400_stays_on_the_base_class_because_the_fix_is_your_program(self): + # A class exists here only if a caller would plausibly write different + # code for it. 401 means fix your key, 429 means wait, 404 in CRUD means + # the row is gone - a 400 means fix the call, and there is nothing to + # branch on. + api, _ = client((400, {"expiry_date": ["Enter a valid date."]})) + with pytest.raises(ApiError) as caught: + api.me() + assert type(caught.value) is ApiError + assert "Enter a valid date." in str(caught.value) + + def test_every_api_error_is_a_nodemaven_error(self): + from nodemaven import NodeMavenError + + for cls in (ApiError, AuthError, NotFoundError, RateLimitError): + assert issubclass(cls, NodeMavenError) + + def test_the_url_is_in_the_message_and_the_key_is_not(self): + api, _ = client((500, {"detail": "boom"})) + with pytest.raises(ApiError) as caught: + api.me() + assert "/users/me" in str(caught.value) + assert "k3y" not in str(caught.value) + + +class TestTheServersOwnMessageSurvives: + def test_a_detail_string(self): + api, _ = client((400, {"detail": "the useful sentence"})) + with pytest.raises(ApiError, match="the useful sentence"): + api.me() + + def test_field_errors_are_flattened_rather_than_str_of_a_dict(self): + # `str()` of a dict shows Python syntax to somebody debugging an HTTP + # call. + api, _ = client((400, {"username": ["too short", "taken"]})) + with pytest.raises(ApiError) as caught: + api.me() + assert "username: too short; taken" in str(caught.value) + assert "{" not in str(caught.value) + + def test_a_bare_list_body(self): + api, _ = client((400, ["first", "second"])) + with pytest.raises(ApiError, match="first; second"): + api.me() + + def test_a_body_that_is_not_json_at_all(self): + api, _ = client((502, b"Bad Gateway")) + with pytest.raises(ApiError, match="Bad Gateway"): + api.me() + + def test_an_empty_error_body_falls_back_to_the_status(self): + api, _ = client((418, b"")) + with pytest.raises(ApiError, match="HTTP 418"): + api.me() + + +class TestSubUsers: + """Five write calls and one read, none of which has ever been sent. + + Each one costs a real object on a production account, so every shape below + is the vendor's OpenAPI document of 2026-09-09 and nothing more. What these + cases pin is that this package sends what that document describes - not that + the document is right, which is a separate claim and an unmeasured one. + + The bodies here were ``username``/``password`` against + ``sub-users/{id}/`` until that document was read. Both were wrong, and the + tests asserting them passed, because they were written from the same + transcription the code was. + """ + + ENVELOPE = { + "success": True, + "description": "", + "errors": [], + "payload": {"id": "4", "proxy_username": "kid"}, + } + + def test_the_rows_come_out_of_payload_and_not_results(self): + # Measured 2026-09-09 and declared by the spec as SubUserManyResponse. + # Reading `results` here returned an empty page against a populated + # account and said nothing about it. + many = dict(self.ENVELOPE, payload=[{"id": "1"}, {"id": "2"}]) + api, _ = client((200, many)) + assert [row["id"] for row in api.sub_users()] == ["1", "2"] + + def test_create_sends_the_fields_the_spec_names_required(self): + api, fake = client((200, self.ENVELOPE)) + api.create_sub_user("kid", "pw") + assert fake.calls[-1]["method"] == "POST" + assert fake.calls[-1]["body"] == { + "proxy_username": "kid", + "proxy_password": "pw", + } + + def test_the_envelope_is_unwrapped_so_callers_never_see_payload(self): + api, _ = client((200, self.ENVELOPE)) + assert api.create_sub_user("kid", "pw") == {"id": "4", "proxy_username": "kid"} + + def test_a_2xx_carrying_success_false_is_refused(self): + # The envelope has a success flag, which is a server reserving the right + # to disagree with its own status line. Reading the status alone would + # report this as done. + api, _ = client( + (200, {"success": False, "description": "no", "errors": [], "payload": None}) + ) + with pytest.raises(ApiError, match="success=false"): + api.create_sub_user("kid", "pw") + + def test_the_optional_fields_are_omitted_when_unset(self): + api, fake = client((200, self.ENVELOPE)) + api.create_sub_user("kid", "pw", traffic_limit=1024) + assert fake.calls[-1]["body"] == { + "proxy_username": "kid", + "proxy_password": "pw", + "traffic_limit": 1024, + } + + def test_extra_fields_pass_through(self): + api, fake = client((200, self.ENVELOPE)) + api.create_sub_user("kid", "pw", note="whatever the server calls it") + assert fake.calls[-1]["body"]["note"] == "whatever the server calls it" + + def test_update_is_a_put_to_the_collection_with_the_id_in_the_body(self): + # There is no `sub-users/{id}/` path in the spec at all. The old + # spelling would have been a 404 - or, on this host, a 200 carrying the + # dashboard's HTML, which is worse. + api, fake = client((200, self.ENVELOPE)) + api.update_sub_user("9", traffic_limit=2048) + assert fake.calls[-1]["method"] == "PUT" + assert fake.calls[-1]["url"].endswith(f"{API_ROOT}/sub-users/") + assert fake.calls[-1]["body"] == {"traffic_limit": 2048, "id": "9"} + + def test_an_empty_change_set_is_refused_and_nothing_is_sent(self): + # A body of nothing but an id can be answered 200, and a call that + # reports success while changing nothing is the failure mode this whole + # package is organised against. + api, fake = client((200, {})) + with pytest.raises(ApiError, match="answered 200"): + api.update_sub_user("9") + assert fake.calls == [] + + def test_delete_sends_the_id_as_a_query_parameter(self): + api, fake = client((204, b"")) + assert api.delete_sub_user("9") == {} + assert fake.calls[-1]["method"] == "DELETE" + assert parse_qs(urlsplit(fake.url).query)["id"] == ["9"] + assert urlsplit(fake.url).path == f"{API_ROOT}/sub-users/" + + def test_reset_usage_takes_a_list_even_for_one(self): + # So the one-and-many cases cannot diverge. `ids` is the endpoint's only + # required field and it is an array. + api, fake = client((200, dict(self.ENVELOPE, payload=[]))) + api.reset_sub_user_usage(["9"]) + assert fake.calls[-1]["method"] == "POST" + assert fake.calls[-1]["url"].endswith(f"{API_ROOT}/sub-users/reset/usage") + assert fake.calls[-1]["body"] == {"ids": ["9"]} + + +class TestWhitelistIps: + def test_the_list_path_has_no_trailing_slash(self): + # `whitelist-ips/` was an invention. Measured 2026-09-09: it answered + # 200 with 6415 bytes of the dashboard's HTML, byte-identical to a path + # nobody registered. + api, fake = client((200, {"results": []})) + api.whitelist_ips() + assert urlsplit(fake.url).path == f"{API_ROOT}/whitelist/ips" + + def test_page_size_is_sent_because_the_documented_default_is_five(self): + api, fake = client((200, {"results": []})) + api.whitelist_ips() + assert parse_qs(urlsplit(fake.url).query)["page_size"] == ["100"] + + def test_page_one_is_sent_because_the_base_was_measured(self): + # This asserted `"page" not in query` until 2026-09-09, when nothing said + # whether the base was 0 or 1. Phase 9 of `probe_account_api.py` says 1: + # page 1 is the only number this endpoint answers 200, against 404 on 0, + # 2 and 9999. + api, fake = client((200, {"results": []})) + api.whitelist_ips() + assert parse_qs(urlsplit(fake.url).query)["page"] == ["1"] + + def test_upsert_posts_protocol_as_well_as_the_required_pair(self): + # `protocol` is sent because the server does not apply the + # `default: "HTTP"` its own document declares for it. Measured + # 2026-09-09 16:18 by the ladder in `lab\probes\probe_account_api.py`: + # ip + ports_count + name is refused 400 "Please enter a valid + # protocol(HTTP or SOCKS5)."; the same body plus protocol is accepted + # 201. Drop it from the body and every call to this endpoint fails. + api, fake = client((200, {"message": "ok"})) + api.upsert_whitelist_ip("203.0.113.7", 4, name="the office") + assert fake.calls[-1]["method"] == "POST" + assert fake.calls[-1]["url"].endswith(f"{API_ROOT}/whitelist/ip/upsert") + assert fake.calls[-1]["body"] == { + "ip": "203.0.113.7", + "ports_count": 4, + "protocol": "HTTP", + "name": "the office", + } + + def test_protocol_can_be_overridden(self): + api, fake = client((200, {"message": "ok"})) + api.upsert_whitelist_ip("203.0.113.7", 4, protocol="SOCKS5") + assert fake.calls[-1]["body"]["protocol"] == "SOCKS5" + + def test_it_is_named_upsert_because_that_is_what_it_does(self): + # Passing an id updates an existing row. A method called `add` that + # silently updates is a worse bug than a wrong path, because the wrong + # path fails. + assert not hasattr(Client, "add_whitelist_ip") + assert hasattr(Client, "upsert_whitelist_ip") + + def test_one_row_is_read_by_id(self): + api, fake = client((200, {"id": "3", "ip": "203.0.113.7"})) + api.whitelist_ip(3) + assert fake.calls[-1]["method"] == "GET" + assert fake.calls[-1]["url"].endswith(f"{API_ROOT}/whitelist/ip/3") + + def test_delete_targets_one_row(self): + api, fake = client((204, b"")) + api.delete_whitelist_ip(3) + assert fake.calls[-1]["method"] == "DELETE" + assert fake.calls[-1]["url"].endswith(f"{API_ROOT}/whitelist/ip/3") + + def test_an_id_cannot_rewrite_the_path(self): + # Ids come from the server, which makes this look unnecessary - and the + # day one comes from a config file or an argv instead, it is the + # difference between a 404 and a DELETE against something else. + # + # The assertion is about slashes and not about the text: the traversal + # still reads as `../../sub-users/1` in the url, escaped, and that is + # harmless. What would not be harmless is one surviving `/`, because + # that is the character that ends a path segment. + api, fake = client((204, b"")) + api.delete_whitelist_ip("../../sub-users/1") + segment = fake.url.split(f"{API_ROOT}/whitelist/ip/")[1] + assert "/" not in segment + assert segment == "..%2F..%2Fsub-users%2F1" + + +class TestStatistics: + """One method became three, because `/statistics/` was never a path. + + Measured 2026-09-09: it answered 200 with the dashboard's HTML. The three + that exist are `statistics/data/`, `.../requests/` and `.../domains/`, and + all three require `proxy_username`. + """ + + def test_there_is_no_bare_statistics_call_any_more(self): + assert not hasattr(Client, "statistics") + + @pytest.mark.parametrize( + "method,path", + [ + ("statistics_data", "/statistics/data/"), + ("statistics_requests", "/statistics/requests/"), + ], + ) + def test_the_two_series_endpoints_return_the_two_arrays_unmodelled( + self, method, path + ): + body = {"labels": ["2026-09-01"], "data": [5]} + api, fake = client((200, body)) + assert getattr(api, method)("acct-1") == body + assert urlsplit(fake.url).path == f"{API_ROOT}{path}" + + def test_proxy_username_is_positional_because_the_server_requires_it(self): + # The difference between a 400 at run time and a TypeError at the call + # site. + api, fake = client((200, {"labels": [], "data": []})) + with pytest.raises(TypeError): + api.statistics_data() + assert fake.calls == [] + + def test_the_optional_filters_are_passed_through_untranslated(self): + # Dates are not validated here. The server has to check them anyway, and + # the spec cannot make up its mind what the format is - the prose says + # "dd-mm-yyyy" and the type says `format: date`, which is `yyyy-mm-dd`. + api, fake = client((200, {"labels": [], "data": []})) + api.statistics_data("acct-1", start="2026-09-01", period="hours24") + query = parse_qs(urlsplit(fake.url).query) + assert query["start"] == ["2026-09-01"] + assert query["period"] == ["hours24"] + assert query["proxy_username"] == ["acct-1"] + + def test_domain_rows_come_out_of_data_and_the_call_does_not_page(self): + # `{"data": [...]}` with no cursor of any kind - `limit` there is a + # top-N cut and there is no `offset` in the spec's parameter list. So + # iterating it yields these rows and makes no second request. + api, fake = client( + (200, {"data": [{"domain_name": "a", "requests": 2, "data": 3}]}) + ) + page = api.domain_statistics("acct-1", limit=10) + assert isinstance(page, Page) + assert [row["domain_name"] for row in page] == ["a"] + assert list(api.iterate(page)) == page.results + assert len(fake.calls) == 1 + assert urlsplit(fake.url).path == f"{API_ROOT}/statistics/domains/" + + +class TestIterate: + def test_it_follows_next_to_the_end(self): + api, fake = client( + (200, {"count": 4, "next": f"{DEFAULT_BASE_URL}/p2", "results": [1, 2]}), + (200, {"count": 4, "next": None, "results": [3, 4]}), + (200, {"count": 4, "next": None, "results": []}), + ) + assert list(api.iterate(api.countries(limit=2))) == [1, 2, 3, 4] + + def test_an_absolute_next_url_on_the_same_host_is_used_as_given(self): + # Paging links come back absolute, so they are followed rather than + # re-derived: the server has said where the next page is, including + # whatever query it needs, and rebuilding that from the path would be + # guessing at a shape this module has never seen from the live API. + api, fake = client( + (200, {"next": f"{DEFAULT_BASE_URL}/api/v2/base/x?offset=50", "results": [1]}), + (200, {"next": None, "results": []}), + ) + list(api.iterate(api.countries())) + assert fake.calls[1]["url"] == f"{DEFAULT_BASE_URL}/api/v2/base/x?offset=50" + + def test_a_next_that_repeats_is_a_loop_and_is_refused(self): + # A server whose `next` points at the page you are on turns `while + # next:` into an unbounded run of real HTTP requests, whose first + # symptom is a rate limit rather than a hang. + api, _ = client((200, {"next": f"{DEFAULT_BASE_URL}/same", "results": [1]})) + with pytest.raises(ApiError, match="already returned"): + list(api.iterate(api.countries())) + + def test_the_bound_raises_rather_than_truncating_silently(self): + api, _ = client( + *[ + (200, {"next": f"{DEFAULT_BASE_URL}/p{n}", "results": [n]}) + for n in range(2, 12) + ] + ) + with pytest.raises(ApiError, match="max_pages"): + list(api.iterate(api.countries(), max_pages=3)) + + def test_an_endpoint_that_does_not_page_makes_no_further_request(self): + # `domain_statistics` is the only one. Everything else costs the spare + # request below, on purpose. + api, fake = client((200, {"data": [1, 2]})) + assert list(api.iterate(api.domain_statistics("acct-1"))) == [1, 2] + assert len(fake.calls) == 1 + + +class TestIterateWalksByOffsetWhenThereIsNoNext: + """What an envelope that fills nothing forces, measured 2026-09-08. + + `countries`, `regions` and `cities` answer with a paging envelope, and that + envelope reported **no `count`** on a page of 50 out of 192. Confirmed + against the vendor's spec on 2026-09-09: `PaginatedCountryList` and its + siblings declare `results` and nothing else, so there is no `count` and no + `next` field to fill. Stopping at an empty `next` would yield the first page + and call it the collection, which the caller could not tell apart from a + complete answer. + + **The walk stops on an empty page, not on a short one, and that changed on + 2026-09-09.** The old rule was "shorter than the limit asked for", and it + truncates against a server that caps the size below the request. This one + does: `cities(limit=10000)` is answered with 1000 rows out of 1965, and the + old rule reads those 1000 as the end. The measurement was already written in + `api.py` and the stop rule was never checked against it. + + This class said "a bare JSON array" and pinned the opposite behaviour for + part of 2026-09-08. See `Client.iterate` for what that reading was built on. + """ + + def test_a_full_page_is_followed_at_the_next_offset(self): + api, fake = client( + (200, {"results": list(range(0, 50))}), + (200, {"results": list(range(50, 100))}), + (200, {"results": list(range(100, 110))}), + (200, {"results": []}), + ) + assert list(api.iterate(api.countries(limit=50))) == list(range(110)) + offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] + assert offsets == [["0"], ["50"], ["100"], ["150"]] + + def test_a_short_page_is_followed_because_the_server_caps_the_limit(self): + # This asserted the opposite until 2026-09-09 and it was the reason the + # old rule survived: a short page really is the end most of the time, so + # the test passed and the truncating case was never written down. + api, fake = client( + (200, {"results": list(range(49))}), (200, {"results": []}) + ) + assert list(api.iterate(api.countries(limit=50))) == list(range(49)) + assert len(fake.calls) == 2 + + def test_the_capped_page_that_the_old_rule_truncated(self): + # The live case, 2026-09-08: `cities(limit=10000)` is answered with 1000 + # rows because 1000 is the server ceiling, and there are 1965. Under + # "stop on a short page" this walk returned 1000 and reported nothing. + api, fake = client( + (200, {"results": list(range(0, 1000))}), + (200, {"results": list(range(1000, 1965))}), + (200, {"results": []}), + ) + assert len(list(api.iterate(api.cities(limit=10000)))) == 1965 + + def test_a_caller_who_raised_the_limit_pages_at_that_limit(self): + api, fake = client( + (200, {"results": list(range(200))}), + (200, {"results": list(range(3))}), + (200, {"results": []}), + ) + list(api.iterate(api.countries(limit=200))) + offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] + assert offsets == [["0"], ["200"], ["400"]] + + def test_the_default_limit_is_what_a_caller_who_asks_for_nothing_pages_at(self): + api, fake = client( + (200, {"results": list(range(DEFAULT_PAGE_SIZE))}), + (200, {"results": [1]}), + (200, {"results": []}), + ) + list(api.iterate(api.countries())) + offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] + assert offsets[:2] == [["0"], [str(DEFAULT_PAGE_SIZE)]] + + def test_a_bare_array_pages_the_same_way(self): + # The shape this whole class was written for and which has never been + # seen from this server. `_page` still accepts it, so it is still + # exercised - a branch nothing tests is a branch that has already rotted. + api, fake = client( + (200, list(range(0, 50))), + (200, list(range(50, 60))), + (200, []), + ) + assert list(api.iterate(api.countries(limit=50))) == list(range(60)) + assert len(fake.calls) == 3 + + def test_a_server_that_ignores_offset_raises_rather_than_repeating(self): + # The guard that makes the offset walk safe to infer. The server was + # measured to *require* `limit` and `offset` on `locations/isps`, never + # to *honour* them across pages, and those are different claims. A + # server that accepts `offset` and ignores it hands back the same 50 + # rows for every page; without this, `iterate()` would return them a + # hundred times over and call it a collection. + api, _ = client( + (200, {"results": list(range(50))}), (200, {"results": list(range(50))}) + ) + with pytest.raises(ApiError, match="accepting `offset` and ignoring it"): + list(api.iterate(api.countries(limit=50))) + + def test_an_envelope_with_an_empty_next_is_not_read_as_the_end(self): + # The inverse of what this asserted for a few hours on 2026-09-08, when + # it was written against a hand-made Django REST Framework envelope + # carrying a real `count`. The live envelope leaves `count` empty on a + # page of 50 out of 192, so an empty `next` beside it is absence and not + # an answer, and honouring it stopped every catalogue read after one + # page. One spare request is the price, and the second answer here is + # what a genuine ending looks like. + api, fake = client( + (200, {"count": None, "next": None, "results": list(range(50))}), + (200, {"count": None, "next": None, "results": []}), + ) + assert list(api.iterate(api.countries(limit=50))) == list(range(50)) + assert len(fake.calls) == 2 + + +class TestIterateRefusesToGuessAPageNumber: + """`sub-users/` and `whitelist/ips` number their pages, and both start at 1. + + **This class used to pin the opposite, and the name is kept so the retired + behaviour stays findable.** Until 2026-09-09 nothing measured said whether + the first page was 0 or 1. The two readings are not symmetric - guessing 1 + against a 0-based server drops the first page in silence - so no cursor was + sent and `iterate()` raised rather than invent one. + + What retired it is `probe_account_api.py --phase 9`, run 2026-09-09 from the + user's own connection, asking each endpoint for pages 0, 1, 2 and 9999 at one + row a page. `sub-users/` answered page 0 with no rows and page 1 with the + account's single sub-user, which a 0-based server cannot produce. + `whitelist/ips` holds nothing and answers `200` on page 1 against `404` on 0, + 2 and 9999, which is the weaker reading and is recorded as such in `Paging`. + """ + + def test_the_first_page_is_asked_for_by_number(self): + body = {"success": True, "description": "", "errors": [], "payload": [{"id": 1}]} + api, fake = client((200, body)) + api.sub_users() + assert parse_qs(urlsplit(fake.calls[0]["url"]).query)["page"] == ["1"] + + def test_an_unnumbered_call_is_walked_from_page_one(self): + pages = [ + {"success": True, "description": "", "errors": [], "payload": [{"id": 1}]}, + {"success": True, "description": "", "errors": [], "payload": [{"id": 2}]}, + {"success": True, "description": "", "errors": [], "payload": []}, + ] + api, fake = client(*[(200, body) for body in pages]) + assert [row["id"] for row in api.iterate(api.sub_users())] == [1, 2] + numbers = [parse_qs(urlsplit(c["url"]).query)["page"] for c in fake.calls] + assert numbers == [["1"], ["2"], ["3"]] + + def test_the_rows_of_the_first_page_are_still_returned(self): + body = {"success": True, "description": "", "errors": [], "payload": [{"id": 1}]} + api, _ = client((200, body)) + assert [row["id"] for row in api.sub_users()] == [1] + + def test_a_caller_who_names_a_page_is_walked_from_it(self): + pages = [ + {"success": True, "description": "", "errors": [], "payload": [{"id": 1}]}, + {"success": True, "description": "", "errors": [], "payload": [{"id": 2}]}, + {"success": True, "description": "", "errors": [], "payload": []}, + ] + api, fake = client(*[(200, body) for body in pages]) + assert [row["id"] for row in api.iterate(api.sub_users(page=1))] == [1, 2] + numbers = [parse_qs(urlsplit(c["url"]).query)["page"] for c in fake.calls] + assert numbers == [["1"], ["2"], ["3"]] + + def test_the_whitelist_walks_by_its_own_parameter_names(self): + api, fake = client( + (200, {"results": [{"id": "a"}]}), + (200, {"results": []}), + ) + list(api.iterate(api.whitelist_ips(page=1))) + query = parse_qs(urlsplit(fake.calls[-1]["url"]).query) + assert query["page"] == ["2"] + assert query["page_size"] == ["100"] + + def test_the_whitelist_ends_its_walk_on_a_404(self): + # Measured 2026-09-09: this endpoint answers a page past the end with + # 404 rather than with an empty page, so the stop-on-empty rule never + # fires and the walk would raise at the end of every collection. + api, _ = client( + (200, {"results": [{"id": "a"}]}), + (404, {"detail": "Invalid page."}), + ) + assert [row["id"] for row in api.iterate(api.whitelist_ips())] == ["a"] + + def test_a_404_still_raises_where_it_was_not_measured(self): + # The two page-number endpoints do not agree: `sub-users/` answers 200 + # with an empty payload past the end. Treating 404 as an ending + # everywhere would swallow a path this module got wrong, which is + # exactly the failure four of these paths had on 2026-09-09. + first = {"success": True, "description": "", "errors": [], "payload": [{"id": 1}]} + api, _ = client((200, first), (404, {"detail": "no such thing"})) + with pytest.raises(NotFoundError): + list(api.iterate(api.sub_users())) + + +class TestTheKeyOnlyEverGoesToOneHost: + """Where an API key is allowed to travel, and what pinned the opposite. + + Until 2026-09-08 the case above this class read + ``test_an_absolute_next_url_is_used_as_given``, answered ``next`` with + ``https://elsewhere.example.test/p2``, and asserted that the request went + there. It was a real property of the paging loop, written deliberately, and + it was pinning the defect: ``_request`` attaches + ``Authorization: x-api-key `` to whatever url it is handed, so "used as + given" is "the credential goes wherever the response says". A test can be + correct about behaviour and wrong about whether that behaviour should exist. + + It was not caught by writing more tests of the same kind. It came from an + external review on 2026-09-08 and was reproduced through this same transport + seam the same day. + """ + + def test_a_next_on_another_host_is_refused_before_the_key_is_sent(self): + api, fake = client( + (200, {"next": "https://evil.example/api/v2/base/x", "data": [1]}), + (200, {"next": None, "data": [2]}), + ) + with pytest.raises(ApiError, match="refusing to send the API key"): + list(api.iterate(api.domain_statistics("acct"))) + assert len(fake.calls) == 1 + + def test_the_measured_leak_was_the_key_in_a_header_and_not_just_a_request(self): + # What made this worth fixing over merely noting: reproduced 2026-09-08, + # the second request carried `Authorization: x-api-key SECRET-KEY` to a + # host nobody configured. The assertion is on the header and not on the + # url, because a request to a stranger with no credential in it is a + # different and much smaller problem. + sent = [] + + def transport(method, url, headers, body): + sent.append((url, headers.get("Authorization"))) + if len(sent) == 1: + return 200, json.dumps( + {"next": "https://evil.example/p2", "data": [1]} + ).encode() + return 200, json.dumps({"next": None, "data": [2]}).encode() + + api = Client(api_key="SECRET-KEY", transport=transport) + with pytest.raises(ApiError): + list(api.iterate(api.domain_statistics("acct"))) + assert not any("evil.example" in url for url, _ in sent) + assert all("SECRET-KEY" in auth for _, auth in sent) + + def test_a_downgrade_to_http_on_the_right_host_is_refused_too(self): + # The same leak on a wire rather than to a stranger. The key travels in + # a header and a header is plaintext, so scheme is part of the rule and + # not decoration on it. + host = DEFAULT_BASE_URL.split("://", 1)[1] + api, _ = client((200, {"next": f"http://{host}/p2", "data": [1]})) + with pytest.raises(ApiError, match="refusing to send the API key"): + list(api.iterate(api.domain_statistics("acct"))) + + def test_a_lookalike_host_is_not_the_same_origin(self): + for url in [ + "https://dashboard.nodemaven.com.evil.example/p2", + "https://evil.example/dashboard.nodemaven.com/p2", + # Userinfo: everything before the `@` is a login, so this one goes + # to `evil.example` while reading as if it went to the dashboard. + "https://dashboard.nodemaven.com@evil.example/p2", + ]: + api, _ = client((200, {"next": url, "data": [1]})) + with pytest.raises(ApiError, match="refusing to send the API key"): + list(api.iterate(api.domain_statistics("acct"))) + + def test_the_default_port_written_out_is_the_same_origin(self): + # The control on the rule, and the direction it is expensive to be wrong + # in: `https://host` and `https://host:443` are one origin, a server + # building paging links from its own absolute URI may emit the explicit + # form behind a proxy, and this check has never run against the live API. + # Comparing `netloc` as text would refuse a legitimate page. + host = DEFAULT_BASE_URL.split("://", 1)[1] + api, fake = client( + (200, {"next": f"https://{host}:443/p2", "data": [1]}), + (200, {"next": None, "data": [2]}), + ) + assert list(api.iterate(api.domain_statistics("acct"))) == [1, 2] + assert fake.calls[-1]["url"] == f"https://{host}:443/p2" + + def test_a_next_with_a_port_that_is_not_a_number_is_refused_and_does_not_crash( + self, + ): + # `urlsplit(...).port` raises ValueError on a port that is not a number, + # so without the catch the paging loop would die of a ValueError inside + # the function added to make it safe - past a module that maps every + # failure onto its own exception type. + api, _ = client((200, {"next": "https://host:notaport/p2", "data": [1]})) + with pytest.raises(ApiError, match="refusing to send the API key"): + list(api.iterate(api.domain_statistics("acct"))) + + def test_a_custom_base_url_moves_the_boundary_with_it(self): + # The rule is "the host this client was pointed at", not a constant. A + # self-hosted or staging dashboard has to page, and hard-coding the + # production host would break it while looking like security. + api, fake = client( + (200, {"next": "https://staging.example.test/p2", "data": [1]}), + (200, {"next": None, "data": [2]}), + base_url="https://staging.example.test", + ) + assert list(api.iterate(api.domain_statistics("acct"))) == [1, 2] + assert fake.calls[-1]["url"] == "https://staging.example.test/p2" + + +class TestValidateAgainstTheLiveCatalogue: + def test_a_country_in_the_catalogue_produces_no_complaints(self): + api, _ = client( + (200, {"next": None, "results": [{"code": "US"}]}), + (200, {"next": None, "results": []}), + ) + proxy = Proxy(login="acct", password="pw", country="us") + assert api.validate(proxy) == [] + + def test_a_country_that_is_not_there_is_named_along_with_the_gateways_answer(self): + # This is the gap the empty `values` table leaves open: the SDK refuses + # a parameter *name* it does not know and passes any *value* through, so + # `country="zz"` builds a username and earns a 407 that does not say + # which parameter was wrong - and reads as a credentials problem, which + # is the whole reason naming the country here is worth the network call. + api, _ = client( + (200, {"next": None, "results": [{"code": "us"}]}), + (200, {"next": None, "results": []}), + ) + proxy = Proxy(login="acct", password="pw", country="zz") + problems = api.validate(proxy) + assert len(problems) == 1 + assert "407" in problems[0] + assert "zz" in problems[0] + + def test_it_is_a_client_method_and_not_a_check_inside_proxy(self): + # A refusal that ships in a release can be wrong forever, and the + # catalogue moves. Asking the live catalogue cannot go stale, and it + # costs a network call - so it has to be the caller's decision. + assert not hasattr(Proxy(login="a", password="b"), "validate") + + def test_no_country_asks_the_catalogue_nothing(self): + api, fake = client((200, {"results": []})) + assert api.validate(Proxy(login="acct", password="pw")) == [] + assert fake.calls == [] + + def test_country_any_is_not_a_place_and_is_skipped(self): + api, _ = client( + (200, {"next": None, "results": [{"code": "us"}]}), + (200, {"next": None, "results": []}), + ) + proxy = Proxy(login="acct", password="pw", country="any") + assert api.validate(proxy) == [] + + def test_the_connection_type_follows_the_proxys_own_type_parameter(self): + # `type` selects the network rather than labelling it - measured + # 2026-08-26, `type=mobile` draws mobile ASNs and `residential` does + # not - so validating a mobile proxy against the residential catalogue + # would refuse a country the mobile pool has. + api, fake = client( + (200, {"next": None, "results": [{"code": "us"}]}), + (200, {"next": None, "results": []}), + ) + proxy = Proxy(login="acct", password="pw", country="us", type="mobile") + api.validate(proxy) + assert "connection_type=mobile" in fake.calls[0]["url"] + + def test_an_unreadable_catalogue_is_a_complaint_and_not_a_pass(self): + # The dangerous outcome is a validator that returns [] because it + # understood nothing. It says so instead, and says whose bug it is. + api, _ = client( + (200, {"next": None, "results": ["not an object"]}), + (200, {"next": None, "results": []}), + ) + proxy = Proxy(login="acct", password="pw", country="us") + problems = api.validate(proxy) + assert len(problems) == 1 + assert "do not treat it as a pass" in problems[0] + + +class TestTheTransportSeam: + def test_a_custom_transport_replaces_the_socket_entirely(self): + # The seam is why this file needs no network. It is also the answer to + # "why no async": a caller who needs one puts it here. + seen = [] + + def transport(method, url, headers, body): + seen.append(url) + return 200, b'{"ok": true}' + + assert Client(api_key="k", transport=transport).me() == {"ok": True} + assert len(seen) == 1 + + def test_the_default_transport_is_the_standard_library(self): + # Zero required dependencies is a property worth a test, because the day + # somebody imports `requests` here it will still pass every other case. + import nodemaven.api as module + + assert module._urllib_transport.__module__ == "nodemaven.api" + source = module.__file__ + with open(source, encoding="utf-8") as handle: + text = handle.read() + assert "import requests" not in text + assert "import httpx" not in text + + def test_the_default_transport_disables_environment_proxies(self): + # `ProxyHandler({})` is load-bearing. Left out, urlopen reads + # http_proxy/https_proxy from the environment - set on exactly the + # machines that use proxies - and an API call goes through a proxy + # nobody asked for. + import nodemaven.api as module + + with open(module.__file__, encoding="utf-8") as handle: + text = handle.read() + assert "ProxyHandler({})" in text + + def test_a_2xx_with_an_empty_body_is_an_empty_dict(self): + # 204 and an empty 200 are both real answers to a DELETE. An empty dict + # rather than None, so a caller can index the result of every method + # without branching on which one they called. + api, _ = client((204, b"")) + assert api.delete_whitelist_ip(1) == {} diff --git a/tests/test_check.py b/tests/test_check.py new file mode 100644 index 0000000..5848dff --- /dev/null +++ b/tests/test_check.py @@ -0,0 +1,804 @@ +"""``check()`` against a socket we control, on loopback only. + +The one module in this package that opens a socket is the one hardest to test +honestly. These cases run a real TCP listener on ``127.0.0.1`` that speaks the +CONNECT half of the exchange and then hangs up, so every assertion below is +about bytes that actually crossed a socket - and no live host is touched, no +traffic is spent, and nothing routes through whatever tunnel the machine happens +to be behind. + +The fake gateway is scripted rather than clever: each test says exactly what +status line and headers to answer with. That is what lets a 407 and a +``Connection established`` carrying ``X-Proxy-Exit-IP`` be tested at all - the +real gateway answers what it feels like answering, and two of its documented +reactions cannot be provoked on demand. +""" + +from __future__ import annotations + +import socket +import threading + +import pytest + +from nodemaven import Check, CheckError, Proxy +from nodemaven.check import connect + +REACTIONS = { + "200": "the tunnel opened; a 200 does not mean every parameter was applied.", + "407": "usually NOT your credentials, despite what the status says.", +} + + +class FakeGateway: + """A one-shot CONNECT responder on loopback. + + Records the request head it was sent, so a test can assert on what the + client emitted as well as on what it parsed. ``answer=None`` means accept + the connection and close it without replying, which is a real failure mode + and one of the two that must raise rather than return. + """ + + def __init__(self, answer, *, answer_bytes=None): + self.answer = answer + self.answer_bytes = answer_bytes + self.received = b"" + self._sock = socket.socket() + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind(("127.0.0.1", 0)) + self._sock.listen(1) + self.address = "%s:%d" % self._sock.getsockname() + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self): + try: + conn, _ = self._sock.accept() + except OSError: + return + with conn: + conn.settimeout(5.0) + try: + while b"\r\n\r\n" not in self.received: + chunk = conn.recv(4096) + if not chunk: + break + self.received += chunk + except OSError: + return + payload = self.answer_bytes + if payload is None and self.answer is not None: + payload = self.answer.encode("latin-1") + if payload is None: + return + try: + conn.sendall(payload) + except OSError: + # The bounded-read test deliberately answers with more than + # `check` will read, so the client closes mid-send. That is the + # behaviour under test, not a failure of the fake. + return + + def close(self): + self._sock.close() + self._thread.join(timeout=5.0) + + +@pytest.fixture +def gateway(request): + made = [] + + def make(answer=None, *, answer_bytes=None): + server = FakeGateway(answer, answer_bytes=answer_bytes) + made.append(server) + return server + + yield make + for server in made: + server.close() + + +ESTABLISHED = ( + "HTTP/1.1 200 Connection established\r\n" + "X-Proxy-Exit-IP: 203.0.113.7\r\n" + "\r\n" +) + + +class TestWhatTheGatewaySaid: + def test_a_200_with_the_exit_header_yields_the_exit_address(self, gateway): + server = gateway(ESTABLISHED) + result = connect( + server.address, "acct-country-us", "pw", + exit_ip_header="X-Proxy-Exit-IP", reactions=REACTIONS, + ) + assert result.ok + assert result.status == 200 + assert result.exit_ip == "203.0.113.7" + # One CONNECT and nothing else: the exit address arrived on the reply + # itself, so it cost no target traffic. That is the whole reason this + # module exists rather than a GET through the tunnel. + assert result.elapsed >= 0.0 + + def test_the_reason_phrase_is_kept_verbatim(self, gateway): + # Measured 2026-08-13: on the shipped gateway a 200 carrying the exit + # header arrives as `Connection established`, while the ones arriving as + # `OK` or `Connection Established` do not carry it. The phrase labels + # which back end answered, so normalising it - even just its case - + # destroys the only key any per-implementation figure can be split on. + server = gateway("HTTP/1.1 200 Connection Established\r\n\r\n") + result = connect(server.address, "acct", "pw") + assert result.reason == "Connection Established" + assert result.exit_ip is None + + def test_a_missing_exit_header_is_normal_and_not_an_error(self, gateway): + server = gateway("HTTP/1.1 200 OK\r\n\r\n") + result = connect( + server.address, "acct", "pw", exit_ip_header="X-Proxy-Exit-IP" + ) + assert result.ok + assert result.exit_ip is None + + def test_headers_are_lowercased_and_kept(self, gateway): + server = gateway( + "HTTP/1.1 200 Connection established\r\n" + "X-Proxy-Exit-IP: 203.0.113.7\r\n" + "Via: 1.1 something\r\n" + "\r\n" + ) + result = connect(server.address, "acct", "pw") + assert result.headers["x-proxy-exit-ip"] == "203.0.113.7" + assert result.headers["via"] == "1.1 something" + + def test_a_header_name_is_lowercased_ascii_only(self, gateway): + # Not a bug that was ever reachable in production - the shipped + # gateway's header is `X-Proxy-Exit-IP` and every byte of it is ASCII - + # but a portability trap, found 2026-09-07 while writing the Rust port, + # and of the kind that surfaces as one SDK finding a header the other + # cannot. + # + # The head is decoded latin-1 on purpose, so byte 0xC0 arrives as + # 'A-grave'. `str.lower()` folds that to 'a-grave' and Rust's + # `to_ascii_lowercase` leaves it alone, so two SDKs would key one + # response two ways. RFC 9110 makes a field name a token and a token is + # ASCII, so ASCII-only is the correct rule as well as the portable one - + # the same decision, for the same reason, as the ASCII-only fold in + # `Provider.normalized`. + server = gateway( + None, answer_bytes=b"HTTP/1.1 200 OK\r\n\xc0-Vendor: yes\r\n\r\n" + ) + result = connect(server.address, "acct", "pw") + assert "\xc0-vendor" in result.headers + assert "\xe0-vendor" not in result.headers + + +class TestARefusalIsAResultAndNotAnException: + def test_a_407_comes_back_as_a_value(self, gateway): + # Raising here would push the status - the thing the caller came for - + # into a traceback. `requests` does exactly that, and the code survives + # only as text inside a nested exception. + server = gateway("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + result = connect(server.address, "acct", "pw", reactions=REACTIONS) + assert isinstance(result, Check) + assert result.status == 407 + assert not result.ok + + def test_the_meaning_comes_from_the_provider_and_not_from_the_status( + self, gateway + ): + # The point of the table: two of the gateway's seven documented + # reactions are 407, and neither is a credentials problem. A caller + # reading the status alone goes and checks a password that is correct. + server = gateway("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + result = connect(server.address, "acct", "pw", reactions=REACTIONS) + assert result.meaning is not None + assert "NOT your credentials" in result.meaning + assert result.meaning in str(result) + + def test_a_status_with_no_entry_has_no_meaning_rather_than_a_wrong_one( + self, gateway + ): + server = gateway("HTTP/1.1 502 Bad Gateway\r\n\r\n") + result = connect(server.address, "acct", "pw", reactions=REACTIONS) + assert result.status == 502 + assert result.meaning is None + + def test_a_200s_meaning_is_not_shown_because_it_is_not_a_diagnosis( + self, gateway + ): + server = gateway(ESTABLISHED) + result = connect(server.address, "acct", "pw", reactions=REACTIONS) + assert result.meaning is not None + assert result.meaning not in str(result) + + +class TestWhenNothingCameBack: + def test_an_immediate_close_raises_and_says_it_is_worth_reporting( + self, gateway + ): + server = gateway(None) + with pytest.raises(CheckError, match="without answering"): + connect(server.address, "acct", "pw") + + def test_something_that_is_not_a_status_line_raises(self, gateway): + # A captive portal, or something other than a proxy on that port. + server = gateway("You must sign in\r\n\r\n") + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + def test_a_non_numeric_status_raises_rather_than_crashing_on_int( + self, gateway + ): + server = gateway("HTTP/1.1 OK Fine\r\n\r\n") + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + @pytest.mark.parametrize("byte", [b"\xb9", b"\xb2", b"\xb3"]) + def test_a_latin1_superscript_status_raises_and_does_not_crash_on_int( + self, gateway, byte + ): + # A real defect, found 2026-09-07 while porting this module to Rust and + # fixed the same day. The gate here was `parts[1].isdigit()`, which is + # True for these three bytes and for which `int()` raises: `'\xb2'` is + # SUPERSCRIPT TWO, `'\xb2'.isdigit()` is True, `int('\xb2')` is a + # `ValueError`. So an uncaught `ValueError` left a module whose docstring + # promises only `CheckError` comes out of it. + # + # And it was reachable only because of a deliberate decision two + # functions away: the head is decoded latin-1 on purpose, so that any + # byte a proxy is entitled to send is accepted rather than raising. The + # widening of the input alphabet is what made the narrow check unsound. + # Sent as bytes, not as a str, because the payload is not ASCII. + server = gateway( + None, answer_bytes=b"HTTP/1.1 " + byte + b" something\r\n\r\n" + ) + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + @pytest.mark.parametrize("status", ["20", "2000", "99999"]) + def test_a_status_that_is_not_three_digits_is_refused(self, gateway, status): + # Three digits and not "one or more" because RFC 9110 calls the status + # code a three-digit integer, and because it is the rule a Rust `u16` + # holds without diverging from Python's arbitrary-precision `int` - + # `99999` parsed fine here and overflows there, which is a port that + # disagrees with its original on a real input. + server = gateway("HTTP/1.1 %s something\r\n\r\n" % status) + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + def test_a_status_line_with_no_http_version_is_refused(self, gateway): + # The review's case, reproduced on loopback 2026-09-08: this answered + # `status=200 ok=True reason='OK'`. `_parse_head` split the line and + # looked only at the second token, so the first was whatever the peer + # felt like sending. + server = gateway("garbage 200 OK\r\n\r\n") + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + def test_a_faked_status_line_cannot_supply_an_exit_address(self, gateway): + # Worse than the status alone and not in the review: whatever is + # listening also gets to name the exit, and the caller reports that + # address as the one its traffic left from. `ok` is what a caller + # branches on and `exit_ip` is what it then prints. + server = gateway("garbage 200 OK\r\nX-Proxy-Exit-IP: 1.2.3.4\r\n\r\n") + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw", exit_ip_header="X-Proxy-Exit-IP") + + @pytest.mark.parametrize( + "version", ["HTTP/1", "HTTP/11", "http/1.1", "HTTP/1.1x", "HTTPS/1.1", ""] + ) + def test_a_version_token_that_is_not_the_grammar_is_refused( + self, gateway, version + ): + # RFC 9112 section 2.3: `HTTP-name "/" DIGIT "." DIGIT`, with the name + # case-sensitive. Eight characters exactly - a CONNECT answered over a + # socket this module opened itself is HTTP/1.x by construction, so there + # is no version negotiation here to be liberal about. `http/1.1` is in + # the list because being liberal about case is the single most likely + # way a port diverges from this one. + server = gateway(version + " 200 OK\r\n\r\n") + with pytest.raises(CheckError, match="not an HTTP status line"): + connect(server.address, "acct", "pw") + + @pytest.mark.parametrize("version", ["HTTP/1.1", "HTTP/1.0", "HTTP/0.9"]) + def test_the_versions_a_proxy_may_answer_with_are_accepted( + self, gateway, version + ): + # The control the rule above needs, and it is what stops the check from + # being tightened into something that refuses a real gateway. Without + # it, "refuse anything that is not HTTP/1.1" passes every test above + # and breaks against a proxy answering 1.0. + server = gateway(version + " 200 OK\r\n\r\n") + assert connect(server.address, "acct", "pw").status == 200 + + def test_a_refused_connection_says_the_gateway_was_never_reached(self): + # Bound and immediately closed, so the port is free and nothing is + # listening. Loopback refuses rather than hanging, which is why this is + # a test and not a 15 s timeout. + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + address = "%s:%d" % sock.getsockname() + sock.close() + with pytest.raises(CheckError, match="never reached"): + connect(address, "acct", "pw", timeout=2.0) + + def test_an_address_without_a_port_is_refused_before_anything_is_sent(self): + with pytest.raises(CheckError, match="Nothing was sent"): + connect("gate.example.com", "acct", "pw") + + def test_a_non_numeric_port_is_refused_before_anything_is_sent(self): + with pytest.raises(CheckError, match="host:port"): + connect("gate.example.com:eight", "acct", "pw") + + @pytest.mark.parametrize("port", ["\xb9", "\xb2", "\xb3"]) + def test_a_latin1_superscript_port_raises_checkerror_and_not_valueerror( + self, port + ): + # The same `isdigit()` defect as the status-line case above, in the + # caller-facing half of the module, and it survived the first fix + # because that fix corrected the predicate where the bug was noticed + # rather than everywhere it was used. Measured 2026-09-07: + # `connect("127.0.0.1:\xb2", "acct", "pw")` raised + # `ValueError: invalid literal for int() with base 10: '2'` from + # check.py, past a docstring promising only `CheckError` leaves here. + # + # `pytest.raises(CheckError)` is the whole assertion: `ValueError` is + # not a `CheckError`, so the old code fails this test by raising the + # wrong class rather than by raising nothing. + with pytest.raises(CheckError, match="host:port"): + connect("127.0.0.1:" + port, "acct", "pw", timeout=2.0) + + def test_a_port_too_large_for_a_socket_raises_checkerror_and_not_overflow( + self + ): + # Measured 2026-09-07: this raised `OverflowError: Python int too large + # to convert to C long`, from inside `socket.create_connection`. That + # one does not derive from `OSError` - it is an `ArithmeticError` - so + # it walked straight past the handler whose entire job is turning + # everything the socket layer raises into a `CheckError`. `isdigit()` + # was True, `int()` succeeded, and the failure happened one layer + # further down than the two above it. + with pytest.raises(CheckError, match="host:port"): + connect("127.0.0.1:99999999999999999999", "acct", "pw", timeout=2.0) + + @pytest.mark.parametrize("port", ["0", "65536", "70000"]) + def test_a_port_outside_1_to_65535_is_refused_before_anything_is_sent( + self, port + ): + # These three already produced a `CheckError` before the fix, so this + # is not a regression pin - it is pinning *where* the refusal happens. + # Measured 2026-09-07, all on this host: `:0` reached the socket layer + # and came back `[WinError 10049]`, and `:70000` came back `timed out` + # after the full timeout. Both are the right exception class carrying + # the wrong explanation, and the second spends 15 s by default to say + # it. Refusing in the parse means the message names the actual mistake + # and costs nothing - and it removes a platform-specific errno from a + # cross-language contract, since Linux answers `:0` with ECONNREFUSED. + with pytest.raises(CheckError, match="Nothing was sent"): + connect("127.0.0.1:" + port, "acct", "pw", timeout=2.0) + + +class TestTheCredentialNeverAppears: + def test_no_message_on_any_failure_path_carries_the_password(self, gateway): + # `connect` holds the request bytes - which contain the base64 + # credential - in scope inside its own except block. An f-string that + # happened to include them would put a working credential into every + # traceback, every CI log and every pasted bug report. + password = "s3cr3t-do-not-print" + cases = [ + gateway(None).address, + gateway("portal\r\n\r\n").address, + ] + for address in cases: + with pytest.raises(CheckError) as caught: + connect(address, "acct", password) + assert password not in str(caught.value) + assert "Basic" not in str(caught.value) + + def test_the_result_carries_no_credential_anywhere(self, gateway): + server = gateway(ESTABLISHED) + result = connect(server.address, "acct", "s3cr3t-do-not-print", reactions=REACTIONS) + assert "s3cr3t" not in str(result) + assert "s3cr3t" not in repr(result) + assert "s3cr3t" not in str(result.headers) + + +class TestWhatIsActuallySentOnTheWire: + def test_the_request_is_a_conditioned_connect(self, gateway): + server = gateway(ESTABLISHED) + connect(server.address, "acct-country-us", "pw", target="example.test:443") + server.close() + head = server.received.decode("latin-1") + assert head.startswith("CONNECT example.test:443 HTTP/1.1\r\n") + assert "Host: example.test:443\r\n" in head + assert "Proxy-Authorization: Basic YWNjdC1jb3VudHJ5LXVzOnB3\r\n" in head + assert head.endswith("\r\n\r\n") + + def test_the_target_is_a_parameter_because_there_is_no_null_connect( + self, gateway + ): + # The gateway has to be asked for some target, and whatever is named + # here sees a TCP connection from the exit address. That makes it the + # caller's business, not a constant buried in the module. + server = gateway(ESTABLISHED) + connect(server.address, "acct", "pw", target="10.254.254.254:443") + server.close() + assert b"CONNECT 10.254.254.254:443" in server.received + + def test_a_target_carrying_crlf_is_refused_before_anything_is_sent(self): + # Measured 2026-09-08: this reached the socket. `target` is interpolated + # into `f"CONNECT {target} HTTP/1.1"`, so the gateway received a request + # line of `CONNECT example.com:443` - no version token at all - and + # `X-Injected: yes HTTP/1.1` as a header of our own request. Header + # injection is the obvious half; the request line losing its version to + # a caller's string is the half that is easy to miss. + with pytest.raises(CheckError, match="Nothing was sent"): + connect( + "127.0.0.1:1", "acct", "pw", + target="example.com:443\r\nX-Injected: yes", + timeout=2.0, + ) + + @pytest.mark.parametrize( + "target", + ["", "a b:443", "a\nb:443", "a\tb:443", "a\x00b:443", "ex\xc3mple:443"], + ) + def test_a_target_that_is_not_one_token_of_visible_ascii_is_refused( + self, target + ): + # Deliberately wider than the characters that hurt. The request line is + # built by interpolation and the set of bytes that change its shape is + # not a thing to enumerate from memory - a space alone splits the line + # into a different request. The address is a port nothing listens on, + # so a test that fails does so by hanging on connect rather than by + # quietly passing. + with pytest.raises(CheckError, match="Nothing was sent"): + connect("127.0.0.1:1", "acct", "pw", target=target, timeout=2.0) + + def test_the_credential_is_not_the_injection_surface_and_is_left_alone( + self, gateway + ): + # The control on the rule above: a login containing CRLF cannot inject + # anything, because it goes through base64 whose output alphabet is + # A-Za-z0-9+/= and holds neither CR nor LF. Validating it would be + # cargo-culting the fix onto the field that was already safe - and the + # actual shape of the defect was the reverse of that. `Provider` + # refuses CRLF in `country`, which is base64-encoded, while `target`, + # which lands in the request line in the clear, was unchecked. + server = gateway(ESTABLISHED) + result = connect(server.address, "acct\r\nX-Injected: yes", "pw") + server.close() + assert result.ok + head = server.received.decode("latin-1") + assert "X-Injected" not in head + assert head.count("\r\n\r\n") == 1 + + +class TestTheHeadIsBounded: + def test_a_long_header_block_does_not_read_forever(self, gateway): + # A gateway that answers with a stream would otherwise be a memory + # exhaustion bug. Bounded at 16 KiB; a real response head is a few + # hundred bytes. + # + # This test asserted `result.status == 200` until 2026-09-08 and that + # assertion was pinning the defect next door. The bound was enforced by + # `break`, so the truncated buffer went to `_parse_head`, which found a + # valid status line at the top of it and reported a complete answer + # assembled from however many headers happened to fit. The bound is the + # feature; parsing what the bound cut off is not. + padding = "X-Pad: " + ("a" * 200) + "\r\n" + server = gateway( + "HTTP/1.1 200 Connection established\r\n" + padding * 200 + "\r\n" + ) + with pytest.raises(CheckError, match="no end to the response head"): + connect(server.address, "acct", "pw") + + +class TestAnUnfinishedHeadIsNotAnAnswer: + """Truncation reported as a 200 was the second half of the CONNECT review. + + Found 2026-09-08 in an external review of both SDKs and reproduced the same + day on a loopback socket. All three cases below returned a ``Check`` rather + than raising, and the first two returned one whose ``ok`` was True. + """ + + def test_a_head_cut_off_mid_block_is_refused_rather_than_reported_as_ok( + self, gateway + ): + # Measured 2026-09-08: this exact payload gave `status=200 ok=True + # headers={}`. Both parts are wrong and the second is the quieter one - + # the exit header is *in* the bytes and `_parse_head` drops it, because + # a header line only enters the table when the blank line that ends the + # block proves it arrived whole. + server = gateway( + "HTTP/1.1 200 Connection established\r\nX-Proxy-Exit-IP: 1.2.3.4" + ) + with pytest.raises(CheckError, match="part-way through"): + connect(server.address, "acct", "pw", exit_ip_header="X-Proxy-Exit-IP") + + def test_a_status_line_with_no_blank_line_after_it_is_refused(self, gateway): + # The minimal case: everything a caller needs is present and the head + # still never ended, so there is no way to know whether a header was on + # its way. A gateway that means to answer 200 sends the blank line. + server = gateway("HTTP/1.1 200 Connection established\r\n") + with pytest.raises(CheckError, match="part-way through"): + connect(server.address, "acct", "pw") + + def test_the_message_separates_truncation_from_an_immediate_close( + self, gateway + ): + # Two different failures with two different causes: nothing at all is a + # documented reaction of the shipped gateway - an empty parameter value + # hangs and then closes - while a head that starts and stops is not, and + # is worth reporting. One message for both would lose that. + cut = gateway("HTTP/1.1 200 OK\r\nX-Pad: a") + with pytest.raises(CheckError) as truncated: + connect(cut.address, "acct", "pw") + silent = gateway(None) + with pytest.raises(CheckError) as nothing: + connect(silent.address, "acct", "pw") + assert "part-way through" in str(truncated.value) + assert "without answering" in str(nothing.value) + + +#: Exactly what the shipped gateway answered on 2026-09-08, byte for byte, +#: read off `gate.nodemaven.com:8080` with a raw dump rather than off formatted +#: output. The success path frames its head with CRLF and every refusal frames +#: it with bare LF, so these are three payloads and one framing question. +#: +#: The exit address is the only edit: TEST-NET-3 per RFC 5737 in place of the +#: address the gateway returned. Everything else including the header *names* is +#: reproduced, because this back end sends `X-Exit-IP` and not the +#: `X-Proxy-Exit-IP` a caller may be looking for. +LIVE_407 = ( + b"HTTP/1.1 407 Proxy Authentication Required\n" + b'Proxy-Authenticate: Basic realm="Invalid credentials"\n' + b"Connection: close\n" + b"\n" +) +LIVE_406 = b"HTTP/1.1 406 Not Acceptable\nConnection: close\n\n" +LIVE_200 = ( + b"HTTP/1.1 200 OK\r\n" + b"X-Exit-IP: 203.0.113.104\r\n" + b"X-Exit-Country: US\r\n" + b"X-Exit-Timezone: America/Los_Angeles\r\n" + b"X-Exit-ASN: 6167\r\n" + b"\r\n" +) + + +class TestTheRealGatewaysBytes: + """The three replies the shipped gateway actually sends. + + RFC 9112 section 2.2 lets a recipient treat a bare LF as a line terminator + and ignore any preceding CR. That is permission rather than obligation + everywhere except here: this gateway frames every refusal with LF and + carries `Connection: close`, so a CRLF-only reader sees the peer hang up + with no blank line, calls it a truncated head, and can report no refusal + code at all - which is the one thing this module exists to do. + """ + + def test_the_407_that_a_bad_filter_value_produces(self, gateway): + server = gateway(None, answer_bytes=LIVE_407) + result = connect( + server.address, "acct", "pw", reactions=REACTIONS + ) + assert result.status == 407 + assert result.reason == "Proxy Authentication Required" + assert not result.ok + assert result.headers["connection"] == "close" + assert result.headers["proxy-authenticate"] == ( + 'Basic realm="Invalid credentials"' + ) + # The whole point of carrying the table: the status says credentials and + # the cause is a value the gateway would not take. + assert result.meaning == REACTIONS["407"] + + def test_the_406_that_a_bad_region_produces(self, gateway): + server = gateway(None, answer_bytes=LIVE_406) + result = connect(server.address, "acct", "pw") + assert result.status == 406 + assert result.reason == "Not Acceptable" + assert result.headers == {"connection": "close"} + + def test_the_200_is_framed_the_other_way_and_still_parses(self, gateway): + server = gateway(None, answer_bytes=LIVE_200) + result = connect(server.address, "acct", "pw") + assert result.ok + assert result.reason == "OK" + assert result.headers["x-exit-ip"] == "203.0.113.104" + assert result.headers["x-exit-asn"] == "6167" + + def test_the_header_this_back_end_omits_reads_as_absent(self, gateway): + # A caller asking for a name this reply does not carry gets None, not a + # wrong address and not an error. The 200 above proves the address was + # on the wire under another name; resolving that is a provider + # definition question and not this function's. + server = gateway(None, answer_bytes=LIVE_200) + result = connect( + server.address, "acct", "pw", exit_ip_header="X-Proxy-Exit-IP" + ) + assert result.exit_ip is None + + +class TestABlankLineHasFourSpellings: + @pytest.mark.parametrize( + "terminator", [b"\r\n\r\n", b"\n\n", b"\r\n\n", b"\n\r\n"] + ) + def test_every_spelling_ends_the_head(self, gateway, terminator): + # Once a bare LF is a terminator, the blank line is any of these four, + # including the two mixed ones - which are what a gateway assembling a + # reply from a template and a variable body produces. + head = b"HTTP/1.1 200 OK\r\nX-Exit-ASN: 6167" + server = gateway(None, answer_bytes=head + terminator) + result = connect(server.address, "acct", "pw") + assert result.status == 200 + assert result.headers == {"x-exit-asn": "6167"} + + def test_body_bytes_in_the_same_segment_are_not_parsed_as_headers( + self, gateway + ): + # The head ends at the blank line and the read stops there. Anything + # after it belongs to the tunnel, and a header table assembled from + # tunnel bytes would be a security bug rather than a parsing one. + server = gateway( + None, + answer_bytes=LIVE_200 + b"\x16\x03\x01\x00\x01X-Exit-IP: 10.0.0.1\r\n", + ) + result = connect(server.address, "acct", "pw") + assert result.headers["x-exit-ip"] == "203.0.113.104" + assert len(result.headers) == 4 + + def test_a_cr_that_is_not_before_the_lf_stays_in_the_value(self, gateway): + # Only a CR immediately before the LF is a line terminator. One in the + # middle of a value is part of the value, and repairing it would be + # this function inventing a reply the gateway did not send. + server = gateway(None, answer_bytes=b"HTTP/1.1 200 OK\nX-Pad: a\rb\n\n") + result = connect(server.address, "acct", "pw") + assert result.headers == {"x-pad": "a\rb"} + + +class TestLfFramingDoesNotUndoTheTruncationGuard: + """The fix widens what counts as a complete head and nothing else. + + Both halves have to hold at once: an LF-framed refusal is an answer, and an + LF-framed head that stops without its blank line is still not one. A fix + that got only the first half would report every truncation as a 200 again. + """ + + def test_an_lf_framed_head_with_no_blank_line_is_still_refused( + self, gateway + ): + server = gateway( + None, + answer_bytes=b"HTTP/1.1 407 Proxy Authentication Required\n" + b"Connection: close\n", + ) + with pytest.raises(CheckError, match="part-way through"): + connect(server.address, "acct", "pw") + + def test_a_bare_lf_status_line_alone_is_still_refused(self, gateway): + server = gateway(None, answer_bytes=b"HTTP/1.1 200 OK\n") + with pytest.raises(CheckError, match="part-way through"): + connect(server.address, "acct", "pw") + + def test_a_single_lf_is_not_a_blank_line(self, gateway): + # The one case that separates "ends a line" from "ends the head": a + # head whose last line ended and whose blank line never came. + server = gateway(None, answer_bytes=b"HTTP/1.1 406 Not Acceptable\n") + with pytest.raises(CheckError, match="part-way through"): + connect(server.address, "acct", "pw") + + +class TestProxyCheckWiresTheProviderIn: + def test_check_passes_the_providers_reaction_table(self, gateway): + # The bug this pins: the meaning depends on the status, and the status + # is not known until the connect returns. Passing a single resolved + # `reaction` from the call site would always be None, which would leave + # the whole table unread while every other test still passed. + server = gateway("HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + host, _, port = server.address.rpartition(":") + proxy = Proxy( + login="acct", password="pw", host=host, port=int(port), country="us" + ) + result = proxy.check(timeout=5.0) + assert result.status == 407 + assert result.meaning is not None + assert "NOT your credentials" in result.meaning + + def test_check_reads_the_exit_header_the_provider_declares(self, gateway): + server = gateway(ESTABLISHED) + host, _, port = server.address.rpartition(":") + proxy = Proxy(login="acct", password="pw", host=host, port=int(port)) + result = proxy.check(timeout=5.0) + assert result.exit_ip == "203.0.113.7" + + def test_check_sends_the_built_username_and_not_the_bare_login(self, gateway): + server = gateway(ESTABLISHED) + host, _, port = server.address.rpartition(":") + proxy = Proxy( + login="acct", password="pw", host=host, port=int(port), + country="us", filter="medium", + ) + proxy.check(timeout=5.0) + server.close() + import base64 + + head = server.received.decode("latin-1") + token = head.split("Basic ", 1)[1].split("\r\n", 1)[0] + assert base64.b64decode(token).decode() == "acct-country-us-filter-medium:pw" + + +class TestSessions: + def test_every_identity_is_distinct(self): + proxy = Proxy(login="acct", password="pw", country="us") + batch = proxy.sessions(25) + assert len({p.username for p in batch}) == 25 + + def test_the_ids_are_hex_and_that_is_load_bearing(self): + # Measured 2026-08-20: the gateway cuts a value at the separator, so an + # id containing one collapses every id sharing a prefix onto a single + # exit - silently, because the connection succeeds. `token_urlsafe` + # emits `-` and `_`, `uuid4` emits `-`, base64 emits `+` and `/`, and + # every one of those is a separator on some gateway. + proxy = Proxy(login="acct", password="pw") + for child in proxy.sessions(10, length=8): + session_id = child.params["sid"] + assert len(session_id) == 16 + assert all(c in "0123456789abcdef" for c in session_id) + + def test_the_parent_is_untouched(self): + proxy = Proxy(login="acct", password="pw", country="us", sid="seed") + proxy.sessions(3) + assert proxy.params["sid"] == "seed" + + def test_a_nonsense_count_is_refused(self): + proxy = Proxy(login="acct", password="pw") + with pytest.raises(Exception, match="no identities"): + proxy.sessions(0) + + def test_a_nonsense_length_is_refused(self): + proxy = Proxy(login="acct", password="pw") + with pytest.raises(Exception, match="length"): + proxy.sessions(2, length=0) + + def test_more_ids_than_exist_is_refused_rather_than_looped_forever(self): + # Measured 2026-09-08, from an external review: `sessions(257, + # length=1)` did not return in 4 s and `sessions(200, length=1)` built + # 200 immediately. Rejection sampling cannot produce more distinct + # values than the space holds, so the 257th draw is an unbounded loop + # holding the CPU - not a slow call, a call that never ends. + # + # This test would hang rather than fail against the old code, which is + # the reason it is written against `length=1`: the failure is cheap to + # provoke at 256 values and impossible to provoke at 2**48. + proxy = Proxy(login="acct", password="pw") + with pytest.raises(Exception, match="at least the whole space"): + proxy.sessions(257, length=1) + + def test_the_whole_space_is_refused_and_one_less_is_not(self): + # Where the boundary is put and why. `count.bit_length() > 8 * length` + # is exactly `count >= 2**bits`, which refuses 256 here and allows 255 - + # one comparison, no exponentiation, the same answer in a language whose + # integers overflow. Asking for the entire space would draw every value + # that exists and leave none for the next process, which is the failure + # the docstring's second paragraph is about. + # + # The message was "asks for more distinct ids than exist" until + # 2026-09-08, and at this exact boundary that sentence is false: 256 of + # them do exist, and the refusal is because taking all of them is a + # coupon-collector loop that leaves the space empty, not because they + # are missing. The guard was right and its explanation was not - caught + # by writing the same bound into the README and finding the two + # sentences could not both be true. + proxy = Proxy(login="acct", password="pw") + with pytest.raises(Exception, match="at least the whole space"): + proxy.sessions(256, length=1) + assert len({p.params["sid"] for p in proxy.sessions(255, length=1)}) == 255 + + def test_the_default_length_refuses_nothing_anybody_would_ask_for(self): + # The control on the guard: a bound that bites a real caller is a + # regression dressed as a fix. The default is 6 bytes, so the ceiling is + # 2**48 identities and no call anyone writes comes near it. + proxy = Proxy(login="acct", password="pw") + assert len(proxy.sessions(64)) == 64 diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 4c129be..2455c5f 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -9,7 +9,15 @@ import pytest -from nodemaven import ParamError, ProviderError, Proxy, available, load, load_file +from nodemaven import ( + CredentialsError, + ParamError, + ProviderError, + Proxy, + available, + load, + load_file, +) CREDS = {"login": "acct", "password": "pw", "host": "gate.example.com", "port": 8080} @@ -54,6 +62,57 @@ def test_a_separator_inside_a_value_is_refused(self): Proxy(country="us-east", **CREDS) +class TestThePort: + """One rule for what a port is, in the one place that already held it. + + Both cases below were measured 2026-09-08 from an external review. The + constructor did `int(raw_port)` and then tested the result for truthiness, + while `check._port_number` two modules away already had the rule written + out with the three defects that produced it recorded beside it. The fix is + the import, not a new predicate. + """ + + @pytest.mark.parametrize("port", ["abc", "8o80", "\xb2", "12.5", " 8080"]) + def test_a_port_that_is_not_a_number_raises_this_packages_own_error(self, port): + # Measured: `port='abc'` raised `ValueError: invalid literal for int() + # with base 10: 'abc'`. A caller catching this package's exceptions did + # not catch that, and the message names neither the parameter nor what a + # port is. `'\xb2'` is in the list for the same reason it is in + # `test_check.py`: `str.isdigit()` is True for it and `int()` refuses. + # `' 8080'` is there because `int()` accepts leading whitespace and a + # port with a space in it is a typo, not a port. + with pytest.raises(CredentialsError, match="not a TCP port"): + Proxy(**{**CREDS, "port": port}) + + @pytest.mark.parametrize("port", [0, "0", -1, 65536, 70000, "99999999999999"]) + def test_a_port_outside_1_to_65535_says_what_is_wrong_with_it(self, port): + # Measured: `port='0'` raised `CredentialsError` - the right class - with + # the message "no gateway address ... pass host= and port=", although + # the port *was* passed. A truthiness test standing in for a range check + # turns 0 into "missing", and the sentence sends the caller to look at + # the one thing they got right. + with pytest.raises(CredentialsError, match="not a TCP port"): + Proxy(**{**CREDS, "port": port}) + + def test_a_port_from_the_environment_is_held_to_the_same_rule(self, monkeypatch): + # The environment is where a port arrives as a string in real use, and + # it was the path where `int()` was most likely to be handed something + # that is not a number. + monkeypatch.setenv("NODEMAVEN_PORT", "not-a-port") + with pytest.raises(CredentialsError, match="not a TCP port"): + Proxy(login="acct", password="pw", host="gate.example.com") + + def test_the_ports_a_caller_actually_uses_still_work(self): + # The control. A guard that refuses a real port is worse than the defect + # it replaces, and `1` and `65535` are the two the range check itself is + # most likely to get wrong. The assertion reads `server` because there is + # no `port` property: the port is a private slot, and the address is the + # only public place its value appears. + for port in (1, 8080, 65535, "8080"): + proxy = Proxy(**{**CREDS, "port": port}) + assert proxy.server == f"gate.example.com:{int(port)}" + + class TestTheUrlIsSafeToHandToAClient: def test_credentials_are_percent_encoded(self): proxy = Proxy(country="us", **{**CREDS, "password": "pa/ss@1:2"}) @@ -213,3 +272,316 @@ def test_the_session_parameter_is_asked_for_rather_than_spelled(self): # Eleven call sites in the benchmark wrote "sid" directly. It is the name # this gateway happens to use, which is exactly why the literal survived. assert load("nodemaven").session_param == "sid" + + def test_type_is_known_and_selects_the_mobile_pool(self): + # Refusing this refused a product tier. It was missing from this list + # until 2026-09-07, so a customer paying for mobile proxies could not + # ask for them and the error they got said the gateway does not know + # the parameter - the exact failure this package exists to prevent, + # produced by the package. + # + # Probed from the VPS on 2026-08-26: a junk value is answered 407, + # which an unrecognised name cannot produce - the negative control + # `zzqqx` gives 200 and the positive control `filter` gives 407. Then + # verified functionally, 5 requests per arm with a fresh sid and + # `country=us`: `type=mobile` drew T-Mobile and Cellco ASNs where + # `type=residential` and the unset arm drew wireline carriers only. + assert "type" in load("nodemaven").known_params + assert Proxy(type="mobile", **CREDS).username == "acct-type-mobile" + + +class TestTheReadmeShowsRealOutput: + """The README quotes this message. This pins it, because a README showing a + message the code no longer produces is worse than one showing none. + + Added 2026-09-07 after the README drifted: it listed nine parameter names + here and the code produced ten, because `type` was added to the shipped + definition and the quoted block was not regenerated. The Rust port carried + this test from the start and it caught the same drift on its first run; + this package did not have it, which is why the drift got as far as being + committed to the source tree. + """ + + def test_the_unknown_parameter_message_is_quoted_verbatim(self): + with pytest.raises(ParamError) as caught: + Proxy(login="u", password="p", contry="us") + assert str(caught.value) == ( + "NodeMaven does not know the parameter 'contry': it is answered " + "with 200 and dropped, so the connection would succeed and your " + "setting would NOT be applied. Known: ['city', 'country', " + "'filter', 'ipv4', 'isp', 'region', 'sid', 'speed', 'ttl', 'type']" + ) + + +class TestValuesAreFoldedToTheFormTheGatewayEmits: + """The fold to the wire form, and the refusal that completes it. + + The evidence for the fold is in the provider TOML: a username this gateway + generated for a real account carries ``region-district_of_columbia``, and + the vendor's own client applies the same transformation. So these cases + assert a form the gateway has been seen to emit, not one that looked tidy. + + The refusal is the other half. A parameter nobody folds cannot carry + whitespace either, because there is no spelling of a space in a proxy + username that works - so between the two, no value with whitespace in it + can reach the wire by any path. + """ + + def test_it_reproduces_a_username_the_gateway_generated(self): + # The case this whole class exists for. The login is `acct` here and + # the real one is not written down anywhere in this repository; + # everything to the right of it is the generated string unchanged. + # Before the fold this same call produced + # `region-District of Columbia`, which is not a thing that can be sent. + proxy = Proxy( + country="us", + region="District of Columbia", + sid="bfd1c859433a4", + filter="medium", + **CREDS, + ) + assert proxy.username == ( + "acct-country-us-region-district_of_columbia" + "-sid-bfd1c859433a4-filter-medium" + ) + + def test_a_space_becomes_an_underscore(self): + assert Proxy(city="New York", **CREDS).username == "acct-city-new_york" + + def test_case_is_folded(self): + assert Proxy(country="US", **CREDS).username == "acct-country-us" + + def test_the_stored_value_is_the_folded_one(self): + # A proxy reporting `District of Columbia` while sending + # `district_of_columbia` would let two callers with the same visible + # configuration sit on different sticky sessions and see no reason why. + proxy = Proxy(city="New York", country="US", **CREDS) + assert proxy.params == {"city": "new_york", "country": "us"} + + def test_folding_a_folded_value_changes_nothing(self): + # Idempotence is what makes `replace` safe to chain. + once = Proxy(city="New York", **CREDS) + twice = once.replace(city=once.params["city"]) + assert once.username == twice.username + + def test_surrounding_whitespace_is_trimmed_rather_than_refused(self): + assert Proxy(country=" US ", **CREDS).username == "acct-country-us" + + def test_a_value_of_only_whitespace_is_refused_as_empty(self): + # It trims to empty, and the empty refusal is the more useful of the + # two messages: the gateway does not answer an empty value at all, it + # hangs for twenty seconds. + with pytest.raises(ParamError, match="empty value"): + Proxy(country=" ", **CREDS) + + def test_a_parameter_nobody_folds_refuses_whitespace_instead(self): + with pytest.raises(ParamError, match="contains whitespace") as caught: + Proxy(sid="order 4417", **CREDS) + # The message names what is folded, so the caller can tell a refusal + # from a parameter that would have been converted. + assert "'region'" in str(caught.value) + + @pytest.mark.parametrize("bad", [" ", "\t", "\n", "\r", "\v", "\f"]) + def test_every_ascii_whitespace_character_is_refused(self, bad): + # The set is spelled out in providers.py rather than delegated - + # `str.isspace` is Unicode-wide and Rust's `is_ascii_whitespace` + # excludes the vertical tab that Python's includes, so a shared + # contract cannot use either name. + with pytest.raises(ParamError, match="contains whitespace"): + Proxy(sid=f"a{bad}b", **CREDS) + + def test_a_no_break_space_is_not_ascii_whitespace_and_passes(self): + # Deliberate rather than an oversight. A no-break space is a character + # the gateway has never been asked about, and the ASCII set is the one + # four languages agree on without depending on their Unicode tables. + # + # Written as an escape and not as the character. As a literal this + # test passes for the right reason only until an editor or a + # formatter normalises the byte to an ordinary space, at which point + # it asserts the opposite of what it says - or gets "fixed". + assert Proxy(sid="a\u00a0b", **CREDS).username == "acct-sid-a\u00a0b" + + def test_a_session_id_keeps_its_case(self): + # `sid` is excluded from the fold on purpose: a session id is opaque + # and caller-chosen, and lowercasing it would silently move the caller + # to a different sticky session than the one they named. + assert Proxy(sid="Order4417", **CREDS).username == "acct-sid-Order4417" + assert Proxy(**CREDS).session("Order4417").username == "acct-sid-Order4417" + + def test_filter_and_ttl_keep_their_case(self): + # The vendor's client does not fold `filter`, not even to lower case, + # and the generated username that evidences the fold says nothing + # either way because `medium` was already lower case. Folding it would + # be a guess. + proxy = Proxy(filter="MEDIUM", ttl="10M", **CREDS) + assert proxy.username == "acct-filter-MEDIUM-ttl-10M" + + def test_the_fold_is_ascii_only(self): + # The Turkish dotted capital I lower-cases to two code points under + # full Unicode rules in some languages and to one in others, so a + # vector built on it would fail in one SDK for a reason that has + # nothing to do with proxies. `str.lower()` would fold this; the + # translate table does not. + assert Proxy(city="\u0130stanbul", **CREDS).username == "acct-city-\u0130stanbul" + + def test_a_definition_that_folds_nothing_still_refuses_whitespace(self, tmp_path): + # The whitespace refusal is about what a username can carry rather than + # about any gateway's dialect, so it applies with no `normalize` at all. + path = tmp_path / "plain.toml" + path.write_text('label = "Plain"\nknown_params = ["country"]\n', encoding="utf-8") + provider = load_file(path) + assert Proxy(provider=provider, country="US", **CREDS).username == "acct-country-US" + with pytest.raises(ParamError, match="contains whitespace"): + Proxy(provider=provider, country="New York", **CREDS) + + +class TestADefinitionCannotDeclareAnImpossibleFold: + """Refused when it loads, not when somebody calls it. + + Same principle as the checks already there: a declaration that reads like a + working setting and can never fire is the class of mistake this package + exists to make loud. + """ + + def test_normalizing_a_parameter_that_is_not_known_is_refused(self, tmp_path): + path = tmp_path / "wrong.toml" + path.write_text( + 'label = "Wrong"\n' + 'known_params = ["country"]\n' + 'normalize = ["country", "city"]\n', + encoding="utf-8", + ) + with pytest.raises(ProviderError, match="normalizes"): + load_file(path) + + def test_folding_into_the_separator_is_refused(self, tmp_path): + # The fold inserts an underscore, so a gateway that separates on one + # would have the value cut in half by the very step meant to make it + # sendable. The caller would be blamed for input that is correct. + path = tmp_path / "underscored.toml" + path.write_text( + 'label = "Underscored"\n' + 'known_params = ["city"]\n' + 'separator = "_"\n' + 'normalize = ["city"]\n', + encoding="utf-8", + ) + with pytest.raises(ProviderError, match="Pick one"): + load_file(path) + + +class TestWhatAStatusMeansIsPerGateway: + """`connect_reactions` - the machine-readable half of the reactions table. + + A status code on this gateway is not a diagnosis. Five parameters answer a + value the gateway will not take with 407, which reads as a credentials + problem and is not one, so the translation from a code to a sentence is part + of the dialect and belongs beside the separators rather than in a dict in a + module. Four languages read this schema, and this is the third key added to + it before the golden vectors freeze. + """ + + def test_the_shipped_definition_explains_the_misleading_407(self): + meaning = load("nodemaven").reaction(407) + assert meaning is not None + assert "NOT your credentials" in meaning + + def test_a_200_says_what_it_does_not_prove(self): + # The one entry that is not about a failure, and the reason it exists: + # an unrecognised parameter name is also answered 200 and dropped, so a + # 200 means the parameters were accepted and not that they were applied. + meaning = load("nodemaven").reaction(200) + assert meaning is not None + assert "not that they were all" in meaning + + def test_each_reaction_names_the_parameters_it_can_come_from(self): + # Naming the parameter is the only thing the status code itself does + # not do. It is deliberately not "four parameters, four codes": + # measured 2026-09-08, `region`, `city` and `isp` all answer 406, so + # the 406 entry has to name all three. + provider = load("nodemaven") + ambiguous = provider.reaction(406) + assert "region" in ambiguous + assert "city" in ambiguous + assert "isp" in ambiguous + assert "city" in provider.reaction(500) + assert "isp" in provider.reaction(410) + assert "country" in provider.reaction(407) + + def test_the_500_names_the_missing_region_and_not_an_unreachable_city(self): + # This entry said the opposite for most of 2026-09-08 - that every + # `city` tried was refused and the value should be treated as unusable. + # Six CONNECTs the same day: `us`/`louisiana`/`abbeville` answers 200 + # and the same city without its region answers 500, so the 500 is an + # incomplete request and `city` works. All seven earlier values had been + # sent without a region, held fixed across every arm and therefore + # invisible in the comparison between them. + meaning = load("nodemaven").reaction(500) + assert "region" in meaning + assert "unusable" not in meaning + + def test_the_406_does_not_claim_to_know_which_parameter_it_means(self): + # A junk `region`, a junk `isp` and `charter` - a real ISP - all answer + # 406, so the entry must neither say which parameter was refused nor + # tell a caller their name was misspelled. This is the row asked to + # report an ambiguity rather than a diagnosis, and the one most likely + # to be tidied into a confident sentence later. + meaning = load("nodemaven").reaction(406) + assert "charter" in meaning + assert "unavailable" in meaning + + def test_the_410_says_how_narrow_its_sample_is(self): + # One ISP produced it. An entry stating a rule from one value is the + # same mistake as a default sentence for an unmeasured status. + meaning = load("nodemaven").reaction(410) + assert "comcast" in meaning + assert "one value" in meaning + + def test_a_status_nobody_measured_has_no_entry_rather_than_a_guess(self): + # The table is only as long as the measurements. A default sentence + # here would be this package inventing a diagnosis, which is the thing + # it exists to stop the gateway doing. + assert load("nodemaven").reaction(502) is None + + def test_the_keys_are_strings_because_no_wire_format_has_integer_keys( + self, tmp_path + ): + # TOML has no integer keys and neither does JSON, and the golden + # vectors are JSON. Storing them as strings and converting at the + # lookup is one line; the alternative is a schema that cannot round + # trip through the format the four SDKs share. + path = tmp_path / "stringkeys.toml" + path.write_text( + 'label = "Keys"\n' + 'known_params = ["country"]\n' + "[connect_reactions]\n" + '407 = "not the password"\n', + encoding="utf-8", + ) + provider = load_file(path) + assert set(provider.connect_reactions) == {"407"} + assert provider.reaction(407) == "not the password" + + def test_an_empty_explanation_is_refused_at_load(self, tmp_path): + # Same principle as an empty `values` list: a key that is present and + # says nothing reads like a working entry and can only ever print a + # blank line next to a status code. + path = tmp_path / "blank.toml" + path.write_text( + 'label = "Blank"\n' + 'known_params = ["country"]\n' + "[connect_reactions]\n" + '407 = ""\n', + encoding="utf-8", + ) + with pytest.raises(ProviderError): + load_file(path) + + def test_a_definition_need_not_have_the_table_at_all(self, tmp_path): + path = tmp_path / "bare.toml" + path.write_text( + 'label = "Bare"\nknown_params = ["country"]\n', encoding="utf-8" + ) + provider = load_file(path) + assert provider.connect_reactions == {} + assert provider.reaction(407) is None diff --git a/tests/test_readme.py b/tests/test_readme.py new file mode 100644 index 0000000..1653fa1 --- /dev/null +++ b/tests/test_readme.py @@ -0,0 +1,453 @@ +"""Every output the README quotes, compared against the real thing. + +This file exists because the one quotation in the README that had no test drifted +without anyone noticing: it listed nine parameter names where the code produced +ten, and the Rust port's equivalent test is what caught it. The rule that came +out of that is the reason this file is here - **a README that quotes real output +needs a test per quotation, or the quotation is a comment.** + +Comparisons collapse whitespace. That is deliberate and it is the only slack +allowed: a paragraph in a fenced block has to be re-wrappable to stay readable at +80 columns, and nothing else about it may change. Every word still has to match. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from nodemaven import Proxy, load +from nodemaven.check import Check + +README = Path(__file__).resolve().parent.parent / "README.md" + + +def flat(text: str) -> str: + return " ".join(text.split()) + + +def _headings(readme: str) -> set: + """Every heading as GitHub would anchor it: lowered, punctuation dropped.""" + return { + re.sub(r"[^a-z0-9 -]", "", line.lstrip("#").strip().lower()).replace(" ", "-") + for line in readme.splitlines() + if line.startswith("#") + } + + +@pytest.fixture(scope="module") +def readme() -> str: + return README.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def prose(readme: str) -> str: + """The README with its HTML comments removed. + + Assertions about what the file *says* run against this rather than against + the raw text. The surviving comments are notes to whoever edits the file - + which links must stay absolute, which blocks a test pins - and none of them + is prose a reader sees, so counting them as prose would let an editing note + satisfy or break an assertion about the documentation. + """ + return re.sub(r"", "", readme, flags=re.DOTALL) + + +@pytest.fixture(scope="module") +def blocks(readme: str): + """Every fenced block in the file, with its language tag.""" + return re.findall(r"```([a-z]*)\n(.*?)```", readme, re.DOTALL) + + +class TestTheReadmeIsSelfConsistent: + def test_it_no_longer_claims_to_open_no_socket(self, prose: str): + # "This library opens no socket" was true of every version up to 0.1.2 + # and false the moment `check()` landed. Pinned as a test because it is + # the kind of sentence that gets restored by somebody tidying the intro. + # + # The record of the correction lives in CLAUDE.md and the CHANGELOG, not + # in the README: shipped documentation describes the product as it is, + # and a reader of the package has no use for what it used to say. + assert "This library opens no socket" not in prose + assert "`Proxy` opens no socket" in prose + + def test_it_no_longer_claims_nothing_is_raised_from_a_response( + self, prose: str + ): + assert "nothing here sends one" not in prose + + def test_the_account_api_separates_what_was_called_from_what_was_not( + self, readme: str + ): + # Documenting a call with an example is a claim that it works, so the + # section has to say which calls that claim rests on. Until 2026-09-08 + # the answer was "none of them" and this test pinned the words + # "transcribed, not measured". Calls were then sent to the live API and + # the paragraph had to change - which is exactly what it was pinned for, + # and it is pinned again for the same reason: the next call that gets + # measured moves the boundary again. + # + # It moved a second time on 2026-09-09, in the other direction. The + # section used to argue that transcribed paths were safe to ship because + # a wrong one earns a 404; this host answers an unrouted path with 200 + # and the dashboard's own HTML, so that argument was never true here. + # The retraction is pinned too, because the tempting edit is to delete a + # wrong sentence rather than to say what it cost. + section = readme.split("## Account API", 1)[1].split("\n## ", 1)[0] + assert "measured" in section and "transcribed" in section + assert "still never been called" in section + assert "negative control" in section + # The five are named. A count with no names cannot be checked by a + # reader, and cannot be checked here either. + for path in ("users/me", "countries", "regions", "cities", "isps"): + assert f"`{path}`" in section, path + + def test_every_error_class_the_package_exports_is_in_the_table( + self, readme: str + ): + import nodemaven + + exported = [ + name + for name in nodemaven.__all__ + if name.endswith("Error") and name != "NodeMavenError" + ] + table = readme.split("## Errors", 1)[1].split("##", 1)[0] + missing = [name for name in exported if f"`{name}`" not in table] + assert missing == [], f"exported and undocumented: {missing}" + + def test_the_nav_line_points_at_headings_that_exist(self, readme: str): + nav = readme.split("\n\n", 1)[0].rsplit("\n", 1)[1] + anchors = re.findall(r"\]\(#([a-z0-9-]+)\)", nav) + assert anchors, "the nav line was not found, so nothing was checked" + assert set(anchors) <= _headings(readme), set(anchors) - _headings(readme) + + def test_every_anchor_in_the_body_points_at_a_heading_too(self, readme: str): + # Widened from the nav line after a section was renamed and two links + # elsewhere in the file went on pointing at the old anchor. GitHub + # renders a dead in-page link as ordinary text that does nothing when + # clicked - no 404, no warning - so nothing but this would have said so. + anchors = set(re.findall(r"\]\(#([a-z0-9-]+)\)", readme)) + assert len(anchors) > 5, "the anchor scan found almost nothing" + assert anchors <= _headings(readme), anchors - _headings(readme) + + +class TestTheParameterTable: + def test_every_known_parameter_has_a_row(self, readme: str): + # The table is what a developer reads instead of the shipped TOML. A + # parameter missing from it is reachable only by reading source, which is + # what the table was added to stop. + table = readme.split("| parameter |", 1)[1].split("\n\n", 1)[0] + for name in load("nodemaven").known_params: + assert f"`{name}`" in table, name + + def test_no_row_names_a_parameter_the_gateway_does_not_know( + self, readme: str + ): + # The other direction, and the one that matters more: a documented + # parameter this package refuses is a promise it breaks. `norotate` was + # exactly this between 2026-08-21 and 2026-08-26. + table = readme.split("| parameter |", 1)[1].split("\n\n", 1)[0] + known = load("nodemaven").known_params + for row in table.splitlines(): + cells = [cell.strip() for cell in row.split("|")] + if len(cells) < 3 or not cells[1].startswith("`"): + continue + assert cells[1].strip("`") in known, cells[1] + + def test_the_ttl_values_it_names_are_the_ones_the_definition_names( + self, readme: str + ): + # `ttl` is the only parameter with measured accepted values, measured + # refused values, and no `values` entry to enforce either - a list of + # four would refuse a fifth the gateway takes. So the README and the + # definition's notes are the only carriers of the unit rule, and nothing + # else makes them agree. + # + # Read out of the table rather than listed here: a list in this file + # would be a third copy of the same contract. + table = readme.split("| parameter |", 1)[1].split("\n\n", 1)[0] + row = next( + line for line in table.splitlines() if line.startswith("| `ttl` |") + ) + named = re.findall(r"`([^`]+)`", row.rsplit("|", 2)[1]) + assert len(named) >= 2, named + + notes = load("nodemaven").notes + assert [value for value in named if value not in notes] == [] + + +class TestTheUnknownParameterMessage: + def test_the_readme_quotes_it_word_for_word(self, readme: str): + # This is the quotation that drifted, and the test that would have caught + # it. It is built from the shipped definition, so adding a parameter to + # the TOML fails here until the README is updated too. + # + # Every block is checked, not the first one. This test used to pin + # `quoting[0]` against a message raised from a typo hardcoded here, so it + # verified whichever block happened to come first and ignored the rest. A + # second quotation was then added to the quickstart with a different + # typo, and the version of the paragraph before it had an invented + # message in it - "Did you mean 'filter'?", which this library does not + # say and never has. That is the failure this test exists to catch and it + # caught it, but only because the new block landed first in the file. + from nodemaven import ParamError + + blocks = re.findall(r"```[a-z]*\n(.*?)```", readme, re.DOTALL) + quoting = [body for body in blocks if "ParamError:" in body] + assert quoting, "the README stopped quoting the message, so nothing was checked" + + for body in quoting: + typo = re.search(r"does not know the parameter '([^']+)'", flat(body)) + assert typo, f"a ParamError block naming no parameter:\n{body}" + with pytest.raises(ParamError) as caught: + Proxy(login="u", password="p", **{typo.group(1): "us"}) + assert flat(str(caught.value)) in flat(body) + + +class TestTheFoldedRegionExample: + def test_the_username_in_the_readme_is_what_the_code_builds(self, readme: str): + # `region-district_of_columbia` is not an invention: it is the form the + # dashboard itself emitted in a username for a real account, read + # 2026-09-07. The example is the one place the README shows the fold, so + # it is the one place a change to the fold has to be reflected. + built = Proxy( + login="u", password="p", region="District of Columbia" + ).username + assert built == "u-region-district_of_columbia" + assert built in readme + + +class TestTheCheckOutput: + """The two blocks under "Asking the gateway". + + Built here rather than captured from the gateway, and that is what makes them + checkable: a `Check` is a plain frozen object, so the README is quoting a + real `__str__` of a real instance and not a hand-typed approximation of one. + """ + + def test_the_200_line(self, readme: str): + result = Check( + status=200, + reason="Connection established", + server="gate.nodemaven.com:8080", + elapsed=0.42, + headers={"x-proxy-exit-ip": "203.0.113.7"}, + exit_ip="203.0.113.7", + meaning=load("nodemaven").reaction(200), + ) + assert flat(str(result)) in flat(readme) + + def test_the_407_block_including_the_gateways_own_explanation( + self, readme: str + ): + # The explanation comes from the provider definition, so this asserts the + # README and the TOML agree. They are two files and one of them is + # published to PyPI. + result = Check( + status=407, + reason="Proxy Authentication Required", + server="gate.nodemaven.com:8080", + elapsed=0.19, + headers={}, + meaning=load("nodemaven").reaction(407), + ) + assert flat(str(result)) in flat(readme) + + def test_a_200s_explanation_is_not_printed_and_the_readme_says_why( + self, readme: str + ): + # `__str__` shows the meaning only for a non-200, so the 200 block above + # must not carry the 200 explanation. This pins the asymmetry rather than + # leaving it as a coincidence of the two fixtures. + result = Check( + status=200, + reason="Connection established", + server="gate.nodemaven.com:8080", + elapsed=0.42, + headers={}, + meaning=load("nodemaven").reaction(200), + ) + assert "\n" not in str(result) + + def test_the_default_target_named_in_the_readme_is_the_default( + self, readme: str + ): + from nodemaven.check import DEFAULT_TARGET + + assert f"`{DEFAULT_TARGET}`" in readme + + def test_the_documented_timeout_is_the_default(self, readme: str): + import inspect + + from nodemaven.check import connect + + default = inspect.signature(connect).parameters["timeout"].default + assert default == 15.0 + assert "defaults to 15 seconds" in readme + + +class TestTheReferenceMatchesTheCode: + """The Reference section is a contract, so it gets the same treatment as a + quoted output: it is compared against the real thing rather than read. + + Added with the section itself, 2026-09-08. The section exists because an + external review said the documentation was overloaded with justifications; + measuring that turned up the sharper version of the complaint - the package + exports 18 names and the README gave a signature for none of them, so for + several calls the rationale was the only coverage there was. A reference + written once and never checked would have been a worse answer than no + reference, because it reads as authoritative. + """ + + def test_the_proxy_signature_names_every_real_argument(self, readme: str): + import inspect + + block = readme.split("### `Proxy`", 1)[1].split("```", 2)[1] + for name in inspect.signature(Proxy.__init__).parameters: + if name == "self": + continue + assert name in block, name + + def test_the_client_signature_names_every_real_argument(self, readme: str): + import inspect + + from nodemaven import Client + + block = readme.split("### `Client` and `Page`", 1)[1].split("```", 2)[1] + for name in inspect.signature(Client.__init__).parameters: + if name == "self": + continue + assert name in block, name + + def test_every_public_proxy_call_is_in_the_reference(self, readme: str): + section = readme.split("### `Proxy`", 1)[1].split("### `Check`", 1)[0] + missing = [ + name + for name in dir(Proxy) + if not name.startswith("_") and f"`.{name}" not in section + ] + assert missing == [], f"public on Proxy and undocumented: {missing}" + + def test_every_public_client_call_is_in_the_reference(self, readme: str): + from nodemaven import Client + + section = readme.split("### `Client` and `Page`", 1)[1] + section = section.split("### `Provider`", 1)[0] + missing = [ + name + for name in dir(Client) + if not name.startswith("_") and f"`.{name}" not in section + ] + assert missing == [], f"public on Client and undocumented: {missing}" + + def test_every_exported_name_appears_in_the_readme(self, readme: str): + # The gap that produced this test: `available()` was exported and + # appeared nowhere in 625 lines, so the only way to find it was to read + # `__init__.py`. + import nodemaven + + missing = [ + name + for name in nodemaven.__all__ + if not name.startswith("_") and f"`{name}" not in readme + ] + assert missing == [], f"exported and unmentioned: {missing}" + + +class TestTheAttributesTheReadmePromises: + def test_every_field_a_check_carries_is_in_the_reference_table( + self, readme: str + ): + """Derived from the dataclass, not from a list written here. + + This test used to carry the seven names by hand and look for + ``result.`` anywhere in the file, which matched a code block under + "Asking the gateway" that said the same thing as the reference table. + Two copies of one contract with the test pinning one of them is how the + other drifts - it is the failure this whole file was written after. The + block is gone, the table is the one copy, and the names now come from + the type so that adding a field fails here until it is documented. + """ + import dataclasses + + names = [field.name for field in dataclasses.fields(Check)] + # `ok` is a property rather than a field, and is the one a caller reads + # first, so it is named explicitly rather than left to the derivation. + names.append("ok") + table = readme.split("### `Check`", 1)[1].split("###", 1)[0] + missing = [name for name in names if f"`.{name}`" not in table] + assert missing == [], f"a Check field the reference does not list: {missing}" + + @pytest.mark.parametrize( + "attribute", + ["ok", "status", "reason", "exit_ip", "elapsed", "headers", "meaning"], + ) + def test_each_one_exists_on_check(self, attribute, readme: str): + result = Check( + status=200, reason="OK", server="h:1", elapsed=0.0, headers={} + ) + assert hasattr(result, attribute) + + @pytest.mark.parametrize( + "method", + [ + "me", + "countries", + "regions", + "cities", + "isps", + "isp_regions", + "isp_cities", + "zip_codes", + "zip_code_regions", + "zip_code_cities", + "statistics_data", + "statistics_requests", + "domain_statistics", + "sub_users", + "create_sub_user", + "update_sub_user", + "delete_sub_user", + "reset_sub_user_usage", + "whitelist_ips", + "whitelist_ip", + "upsert_whitelist_ip", + "delete_whitelist_ip", + "iterate", + "validate", + ], + ) + def test_every_client_method_the_readme_shows_exists(self, method, readme: str): + from nodemaven import Client + + assert hasattr(Client, method) + assert f"client.{method}(" in readme or f"`{method}()`" in readme + + +class TestTheEnvironmentVariableNames: + @pytest.mark.parametrize( + "name", + [ + "NODEMAVEN_LOGIN", + "NODEMAVEN_PASSWORD", + "NODEMAVEN_HOST", + "NODEMAVEN_PORT", + "NODEMAVEN_APIKEY", + ], + ) + def test_each_one_the_readme_names_is_read_by_the_code(self, name, readme: str): + # A documented variable the code does not read is worse than an + # undocumented one: it silently does nothing and the developer concludes + # their credentials are wrong. + import nodemaven.api + import nodemaven.proxy + + sources = "".join( + Path(module.__file__).read_text(encoding="utf-8") + for module in (nodemaven.proxy, nodemaven.api) + ) + assert name in readme + assert name in sources From 9e6122d3e818b4c9ba80533fa6a97d0407b75a0f Mon Sep 17 00:00:00 2001 From: aleekaz Date: Wed, 9 Sep 2026 16:39:40 +0300 Subject: [PATCH 2/9] Bump the version to 0.1.3 Step 1 of the release procedure and nothing more: no tag, no GitHub Release, so `publish.yml` has not fired and PyPI still serves 0.1.2. The changelog heading carries no time for that reason - every other heading in the file is a PyPI upload record and there is nothing to record yet. The bump is worth making before the release rather than with it. Committed `main` and published 0.1.2 have been the same library since 0.1.2 shipped; this branch is not, it adds `check()` and the dashboard API client, and while both trees answered `0.1.2` there was no way to tell from the outside which one a bug report was against. That is the exact failure this repository hit with 0.1.1, logged in CHANGELOG.md. --- CHANGELOG.md | 9 ++++++++- src/nodemaven/__init__.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f1e95..f89037e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,14 @@ itself is built on: a change to what the gateway is believed to accept carries the probe that established it and the date it was run. "The vendor's documentation says so" is not one of those, and an entry that rests on it says so outright. -## Unreleased +## 0.1.3 - not released yet + +`__version__` says 0.1.3 as of 2026-09-09 and nothing has been uploaded. The +heading carries no time because the times above are PyPI's upload records and +there is no upload to record; it gets one when a `v0.1.3` tag and a GitHub +Release fire `publish.yml`. Anyone quoting what this library does should say +which tree they read until then, because PyPI still serves 0.1.2 and the two +are different libraries. ### The library opens sockets now, and the README said it did not diff --git a/src/nodemaven/__init__.py b/src/nodemaven/__init__.py index bbcb81c..45b7002 100644 --- a/src/nodemaven/__init__.py +++ b/src/nodemaven/__init__.py @@ -43,7 +43,7 @@ from .providers import Provider, available, load, load_file from .proxy import Proxy -__version__ = "0.1.2" +__version__ = "0.1.3" __all__ = [ "Proxy", From 8072c1af0376d707805f80b528026608c4a36cec Mon Sep 17 00:00:00 2001 From: Alexandr Kazmin Date: Wed, 9 Sep 2026 19:49:10 +0300 Subject: [PATCH 3/9] Update src/nodemaven/api.py Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- src/nodemaven/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nodemaven/api.py b/src/nodemaven/api.py index ed6e30b..b18b1d2 100644 --- a/src/nodemaven/api.py +++ b/src/nodemaven/api.py @@ -1603,7 +1603,7 @@ def _next_step(page: Page) -> Optional[Tuple[str, Dict[str, Any]]]: return None if size <= 0: return None - following[paging.cursor_key] = cursor + size + following[paging.cursor_key] = cursor + len(page.results) else: following[paging.cursor_key] = cursor + 1 return path, following From 625e29464d2c365e5ee8270dffdf190d4ffff095 Mon Sep 17 00:00:00 2001 From: Alexandr Kazmin Date: Wed, 9 Sep 2026 19:49:27 +0300 Subject: [PATCH 4/9] Update README.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index fb22a46..b216fca 100644 --- a/README.md +++ b/README.md @@ -709,9 +709,7 @@ client.upsert_whitelist_ip("203.0.113.7", 10, name="the office") client.delete_whitelist_ip(id) ``` -The dates go as `yyyy-mm-dd`. The vendor's documentation says `dd-mm-yyyy` in -its prose and types the same fields as ISO dates two lines below; the type is -what the server parses. +The dates go as `dd-mm-yyyy`. The vendor's documentation types them as ISO dates (`format: date`), but the server parses `dd-mm-yyyy` and answers `yyyy-mm-dd` with a 400 - so send `20-08-2026`, not `2026-08-20`. **`sub_users()` returns each sub-user's `proxy_password` in clear text**, on every row, by the specification's own required-field list. So does `me()`. Do From d1f43ddcb3a13eb514730ee4ee9472ebb43a9d25 Mon Sep 17 00:00:00 2001 From: Alexandr Kazmin Date: Wed, 9 Sep 2026 19:51:05 +0300 Subject: [PATCH 5/9] Update src/nodemaven/proxy.py Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- src/nodemaven/proxy.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/nodemaven/proxy.py b/src/nodemaven/proxy.py index 21d85bb..a8d9ba4 100644 --- a/src/nodemaven/proxy.py +++ b/src/nodemaven/proxy.py @@ -99,17 +99,16 @@ def __init__( # accepting more than a port is and then a truthiness test standing in # for a range check. if raw_port in (None, ""): - self._port = self._provider.port - else: - self._port = _port_number(str(raw_port)) - if self._port is None: - raise CredentialsError( - f"port={raw_port!r} is not a TCP port: it has to be a whole " - f"number from 1 to 65535. Nothing was built. The gateway's " - f"own ports are {self._provider.port} and the ones its " - f"documentation lists; 0 is not one of them - it means 'any " - f"free port' when binding and is meaningless when connecting." - ) + raw_port = self._provider.port + self._port = None if raw_port is None else _port_number(str(raw_port)) + if raw_port is not None and self._port is None: + raise CredentialsError( + f"port={raw_port!r} is not a TCP port: it has to be a whole " + f"number from 1 to 65535. Nothing was built. The gateway's " + f"own ports are {self._provider.port} and the ones its " + f"documentation lists; 0 is not one of them - it means 'any " + f"free port' when binding and is meaningless when connecting." + ) if not self._login or not self._password: missing = [ From ab560bf7879f037b716a3c3abe38e035fb379b05 Mon Sep 17 00:00:00 2001 From: Alexandr Kazmin Date: Wed, 9 Sep 2026 19:51:50 +0300 Subject: [PATCH 6/9] Update src/nodemaven/providers.py Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- src/nodemaven/providers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nodemaven/providers.py b/src/nodemaven/providers.py index 76d4e8f..a0a7924 100644 --- a/src/nodemaven/providers.py +++ b/src/nodemaven/providers.py @@ -275,7 +275,12 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: # entry here cannot refuse anything, so a wrong one is a misleading sentence # and not a blocked request. connect_reactions: Dict[str, str] = {} - for status, meaning in (raw.get("connect_reactions") or {}).items(): + raw_reactions = raw.get("connect_reactions") + if raw_reactions is not None and not isinstance(raw_reactions, dict): + raise ProviderError( + f"{path} gives connect_reactions as {raw_reactions!r}; it has to be a table." + ) + for status, meaning in (raw_reactions or {}).items(): if not isinstance(meaning, str) or not meaning: raise ProviderError( f"{path} describes CONNECT status {status!r} as {meaning!r}. It has " From c12fdb123a0cf980ecffb19f96b42b1921d4de8f Mon Sep 17 00:00:00 2001 From: Alexandr Kazmin Date: Wed, 9 Sep 2026 19:56:08 +0300 Subject: [PATCH 7/9] Update src/nodemaven/data/providers/nodemaven.toml Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- src/nodemaven/data/providers/nodemaven.toml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/nodemaven/data/providers/nodemaven.toml b/src/nodemaven/data/providers/nodemaven.toml index 9111f7d..4c7c359 100644 --- a/src/nodemaven/data/providers/nodemaven.toml +++ b/src/nodemaven/data/providers/nodemaven.toml @@ -332,9 +332,6 @@ looks like a name the gateway knows and a pool this account cannot reach. That \ is one ISP, so read it as the narrower fact and not as a rule. The account \ API's ISP catalogue, filtered by country, is what says which names exist.""" 500 = """\ -`city` was sent without a `region`. `country`+`region`+`city` opens the tunnel \ -on two different cities in two different regions, and dropping the `region` \ -from either of them turns the same request into a 500 - so this is an \ -incomplete request answered as a server fault, and not a fault. Add the region \ -the account API's city catalogue files that city under. A city name the \ -catalogue does not hold is answered 406 instead.""" +a 500 was measured when `city` was sent without a `region`; this status may also \ +represent another gateway failure. Add the region the account API's city catalogue \ +files that city under; a city name the catalogue does not hold is answered 406.""" From e495d52477a8ab03e0788aa1ac4d315d7a134823 Mon Sep 17 00:00:00 2001 From: aleekaz Date: Wed, 9 Sep 2026 20:14:03 +0300 Subject: [PATCH 8/9] Finish the review fixes, and pin the three tests that hid the paging bug The five suggestions applied through the web interface were each half of a fix. This is the other half, plus what the suite should have caught. iterate() advanced the offset by the limit asked for instead of by the rows returned, so cities(limit=10000) walked past the end of a collection this server caps at 1000 rows of 1965. Three tests covered that case and none could fail: two asserted the buggy offsets, one of them saying so in its own name, and the third counted rows only, which the fake transport cannot distinguish because it replays a queued list whatever query string it gets. All three now assert the offset sequence. The 429 message told callers to read retry_after, which this client can never fill - Transport returns (status, bytes) and the headers are gone before anything sees Retry-After. The message and the class docstring now say so; the attribute stays, because it belongs to the class rather than to this transport. The README's statistics examples still sent ISO dates while the prose two paragraphs below them had already been corrected against --phase 10. A test reads the dates out of the fenced blocks now. A provider port is validated at load rather than coerced with int(), which accepted 0. A bad port from the definition no longer raises a message blaming a port= the caller did not pass. normalize and connect_reactions are type-checked, so normalize = "city" stops being iterated character by character. A values list is folded when its parameter is normalized, through one _fold() called from both sites instead of two copies of three steps. test_check.py caught Exception in four places where it meant ParamError, and _headings() in test_readme.py read every code comment as a heading, so the two anchor tests were checking a superset of their subject. 341 tests green. Two of the review's twelve findings are declined and the CHANGELOG says why: carrying repeated headers breaks Check.headers as a dict for no measured reason, and the proposed import-scan test was too vague to pin anything. --- CHANGELOG.md | 100 ++++++++++++++++++ README.md | 19 +++- src/nodemaven/api.py | 37 ++++++- src/nodemaven/data/providers/nodemaven.toml | 10 +- src/nodemaven/errors.py | 15 +++ src/nodemaven/providers.py | 64 +++++++++++- src/nodemaven/proxy.py | 26 ++++- tests/test_api.py | 40 +++++++- tests/test_check.py | 10 +- tests/test_proxy.py | 108 ++++++++++++++++++++ tests/test_readme.py | 82 +++++++++++++-- 11 files changed, 484 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f89037e..d7991bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -808,6 +808,106 @@ Five of the vendor's 24 paths are deliberately not wrapped: `locations/all-doc/` `notifications/`, `llm/submit/`, `llm/results/{id}/` and `llm/balance/`. The omission is listed rather than left silent. +### What a review of the above found, 2026-09-09 + +Ten fixes, from an automated review of the pull request carrying the section +above. Two of its twelve findings were declined and are recorded here as well, +because a declined finding is a decision and reads as an oversight if it is not +written down. + +Worth stating as one thing rather than ten, because it is the pattern and not +the bugs: **in three of these the reasoning was written correctly in the comment +directly above the code, and the code beside it did something else.** The +paging comment described advancing by rows returned while the line added the +limit; the port comment said the value could come from the definition while the +message blamed the caller; the 429 message told the reader to consult a field +this client cannot fill. Prose next to code is not a test of that code, and it +is worse than no prose, because it is what a later reader checks against. + +- **`iterate()` advanced the offset by the limit asked for rather than by the + rows returned.** `cities(limit=10000)` requested offset 0 then offset 10000 + against a server that caps that collection at 1000 rows of 1965, so it walked + off the end and returned 1000 of 1965. The cap was already measured, already + written into this module's docstrings, and the fix had been applied to the + stop condition - stop on an *empty* page, not a short one - and not to the + advance, in the same sitting. + + The suite did not catch it and the reason is worth more than the fix. Three + tests covered the case. Two asserted the wrong offsets, one of them saying so + in its own name - `test_a_caller_who_raised_the_limit_pages_at_that_limit` - + and the third counted rows only, which cannot fail, because the fake transport + replays a queued list of pages whatever query string it is handed. So a test + named after the capped page passed under the broken rule. All three now assert + the offset sequence. + +- **`RateLimitError.retry_after` is always `None` from this client, and both the + message and the class said otherwise.** The 429 said to wait the server's own + interval "in `retry_after`" - an instruction that could not be followed on any + 429 this package raises, because `Transport` returns `(status, bytes)` and the + response headers are gone before anything could read `Retry-After`. The + message now says the client cannot report the interval and that you should + back off on your own schedule. The attribute stays: it belongs to the class + rather than to this one transport, and a caller raising it by hand can set it. + What was wrong was the promise, not the field. Surfacing the header means + widening a public protocol implemented in four languages, and that is a + version's worth of change. + +- **The README's statistics examples sent ISO dates**, which this server answers + `400`. The prose two paragraphs below them had already been corrected against + `--phase 10` on the same day; the examples were left, so the section explained + the right rule and demonstrated the wrong one. A test now reads the dates out + of the fenced blocks and refuses anything that is not `dd-mm-yyyy`. It is + scoped to the blocks deliberately - the prose quotes `start=2026-08-20` as the + form that fails, and a whole-file scan would have to permit the exact string + it is hunting for. + +- **A provider definition's `port` is validated when the definition loads**, not + coerced with `int()` and not left to fail later. `int()` accepted `0`, which + means "any free port" when binding and nothing at all when connecting, and it + raised `ValueError` rather than `ProviderError` on text. A bad port in a + shipped TOML now fails at load with the file named. + +- **A bad port from the provider definition no longer blames the caller.** + `Proxy` falls back to the definition's port when none is passed, and the + refusal said `port=... is not a TCP port`, naming an argument the caller did + not supply. The two cases now raise separately: one names the definition and + says to fix it or pass `port=` to override, the other is unchanged. + +- **`normalize` and `connect_reactions` are type-checked at load.** A + `normalize` given as a string was iterated character by character, so + `normalize = "city"` silently produced the parameter names `c`, `i`, `t`, `y` + and normalized nothing. Both now raise `ProviderError` naming the file. + +- **A `values` list is folded when its parameter is normalized.** The fold - + strip, ASCII lower-case, spaces to `_` - is applied to what the caller passes, + and was not applied to the legal values it is checked against, so a definition + listing `District of Columbia` refused the value it was written to allow. The + fold is now one function, `_fold`, called from both places; it was two copies + of three operations before, which is how they came apart. + +- **`tests/test_check.py` caught `Exception` in four places** where it meant + `ParamError`. `pytest.raises(Exception)` passes on a `TypeError` from a + refactor that broke the call, which is the failure those tests exist to + report. + +- **`_headings()` in `tests/test_readme.py` read every `#` comment in every + example as a heading.** The two tests that check in-page links point at real + headings were therefore checking a superset: a link pointing at a code comment + would have passed. They were green throughout and had stopped testing their + subject. + +Two findings were declined: + +- **Repeated headers in a `Check` are still collapsed to the last one.** + Carrying them all means `Check.headers` stops being a `dict`, which is a + breaking change to a documented attribute, and no measured gateway reaction + depends on a repeated header. It is a design decision for a later version, not + a fix. + +- **A test that scans the source for imports** was proposed as a replacement for + a narrower one. The replacement was not specific enough to pin anything, and a + test that is vague about what it forbids is the shape the two above had. + ## 0.1.2 - 2026-08-26 09:23 - **`values`**, a per-parameter list of legal values, added to the provider diff --git a/README.md b/README.md index b216fca..f374e7d 100644 --- a/README.md +++ b/README.md @@ -693,8 +693,9 @@ client.zip_code_regions(country__code="us") client.zip_code_cities(country__code="us", region__code="dc") # Statistics are per proxy username, and the username is required. -client.statistics_data("acct-1", start_date="2026-09-01", end_date="2026-09-07") -client.statistics_requests("acct-1", start_date="2026-09-01") +# The dates are `dd-mm-yyyy`. ISO is answered 400 - see below. +client.statistics_data("acct-1", start_date="01-09-2026", end_date="07-09-2026") +client.statistics_requests("acct-1", start_date="01-09-2026") client.domain_statistics("acct-1") client.sub_users(page=1) @@ -709,7 +710,19 @@ client.upsert_whitelist_ip("203.0.113.7", 10, name="the office") client.delete_whitelist_ip(id) ``` -The dates go as `dd-mm-yyyy`. The vendor's documentation types them as ISO dates (`format: date`), but the server parses `dd-mm-yyyy` and answers `yyyy-mm-dd` with a 400 - so send `20-08-2026`, not `2026-08-20`. +The dates go as `dd-mm-yyyy`, measured 2026-09-09 by `--phase 10`: +`start=20-08-2026` is answered **200 with 21 data points** and +`start=2026-08-20` is answered **400**, with a body byte-identical to the one +`start=not-a-date` draws. The vendor's document writes the format both ways - +`dd-mm-yyyy` in the prose, `format: date` in the type - and **the prose is the +half that is right**. So ISO is not a rival spelling the server declines, it is +a string the server cannot parse, and every client generated from that +specification sends the one form that fails. + +This paragraph said the opposite until 2026-09-09, and said it for the worst +possible reason: "the type is what the server parses" was an inference about +which half of a self-contradicting document to trust, written before either +half had been sent. The measurement cost one request. **`sub_users()` returns each sub-user's `proxy_password` in clear text**, on every row, by the specification's own required-field list. So does `me()`. Do diff --git a/src/nodemaven/api.py b/src/nodemaven/api.py index b18b1d2..d391c15 100644 --- a/src/nodemaven/api.py +++ b/src/nodemaven/api.py @@ -1070,6 +1070,18 @@ def iterate(self, page: Page, *, max_pages: int = 100) -> Iterator[Any]: Stopping on empty costs one extra request per walk and cannot truncate. + **The offset advances by the rows returned, not by the limit asked + for**, corrected 2026-09-09 from the same measurement and some hours + after it. Advancing by the limit skips whatever the cap withheld: + ``cities(limit=10000)`` asked for offset 10000 next, which is 8035 rows + past the end, so the walk collected 1000 rows of 1965 - the exact + truncation the stop rule above had just been rewritten to prevent, by a + different route. The suite hid it. ``test_the_capped_page_that_the_old_ + rule_truncated`` counted rows and never read the query string, and the + fake transport replays its queue whatever the offset says, so it passed + under both rules; the two tests that did read the offsets asserted the + wrong ones, one of them in its own name. + **An empty ``next`` is not read as the end of the collection** either, and that is the point of this method rather than a detail of it. A full page with no total and no next url is byte-for-byte what a complete @@ -1492,9 +1504,17 @@ def _interpret(status: int, raw: bytes, method: str, url: str) -> Any: f"{where} found nothing: {detail}.", status=status, body=parsed ) if status == 429: + # This said "wait the server's own interval if it gave one, in + # retry_after" until 2026-09-09, and `retry_after` is always `None` + # here: `Transport` hands back `(status, bytes)` and the headers are + # discarded before this function sees them, so a `Retry-After` the + # server did send cannot reach the attribute the message points at. + # A test one file over pinned it as `None` the whole time. raise RateLimitError( - f"{where} was rate limited: {detail}. Nothing here retries - wait the " - f"server's own interval if it gave one, in retry_after.", + f"{where} was rate limited: {detail}. Nothing here retries, and this " + f"client cannot tell you the server's interval - it reads no " + f"response headers, so retry_after is always None. Back off on your " + f"own schedule.", status=status, body=parsed, ) @@ -1579,6 +1599,19 @@ def _next_step(page: Page) -> Optional[Tuple[str, Dict[str, Any]]]: that rule truncates against a server that caps the size below the request - which this one does, at ``limit=1000`` on ``cities``, where the collection is 1965. See :meth:`Client.iterate`. + + **The cursor advances by the rows that came back, not by the size that was + asked for**, and that is the same measurement applied a second time. It was + ``cursor + size`` until 2026-09-09, which is only the same number while the + server never returns fewer rows than requested; against the ceiling above, + ``cities(limit=10000)`` walked 0 then 10000, landed 8035 rows past the end of + the collection, and returned 1000 of 1965. Both halves of this function were + written from one measurement in one sitting and only one of them was + changed, so the docstring stating the ceiling sat directly above the code + ignoring it. + + ``size`` is still read, and only to refuse a walk whose size key is missing + or not a positive whole number. It no longer takes part in the arithmetic. """ path = page.request_path asked = page.request_params diff --git a/src/nodemaven/data/providers/nodemaven.toml b/src/nodemaven/data/providers/nodemaven.toml index 4c7c359..d2f9cbe 100644 --- a/src/nodemaven/data/providers/nodemaven.toml +++ b/src/nodemaven/data/providers/nodemaven.toml @@ -332,6 +332,10 @@ looks like a name the gateway knows and a pool this account cannot reach. That \ is one ISP, so read it as the narrower fact and not as a rule. The account \ API's ISP catalogue, filtered by country, is what says which names exist.""" 500 = """\ -a 500 was measured when `city` was sent without a `region`; this status may also \ -represent another gateway failure. Add the region the account API's city catalogue \ -files that city under; a city name the catalogue does not hold is answered 406.""" +one cause is measured and it is not the only cause a 500 can have: `city` sent \ +without a `region`. `country`+`region`+`city` opens the tunnel on two different \ +cities in two different regions, and dropping the `region` from either of them \ +turns the same request into a 500 - that much is an incomplete request answered \ +as a server fault. Try the region the account API's city catalogue files that \ +city under. A city name the catalogue does not hold is answered 406 instead. \ +Any other 500 is the gateway's own, and this sentence cannot tell them apart.""" diff --git a/src/nodemaven/errors.py b/src/nodemaven/errors.py index c7e982a..e8b8284 100644 --- a/src/nodemaven/errors.py +++ b/src/nodemaven/errors.py @@ -98,6 +98,21 @@ class RateLimitError(ApiError): for the same reason applies less here. Waiting the number the server gave you is not the behaviour that measurement warns about; a loop that ignores it is. + + **``nodemaven.api.Client`` never fills it in, so from that client it is + always ``None``**, said here from 2026-09-09. The reason is structural + rather than an oversight to be worked around: ``api.Transport`` returns + ``(status, bytes)`` and discards the response headers, so ``Retry-After`` + is gone before anything could read it. The paragraph above described the + attribute as though the client populated it, and the 429 message told + callers to go and read it - an instruction that could not be followed on + any 429 this package raises. The attribute stays, because it is part of + this class rather than of that one transport and a caller raising it by + hand can set it; what was wrong was the promise, not the field. + + Surfacing the header means widening the ``Transport`` return type, which is + a public protocol implemented in four languages. That is a version's worth + of change and it is not made here. """ def __init__( diff --git a/src/nodemaven/providers.py b/src/nodemaven/providers.py index a0a7924..f6a990f 100644 --- a/src/nodemaven/providers.py +++ b/src/nodemaven/providers.py @@ -22,6 +22,7 @@ from pathlib import Path from typing import Any, Dict, FrozenSet, Optional, Tuple +from .check import _port_number from .errors import ProviderError if sys.version_info >= (3, 11): # pragma: no cover - version dependent @@ -55,6 +56,21 @@ ) +def _fold(value: str) -> str: + """A value in its wire form: strip, ASCII lower-case, spaces to ``_``. + + Module-level from 2026-09-09 so that ``load_file`` folds a configured + ``values`` list through the same three steps ``Provider.normalized`` folds a + caller's input through. It was inline in that method, and the two sides of + the comparison were therefore folded by one implementation and none: a + definition that both normalized ``region`` and listed + ``values.region = ["District of Columbia"]`` refused ``District of + Columbia``, because the input arrived at the check as + ``district_of_columbia`` and the legal list had never been touched. + """ + return value.strip(ASCII_WHITESPACE).translate(_ASCII_LOWER).replace(" ", "_") + + @dataclass(frozen=True, eq=False) class Provider: """One gateway's username dialect. @@ -146,7 +162,7 @@ def normalized(self, name: str, value: str) -> str: """ if name not in self.normalize: return value - return value.strip(ASCII_WHITESPACE).translate(_ASCII_LOWER).replace(" ", "_") + return _fold(value) @property def is_measured(self) -> bool: @@ -254,7 +270,17 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: # cannot be observed, because a country code and a pool name have no spaces # in them to convert. One rule that four languages have to agree on beats # two. - normalize = frozenset(str(name) for name in (raw.get("normalize") or ())) + raw_normalize = raw.get("normalize") + if raw_normalize is not None and not isinstance(raw_normalize, list): + # `normalize = 1` raised a bare `TypeError: 'int' object is not + # iterable` until 2026-09-09, and `normalize = "region"` was worse than + # that: a string iterates into its characters, so it reached the check + # below and was reported as five unknown parameter names. + raise ProviderError( + f"{path} gives normalize as {raw_normalize!r}. It has to be a list " + f"of parameter names." + ) + normalize = frozenset(str(name) for name in (raw_normalize or ())) unknown_normalize = sorted(normalize - known) if unknown_normalize: raise ProviderError( @@ -263,6 +289,17 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: f"fold can never run." ) + # A legal-values list for a folded parameter is folded too, from 2026-09-09. + # The caller's value is folded before it is checked, so an unfolded list + # refuses exactly the values a definition went to the trouble of declaring + # legal. Doing it here rather than at the comparison keeps one folded form + # in the object, so the message that lists the legal values quotes what the + # check actually compared against. + values = { + name: tuple(_fold(item) for item in allowed) if name in normalize else allowed + for name, allowed in values.items() + } + # What each CONNECT status means on this gateway. Keys are the status code as # a string, because TOML has no integer keys and JSON has none either - and # the golden vectors are JSON, so a schema that used integers here would @@ -309,7 +346,28 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: f"inserted. Pick one." ) + # Through `check._port_number`, the same one `Proxy` uses, and for the same + # reason a comment in `proxy.py` gives at length: a rule with two + # implementations is how the two languages here drifted apart once already. + # + # This check did not exist until 2026-09-09. It was `int(port)`, which + # accepts `70000` and `-1` and raises a bare `ValueError` on `"abc"` instead + # of this module's `ProviderError`. A definition carrying `port = 70000` + # loaded without complaint and `Proxy` then built `gateway.example:70000`, + # because the branch taking the provider's port as a fallback was the one + # branch that skipped the port rule. That branch was corrected the same day; + # correcting it there alone would have left the wrong layer reporting it - + # the value comes from this file, so the error has to name this file. port = raw.get("port") + if port is not None: + checked = _port_number(str(port)) + if checked is None: + raise ProviderError( + f"{path} gives port as {port!r}. It has to be a whole number " + f"from 1 to 65535; 0 means 'any free port' when binding and is " + f"meaningless when connecting." + ) + port = checked return Provider( id=provider_id or path.stem, label=str(raw["label"]), @@ -320,7 +378,7 @@ def load_file(path, provider_id: Optional[str] = None) -> Provider: pair_separator=pair_separator, session_param=session_param, host=raw.get("host"), - port=int(port) if port is not None else None, + port=port, aliases=aliases, values=values, normalize=normalize, diff --git a/src/nodemaven/proxy.py b/src/nodemaven/proxy.py index a8d9ba4..aa08730 100644 --- a/src/nodemaven/proxy.py +++ b/src/nodemaven/proxy.py @@ -98,10 +98,34 @@ def __init__( # the wrong class, the second the wrong sentence - both from `int()` # accepting more than a port is and then a truthiness test standing in # for a range check. - if raw_port in (None, ""): + # + # The provider's own port went through this rule from 2026-09-09; until + # then that fallback was assigned unchecked, so a definition carrying + # `port = 70000` built `host:70000` and nothing refused it. The value is + # now also checked in `providers.load_file`, which is the layer that can + # name the file it came from - this branch stays because `Provider` is a + # public dataclass and can be built in code without going through the + # loader. + # + # **Which of the two it was has to be said**, and the first version of + # this fix did not say it: one message served both, so a bad port in a + # TOML file was reported as `port=70000 is not a TCP port` followed by + # `the gateway's own ports are 70000`, blaming a `port=` argument the + # caller never passed and quoting the bad value back as the good one. + from_provider = raw_port in (None, "") + if from_provider: raw_port = self._provider.port self._port = None if raw_port is None else _port_number(str(raw_port)) if raw_port is not None and self._port is None: + if from_provider: + raise CredentialsError( + f"the {self._provider.label} definition gives its port as " + f"{raw_port!r}, which is not a TCP port: it has to be a " + f"whole number from 1 to 65535. Nothing was built. You " + f"passed no port=, so this is the provider definition and " + f"not your call - fix the definition, or pass port= to " + f"override it." + ) raise CredentialsError( f"port={raw_port!r} is not a TCP port: it has to be a whole " f"number from 1 to 65535. Nothing was built. The gateway's " diff --git a/tests/test_api.py b/tests/test_api.py index 85b4009..062fec0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -404,11 +404,22 @@ def test_404_is_its_own_class_because_crud_branches_on_it(self): api.delete_sub_user(7) assert caught.value.status == 404 - def test_429_carries_the_servers_own_interval_and_retries_nothing(self): + def test_429_says_it_cannot_report_the_interval_and_retries_nothing(self): + # This was called `..._carries_the_servers_own_interval_...` and the + # message told the caller to read `retry_after`, while this same + # assertion pinned it as `None`. It can never be anything else from + # this client: `Transport` returns `(status, bytes)` and the headers + # are gone before `_interpret` sees them, so a `Retry-After` the server + # sent cannot reach the attribute. Found by review 2026-09-09. + # + # The test and the defect were three lines apart. The assertion was + # read as "the server did not send one" for as long as it existed, and + # nothing said the other branch was unreachable. api, _ = client((429, {"detail": "Request was throttled."})) with pytest.raises(RateLimitError, match="Nothing here retries") as caught: api.me() assert caught.value.retry_after is None + assert "retry_after is always None" in str(caught.value) def test_5xx_says_that_hammering_it_is_the_thing_we_decline_to_do(self): api, _ = client((503, {"detail": "upstream down"})) @@ -794,7 +805,11 @@ def test_a_full_page_is_followed_at_the_next_offset(self): ) assert list(api.iterate(api.countries(limit=50))) == list(range(110)) offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] - assert offsets == [["0"], ["50"], ["100"], ["150"]] + # The third page is short - 10 rows against a limit of 50 - so the + # fourth offset is 110 and not 150. This asserted 150 until 2026-09-09: + # the cursor advanced by the limit that was *asked for*, which is only + # the same number while the server never returns fewer. + assert offsets == [["0"], ["50"], ["100"], ["110"]] def test_a_short_page_is_followed_because_the_server_caps_the_limit(self): # This asserted the opposite until 2026-09-09 and it was the reason the @@ -810,14 +825,31 @@ def test_the_capped_page_that_the_old_rule_truncated(self): # The live case, 2026-09-08: `cities(limit=10000)` is answered with 1000 # rows because 1000 is the server ceiling, and there are 1965. Under # "stop on a short page" this walk returned 1000 and reported nothing. + # + # **The offsets are asserted here from 2026-09-09, and until they were + # this test could not fail on the case it is named after.** It checked + # the row count only, and `FakeTransport` replays its queue whatever the + # query string says, so a walk asking for offset 0, 10000, 20000 gets + # the same three responses as one asking for 0, 1000, 1965 and counts + # 1965 either way. Against the real server the second request would have + # started 8035 rows past the end of the collection. The two tests in + # this class that do read the offsets were asserting the wrong numbers, + # so the defect sat between a test that could not see it and two that + # pinned it. api, fake = client( (200, {"results": list(range(0, 1000))}), (200, {"results": list(range(1000, 1965))}), (200, {"results": []}), ) assert len(list(api.iterate(api.cities(limit=10000)))) == 1965 + offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] + assert offsets == [["0"], ["1000"], ["1965"]] - def test_a_caller_who_raised_the_limit_pages_at_that_limit(self): + def test_a_caller_who_raised_the_limit_pages_by_the_rows_returned(self): + # This was called `..._pages_at_that_limit` and asserted 400 as the + # third offset until 2026-09-09, so its own name stated the defect as + # the intended behaviour. The second page holds 3 rows, so everything + # from 203 to 399 was skipped whenever the server had it. api, fake = client( (200, {"results": list(range(200))}), (200, {"results": list(range(3))}), @@ -825,7 +857,7 @@ def test_a_caller_who_raised_the_limit_pages_at_that_limit(self): ) list(api.iterate(api.countries(limit=200))) offsets = [parse_qs(urlsplit(c["url"]).query)["offset"] for c in fake.calls] - assert offsets == [["0"], ["200"], ["400"]] + assert offsets == [["0"], ["200"], ["203"]] def test_the_default_limit_is_what_a_caller_who_asks_for_nothing_pages_at(self): api, fake = client( diff --git a/tests/test_check.py b/tests/test_check.py index 5848dff..73586dd 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -21,7 +21,7 @@ import pytest -from nodemaven import Check, CheckError, Proxy +from nodemaven import Check, CheckError, ParamError, Proxy from nodemaven.check import connect REACTIONS = { @@ -754,12 +754,12 @@ def test_the_parent_is_untouched(self): def test_a_nonsense_count_is_refused(self): proxy = Proxy(login="acct", password="pw") - with pytest.raises(Exception, match="no identities"): + with pytest.raises(ParamError, match="no identities"): proxy.sessions(0) def test_a_nonsense_length_is_refused(self): proxy = Proxy(login="acct", password="pw") - with pytest.raises(Exception, match="length"): + with pytest.raises(ParamError, match="length"): proxy.sessions(2, length=0) def test_more_ids_than_exist_is_refused_rather_than_looped_forever(self): @@ -773,7 +773,7 @@ def test_more_ids_than_exist_is_refused_rather_than_looped_forever(self): # the reason it is written against `length=1`: the failure is cheap to # provoke at 256 values and impossible to provoke at 2**48. proxy = Proxy(login="acct", password="pw") - with pytest.raises(Exception, match="at least the whole space"): + with pytest.raises(ParamError, match="at least the whole space"): proxy.sessions(257, length=1) def test_the_whole_space_is_refused_and_one_less_is_not(self): @@ -792,7 +792,7 @@ def test_the_whole_space_is_refused_and_one_less_is_not(self): # by writing the same bound into the README and finding the two # sentences could not both be true. proxy = Proxy(login="acct", password="pw") - with pytest.raises(Exception, match="at least the whole space"): + with pytest.raises(ParamError, match="at least the whole space"): proxy.sessions(256, length=1) assert len({p.params["sid"] for p in proxy.sessions(255, length=1)}) == 255 diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 2455c5f..82d7935 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -12,6 +12,7 @@ from nodemaven import ( CredentialsError, ParamError, + Provider, ProviderError, Proxy, available, @@ -241,6 +242,113 @@ def test_an_empty_list_is_refused_at_load(self, tmp_path): values = { filter = [] } """) + def test_a_legal_value_list_is_folded_when_the_parameter_is_normalized( + self, tmp_path + ): + # Found by review 2026-09-09. The caller's value is folded before it is + # checked, so an unfolded list refuses exactly the values the definition + # declared legal: `District of Columbia` arrived at the check as + # `district_of_columbia` and was compared against `District of + # Columbia`. Both sides go through one fold now. + provider = self._provider(tmp_path, """ +label = "P" +known_params = ["region"] +normalize = ["region"] +values = { region = ["District of Columbia"] } +""") + assert provider.allowed("region") == ("district_of_columbia",) + built = Proxy(provider=provider, region="District of Columbia", **CREDS) + assert built.username == "acct-region-district_of_columbia" + + def test_an_unnormalized_value_list_is_left_alone(self, tmp_path): + # The control for the test above: the fold has to follow `normalize` + # and not apply to every list, or a definition that deliberately keeps + # a case-sensitive value silently loses the distinction. + provider = self._provider(tmp_path, """ +label = "P" +known_params = ["region"] +values = { region = ["District of Columbia"] } +""") + assert provider.allowed("region") == ("District of Columbia",) + + def test_a_normalize_that_is_not_a_list_is_refused_at_load(self, tmp_path): + # `normalize = 1` raised a bare TypeError until 2026-09-09, which is + # neither of this package's exception types. + with pytest.raises(ProviderError, match="has to be a list"): + self._provider(tmp_path, """ +label = "P" +known_params = ["region"] +normalize = 1 +""") + + def test_a_normalize_given_as_a_string_is_refused_at_load(self, tmp_path): + # Worse than the TypeError above and the reason the check is on the + # type rather than on iterability: a string iterates into characters, + # so this used to be reported as six unknown parameter names. + with pytest.raises(ProviderError, match="has to be a list"): + self._provider(tmp_path, """ +label = "P" +known_params = ["region"] +normalize = "region" +""") + + def test_connect_reactions_that_are_not_a_table_are_refused_at_load( + self, tmp_path + ): + # Found by review 2026-09-09: `.items()` on a list is an AttributeError, + # which says nothing about the file it came from. + with pytest.raises(ProviderError, match="has to be a table"): + self._provider(tmp_path, """ +label = "P" +known_params = ["country"] +connect_reactions = ["407 is a refusal"] +""") + + def test_a_port_outside_the_range_is_refused_at_load(self, tmp_path): + # Found by review 2026-09-09. `int(port)` accepted this and `Proxy` + # then built `host:70000`, because taking the provider's port as a + # fallback was the one path that skipped the port rule. + with pytest.raises(ProviderError, match="1 to 65535"): + self._provider(tmp_path, """ +label = "P" +known_params = ["country"] +host = "gateway.example" +port = 70000 +""") + + def test_a_port_that_is_not_a_number_is_refused_at_load(self, tmp_path): + with pytest.raises(ProviderError, match="1 to 65535"): + self._provider(tmp_path, """ +label = "P" +known_params = ["country"] +host = "gateway.example" +port = "abc" +""") + + def test_a_bad_provider_port_reached_in_code_names_the_definition(self): + # `load_file` refuses the value above, but `Provider` is a public + # dataclass and can be built without the loader, so `Proxy` keeps its + # own guard. What is pinned here is the sentence: the first version of + # this fix reported a bad definition as `port=70000 is not a TCP port` + # and then `the gateway's own ports are 70000`, blaming a `port=` + # argument the caller never passed. + # + # `CREDS` carries a port, so it is deliberately not used here - passing + # one would take the caller branch and the test would pass while + # measuring the other sentence. + provider = Provider(id="p", label="P", known_params=frozenset({"country"}), + host="gateway.example", port=70000) + with pytest.raises(CredentialsError, match="You passed no port="): + Proxy(provider=provider, login="acct", password="pw") + + def test_a_bad_caller_port_still_blames_the_caller(self): + # The control for the test above: the same bad number, supplied the + # other way, has to produce the other sentence. + provider = Provider(id="p", label="P", known_params=frozenset({"country"}), + host="gateway.example", port=8080) + with pytest.raises(CredentialsError, match="port=70000 is not a TCP port"): + Proxy(provider=provider, login="acct", password="pw", port=70000) + class TestTheShippedDefinition: def test_nodemaven_is_shipped_and_is_measured(self): diff --git a/tests/test_readme.py b/tests/test_readme.py index 1653fa1..76474fe 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -29,12 +29,36 @@ def flat(text: str) -> str: def _headings(readme: str) -> set: - """Every heading as GitHub would anchor it: lowered, punctuation dropped.""" - return { - re.sub(r"[^a-z0-9 -]", "", line.lstrip("#").strip().lower()).replace(" ", "-") - for line in readme.splitlines() - if line.startswith("#") - } + """Every heading as GitHub would anchor it: lowered, punctuation dropped. + + **Lines inside fenced code blocks are skipped, and a heading needs a space + after its hashes**, both from 2026-09-09. Before that this took any line + beginning with `#`, which in this README means every Python and TOML + comment inside every example - `# Statistics are per proxy username` + became the anchor `statistics-are-per-proxy-username`, and GitHub creates + no such anchor. + + That made the two anchor tests below weaker than they read. They check that + every in-page link points at a heading that exists; with comments in the + set, a link could point at a comment and pass. The tests were not wrong + about their subject, they were quietly checking a superset of it, and a + superset is exactly the shape a passing test takes when it has stopped + testing anything. + """ + headings = set() + fenced = False + for line in readme.splitlines(): + if line.lstrip().startswith("```"): + fenced = not fenced + continue + if fenced: + continue + stripped = line.strip() + if not re.match(r"^#{1,6} ", stripped): + continue + text = stripped.lstrip("#").strip().lower() + headings.add(re.sub(r"[^a-z0-9 -]", "", text).replace(" ", "-")) + return headings @pytest.fixture(scope="module") @@ -133,6 +157,52 @@ def test_every_anchor_in_the_body_points_at_a_heading_too(self, readme: str): assert len(anchors) > 5, "the anchor scan found almost nothing" assert anchors <= _headings(readme), anchors - _headings(readme) + def test_a_comment_in_a_code_block_is_not_read_as_a_heading(self): + # The two tests above check that every in-page link points at a heading + # that exists. That is only worth something if the heading set is the + # headings - until 2026-09-09 it also held every `#` comment in every + # example, so a link could point at a comment and pass. Both tests were + # green the whole time, on a superset of their own subject. + # + # Written against a fixture rather than against the README, because the + # rule is about `_headings` and a README that happens to have no + # comment starting with a word would satisfy it by accident. + sample = "\n".join( + [ + "# Real Heading", + "", + "```python", + "# Statistics are per proxy username", + "#### not a heading either", + "```", + "", + "## Second Real Heading", + "#no space after the hashes", + ] + ) + assert _headings(sample) == {"real-heading", "second-real-heading"} + + def test_no_statistics_example_carries_an_iso_date(self, blocks): + # The server parses `dd-mm-yyyy` and answers an ISO date 400, measured + # 2026-09-09 by `--phase 10`. The examples carried ISO until that day. + # + # Scoped to the fenced blocks on purpose: the prose two paragraphs down + # quotes `start=2026-08-20` as the form that fails, and a scan over the + # whole file would have to be written to permit the very string it is + # looking for. A test that has to make an exception for the correct case + # is one edit away from making it for the wrong one. + dated = [ + value + for _language, body in blocks + for value in re.findall(r"(?:start|end)_date=\"([^\"]+)\"", body) + ] + assert dated, "no dated example was found, so nothing was checked" + for value in dated: + assert re.fullmatch(r"\d{2}-\d{2}-\d{4}", value), ( + f"a statistics example dates as {value!r}; this server parses " + f"dd-mm-yyyy and answers ISO 400" + ) + class TestTheParameterTable: def test_every_known_parameter_has_a_row(self, readme: str): From 1ce201b049d6d1808aa01d20b0422640be1b4a95 Mon Sep 17 00:00:00 2001 From: aleekaz Date: Wed, 9 Sep 2026 20:25:55 +0300 Subject: [PATCH 9/9] Name the statistics filters start and end, not start_date and end_date The review's one remaining finding, and it is a defect the previous commit introduced. Correcting the ISO dates in those examples left the keyword names alone: this server's filters are start and end, confirmed against the vendor specification's parameter list and against --phase 10, which got 200 from start=20-08-2026. Client forwards **filters as query parameters unaltered and this server ignores a query parameter it does not know, so start_date= was accepted by Python, sent, and dropped - and with the range then absent the call answers 500 about the missing range rather than about the name. That is the norotate failure mode one layer up: Proxy refuses an unknown gateway parameter because the gateway answers 200 and drops it, and Client forwards an unknown API parameter in silence for the same server behaviour. Validating **filters needs a per-endpoint list of legal names and is not done here. Three tests pin the examples instead: the filter names, the date format, and that no example omits start, end and period together - domain_statistics ("acct-1") did, and all three statistics endpoints answer 500 to that. The docstrings on statistics_requests and domain_statistics now repeat the rule rather than pointing at statistics_data, because the one carrying it was not the one anybody read. The date test from the previous commit would not have caught this and would have gone quiet on the fix: it scanned for start_date=, written from the examples rather than from the API, so correcting the names would have left it matching nothing and passing on an empty list. It survives on its assert dated guard. 343 tests green. --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++- README.md | 9 ++++--- src/nodemaven/api.py | 10 +++++++- tests/test_readme.py | 59 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7991bb..134d93f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -586,7 +586,11 @@ taken effect. Nothing failing, nothing to notice. softened - the README still says which behaviours were measured and which were transcribed, and it now names this file as where the provenance lives. Counted after the sweep rather than during it: `README.md` holds two dates and - both are the arguments in `statistics(start_date=..., end_date=...)`, the Rust + both are the arguments in `statistics(start_date=..., end_date=...)` - which + named two filters this server does not have, corrected in the README on + 2026-09-09 and left standing here because the count was right about the dates + and the sentence around them was quoting the example rather than the API - the + Rust README holds none, all five `connect_reactions` hold none, and `notes` keeps its six. @@ -896,6 +900,42 @@ is worse than no prose, because it is what a later reader checks against. would have passed. They were green throughout and had stopped testing their subject. +**An eleventh fix, from a second pass of the same review, and it is a defect +the first pass introduced.** Correcting the ISO dates in the statistics examples +left the keyword names alone: they read `start_date=` and `end_date=`, and this +server's filters are `start` and `end`. The examples had been wrong about the +names since the section was written, and the edit that touched the very same two +lines did not look at them, because the finding was about the date format and +the date format is what got checked. + +The consequence is worse than a wrong example. `Client` forwards `**filters` as +query parameters unaltered, and **this server ignores a query parameter it does +not know**, measured 2026-09-09. So `start_date=` is accepted by Python, sent, +and dropped; the range is then absent, and the call answers 500 with the server +complaining about a missing range rather than about the name. Nothing between +the caller and the response says the word was wrong. + +That is the `norotate` failure mode one layer up, and it is worth naming as an +inconsistency in this package rather than as a typo: `Proxy` refuses an unknown +gateway parameter before sending, because the gateway answers 200 and drops it - +that refusal is most of what this library is for - and `Client` forwards an +unknown API parameter in silence for exactly the same server behaviour. +Validating `**filters` means shipping a per-endpoint list of legal names, which +is a version's worth of decision and is not made here. Three tests now pin the +examples instead: the filter names, the date format, and that no example omits +`start`, `end` and `period` together - `domain_statistics("acct-1")` did, and +all three statistics endpoints answer 500 to that. + +**The test written in the first pass would not have caught it, and would have +gone quiet on being fixed.** It scanned for `(?:start|end)_date="..."`, because +it was written from the examples rather than from the API, so it pinned the date +format of two arguments that did nothing - and the moment the names were +corrected its regex would have matched nothing and it would have passed on an +empty list. It survives only because it carries an `assert dated` guard first. A +test that scans for something and then checks what it found needs to assert it +found something, or it turns into a green no-op the first time the thing it +scans for is renamed. + Two findings were declined: - **Repeated headers in a `Check` are still collapsed to the last one.** diff --git a/README.md b/README.md index f374e7d..af787ad 100644 --- a/README.md +++ b/README.md @@ -693,10 +693,11 @@ client.zip_code_regions(country__code="us") client.zip_code_cities(country__code="us", region__code="dc") # Statistics are per proxy username, and the username is required. -# The dates are `dd-mm-yyyy`. ISO is answered 400 - see below. -client.statistics_data("acct-1", start_date="01-09-2026", end_date="07-09-2026") -client.statistics_requests("acct-1", start_date="01-09-2026") -client.domain_statistics("acct-1") +# The range is `start` and `end`, not `start_date` and `end_date`, and the +# dates are `dd-mm-yyyy`. ISO is answered 400 - see below. +client.statistics_data("acct-1", start="01-09-2026", end="07-09-2026") +client.statistics_requests("acct-1", start="01-09-2026") +client.domain_statistics("acct-1", period="hours24") client.sub_users(page=1) client.create_sub_user("worker-1", "a-password", traffic_limit=1024) diff --git a/src/nodemaven/api.py b/src/nodemaven/api.py index d391c15..f6e1c80 100644 --- a/src/nodemaven/api.py +++ b/src/nodemaven/api.py @@ -676,7 +676,8 @@ def statistics_data(self, proxy_username: str, **filters: Any) -> Dict[str, Any] def statistics_requests(self, proxy_username: str, **filters: Any) -> Dict[str, Any]: """Request counts over time. Same two-array shape as - :meth:`statistics_data`.""" + :meth:`statistics_data`, and the same filters: ``start`` and ``end`` in + ``dd-mm-yyyy``, or a ``period``, and a 500 if all three are missing.""" return self._get( f"{API_ROOT}/statistics/requests/", dict(filters, proxy_username=proxy_username), @@ -693,6 +694,13 @@ def domain_statistics(self, proxy_username: str, **filters: Any) -> Page: paging convention and :meth:`iterate` over it yields exactly these rows and stops. It is a ``Page`` at all only so that iterating the result reads the same as iterating the catalogue. + + Send a ``period`` or a ``start``/``end`` range here too: this endpoint + is in the same 500 as the other two when all three are omitted. The + warning is repeated on each of the three rather than written once on + :meth:`statistics_data`, because the README showed + ``domain_statistics("acct-1")`` bare until 2026-09-09 - the docstring + carrying it was not the docstring anybody read. """ return self._list( f"{API_ROOT}/statistics/domains/", diff --git a/tests/test_readme.py b/tests/test_readme.py index 76474fe..5f701be 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -182,6 +182,39 @@ def test_a_comment_in_a_code_block_is_not_read_as_a_heading(self): ) assert _headings(sample) == {"real-heading", "second-real-heading"} + def test_the_statistics_examples_name_the_filters_the_server_has( + self, blocks + ): + """The range is ``start`` and ``end``, and nothing else catches a typo. + + `Client` forwards `**filters` as query parameters unaltered, and this + server **ignores a query parameter it does not know** - measured + 2026-09-09. So `start_date=` is accepted by Python, sent, and dropped, + and because the range is then absent the call answers 500 with the + server complaining about something else. That is the `norotate` failure + mode one layer up: the package refuses an unknown *gateway* parameter + and forwards an unknown *API* parameter in silence. + + The names are written out here, which is a second copy of a contract + this file's own docstring warns against. It is deliberate, because + there is no first copy to point at: `**filters` means the package holds + no list of legal filter names anywhere. The copy is the finding. + """ + names = {"proxy_username", "timezone", "start", "end", "period", + "request_source", "limit"} + calls = [ + (call, args) + for _language, body in blocks + for call, args in re.findall(r"client\.(statistics_\w+|domain_statistics)\(([^)]*)\)", body) + ] + assert calls, "no statistics example was found, so nothing was checked" + for call, args in calls: + for keyword in re.findall(r"(\w+)=", args): + assert keyword in names, ( + f"{call} is shown with {keyword}=, which this server has no " + f"such filter for; it would be dropped in silence" + ) + def test_no_statistics_example_carries_an_iso_date(self, blocks): # The server parses `dd-mm-yyyy` and answers an ISO date 400, measured # 2026-09-09 by `--phase 10`. The examples carried ISO until that day. @@ -191,10 +224,18 @@ def test_no_statistics_example_carries_an_iso_date(self, blocks): # whole file would have to be written to permit the very string it is # looking for. A test that has to make an exception for the correct case # is one edit away from making it for the wrong one. + # + # This matched `start_date=`/`end_date=` when it was written, because it + # was written from the examples rather than from the API. Those names + # are not filters this server has, so the test pinned the date format of + # two arguments that did nothing - and the moment they were corrected it + # would have matched nothing and passed on an empty list. The `assert + # dated` guard below is the only reason that would have been noticed, + # and it is why a scan-and-check test needs one. dated = [ value for _language, body in blocks - for value in re.findall(r"(?:start|end)_date=\"([^\"]+)\"", body) + for value in re.findall(r"\b(?:start|end)=\"([^\"]+)\"", body) ] assert dated, "no dated example was found, so nothing was checked" for value in dated: @@ -203,6 +244,22 @@ def test_no_statistics_example_carries_an_iso_date(self, blocks): f"dd-mm-yyyy and answers ISO 400" ) + def test_no_statistics_example_omits_the_range_and_the_period(self, blocks): + # All three statistics endpoints answer **500** - not 400 - when + # `start`, `end` and `period` are all omitted, measured 2026-09-09 by + # `--phase 10`, though the document marks all three optional. So an + # example calling one with the username alone is an example of a 500. + # `domain_statistics("acct-1")` was exactly that until this test. + for _language, body in blocks: + for call, args in re.findall( + r"client\.(statistics_\w+|domain_statistics)\(([^)]*)\)", body + ): + keywords = set(re.findall(r"(\w+)=", args)) + assert keywords & {"start", "end", "period"}, ( + f"{call} is shown with no start=, end= or period=, which " + f"this server answers 500" + ) + class TestTheParameterTable: def test_every_known_parameter_has_a_row(self, readme: str):