diff --git a/CHANGELOG.md b/CHANGELOG.md index f651b43ea..ad98926ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The public API snapshot records this version update; publication remains pending ### Added +- Python `SDMClient.for_databricks_serving(max_request_bytes=...)` sets the most bytes one wrapped call may take, the gateway's 16 MB by default; a call past it is split before it is sent. - Python `Schema.from_relbench` and Java `Schema.fromRelbench` load the same pinned RelBench datasets with verified caches, offline mode and declared schema metadata. - Python and Java requests and fitted handles expose immutable sanitization reports for in-memory relational preparation, with per-table row exclusions and explicit unavailable/not-applicable states. @@ -50,6 +51,11 @@ The public API snapshot records this version update; publication remains pending ### Changed +- A single-table request is scored in calls of 10,000 rows, each with the whole context, split further as a relational request is. +- A `500` is retried with backoff as a `502` or `503` is, at the same size; through Databricks model serving, a `500` or `502` once more. +- `rows_per_call` is taken as asked rather than lowered to 10,000 (200 for a link prediction), which is now only its default. +- `num_context_samples` past 10,000 and more than 6 hops are accepted. + - Sampled Tabular context uses stable Rust index ordering, including duplicate labels and mixed numeric labels. Equal labels retain sampled order; mixed numbers compare by their represented values. Context order and predictions can differ from platform-specific NumPy ordering; sampled membership and warehouse sampling are unchanged. @@ -64,6 +70,13 @@ The public API snapshot records this version update; publication remains pending Endpoint capabilities and ordinary request limits still apply. The singular raw `result.explanation` view remains supported. +### Removed + +- The client's own size limits: the 10,000 context rows, the 30 MiB request body, the 100,000 entities and 100 calls of one `materialize()`, and the 15 tables and 6 hops of a relational request. + The client sends its best attempt: a call the endpoint refuses as too large (`413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB) or times out on (`504`, a timeout) is asked again in halves, each with the whole context, at most 3 times per batch, then refused naming its rows, bytes and status. A session's create call, which carries only the context, is refused at once when too large; a session's scoring call is halved as a prediction is. A `500`, `502` or `503` is retried at the same size and never split. + A `422`, or a refusal of a call of one row, is shown as the endpoint said it. + The context is sampled only to a `num_context_samples` the caller passes; through Databricks model serving, a call is still split up front to fit its 16 MB. + ### Fixed - Explicit empty outputs remain invalid with `EXPLAIN`; unsupported local-runtime outputs are refused before model weights load. diff --git a/contract/cases/s1/multicall.calls.101.json b/contract/cases/s1/multicall.calls.101.json index 804d9d27b..2d2ab3d32 100644 --- a/contract/cases/s1/multicall.calls.101.json +++ b/contract/cases/s1/multicall.calls.101.json @@ -579,6 +579,6 @@ "code": "UNSUPPORTED_REQUEST", "message": "101 entities are gathered as 101 calls of 1, and every one is built, with its own context, before any is sent. That is past the 100 calls this will hold at once; ask about fewer entities at a time" }, - "why": "rows_per_call=1 over 101 entities: 101 calls, one past the 100 one request is asked in; refused UNSUPPORTED_REQUEST at materialize", + "why": "rows_per_call=1 over 101 entities: 101 calls, one past the 100 the reference asked one request in; the reference refused it UNSUPPORTED_REQUEST at materialize; this client sets no most on calls and asks all 101 (contract/expected)", "source": "Correctness case multicall.calls.101 using the shop fixture." } diff --git a/contract/cases/s1/multicall.entities.100000.json b/contract/cases/s1/multicall.entities.100000.json index 6e732e562..ad74de405 100644 --- a/contract/cases/s1/multicall.entities.100000.json +++ b/contract/cases/s1/multicall.entities.100000.json @@ -69,7 +69,7 @@ "outcome": "success", "stage": "materialize" }, - "why": "100,000 entities, the most one materialize() gathers: gathered as ten calls of 10,000 (until materialize, so no payload of 100,000 rows is recorded); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", + "why": "100,000 entities, the most the reference's materialize() gathered (this client sets no most): gathered as ten calls of 10,000 (until materialize, so no payload of 100,000 rows is recorded); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", "source": "Correctness case multicall.entities.100000 using the shop fixture.", "input_factory": "entity_limit_100000" } diff --git a/contract/cases/s1/multicall.entities.100001.json b/contract/cases/s1/multicall.entities.100001.json index f78f7c05b..07b29b776 100644 --- a/contract/cases/s1/multicall.entities.100001.json +++ b/contract/cases/s1/multicall.entities.100001.json @@ -72,7 +72,7 @@ "code": "UNSUPPORTED_REQUEST", "message": "100,001 entities is past the 100,000 one materialize() will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time" }, - "why": "100,001 entities, one past the 100,000 one materialize() gathers: refused UNSUPPORTED_REQUEST from the count, before anything is read; user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", + "why": "100,001 entities, one past the 100,000 the reference's materialize() gathered: the reference refused it UNSUPPORTED_REQUEST from the count; this client sets no most and gathers it (contract/expected); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", "source": "Correctness case multicall.entities.100001 using the shop fixture.", "input_factory": "entity_limit_100001" } diff --git a/contract/cases/s1/rank.entities-by-candidates.over.json b/contract/cases/s1/rank.entities-by-candidates.over.json index c8bf3cdfd..41c9341bc 100644 --- a/contract/cases/s1/rank.entities-by-candidates.over.json +++ b/contract/cases/s1/rank.entities-by-candidates.over.json @@ -76,6 +76,6 @@ "error": "StructuredDataError", "code": "UNSUPPORTED_REQUEST" }, - "why": "The entity ceiling over a ranking (main's MOST_ENTITIES, 100,000): the entities gathered once per candidate, 501 by 200, one past it by 200, refused before anything is read", + "why": "The reference's entity ceiling over a ranking (main's MOST_ENTITIES, 100,000): the entities gathered once per candidate, 501 by 200, one past it by 200, refused before anything is read; this client sets no such ceiling, and refuses the call only for the entities that name no user (contract/expected)", "source": "Correctness case rank.entities-by-candidates.over using the shop fixture." } diff --git a/contract/cases/s1/sample.num-hops.seven.json b/contract/cases/s1/sample.num-hops.seven.json index bcd5a8467..3b20bbc5e 100644 --- a/contract/cases/s1/sample.num-hops.seven.json +++ b/contract/cases/s1/sample.num-hops.seven.json @@ -87,6 +87,6 @@ "code": "UNSUPPORTED_REQUEST", "message": "this request samples 7 hops and the endpoint reads at most 6" }, - "why": "num_hops=7 is past the six hops the endpoint reads: refused UNSUPPORTED_REQUEST at materialize", + "why": "num_hops=7 is past the six hops the reference's endpoint read: the reference refused it UNSUPPORTED_REQUEST at materialize; this client sets no most on hops and gathers it (contract/expected)", "source": "Correctness case sample.num-hops.seven using the shop fixture." } diff --git a/contract/expected/README.md b/contract/expected/README.md index c3422e18f..9bf9135b3 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -29,6 +29,13 @@ Their differences from DuckDB follow the fixture loader's explicit mappings in ` | SQLite / Postgres | Malformed text-time values are refused; the US-date fixture completes under these backends' existing read behavior | | Postgres | NaN-key fixtures refuse at materialize; a negative neighbor bound retains the existing `PROVIDER_ERROR` outcome | +`sample.num-hops.seven` and `sample.num-neighbors.seven-hops` now reach predict on every backend: the client sets no most on hops, since the model reads as many as a request carries, where the reference refused past six. +The `multicall.body-30mib` cases now reach predict on every backend: the client holds a call to no size of its own, and a call the endpoint refuses as too large is asked again in halves. +`multicall.entities.100001` and `multicall.calls.101` are now gathered: the client sets no most on the entities or the calls of one `materialize()`. +`rank.entities-by-candidates.over` is now refused at materialize only for the entities that name no user, as `rank.entities-by-candidates.at` is, since the client sets no entity ceiling over a ranking. +`multicall.fail-call-0`, `multicall.fail-call-2` and `multicall.forecast.fail-call-1` now succeed: their scripted endpoint fails one call with a `500`, which the client tries once more. +These outcomes were recorded from the runner's observations (`run_python.py`, then `check_observations.py --expect`) on DuckDB and SQLite; the Postgres entries take the same outcomes, as no Postgres server was available. + These expectations describe the declared fixture mappings; they do not claim that every backend supports identical physical types. `explain.two-entities` now succeeds on all three backends: The client supports explanations for multiple prediction rows. `contract/tests/test_query_options.py::test_each_explained_entity_retains_its_own_raw_response` independently checks both rows' distinct explanation values, the request batch size and the public success outcome. diff --git a/contract/expected/duckdb-python.json b/contract/expected/duckdb-python.json index 2a5376ff1..1ef2af283 100644 --- a/contract/expected/duckdb-python.json +++ b/contract/expected/duckdb-python.json @@ -108,17 +108,17 @@ "metadata.items": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-across-calls": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -247,7 +247,7 @@ "sample.num-hops-3": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -256,7 +256,7 @@ "sample.num-neighbors.minus-one": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.number": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/postgres-python.json b/contract/expected/postgres-python.json index de5fe0707..f4d7f3ff6 100644 --- a/contract/expected/postgres-python.json +++ b/contract/expected/postgres-python.json @@ -108,17 +108,17 @@ "metadata.items-across-calls@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000@postgres": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001@postgres": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -245,7 +245,7 @@ "sample.num-hops-3@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -254,7 +254,7 @@ "sample.num-neighbors.minus-one@postgres": {"code": "PROVIDER_ERROR", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.number@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.relbench-avito@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/sqlite-python.json b/contract/expected/sqlite-python.json index 209fbf7ab..f8d2d832b 100644 --- a/contract/expected/sqlite-python.json +++ b/contract/expected/sqlite-python.json @@ -108,17 +108,17 @@ "metadata.items-across-calls@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000@sqlite": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001@sqlite": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -229,7 +229,7 @@ "sample.num-hops-3@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -238,7 +238,7 @@ "sample.num-neighbors.minus-one@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.number@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.relbench-avito@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/runners/check_observations.py b/contract/runners/check_observations.py index 0d6fdeaf8..7e653f7c2 100644 --- a/contract/runners/check_observations.py +++ b/contract/runners/check_observations.py @@ -215,9 +215,14 @@ def check( for entry in (item.get('public') or {}).get('payloads', []) ] paths = (('metadata', 'request_id'),) - if [_payload(value, paths) for value in calls] != [ - _payload(value, paths) for value in received - ]: + # A body the transport sent again, a 500 tried once more, is the + # one call the core made. + sent = [] + for value in received: + value = _payload(value, paths) + if not sent or sent[-1] != value: + sent.append(value) + if [_payload(value, paths) for value in calls] != sent: failures.append( 'endpoint payload differs from native prediction' ) diff --git a/contract/support/error_wording_scenarios.py b/contract/support/error_wording_scenarios.py index b0b407189..67e5ff548 100644 --- a/contract/support/error_wording_scenarios.py +++ b/contract/support/error_wording_scenarios.py @@ -68,12 +68,10 @@ 'sample.max-pq-iterations.zero', 'sample.num-hops.bool', 'sample.num-hops.fraction', - 'sample.num-hops.seven', 'sample.num-hops.text', 'sample.num-hops.zero', 'sample.num-neighbors.empty', 'sample.num-neighbors.number', - 'sample.num-neighbors.seven-hops', 'sample.seed.negative', 'shape.star.customers', 'target.filter.sum', diff --git a/contract/tests/test_contract_loader.py b/contract/tests/test_contract_loader.py index b8658b923..c82849c7a 100644 --- a/contract/tests/test_contract_loader.py +++ b/contract/tests/test_contract_loader.py @@ -521,6 +521,8 @@ def test_entity_limit_factory_expands_exact_keys_and_preserves_source_digest( assert case['core']['materialize']['indices'] == [ {'t': 'i64', 'v': str(i)} for i in range(count) ] + # The case states the reference's outcome, which refused 100,001; this + # client gathers it, a departure recorded in contract/expected. assert case['expect']['outcome'] == ( 'success' if count == 100_000 else 'refusal' ) diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 7105edb25..8374f8ad5 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -102,6 +102,17 @@ 'the call: materialize(rows_per_call=N) below the 3 this one holds' ), } +#: The cases this client departs from the reference on, on purpose: it holds a +#: request to no size of its own (no entity, call or body ceiling), so each is +#: gathered where the reference refused it at materialize; and it tries a call +#: the endpoint failed with a 500 again, so a scripted one-off failure is +#: answered where the reference ended the request. +DEPARTED = ( + 'multicall.entities.100001', + 'multicall.calls.101', + *BODY, + *FAILED, +) CASE_IDS = ( 'multicall.rows-per-call-3', 'multicall.fail-call-2', @@ -609,6 +620,8 @@ def test_python_core_answers_each_case_as_the_reference_does( database = case_key.partition('@')[2] or 'duckdb' if database == 'postgres' and not _server(): return + if case_key.split('@')[0] in DEPARTED: + pytest.skip('a departure from the reference, checked on its own') def check() -> None: run = core_run(database) @@ -656,6 +669,8 @@ def test_python_core_carries_what_the_reference_carries(case_key: str) -> None: case_id, _, database = case_key.partition('@') if database == 'postgres' and not _server(): return + if case_id in DEPARTED: + pytest.skip('a departure from the reference, checked on its own') def check() -> None: want = vectors()['vectors'][case_key] @@ -780,3 +795,13 @@ def check() -> None: assert not found, f'{case_key}: ' + '; '.join(found) check() + + +@pytest.mark.parametrize('case_id', DEPARTED) +def test_python_core_gathers_and_predicts_what_the_reference_refused( + case_id: str, +) -> None: + r"""The client sets no most on the entities, the calls or the body, and + tries a call failed with a 500 again: each case is answered.""" + got = core_carried(case_id) + assert got['outcome']['result'] == 'success', got['outcome'] diff --git a/contract/tests/test_public_error_messages.py b/contract/tests/test_public_error_messages.py index c63b65eda..28869d3dc 100644 --- a/contract/tests/test_public_error_messages.py +++ b/contract/tests/test_public_error_messages.py @@ -111,12 +111,10 @@ def runner() -> ModuleType: 'sample.max-pq-iterations.zero': ['max_pq_iterations', '0'], 'sample.num-hops.bool': ['num_hops', 'True'], 'sample.num-hops.fraction': ['num_hops', '1.5'], - 'sample.num-hops.seven': ['7 hops', '6'], 'sample.num-hops.text': ['num_hops', "'2'"], 'sample.num-hops.zero': ['num_hops', 'num_neighbors=[8, 8]'], 'sample.num-neighbors.empty': ['num_neighbors', 'no hop'], 'sample.num-neighbors.number': ['num_neighbors', 'sequence'], - 'sample.num-neighbors.seven-hops': ['7 hops', '6'], 'sample.seed.negative': ['random_seed', '-1'], 'wording.rank-text-candidate': ['buys.item_id', "'1'"], 'wording.float32-classification': ['non-finite', 'inf'], diff --git a/contract/tests/test_ranking.py b/contract/tests/test_ranking.py index 59f86c088..29fb80fc3 100644 --- a/contract/tests/test_ranking.py +++ b/contract/tests/test_ranking.py @@ -51,8 +51,8 @@ 'holds one class left out, unscored, before any call', 'rank.all-unscored': 'no candidate left to rank, refused before any call', 'rank.entities-by-candidates.over': 'the entities gathered once ' - 'per candidate past the 100,000 one materialize holds (501 by 200), ' - 'refused before anything is read', + "per candidate past the reference's 100,000 (501 by 200), which it " + 'refused; this client sets no such ceiling (DEPARTED)', 'rank.entities-by-candidates.at': 'the same at exactly 100,000 ' '(500 by 200), not refused by that ceiling', 'forecast.top-k': "the forecast's top_k refusal in the core", @@ -373,6 +373,12 @@ def test_the_vectors_are_the_reference_s_and_not_vacuous() -> None: } +#: The cases this client departs from the reference on, on purpose: it sets +#: no ceiling on the entities a ranking gathers across its candidates, so the +#: reference's refusal past 100,000 is gathered, and refused only for the +#: entities that name no user, as at the ceiling. +DEPARTED = ('rank.entities-by-candidates.over',) + #: What the reference's refusal of each case refused before any call says, #: so it is refused for the argument the case names. REFUSED_FOR = { @@ -531,6 +537,8 @@ def test_python_core_answers_each_case_as_the_reference_does( database = case_key.partition('@')[2] or 'duckdb' if database == 'postgres' and not _server(): return + if case_key.split('@')[0] in DEPARTED: + pytest.skip('a departure from the reference, checked on its own') def check() -> None: run = core_run(database) @@ -658,3 +666,19 @@ def check() -> None: held_to(recorder().observe(inputs), spec['expected'], name) check() + + +@pytest.mark.parametrize('case_id', DEPARTED) +def test_python_core_gathers_past_the_reference_s_entity_ceiling( + case_id: str, +) -> None: + r"""Past the reference's 100,000, the ranking is gathered and refused + only for the entities 8 to 500, which name no user, as at the ceiling: + the caller's ``INVALID_REQUEST``, not the ceiling's + ``UNSUPPORTED_REQUEST``.""" + line = core_run('duckdb').get(case_id) + assert line is not None, f'{case_id}: not observed' + assert (line['outcome']['result'], line['outcome']['code']) == ( + 'refusal', + 'INVALID_REQUEST', + ) diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 4424d8c2d..34286e76a 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -1,125 +1,52 @@ # Limits -The endpoint enforces several size limits. `materialize()` checks them before -sending, then refuses or samples while naming the limit. Otherwise, a rejected -request returns a 500 or 422 that names neither the limit nor how to meet it. - -`Request(context=..., predict=...)` receives exact frames and does not trim -them. It refuses a context above the model's cap instead of discarding named -rows, and reports the count and cap. - -`materialize()` keeps the context under each model's cap. On `kumo-tabular`, it -samples to the cap and reports the row count. On `kumo-relational`, it gathers -the context. It refuses `num_context_samples` above the cap. Omitting the -argument gathers and reports the 1,000-row budget; it never samples down to -10,000. - -## Sizes - -Both endpoints enforce sizes, and most of these are the relational one's. - -| Limit | Model | Value | Checked | -|---|---|---|---| -| context rows | relational | 1,000 by default, up to 10,000 with `num_context_samples` | at `materialize()` | -| context rows | tabular | 10,000 | sampled at `materialize()`, refused before sending on a direct `Request` | -| rows scored per call | relational | 10,000, or 200 for link prediction | more are gathered as several calls at `materialize()` | -| entities one question gathers | relational | 100,000, in at most 100 calls | at `materialize()`, and this client's own: the endpoint is never asked for more than one call's worth | -| calls a question asked in parts makes | relational | 200, one per candidate or step | at `materialize()`, as `sdm_client.decompose.MOST_PARTS` | -| request body | relational | 30 MiB per call | at `materialize()` | -| request body | tabular or relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | before sending, on the wrapped envelope | -| gradient explanations | relational and tabular | endpoint-dependent; no separate client row/context limit | endpoint validates support | -| hops | relational | 6 | at `materialize()` | -| tables | relational | 15 | against the graph the request carries, not the schema it was built from | - -A client opened with `SDMClient.in_process()` is held to this table as well, -since this client checks it. A NIM also has limits of its own, which each -deployment sets, such as the longest string a cell may hold or the most -distinct values a column may have. Those protect a shared server, and are -not applied when the model runs in your own process. - -A relational request for more entities than one call scores is not refused. -`materialize()` gathers it in batches of at most that many, each with a -context of its own, sampled around that batch's rows rather than copied from -the first, and `predict()` sends one call per batch and returns one result. -Its `row_index` counts across the whole request rather than from zero per -call, so `to_pandas()`, which orders by it, gives the indices back in the -order they were asked for. A session scores those batches against the context -it pinned, so the context is uploaded once rather than once per call. `fit()` -itself takes a request of one call, since a session pins one context: fit a -small request and score the batched one against the handle. - -**The 10,000 is the deployment's, not this client's.** It is the serving -layer's `TFM_MAX_TEST_ROWS`, whose default it is, so a deployment may hold to -less. Measured at the boundary rather than read from a document: 10,000 rows -to score are answered and 10,001 come back as - -``` -422 UNSUPPORTED_SHAPE N_test 10001 exceeds limit 10000 -``` - -The earlier `kumo-relational` container holds to 1,000 instead, through a -model-specific admission check of its own, so a request gathered at this -default is refused there on its first call. - -Pass `rows_per_call` to `materialize()` to ask in smaller batches than that. -It only ever goes down, since a larger call buys a 422 rather than a bigger -call. Two reasons to reach for it: a graph wide enough that a full batch -outruns the 30 MiB body, and a deployment whose cap is lower than the one -above. - -It is not the answer to every 30 MiB refusal. A batch carries its context as -well as its rows, so where the context is itself over the limit, no batch is -small enough: a wide graph at 10,000 context rows was refused at every batch -size tried, and the same graph at the default context was gathered as 15 -calls of 200. `num_context_samples`, `num_neighbors` and the columns the graph -carries are what come down in the first case. - -**That 100,000 is this client's, not the endpoint's.** The endpoint is never -asked for more than one call holds. Every batch is built before the first is -sent, each with its own context, so the whole population is held in memory at -once, and 100,000 entities is about a gigabyte of it. One `materialize()` -gathers at most that many, in at most 100 calls, and the refusal names -whichever of the two it met. The second is what a small `rows_per_call` -reaches, and what a temporal link prediction reaches at 20,001 entities, since -it scores 200 a call. - -Both govern the batches of one question, and neither counts the parts of a -question asked in several. A ranking is one call per candidate and a forecast -one per step, and what bounds those is `sdm_client.decompose.MOST_PARTS`, 200, -which is why a 101-candidate ranking makes 101 calls without meeting the 100 -above. What the two ceilings hold for such a question is the population each -part gathers, counted across all of them. - -A call that fails ends the request. The answers already paid for are carried -on the error as `details['answered']`, and the rows they cover as -`details['rows_answered']`, so the rest can be asked for again without buying -the whole population twice. A forecast keeps its answered steps the same way, -under `details['answered']`, numbered by step, with how many of them under -`details['parts_answered']`; a ranking keeps only the count, because an order -over some of its candidates is a different answer rather than part of the one -that was asked for. - -**Batch size is chosen from the row count, not from the bytes.** The 30 MiB -body is checked per batch at `materialize()`, before anything is sent. -Databricks model serving weighs its own wrapped envelope against a smaller -16 MB, and only at send time, so a wide graph can pass `materialize()` and be -refused on the first call. Ask in smaller calls there, with `rows_per_call`, -unless the context alone is over the gateway's limit, which a single-entity -request can be: then it is `num_context_samples`, `num_neighbors` or the -columns the graph carries, as it is for the 30 MiB body. - -**A question asked in parts is not batched.** A ranking is asked as one call -per candidate and a forecast as one per step, and `MOST_PARTS` counts those -calls, so a batched part would send the parts times the batches: a ranking -over 200 candidates and 50,000 entities would pass as 200 calls and make -1,000. More entities than one call scores is refused for both, before -anything is gathered. A ranking could not be batched in any case, since a -candidate is reported unscored for the whole ranking rather than per batch, so -one batch whose context held a single class would drop it from every row. - -The tabular cap was measured at the boundary rather than read from a document: -10,000 context rows are answered and 10,001 come back as -`422 UNSUPPORTED_SHAPE N_train 10001 exceeds limit 10000`. +The client holds a request to no size of its own. +It sends its best attempt and recovers from what the endpoint says: + +| The endpoint answers | The client | +|---|---| +| `413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB, a `504`, or the call times out | asks the call's rows again in two halves at once, each with the whole context, and keeps that smaller size for the request's later calls; a batch is halved at most 3 times, to an eighth of its rows, and then refused naming the call's rows, bytes and status | +| `503` or `429` | waits as `Retry-After` says and tries again at the same size | +| a connection error, a `500` or a `502` | tries again at the same size with backoff, up to `max_retries` (3 by default), and never splits; through Databricks model serving, a `500` or `502` is tried once more | +| `422`, or a refusal of a call of one row | does not try again, and shows the endpoint's message | + +What a call too large or too slow comes to depends on what it carries: + +| Call | Too large or too slow | +|---|---| +| a prediction, context and rows to score | its rows are halved, at most 3 times, each smaller call with the whole context; then refused, naming the rows, bytes and status | +| a session's create (`fit()`, or an automatic batch session), the context alone | refused at once: the context is too large for the endpoint; send fewer context rows or columns | +| a session's scoring call, rows to score alone | its rows are halved, at most 3 times | + +A temporary failure is tried again at the same size on every call; none is split. +The context is never split. +The context is sampled only to a `num_context_samples` you pass. +`timeout` and `max_retries` are optional settings of the client. + +Through Databricks model serving, an oversized call is dropped as an opaque connection error, so `predict()` splits a call whose wrapped envelope is past the endpoint's `max_request_bytes` (16 MB, the gateway's own decimal limit) before sending it. +A call of one row past it is refused, since only a smaller context helps. +There, a deadline spent waiting on a `429`, a `503` or a connection is reported as a `503` and not split; a `504` is a call that timed out. + +## Batches + +A relational request is gathered in batches of 10,000 rows to score, or 200 for a link prediction: a default, not a cap. +Pass `rows_per_call` to `materialize()` to ask in batches of another size; it is taken as asked. +A single-table request is scored in calls of 10,000 rows too, each with the whole context, and the answers are put back together as one. +Each batch has a context of its own, sampled around that batch's rows rather than copied from the first, and `predict()` sends one call per batch and returns one result. +Its `row_index` counts across the whole request rather than from zero per call, so `to_pandas()`, which orders by it, gives the indices back in the order they were asked for. +A session scores those batches against the context it pinned, so the context is uploaded once rather than once per call. +`fit()` itself takes a request of one call, since a session pins one context: fit a small request and score the batched one against the handle. + +Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once; the client sets no most on the entities or the calls. +A question asked in parts is different: a ranking is one call per candidate and a forecast one per step, bounded by `sdm_client.decompose.MOST_PARTS`, 200, a limit on the wait rather than on the size. +A question asked in parts is not batched by rows, since a batched part would send the parts times the batches. + +A call that fails ends the request. +The answers already paid for, the halves of a split call included, are carried on the error as `details['answered']`, and the rows they cover as `details['rows_answered']`, so the rest can be asked for again without buying the whole population twice. +A forecast keeps its answered steps the same way, under `details['answered']`, numbered by step, with how many of them under `details['parts_answered']`; a ranking keeps only the count, because an order over some of its candidates is a different answer rather than part of the one that was asked for. + +The relational model reads as many tables and hops as a request carries. +A walk that reaches no new row ends there, so a hop count past the data's depth costs nothing. ## Aggregation windows open after the anchor and close on their last day @@ -151,13 +78,11 @@ values. Omit it on a relational request to use the smaller budget, up to 1,000 examples. This is the endpoint's long-standing default. Omit it on a single-table `materialize()` with either `indices=` or `predict=` to use every -labelled row, up to the endpoint's 10,000-row limit. Above that, it samples -10,000 rows and reports doing so. Use the argument to request more. +labelled row. Ask for more than exists and you get what exists; there are only so many -labelled examples, and that is not an error. Ask for more than 10,000 and you -are told the ceiling rather than silently given fewer. Zero, a negative, a -fraction or a string are refused by name. +labelled examples, and that is not an error. Zero, a negative, a fraction or a +string are refused by name. **Which examples you get** when the count narrows: the first `n` of what was gathered, and for a temporal question that is **not a sample across time**. diff --git a/java/src/main/java/com/nvidia/sdm/ClientOptions.java b/java/src/main/java/com/nvidia/sdm/ClientOptions.java index 292fee843..7a412162b 100644 --- a/java/src/main/java/com/nvidia/sdm/ClientOptions.java +++ b/java/src/main/java/com/nvidia/sdm/ClientOptions.java @@ -64,7 +64,7 @@ public Builder timeout(Duration timeout) { return this; } - /** How many times a transient failure (408, 429, 502, 503, 504, a dropped connection) is retried. */ + /** How many times a transient failure (429, 500, 502, 503, a failed connection) is retried. */ public Builder maxRetries(int maxRetries) { this.maxRetries = maxRetries; return this; diff --git a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java index edfa4e823..c2144df7e 100644 --- a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java +++ b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java @@ -202,11 +202,13 @@ boolean healthReady() throws Host.EndpointFailed, Host.TransportFailed { private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.TransportFailed { long deadline = System.nanoTime() + timeout.toNanos(); + boolean retried500 = false; for (int attempt = 1; ; attempt++) { Authentication granted = authenticate(deadline); long remaining = deadline - System.nanoTime(); if (remaining <= 0) { - throw new Host.EndpointFailed(504, new byte[0]); + // The deadline spent before the call was made: the endpoint unavailable. + throw new Host.EndpointFailed(503, new byte[0]); } // Native SDK authentication can discover the workspace host. Validate it afterwards, // before any credential is sent, and on every attempt if the caller edits its config. @@ -236,11 +238,19 @@ private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.T if (status >= 200 && status < 300) { return response.body(); } - if (status != 429 && status != 503) { + // A 429 or a 503 is tried again until the deadline, as the SDK tries it; a + // 500 or a 502 once more, as Python's transport tries it, then reported as it is. + if ((status == 500 || status == 502) && !retried500) { + retried500 = true; + } else if (status != 429 && status != 503) { throw new Host.EndpointFailed(status, response.body()); } pause = retryAfter(response.headers().firstValue("Retry-After").orElse(null)); + } catch (java.net.http.HttpConnectTimeoutException e) { + // No connection made: tried again as any connection is, never split. + pause = Math.min(10, attempt) * 1000L; } catch (HttpTimeoutException e) { + // The call itself took too long: a 504, which the core splits on. throw new Host.EndpointFailed(504, new byte[0]); } catch (IOException e) { pause = Math.min(10, attempt) * 1000L; @@ -250,8 +260,9 @@ private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.T } pause += ThreadLocalRandom.current().nextLong(1000); if (System.nanoTime() + pause * 1_000_000L >= deadline) { - // The SDK's retries end as its own timeout, which Python reports as 504. - throw new Host.EndpointFailed(504, new byte[0]); + // The deadline spent waiting on a 429, a 503, a 500 or a connection: the + // endpoint is unavailable, a 503, which no smaller call fixes. + throw new Host.EndpointFailed(503, new byte[0]); } try { Thread.sleep(pause); diff --git a/java/src/main/java/com/nvidia/sdm/SdmClient.java b/java/src/main/java/com/nvidia/sdm/SdmClient.java index 6606e187d..a59395e72 100644 --- a/java/src/main/java/com/nvidia/sdm/SdmClient.java +++ b/java/src/main/java/com/nvidia/sdm/SdmClient.java @@ -206,7 +206,10 @@ public byte[] call(Object endpoint, byte[] body) throws Host.TransportFailed, Ho if (serving != null) throw new StructuredDataException("this transport does not support sessions", "SESSIONS_UNSUPPORTED", Map.of(), null); try { return transport.call(endpoint, body); } - catch (HttpTransport.Failed e) { throw new Host.TransportFailed(); } + catch (HttpTransport.Failed e) { + if (e.status() != null) throw smaller(e); + throw new Host.TransportFailed(); + } catch (ModelEndpointException e) { throw new Host.Refused(e::with, e.status()); } } }; @@ -268,12 +271,25 @@ private byte[] call(byte[] body) try { return transport.predict(body); } catch (HttpTransport.Failed failed) { + if (failed.status() != null) throw smaller(failed); throw new Host.TransportFailed(); } catch (ModelEndpointException refused) { throw new Host.Refused(refused::with, refused.status()); } } + /** + * A failed transport a smaller call can fix (a timeout, an answer too large), told to the core by + * its status, so the core asks the rows in halves; still a transport error if no smaller call + * answers. + */ + private static Host.Refused smaller(HttpTransport.Failed failed) { + return new Host.Refused( + answered -> new StructuredDataException( + failed.getMessage(), "TRANSPORT_ERROR", answered, "model_endpoint", false, null), + failed.status()); + } + /** A call's id, as the adapter draws one: {@code sdm-} and twelve hexadecimal digits. */ static String requestId() { byte[] random = new byte[6]; diff --git a/java/src/main/java/com/nvidia/sdm/internal/Host.java b/java/src/main/java/com/nvidia/sdm/internal/Host.java index 1807bc2db..9ec069a81 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Host.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Host.java @@ -409,7 +409,8 @@ private static Finished steps(Run run, Call call) { } } StructuredDataException error = error(failed, call.held(), details); - if (refused != null && (!cleanupRefusal + // A call the core split and gave up on is the core's to name. + if (refused != null && !Boolean.TRUE.equals(error.details().get("after_split")) && (!cleanupRefusal || (refused.status != null && error.details().get("status") instanceof Number status && status.intValue() == refused.status) || (refused.status == null && "TRANSPORT_ERROR".equals(error.code())))) { throw carried(refused, error); diff --git a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java index 2677af03e..d5b5db292 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java +++ b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java @@ -31,7 +31,7 @@ * Transport} ({@code sdm_client/core/transport.py}): it posts the core's call to {@code * /v1/predictions} as the bytes the core wrote, never parsing or re-encoding them (design choice * 10, "Model calls"), and gives back the endpoint's answer as the bytes it sent, which the core - * reads. A transient failure (a status of 429, 502, 503 or 504, or a connection that failed) is + * reads. A transient failure (a status of 429, 500, 502 or 503, or a connection that failed) is * tried again with exponential backoff, a {@code Retry-After} the endpoint sends honoured up to a * minute; the timeout bounds each attempt whole, its answer's body included; a status of 400 or more is the endpoint's refusal ({@link ModelEndpointException}); an * endpoint that cannot be reached, or an answer past 64 MiB or that is not a JSON object, is a @@ -42,7 +42,11 @@ public final class HttpTransport { public static final String PREDICTIONS = "/v1/predictions"; /** The statuses tried again. */ - static final Set RETRY_STATUS = Set.of(408, 429, 502, 503, 504); + /** + * Tried again at the same size: overload, and a server or gateway fault. A 408, 413 or 504, or a + * call that timed out, is not: the core asks its rows in smaller calls at once. + */ + static final Set RETRY_STATUS = Set.of(429, 500, 502, 503); /** The largest answer read. */ static final int MAX_RESPONSE_BYTES = 64 * 1024 * 1024; @@ -61,9 +65,23 @@ public final class HttpTransport { /** The transport could not reach the endpoint or read its answer. */ public static final class Failed extends Exception { private static final long serialVersionUID = 1L; + private final Integer status; Failed(String message) { + this(message, null); + } + + Failed(String message, Integer status) { super(message); + this.status = status; + } + + /** + * The status the core is told, so it asks the call's rows in smaller calls: 408 for an answer + * slower than the timeout, 413 for one past {@link #MAX_RESPONSE_BYTES}; none otherwise. + */ + public Integer status() { + return status; } } @@ -211,12 +229,16 @@ else if (endpoint instanceof java.util.Map map) { HttpResponse response; try { response = attempt(built); - } catch (HttpTimeoutException e) { + } catch (java.net.http.HttpConnectTimeoutException e) { + // No connection made: a failed transport, which no smaller call fixes. if (attempt < retries) { backoff(attempt, null); continue; } - throw new Failed("the request timed out after " + timeout + ", the timeout given"); + throw new Failed("the request failed: " + e.getClass().getSimpleName()); + } catch (HttpTimeoutException e) { + // Not tried again at its size: told as a 408, so the core asks the rows in halves. + throw new Failed("the request timed out after " + timeout + ", the timeout given", 408); } catch (IOException e) { if (attempt < retries) { backoff(attempt, null); @@ -247,13 +269,15 @@ else if (endpoint instanceof java.util.Map map) { * * @throws HttpTimeoutException the attempt did not end within the timeout * @throws IOException the endpoint could not be reached or its answer read - * @throws Failed the answer is past {@link #MAX_RESPONSE_BYTES}, or the call was interrupted + * @throws Failed the answer is past {@link #MAX_RESPONSE_BYTES} (its {@link Failed#status} a + * 413), or the call was interrupted */ private HttpResponse attempt(HttpRequest built) throws IOException, Failed { try { return exchange(client, built, info -> new Capped(), timeout.toNanos()); } catch (TooLarge e) { - throw new Failed("the response body exceeds the " + MAX_RESPONSE_BYTES + " byte limit"); + // Told as a 413, the call too large, so the core asks its rows in smaller calls. + throw new Failed("the response body exceeds the " + MAX_RESPONSE_BYTES + " byte limit", 413); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new Failed("the request was interrupted"); diff --git a/java/src/main/java/com/nvidia/sdm/internal/Wording.java b/java/src/main/java/com/nvidia/sdm/internal/Wording.java index 50ae80ad6..4a1158a6e 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Wording.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Wording.java @@ -99,7 +99,6 @@ private static Map, String>> advice() { + ") && List.of(" + String.join(", ", quotedAll(given.get(2))) + ").contains(link.sourceColumn()))).toList())"); - spelled.put("materialize_call", given -> "materialize()"); spelled.put("missing_time", given -> "null"); spelled.put( "name_candidates", @@ -135,9 +134,6 @@ private static Map, String>> advice() { given -> "Name another key with FromSourceOptions.builder().primaryKey(...), or pass" + " checkKeys(false) to skip (one full scan per table)."); - spelled.put( - "smaller_calls", - given -> "MaterializeOptions.builder().option(\"rows_per_call\", N)"); spelled.put("trust_expressions", given -> "semanticModel(name, true)"); return Map.copyOf(spelled); } diff --git a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java index 0f83ddb7f..5ee820cc8 100644 --- a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java +++ b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java @@ -48,8 +48,8 @@ static final class Endpoint implements AutoCloseable { scores++; scoredBodies.add(payload); if(mode.equals("transport") && scores>=2) { exchange.close();return; } if(mode.equals("expired") && scores==2) status=404; - else if(mode.equals("retry") && scores==1) status=408; - else if(mode.equals("fail") && scores==2) status=500; + else if(mode.equals("retry") && scores==1) status=503; + else if(mode.equals("fail") && scores==2) status=422; else if(mode.equals("malformed") && scores==2) answer=Map.of("predictions","invalid"); else { var table=object(object(payload.get("predict")).get("instance_table")); @@ -134,7 +134,7 @@ void livePreparedBatchPrediction() throws Exception { @Test void failedBatchKeepsPaidAnswersAndCloses() throws Exception { try(Endpoint endpoint=new Endpoint("fail"); SdmClient client=endpoint.client()) { var error=assertThrows(ModelEndpointException.class,()->client.predict(request(),SdmClient.KUMO_RELATIONAL)); - assertEquals(500,error.status()); assertEquals(2,endpoint.scores);assertEquals(1,endpoint.deletes); + assertEquals(422,error.status()); assertEquals(2,endpoint.scores);assertEquals(1,endpoint.deletes); assertNotNull(error.details().get("answered")); } } diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index ac1fc4ed6..adae7fd06 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -46,8 +46,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.table.Table; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -406,11 +408,11 @@ void utf8FeatureValuesUseTheSharedEnvelopeAndActualByteLimit() throws Exception assertTrue(new String(body, StandardCharsets.UTF_8).contains("é😀")); try (Workspace limited = new Workspace(); SdmClient capped = SdmClient.forDatabricksServing(limited.serving(body.length - 1))) { - StructuredDataException error = assertThrows(StructuredDataException.class, - () -> capped.predict(request, SdmClient.KUMO_RELATIONAL)); - assertEquals("INVALID_REQUEST", error.code()); - assertEquals(body.length, ((Number) error.details().get("bytes")).intValue()); - assertTrue(limited.bodies.isEmpty()); + // Past the endpoint's limit, the call is split before it is sent, + // each part within it. + capped.predict(request, SdmClient.KUMO_RELATIONAL); + assertTrue(limited.bodies.size() > 1); + for (byte[] part : limited.bodies) assertTrue(part.length <= body.length - 1); } } } @@ -461,7 +463,8 @@ void theFourBaseJ2ShapesUseTheSameCoreWithDirectAndServingEndpoints() throws Exc private static ModelEndpointException refused(Answer answer) throws Exception { try (Connection connection = Shop.open("duckdb"); - Workspace workspace = new Workspace(body -> answer); + // Answered twice: a 500 is tried once more, and a 502 asked in halves. + Workspace workspace = new Workspace(body -> answer, body -> answer); SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { Request request = asked(connection); StructuredDataException error = @@ -575,6 +578,38 @@ void aCallTheEndpointThrottlesIsTriedAgainWithTheSameBytes() throws Exception { } } + @Test + void a500OrA502IsTriedAgainAtItsSize() throws Exception { + for (int status : new int[] {500, 502}) { + try (Connection connection = Shop.open("duckdb"); + Workspace workspace = new Workspace(body -> Answer.of(status, "")); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); + assertEquals(2, workspace.bodies.size(), "status " + status); + assertArrayEquals(workspace.bodies.get(0), workspace.bodies.get(1), "never split"); + } + } + } + + @Test + void aCallTheGatewayRefusesAsTooLargeIsAskedInHalves() throws Exception { + String oversized = "{\"error_code\":\"BAD_REQUEST\",\"message\":\"Request size cannot exceed 16MB\"}"; + for (int status : new int[] {413, 400}) { + String text = status == 400 ? oversized : ""; + Request request; + try (RootAllocator allocator = new RootAllocator(); + Table context = TabularTest.table(allocator, true, 1); + Table predict = TabularTest.table(allocator, false, 2, 3)) { + request = new Request(context, predict, "y", "regression"); + } + try (Workspace workspace = new Workspace(body -> Answer.of(status, text)); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + assertEquals(2, client.predict(request, SdmClient.KUMO_TABULAR).rows()); + assertEquals(3, workspace.bodies.size(), "status " + status); + } + } + } + @Test void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { int port; @@ -591,7 +626,8 @@ void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { assertThrows( ModelEndpointException.class, () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); - assertEquals(504, error.status()); + // The deadline spent on connections never made: the endpoint unavailable, never split. + assertEquals(503, error.status()); assertNull(error.code()); } } diff --git a/java/src/test/java/com/nvidia/sdm/Shop.java b/java/src/test/java/com/nvidia/sdm/Shop.java index 738671908..25c29ef7d 100644 --- a/java/src/test/java/com/nvidia/sdm/Shop.java +++ b/java/src/test/java/com/nvidia/sdm/Shop.java @@ -216,7 +216,9 @@ static byte[] scripted(byte[] body) { for (int at = 0; at < rows.size(); at++) { Map item = new LinkedHashMap<>(); // The instance id as its text, as the scripted endpoint gives it. - item.put("id", String.valueOf(((List) rows.get(at)).get(id))); + if (id >= 0) { + item.put("id", String.valueOf(((List) rows.get(at)).get(id))); + } item.put("row_index", (long) at); item.put("prediction", prediction); if (call.get("output") instanceof Map output diff --git a/java/src/test/java/com/nvidia/sdm/TabularTest.java b/java/src/test/java/com/nvidia/sdm/TabularTest.java index 2564301ad..b491899ff 100644 --- a/java/src/test/java/com/nvidia/sdm/TabularTest.java +++ b/java/src/test/java/com/nvidia/sdm/TabularTest.java @@ -58,6 +58,8 @@ static final class Endpoint implements AutoCloseable { int deleteFailuresRemaining; boolean wrongId; boolean gradients; + int createStatus = 200; + int failScores; Endpoint() throws Exception { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); server.createContext("/", exchange -> { @@ -72,7 +74,11 @@ static final class Endpoint implements AutoCloseable { @SuppressWarnings("unchecked") Map body = (Map)Json.read(exchange.getRequestBody().readAllBytes()); calls.add(body); byte[] answer; - if (path.equals("/v1/sessions")) { + if (path.equals("/v1/sessions") && createStatus != 200) { + creates++; exchange.sendResponseHeaders(createStatus, -1); return; + } else if (path.endsWith("/predictions") && path.startsWith("/v1/sessions/") && failScores > 0) { + failScores--; exchange.sendResponseHeaders(413, -1); return; + } else if (path.equals("/v1/sessions")) { creates++; answer = Json.write(Map.of("session_id", "s" + creates)); } else { @SuppressWarnings("unchecked") Map predict = (Map)body.get("predict"); @@ -105,6 +111,60 @@ static final class Endpoint implements AutoCloseable { @Override public void close() { server.stop(0); } } + @Test void aSingleTableRequestIsScoredInCallsOf10000RowsEachWithTheWholeContext() throws Exception { + long[] scoring = java.util.stream.LongStream.range(100, 10_101).toArray(); + try (RootAllocator allocator = new RootAllocator(); + Table context = table(allocator, true, 1, 2, 3); + Table predict = table(allocator, false, scoring); + Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + Request request = new Request(context, predict, "y", "regression"); + assertEquals(10_001, client.predict(request, SdmClient.KUMO_TABULAR).rows()); + List sizes = new ArrayList<>(); + for (Map call : endpoint.calls) { + @SuppressWarnings("unchecked") Map half = (Map) call.get("predict"); + @SuppressWarnings("unchecked") Map rows = (Map) half.get("instance_table"); + sizes.add(((List) rows.get("rows")).size()); + assertEquals(endpoint.calls.get(0).get("context"), call.get("context")); + } + assertEquals(List.of(10_000, 1), sizes); + } + } + + @Test void aSessionWhoseContextIsTooLargeIsRefusedNamingIt() throws Exception { + try (RootAllocator allocator = new RootAllocator(); Table context = table(allocator, true, 1, 2); + Table predict = table(allocator, false, 3); Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + endpoint.createStatus = 413; + Request request = new Request(context, predict, "y", "regression"); + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> client.fit(request, SdmClient.KUMO_TABULAR)); + assertTrue(refused.getMessage().contains("the context is too large for this endpoint"), + refused.getMessage()); + assertEquals(1, endpoint.creates); + } + } + + @Test void aPinnedSessionsScoringCallIsAskedInHalves() throws Exception { + try (RootAllocator allocator = new RootAllocator(); Table context = table(allocator, true, 1, 2); + Table predict = table(allocator, false, 3, 4, 5, 6); Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + Request request = new Request(context, predict, "y", "regression"); + FittedModel fitted = client.fit(request, SdmClient.KUMO_TABULAR); + endpoint.failScores = 1; + assertEquals(4, fitted.predict(request).rows()); + List sizes = new ArrayList<>(); + for (Map call : endpoint.calls.subList(1, endpoint.calls.size())) { + @SuppressWarnings("unchecked") Map half = (Map) call.get("predict"); + @SuppressWarnings("unchecked") Map rows = (Map) half.get("instance_table"); + sizes.add(((List) rows.get("rows")).size()); + assertFalse(call.containsKey("context")); + } + assertEquals(List.of(4, 2, 2), sizes); + fitted.close(); + } + } + @Test void directRequestSurvivesCallerClosureAndRestoresRowOrder() throws Exception { Request request; try (RootAllocator allocator = new RootAllocator(); diff --git a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java index 172317164..a24e5e1b2 100644 --- a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java +++ b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java @@ -96,7 +96,8 @@ static List sizes(Request request) throws Exception { assertEquals(java.util.stream.IntStream.rangeClosed(1, count).boxed().toList(), ids); } assertEquals(List.of(100, 100, 1), sizes(materialize(schema, QUERY, 201, 100))); - assertEquals(List.of(200, 1), sizes(materialize(schema, QUERY, 201, 1000))); + // A size asked is taken as asked; 200 is only the default. + assertEquals(List.of(201), sizes(materialize(schema, QUERY, 201, 1000))); assertEquals(List.of(201), sizes(materialize(schema, "PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id", 201, null))); } @@ -113,7 +114,7 @@ static List sizes(Request request) throws Exception { Table predict = arrow(allocator, predicted, "temporal_link_prediction")) { Request request = memory(fixture, false).materializeExamples(task(test), context, predict, ExamplesOptions.builder().rowsPerCall(1000).build()); - assertEquals(List.of(200, 1), sizes(request)); + assertEquals(List.of(201), sizes(request)); } } diff --git a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java index db617ab5b..a13791b8c 100644 --- a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java +++ b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java @@ -27,8 +27,9 @@ /** * The model transport over {@code java.net.http}: the call is sent as the bytes given, - * one past 16 MB included; a transient status is tried again; a refusal is the endpoint's; an - * answer that is not JSON, or one past 64 MiB, is a failed transport. + * one past 16 MB included; a transient status or a 500 is tried again; a refusal is the endpoint's; + * a timeout is a failed transport told as a 408, not tried again; an + * answer that is not JSON is a failed transport, and one past 64 MiB too, told as a 413. */ class TransportTest { /** An endpoint answering {@code statuses} in turn, then 200, keeping every body. */ @@ -83,20 +84,20 @@ void aCallPast16MiBIsSentAsTheBytesGiven() throws Exception { @Test void aTransientStatusIsTriedAgain() throws Exception { - try (Endpoint endpoint = new Endpoint(OBJECT, 503, 429)) { + try (Endpoint endpoint = new Endpoint(OBJECT, 503, 429, 500)) { assertArrayEquals(OBJECT, endpoint.transport(3).predict("{}".getBytes(StandardCharsets.UTF_8))); - assertEquals(3, endpoint.received.size()); + assertEquals(4, endpoint.received.size()); } } @Test void aRefusalIsTheEndpointsAndIsNotTriedAgain() throws Exception { - try (Endpoint endpoint = new Endpoint(OBJECT, 500)) { + try (Endpoint endpoint = new Endpoint(OBJECT, 422)) { ModelEndpointException refused = assertThrows( ModelEndpointException.class, () -> endpoint.transport(3).predict("{}".getBytes(StandardCharsets.UTF_8))); - assertEquals(500, refused.status()); + assertEquals(422, refused.status()); assertEquals("later", refused.getMessage()); assertEquals(1, endpoint.received.size()); } @@ -113,6 +114,19 @@ void anAnswerThatIsNoJsonObjectIsAFailedTransport() throws Exception { } } + @Test + void anAnswerPast64MiBIsAFailedTransportToldAsA413() throws Exception { + byte[] answer = new byte[HttpTransport.MAX_RESPONSE_BYTES + 1]; + Arrays.fill(answer, (byte) ' '); + try (Endpoint endpoint = new Endpoint(answer)) { + HttpTransport.Failed failed = + assertThrows( + HttpTransport.Failed.class, + () -> endpoint.transport(0).predict("{}".getBytes(StandardCharsets.UTF_8))); + assertEquals(413, failed.status()); + } + } + @Test void anEmptyAnswerIsAnEmptyObject() throws Exception { try (Endpoint endpoint = new Endpoint(new byte[0])) { @@ -161,10 +175,12 @@ void anAnswerWhoseBodyIsHeldBackTimesOut() throws Exception { new HttpTransport( "http://127.0.0.1:" + server.getAddress().getPort(), null, Duration.ofMillis(300), 0); long started = System.nanoTime(); + // A failed transport, told as a 408 so the core asks the rows again in halves. HttpTransport.Failed failed = assertThrows( HttpTransport.Failed.class, () -> transport.predict("{}".getBytes(StandardCharsets.UTF_8))); + assertEquals(408, failed.status()); long millis = (System.nanoTime() - started) / 1_000_000; assertTrue(millis < 5_000, "the call took " + millis + " ms"); assertTrue(failed.getMessage().contains("timed out"), failed.getMessage()); diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py index 9da5854b0..6f7efb39b 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py @@ -68,15 +68,12 @@ } #: The refusals the reference raises as its base class under their code, -#: by code and rule: ``main``'s 1fc5a686 ceilings on what one -#: ``materialize`` gathers and its refusal of a batched forecast, each a +#: by code and rule: ``main``'s 1fc5a686 refusal of a batched forecast, a #: ``StructuredDataError(code='UNSUPPORTED_REQUEST')`` in the reference's #: client, so a caller's ``except`` sees the class it would there ( fix #: 2). Every other refusal takes its code's class (data:`ERRORS`). AS_THE_BASE_CLASS: frozenset[tuple[str, str]] = frozenset( { - ('UNSUPPORTED_REQUEST', 'most_entities'), - ('UNSUPPORTED_REQUEST', 'most_calls'), ('UNSUPPORTED_REQUEST', 'batched_part'), } ) @@ -143,7 +140,6 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name f'not (link.source_table == {source!r} and link.target_table == ' f'{target!r} and link.source_column in {others!r})]' ), - 'materialize_call': lambda: 'materialize()', 'name_candidates': lambda column: ( f'materialize(..., candidates=frame[{column!r}])' ), @@ -165,7 +161,6 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name ), 'missing_time': lambda: "'NaT'", 'result_frame': lambda: 'to_pandas()', - 'smaller_calls': lambda: 'materialize(rows_per_call=N)', 'trust_expressions': lambda: 'trust_expressions=True', } diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py index 0582c9574..6052fdbef 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py @@ -104,7 +104,6 @@ def _gather_locked( candidates: Sequence[Any] | None, ) -> tuple[tuple[dict[str, Any], ...], results.QueryFacts, results.CoreRequest]: from sdm_client import decompose - from sdm_client.materializers import relational anchor = options.get('anchor_time') call = { @@ -116,8 +115,6 @@ def _gather_locked( 'unseeded': 'random_seed' in options and options['random_seed'] is None, 'options': sorted(set(options) - TAKEN), 'most_parts': max(0, min(int(decompose.MOST_PARTS), 2**32 - 1)), - 'most_entities': max(0, min(int(relational.MOST_ENTITIES), 2**64 - 1)), - 'most_calls': max(0, min(int(relational.MOST_CALLS), 2**32 - 1)), } attachments = { 1: _graph(materializer), diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py index 4c62bd3f9..b226618ac 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py @@ -289,9 +289,7 @@ def gather( A request of more entities than one call scores is gathered by the core as several calls (``main``'s 1fc5a686): its payloads are each call's, in order, and the state keeps how many rows each later call - scores. The most entities and calls one ``materialize`` holds are this - client's (``MOST_ENTITIES``, ``MOST_CALLS``), which the core holds the - call to. + scores, however many entities there are. Raises: StructuredDataError: The core's refusal, as:mod:`.errors` maps it, @@ -344,14 +342,6 @@ def gather( }, ), } - # The ceilings on what one materialize holds, which the caller may set - # as it sets the most steps of a forecast. - from sdm_client.materializers import relational - - operation['most_entities'] = max( - 0, min(int(relational.MOST_ENTITIES), 2**64 - 1) - ) - operation['most_calls'] = max(0, min(int(relational.MOST_CALLS), 2**32 - 1)) if candidates is not None: operation['candidates'] = 6 attachments[6] = codec.attachment(_candidates(candidates)) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index c4feccaa3..44891e076 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -19,7 +19,7 @@ from sdm_client._rust import codec, errors, results from sdm_client._rust.run import Refused, TransportFailed, run -from sdm_client.core.transport import EndpointFailed +from sdm_client.core.transport import EndpointFailed, read_timed_out from sdm_client.decompose import RANKING from sdm_client.errors import NimRequestError, StructuredDataError @@ -606,6 +606,8 @@ def model_call(body: bytes) -> bytes: failure: StructuredDataError | None = None unexpected: Exception | None = None transport_failed = False + timed_out: StructuredDataError | None = None + split_status = 408 refused: NimRequestError | None = None try: # The core's call, its bytes as written (register PD-2), @@ -619,7 +621,18 @@ def model_call(body: bytes) -> bytes: except NimRequestError as refusal: refused = refusal except StructuredDataError as failed: - if failed.code == 'TRANSPORT_ERROR': + if failed.code == 'TRANSPORT_ERROR' and ( + read_timed_out(failed) + or failed.details.get('response_too_large') + ): + # Told to the core as a 408 or a 413, so it asks the rows + # again in halves; this is what is raised if no half answers. + if failed.details.get('response_too_large'): + split_status = 413 + timed_out = StructuredDataError( + failed.message, code=failed.code, details=failed.details + ) + elif failed.code == 'TRANSPORT_ERROR': transport_failed = True else: from sdm_client.in_process.backend import InProcessBackend @@ -641,6 +654,8 @@ def model_call(body: bytes) -> bytes: unexpected = other if transport_failed: raise TransportFailed + if timed_out is not None: + raise Refused(timed_out, split_status) raised: StructuredDataError | None = failure if refused is not None: # Built outside the handler, so nothing failing in the copy is diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index 570bcb51e..5e8b266e3 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -301,13 +301,21 @@ def _steps( _answered(handle, failed, error) if error is not None: # Raised outside the handler, so nothing is its context. - if refused is not None and ( - not cleanup_refusal - or ( - refused.status is not None - and error.details.get('status') == refused.status + # A call the core split and gave up on is the core's to name. + if ( + refused is not None + and not error.details.get('after_split') + and ( + not cleanup_refusal + or ( + refused.status is not None + and error.details.get('status') == refused.status + ) + or ( + refused.status is None + and error.code == 'TRANSPORT_ERROR' + ) ) - or (refused.status is None and error.code == 'TRANSPORT_ERROR') ): # The call's own failure, as the caller sees it, carrying # what the core carried on its end of the run. diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py index e619de18b..d2042e1e4 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py @@ -132,21 +132,47 @@ def send( primitive = inference is None or direct_options.can_encode(inference) def transmit(target: str | dict[str, Any], body: bytes) -> bytes: + from sdm_client._rust.run import Refused + native = json.loads(body) if 'inference' in native and not primitive: native['inference'] = inference - observed['payload'] = native - if target == 'create_session': - observed['reply'] = client._transport.create_session(native) - elif isinstance(target, dict) and set(target) == {'session_predict'}: - observed['reply'] = client._transport.session_predict( - target['session_predict']['session_id'], native - ) - else: - raise NativeBoundaryError('unexpected session transport endpoint') - return b'{}' + # The call as asked; the core may ask a scoring call's rows in + # smaller calls, whose answers it puts back together. + observed.setdefault('payload', native) + try: + if target == 'create_session': + reply = client._transport.create_session(native) + elif isinstance(target, dict) and set(target) == { + 'session_predict' + }: + reply = client._transport.session_predict( + target['session_predict']['session_id'], native + ) + else: + raise NativeBoundaryError( + 'unexpected session transport endpoint' + ) + except NativeBoundaryError: + raise + except Exception as error: + # The endpoint's status, for the core to split a call too large + # or refuse a context too large; the error raised as it is. A + # timeout is told as a 408, an answer too large to read a 413. + from sdm_client.core.transport import read_timed_out + + status = getattr(error, 'status_code', None) + if status is None and read_timed_out(error): + status = 408 + if status is None and getattr(error, 'details', {}).get( + 'response_too_large' + ): + status = 413 + raise Refused(error, status) from None + observed.setdefault('replies', []).append(reply) + return json.dumps(reply).encode() - run( + finished = run( {'op': 'send_tabular_session', 'input': 1, 'payload': 2}, { 1: codec.attachment( @@ -162,7 +188,15 @@ def transmit(target: str | dict[str, Any], body: bytes) -> bytes: }, model_endpoints=transmit, ) - return observed['reply'], observed['payload'] + answer = results.expect(finished.result, results.DirectValue) + sent = json.loads(finished.take(answer.value)) + merged = finished.take(sent['response']) + replies = observed['replies'] + # One call is answered by the transport's own reply, as it gave it; a + # call asked in smaller calls, by the answers the core put together. + if len(replies) == 1: + return replies[0], observed['payload'] + return json.loads(merged), observed['payload'] def fit( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py index 9b15ae10f..8e37a6295 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py @@ -215,30 +215,13 @@ def _context( has_custom_sorter, observe, ) - from sdm_client.models import TABULAR_LIMITS - limit = TABULAR_LIMITS['context_rows_max'] + # The context is sampled only to a count the caller asked for. if decision is None: - over_limit = wanted is not None and bool(wanted > limit) - accepted = _call('check_tabular_cap', {'over_limit': over_limit})[ - 'accepted' - ] - if not accepted: - decision = {'kind': 'refused'} - else: - count = int(wanted) if isinstance(wanted, np.integer) else wanted - decision = _call( - 'decide_tabular_context', - { - 'population': len(candidates), - 'requested': count, - }, - ) - if decision['kind'] == 'refused': - raise StructuredDataError( - f'num_context_samples is {wanted:,}, and the ' - f'endpoint accepts at most {limit:,} context rows', - code='UNSUPPORTED_REQUEST', + count = int(wanted) if isinstance(wanted, np.integer) else wanted + decision = _call( + 'decide_tabular_context', + {'population': len(candidates), 'requested': count}, ) if not decision['reduced']: return candidates diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 64e2d41e9..c655cb754 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -285,10 +285,12 @@ def __init__( verify_ssl: Whether to verify the server's TLS certificate. timeout: Seconds to wait for each individual attempt, not for the call as a whole. - max_retries: Transport-level retries of a transient failure (408, 429, - 502, 503, 504, or a dropped connection) with exponential - backoff. A 500 is not retried: the payload and the model are - the same on every attempt. ``POST /v1/sessions`` is excluded: + max_retries: Transport-level retries of a transient failure (429, + 500, 502, 503, or a connection that failed) with exponential + backoff, a ``Retry-After`` honoured. A 408, 413 or 504, or a + call slower than ``timeout``, is not tried again at its size: + its rows are asked in halves. ``POST /v1/sessions`` is + excluded: a re-sent create would orphan a pinned context on the endpoint. """ @@ -336,6 +338,7 @@ def for_databricks_serving( *, workspace_client: Any | None = None, timeout: float | None = None, + max_request_bytes: int | None = None, ) -> SDMClient: r"""Open a client against a Databricks Model Serving endpoint. @@ -367,6 +370,11 @@ def for_databricks_serving( timeouts. Given with a ``WorkspaceClient``, it is set on the copy, never on the caller's client; with a subclass or test double, which cannot be copied, it is refused. + max_request_bytes: The most bytes one call's wrapped body may + take, the gateway's 16 MB when omitted. The endpoint drops a + call past it as a connection, so a call past it is split + into smaller ones before it is sent, each with the whole + context. Raises: MissingExtraError: If ``databricks-sdk`` is not installed; install @@ -384,7 +392,14 @@ def for_databricks_serving( client = cls.__new__(cls) client._start( DatabricksServingTransport( - endpoint, workspace_client, timeout=timeout + endpoint, + workspace_client, + timeout=timeout, + **( + {} + if max_request_bytes is None + else {'max_request_bytes': max_request_bytes} + ), ) ) return client diff --git a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py index 66c4f302c..a3732234f 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py +++ b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py @@ -22,6 +22,7 @@ import math import re import threading +import time from collections.abc import Mapping from typing import Any @@ -31,6 +32,7 @@ _check_encodable, _diagnosis_in, _validate_limits, + read_timed_out, ) from sdm_client.errors import ( MissingExtraError, @@ -222,23 +224,30 @@ def predict_body(self, body: bytes) -> bytes: says, for the core to name; anything but an ``OSError`` escapes, as from:meth:`predict`.""" self._require_open() - try: - answered = self._workspace.api_client.do( - 'POST', - f'/serving-endpoints/{self._endpoint}/invocations', - headers={'Content-Type': 'application/json'}, - data=_Body(body), - raw=True, - ) - contents = answered['contents'] + # The SDK retries a 429 or a 503, not a 500 or a 502, which this + # client tries once more, after a pause, as temporary. + for attempt in (0, 1): try: - return bytes(contents.read()) - finally: - contents.close() - # The SDK's errors and its retry deadline are all OSError, as are - # requests' own, as in predict. - except OSError as error: - raise _failed(error) from None + answered = self._workspace.api_client.do( + 'POST', + f'/serving-endpoints/{self._endpoint}/invocations', + headers={'Content-Type': 'application/json'}, + data=_Body(body), + raw=True, + ) + contents = answered['contents'] + try: + return bytes(contents.read()) + finally: + contents.close() + # The SDK's errors and its retry deadline are all OSError, as are + # requests' own, as in predict. + except OSError as error: + if attempt == 0 and _status_of(error) in (500, 502): + time.sleep(_RETRY_PAUSE_SECONDS) + continue + raise _failed(error) from None + raise AssertionError('unreachable') def _answered(self, payload: dict[str, Any]) -> str: r"""The served model's answer to ``payload``, as the text it @@ -566,6 +575,25 @@ def _response_body(reply: Any, endpoint: str) -> str: return body +#: The pause before a 500 or a 502 is tried once more. +_RETRY_PAUSE_SECONDS = 1.0 + +#: The statuses the SDK names only by an ``error_code``: a call too large, or +#: a gateway's failure, which the core asks again in halves. +_STATUS_OF_CODE = { + 'REQUEST_ENTITY_TOO_LARGE': 413, + 'PAYLOAD_TOO_LARGE': 413, + 'CONTENT_TOO_LARGE': 413, + 'BAD_GATEWAY': 502, + 'GATEWAY_TIMEOUT': 504, +} + +#: The SDK's and ``requests``' names for a deadline reached. +_TIMEOUTS = frozenset( + {'DeadlineExceeded', 'OperationTimeout', 'TimeoutError', 'Timeout'} +) + + def _status_of(error: BaseException) -> int: r"""Map a Databricks SDK error onto an HTTP-equivalent status. @@ -583,15 +611,27 @@ def _status_of(error: BaseException) -> int: return 409 if names & {'TooManyRequests', 'RequestLimitExceeded'}: return 429 - if names & { - 'DeadlineExceeded', - 'OperationTimeout', - 'TimeoutError', - 'Timeout', - }: + if names & _TIMEOUTS: + # The SDK's deadline spent retrying a 429, a 503 or a connection is + # the endpoint being unavailable, not a call too slow to answer: + # 504 only where the call itself timed out, which a smaller call + # can fix. + cause = error.__cause__ + if cause is not None and not ( + {c.__name__ for c in type(cause).__mro__} & _TIMEOUTS + or read_timed_out(cause) + ): + return 503 return 504 if 'BadRequest' in names: return 400 + if 'InternalError' in names: + return 500 + # The SDK has no class for a 413 or a 502; it names one by the HTTP + # reason phrase, as its ``error_code``, where the body is no JSON. + code = getattr(error, 'error_code', None) + if code in _STATUS_OF_CODE: + return _STATUS_OF_CODE[code] status = getattr(error, 'status_code', None) return status if isinstance(status, int) else 503 diff --git a/packages/structured-data-models-client/src/sdm_client/core/transport.py b/packages/structured-data-models-client/src/sdm_client/core/transport.py index 0b1a13300..77a8d119c 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/transport.py +++ b/packages/structured-data-models-client/src/sdm_client/core/transport.py @@ -31,11 +31,11 @@ _DEFAULT_TIMEOUT_SECONDS = 60.0 _DEFAULT_MAX_RETRIES = 3 _DEFAULT_BACKOFF_FACTOR = 0.5 -#: The transient ones. A 500 is not among them: the payload and the model are -#: the same on every attempt, so a server fault reproduces, and the caller -#: mistakes this endpoint reports as a 500 are reclassified from the body as -#: the caller's after three more uploads of the whole gathered context. -_RETRY_STATUS = (408, 429, 502, 503, 504) +#: The statuses tried again with backoff, up to ``max_retries``: overload, and +#: a server or gateway fault, a 500 included. A 408, 413 or 504, or an answer +#: slower than the timeout, is not tried again at its size: the core asks the +#: call's rows in halves at once. +_RETRY_STATUS = (429, 500, 502, 503) _RETRY_AFTER_MAX_SECONDS = 60 _MAX_BODY_SNIPPET = 512 _LOCAL_HOSTS = frozenset({'localhost', '127.0.0.1', '::1'}) @@ -301,7 +301,9 @@ def _build_retry( options: dict[str, Any] = { 'total': max_retries, 'connect': max_retries, - 'read': max_retries, + # A read error, an answer slower than the timeout among them, is not + # tried again at its size: the core asks the call's rows in halves. + 'read': 0, 'status': max_retries, 'backoff_factor': backoff_factor, 'status_forcelist': _RETRY_STATUS, @@ -315,6 +317,21 @@ def _build_retry( return Retry(**options) +def read_timed_out(error: BaseException | None) -> bool: + r"""Whether a request failure is the answer taking longer than the + timeout, which a smaller call can fix, rather than a connection that + could not be made (``NewConnectionError`` is a ``TimeoutError`` too).""" + seen: set[int] = set() + while error is not None and id(error) not in seen: + seen.add(id(error)) + if isinstance( + error, (requests.ReadTimeout, urllib3.exceptions.ReadTimeoutError) + ): + return True + error = error.__cause__ or error.__context__ + return False + + def _timed_out(error: BaseException | None) -> bool: r"""Whether a request failure is a timeout, however it reached us. @@ -404,10 +421,13 @@ def _read_capped(response: requests.Response, url: str) -> bytes: body = bytearray() for chunk in response.iter_content(_RESPONSE_CHUNK_BYTES): if len(body) + len(chunk) > _MAX_RESPONSE_BYTES: + # An answer too large to read is the call too large: a call of + # fewer rows is asked instead (``response_too_large``). raise StructuredDataError( f'Response body from {redact_url(url)} exceeds the ' f'{_MAX_RESPONSE_BYTES} byte limit', code='TRANSPORT_ERROR', + details={'response_too_large': True}, ) body.extend(chunk) return bytes(body) diff --git a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py index b18afcfe9..615c51cf6 100644 --- a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py @@ -19,11 +19,6 @@ from sdm_client.sanitization import SanitizationReport -# Host memory limits passed to Rust for request-size enforcement. -MOST_ENTITIES = 100_000 -MOST_CALLS = 100 - - class RelationalMaterializer: """Retain host resources while Rust owns sampling and request construction.""" diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 5a8a36717..eb48a78b4 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -37,28 +37,11 @@ from sdm_client.base import ModelSpec, RuntimeContract from sdm_client.errors import StructuredDataError, UnknownModelError -# Sizes the relational endpoint enforces. `predict_rows_max` is the serving -# layer's own `TFM_MAX_TEST_ROWS`, measured at the boundary rather than read -# from a document: 10,000 rows to score are answered, and 10,001 come back -# -# [422 UNSUPPORTED_SHAPE] N_test 10001 exceeds limit 10000. -# -# A deployment may be configured to hold to less, which is what the -# `rows_per_call` option on materialize() is for. -# -# The endpoint refuses past these, so the client refuses first with a message -# that names the limit. See the architecture note on inference knobs: -# `run_mode` is derived from these rather than being set. -# -# Prediction row limits also bound the batches requested from the Rust core. -# Keep endpoint limits and batching consistent so each batch is accepted. +# Fixed properties of the relational model. The client holds a request to no +# size of its own: it sends its best attempt, and a call the endpoint refuses +# as too large, or times out on, is asked again in halves. RELATIONAL_LIMITS = { 'context_rows_fast': 1_000, - 'context_rows_max': 10_000, - 'predict_rows_max': 10_000, - 'predict_rows_temporal_link_prediction': 200, - 'hops_max': 6, - 'tables_max': 15, 'classes_max': 10, } @@ -69,18 +52,6 @@ PREDICTION_STATISTICS = frozenset({'mean', 'median'}) SUMMARISED_TASK_KINDS = frozenset({'regression', 'forecasting'}) -# The tabular endpoint refuses a context past this, exactly: -# -# [422 UNSUPPORTED_SHAPE] N_train 10001 exceeds limit 10000. -# -# Measured against the deployment rather than read from a document, at the -# boundary: 10,000 rows answer, 10,001 refuse. Declared so the client refuses -# first, because the alternative is encoding and transmitting an arbitrarily -# large context to be told the limit at the far end. -TABULAR_LIMITS = { - 'context_rows_max': 10_000, -} - KUMO_TABULAR = ModelSpec( model_id=TFM_MODEL_KUMO_TABULAR, profile='TABULAR', @@ -91,7 +62,6 @@ outputs=frozenset( {'prediction', 'probabilities', 'quantiles', 'explanation'} ), - limits=TABULAR_LIMITS, explanation_formats=('sdm_gradient_v1',), ), ) diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index d3e97a2c8..1f326433c 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -193,12 +193,10 @@ def materialize( and ``anchor_time``; ``prediction_statistic``: what a regression's prediction reports of its predicted distribution, ``'median'`` (the default) or ``'mean'``; and - ``rows_per_call``: how many entities one call scores, which - defaults to what the endpoint takes, 10,000 or 200 for a - temporal link prediction, and may only be lowered. Lower it - for a graph wide enough that a full batch outruns the 30 MiB - body, or for a deployment holding to less than this client - knows of. + ``rows_per_call``: how many entities one call scores, 10,000 + by default, or 200 for a temporal link prediction, taken as + asked. A call the endpoint refuses as too large, or times out + on, is asked again in halves, each with the whole context. Returns: A:class:`~sdm_client.Request` carrying the data. diff --git a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py index 85baf1abd..98d023a01 100644 --- a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py +++ b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py @@ -34,10 +34,9 @@ class StopWarning(Exception): def observe_case(name: str) -> dict[str, Any]: from sdm_client import KumoTabular, PredictiveQuery, SDMClient - from sdm_client.models import TABULAR_LIMITS assert name in CASES - cap = TABULAR_LIMITS['context_rows_max'] + cap = 10_000 events: list[Any] = [] recorded: list[dict[str, str]] = [] outcome: dict[str, Any] = {} diff --git a/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py b/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py index 360cfa4ef..418f006fc 100644 --- a/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py +++ b/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py @@ -256,7 +256,7 @@ def test_the_limits_page_records_the_default_the_code_has() -> None: Path(__file__).resolve().parents[3] / 'docs' / 'reference' / 'limits.md' ).read_text() assert '1,000 when explaining, otherwise 10,000' not in page - assert '1,000 by default, up to 10,000 with `num_context_samples`' in page + assert 'The context is sampled only to a `num_context_samples`' in page def test_a_concrete_timestamp_materializes() -> None: diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index 641cc7926..b34edd12f 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -97,9 +97,7 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( ): """A part is one call, which is what makes the fan-out countable. The refusal reads the count it was given, so it costs no gathering.""" - from sdm_client.models import RELATIONAL_LIMITS - - count = RELATIONAL_LIMITS['predict_rows_max'] + 1 + count = 10_000 + 1 # one past the default batch with pytest.raises(StructuredDataError) as refused: built(query, indices=[1] * count, **how) assert refused.value.code == 'UNSUPPORTED_REQUEST' @@ -108,8 +106,8 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( def test_a_smaller_batch_lowers_what_a_part_may_ask_for(monkeypatch): - """`rows_per_call` only goes down, and a part is one call, so asking for a - smaller one asks for fewer entities rather than for more calls.""" + """A part is one call, so asking for a smaller ``rows_per_call`` asks for + fewer entities rather than for more calls.""" with pytest.raises(StructuredDataError) as refused: built(FORECAST, indices=[1, 2, 3], rows_per_call=2) @@ -117,22 +115,6 @@ def test_a_smaller_batch_lowers_what_a_part_may_ask_for(monkeypatch): assert built(FORECAST, indices=[1, 2], rows_per_call=2).parts -def test_the_whole_decomposed_question_is_held_to_the_memory_ceiling( - monkeypatch, -): - """Each part gathers the whole population with a context of its own, so a - ranking over 200 candidates holds 200 times what one of them does. Bounding - each part on its own left the question itself unbounded.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_ENTITIES', 5) - built(RANK, indices=[1, 2], candidates=[0, 1]) - with pytest.raises(StructuredDataError) as refused: - built(RANK, indices=[1, 2, 3], candidates=[0, 1]) - assert refused.value.details['total'] == 6 - assert refused.value.details['gathered_times'] == 2 - - def test_a_step_that_fails_keeps_the_steps_already_paid_for(): """A question asked in parts pays per part, and the steps answered before a failure are as bought as the batches of one are. diff --git a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py index 60ee44e3a..5e9673f04 100644 --- a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py +++ b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py @@ -78,9 +78,9 @@ def do_POST(self): elif outer.mode == 'malformed' and outer.scored == 2: answer = {'predictions': 'invalid'} elif outer.mode == 'fail' and outer.scored == 2: - status = 500 + status = 422 elif outer.mode == 'retry' and outer.scored == 1: - status = 408 + status = 503 else: table = payload['predict']['instance_table'] index = table['columns'].index('instance_id') @@ -165,9 +165,9 @@ def test_failed_batch_preserves_paid_answers_and_cleanup(): with SDMClient(url=endpoint.url, max_retries=1) as client: with pytest.raises(StructuredDataError) as caught: client.predict(request(), model=KumoRelational()) - assert endpoint.scored == 2 # HTTP500 is not retried. + assert endpoint.scored == 2 # a 422 is not retried assert endpoint.calls[-1][:2] == ('DELETE', '/v1/sessions/s1') - assert getattr(caught.value, 'status_code', None) == 500 + assert getattr(caught.value, 'status_code', None) == 422 assert caught.value.details['answered'] is not None finally: endpoint.close() diff --git a/packages/structured-data-models-client/tests/test_context_samples.py b/packages/structured-data-models-client/tests/test_context_samples.py index d41d87c62..0ff71744a 100644 --- a/packages/structured-data-models-client/tests/test_context_samples.py +++ b/packages/structured-data-models-client/tests/test_context_samples.py @@ -25,7 +25,7 @@ from sdm_client.models import RELATIONAL_LIMITS FAST = RELATIONAL_LIMITS['context_rows_fast'] -MAX = RELATIONAL_LIMITS['context_rows_max'] +MAX = 10_000 # --- which value goes on the wire ----------------------------------------- @@ -41,6 +41,7 @@ (FAST + 1, 'best'), (MAX - 1, 'best'), (MAX, 'best'), + (MAX + 1, 'best'), ], ) def test_the_wire_value_is_the_one_that_admits_the_count(asked, expected): @@ -49,18 +50,10 @@ def test_the_wire_value_is_the_one_that_admits_the_count(asked, expected): assert request._payloads[0]['inference']['run_mode'] == expected -def test_asking_for_more_than_the_endpoint_allows_names_the_ceiling(): - with pytest.raises(StructuredDataError) as error: - _materialize(num_context_samples=MAX + 1) - message = str(error.value) - assert '10,000' in message - assert error.value.code == 'UNSUPPORTED_REQUEST' - - -def test_the_boundary_is_at_the_documented_row_counts(): - """A pin: these are the two numbers the endpoint actually distinguishes.""" +def test_the_boundary_is_at_the_documented_row_count(): + """A pin: the count past which a request asks for the larger setting. + The client sets no most on the count.""" assert FAST == 1_000 - assert MAX == 10_000 # --- how many examples are actually gathered ------------------------------ diff --git a/packages/structured-data-models-client/tests/test_custom_examples.py b/packages/structured-data-models-client/tests/test_custom_examples.py index 9968f8f0d..0abc2d5fc 100644 --- a/packages/structured-data-models-client/tests/test_custom_examples.py +++ b/packages/structured-data-models-client/tests/test_custom_examples.py @@ -385,6 +385,7 @@ def test_supplied_link_batches_obey_model_limit(): predicted, rows_per_call=500, ) + # A batch size asked is taken as asked; 200 is only the default. assert [ len(p['predict']['instance_table']['rows']) for p in request.payloads - ] == [200, 1] + ] == [201] diff --git a/packages/structured-data-models-client/tests/test_databricks_from_source.py b/packages/structured-data-models-client/tests/test_databricks_from_source.py index 5979eba8d..1e8231721 100644 --- a/packages/structured-data-models-client/tests/test_databricks_from_source.py +++ b/packages/structured-data-models-client/tests/test_databricks_from_source.py @@ -1576,25 +1576,6 @@ def test_refusals_preserve_source_read_order( assert _where_read(schema, connection) == {where} -def test_too_many_entities_are_refused_before_the_source_is_read( - monkeypatch: pytest.MonkeyPatch, -) -> None: - r"""More entities than one ``materialize`` holds are refused from the - count alone, before the old route's engine reads anything - (``relational.py:339``): the tracker's entity-count row.""" - from sdm_client.errors import StructuredDataError - from sdm_client.materializers import relational - - monkeypatch.setattr(relational, 'MOST_ENTITIES', 1) - connection = _CompletingConnection() - schema = _rebuilt(connection) - mark = len(connection.calls) - with pytest.raises(StructuredDataError) as refused: - _asked(schema) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert len(connection.calls) == mark - - class _EmptyingWarehouse(_CompletingWarehouse): r"""The completing warehouse, whose ``shop.users`` holds no row while ``empty`` says so: every read of its rows answers none, and its count diff --git a/packages/structured-data-models-client/tests/test_databricks_splitting.py b/packages/structured-data-models-client/tests/test_databricks_splitting.py new file mode 100644 index 000000000..875e37611 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_databricks_splitting.py @@ -0,0 +1,218 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A Databricks serving endpoint drops a call too large as a connection, so a +call past its byte limit is split before it is sent; a 413, the gateway's +400 saying a request is too large, a 504 or a call +that timed out is asked again in halves; a deadline spent waiting on an +unavailable endpoint is a 503, never split; and a 500 is tried once more.""" + +from __future__ import annotations + +import io +import json +from typing import Any + +import pytest +import requests +from test_databricks_serving import ( + ENDPOINT, + ApiClient, + Endpoints, + Workspace, + asked, +) + +from sdm_client import SDMClient +from sdm_client.core.databricks_serving import _status_of + + +def answer(sent: dict[str, Any]) -> dict[str, Any]: + r"""The served model's answer to ``sent``: one prediction per row.""" + table = sent['predict']['instance_table'] + keys = ( + sent.get('schema', {}).get('instance_table', {}).get('primary_key') + or [] + ) + at = table['columns'].index(keys[0]) if keys else None + return { + 'object': 'prediction.response', + 'id': 'pred', + 'model': 'kumo-tabular', + 'metadata': {}, + 'predictions': [ + { + 'id': f'[{row[at]}]' if at is not None else str(index), + 'row_index': index, + 'prediction': 'yes', + } + for index, row in enumerate(table['rows']) + ], + } + + +class Failing(Endpoints): + r"""Raises each of ``failures`` on the calls in turn, then answers.""" + + def __init__(self, failures: list[BaseException]) -> None: + super().__init__(answer=answer) + self.failures = list(failures) + + def query(self, *, name: str, **kwargs: Any) -> Any: + if self.failures: + self.calls.append({'name': name, **kwargs}) + raise self.failures.pop(0) + return super().query(name=name, **kwargs) + + +class Sized(ApiClient): + r"""Records the bytes of each envelope posted.""" + + sizes: list[int] + + def do(self, method: str, path: str, *, headers, data, raw) -> Any: + body = data.read() + self.sizes.append(len(body)) + return super().do( + method, path, headers=headers, data=io.BytesIO(body), raw=raw + ) + + +def opened(failures=(), **kwargs: Any) -> tuple[SDMClient, Workspace]: + workspace = Workspace() + workspace.serving_endpoints = Failing(list(failures)) + workspace.api_client = Sized(workspace) + workspace.api_client.sizes = [] + client = SDMClient.for_databricks_serving( + ENDPOINT, workspace_client=workspace, **kwargs + ) + return client, workspace + + +def sdk_error(status: int, reason: str, body: bytes, kind: str) -> OSError: + r"""The error the installed Databricks SDK raises for an HTTP answer of + ``status``, read by its own parser, as ``api_client.do`` raises it.""" + parser = pytest.importorskip('databricks.sdk.errors.parser') + response = requests.Response() + response.status_code = status + response.reason = reason + response._content = body + response.headers['Content-Type'] = kind + response.request = requests.Request( + 'POST', + f'https://example.invalid/serving-endpoints/{ENDPOINT}/invocations', + ).prepare() + response.url = response.request.url + error = parser._Parser().get_api_error(response) + assert error is not None + return error + + +HTML = 'text/html' +JSON = 'application/json' +#: The SDK's errors for each status, as the gateway answers them. +STATUS = { + 413: lambda: sdk_error( + 413, + 'Request Entity Too Large', + b'
too large
', + HTML, + ), + 502: lambda: sdk_error( + 502, 'Bad Gateway', b'Bad Gateway', HTML + ), + 504: lambda: sdk_error(504, 'Gateway Timeout', b'', 'text/plain'), + 500: lambda: sdk_error( + 500, + 'Internal Server Error', + b'{"error_code": "INTERNAL_ERROR", "message": "boom"}', + JSON, + ), + 429: lambda: sdk_error( + 429, + 'Too Many Requests', + b'{"error_code": "REQUEST_LIMIT_EXCEEDED", "message": "slow"}', + JSON, + ), + # The gateway's refusal of a call too large. + 400: lambda: sdk_error( + 400, + 'Bad Request', + b'{"error_code": "BAD_REQUEST", "message": "Request size cannot exceed 16MB"}', + JSON, + ), + 503: lambda: sdk_error( + 503, + 'Service Unavailable', + b'{"error_code": "TEMPORARILY_UNAVAILABLE", "message": "later"}', + JSON, + ), +} + + +def test_max_request_bytes_is_passed_through(): + client, _ = opened(max_request_bytes=1234) + assert client._transport._max_request_bytes == 1234 + + +def test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent(caplog): + whole, workspace = opened() + whole.predict(asked(), model='kumo-tabular') + (size,) = workspace.api_client.sizes + caplog.set_level('INFO', logger='sdm_core') + split, workspace = opened(max_request_bytes=size - 1) + result = split.predict(asked(), model='kumo-tabular') + assert len(workspace.api_client.sizes) == 2 + assert all(sent <= size - 1 for sent in workspace.api_client.sizes) + assert result.to_pandas()['row_index'].tolist() == [0, 1] + assert any('smaller calls' in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize('status', [413, 400, 504]) +def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): + error = STATUS[status]() + assert _status_of(error) == status, type(error).__mro__ + client, workspace = opened([error]) + result = client.predict(asked(), model='kumo-tabular') + calls = workspace.serving_endpoints.calls + assert len(calls) == 3 + sent = [ + json.loads(c['dataframe_records'][0]['request_json']) for c in calls + ] + assert [len(s['predict']['instance_table']['rows']) for s in sent] == [ + 2, + 1, + 1, + ] + assert result.to_pandas()['row_index'].tolist() == [0, 1] + + +@pytest.mark.parametrize('status', [500, 502]) +def test_a_500_or_a_502_is_tried_once_more_at_its_size(status, monkeypatch): + from sdm_client.core import databricks_serving + + monkeypatch.setattr(databricks_serving, '_RETRY_PAUSE_SECONDS', 0) + client, workspace = opened([STATUS[status]()]) + client.predict(asked(), model='kumo-tabular') + calls = workspace.serving_endpoints.calls + assert len(calls) == 2 + # Never split: both calls carry every row. + sent = [ + json.loads(c['dataframe_records'][0]['request_json']) for c in calls + ] + assert [len(s['predict']['instance_table']['rows']) for s in sent] == [2, 2] + + +def test_a_deadline_spent_waiting_is_a_503_and_a_call_timed_out_a_504(): + def timed_out(cause: BaseException | None) -> TimeoutError: + error = TimeoutError('Timed out after 0:05:00') + error.__cause__ = cause + return error + + assert _status_of(timed_out(STATUS[429]())) == 503 + assert _status_of(timed_out(STATUS[503]())) == 503 + assert _status_of(timed_out(requests.ConnectionError('refused'))) == 503 + assert _status_of(timed_out(requests.ReadTimeout('slow'))) == 504 + assert _status_of(timed_out(None)) == 504 + assert _status_of(STATUS[504]()) == 504 diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 33e837fe8..6f8b65c61 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -4,10 +4,9 @@ """What the endpoint will not take, and what the answer did not contain. -Two silences. The tabular endpoint caps a context at 10,000 rows and the client -did not know it, so a caller could encode and transmit a table of any size to -be refused at the far end, while the identical overage on the other model was -refused locally with a message naming the limit. +The context the client sends: whole, or sampled only to a count the caller +names. The client holds it to no size of its own; a context larger than the +endpoint takes is the endpoint's refusal. And a field a caller names is validated against the model and never against what comes back. The endpoint answers 200, leaves the field out, and the frame @@ -27,7 +26,6 @@ from sdm_client import PredictiveQuery, Schema from sdm_client.client import _report_absent from sdm_client.errors import StructuredDataError -from sdm_client.models import KUMO_TABULAR from sdm_client.result import PredictionResult @@ -59,16 +57,12 @@ def query(schema: Schema) -> PredictiveQuery: # --- the context the endpoint will take ------------------------------------ -def test_the_declared_cap_is_the_one_the_endpoint_enforces(): - """Measured at the boundary: 10,000 answer, 10,001 refuse.""" - assert KUMO_TABULAR.runtime.limits['context_rows_max'] == 10_000 - - -def test_a_context_above_the_cap_is_refused_before_it_is_encoded(): +def test_a_count_past_the_rows_there_takes_them_all(): schema = table(200) - with pytest.raises(StructuredDataError) as raised: - query(schema).materialize(indices=[0, 1], num_context_samples=50_000) - assert 'accepts at most 10,000 context rows' in str(raised.value) + request = query(schema).materialize( + indices=[0, 1], num_context_samples=50_000 + ) + assert len(request.context.instance_table) == 198 def test_the_cap_itself_is_allowed(): @@ -79,14 +73,13 @@ def test_the_cap_itself_is_allowed(): assert len(request.context.instance_table) == 198 -def test_an_unnamed_context_is_capped_and_says_so(): +def test_an_unnamed_context_is_sent_whole(): schema = table(12_000) request, said = warnings_from( partial(query(schema).materialize, indices=[0, 1]) ) - assert len(request.context.instance_table) == 10_000 - assert any('were sampled' in s for s in said), said - assert any('num_context_samples' in s for s in said) + assert len(request.context.instance_table) == 11_998 + assert not any('were sampled' in s for s in said), said def test_a_context_under_the_cap_is_left_whole_and_unremarked(): @@ -98,27 +91,26 @@ def test_a_context_under_the_cap_is_left_whole_and_unremarked(): assert not any('were sampled' in s for s in said) -def test_the_capped_context_is_a_sample_and_not_the_first_rows(): +def test_a_counted_context_is_a_sample_and_not_the_first_rows(): """Taking the head would bias the context toward whatever the table is ordered by, which for a table ordered by time is the oldest rows.""" schema = table(12_000) - request = query(schema).materialize(indices=[0]) + request = query(schema).materialize(indices=[0], num_context_samples=10_000) taken = request.context.instance_table['rid'] assert taken.max() > 11_000 assert len(taken) == 10_000 -def test_the_predict_frame_route_is_capped_too(): - """The route with no knob at all. The schema's table became the context - whole, so a large table was encoded and sent in full every time, to be - refused.""" +def test_the_predict_frame_route_sends_the_table_whole(): + """The route with no knob at all: the schema's table, less the rows + scored, is the context.""" schema = table(12_000) rows = schema['t'].data.head(3).drop(columns=['amount']) request, said = warnings_from( partial(query(schema).materialize, predict=rows) ) - assert len(request.context.instance_table) == 10_000 - assert any('were sampled' in s for s in said) + assert len(request.context.instance_table) == 11_997 + assert not any('were sampled' in s for s in said) # --- what the answer did not carry ----------------------------------------- @@ -220,28 +212,18 @@ def test_the_result_is_returned_unchanged(): assert _report_absent(answer, ['quantiles'], 'kumo-x') is answer -def test_a_materialized_context_above_the_cap_is_refused_not_merely_the_number(): - """A small table makes this a check on the argument. The rows have to be - there for it to be a check on the context.""" +def test_a_count_past_the_old_cap_is_gathered(): + """The rows are there, so the count is the context's size.""" schema = table(12_000) - with pytest.raises(StructuredDataError) as raised: - query(schema).materialize(indices=[0, 1], num_context_samples=10_001) - assert 'accepts at most 10,000 context rows' in str(raised.value) + request = query(schema).materialize( + indices=[0, 1], num_context_samples=10_001 + ) + assert len(request.context.instance_table) == 10_001 -def test_the_predict_route_advises_something_that_route_allows(): - """Its own refusal says `num_context_samples` cannot be used here, so the - warning must not tell the caller to pass it.""" +def test_the_predict_route_takes_no_count(): schema = table(12_000) rows = schema['t'].data.head(3).drop(columns=['amount']) - _, said = warnings_from(partial(query(schema).materialize, predict=rows)) - capped = [s for s in said if 'were sampled' in s] - assert capped - remedy = capped[0].split('sampled.')[1] - assert 'Pass num_context_samples' not in remedy - assert 'takes no num_context_samples' in remedy - assert 'building the schema' in remedy - with pytest.raises(StructuredDataError) as raised: query(schema).materialize(predict=rows, num_context_samples=100) assert 'cannot be used with predict=' in str(raised.value) @@ -397,15 +379,14 @@ def test_a_target_that_varies_is_left_alone(): assert not any('every context row' in s for s in said) -def test_a_context_past_the_models_cap_is_refused_before_it_is_sent(): - """The client's own preamble says it refuses first where it can tell, and - the model is known by the time this check runs, so sending 12,500 rows to - be told the limit at the far end was work nobody needed to do.""" +def test_a_context_past_the_old_cap_is_sent(): + """The client holds a context to no size of its own: 12,500 rows are + sent, for the endpoint to answer or refuse.""" import numpy as np import pandas as pd from test_adapter_and_result import envelope - from sdm_client import Request, UnsupportedRequestError + from sdm_client import Request rng = np.random.default_rng(0) rows = 12_500 @@ -418,8 +399,7 @@ def test_a_context_past_the_models_cap_is_refused_before_it_is_sent(): task='classification', ) - with pytest.raises(UnsupportedRequestError, match='12,500'): - envelope(request) + envelope(request) def test_a_context_within_the_cap_is_not_refused(): diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 9940e7c6f..61984a506 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -58,15 +58,11 @@ def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): """A pin, not a probe. These numbers were read from the relational NIM. This asserts they have - not drifted in the client, which is all it can do offline. The live suites - check that the endpoint still agrees. + not drifted in the client, which is all it can do offline. The client + holds a request to no size of its own, so no most is set on context, + rows scored, bytes, tables or hops. """ - assert RELATIONAL_LIMITS['context_rows_fast'] == 1_000 - assert RELATIONAL_LIMITS['context_rows_max'] == 10_000 - assert RELATIONAL_LIMITS['predict_rows_max'] == 10_000 - assert RELATIONAL_LIMITS['predict_rows_temporal_link_prediction'] == 200 - assert RELATIONAL_LIMITS['hops_max'] == 6 - assert RELATIONAL_LIMITS['tables_max'] == 15 + assert RELATIONAL_LIMITS == {'context_rows_fast': 1_000, 'classes_max': 10} @pytest.mark.parametrize('bad', ['2', True]) @@ -242,20 +238,10 @@ def gathered_in_calls(*, calls: int, context_rows: int): ) -@pytest.mark.parametrize('over', [False, True], ids=['under', 'over']) -def test_the_context_cap_is_held_to_each_call_and_not_their_sum(over): - """Both sides of the cap, because only the allowed one was covered: a - per-call count that always read nought would have passed that alone.""" - - cap = RELATIONAL_LIMITS['context_rows_max'] - each = cap + 1 if over else cap // 2 + 1 - request = gathered_in_calls(calls=2, context_rows=each) - if not over: - sent(request) - return - with pytest.raises(StructuredDataError) as refused: - sent(request) - assert f'{each:,} context rows' in str(refused.value) +def test_each_call_s_context_is_sent_at_any_size(): + """The client holds a context to no size of its own, in each call or + across them.""" + sent(gathered_in_calls(calls=2, context_rows=10_001)) def test_a_batched_request_reports_the_context_one_call_carries(): diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index deccb4296..4fd99bd8b 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -193,9 +193,7 @@ def test_the_endpoint_scores_what_this_client_says_it_does(client, query): """ import copy - from sdm_client.models import RELATIONAL_LIMITS - - cap = RELATIONAL_LIMITS['predict_rows_max'] + cap = 10_000 # the default batch payload = copy.deepcopy(query.materialize(indices=[0, 1]).payloads[0]) rows = payload['predict']['instance_table']['rows'] diff --git a/packages/structured-data-models-client/tests/test_materialize_options.py b/packages/structured-data-models-client/tests/test_materialize_options.py index 0c2e91ec5..10f638a55 100644 --- a/packages/structured-data-models-client/tests/test_materialize_options.py +++ b/packages/structured-data-models-client/tests/test_materialize_options.py @@ -372,21 +372,8 @@ def _refusal_of(**options: Any) -> StructuredDataError: def test_a_limit_is_raised_as_the_subclass_on_the_core_register_p1b_20() -> ( None ): - for options in ({'num_context_samples': 10001}, {'num_hops': 7}): - core = _refusal_of(**options) - assert core.code == 'UNSUPPORTED_REQUEST', options - assert type(core).__name__ == 'UnsupportedRequestError', options - assert 'rule' not in core.details, options - for options, rule in ( - ({'indices': list(range(100001))}, 'most_entities'), - ({'indices': list(range(101)), 'rows_per_call': 1}, 'most_calls'), - ): - core = _refusal_of(**options) - assert (type(core), core.code) == ( - StructuredDataError, - 'UNSUPPORTED_REQUEST', - ), options - assert core.details.get('rule') == rule, options + # The client sets no most on the entities, the calls or the context: + # entities the source does not hold are refused for that alone. core = _refusal_of(indices=list(range(1001))) assert (type(core), core.code) == (StructuredDataError, 'INVALID_REQUEST') diff --git a/packages/structured-data-models-client/tests/test_materialize_state.py b/packages/structured-data-models-client/tests/test_materialize_state.py index fc0588434..20f372cbe 100644 --- a/packages/structured-data-models-client/tests/test_materialize_state.py +++ b/packages/structured-data-models-client/tests/test_materialize_state.py @@ -602,8 +602,8 @@ def test_a_call_the_core_refuses_before_its_check_leaves_it_to_the_next() -> ( [query], connection, indices=[0, 1], - num_hops=7, - expected_error='UNSUPPORTED_REQUEST', + num_hops=0, + expected_error='INVALID_REQUEST', ) == [(0, 0)] assert _calls([query], connection, indices=[0, 1]) == [(6, 1)] other = PredictiveQuery(QUERY, schema) diff --git a/packages/structured-data-models-client/tests/test_payload_rules.py b/packages/structured-data-models-client/tests/test_payload_rules.py index 8e1fac88c..1a2cb693d 100644 --- a/packages/structured-data-models-client/tests/test_payload_rules.py +++ b/packages/structured-data-models-client/tests/test_payload_rules.py @@ -24,12 +24,11 @@ the reference's refusals of an unknown statistic or one on a classification; - exit "payload limits and num_context_samples' bound": the reference's - refusals at each limit (num_context_samples 0 and 10,001, 1,001 entities, - a 16-table graph) and its acceptance at the limit (10,000 context - samples, 1,000 entities, 15 tables); and PD-2: at the byte limit, the - longest payload the reference takes is taken and the bytes the core - produced are the count the reference made, and one byte-step past it is - refused as the reference refuses it; + refusals (num_context_samples 0, 1,001 entities the base does not hold), + and the cases this client departs on (``DEPARTED``): it sets no most on + the context, the tables or the bytes, so a payload the reference refused + past one of them is gathered; and PD-2: the bytes the core produced are + the count the reference made; - exit "empty and reserved-name refusals": the class and code; - exit "inference_config written into the call": the payload's inference section, and the call's model, output and inference as the endpoint @@ -189,6 +188,20 @@ def test_vectors_cover_required_success_and_failure_cases() -> None: CASES = RECORDED['cases'] +#: The recorded cases this client departs from the reference on, on purpose, +#: and why: it gathers each of them where the reference refused. +DEPARTED = { + # An earlier endpoint read at most 15 tables; the model reads as many as + # a request carries, so the client sets no most. + 'star.tables-16': 'no table limit', + # The client holds a payload to no size of its own: a call the endpoint + # refuses as too large is asked again in halves when it is predicted. + 'size.one-past-the-limit': 'no byte limit', + # Nor on the context: a count past 10,000 is gathered. + 'base.regression.num-context-10001': 'no context limit', +} + + @pytest.mark.parametrize('spec', CASES, ids=[c['name'] for c in CASES]) def test_the_core_s_payload_rules_are_the_reference_s( spec: dict[str, Any], monkeypatch: pytest.MonkeyPatch @@ -197,6 +210,11 @@ def test_the_core_s_payload_rules_are_the_reference_s( got, attachments = _on_the_core(spec, monkeypatch) covers = spec['covers'] + if spec['name'] in DEPARTED: + assert expected['outcome'] == 'refused' + assert got['outcome'] == 'payload', got + return + if covers == R.AVG_NULL: # The reference crashes (INTERNAL_ERROR); the core refuses it by # name and does not accept it. @@ -279,22 +297,20 @@ def test_the_core_s_payload_rules_are_the_reference_s( pytest.fail(f'no exit named: {covers}') -def test_a_payload_past_the_byte_limit_is_refused_by_name( +def test_a_payload_past_the_byte_limit_is_gathered_to_be_split( monkeypatch: pytest.MonkeyPatch, ) -> None: - # PD-2, past the limit: the core's refusal is named (a rule or a code - # the reference gives), never a crash, and the case before it is taken. + # PD-2, past the limit: a payload of two rows whose context fits with + # one is gathered, to be asked as smaller calls when it is predicted. past = next(c for c in CASES if c['name'] == 'size.one-past-the-limit') got, attachments = _on_the_core(past, monkeypatch) - assert got['outcome'] == 'refused' - assert not _not_held(got), got - assert got['code'] == past['expected']['code'] - assert not attachments + assert got['outcome'] == 'payload', got + assert len(attachments) == 1 -@pytest.mark.parametrize('padding, accepted', [(5, True), (8, False)]) +@pytest.mark.parametrize('padding, within', [(5, True), (8, False)]) def test_client_only_provenance_does_not_consume_model_payload_limit( - padding, accepted, monkeypatch + padding, within, monkeypatch ): import copy @@ -305,13 +321,13 @@ def test_client_only_provenance_does_not_consume_model_payload_limit( f"UPDATE users SET note=note||'{'a' * padding}' WHERE user_id=0" ) got, attachments = _on_the_core(spec, monkeypatch) - if accepted: - assert got['outcome'] == 'payload', got - state = bytes(attachments[0]) - prefix = b'{"client_generated_keys":{"orders":"__node_id"},' - assert state.startswith(prefix) - model_bytes = b'{' + state[len(prefix) :] + # Past the limit, the two rows are gathered to be split when predicted. + assert got['outcome'] == 'payload', got + state = bytes(attachments[0]) + prefix = b'{"client_generated_keys":{"orders":"__node_id"},' + assert state.startswith(prefix) + model_bytes = b'{' + state[len(prefix) :] + if within: assert len(model_bytes) <= R.PAYLOAD_LIMIT < len(state) else: - assert got['outcome'] == 'refused' - assert '30MB' in got['message'] + assert R.PAYLOAD_LIMIT < len(model_bytes) diff --git a/packages/structured-data-models-client/tests/test_pql_for_tabular.py b/packages/structured-data-models-client/tests/test_pql_for_tabular.py index 4bf351533..c7438da9b 100644 --- a/packages/structured-data-models-client/tests/test_pql_for_tabular.py +++ b/packages/structured-data-models-client/tests/test_pql_for_tabular.py @@ -154,10 +154,10 @@ def test_tabular_context_budget_limits_actual_rows(count): assert request.predict_rows == 1 -def test_tabular_context_budget_above_model_limit_is_refused(): - with pytest.raises(StructuredDataError) as error: - query().materialize(indices=[1], num_context_samples=10_001) - assert error.value.code == 'UNSUPPORTED_REQUEST' +def test_tabular_context_budget_past_the_old_limit_is_taken(): + # The client sets no most: more than the rows there takes them all. + request = query().materialize(indices=[1], num_context_samples=10_001) + assert request.context_rows > 0 @pytest.mark.parametrize('count', [500, 10_001]) diff --git a/packages/structured-data-models-client/tests/test_predict_result.py b/packages/structured-data-models-client/tests/test_predict_result.py index 3cbdfc196..9a9cfa739 100644 --- a/packages/structured-data-models-client/tests/test_predict_result.py +++ b/packages/structured-data-models-client/tests/test_predict_result.py @@ -285,14 +285,10 @@ def test_the_core_s_predict_is_the_reference_s( ) return if spec['covers'] == R.SIZE: - # The reference's code, and a message that explains the limit as - # the old route's does (it names 30MB). - assert got['outcome'] == 'refused', 'past the limit, taken' - assert got['code'] == expected['code'], got['message'] + # The client holds a call to no size of its own, where the + # reference refused one past 30 MiB: it is sent. assert '30MB' in expected['message'] - assert re.search(r'30 ?MB|30 ?MiB|31,?457,?280', got['message']), ( - f'the size refusal does not explain the limit: {got["message"]!r}' - ) + assert got['outcome'] != 'refused', got.get('message') return if spec['name'] in { 'prepare.regression.outputs-explanation-two-rows', diff --git a/packages/structured-data-models-client/tests/test_query_engine_boundary.py b/packages/structured-data-models-client/tests/test_query_engine_boundary.py index 46b3393df..5fea16f30 100644 --- a/packages/structured-data-models-client/tests/test_query_engine_boundary.py +++ b/packages/structured-data-models-client/tests/test_query_engine_boundary.py @@ -300,59 +300,41 @@ def test_a_rows_per_call_that_is_not_a_size_is_refused(given, text): assert 'a call scores a whole number of rows' in str(refused.value) -def test_a_batch_size_of_one_is_refused_for_the_contexts_it_would_hold( - monkeypatch, -): - """A context is gathered per call, so a small enough batch holds one per - entity however few entities there are.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_CALLS', 3) +def test_a_batch_size_of_one_gathers_a_call_per_entity(): + """A context is gathered per call, and the client sets no most on the + calls: a batch of one is a call for each entity.""" with warnings.catch_warnings(): warnings.simplefilter('ignore') - with pytest.raises(StructuredDataError) as refused: - query().materialize( - indices=[5, 6, 7, 8, 9], - num_context_samples=20, - rows_per_call=1, - ) - assert refused.value.details['calls'] == 5 - assert '3 calls' in str(refused.value) - - -def test_a_population_past_the_ceiling_is_refused_before_the_engine_is_reached( - monkeypatch, -): - """Reaching the engine means building the graph and labelling every index, - which for a population far past the ceiling is minutes of work and the - memory with it before anything says no.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_ENTITIES', 4) - with pytest.raises(StructuredDataError) as refused: - query().materialize(indices=list(range(5)), num_context_samples=20) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert refused.value.details['entities'] == 5 + request = query().materialize( + indices=[5, 6, 7, 8, 9], + num_context_samples=20, + rows_per_call=1, + ) + assert len(request.payloads) == 5 -def test_more_calls_than_fit_in_memory_are_refused_before_they_are_built( - monkeypatch, -): - """The calls a population comes to is known only once the engine has said - how many entities it really holds, so this one is checked there, and still - before the batches themselves are built. - """ - from sdm_client.materializers import relational as materializer +def test_more_than_100_000_entities_are_gathered(): + """The client sets no most on the entities one materialize gathers.""" + from sdm_client import KumoRelational - monkeypatch.setattr(materializer, 'MOST_CALLS', 2) + count = 100_001 + frame = pd.DataFrame( + {'uid': range(count + 10), 'label': [*range(count), *[None] * 10]} + ) + users = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + users['users']['label'].stype = 'numerical' with warnings.catch_warnings(): warnings.simplefilter('ignore') - with pytest.raises(StructuredDataError) as refused: - query().materialize( - indices=[5, 6, 7, 8, 9], num_context_samples=20, rows_per_call=2 - ) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert refused.value.details['calls'] == 3 + request = PredictiveQuery( + 'PREDICT users.label FOR EACH users.uid', users + ).materialize( + indices=list(range(count)), + model=KumoRelational(), + rows_per_call=50_001, + ) + assert [ + len(p['predict']['instance_table']['rows']) for p in request.payloads + ] == [50_001, 50_000] def test_a_call_that_fails_keeps_the_answers_already_paid_for(monkeypatch): @@ -731,22 +713,18 @@ def gathered_from(built: Schema) -> set[str]: return set(request.payloads[0]['schema']['related_tables']) -def test_a_schema_wider_than_the_cap_is_gathered_when_its_subgraph_is_not(): - """21 declared tables, 6 gathered, and the endpoint reads 15.""" - from sdm_client.models import RELATIONAL_LIMITS - +def test_a_schema_wider_than_fifteen_tables_is_gathered_as_its_subgraph(): + """21 declared tables, 6 gathered: what is sent is what the query reaches.""" built = star(satellites=20, joining=5) - assert len(built) > RELATIONAL_LIMITS['tables_max'] + assert len(built) > 15 assert len(gathered_from(built)) == 6 -def test_a_subgraph_wider_than_the_cap_is_still_refused(): - from sdm_client.models import RELATIONAL_LIMITS - - with pytest.raises(StructuredDataError) as error: - gathered_from(star(satellites=20, joining=20)) - assert str(RELATIONAL_LIMITS['tables_max']) in str(error.value) +def test_a_subgraph_wider_than_fifteen_tables_is_gathered_whole(): + """An earlier endpoint read at most 15 tables; the model reads as many as + a request carries, so a subgraph of 21 is gathered rather than refused.""" + assert len(gathered_from(star(satellites=20, joining=20))) == 21 # --- a table the request leaves out is said to be left out ------------------ @@ -1095,8 +1073,8 @@ def test_a_lone_table_is_not_told_which_was_taken(): ids=['an empty num_neighbors', 'num_hops of zero'], ) def test_both_spellings_of_no_hop_are_refused_alike(option, names): - """They are one request written two ways, and the endpoint reads between 1 - and 6 hops, so one slipping through to be taken literally was the defect.""" + """They are one request written two ways, and a request samples at least + one hop, so one slipping through to be taken literally was the defect.""" built = catalogue_schema() with pytest.raises(StructuredDataError, match='no hop at all') as raised: diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py new file mode 100644 index 000000000..673c037c4 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The client holds a call to no size of its own and recovers from what the +endpoint says: a call refused as too large, by a gateway, or timed out is +asked again in halves, each with the whole context; a 422 is not.""" + +from __future__ import annotations + +import json +import threading +import warnings +from collections.abc import Iterator +from contextlib import contextmanager + +import pytest +import requests +from test_limits_and_absent_outputs import gathered +from test_tabular_direct_core import client + +from sdm_client import SDMClient +from sdm_client.core.transport import _to_nim_error +from sdm_client.errors import StructuredDataError + + +class Scripted: + r"""An endpoint that fails its calls in turn as ``failures`` say (``None`` + answering), then answers every call with a prediction for each row it + scores.""" + + url = 'http://localhost' + + def __init__(self, *failures: Exception | None) -> None: + self.failures = list(failures) + self.calls: list[dict] = [] + + def predict_body(self, body: bytes) -> bytes: + payload = json.loads(body) + self.calls.append(payload) + failure = self.failures.pop(0) if self.failures else None + if failure is not None: + raise failure + table = payload['predict']['instance_table'] + key = table['columns'].index('instance_id') + return json.dumps( + { + 'id': 'answer', + 'model': 'kumo-relational', + 'metadata': {}, + 'predictions': [ + {'id': str(row[key]), 'row_index': at, 'prediction': 1.0} + for at, row in enumerate(table['rows']) + ], + } + ).encode() + + def predict(self, payload: dict) -> dict: + return json.loads(self.predict_body(json.dumps(payload).encode())) + + def close(self) -> None: + pass + + +def _timed_out() -> StructuredDataError: + r"""The transport's account of an answer slower than its timeout.""" + error = StructuredDataError( + 'Request to http://localhost/v1/predictions timed out after 1 seconds', + code='TRANSPORT_ERROR', + details={'timeout': 1.0}, + ) + error.__cause__ = requests.ReadTimeout() + return error + + +@pytest.mark.parametrize( + 'failure', + [ + _to_nim_error(413, b'{"detail": "too large"}'), + _to_nim_error(504, b'{}'), + _timed_out(), + ], + ids=['413', '504', 'timeout'], +) +def test_a_call_too_large_or_too_slow_is_asked_again_in_halves(failure): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(failure) + result = client(endpoint).predict(request, model='kumo-relational') + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + whole, *halves = endpoint.calls + assert [len(c['predict']['instance_table']['rows']) for c in halves] == [ + 2, + 2, + ] + assert all(c['context'] == whole['context'] for c in halves) + + +def test_a_422_is_not_asked_again_and_says_what_the_endpoint_said(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_to_nim_error(422, b'{"detail": "N_test 4 exceeds"}')) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert len(endpoint.calls) == 1 + assert 'N_test 4 exceeds' in str(refused.value) + + +def test_a_call_of_one_row_refused_as_too_large_is_the_endpoint_s_refusal(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=1) + endpoint = Scripted(_to_nim_error(413, b'{"detail": "context too large"}')) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert len(endpoint.calls) == 1 + assert 'context too large' in str(refused.value) + + +@contextmanager +def _nim( + *failures: int, create: int = 200 +) -> Iterator[tuple[SDMClient, list[dict]]]: + r"""A NIM on a real socket, reached over the client's own HTTP transport: + each call answered row by row, the first ones failed with ``failures`` + in turn, a 0 among them an answer too large to read.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + from test_databricks_splitting import answer + + calls: list[dict] = [] + script = list(failures) + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: + pass + + def _send(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + self._send(200, b'{"data": [{"id": "kumo-tabular"}]}') + + def do_DELETE(self) -> None: + self._send(204, b'') + + def do_POST(self) -> None: + length = int(self.headers.get('Content-Length') or 0) + payload = json.loads(self.rfile.read(length)) + calls.append(payload) + if self.path == '/v1/sessions': + body = b'{"detail": "too large"}' if create != 200 else b'' + self._send(create, body or b'{"session_id": "s1"}') + return + if script: + status = script.pop(0) + if status == 0: + self._send(200, b' ' * (64 * 1024 * 1024 + 1)) + else: + self._send(status, b'{"detail": "refused"}') + return + self._send(200, json.dumps(answer(payload)).encode()) + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread( + target=lambda: server.serve_forever(poll_interval=0.01), daemon=True + ) + thread.start() + try: + url = f'http://127.0.0.1:{server.server_address[1]}' + with SDMClient(url, max_retries=0) as nim: + yield nim, calls + finally: + server.shutdown() + server.server_close() + + +def _tabular(rows: int): + import numpy as np + import pandas as pd + + from sdm_client import Request + + context = pd.DataFrame( + { + 'row_id': np.arange(40), + 'feature': np.arange(40) % 7, + 'target': np.arange(40) % 2, + } + ) + scoring = pd.DataFrame( + {'row_id': np.arange(100, 100 + rows), 'feature': np.arange(rows) % 7} + ) + return Request( + context, + scoring, + target='target', + task='classification', + ) + + +def test_a_tabular_request_is_scored_in_calls_of_10_000_rows(): + with _nim() as (nim, calls): + result = nim.predict(_tabular(10_001), model='kumo-tabular') + assert [len(c['predict']['instance_table']['rows']) for c in calls] == [ + 10_000, + 1, + ] + assert all(c['context'] == calls[0]['context'] for c in calls) + assert list(result.to_pandas()['row_index']) == list(range(10_001)) + + +@pytest.mark.parametrize( + 'failure', [413, 504, 0], ids=['413', '504', 'answer-too-large'] +) +def test_a_tabular_call_too_large_is_asked_again_in_halves(failure, caplog): + caplog.set_level('INFO', logger='sdm_core') + with _nim(failure) as (nim, calls): + result = nim.predict(_tabular(4), model='kumo-tabular') + assert [len(c['predict']['instance_table']['rows']) for c in calls] == [ + 4, + 2, + 2, + ] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + assert any('smaller calls' in r.getMessage() for r in caplog.records) + + +def _sizes(endpoint: Scripted) -> list[int]: + return [len(c['predict']['instance_table']['rows']) for c in endpoint.calls] + + +def _too_large(): + return _to_nim_error(413, b'{"detail": "too large"}') + + +def test_a_half_still_too_large_is_split_again(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_too_large(), _too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + # 4 refused, then 2, then each run of one; the size kept, the second + # half is asked one row at a time without trying two. + assert _sizes(endpoint) == [4, 2, 1, 1, 1, 1] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + + +def test_an_odd_count_is_split_into_the_larger_half_first(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=3) + endpoint = Scripted(_too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [3, 2, 1] + assert list(result.to_pandas()['row_index']) == [0, 1, 2] + + +def test_the_size_that_worked_is_kept_for_the_later_calls(): + import pandas as pd + + from sdm_client import KumoRelational, PredictiveQuery, Schema + + frame = pd.DataFrame({'uid': range(38), 'label': [*range(30), *[None] * 8]}) + schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + schema['users']['label'].stype = 'numerical' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + request = PredictiveQuery( + 'PREDICT users.label FOR EACH users.uid', schema + ).materialize( + indices=list(range(30, 38)), + model=KumoRelational(), + rows_per_call=4, + ) + endpoint = Scripted(_too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + # The first batch's 4 refused and asked as 2 and 2; the second batch is + # asked as 2 and 2 at once. Rows are numbered across the whole request. + assert _sizes(endpoint) == [4, 2, 2, 2, 2] + assert list(result.to_pandas()['row_index']) == list(range(8)) + + +def test_the_halves_answered_before_one_fails_are_kept(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted( + _too_large(), None, _to_nim_error(422, b'{"detail": "N_test"}') + ) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [4, 2, 2] + assert refused.value.details['rows_answered'] == 2 + assert len(refused.value.details['answered']) == 2 + + +@pytest.mark.parametrize('status', [502, 503, 500]) +def test_a_temporary_failure_is_never_split(status): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_to_nim_error(status, b'{"detail": "later"}')) + with pytest.raises(StructuredDataError): + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [4] + + +def test_a_batch_is_halved_at_most_three_times(): + request = gathered(context_rows=5, run_mode='fast', predict_rows=16) + endpoint = Scripted(*[_too_large() for _ in range(20)]) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [16, 8, 4, 2] + assert 'halved 3 times' in str(refused.value) + assert refused.value.details['halvings'] == 3 + + +def test_a_timeout_counts_toward_the_three_halvings(): + request = gathered(context_rows=5, run_mode='fast', predict_rows=16) + endpoint = Scripted(_timed_out(), *[_too_large() for _ in range(20)]) + with pytest.raises(StructuredDataError): + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [16, 8, 4, 2] + + +def test_a_session_whose_context_is_too_large_is_refused_naming_it(): + """A create call carries only the context, which cannot be split.""" + with _nim(create=413) as (nim, calls): + with pytest.raises(StructuredDataError) as refused: + nim.fit(_tabular(4), model='kumo-tabular') + assert len(calls) == 1 + assert 'the context is too large for this endpoint' in str(refused.value) + + +def test_a_pinned_session_s_scoring_call_is_asked_in_halves(): + """A scoring call against a pinned context carries no context: its rows + are halved, whatever the context's size.""" + with _nim(413) as (nim, calls): + fitted = nim.fit(_tabular(4), model='kumo-tabular') + result = fitted.predict(_tabular(4)) + scoring = [c for c in calls if 'context' not in c] + assert [len(c['predict']['instance_table']['rows']) for c in scoring] == [ + 4, + 2, + 2, + ] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] diff --git a/packages/structured-data-models-client/tests/test_serving_route.py b/packages/structured-data-models-client/tests/test_serving_route.py index a8bc2287f..11e8b7f19 100644 --- a/packages/structured-data-models-client/tests/test_serving_route.py +++ b/packages/structured-data-models-client/tests/test_serving_route.py @@ -308,8 +308,10 @@ def test_a_call_the_sdk_retries_sends_the_same_bytes_again() -> None: ], ) def test_endpoint_failure_is_sanitized_and_classified(reply: Reply) -> None: - error, received = _predict(_materialized(), reply) status, body, _ = reply + # A 500 is tried once more, so it is answered twice. + replies = (reply, reply) if status == 500 else (reply,) + error, received = _predict(_materialized(), *replies) if isinstance(body, dict): provider_code = body['error_code'] code = { @@ -330,7 +332,7 @@ def test_endpoint_failure_is_sanitized_and_classified(reply: Reply) -> None: 403 if status == 403 else 503, 'PERMISSION_DENIED' if status == 403 else None, ) - assert len(received) == 1 + assert len(received) == len(replies) @pytest.mark.parametrize( @@ -395,17 +397,21 @@ def test_malformed_serving_response_is_a_transport_error( assert len(received) == 1 -def test_a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() -> None: +def test_a_call_past_the_caller_s_limit_is_split_before_it_is_sent() -> None: request = _materialized() _, received = _predict(request) [(_, body, _)] = received - # The limit applies to the actual UTF-8 envelope posted to the SDK. + # The limit applies to the actual UTF-8 envelope posted to the SDK: past + # it, the call's rows are sent in calls that fit, the context whole. core, on_core = _predict(request, max_request_bytes=len(body) - 1) - assert on_core == [] - assert isinstance(core, StructuredDataError), core - assert core.details['limit'] == len(body) - 1 - assert core.details['bytes'] == len(body) - # The size itself is admitted. + assert not isinstance(core, Exception), core + assert len(on_core) == 2 + assert all(len(sent) <= len(body) - 1 for (_, sent, _) in on_core) + assert all( + _call(sent)['context'] == _call(body)['context'] + for (_, sent, _) in on_core + ) + # The size itself is admitted as one call. core, received = _predict(request, max_request_bytes=len(body)) assert not isinstance(core, Exception), core [(_, sent, _)] = received @@ -426,11 +432,13 @@ def test_utf8_serving_preserves_public_payload_and_measures_the_sent_body() -> ( assert call[field] == request.payloads[0][field], field assert result.to_pandas()['id'].tolist() == [0, 1] assert result.to_pandas()['prediction'].tolist() == [1.0, 1.0] - refused, sent = _predict(request, max_request_bytes=len(body) - 1) - assert isinstance(refused, StructuredDataError) - assert sent == [] - assert refused.details['bytes'] == len(body) - assert refused.details['limit'] == len(body) - 1 + # Past the endpoint's limit, the call is split before it is sent, each + # part within it, and the answer is the same. + split, sent = _predict(request, max_request_bytes=len(body) - 1) + assert not isinstance(split, Exception), split + assert len(sent) == 2 + assert all(len(part) <= len(body) - 1 for (_, part, _) in sent) + assert split.to_pandas()['id'].tolist() == [0, 1] @pytest.mark.parametrize( @@ -444,7 +452,8 @@ def test_the_sdk_s_debug_log_carries_no_rows_on_the_core_route( ) -> None: request = _materialized() caplog.set_level(logging.DEBUG) - replies = () if reply is None else (reply,) + # A 500 is tried once more, so it is answered twice. + replies = () if reply is None else (reply, reply) outcome, received = _predict(request, *replies) assert received if reply is not None: diff --git a/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py b/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py index 65d4d02c7..35f5bd4d0 100644 --- a/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py +++ b/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py @@ -11,9 +11,14 @@ def expected_case(name): """Same-runtime pandas ordering, independent of client implementation.""" family, entry, shape = name.split('/') - n, wanted = (81, 40) if entry == 'indices' else (10002, 10000) + # The predict route sends the frame's labelled rows whole: no sampling. + n, wanted = (81, 40) if entry == 'indices' else (10002, 10001) frame = frame_for(family, n, shape) - selected = frame.iloc[:-1].sample(n=wanted, random_state=42).sort_index() + selected = ( + frame.iloc[:-1] + if entry == 'predict' + else frame.iloc[:-1].sample(n=wanted, random_state=42).sort_index() + ) ids = selected['id'].tolist() return { 'context_count': wanted, diff --git a/packages/structured-data-models-client/tests/test_tabular_direct_core.py b/packages/structured-data-models-client/tests/test_tabular_direct_core.py index 0070d69a7..802f6fedf 100644 --- a/packages/structured-data-models-client/tests/test_tabular_direct_core.py +++ b/packages/structured-data-models-client/tests/test_tabular_direct_core.py @@ -340,9 +340,10 @@ def test_direct_context_limit_never_samples() -> None: scoring = pd.DataFrame({'feature': [30, 31]}) request = Request(context, scoring, target='target', task='classification') endpoint = Endpoint() - with pytest.raises(StructuredDataError, match='reads at most 10,000'): - client(endpoint).predict(request, model='kumo-tabular') - assert endpoint.calls == [] + # A direct context is sent as it was given, held to no size. + client(endpoint).predict(request, model='kumo-tabular') + (sent,) = endpoint.calls + assert len(sent['context']['instance_table']['rows']) == 10001 def test_target_container_guard_is_immediate() -> None: diff --git a/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py b/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py index 5c2b021fb..553fdde3b 100644 --- a/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py +++ b/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py @@ -14,7 +14,8 @@ def expected_case(name): """Pandas ordering and Decimal effects without invoking any client code.""" family, entry, shape, mode = name.split('/') - n, wanted = (81, 40) if entry == 'indices' else (10002, 10000) + # The predict route sends the frame's labelled rows whole: no sampling. + n, wanted = (81, 40) if entry == 'indices' else (10002, 10001) with localcontext() as context: context.prec = 2 context.traps[FloatOperation] = False @@ -25,6 +26,8 @@ def expected_case(name): try: selected = ( frame.iloc[:-1] + if entry == 'predict' + else frame.iloc[:-1] .sample(n=wanted, random_state=42) .sort_index(kind='stable') ) @@ -38,17 +41,20 @@ def expected_case(name): ), } ids = selected['id'].tolist() + flags = { + signal.__name__ for signal, set_ in context.flags.items() if set_ + } + if entry == 'predict' and family == 'decimal_float': + # The whole frame is encoded unsorted, which compares its + # decimals with floats where a sample's sort would have first. + flags.add('FloatOperation') return { 'context_count': wanted, 'context_ids_sha256': digest(ids), 'encoded_ids_sha256': digest(ids), 'index_dtype': str(selected.index.dtype), 'predictions': [{'row_index': 0, 'id': n - 1, 'prediction': 0.0}], - 'flags': sorted( - signal.__name__ - for signal, set_ in context.flags.items() - if set_ - ), + 'flags': sorted(flags), } diff --git a/packages/structured-data-models-client/tests/test_tabular_pql_counts.py b/packages/structured-data-models-client/tests/test_tabular_pql_counts.py index 49ae3f5b3..8536d6aba 100644 --- a/packages/structured-data-models-client/tests/test_tabular_pql_counts.py +++ b/packages/structured-data-models-client/tests/test_tabular_pql_counts.py @@ -36,23 +36,17 @@ def test_public_count_task_journey(monkeypatch, route, name): } } assert not events and not warnings - elif name == 'warning_raise': - assert outcome == { - 'error': { - 'type': 'StopWarning', - 'code': None, - 'message': 'warning handler stopped', - } - } - assert len(events) == 1 and events[0][0] == 'warning' - assert len(warnings) == 1 and 'were sampled' in warnings[0]['message'] else: assert 'error' not in outcome + # The context is sampled only to a count the caller names: past the + # old cap, with none named, every labelled row is sent, unremarked. expected_count = ( 12 if name.startswith('task_') else 4 if name == 'count_owner' + else actual['cap'] + 1 + if name in ('over_default', 'warning_record', 'warning_raise') else actual['cap'] ) assert len(outcome['context_ids']) == expected_count @@ -67,12 +61,7 @@ def test_public_count_task_journey(monkeypatch, route, name): ['endpoint', expected_count], ['predicted'], ] - if name == 'count_owner': - assert events[:-3] == [['count_gt', actual['cap']]] - elif name.startswith('task_') or name in ( - 'over_default', - 'warning_record', - ): + if name.startswith('task_'): assert len(events) == 4 and events[0][0] == 'warning' else: assert len(events) == 3 @@ -81,9 +70,5 @@ def test_public_count_task_journey(monkeypatch, route, name): len(warnings) == 1 and 'two whole numbers' in warnings[0]['message'] ) - elif name in ('over_default', 'warning_record'): - assert ( - len(warnings) == 1 and 'were sampled' in warnings[0]['message'] - ) else: assert not warnings diff --git a/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py b/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py index 07e3ff93e..684e7631b 100644 --- a/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py +++ b/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py @@ -48,7 +48,9 @@ def materialize(frame, entry, wanted): return query(frame).materialize(**arguments, task='regression') -@pytest.mark.parametrize('entry', ['indices', 'predict']) +# Only a count the caller names reduces the context; the predict route takes +# no count, and sends its context whole. +@pytest.mark.parametrize('entry', ['indices']) @pytest.mark.parametrize('owner', ['frame_sort', 'sampled_frame_sort', 'label']) def test_reduced_custom_sorting_is_refused_without_execution( monkeypatch, entry, owner diff --git a/packages/structured-data-models-client/tests/test_temporal_link_batching.py b/packages/structured-data-models-client/tests/test_temporal_link_batching.py index 02c6f1bf1..c1e0d2e6c 100644 --- a/packages/structured-data-models-client/tests/test_temporal_link_batching.py +++ b/packages/structured-data-models-client/tests/test_temporal_link_batching.py @@ -106,7 +106,8 @@ def materialize(schema, count=201, rows_per_call=None, query=QUERY): (200, None, [200]), (201, None, [200, 1]), (201, 100, [100, 100, 1]), - (201, 1000, [200, 1]), + # A size asked is taken as asked; 200 is only the default. + (201, 1000, [201]), ], ) def test_pql_link_batches_preserve_every_entity( @@ -135,15 +136,12 @@ def test_node_predictions_keep_the_ordinary_limit(link_schema): ] == [201] -def test_link_batches_respect_total_call_limit(link_schema, monkeypatch): - from sdm_client.materializers import relational - - monkeypatch.setattr(relational, 'MOST_CALLS', 1) - with pytest.raises(StructuredDataError) as caught: - materialize(link_schema) - assert caught.value.code == 'UNSUPPORTED_REQUEST' - assert caught.value.details['calls'] == 2 - assert caught.value.details['rows_per_call'] == 200 +def test_link_batches_are_as_many_calls_as_the_entities_need(link_schema): + request = materialize(link_schema) + assert [ + len(p['predict']['instance_table']['rows']) for p in request.payloads + ][:1] == [200] + assert len(request.payloads) == 2 def test_link_batch_result_order_and_partial_failure( diff --git a/packages/structured-data-models-client/tests/test_the_predict_route.py b/packages/structured-data-models-client/tests/test_the_predict_route.py index ad1bb939b..f7f94737b 100644 --- a/packages/structured-data-models-client/tests/test_the_predict_route.py +++ b/packages/structured-data-models-client/tests/test_the_predict_route.py @@ -761,21 +761,13 @@ def recording(self: Any, adapter: Any, body: bytes) -> Any: return written, received, refused.value -def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( +def test_the_bytes_sent_are_the_core_s_at_any_size( monkeypatch: pytest.MonkeyPatch, ) -> None: - # Register PD-2: the HTTP transport sends the - # core's call as the core wrote it, so the bytes the endpoint receives - # are the bytes the core counted against the limit, characters past - # ASCII as their own bytes, never json.dumps' escapes. At the limit's - # edge the call is sent, at most the limit; one character past it, - # nothing is sent. Since the reference's 26ecdd28 the call - # leaves out the engine's inference config, which the payload holds, - # so the payload is the larger and meets the limit first: the edge is - # the payload's, which materialize names in its refusal, and one past - # it materialize refuses (the call's own refusal, past the limit with - # the payload within it, is sdm-core's - # a_call_past_the_limit_is_refused_before_it_is_sent). + # Register PD-2: the HTTP transport sends the core's call as the core + # wrote it, characters past ASCII as their own bytes, never json.dumps' + # escapes; and the client holds a call to no size of its own, so one + # past the 30 MiB the reference refused is sent. limit = 30 * 1024 * 1024 lengths = [] for emoji, ascii in ((0, 0), (1, 0), (0, 1)): @@ -784,39 +776,17 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( ) assert written == received lengths.append(len(received[0])) - base, wide, narrow = ( - lengths[0], - lengths[1] - lengths[0], - lengths[2] - lengths[0], - ) - assert wide % 4 == 0 and narrow == 1 - emoji = (limit - base) // wide - ascii = (limit - base - emoji * wide) // narrow - # The call's edge is past the payload's: materialize refuses it, naming - # the payload's size, and the payload's edge is that much nearer. - with pytest.raises(StructuredDataError) as over: - _emoji_request(monkeypatch, emoji, ascii) - details = over.value.details - assert over.value.code == 'INVALID_REQUEST' - assert details['field'] == 'payload' and details['limit'] == limit - excess = details['bytes'] - limit - assert 0 < excess < wide * 64 - room = emoji * wide + ascii * narrow - excess - emoji = room // wide - ascii = (room - emoji * wide) // narrow + base, wide = lengths[0], lengths[1] - lengths[0] + assert wide % 4 == 0 and lengths[2] - lengths[0] == 1 + emoji = (limit - base) // wide + 1 written, received, refused = _sent( - monkeypatch, _emoji_request(monkeypatch, emoji, ascii) + monkeypatch, _emoji_request(monkeypatch, emoji, 0) ) assert refused.code == 'RECORDED' assert written == received - assert len(received[0]) == base + emoji * wide + ascii * narrow <= limit + assert len(received[0]) == base + emoji * wide > limit assert b'\\ud83d' not in received[0] assert '\U0001f600'.encode() in received[0] - with pytest.raises(StructuredDataError) as past: - _emoji_request(monkeypatch, emoji, ascii + 1) - assert past.value.code == 'INVALID_REQUEST' - assert past.value.details['field'] == 'payload' - assert past.value.details['bytes'] == limit + 1 def test_embeddings_are_the_old_route_s_on_the_core( diff --git a/packages/structured-data-models-client/tests/test_transport_socket.py b/packages/structured-data-models-client/tests/test_transport_socket.py index 1e4ae9faa..0954ef974 100644 --- a/packages/structured-data-models-client/tests/test_transport_socket.py +++ b/packages/structured-data-models-client/tests/test_transport_socket.py @@ -377,17 +377,16 @@ def test_a_detail_that_is_not_text_names_no_diagnosis(): def test_a_request_is_sent_again_only_for_what_sending_it_again_can_fix(): - """A 500 is not retried. The payload and the model are the same on every - attempt, so a server fault reproduces. Caller mistakes this endpoint - reports as a 500 are reclassified from the body, but only after three more - uploads of the whole gathered context. Overload, - gateway and timeout are the transient ones and are still retried.""" + """A connection error, a 500, a 502 or a 503 is retried with backoff, up + to ``max_retries``; overload is too. A 504 is not tried again at its + size, as the core asks its rows in halves; a 422 is the caller's, and + sending the same request again cannot fix it.""" for status, sends in ( - (500, 1), + (500, 4), (429, 4), (502, 4), (503, 4), - (504, 4), + (504, 1), (422, 1), ): with _serve(_Reply(status=status, body=_EMPTY_CONTEXT)) as ( diff --git a/packages/structured-data-models-client/tests/test_wording.py b/packages/structured-data-models-client/tests/test_wording.py index 12ac5eef7..2ca7c8b3e 100644 --- a/packages/structured-data-models-client/tests/test_wording.py +++ b/packages/structured-data-models-client/tests/test_wording.py @@ -515,16 +515,10 @@ def _users() -> Schema: def test_the_context_size_refused_is_named_as_the_caller_gave_it() -> None: # The old client's words, with the value as given, however large - # (review r1, P1-2 and P1-3). + # (review r1, P1-2 and P1-3). No size past one is refused. query = PredictiveQuery( 'PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', _users() ) - with pytest.raises(StructuredDataError) as past: - query.materialize(indices=[1], num_context_samples=10_001) - assert str(past.value).split('] ', 1)[1] == ( - 'num_context_samples is 10,001, and the endpoint accepts at most ' - '10,000 context rows' - ) with pytest.raises(StructuredDataError) as below: query.materialize(indices=[1], num_context_samples=-(10**100)) assert str(below.value).split('] ', 1)[1] == ( diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 73a67e69f..91ec993ea 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -71 sha256:6041dbc10ec757561b56eb6cc0decdedf1c9e7f7e1268e641e03401b492ea8be sdm-contract-71.json +72 sha256:0aed25018f2ee08388374658ca173d9d8aa960af18e6223d65d24a90a1896dd3 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-71.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json similarity index 99% rename from rust/crates/sdm-contract/schema/sdm-contract-71.json rename to rust/crates/sdm-contract/schema/sdm-contract-72.json index dcd05e49d..723b6467a 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-71.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -1,5 +1,5 @@ { - "$id": "sdm-contract/71", + "$id": "sdm-contract/72", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ControlMessage", "description": "Every control message that crosses between a host and the core.", @@ -306,12 +306,6 @@ "DirectValue.select_tabular_rows": { "$ref": "#/$defs/SelectedTabularRows" }, - "CheckTabularCap.input": { - "$ref": "#/$defs/CheckTabularCap" - }, - "DirectValue.check_tabular_cap": { - "$ref": "#/$defs/CheckedTabularCap" - }, "ObserveTabularMembership.input": { "$ref": "#/$defs/ObserveTabularMembership" }, @@ -699,8 +693,6 @@ "tabular_arrow_session", "SelectTabularRows.input", "DirectValue.select_tabular_rows", - "CheckTabularCap.input", - "DirectValue.check_tabular_cap", "ObserveTabularMembership.input", "DirectValue.observe_tabular_membership", "DecideTabularTask.input", @@ -3806,31 +3798,6 @@ "above_population" ] }, - "CheckTabularCap": { - "description": "Reached native comparison against the model cap, before frame length reads.", - "type": "object", - "properties": { - "over_limit": { - "type": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "over_limit" - ] - }, - "CheckedTabularCap": { - "type": "object", - "properties": { - "accepted": { - "type": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "accepted" - ] - }, "ObserveTabularMembership": { "type": "object", "properties": { @@ -7930,22 +7897,6 @@ ], "format": "uint32", "minimum": 0 - }, - "most_entities": { - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 0 - }, - "most_calls": { - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0 } }, "additionalProperties": false, @@ -10915,7 +10866,7 @@ ] }, "NamedTable": { - "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads (design choice 12), which a host naming a table\nby a string gives as one part.", + "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads, which a host naming a table\nby a string gives as one part.", "type": "object", "properties": { "name": { @@ -10993,7 +10944,7 @@ "type": "string" }, "source": { - "description": "The relation of the source this table reads (design choice 12): its\nown name, one part, unless the caller named it otherwise (decision\nE59).", + "description": "The relation of the source this table reads: its\nown name, one part, unless the caller named it otherwise (decision\nE59).", "$ref": "#/$defs/Relation" }, "columns": { @@ -13689,24 +13640,6 @@ "input" ] }, - { - "description": "An input-sized direct value, carried as typed JSON bytes.", - "type": "object", - "properties": { - "input": { - "$ref": "#/$defs/AttachmentId" - }, - "op": { - "type": "string", - "const": "check_tabular_cap" - } - }, - "additionalProperties": false, - "required": [ - "op", - "input" - ] - }, { "description": "An input-sized direct value, carried as typed JSON bytes.", "type": "object", @@ -14481,24 +14414,6 @@ "format": "uint32", "minimum": 0 }, - "most_entities": { - "description": "The most entities one `materialize` gathers, across the calls it is\nasked in, which the host lets its caller set (`main`'s\n`MOST_ENTITIES`); 100,000 where it names none (design choice 1,\nmulti-call `predict`, read which asks a request of more\nrows than one call scores in several calls; until then the core\nrefuses more than one call's rows, a lower bound than either).", - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 0 - }, - "most_calls": { - "description": "The most model calls one request is asked in (`main`'s\n`MOST_CALLS`); 100 where the host names none (read as\n`most_entities`).", - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0 - }, "sampler_state": { "description": "The sampler state the materializer's last run left, which the host\nhands back as it took it; none for a materializer's\nfirst run.", "anyOf": [ @@ -14740,7 +14655,7 @@ "$ref": "#/$defs/Determinism" }, "max_seed": { - "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n(design choice 1). The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", + "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n. The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", "type": "integer", "format": "uint64", "minimum": 0 @@ -14779,7 +14694,7 @@ ] }, "fallback": { - "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads (design choice 1, the in-memory fallback); left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", + "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads; left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", "anyOf": [ { "$ref": "#/$defs/FallbackLimits" @@ -17456,7 +17371,6 @@ "direct_scored", "direct_target_missing", "direct_target_scored", - "direct_context_limit", "tabular_missing_row_index", "direct_multilabel", "direct_relational_frames", @@ -17611,7 +17525,6 @@ "undeclared", "seed_past_range", "big_integer_past_type", - "lag_past_payload", "date_literal_packed", "local_zone_unread", "clock_past_years", @@ -17663,7 +17576,6 @@ "model_lacks_shape", "model_lacks_task", "model_lacks_outputs", - "context_past_model", "no_context_rows", "too_many_classes", "one_class_column", @@ -17682,9 +17594,6 @@ "explanation_fast_mode", "options_unread", "options_unread_statistic", - "payload_smaller_calls", - "payload_no_smaller_call", - "call_past_endpoint", "outputs_none", "not_finite", "past_int64", @@ -17695,7 +17604,6 @@ "lag_past_times", "num_context_samples_not_whole", "num_context_samples_below_one", - "num_context_samples_past_most", "num_hops_not_whole", "num_hops_none", "random_seed_not_whole", @@ -17720,8 +17628,6 @@ "statistic_not_a_regression", "no_hop_num_hops", "no_hop_num_neighbors", - "hops_past_endpoint", - "hops_past_endpoint_uncounted", "inference_run_mode_invalid", "inference_config_not_an_object", "inference_option_unknown", @@ -17760,7 +17666,6 @@ "entity_table_empty", "entity_key_missing", "target_not_a_feature", - "context_past_rows", "held_by_several_rows_one", "held_by_several_rows", "held_by_several_rows_many", @@ -17769,7 +17674,6 @@ "frame_key_missing_parts", "frame_no_valid_rows", "not_in_table", - "subgraph_past_tables", "unresolved", "anchor_not_a_time", "anchor_before_data", @@ -17786,9 +17690,6 @@ "no_entity_passed_several", "anchor_too_late_several", "forecast_top_k", - "most_entities", - "most_entities_parts", - "most_calls", "not_yet_materialize", "missing_index_table", "missing_index_tables", @@ -17898,6 +17799,8 @@ "seed_ignored", "serving_value_not_json", "serving_past_limit", + "call_too_large_after_halving", + "context_too_large_for_endpoint", "serving_not_json", "serving_not_an_object", "serving_refused", @@ -18010,7 +17913,7 @@ } }, "answered": { - "description": "What a request asked in several model calls had been answered when\na later call failed (design choice 1, `main`'s answered parts,\n): the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", + "description": "What a request asked in several model calls had been answered when\na later call failed: the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", "anyOf": [ { "$ref": "#/$defs/AnsweredParts" diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index e2a198142..af9b0bbe0 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -787,7 +787,6 @@ sentences! { DirectScored = "direct_scored", E::InvalidRequest; DirectTargetMissing = "direct_target_missing", E::InvalidRequest; DirectTargetScored = "direct_target_scored", E::InvalidRequest; - DirectContextLimit = "direct_context_limit", E::UnsupportedRequest; TabularMissingRowIndex = "tabular_missing_row_index", E::InvalidResponse; DirectMultilabel = "direct_multilabel", E::UnsupportedRequest; DirectRelationalFrames = "direct_relational_frames", E::UnsupportedRequest; @@ -942,7 +941,6 @@ sentences! { Undeclared = "undeclared", E::UnsupportedRequest; SeedPastRange = "seed_past_range", E::InvalidRequest; BigIntegerPastType = "big_integer_past_type", E::ValueOutOfRange; - LagPastPayload = "lag_past_payload", E::InvalidRequest; DateLiteralPacked = "date_literal_packed", E::InvalidRequest, rule "date_literal_packed"; LocalZoneUnread = "local_zone_unread", E::InvalidRequest; ClockPastYears = "clock_past_years", E::InvalidRequest; @@ -994,7 +992,6 @@ sentences! { ModelLacksShape = "model_lacks_shape", E::UnsupportedRequest; ModelLacksTask = "model_lacks_task", E::UnsupportedRequest; ModelLacksOutputs = "model_lacks_outputs", E::UnsupportedRequest; - ContextPastModel = "context_past_model", E::UnsupportedRequest; NoContextRows = "no_context_rows", E::UnsupportedRequest; TooManyClasses = "too_many_classes", E::UnsupportedRequest; OneClassColumn = "one_class_column", E::UnsupportedRequest; @@ -1013,9 +1010,6 @@ sentences! { ExplanationFastMode = "explanation_fast_mode", E::UnsupportedRequest; OptionsUnread = "options_unread", E::UnsupportedRequest; OptionsUnreadStatistic = "options_unread_statistic", E::UnsupportedRequest; - PayloadSmallerCalls = "payload_smaller_calls", E::InvalidRequest; - PayloadNoSmallerCall = "payload_no_smaller_call", E::InvalidRequest; - CallPastEndpoint = "call_past_endpoint", E::InvalidRequest; OutputsNone = "outputs_none", E::InvalidRequest; NotFinite = "not_finite", E::InvalidRequest; PastInt64 = "past_int64", E::InvalidRequest; @@ -1026,7 +1020,6 @@ sentences! { LagPastTimes = "lag_past_times", E::ValueOutOfRange; NumContextSamplesNotWhole = "num_context_samples_not_whole", E::InvalidRequest; NumContextSamplesBelowOne = "num_context_samples_below_one", E::InvalidRequest; - NumContextSamplesPastMost = "num_context_samples_past_most", E::UnsupportedRequest; NumHopsNotWhole = "num_hops_not_whole", E::InvalidRequest; NumHopsNone = "num_hops_none", E::InvalidRequest; RandomSeedNotWhole = "random_seed_not_whole", E::InvalidRequest; @@ -1051,8 +1044,6 @@ sentences! { StatisticNotARegression = "statistic_not_a_regression", E::InvalidRequest; NoHopNumHops = "no_hop_num_hops", E::InvalidRequest; NoHopNumNeighbors = "no_hop_num_neighbors", E::InvalidRequest; - HopsPastEndpoint = "hops_past_endpoint", E::UnsupportedRequest; - HopsPastEndpointUncounted = "hops_past_endpoint_uncounted", E::UnsupportedRequest; InferenceRunModeInvalid = "inference_run_mode_invalid", E::InvalidRequest; InferenceConfigNotAnObject = "inference_config_not_an_object", E::InvalidRequest; InferenceOptionUnknown = "inference_option_unknown", E::InvalidRequest; @@ -1091,7 +1082,6 @@ sentences! { EntityTableEmpty = "entity_table_empty", E::InvalidRequest; EntityKeyMissing = "entity_key_missing", E::InvalidRequest; TargetNotAFeature = "target_not_a_feature", E::InvalidRequest; - ContextPastRows = "context_past_rows", E::InvalidRequest; HeldBySeveralRowsOne = "held_by_several_rows_one", E::InvalidRequest; HeldBySeveralRows = "held_by_several_rows", E::InvalidRequest; HeldBySeveralRowsMany = "held_by_several_rows_many", E::InvalidRequest; @@ -1100,7 +1090,6 @@ sentences! { FrameKeyMissingParts = "frame_key_missing_parts", E::InvalidRequest; FrameNoValidRows = "frame_no_valid_rows", E::InvalidRequest; NotInTable = "not_in_table", E::InvalidRequest; - SubgraphPastTables = "subgraph_past_tables", E::InvalidRequest; Unresolved = "unresolved", E::InvalidRequest; AnchorNotATime = "anchor_not_a_time", E::InvalidRequest; AnchorBeforeData = "anchor_before_data", E::InvalidRequest; @@ -1117,9 +1106,6 @@ sentences! { NoEntityPassedSeveral = "no_entity_passed_several", E::InvalidRequest; AnchorTooLateSeveral = "anchor_too_late_several", E::InvalidRequest; ForecastTopK = "forecast_top_k", E::InvalidRequest, rule "forecast_top_k"; - MostEntities = "most_entities", E::UnsupportedRequest, rule "most_entities"; - MostEntitiesParts = "most_entities_parts", E::UnsupportedRequest, rule "most_entities"; - MostCalls = "most_calls", E::UnsupportedRequest, rule "most_calls"; NotYetMaterialize = "not_yet_materialize", E::UnsupportedRequest; MissingIndexTable = "missing_index_table", W::MissingIndices; MissingIndexTables = "missing_index_tables", W::MissingIndices; @@ -1229,6 +1215,8 @@ sentences! { SeedIgnored = "seed_ignored", W::SeedIgnored; ServingValueNotJson = "serving_value_not_json", E::InvalidRequest; ServingPastLimit = "serving_past_limit", E::InvalidRequest; + CallTooLargeAfterHalving = "call_too_large_after_halving", E::InvalidRequest; + ContextTooLargeForEndpoint = "context_too_large_for_endpoint", E::InvalidRequest; ServingNotJson = "serving_not_json", E::TransportError; ServingNotAnObject = "serving_not_an_object", E::TransportError; ServingRefused = "serving_refused", E::EndpointRefused; diff --git a/rust/crates/sdm-contract/src/frames.rs b/rust/crates/sdm-contract/src/frames.rs index a84df1c27..9e2c4c48f 100644 --- a/rust/crates/sdm-contract/src/frames.rs +++ b/rust/crates/sdm-contract/src/frames.rs @@ -83,8 +83,6 @@ pub struct FrameCall { #[serde(default)] pub options: Vec, pub most_parts: Option, - pub most_entities: Option, - pub most_calls: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/rust/crates/sdm-contract/src/lib.rs b/rust/crates/sdm-contract/src/lib.rs index 024697026..47893f2cb 100644 --- a/rust/crates/sdm-contract/src/lib.rs +++ b/rust/crates/sdm-contract/src/lib.rs @@ -56,7 +56,7 @@ pub use scalar::Scalar; pub use types::{ColumnType, Field, Unit}; /// The contract version this build speaks. -pub const CONTRACT_VERSION: u32 = 71; +pub const CONTRACT_VERSION: u32 = 72; /// The JSON Schema of [`ControlMessage`], as committed: pretty-printed, with /// a trailing newline. Its `$defs` hold [`ops::Attachments`] too, the JSON diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 6c9daf634..07b8954e2 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -68,7 +68,6 @@ pub enum Operation { DecideTabularTask(crate::direct::DirectInput), OrderSampledTabularRows(crate::direct::DirectInput), SelectTabularRows(crate::direct::DirectInput), - CheckTabularCap(crate::direct::DirectInput), CheckDirectShape(crate::direct::DirectInput), CheckDirectCell(crate::direct::DirectInput), CheckColumn(crate::direct::DirectInput), @@ -160,7 +159,6 @@ impl Operation { Operation::DecideTabularTask(_) => "decide_tabular_task", Operation::OrderSampledTabularRows(_) => "order_sampled_tabular_rows", Operation::SelectTabularRows(_) => "select_tabular_rows", - Operation::CheckTabularCap(_) => "check_tabular_cap", Operation::CheckDirectShape(_) => "check_direct_shape", Operation::CheckDirectCell(_) => "check_direct_cell", Operation::CheckColumn(_) => "check_column", @@ -332,7 +330,6 @@ impl Operation { | Operation::DecideTabularTask(v) | Operation::OrderSampledTabularRows(v) | Operation::SelectTabularRows(v) - | Operation::CheckTabularCap(v) | Operation::CheckDirectShape(v) | Operation::CheckDirectCell(v) | Operation::FeatureImportance(v) @@ -1322,19 +1319,6 @@ pub struct Materialize { /// `too_many_parts`. #[serde(default, skip_serializing_if = "Option::is_none")] pub most_parts: Option, - /// The most entities one `materialize` gathers, across the calls it is - /// asked in, which the host lets its caller set (`main`'s - /// `MOST_ENTITIES`); 100,000 where it names none (design choice 1, - /// multi-call `predict`, read which asks a request of more - /// rows than one call scores in several calls; until then the core - /// refuses more than one call's rows, a lower bound than either). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub most_entities: Option, - /// The most model calls one request is asked in (`main`'s - /// `MOST_CALLS`); 100 where the host names none (read as - /// `most_entities`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub most_calls: Option, /// The sampler state the materializer's last run left, which the host /// hands back as it took it; none for a materializer's /// first run. @@ -1944,10 +1928,6 @@ pub struct Attachments { pub select_tabular_rows_input: crate::tabular::SelectTabularRows, #[schemars(rename = "DirectValue.select_tabular_rows")] pub select_tabular_rows_output: crate::tabular::SelectedTabularRows, - #[schemars(rename = "CheckTabularCap.input")] - pub check_tabular_cap_input: crate::tabular::CheckTabularCap, - #[schemars(rename = "DirectValue.check_tabular_cap")] - pub check_tabular_cap_output: crate::tabular::CheckedTabularCap, #[schemars(rename = "ObserveTabularMembership.input")] pub observe_tabular_membership_input: crate::tabular::ObserveTabularMembership, #[schemars(rename = "DirectValue.observe_tabular_membership")] diff --git a/rust/crates/sdm-contract/src/tabular.rs b/rust/crates/sdm-contract/src/tabular.rs index 443c1afad..1babdc910 100644 --- a/rust/crates/sdm-contract/src/tabular.rs +++ b/rust/crates/sdm-contract/src/tabular.rs @@ -472,19 +472,6 @@ pub enum TabularOrderError { TemporalYearOutOfRange, } -/// Reached native comparison against the model cap, before frame length reads. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct CheckTabularCap { - pub over_limit: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct CheckedTabularCap { - pub accepted: bool, -} - /// One ordinary selection composition. Only proven-passive native producers /// may supply the complete population before earlier semantic guards finish. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 7970f1dc6..ea72f0c42 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -201,7 +201,6 @@ advice! { EmptyRequest = "empty_request", []; GiveAHop = "give_a_hop", []; KeepOneLink = "keep_one_link", ["source_table", "target_table", "others"]; - MaterializeCall = "materialize_call", []; MissingTime = "missing_time", []; NameCandidates = "name_candidates", ["column"]; NamePrimaryKey = "name_primary_key", ["table"]; @@ -214,7 +213,6 @@ advice! { ResultFrame = "result_frame", []; SetPrimaryKey = "set_primary_key", ["target_table", "target_column"]; SkipKeyCheck = "skip_key_check", []; - SmallerCalls = "smaller_calls", []; TrustExpressions = "trust_expressions", []; } @@ -340,9 +338,6 @@ pub fn template(sentence: Sentence) -> Template { S::DirectTargetScored => said( r#"predict carries the target column {target!r}. Those are the rows whose answer is being asked for, so the column has to be absent; drop it with predict.drop(columns=[{target!r}])"#, ), - S::DirectContextLimit => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with materialize(), which keeps the context under the cap and says how"#, - ), S::TabularMissingRowIndex => said( r#"kumo-tabular returned a response that does not match the contract: PredictionItem response is missing required field 'row_index'"#, ), @@ -744,9 +739,6 @@ pub fn template(sentence: Sentence) -> Template { S::BigIntegerPastType => said( r#"column {column!r} holds a value that {admitted_type}, the type it was described as, cannot hold"#, ), - S::LagPastPayload => said( - r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes the endpoint takes; ask for fewer steps"#, - ), S::DateLiteralPacked => said( r#"{query!r} is not a query a relational model can answer: {literal} is not a valid instance of time"#, ), @@ -882,9 +874,6 @@ pub fn template(sentence: Sentence) -> Template { S::ModelLacksOutputs => said( r#"{model} does not produce {unsupported_outputs!r}; it produces {supported_outputs!r}"#, ), - S::ContextPastModel => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with {advice:materialize_call()}, which keeps the context under the cap and says how"#, - ), S::NoContextRows => said( r#"no context rows are left: every row that could be context is also being scored, and a row carrying the answer being asked for is not sent as context. Score fewer rows, so that the rest are there to learn from"#, ), @@ -936,15 +925,6 @@ pub fn template(sentence: Sentence) -> Template { S::OptionsUnreadStatistic => said( r#"{model} does not read inference option(s) {unsupported!r}; drop them from inference_config to send the request without them. The keys this client forwards are {forwarded!r}, and the endpoint decides for itself whether it serves the value given for one. A regression's statistic is chosen with {advice:choose_statistic()} instead: 'median', the default, or 'mean'"#, ), - S::PayloadSmallerCalls => said( - r#"Context size exceeds the 30MB limit. This call is {bytes:,} bytes, and the endpoint takes at most {limit:,}. What comes down is one of three. For an ordinary question it is the call: {advice:smaller_calls()} below the {rows_per_call:,} this one holds asks for the same entities in more and smaller calls. Where the context is itself over the limit, no call is small enough, and it is num_context_samples, num_neighbors or the columns the graph carries. For a ranking or a forecast, which is one call per candidate or step and is never batched, it is the entities."#, - ), - S::PayloadNoSmallerCall => said( - r#"Context size exceeds the 30MB limit. This call is {bytes:,} bytes, and the endpoint takes at most {limit:,}. A call already scores one row, so there is no smaller one to ask for: what comes down is num_context_samples, num_neighbors or the columns the graph carries."#, - ), - S::CallPastEndpoint => said( - r#"the request is {bytes:,} bytes and the endpoint takes at most {limit:,} bytes; reduce the number of tables in the graph, their columns, the neighbourhood sampled or the context gathered (num_context_samples)"#, - ), S::OutputsNone => said(r#"outputs must name at least one field"#), S::NotFinite => said( r#"encountered a non-finite value ({value}); NIM requests must contain finite numbers or nulls"#, @@ -973,9 +953,6 @@ pub fn template(sentence: Sentence) -> Template { S::NumContextSamplesBelowOne => said( r#"num_context_samples must be at least 1; a model predicts from labelled examples, and {given!S} leaves it none"#, ), - S::NumContextSamplesPastMost => said( - r#"num_context_samples is {given!I:,}, and the endpoint accepts at most {limit:,} context rows"#, - ), S::NumHopsNotWhole => said(r#"num_hops must be a whole number, got {given!N} {given!G}"#), S::NumHopsNone => said(r#"num_hops must be a whole number, got {advice:nothing()}"#), S::RandomSeedNotWhole => said(r#"random_seed must be a whole number"#), @@ -1030,17 +1007,11 @@ pub fn template(sentence: Sentence) -> Template { r#"prediction_statistic applies only to a regression, and this question is a {task}; ask it without prediction_statistic"#, ), S::NoHopNumHops => said( - r#"num_hops is {hops}, which asks for no hop at all, and the endpoint reads between 1 and {limit}. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, + r#"num_hops is {hops}, which asks for no hop at all, and a request samples at least 1. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, ), S::NoHopNumNeighbors => said( - r#"num_neighbors is empty, which asks for no hop at all, and the endpoint reads between 1 and {limit}. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, + r#"num_neighbors is empty, which asks for no hop at all, and a request samples at least 1. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, ), - S::HopsPastEndpoint => { - said(r#"this request samples {hops} hops and the endpoint reads at most {limit}"#) - } - S::HopsPastEndpointUncounted => { - said(r#"this request samples more hops than the endpoint reads, at most {limit}"#) - } S::InferenceRunModeInvalid => { said("inference run_mode is text of at most 1024 characters, or null") } @@ -1147,7 +1118,6 @@ pub fn template(sentence: Sentence) -> Template { S::TargetNotAFeature => { said(r#"the column the query predicts is not one of the features of the entity table"#) } - S::ContextPastRows => said(r#"the context holds more rows than a request takes"#), S::HeldBySeveralRowsOne => said( r#"Cannot score {keys[0]!r}: more than one row of table '{table}' holds this value of '{key}', so which row is meant would be a guess. The source does not enforce the key; keep one row per value, or key the table on columns that identify a row."#, ), @@ -1167,9 +1137,6 @@ pub fn template(sentence: Sentence) -> Template { said("Found no valid rows in table '{table}'; supply valid non-missing {requirements}.") } S::NotInTable => said(r#""The primary keys {keys!r} do not exist in the '{table}' table""#), - S::SubgraphPastTables => said( - r#"the request's subgraph carries {tables:,} tables, and the endpoint reads at most {limit:,}"#, - ), S::Unresolved => { said(r#"{query!r} is not a query a relational model can answer: {reasons!R}"#) } @@ -1216,15 +1183,6 @@ pub fn template(sentence: Sentence) -> Template { S::ForecastTopK => said( r#"top_k is how many candidates a ranking keeps, and a forecast ranks nothing; it gives back one value per step"#, ), - S::MostEntities => said( - r#"{entities:,} entities is past the {limit:,} one {advice:materialize_call()} will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time"#, - ), - S::MostEntitiesParts => said( - r#"{entities:,} entities gathered {gathered_times:,} times over, {total:,} in all, is past the {limit:,} one {advice:materialize_call()} will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time"#, - ), - S::MostCalls => said( - r#"{entities:,} entities are gathered as {calls:,} calls of {rows_per_call:,}, and every one is built, with its own context, before any is sent. That is past the {limit:,} calls this will hold at once; ask about fewer entities at a time"#, - ), S::NotYetMaterialize => said( r#"materializing this request needs the {rule!r} rule, which is not one this client's core supports yet"#, ), @@ -1555,6 +1513,12 @@ pub fn template(sentence: Sentence) -> Template { S::ServingValueNotJson => { said(r#"this request carries a value that cannot be sent as JSON: {reason}"#) } + S::CallTooLargeAfterHalving => said( + r#"the endpoint refused a call of {rows:,} rows to score, {bytes:,} bytes, with status {status}, after it was halved {halvings} times; ask in calls of fewer rows (rows_per_call), or send fewer context rows or columns"#, + ), + S::ContextTooLargeForEndpoint => said( + r#"the context is too large for this endpoint: the call that sends it, {bytes:,} bytes, was refused with status {status}; send fewer context rows or columns"#, + ), S::ServingPastLimit => said( r#"this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller num_context_samples, or fewer entities to score"#, ), diff --git a/rust/crates/sdm-contract/tests/corpus/mod.rs b/rust/crates/sdm-contract/tests/corpus/mod.rs index 79e7f9793..ee94a7a76 100644 --- a/rust/crates/sdm-contract/tests/corpus/mod.rs +++ b/rust/crates/sdm-contract/tests/corpus/mod.rs @@ -151,8 +151,6 @@ pub fn messages() -> Vec { values: None, options: vec!["num_neighbors".into()], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: Some(SamplerState { version: 1, diff --git a/rust/crates/sdm-contract/tests/schema_lock.rs b/rust/crates/sdm-contract/tests/schema_lock.rs index bbe065843..4daaa96af 100644 --- a/rust/crates/sdm-contract/tests/schema_lock.rs +++ b/rust/crates/sdm-contract/tests/schema_lock.rs @@ -552,11 +552,6 @@ const JSON: &[(&str, &str)] = &[ "OperationResult/direct_value.value", "DirectValue.select_tabular_rows", ), - ("Operation/check_tabular_cap.input", "CheckTabularCap.input"), - ( - "OperationResult/direct_value.value", - "DirectValue.check_tabular_cap", - ), ( "Operation/observe_tabular_membership.input", "ObserveTabularMembership.input", diff --git a/rust/crates/sdm-contract/tests/wording.rs b/rust/crates/sdm-contract/tests/wording.rs index 0baf8070f..bc0113cec 100644 --- a/rust/crates/sdm-contract/tests/wording.rs +++ b/rust/crates/sdm-contract/tests/wording.rs @@ -377,7 +377,6 @@ fn the_sentences_the_hosts_choose_between_today_are_the_core_s() { Sentence::TimeColumnsRefusedEmpty, ), (Sentence::KeyNotNames, Sentence::KeyNotNamesGiven), - (Sentence::MostEntities, Sentence::MostEntitiesParts), (Sentence::BatchedPartForecast, Sentence::BatchedPartRanking), ( Sentence::CandidatesUnscoredOne, diff --git a/rust/crates/sdm-core/src/batch_session.rs b/rust/crates/sdm-core/src/batch_session.rs index 84d81106a..254699362 100644 --- a/rust/crates/sdm-core/src/batch_session.rs +++ b/rust/crates/sdm-core/src/batch_session.rs @@ -90,6 +90,11 @@ impl Session { Ok(true) } + /// The session the batches are scored against, once it is open. + pub(crate) fn current(&self) -> Option<&str> { + self.current.as_deref() + } + /// None requests ordinary stateless fallback, never another retry layer. pub(crate) async fn score( &mut self, diff --git a/rust/crates/sdm-core/src/direct.rs b/rust/crates/sdm-core/src/direct.rs index e7ab051b0..ee2584f16 100644 --- a/rust/crates/sdm-core/src/direct.rs +++ b/rust/crates/sdm-core/src/direct.rs @@ -844,8 +844,7 @@ pub(crate) async fn predict( })); } let answer = - crate::predict::call_answer(&asker, payload.to_bytes(), request.model_endpoint.as_ref()) - .await?; + crate::predict::direct_answer(&asker, &payload, request.model_endpoint.as_ref()).await?; let mut payload = payload; if let Some(analysis) = analysis { payload.set("client_column_analysis", Value::Text(analysis)); diff --git a/rust/crates/sdm-core/src/dispatch.rs b/rust/crates/sdm-core/src/dispatch.rs index 04aeffe69..dbce2b3b4 100644 --- a/rust/crates/sdm-core/src/dispatch.rs +++ b/rust/crates/sdm-core/src/dispatch.rs @@ -324,13 +324,6 @@ pub fn start( crate::direct::value(&asker, &crate::tabular_selection::select(input)?) }) } - Operation::CheckTabularCap(request) => { - let input: sdm_contract::tabular::CheckTabularCap = - take_json(&mut attachments, request.input)?; - Run::new(move |asker| async move { - crate::direct::value(&asker, &sdm_request::tabular::check_materialized_cap(input)) - }) - } Operation::CheckDirectShape(request) => { let input: sdm_contract::direct::DirectShape = take_json(&mut attachments, request.input)?; @@ -786,8 +779,6 @@ pub(crate) fn materialize( values, options: request.options, most_parts: request.most_parts, - most_entities: request.most_entities, - most_calls: request.most_calls, facts, zones: Arc::clone(&zones), #[cfg(feature = "test-ops")] diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 7650900c7..9120d747a 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -63,12 +63,10 @@ pub(crate) async fn call( endpoint: Endpoint, body: Value, ) -> Result { - match asker - .call_model( - endpoint, - "application/json".into(), - predict::call_bytes(&body)?, - ) + // A create call carries only the context, which cannot be split. + let create = matches!(endpoint, Endpoint::CreateSession); + let result = match asker + .call_model(endpoint, "application/json".into(), body.to_bytes()) .await { Reply::Model(ModelResult::Response { status, body, .. }) @@ -88,6 +86,10 @@ pub(crate) async fn call( .with_field("status", Detail::Scalar(Scalar::UInt16(status)))) } _ => Err(internal("session transport returned another reply kind")), + }; + match create { + true => result.map_err(|error| crate::predict::created(error, &body)), + false => result, } } pub(crate) async fn run( @@ -295,22 +297,53 @@ pub(crate) async fn run( crate::columns::expose(&asker, state.column_analysis.as_deref(), &state.outputs)?; let mut parts = Vec::new(); let mut start = 0; + // A scoring call carries no context: its rows are asked in halves + // where the endpoint refuses it as too large or too slow, the + // size kept for the calls after it. + let mut most = None; + let write = |call: &Value| { + Ok(subset( + call, + &["predict", "output", "inference", "metadata"], + )) + }; + let session_id = state.session_id.clone(); + let send = |call: Value, + bytes: Vec| + -> crate::predict::Sent<'_, sdm_request::predict::Part> { + let session_id = session_id.clone(); + let asker = &asker; + Box::pin(async move { + let answer = self::call( + asker, + Endpoint::SessionPredict { session_id }, + crate::predict::payload(&bytes)?, + ) + .await?; + let response = sdm_wire::response::PredictionResponse::read(&answer) + .map_err(predict::refused)?; + predict::layout(&call, &response, crate::pql::float_text) + }) + }; + let how = crate::predict::Asking { + write: &write, + send: &send, + merge: crate::predict::numbered_parts, + }; for (body, layout) in plans { let answered: Result<_, CoreError> = async { let part = if predict::nothing_to_score(&layout) { predict::empty(&state.model, &layout) } else { - let answer = call( - &asker, - Endpoint::SessionPredict { - session_id: state.session_id.clone(), - }, - body, + crate::predict::answered( + &how, + layout.clone(), + body.to_bytes(), + &mut most, + 0, ) - .await?; - let response = sdm_wire::response::PredictionResponse::read(&answer) - .map_err(predict::refused)?; - predict::layout(&layout, &response, crate::pql::float_text)? + .await + .map_err(|unanswered| unanswered.error)? }; predict::numbered_from(part, start) } diff --git a/rust/crates/sdm-core/src/lag.rs b/rust/crates/sdm-core/src/lag.rs index 55812a9e5..715e7534c 100644 --- a/rust/crates/sdm-core/src/lag.rs +++ b/rust/crates/sdm-core/src/lag.rs @@ -34,23 +34,15 @@ //! than their own; the core puts each on its own row ( fix 1, a //! registered difference decided under ). -use sdm_contract::{CoreError, Detail, Scalar, Sentence}; +use sdm_contract::{CoreError, Sentence}; use sdm_engine::labels::{self, Labels, Target}; use sdm_engine::sample::out_of_range; use sdm_engine::time::offset::Offset; -use sdm_request::payload::{LagColumn, LagValues, PAYLOAD_BYTES}; +use sdm_request::payload::{LagColumn, LagValues}; /// The field a lag's refusal names. const FIELD: &str = "lag_timesteps"; -/// The fewest bytes a lagged column adds to a payload: its declaration in -/// the instance table's schema (`"__kumo_arl0__": {"dtype": "bool"}`). -const COLUMN_BYTES: usize = 30; - -/// The fewest bytes one lagged cell adds: the shortest value (`0.0`, `true` -/// or `null`) and its separator. -const CELL_BYTES: usize = 4; - /// `offset` taken `times` times, as pandas multiplies a `DateOffset`: a /// count of its unit `times` as large; past what the count holds, refused /// as a time out of range naming `lag_timesteps`, as pandas overflows. @@ -143,33 +135,6 @@ pub(crate) struct Lag { } impl Lag { - /// Refuses lags that would make a payload of `rows` instance rows - /// larger than the endpoint takes, before any column is computed, as - /// the payload's limit refuses it. - pub fn fits(&self, rows: usize) -> Result<(), CoreError> { - let least = usize::try_from(self.steps) - .ok() - .and_then(|steps| { - rows.checked_mul(CELL_BYTES)? - .checked_add(COLUMN_BYTES)? - .checked_mul(steps) - }) - .unwrap_or(usize::MAX); - if least > PAYLOAD_BYTES { - return Err(CoreError::new( - Sentence::LagPastPayload, - "the request is larger than the endpoint takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(PAYLOAD_BYTES as u64)), - )); - } - Ok(()) - } - /// The lagged columns of the examples at `kept` (positions among the /// examples the lags were read for: the whole request's, the gathered /// context before narrowing and every entity predicted for), @@ -239,6 +204,7 @@ impl Lag { #[cfg(test)] mod tests { use super::*; + use sdm_contract::Detail; use sdm_contract::ErrorCode; use sdm_contract::pql::TimeUnit; use sdm_engine::labels::{Aggregation, Constant, RelOp}; @@ -520,8 +486,7 @@ mod tests { } /// Steps that reach outside what nanoseconds hold, or past what a count - /// holds, are refused naming `lag_timesteps`; a payload they would make - /// larger than the endpoint takes, before anything is computed. + /// holds, are refused naming `lag_timesteps`. #[test] fn too_many_steps_are_refused_by_name() { for steps in [100_000, u64::MAX] { @@ -529,21 +494,5 @@ mod tests { assert_eq!(e.code, ErrorCode::ValueOutOfRange); assert_eq!(e.fields["field"], Detail::text(FIELD)); } - let lag = single( - 10_000_000, - Aggregation::Count, - Offset::of(0, TimeUnit::Seconds), - Offset::of(1, TimeUnit::Seconds), - None, - vec![0, 0], - Rows { - batches: vec![], - values: labels::Values::Ones(0), - times: vec![], - }, - ); - let e = lag.fits(2).err().unwrap(); - assert_eq!(e.code, ErrorCode::InvalidRequest); - assert_eq!(e.fields["field"], Detail::text("payload")); } } diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index d171e3ebd..89624220b 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -49,8 +49,7 @@ //! narrowed to the size asked, the hops the options ask bounded, the //! entity count, the reserved names, the statistic, the inference //! options, a static target's columns kept out of what is read and sent, -//! the subgraph's 15 tables, the payload's layout and byte limit -//! ([`sdm_request::payload`]), and the notices of what the request leaves +//! the payload's layout ([`sdm_request::payload`]), and the notices of what the request leaves //! out, each a code and fields the host words; and it refuses with //! `UNSUPPORTED_REQUEST` naming the rule ([`sdm_engine::not_yet`]) every //! input whose request takes a rule it does not port yet (register entry @@ -139,18 +138,6 @@ const DEFAULT_SEED: u128 = 42; /// How many times as many candidates as examples a run tries /// (`max_pq_iterations`' default). const ITERATIONS: usize = 10; -/// The most tables a request's subgraph carries (`_MAX_SUBGRAPH_TABLES`). -const SUBGRAPH_TABLES: usize = 15; -/// The most entities one `materialize` gathers where the host names no -/// limit (`main`'s `MOST_ENTITIES`): every call's payload is built, with -/// its own context, before any is sent, so the whole population is held at -/// once. -pub const MOST_ENTITIES: u64 = 100_000; -/// The most model calls one request is gathered as where the host names no -/// limit (`main`'s `MOST_CALLS`): a context is gathered for every call. -pub const MOST_CALLS: u32 = 100; -/// The most rows an instance table may hold (`MAX_TABLE_ROWS`). -const TABLE_ROWS: usize = 10_000; /// How many examples the reference labels at once, stopping once it has /// the run mode's (`_sample_target_set`'s `batch_size`): the first batch /// holds every example a context takes whenever all of it is labelled. @@ -183,12 +170,6 @@ pub(crate) struct Inputs { pub options: Vec, /// The most steps a forecast is asked as (`Materialize.most_parts`). pub most_parts: Option, - /// The most entities one `materialize` gathers - /// (`Materialize.most_entities`). - pub most_entities: Option, - /// The most calls one request is gathered as - /// (`Materialize.most_calls`). - pub most_calls: Option, pub facts: HostFacts, /// The zones the run resolves names with: the host's local zone, which /// places a date literal's two-digit year (`pql::local_year`). @@ -2899,7 +2880,6 @@ pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result, -) -> Result<(), CoreError> { - let most = most.unwrap_or(MOST_ENTITIES); - let total = entities.saturating_mul(times); - if total <= most { - return Ok(()); - } - Err(CoreError::new( - if times == 1 { - Sentence::MostEntities - } else { - Sentence::MostEntitiesParts - }, - "more entities were asked for than one materialize holds at once", - ) - .with_field("field", Detail::text("indices")) - .with_field("entities", Detail::Scalar(Scalar::UInt64(entities))) - .with_field("gathered_times", Detail::Scalar(Scalar::UInt64(times))) - .with_field("total", Detail::Scalar(Scalar::UInt64(total))) - .with_field("limit", Detail::Scalar(Scalar::UInt64(most)))) -} - /// A request's payload and query, attached. pub(crate) fn attach_request( asker: &Asker, @@ -3105,8 +3055,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result mode.context_examples(), + options::RunMode::Best => mode.context_examples().max(asked_examples.unwrap_or(0)), + }; // `max_pq_iterations` times as many candidates as examples; a // float's count is no slice bound for a temporal query's walk, the // engine's `TypeError`, the caller's error. @@ -3837,19 +3784,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result most as usize { - return Err(CoreError::new( - Sentence::MostCalls, - "the entities asked for are gathered as more calls than one materialize holds at once", - ) - .with_field("field", Detail::text("indices")) - .with_field("entities", count(num_test)) - .with_field("calls", count(calls)) - .with_field("rows_per_call", count(per_call)) - .with_field("limit", Detail::Scalar(Scalar::UInt32(most)))); - } // How many candidates the call asked be kept (`_candidates_to_rank`, // ), which the engine writes into the task; then the inference // options, read for the task's config class, as the engine reads them @@ -3912,16 +3846,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result TABLE_ROWS { - return Err(invalid( - Sentence::ContextPastRows, - "the context holds more rows than a request takes", - "query", - )); - } - // The rows the largest call scores (`_the_largest_batch`), which a - // payload past the endpoint's body names as the size to come down from. - let largest = per_call.min(num_test); let train_anchors = anchors.len(); let entity_names = crate::subgraph::sample_columns(&graph, entity, &excluded); // The labelled context's examples before narrowing, then the entities @@ -3941,7 +3865,7 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result SUBGRAPH_TABLES { - return Err(invalid( - Sentence::SubgraphPastTables, - "the request's subgraph carries more tables than the endpoint reads", - "tables", - ) - .with_field("tables", count(walked.tables.len())) - .with_field("limit", count(SUBGRAPH_TABLES))); - } sent.extend(walked.tables.iter().map(|t| t.name.clone())); // The lagged columns of this call's examples: the narrowed // context's, then this call's own entities, each its own row's, // sliced from the whole request's columns, so each column's type // is the request's whatever this call's rows hold, typed // over the gathered context before narrowing; the - // columns are computed at the first call's payload, where the - // reference's size refusal falls, held to the largest call's rows. + // columns are computed at the first call's payload. if lag_columns.is_none() && let Some(lag) = &lag { - lag.fits(num_train + largest)?; lag_columns = Some( lag.columns( &(0..num_train) @@ -4178,7 +4088,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result TABLE_ROWS { - return Err(refuse("context exceeds the model row limit")); - } - refuse_entities(predict.num_rows() as u64, 1, None)?; let seed = call .random_seed .to_u128() @@ -420,9 +416,6 @@ async fn run( Some(seed), ) .await?; - if walked.tables.len() > SUBGRAPH_TABLES { - return Err(refuse("sampled graph exceeds the model table limit")); - } if let Some(destination) = destination && !walked.tables.iter().any(|t| t.name == destination.name) { @@ -447,7 +440,6 @@ async fn run( options: options.clone(), statistic: None, top_k: None, - rows_per_call: per_call, lags: vec![], float_text: crate::pql::float_text, }; diff --git a/rust/crates/sdm-core/src/materialize/frames.rs b/rust/crates/sdm-core/src/materialize/frames.rs index cf826199c..46da643d4 100644 --- a/rust/crates/sdm-core/src/materialize/frames.rs +++ b/rust/crates/sdm-core/src/materialize/frames.rs @@ -133,7 +133,7 @@ impl FrameGraph { &self, graph: &Graph, seeds: SeedRows<'_>, - hops: &[options::Hop], + hops: &options::Hops, excluded: &[(String, String)], seed: Option, ) -> Result { @@ -144,7 +144,10 @@ impl FrameGraph { } = seeds; let rows = self.0.snapshot.entity_rows(&entity.name, keys)?; let mut counts = Vec::new(); - for hop in hops { + // Hops past as many as the snapshot holds rows reach nothing more, + // so a count of hops past it is walked as that many. + let most = self.0.snapshot.rows().saturating_add(1); + for hop in (0..most).map_while(|at| hops.get(at)) { match hop { options::Hop::Keep(n) => { counts.push(i64::try_from(*n).map_err(|_| not_yet("frame_neighbor_count"))?) @@ -365,8 +368,6 @@ pub(crate) async fn materialize( values, options: call.options, most_parts: call.most_parts, - most_entities: call.most_entities, - most_calls: call.most_calls, facts, zones, memo: None, diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 9b4d03f2e..a2df3b6e7 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -36,9 +36,9 @@ use sdm_contract::ops::{ModelEndpoint, OperationResult, Predicted}; use sdm_contract::run::Endpoint; use sdm_contract::{CoreError, CoreWarning, Detail, Scalar, Sentence}; use sdm_request::predict::{ - Notice, Part, absent, absent_notice, bad_response, body, by_step, call_bytes, call_starts, - empty, layout, left_out, merge, nothing_to_score, numbered_from, predict_rows, preflight, - prepare, refused, steps_unexplained, table, + Notice, Part, absent, absent_notice, bad_response, body, by_step, call_starts, empty, layout, + left_out, merge, nothing_to_score, numbered_from, predict_rows, preflight, prepare, refused, + steps_unexplained, table, }; use sdm_request::serving::{self, Failure}; use sdm_wire::json::{self, Value}; @@ -430,27 +430,36 @@ pub(crate) async fn one( } // Every call written before the first is sent, as the client's // plan is listed whole (`_numbered_across_calls`), then each call's - // rows counted. + // rows counted. To a serving endpoint, whose refusal of a call too + // large is a dropped connection, a call past its limit is split up + // front into calls that fit. + let mut calls = Vec::with_capacity(payloads.len()); let mut bodies = Vec::with_capacity(payloads.len()); for (payload, request_id) in payloads.iter().zip(&ids) { - bodies.push(body( - payload, - &inputs.model, - request_id, - &outputs, - &inputs.inference_asked, - )?); + let write = |call: &Value| { + body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + ) + }; + for (call, body) in fitted(inputs.endpoint.as_ref(), payload, &write)? { + calls.push((call, request_id.as_str())); + bodies.push(body); + } } + let payloads: Vec = calls.iter().map(|(call, _)| call.clone()).collect(); let mut session = crate::batch_session::Session::eligible( &bodies, inputs.supports_sessions && inputs.endpoint.is_none(), ); - // Validate all full calls before any network effect, including their size. // Retain only the small scoring half alongside encoded fallback bytes. let plans = bodies .into_iter() .map(|body| { - let bytes = call_bytes(&body)?; + let bytes = body.to_bytes(); let scoring = if session.is_some() { sdm_request::frame_session::subset( &body, @@ -459,36 +468,112 @@ pub(crate) async fn one( } else { Value::Null }; - Ok((scoring, bytes)) + (scoring, bytes) }) - .collect::, CoreError>>()?; + .collect::>(); let starts = call_starts(&payloads)?; let mut parts = Vec::with_capacity(payloads.len()); - for ((payload, (body, bytes)), start) in payloads.iter().zip(plans).zip(starts) { + // The rows a call the endpoint took after a smaller one was asked, + // kept for the calls after it. + let mut most = None; + for (((payload, request_id), (body, bytes)), start) in calls.iter().zip(plans).zip(starts) { let response = async { - if let Some(session) = session.as_mut() - && let Some(answer) = session.score(asker, &body).await? - { - let response = PredictionResponse::read(&answer).map_err(refused)?; - return layout(payload, &response, crate::pql::float_text); + if let Some(session) = session.as_mut() { + match session.score(asker, &body).await { + Ok(Some(answer)) => { + let response = PredictionResponse::read(&answer).map_err(refused)?; + return layout(payload, &response, crate::pql::float_text) + .map_err(Unanswered::from); + } + Ok(None) => {} + // The session's scoring call carries no context: its + // rows are asked in halves as a call's are. + Err(error) if too_large(&error) && session.current().is_some() => { + let session_id = session.current().unwrap_or_default().to_owned(); + let write = |call: &Value| { + let written = sdm_request::predict::body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + )?; + Ok(sdm_request::frame_session::subset( + &written, + &["predict", "output", "inference", "metadata"], + )) + }; + let send = |call: Value, bytes: Vec| -> Sent<'_, Part> { + let session_id = session_id.clone(); + Box::pin(async move { + let answer = crate::frame_session::call( + asker, + Endpoint::SessionPredict { session_id }, + self::payload(&bytes)?, + ) + .await?; + let response = + PredictionResponse::read(&answer).map_err(refused)?; + layout(&call, &response, crate::pql::float_text) + }) + }; + let how = Asking { + write: &write, + send: &send, + merge: numbered_parts, + }; + let sent = body.to_bytes().len(); + let (size, halvings) = split(payload, &error, sent, &mut most, 0)?; + return in_runs(&how, payload.clone(), size, &mut most, halvings).await; + } + Err(error) => return Err(Unanswered::from(error)), + } } - call(asker, payload, bytes, inputs.endpoint.as_ref()).await + let write = |call: &Value| { + sdm_request::predict::body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + ) + }; + let endpoint = inputs.endpoint.as_ref(); + let send = |call: Value, bytes: Vec| -> Sent<'_, Part> { + Box::pin(async move { self::call(asker, &call, bytes, endpoint).await }) + }; + let how = Asking { + write: &write, + send: &send, + merge: numbered_parts, + }; + answered(&how, payload.clone(), bytes, &mut most, 0).await } .await; let answered = match response { - Ok(part) => numbered_from(part, start), - Err(error) => Err(error), + Ok(part) => numbered_from(part, start).map_err(Unanswered::from), + Err(unanswered) => Err(unanswered), }; match answered { Ok(mut part) => { part.columns = analysis.clone(); parts.push(part) } - Err(error) => { + Err(Unanswered { error, answered }) => { if let Some(session) = session.take() { session.close(asker).await; } - return Err(carry_calls(asker, error, parts, &outputs, start)); + // The runs of a split call answered before one failed + // are carried as the calls before it are. + let mut rows = start; + if let Some((partial, more)) = answered + && let Ok(mut partial) = numbered_from(partial, start) + { + partial.columns = analysis.clone(); + parts.push(partial); + rows += more; + } + return Err(carry_calls(asker, error, parts, &outputs, rows)); } } } @@ -507,6 +592,349 @@ pub(crate) async fn one( Ok((merged, outputs, scored)) } +/// `payload` as the calls it is sent in, each with its body (`write`): one +/// call, or, to a serving endpoint, whose refusal of a call too large is a +/// dropped connection, as many as it is split into up front so that each +/// one's envelope fits the endpoint's `max_request_bytes` +/// ([`sdm_request::batching::fitted`]), each scoring a run of its rows with +/// the whole context. A call of one row past it is refused. +fn fitted( + endpoint: Option<&ModelEndpoint>, + payload: &Value, + write: &Write<'_>, +) -> Result, CoreError> { + let Some(ModelEndpoint::ServingEndpoint { + name, + max_request_bytes, + }) = endpoint + else { + return Ok(vec![(payload.clone(), write(payload)?)]); + }; + let mut fit = |call: &Value| -> sdm_request::batching::Fit { + let body = write(call)?; + let size = serving::wrapped(&body)?.len() as u64; + Ok(if size > *max_request_bytes { + Err(serving::past_limit(name, *max_request_bytes, size)) + } else { + Ok(body) + }) + }; + let calls = sdm_request::batching::fitted(payload, &mut fit)?; + if calls.len() > 1 { + tracing::info!( + target: "sdm_core::predict", + rows = predict_rows(payload).unwrap_or(0), + calls = calls.len(), + limit = *max_request_bytes, + "a call past the serving endpoint's limit is sent as smaller calls" + ); + } + Ok(calls) +} + +/// The statuses that say a call was too large or too slow to answer: the +/// host's timeout (told as 408) or an answer too large to read (told as +/// 413), the endpoint's 413, and a gateway's 504, which a call it gave up +/// on comes back as. A 502, a 503 or a 500 is temporary: the host tries it +/// again at its size, and it is never split. +const TOO_LARGE: [u16; 3] = [408, 413, 504]; + +/// The most times one batch is halved (to an eighth of its rows) before the +/// call is refused as too large. +const MOST_HALVINGS: u32 = 3; + +/// A call sent, its answer to come. +pub(crate) type Sent<'a, T> = + std::pin::Pin> + Send + 'a>>; + +/// How a call is asked and, split, put back together: the body a run of its +/// rows is sent as, the sending, and the answers merged, each with the row +/// of the call it starts at. +pub(crate) struct Asking<'a, T> { + pub write: &'a Write<'a>, + pub send: &'a Sender<'a, T>, + pub merge: Merge, +} + +/// The answers to runs of a call's rows, each with the row it starts at, +/// as the one answer to the call. +pub(crate) type Merge = fn(Vec<(usize, T)>) -> Result; + +/// The sending of a run of a call's rows, its body's bytes given. +pub(crate) type Sender<'a, T> = dyn Fn(Value, Vec) -> Sent<'a, T> + Sync + 'a; + +/// The body a run of a call's rows is sent as. +pub(crate) type Write<'a> = dyn Fn(&Value) -> Result + Sync + 'a; + +/// `payload`'s answer, its rows numbered from zero, sent as `bytes`. Where +/// the endpoint refuses the call as too large or too slow ([`TOO_LARGE`]), +/// its rows are asked again in calls of half as many, each with the whole +/// context, and that size is kept in `most` for the calls after it; a call +/// of more rows than `most` is asked in calls of that many. A call of one +/// row is not split: its refusal, the endpoint's own, stands, as does any +/// other. Where a later run of a split call fails, the runs answered before +/// it are kept, with how many rows they answer ([`Unanswered`]). +pub(crate) async fn answered( + how: &Asking<'_, T>, + payload: Value, + bytes: Vec, + most: &mut Option, + halvings: u32, +) -> Result> { + let rows = predict_rows(&payload).unwrap_or(0); + let (size, halvings) = match *most { + Some(most) if rows > most => (most, halvings), + _ => { + let sent = bytes.len(); + match (how.send)(payload.clone(), bytes).await { + Err(error) if rows > 1 && too_large(&error) => { + split(&payload, &error, sent, most, halvings)? + } + answer => return answer.map_err(Unanswered::from), + } + } + }; + in_runs(how, payload, size, most, halvings).await +} + +/// The size a call refused as too large (`error`, `sent` bytes) is asked +/// again in, and the halvings that make: half its rows, at most +/// [`MOST_HALVINGS`] times a batch. +pub(crate) fn split( + payload: &Value, + error: &CoreError, + sent: usize, + most: &mut Option, + halvings: u32, +) -> Result<(usize, u32), Unanswered> { + let rows = predict_rows(payload).unwrap_or(0); + if halvings >= MOST_HALVINGS { + return Err(Unanswered::from(still_too_large( + error, rows, sent, halvings, + ))); + } + let half = rows.div_ceil(2); + tracing::info!( + target: "sdm_core::predict", + rows, + status = ?error.fields.get(sdm_contract::error::REFUSED_STATUS), + size = half, + halving = halvings + 1, + "a call refused as too large or too slow is asked again in smaller calls" + ); + *most = Some(half); + Ok((half, halvings + 1)) +} + +/// `payload`'s answer asked in runs of `size` rows, each as [`answered`] +/// asks it, merged; where a run fails, the runs before it are kept. +pub(crate) async fn in_runs( + how: &Asking<'_, T>, + payload: Value, + size: usize, + most: &mut Option, + halvings: u32, +) -> Result> { + let rows = predict_rows(&payload).unwrap_or(0); + let mut answers = Vec::new(); + for start in (0..rows).step_by(size.max(1)) { + let range = start..(start + size).min(rows); + let written = sdm_request::batching::predict_slice(&payload, range).and_then(|call| { + let bytes = (how.write)(&call)?.to_bytes(); + Ok((call, bytes)) + }); + let outcome = match written { + Ok((call, bytes)) => Box::pin(answered(how, call, bytes, most, halvings)).await, + Err(error) => Err(Unanswered::from(error)), + }; + match outcome { + Ok(answer) => answers.push((start, answer)), + Err(Unanswered { error, answered }) => { + // The runs before this one answered whole, and this one's own + // answered runs, a prefix of its rows. + let mut rows = start; + if let Some((partial, more)) = answered { + answers.push((start, partial)); + rows += more; + } + let answered = match answers.is_empty() { + true => None, + false => (how.merge)(answers).ok().map(|merged| (merged, rows)), + }; + return Err(Unanswered { error, answered }); + } + } + } + (how.merge)(answers).map_err(Unanswered::from) +} + +/// A call that failed, with what of it was answered before it did: the +/// runs of its rows answered, merged, and how many rows they answer, a +/// prefix of the call's. +pub(crate) struct Unanswered { + pub error: CoreError, + pub answered: Option<(T, usize)>, +} + +impl From for Unanswered { + fn from(error: CoreError) -> Self { + Unanswered { + error, + answered: None, + } + } +} + +/// The refusal of a call the endpoint still refused as too large after +/// [`MOST_HALVINGS`] halvings: its rows, bytes and status, and what to +/// change. `after_split` tells a host to raise it in place of its own. +fn still_too_large(error: &CoreError, rows: usize, bytes: usize, halvings: u32) -> CoreError { + CoreError::new( + Sentence::CallTooLargeAfterHalving, + "a call was still too large after it was halved as often as it is", + ) + .with_field("rows", count(rows)) + .with_field("bytes", count(bytes)) + .with_field("halvings", Detail::Scalar(Scalar::UInt32(halvings))) + .with_field("status", refused_status(error)) + .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) +} + +/// The refusal of a session's create call, which carries only the context, +/// refused as too large or too slow: `bytes` cannot be split. +fn context_too_large(error: &CoreError, bytes: usize) -> CoreError { + CoreError::new( + Sentence::ContextTooLargeForEndpoint, + "a call that sends only the context was refused as too large", + ) + .with_field("bytes", count(bytes)) + .with_field("status", refused_status(error)) + .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) +} + +/// The status an endpoint refused a call with. +fn refused_status(error: &CoreError) -> Detail { + error + .fields + .get(sdm_contract::error::REFUSED_STATUS) + .cloned() + .unwrap_or(Detail::Scalar(Scalar::Null)) +} + +/// The answer to a single-table session's scoring call, `payload`, sent by +/// `send`: it carries no context, so its rows are asked in halves where the +/// endpoint refuses it as too large or too slow ([`answered`]), the answers +/// merged as one ([`merged_answers`]). +pub(crate) async fn session_answer<'a>( + payload: &Value, + send: &'a (dyn Fn(Value) -> Sent<'a, Value> + Sync), +) -> Result { + let write = |call: &Value| Ok(call.clone()); + let sender = |call: Value, _: Vec| send(call); + let how = Asking { + write: &write, + send: &sender, + merge: merged_answers, + }; + answered(&how, payload.clone(), payload.to_bytes(), &mut None, 0) + .await + .map_err(|unanswered| unanswered.error) +} + +/// A session's create call's refusal: the call carries only the context, +/// which cannot be split, so a call refused as too large or too slow is +/// refused naming the context's bytes; any other refusal stands. +pub(crate) fn created(error: CoreError, body: &Value) -> CoreError { + if !too_large(&error) { + return error; + } + context_too_large(&error, body.to_bytes().len()) +} + +/// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. +fn too_large(error: &CoreError) -> bool { + matches!( + error.fields.get(sdm_contract::error::REFUSED_STATUS), + Some(Detail::Scalar(Scalar::UInt16(status))) if TOO_LARGE.contains(status) + ) +} + +/// Parts of one call's rows, each numbered from the row it starts at. +pub(crate) fn numbered_parts(parts: Vec<(usize, Part)>) -> Result { + merge( + parts + .into_iter() + .map(|(start, part)| numbered_from(part, start)) + .collect::, _>>()?, + ) +} + +/// The answers to runs of one call's rows as the one answer to it: the +/// first's own members (its `id`, `model` and `metadata`, which the runs +/// share but for their ids), with every answer's predictions in order, each +/// numbered from the row its run starts at (`row_index`). +fn merged_answers(answers: Vec<(usize, Value)>) -> Result { + let mut answers = answers.into_iter(); + let (first, mut whole) = answers + .next() + .ok_or_else(|| internal("a call answered by none"))?; + let mut predictions = Vec::new(); + for (start, answer) in std::iter::once((first, whole.clone())).chain(answers) { + let Some(Value::Array(items)) = answer.get("predictions") else { + // An answer that is not a list of predictions is left to be read + // and refused as it is. + return Ok(answer); + }; + for item in items { + let mut item = item.clone(); + if start > 0 + && let Some(Value::Number(at)) = item.get("row_index") + && let Ok(at) = at.parse::() + { + item.set("row_index", Value::Number((at + start as u64).to_string())); + } + predictions.push(item); + } + } + whole.set("predictions", Value::Array(predictions)); + Ok(whole) +} + +/// The answer to a call sent as it was built, a single-table request's: +/// asked in calls of at most [`sdm_request::options::PREDICT_ROWS`] rows, +/// each with the whole context, split up front to fit a serving endpoint +/// and in halves where the endpoint refuses one as too large or too slow +/// ([`answered`]), the answers merged as one ([`merged_answers`]). +pub(crate) async fn direct_answer( + asker: &Asker, + payload: &Value, + endpoint: Option<&ModelEndpoint>, +) -> Result { + let write = |call: &Value| Ok(call.clone()); + let send = |_: Value, bytes: Vec| -> Sent<'_, Value> { + Box::pin(call_answer(asker, bytes, endpoint)) + }; + let how = Asking { + write: &write, + send: &send, + merge: merged_answers, + }; + let mut most = usize::try_from(sdm_request::options::PREDICT_ROWS).ok(); + let mut answers = Vec::new(); + let mut start = 0; + for (call, body) in fitted(endpoint, payload, &write)? { + let rows = predict_rows(&call).unwrap_or(0); + answers.push(( + start, + answered(&how, call, body.to_bytes(), &mut most, 0) + .await + .map_err(|unanswered| unanswered.error)?, + )); + start += rows; + } + merged_answers(answers) +} + /// One payload's call, its body as written, sent and its answer laid out; /// to a serving endpoint, in its envelope, its answer read out of its row. async fn call( diff --git a/rust/crates/sdm-core/src/ranking.rs b/rust/crates/sdm-core/src/ranking.rs index 97d311259..2919078d4 100644 --- a/rust/crates/sdm-core/src/ranking.rs +++ b/rust/crates/sdm-core/src/ranking.rs @@ -34,7 +34,7 @@ use sdm_wire::json::Value; use crate::materialize::{ Call, Gathered, Inputs, MOST_PARTS, REQUEST_STATE_VERSION, asked_rows_per_call, attach_request, - one, refuse_entities, + one, }; use crate::pql::RankingPlan; use crate::run::Asker; @@ -168,14 +168,11 @@ pub(crate) async fn materialize( .map(|digits| digits.parse::().unwrap_or(u32::MAX)), }; // `_in_parts`, before anything is gathered, as the client refuses them - // (`main`'s 1fc5a686): the batch size asked for; a question over more - // entities than one call scores, since a part is one call and a + // (`main`'s 1fc5a686): the batch size asked for; and a question over + // more entities than one call scores, since a part is one call and a // candidate is reported unscored for the whole ranking, so one cold // batch would drop it from every row - // (`refuse_a_part_that_would_take_several_calls`); and the population - // each question gathers whole, held as many times over as there are - // questions (`refuse_more_entities_than_fit_in_memory`), past the - // host's limit, 100,000 where it names none. + // (`refuse_a_part_that_would_take_several_calls`). let per_call = sdm_request::options::rows_one_call_scores(asked_rows_per_call(inputs.values.as_ref())?); let entities = inputs.indices.as_ref().map_or(0, |data| { @@ -195,7 +192,6 @@ pub(crate) async fn materialize( .with_field("parts", Detail::Scalar(Scalar::UInt64(parts))) .with_field("rows_per_call", Detail::Scalar(Scalar::UInt64(per_call)))); } - refuse_entities(entities, parts, inputs.most_entities)?; // Each question gathered with the call's values but `top_k`, which is // the whole ranking's, each by a materializer of its own, so with no // memo of another's reads; the materializer's own memo is left as it diff --git a/rust/crates/sdm-core/src/session.rs b/rust/crates/sdm-core/src/session.rs index 4da783244..2120da0f5 100644 --- a/rust/crates/sdm-core/src/session.rs +++ b/rust/crates/sdm-core/src/session.rs @@ -281,32 +281,54 @@ pub(crate) async fn send( } } sdm_request::inference::envelope(&mut payload)?; - let reply = asker - .call_model( - request.endpoint, - "application/json".into(), - payload.to_bytes(), - ) - .await; - let (status, body) = match reply { + // A create call carries only the context, which cannot be split; a + // scoring call's rows are asked in halves where the endpoint refuses it + // as too large or too slow. + let (status, body) = match request.endpoint { + Endpoint::CreateSession => { + let created = sent(asker, Endpoint::CreateSession, &payload) + .await + .map_err(|error| crate::predict::created(error, &payload))?; + (200, created) + } + endpoint => { + let send = |call: Value| -> crate::predict::Sent<'_, Value> { + let endpoint = endpoint.clone(); + Box::pin( + async move { crate::predict::payload(&sent(asker, endpoint, &call).await?) }, + ) + }; + let answer = crate::predict::session_answer(&payload, &send).await?; + (200, answer.to_bytes()) + } + }; + let response = asker.attach(Attachment::Bytes(body))?; + crate::direct::value(asker, &SessionTransportReply { status, response }) +} + +/// The answer to `payload` sent to `endpoint`, its bytes; a status that is no +/// success, or a failed transport, refused. +async fn sent(asker: &Asker, endpoint: Endpoint, payload: &Value) -> Result, CoreError> { + match asker + .call_model(endpoint, "application/json".into(), payload.to_bytes()) + .await + { Reply::Model(ModelResult::Response { status, body, .. }) if (200..300).contains(&status) => { - (status, body) + Ok(body) } Reply::Model(ModelResult::Response { status, .. } | ModelResult::Failed { status, .. }) => { - return Err( + Err( sdm_request::predict::bad_response("status", "not_a_success") .with_field("status", Detail::Scalar(Scalar::UInt16(status))), - ); + ) } Reply::Model(ModelResult::TransportError { kind, retryable }) => { - return Err(CoreError::transport(kind, retryable)); + Err(CoreError::transport(kind, retryable)) } - Reply::Fetch(_) => return Err(internal("session call was answered as a fetch")), - }; - let response = asker.attach(Attachment::Bytes(body))?; - crate::direct::value(asker, &SessionTransportReply { status, response }) + Reply::Fetch(_) => Err(internal("session call was answered as a fetch")), + } } /// A deletion has no response document. Only a successful transport outcome diff --git a/rust/crates/sdm-core/src/subgraph.rs b/rust/crates/sdm-core/src/subgraph.rs index 4d9751860..6199311ae 100644 --- a/rust/crates/sdm-core/src/subgraph.rs +++ b/rust/crates/sdm-core/src/subgraph.rs @@ -48,7 +48,7 @@ use sdm_contract::{CoreError, CoreWarning, Detail, Scalar, Sentence}; use sdm_engine::graph::{Graph, Table}; use sdm_engine::keys::{self, Key}; use sdm_engine::sample::{self, SEED_TIME}; -use sdm_request::options::Hop; +use sdm_request::options::{Hop, Hops}; use sdm_request::payload::{SampledColumn, SampledLink, SampledTable}; use crate::describe::{ask, ask_all_vec}; @@ -64,24 +64,38 @@ struct Visit { /// Each row's number, by its key and example (the reference's /// `Mapper`), for a table with a key. nodes: BTreeMap<(Key, usize), usize>, - /// The rows reached at each hop, the entity's first. + /// The rows reached at each hop, the entity's first; a hop past the + /// end reached none. counts: Vec, /// The frames visited so far. offset: usize, } impl Visit { - fn new(name: &str, hops: usize) -> Visit { + fn new(name: &str) -> Visit { Visit { name: name.to_owned(), frames: Vec::new(), batches: Vec::new(), nodes: BTreeMap::new(), - counts: vec![0; hops + 1], + counts: Vec::new(), offset: 0, } } + /// The rows reached at hop `hop`. + fn reached(&self, hop: usize) -> usize { + self.counts.get(hop).copied().unwrap_or(0) + } + + /// Counts `rows` more reached at hop `hop`. + fn add_reached(&mut self, hop: usize, rows: usize) { + if self.counts.len() <= hop { + self.counts.resize(hop + 1, 0); + } + self.counts[hop] += rows; + } + fn rows(&self) -> usize { self.counts.iter().sum() } @@ -119,10 +133,11 @@ pub(crate) struct Walk<'a> { /// The hops walked, each with the neighbors it keeps of each row: the /// run mode's two by default, or as the sampling options ask /// (`num_hops`, `num_neighbors`, [`options::neighbors`]); the walk ends - /// at a hop of size zero. + /// at a hop of size zero, after the last, or at a hop no row is walked + /// from, past which nothing is reached. /// /// [`options::neighbors`]: sdm_request::options::neighbors - pub neighbors: &'a [Hop], + pub neighbors: &'a Hops, /// The features left out, by table: a static target's columns, which /// the reference's sampler neither reads nor sends /// (`get_exclude_cols_dict`, `exclude_cols_dict`). @@ -293,7 +308,7 @@ impl Walk<'_> { /// [`sample_columns`]. pub(crate) async fn run(&self, entity_rows: RecordBatch) -> Result { let examples = entity_rows.num_rows(); - let mut visits = vec![Visit::new(&self.entity.name, self.neighbors.len())]; + let mut visits = vec![Visit::new(&self.entity.name)]; let mut edges: Vec = Vec::new(); { let entity = &mut visits[0]; @@ -311,9 +326,17 @@ impl Walk<'_> { } entity.frames.push(entity_rows); entity.batches.push((0..examples).collect()); - entity.counts[0] = examples; + entity.add_reached(0, examples); } - for (hop, neighbors) in self.neighbors.iter().enumerate() { + for hop in 0.. { + let Some(neighbors) = self.neighbors.get(hop) else { + break; + }; + // A hop no table has rows to walk from reaches none, nor does + // any after it: the walk ends, as it would hop by hop. + if visits.iter().all(|visit| visit.reached(hop) == 0) { + break; + } // A hop of size zero ends the walk (`neighbors == 0`). let neighbors = match neighbors { Hop::Stop => break, @@ -352,7 +375,7 @@ impl Walk<'_> { neighbors: &sdm_contract::data::NeighborLimit, ) -> Result<(), CoreError> { let at = position(visits, name)?; - if visits[at].counts[hop] == 0 { + if visits[at].reached(hop) == 0 { return Ok(()); } let table = table_named(self.graph, name)?; @@ -479,7 +502,7 @@ impl Walk<'_> { return Err(internal("a row looked up was already reached")); } } - dst_visit.counts[hop + 1] += examples.len(); + dst_visit.add_reached(hop + 1, examples.len()); dst_visit.frames.push(added); dst_visit.batches.push(examples); for &i in &missing { @@ -641,7 +664,7 @@ impl Walk<'_> { ), }; let kept_examples: Vec = kept.iter().map(|&i| examples[i]).collect(); - src_visit.counts[hop + 1] += kept.len(); + src_visit.add_reached(hop + 1, kept.len()); src_visit.frames.push(take(&fetched, &kept)?); src_visit.batches.push(kept_examples); edge(edges, src, fkey, name, false).extend(rows, cols); @@ -840,8 +863,7 @@ fn reach(visits: &mut Vec, name: &str) -> usize { match visits.iter().position(|v| v.name == name) { Some(at) => at, None => { - let hops = visits[0].counts.len() - 1; - visits.push(Visit::new(name, hops)); + visits.push(Visit::new(name)); visits.len() - 1 } } diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 4ff4d8346..7b8a62c29 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -396,8 +396,7 @@ async fn predict( part.columns = analysis; return crate::predict::finish(asker, part, prepared.bound.outputs); } - let answer = - crate::predict::call_answer(asker, payload.to_bytes(), input.endpoint.as_ref()).await?; + let answer = crate::predict::direct_answer(asker, &payload, input.endpoint.as_ref()).await?; finish(asker, payload, answer, prepared.bound.outputs, analysis) } fn finish( @@ -549,7 +548,9 @@ async fn fit( tabular::typed_inference(input.prediction.request.inference.clone())?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&payload))?); crate::columns::expose(asker, column_analysis.as_deref(), &prepared.bound.outputs)?; - let answer = session_call(asker, Endpoint::CreateSession, payload).await?; + let answer = session_call(asker, Endpoint::CreateSession, payload.clone()) + .await + .map_err(|error| crate::predict::created(error, &payload))?; let session_id = answer .get("session_id") .or_else(|| answer.get("id")) @@ -720,13 +721,15 @@ async fn score( part.columns = analysis; return crate::predict::finish(asker, part, state.bound.outputs.clone()); } - let answer = session_call( - asker, - sdm_contract::run::Endpoint::SessionPredict { - session_id: state.session_id.clone(), - }, - payload, - ) + let answer = crate::predict::session_answer(&payload, &|body| { + Box::pin(session_call( + asker, + sdm_contract::run::Endpoint::SessionPredict { + session_id: state.session_id.clone(), + }, + body, + )) + }) .await?; finish(asker, layout, answer, state.bound.outputs.clone(), analysis) } @@ -873,7 +876,8 @@ fn materialize( let decision = sdm_engine::tabular::context_decision( candidates.len(), input.num_context_samples, - sdm_request::predict::CONTEXT_ROWS_MAX, + // No most: the context is sampled only to the count asked for. + usize::MAX, ) .map_err(|_| invalid("context sample count is invalid"))?; let context_rows = if decision.reduced { diff --git a/rust/crates/sdm-core/src/tabular_selection.rs b/rust/crates/sdm-core/src/tabular_selection.rs index c11020253..6dae80d39 100644 --- a/rust/crates/sdm-core/src/tabular_selection.rs +++ b/rust/crates/sdm-core/src/tabular_selection.rs @@ -39,7 +39,8 @@ pub(crate) fn decide(input: DecideTabularContext) -> TabularContextDecision { match tabular::context_decision( input.population, input.requested, - sdm_request::predict::CONTEXT_ROWS_MAX, + // No most: the context is sampled only to the count asked for. + usize::MAX, ) { Ok(value) => TabularContextDecision::Selected { wanted: value.wanted, diff --git a/rust/crates/sdm-core/tests/attachments.rs b/rust/crates/sdm-core/tests/attachments.rs index 64d0d3bf0..57dd02b97 100644 --- a/rust/crates/sdm-core/tests/attachments.rs +++ b/rust/crates/sdm-core/tests/attachments.rs @@ -49,7 +49,6 @@ const READ: &[&str] = &[ "DecideTabularTask.input", "OrderSampledTabularRows.input", "SelectTabularRows.input", - "CheckTabularCap.input", "CheckDirectShape.input", "BindDirect.input", "CheckDirectRows.input", diff --git a/rust/crates/sdm-core/tests/dispatch.rs b/rust/crates/sdm-core/tests/dispatch.rs index 63acae7a6..067893ab9 100644 --- a/rust/crates/sdm-core/tests/dispatch.rs +++ b/rust/crates/sdm-core/tests/dispatch.rs @@ -672,8 +672,6 @@ fn materialize(seed: Option) -> Operation { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }) diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 75a4e644a..09a74fd46 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -240,8 +240,6 @@ fn materialize(held: &HeldSchema) -> Run { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }); diff --git a/rust/crates/sdm-core/tests/materialize.rs b/rust/crates/sdm-core/tests/materialize.rs index 906629481..b897b2fa0 100644 --- a/rust/crates/sdm-core/tests/materialize.rs +++ b/rust/crates/sdm-core/tests/materialize.rs @@ -228,8 +228,6 @@ fn materialize_start( values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }); @@ -1110,11 +1108,6 @@ fn the_options_the_host_passes_through_are_refused_by_name() { for (asked, code) in [ ("0", ErrorCode::InvalidRequest), ("-3", ErrorCode::InvalidRequest), - ("10001", ErrorCode::UnsupportedRequest), - ( - "1267650600228229401496703205376", - ErrorCode::UnsupportedRequest, - ), ("2.5", ErrorCode::InvalidRequest), ] { let err = valued( @@ -1136,13 +1129,13 @@ fn the_options_the_host_passes_through_are_refused_by_name() { assert_eq!(err.code, ErrorCode::InvalidRequest, "{values}"); assert_eq!(err.fields["field"], Detail::text(field), "{values}"); } - let past = valued(r#"{"num_hops": 7}"#, &[]).unwrap_err(); - assert_eq!(past.code, ErrorCode::UnsupportedRequest); - assert_eq!(past.fields["field"], Detail::text("num_hops")); - assert!(!past.fields.contains_key("rule"), "the bound, not the rule"); - // The sampling options taken reach the first read. + // The sampling options taken reach the first read: as many hops as are + // asked, the six an earlier endpoint read and past them. for values in [ r#"{"num_hops": 3}"#, + r#"{"num_hops": 7}"#, + r#"{"num_hops": 1000000}"#, + r#"{"num_neighbors": [8, 8, 4, 4, 2, 2, 1, 1]}"#, r#"{"num_neighbors": [4, 0]}"#, r#"{"random_seed": null}"#, r#"{"max_pq_iterations": 2, "use_prediction_time": true}"#, diff --git a/rust/crates/sdm-core/tests/predict.rs b/rust/crates/sdm-core/tests/predict.rs index 3a37675bd..cdb78ec4f 100644 --- a/rust/crates/sdm-core/tests/predict.rs +++ b/rust/crates/sdm-core/tests/predict.rs @@ -34,7 +34,8 @@ use sdm_contract::ops::{ }; use sdm_contract::run::{Endpoint, StartRun}; use sdm_contract::{ - CONTRACT_VERSION, CoreError, Detail, ErrorCode, ErrorKind, HostFacts, Scalar, Step, Unit, + CONTRACT_VERSION, CoreError, Detail, ErrorCode, ErrorKind, HostFacts, Scalar, Sentence, Step, + Unit, }; use sdm_core::{ Attachment, Attachments, CompositionRoot, ModelResult, Run, UnknownZone, Zones, start, @@ -1260,40 +1261,140 @@ fn an_output_named_and_left_out_is_noted() { } /// The call's size: the core writes the call's bytes, which the host sends -/// unchanged, and refuses a call past the endpoint's limit before the model -/// is called (register PD-2, fix round 1); counted to the byte, -/// with characters past ASCII counted as the bytes written. +/// unchanged (register PD-2), and holds a call to no size of its own. #[test] -fn a_call_past_the_limit_is_refused_before_it_is_sent() { - // Pad a non-truncated metadata field; diagnostic query clipping is independent. - let padded = |padding: &str| { - payload_text().replace( - "\"metadata\":{", - &format!("\"metadata\":{{\"caller_note\":\"{padding}\","), +fn a_call_of_any_size_is_sent() { + let padding = "a".repeat(31 * 1024 * 1024); + let payload = payload_text().replace( + "\"metadata\":{", + &format!("\"metadata\":{{\"caller_note\":\"{padding}\","), + ); + let mut run = predict_run(REQUEST_STATE_VERSION, payload, "kumo-relational").unwrap(); + let (_, body) = call(&mut run); + assert!(body.len() > 31 * 1024 * 1024); +} + +/// The model's answer to a call of the instance ids `ids`, each its row. +fn rows_answered(ids: &[i64]) -> ModelResult { + let predictions: Vec = ids + .iter() + .enumerate() + .map(|(at, id)| format!(r#"{{"id":"{id}","row_index":{at},"prediction":1.0}}"#)) + .collect(); + ModelResult::Response { + status: 200, + content_type: "application/json".into(), + body: format!( + r#"{{"id":"r1","model":"m","metadata":{{}},"predictions":[{}]}}"#, + predictions.join(",") ) - }; - let called = |payload: String| -> Result { - let mut run = predict_run(REQUEST_STATE_VERSION, payload, "kumo-relational").unwrap(); - match run.next()? { - Step::CallModel(call) => match run.take_attachment(call.body) { - Some(Attachment::Bytes(body)) => Ok(body.len()), - _ => panic!("the body is attached"), - }, - other => panic!("not a call: {other:?}"), + .into_bytes(), + } +} + +fn refused_with(status: u16) -> ModelResult { + ModelResult::Response { + status, + content_type: "application/json".into(), + body: b"{}".to_vec(), + } +} + +/// The rows a call scores: its predict half's instance ids. +fn scored(body: &[u8]) -> Vec { + let read: serde_json::Value = serde_json::from_slice(body).unwrap(); + read["predict"]["instance_table"]["rows"] + .as_array() + .unwrap() + .iter() + .map(|row| row[0].as_i64().unwrap_or(-1)) + .collect() +} + +/// A call refused as too large (413), or by a gateway (502, 504), or timed +/// out (the host's 408), is asked again in halves, each with the whole +/// context, and the answers are numbered as one. +#[test] +fn a_call_too_large_or_too_slow_is_asked_again_in_halves() { + for status in [408, 413, 504] { + let mut run = + predict_run(REQUEST_STATE_VERSION, payload_text(), "kumo-relational").unwrap(); + let (id, whole) = call(&mut run); + assert_eq!(scored(&whole).len(), 2); + run.submit_model(id, refused_with(status)).unwrap(); + for _ in 0..2 { + let (id, half) = call(&mut run); + let read: serde_json::Value = serde_json::from_slice(&half).unwrap(); + let sent: serde_json::Value = serde_json::from_slice(&whole).unwrap(); + assert_eq!(read["context"], sent["context"], "{status}"); + let ids = scored(&half); + assert_eq!(ids.len(), 1, "{status}"); + run.submit_model(id, rows_answered(&ids)).unwrap(); + } + assert!(matches!(run.next().unwrap(), Step::Done { .. }), "{status}"); + } +} + +/// A `422`, a `502`, a `503` or a `500`, or any refusal not of a size, is +/// not asked again by the core: the run ends with the endpoint's refusal +/// (the host has tried a temporary one again at its size). +#[test] +fn a_refusal_not_of_a_size_is_not_split() { + for status in [422, 502, 503, 500] { + let (_, end) = answered(status, b"{}"); + let error = end.unwrap_err(); + assert_eq!( + error.fields["status"], + Detail::Scalar(Scalar::UInt16(status)), + "{status}" + ); + } +} + +/// `payload_text` scoring `rows` rows, its context of `context` rows. +fn sized_payload(rows: usize, context: usize) -> String { + let scored: Vec = (0..rows) + .map(|i| format!("[{},{}]", context + i, i)) + .collect(); + let known: Vec = (0..context).map(|i| format!("[{i},{i}.5]")).collect(); + payload_text() + .replace( + r#"[[2,7],[3,"9007199254740993"]]"#, + &format!("[{}]", scored.join(",")), + ) + .replace( + "[[0,1e-05],[1,12345678.900000001]]", + &format!("[{}]", known.join(",")), + ) +} + +/// A batch is halved at most three times, to an eighth of its rows; still +/// refused then, the call is refused naming its rows, bytes and status. +#[test] +fn a_batch_is_halved_at_most_three_times() { + let mut run = predict_run( + REQUEST_STATE_VERSION, + sized_payload(16, 2), + "kumo-relational", + ) + .unwrap(); + let mut sizes = Vec::new(); + let end = loop { + match run.next() { + Ok(Step::CallModel(call)) => { + let Some(Attachment::Bytes(body)) = run.take_attachment(call.body) else { + panic!("the body is attached"); + }; + sizes.push(scored(&body).len()); + run.submit_model(call.id, refused_with(413)).unwrap(); + } + other => break other, } }; - let limit = sdm_request::payload::PAYLOAD_BYTES; - let base = called(padded("")).unwrap(); - // ASCII to the limit, exactly: taken; one byte more: refused. - assert_eq!(called(padded(&"a".repeat(limit - base))).unwrap(), limit); - let past = called(padded(&"a".repeat(limit - base + 1))).unwrap_err(); - assert_eq!(past.code, ErrorCode::InvalidRequest); - assert_eq!(past.fields["field"], text("call")); - // Three-byte characters, each counted as its three bytes written. - let euros = (limit - base) / 3; - let rest = limit - base - 3 * euros; - let at = format!("{}{}", "\u{20ac}".repeat(euros), "a".repeat(rest)); - assert_eq!(called(padded(&at)).unwrap(), limit); - let over = format!("{}{}", "\u{20ac}".repeat(euros + 1), "a".repeat(rest)); - assert!(called(padded(&over)).is_err()); + assert_eq!(sizes, [16, 8, 4, 2]); + let error = end.unwrap_err(); + assert_eq!(error.sentence, Sentence::CallTooLargeAfterHalving); + assert_eq!(error.fields["rows"], Detail::Scalar(Scalar::UInt64(2))); + assert_eq!(error.fields["halvings"], Detail::Scalar(Scalar::UInt32(3))); + assert_eq!(error.fields["status"], Detail::Scalar(Scalar::UInt16(413))); } diff --git a/rust/crates/sdm-core/tests/serving.rs b/rust/crates/sdm-core/tests/serving.rs index 92f611d81..6fefd590c 100644 --- a/rust/crates/sdm-core/tests/serving.rs +++ b/rust/crates/sdm-core/tests/serving.rs @@ -349,10 +349,10 @@ fn each_call_is_wrapped_as_the_endpoint_reads_it_and_answered_from_its_row() { } #[test] -fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { - // The first call fits; the second, larger, does not: it is refused - // when it would be sent, carrying the first call's answers, as the old - // transport weighs each call as it sends it. +fn a_call_past_the_endpoint_s_limit_is_split_up_front_and_one_row_past_it_refused() { + // A serving endpoint drops a call too large as a connection, so a call + // past its limit is split before anything is sent, each part with the + // whole context; a call of one row past it is refused, nothing sent. let mut probe = predict(&[&[10]], serving(16_000_000)); let Ok(Step::CallModel(call)) = probe.next() else { panic!("a call"); @@ -367,18 +367,25 @@ fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { retryable: false, }; let (end, bodies) = drive(&mut run, None, &failure); - assert_eq!(bodies.len(), 1); - assert_eq!(bodies[0], one); - let error = end.unwrap_err(); - assert_eq!(error.sentence, Sentence::ServingPastLimit); - assert_eq!(error.code, ErrorCode::InvalidRequest); - assert_eq!(error.fields["limit"], Detail::Scalar(Scalar::UInt64(limit))); - assert_eq!(count(&error, "calls_answered"), Some(1)); - // A limit the first call misses: nothing is sent. + assert_eq!(bodies.len(), 4); + assert!(bodies.iter().all(|body| body.len() as u64 <= limit)); + let Ok(Step::Done { + result: OperationResult::Predict(predicted), + }) = end + else { + panic!("the run predicts: {end:?}"); + }; + let result = arrow(&mut run, predicted.result); + assert_eq!(int64s(&result, "row_index"), (0..4).collect::>()); let mut run = predict(&[&[10]], serving(limit - 1)); let (end, bodies) = drive(&mut run, None, &failure); assert!(bodies.is_empty()); - assert_eq!(end.unwrap_err().sentence, Sentence::ServingPastLimit); + let error = end.unwrap_err(); + assert_eq!(error.sentence, Sentence::ServingPastLimit); + assert_eq!( + error.fields["limit"], + Detail::Scalar(Scalar::UInt64(limit - 1)) + ); } #[test] diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index 23fa7ebe7..8dcb873ca 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -908,8 +908,6 @@ fn materialize_query( values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }), diff --git a/rust/crates/sdm-engine/src/frames.rs b/rust/crates/sdm-engine/src/frames.rs index 1c392bd1b..f0b4dd395 100644 --- a/rust/crates/sdm-engine/src/frames.rs +++ b/rust/crates/sdm-engine/src/frames.rs @@ -237,6 +237,13 @@ pub enum FrameNeighbor { } impl Snapshot { + /// How many rows the snapshot admits across its tables: a walk reaches + /// no row it has not reached at any hop past this many, since each hop + /// that reaches anything reaches at least one row anew. + pub fn rows(&self) -> usize { + self.tables.values().map(|table| table.valid.len()).sum() + } + /// Physical storage of one admitted column, without scanning or sampling rows. pub fn column_type(&self, table: &str, column: &str) -> Result { let rows = &self diff --git a/rust/crates/sdm-host/tests/runs.rs b/rust/crates/sdm-host/tests/runs.rs index 94393bf4e..01ea45f51 100644 --- a/rust/crates/sdm-host/tests/runs.rs +++ b/rust/crates/sdm-host/tests/runs.rs @@ -720,8 +720,6 @@ fn request_state_leaves_one_run_as_bytes_and_returns_under_the_host_s_id() { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }), diff --git a/rust/crates/sdm-request/src/batching.rs b/rust/crates/sdm-request/src/batching.rs new file mode 100644 index 000000000..36195b830 --- /dev/null +++ b/rust/crates/sdm-request/src/batching.rs @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A call's predict rows taken apart, so a call too large is asked as +//! several smaller ones: up front where a serving endpoint's byte limit is +//! known ([`fitted`]), or in halves where the endpoint refuses it. +//! +//! A payload carries its context whole and the rows it scores beside it, +//! each row of its predict half's tables belonging to one of the rows +//! scored by its `instance_id`. [`predict_slice`] keeps the context as it +//! is and a run of the rows scored, with the rows of every related table +//! that belong to them, numbered as a payload of only those rows would be. +//! The context is sent with every call, so it is never split. + +use std::collections::BTreeSet; +use std::ops::Range; + +use sdm_contract::{CoreError, Sentence}; +use sdm_wire::json::Value; + +use crate::payload::INSTANCE_ID; +use crate::predict::predict_rows; + +fn bad(reason: &str) -> CoreError { + CoreError::new( + Sentence::Internal, + format!("a payload cannot be split: {reason}"), + ) +} + +/// The rows a table of the payload holds: its column names and rows. +pub(crate) fn table_rows(table: &Value) -> Option<(Vec<&str>, &[Value])> { + let Some(Value::Array(columns)) = table.get("columns") else { + return None; + }; + let Some(Value::Array(rows)) = table.get("rows") else { + return None; + }; + let names: Vec<&str> = columns.iter().filter_map(Value::as_text).collect(); + (names.len() == columns.len()).then_some((names, rows.as_slice())) +} + +/// An instance id as a whole number, where it is one. +fn id_of(value: &Value) -> Option { + match value { + Value::Number(token) => token.parse::().ok(), + _ => None, + } +} + +/// `table` with only the rows `keep` takes, each row's `instance_id` moved +/// `shift` places down where it is a whole number. +fn kept( + table: &Value, + keep: impl Fn(usize, &[Value]) -> bool, + id_at: Option, + shift: i64, +) -> Result { + let (_, rows) = table_rows(table).ok_or_else(|| bad("a table has no rows to take"))?; + let rows = rows + .iter() + .enumerate() + .filter_map(|(at, row)| { + let Value::Array(cells) = row else { + return Some(Err(bad("a row is not a list"))); + }; + if !keep(at, cells) { + return None; + } + let mut cells = cells.clone(); + if shift != 0 + && let Some(at) = id_at + && let Some(id) = cells.get(at).and_then(id_of) + { + cells[at] = Value::Number((id - shift).to_string()); + } + Some(Ok(Value::Array(cells))) + }) + .collect::, _>>()?; + let mut table = table.clone(); + table.set("rows", Value::Array(rows)); + Ok(table) +} + +/// `payload` scoring only its predict rows `range`, in order, with the rows +/// of each related table of its predict half that belong to them, and its +/// context whole. The rows kept are numbered from where the payload's own +/// were (`instance_id`), as a payload built of only those rows numbers +/// them, and its metadata counts them (`num_prediction_examples`). +pub fn predict_slice(payload: &Value, range: Range) -> Result { + let predict = payload + .get("predict") + .ok_or_else(|| bad("there is no predict half"))?; + let instance = predict + .get("instance_table") + .ok_or_else(|| bad("there is no instance table"))?; + let (names, rows) = table_rows(instance).ok_or_else(|| bad("the instance table is unread"))?; + if range.end > rows.len() || range.start > range.end { + return Err(bad("the rows asked for are not there")); + } + let id_at = names.iter().position(|name| *name == INSTANCE_ID); + // The ids of the rows kept, and how far they move down: a run of rows + // numbered on from the first row of the half is numbered from it again. + let ids = |row: &Value| match (row, id_at) { + (Value::Array(cells), Some(at)) => cells.get(at).and_then(id_of), + _ => None, + }; + let kept_ids: BTreeSet = rows[range.clone()].iter().filter_map(ids).collect(); + let shift = match ( + rows.first().and_then(ids), + rows.get(range.start).and_then(ids), + ) { + (Some(first), Some(start)) if kept_ids.len() == range.len() => start - first, + _ => 0, + }; + let mut half = predict.clone(); + half.set( + "instance_table", + kept(instance, |at, _| range.contains(&at), id_at, shift)?, + ); + if let Some(Value::Object(related)) = predict.get("related_tables") { + let mut tables = Vec::with_capacity(related.len()); + for (name, table) in related { + let (columns, _) = table_rows(table).ok_or_else(|| bad("a related table is unread"))?; + let at = columns.iter().position(|column| *column == INSTANCE_ID); + let Some(at) = at.filter(|_| id_at.is_some()) else { + // A related table whose rows name no instance belongs to no + // one row scored: kept whole. + tables.push((name.clone(), table.clone())); + continue; + }; + let belongs = |_: usize, cells: &[Value]| { + cells + .get(at) + .and_then(id_of) + .is_some_and(|id| kept_ids.contains(&id)) + }; + tables.push((name.clone(), kept(table, belongs, Some(at), shift)?)); + } + half.set("related_tables", Value::Object(tables)); + } + let mut sliced = payload.clone(); + sliced.set("predict", half); + if let Some(metadata @ Value::Object(_)) = payload.get("metadata") + && metadata.get("num_prediction_examples").is_some() + { + let mut metadata = metadata.clone(); + metadata.set( + "num_prediction_examples", + Value::Number(range.len().to_string()), + ); + sliced.set("metadata", metadata); + } + Ok(sliced) +} + +/// What a call comes to ([`fitted`]): what it carries where it fits, or the +/// refusal of a call too large; the outer error ends the split. +pub type Fit = Result, CoreError>; + +/// The calls `payload` is asked as so that each fits, in order, each with +/// what `fit` made of it: the whole where it fits, else halves, each split +/// again until it fits. `fit` answers what a call carries, or the refusal +/// of a call too large (the inner error); any other error ends the split. A +/// call of one row too large cannot be split, and its refusal is returned: +/// the context is sent with every call. +pub fn fitted( + payload: &Value, + fit: &mut dyn FnMut(&Value) -> Fit, +) -> Result, CoreError> { + let past = match fit(payload)? { + Ok(fitted) => return Ok(vec![(payload.clone(), fitted)]), + Err(past) => past, + }; + let rows = match predict_rows(payload) { + Some(rows) if rows > 1 => rows, + _ => return Err(past), + }; + let mut done = Vec::new(); + // Ranges still to fit, the first next; the whole is known not to. + let middle = rows / 2; + let mut pending: Vec> = vec![middle..rows, 0..middle]; + while let Some(range) = pending.pop() { + let sliced = predict_slice(payload, range.clone())?; + match fit(&sliced)? { + Ok(fitted) => done.push((sliced, fitted)), + Err(past) if range.len() <= 1 => return Err(past), + Err(_) => { + let middle = range.start + range.len() / 2; + pending.push(middle..range.end); + pending.push(range.start..middle); + } + } + } + Ok(done) +} + +#[cfg(test)] +mod tests { + use super::*; + use sdm_wire::json; + + fn payload() -> Value { + json::read( + br#"{ + "model": "kumo-relational", + "context": {"instance_table": {"format": "rows", "columns": ["instance_id", "TARGET"], "rows": [[0, 1], [1, 2]]}, + "related_tables": {"users": {"format": "rows", "columns": ["instance_id", "user_id"], "rows": [[0, 10], [1, 11]]}}}, + "predict": {"instance_table": {"format": "rows", "columns": ["instance_id", "__kumo_entity_ref_0"], "rows": [[2, 20], [3, 21], [4, 22], [5, 23]]}, + "related_tables": {"users": {"format": "rows", "columns": ["instance_id", "user_id"], "rows": [[2, 20], [3, 21], [4, 22], [5, 23], [3, 99]]}, + "plain": {"format": "rows", "columns": ["x"], "rows": [[7]]}}}, + "metadata": {"num_context_examples": 2, "num_prediction_examples": 4} +}"#, + ) + .unwrap() + } + + fn rows(value: &Value, path: &[&str]) -> String { + let mut at = value; + for key in path { + at = at.get(key).unwrap(); + } + String::from_utf8(at.get("rows").unwrap().to_bytes()).unwrap() + } + + #[test] + fn a_slice_keeps_the_context_and_the_rows_of_its_instances_renumbered() { + let whole = payload(); + let sliced = predict_slice(&whole, 1..3).unwrap(); + assert_eq!(sliced.get("context"), whole.get("context")); + assert_eq!( + rows(&sliced, &["predict", "instance_table"]), + "[[2,21],[3,22]]" + ); + assert_eq!( + rows(&sliced, &["predict", "related_tables", "users"]), + "[[2,21],[3,22],[2,99]]" + ); + assert_eq!( + rows(&sliced, &["predict", "related_tables", "plain"]), + "[[7]]" + ); + assert_eq!( + sliced + .get("metadata") + .unwrap() + .get("num_prediction_examples"), + Some(&Value::Number("2".into())) + ); + // The first run is numbered as it was. + let first = predict_slice(&whole, 0..1).unwrap(); + assert_eq!(rows(&first, &["predict", "instance_table"]), "[[2,20]]"); + assert_eq!(predict_slice(&whole, 0..4).unwrap(), whole); + assert!(predict_slice(&whole, 2..5).is_err()); + } + + fn size(value: &Value) -> usize { + value.to_bytes().len() + } + + /// The rows each call of `whole` scores where a call takes `most` bytes. + fn calls(whole: &Value, most: usize) -> Result, CoreError> { + let mut fit = |p: &Value| { + Ok(if size(p) <= most { + Ok(()) + } else { + Err(CoreError::new(Sentence::Internal, "refused")) + }) + }; + Ok(fitted(whole, &mut fit)? + .iter() + .map(|(call, ())| predict_rows(call).unwrap()) + .collect()) + } + + #[test] + fn a_call_that_fits_is_one_call_and_one_past_is_halved_until_each_fits() { + let whole = payload(); + let one = size(&predict_slice(&whole, 0..1).unwrap()); + let two = size(&predict_slice(&whole, 0..2).unwrap()); + let all = size(&whole); + assert_eq!(calls(&whole, all).unwrap(), [4]); + // One byte under the whole: two calls of two rows. + assert_eq!(calls(&whole, all - 1).unwrap(), [2, 2]); + // Exactly two rows' worth: still two calls. One byte under it: the + // last two rows carry fewer related rows than the first two, so + // together they fit where the first two do not. + assert_eq!(calls(&whole, two).unwrap(), [2, 2]); + assert_eq!(calls(&whole, two - 1).unwrap(), [1, 1, 2]); + // Under one row's worth, the context alone does not fit. + assert_eq!(calls(&whole, one - 1).unwrap_err().message, "refused"); + } +} diff --git a/rust/crates/sdm-request/src/lib.rs b/rust/crates/sdm-request/src/lib.rs index 41f0b89cb..7371cebbf 100644 --- a/rust/crates/sdm-request/src/lib.rs +++ b/rust/crates/sdm-request/src/lib.rs @@ -14,6 +14,7 @@ #![forbid(unsafe_code)] +pub mod batching; pub mod inference; pub mod options; pub mod payload; diff --git a/rust/crates/sdm-request/src/options.rs b/rust/crates/sdm-request/src/options.rs index ffa61d7e8..b233aa83b 100644 --- a/rust/crates/sdm-request/src/options.rs +++ b/rust/crates/sdm-request/src/options.rs @@ -591,14 +591,44 @@ pub enum Hop { Stop, } +/// The hops a walk takes, each with the neighbors it keeps of each row: the +/// sizes listed, then the last of them again for as many hops more as were +/// asked. A walk reads them one hop at a time ([`Hops::get`]), so a count +/// of hops far past what the data holds costs nothing: the walk ends at +/// the first hop that reaches no row. +#[derive(Debug, Clone, PartialEq)] +pub struct Hops { + listed: Vec, + /// How many hops follow the listed ones, each the last listed size. + more: usize, +} + +impl Hops { + /// The size of hop `hop`, counted from zero; none past the last. + pub fn get(&self, hop: usize) -> Option<&Hop> { + match self.listed.get(hop) { + Some(size) => Some(size), + None if hop - self.listed.len() < self.more => self.listed.last(), + None => None, + } + } +} + +impl From> for Hops { + fn from(listed: Vec) -> Self { + Hops { listed, more: 0 } + } +} + /// The hops the walk takes, each with its size, as the engine resolves /// them (`_resolve_options`): `num_neighbors` where it is given, each size /// read as [`hop`] reads it; else the first `num_hops` of the run mode's sizes -/// (`_DEFAULT_NUM_NEIGHBORS`), two where it is not named, and a `num_hops` +/// (`_DEFAULT_NUM_NEIGHBORS`), two where it is not named, each hop past the +/// sizes the mode lists keeping as many as its last; and a `num_hops` /// named as none, which the engine cannot compare with its bounds, /// `INVALID_REQUEST` (`not_a_whole_number`), as the engine's `TypeError` -/// is the caller's. Call it after [`hops`] has bounded the count. -pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { +/// is the caller's. Call it after [`hops`] has checked the count. +pub fn neighbors(asked: &Asked, mode: RunMode) -> Result { let Some(given) = &asked.neighbors else { if asked.num_hops_none { return Err(mistyped( @@ -609,14 +639,13 @@ pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { } let hops = asked .num_hops - .and_then(|n| n.magnitude) - .map_or(2, |n| usize::try_from(n).unwrap_or(usize::MAX)); - return Ok(mode - .all_neighbors() - .into_iter() - .take(hops) - .map(Hop::Keep) - .collect()); + .map_or(Some(2), |n| n.magnitude) + .map_or(usize::MAX, |n| usize::try_from(n).unwrap_or(usize::MAX)); + let sizes = mode.all_neighbors(); + return Ok(Hops { + listed: sizes.into_iter().take(hops).map(Hop::Keep).collect(), + more: hops.saturating_sub(sizes.len()), + }); }; given .iter() @@ -624,7 +653,8 @@ pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { .map(|(at, size)| { hop(size).map_err(|e| e.with_field("hop", Detail::Scalar(Scalar::UInt64(at as u64)))) }) - .collect() + .collect::, _>>() + .map(Hops::from) } /// One hop's size, as the reference's DuckDB sampler reads the value it @@ -790,14 +820,14 @@ pub fn rows_per_call(given: &Value) -> Result, CoreError> { } } -/// The most rows one model call scores (`predict_rows_max`, the serving -/// layer's `TFM_MAX_TEST_ROWS`, which the reference's client and engine -/// hold at 10,000 from `main`'s 1fc5a686). +/// How many rows one model call scores where the caller names no size +/// (`predict_rows_max`, the serving layer's `TFM_MAX_TEST_ROWS` default): +/// a batching default, not a cap; a call the endpoint refuses as too large +/// is asked again in halves. pub const PREDICT_ROWS: u64 = 10_000; -/// How many entities one call scores, as the caller asked or as the -/// endpoint allows (`rows_one_call_scores`): the caller's size only ever -/// lowers the endpoint's. +/// How many entities one call scores, as the caller asked, else the +/// default (`rows_one_call_scores`). pub fn rows_one_call_scores(asked: Option) -> u64 { rows_one_task_call_scores(asked, BatchTask::Ordinary) } @@ -810,14 +840,13 @@ pub enum BatchTask { TemporalLink, } -/// Apply the task's serving limit after validating the requested positive size. -/// PQL and supplied examples use the same policy; hosts do not split rows. +/// The rows one call scores: the requested positive size, else the task's +/// default batch. PQL and supplied examples use the same policy. pub fn rows_one_task_call_scores(asked: Option, task: BatchTask) -> u64 { - let limit = match task { + asked.unwrap_or(match task { BatchTask::Ordinary => PREDICT_ROWS, BatchTask::TemporalLink => 200, - }; - asked.map_or(limit, |n| n.min(limit)) + }) } /// A task's kind, as the engine names it. @@ -860,7 +889,8 @@ impl RunMode { } } - /// The most context examples the mode gathers. + /// The most context examples the mode gathers where the caller asks + /// for no more: a best run gathers as many as are asked past it. pub fn context_examples(self) -> usize { match self { RunMode::Fast => 1_000, @@ -881,9 +911,8 @@ impl RunMode { /// The run mode a context size asks for, as the client chooses it /// (`_run_mode_for`, with `num_context_samples`' bound of /// `_validate_materialize_args`): none or up to a fast run's examples, a -/// fast run; up to a best run's, a best one. Below 1 is `INVALID_REQUEST`; -/// past a best run's examples, `UNSUPPORTED_REQUEST`, as the endpoint takes -/// no more. +/// fast run; past it, a best one, gathering as many as are asked. Below 1 +/// is `INVALID_REQUEST`. pub fn run_mode(asked: Option, given: Option<&str>) -> Result { // A refusal names the size as the caller gave it, its JSON text. let named = |refused: CoreError| match given { @@ -908,29 +937,16 @@ pub fn run_mode(asked: Option, given: Option<&str>) -> Result Ok(RunMode::Fast), - Some(n) if n <= RunMode::Best.context_examples() as u128 => Ok(RunMode::Best), - _ => Err(named( - CoreError::new( - Sentence::NumContextSamplesPastMost, - "num_context_samples is past the most context the endpoint takes", - ) - .with_field("field", Detail::text("num_context_samples")) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(RunMode::Best.context_examples() as u64)), - ), - )), + _ => Ok(RunMode::Best), } } -/// The most hops the endpoint reads (`hops_max`). -pub const HOPS_MAX: u128 = 6; - -/// The hops the sampling options ask for, as the client bounds them +/// The hops the sampling options ask for, as the client checks them /// (`_check_shape`): `num_neighbors`' length where it is given, else /// `num_hops`; none, or fewer than one, is `INVALID_REQUEST` (a request -/// that crosses no relationship), more than [`HOPS_MAX`] -/// `UNSUPPORTED_REQUEST`. +/// that crosses no relationship). The client sets no most: the model reads +/// as many hops as the request carries, and a walk ends at the first hop +/// that reaches no row. pub fn hops(num_hops: Option, num_neighbors: Option) -> Result<(), CoreError> { let ( field, @@ -943,34 +959,19 @@ pub fn hops(num_hops: Option, num_neighbors: Option) -> Result (None, Some(n)) => ("num_hops", n), (None, None) => return Ok(()), }; - // How many hops were asked, as the host's words name them. - let asked = value.map(|n| format!("{}{n}", if negative && n > 0 { "-" } else { "" })); - let named = |error: CoreError| match &asked { - Some(n) => error.with_field("hops", Detail::text(n)), - None => error, - }; if negative || value == Some(0) { + // How many hops were asked, as the host's words name them. + let asked = value.map(|n| format!("{}{n}", if negative && n > 0 { "-" } else { "" })); let sentence = if field == "num_hops" { Sentence::NoHopNumHops } else { Sentence::NoHopNumNeighbors }; - return Err(named(invalid(sentence, field, "no_hop")) - .with_field("limit", Detail::Scalar(Scalar::UInt64(HOPS_MAX as u64)))); - } - if value.is_none_or(|n| n > HOPS_MAX) { - return Err(named( - CoreError::new( - if asked.is_some() { - Sentence::HopsPastEndpoint - } else { - Sentence::HopsPastEndpointUncounted - }, - "the sampling options ask more hops than the endpoint reads", - ) - .with_field("field", Detail::text(field)) - .with_field("limit", Detail::Scalar(Scalar::UInt64(HOPS_MAX as u64))), - )); + let refused = invalid(sentence, field, "no_hop"); + return Err(match asked { + Some(n) => refused.with_field("hops", Detail::text(n)), + None => refused, + }); } Ok(()) } @@ -1370,26 +1371,23 @@ mod tests { #[test] fn the_run_mode_is_the_client_s() { assert_eq!(run_mode(None, None).unwrap(), RunMode::Fast); - assert_eq!(run_mode(int(false, Some(1)), None).unwrap(), RunMode::Fast); assert_eq!( run_mode(int(false, Some(1_000)), None).unwrap(), RunMode::Fast ); - assert_eq!( - run_mode(int(false, Some(1_001)), None).unwrap(), - RunMode::Best - ); - assert_eq!( - run_mode(int(false, Some(10_000)), None).unwrap(), - RunMode::Best - ); - for (asked, code) in [ - (int(false, Some(0)), ErrorCode::InvalidRequest), - (int(true, Some(5)), ErrorCode::InvalidRequest), - (int(false, Some(10_001)), ErrorCode::UnsupportedRequest), - (int(false, None), ErrorCode::UnsupportedRequest), + // Past a fast run's examples, a best run, however many are asked. + for asked in [ + int(false, Some(1_001)), + int(false, Some(200_000)), + int(false, None), ] { - assert_eq!(run_mode(asked, None).unwrap_err().code, code, "{asked:?}"); + assert_eq!(run_mode(asked, None).unwrap(), RunMode::Best); + } + for asked in [int(false, Some(0)), int(true, Some(5))] { + assert_eq!( + run_mode(asked, None).unwrap_err().code, + ErrorCode::InvalidRequest + ); } } @@ -1398,6 +1396,11 @@ mod tests { let some = |n: u128| int(false, Some(n)); assert!(hops(None, None).is_ok()); assert!(hops(some(6), None).is_ok()); + // No most: past the six an earlier endpoint read is taken. + assert!(hops(some(7), None).is_ok()); + assert!(hops(some(1_000), None).is_ok()); + assert!(hops(None, some(12)).is_ok()); + assert!(hops(int(false, None), None).is_ok()); assert!( hops(some(9), some(2)).is_ok(), "the neighbors' length counts" @@ -1414,14 +1417,6 @@ mod tests { hops(int(true, Some(1)), None).unwrap_err().code, ErrorCode::InvalidRequest ); - assert_eq!( - hops(some(7), None).unwrap_err().code, - ErrorCode::UnsupportedRequest - ); - assert_eq!( - hops(int(false, None), None).unwrap_err().code, - ErrorCode::UnsupportedRequest - ); } #[test] @@ -1942,15 +1937,32 @@ mod tests { #[test] fn the_hops_are_the_run_mode_s_first_num_hops_two_by_default() { let fast = |text: &str| neighbors(&asked(text), RunMode::Fast).unwrap(); - assert_eq!(fast("{}"), vec![Hop::Keep(32), Hop::Keep(32)]); + assert_eq!(fast("{}"), Hops::from(vec![Hop::Keep(32), Hop::Keep(32)])); assert_eq!( fast(r#"{"num_hops": 4}"#), - vec![Hop::Keep(32), Hop::Keep(32), Hop::Keep(8), Hop::Keep(8)] + Hops::from(vec![ + Hop::Keep(32), + Hop::Keep(32), + Hop::Keep(8), + Hop::Keep(8) + ]) ); assert_eq!( neighbors(&asked(r#"{"num_hops": 3}"#), RunMode::Best).unwrap(), - vec![Hop::Keep(64), Hop::Keep(64), Hop::Keep(8)] + Hops::from(vec![Hop::Keep(64), Hop::Keep(64), Hop::Keep(8)]) ); + // Past the six sizes a mode lists, each hop keeps the last. + let eight = fast(r#"{"num_hops": 8}"#); + let sizes: Vec> = (0..9).map(|hop| eight.get(hop).cloned()).collect(); + let mut expected: Vec> = [32, 32, 8, 8, 4, 4, 4, 4] + .iter() + .map(|&n| Some(Hop::Keep(n))) + .collect(); + expected.push(None); + assert_eq!(sizes, expected); + // A count far past any graph is read lazily, never laid out. + let far = fast(r#"{"num_hops": 100000000000000000000000000000000000000000}"#); + assert_eq!(far.get(1_000_000), Some(&Hop::Keep(4))); // num_hops named as none, with no num_neighbors, is the engine's // TypeError: the caller's. let e = neighbors(&asked(r#"{"num_hops": null}"#), RunMode::Fast).unwrap_err(); @@ -1963,7 +1975,7 @@ mod tests { let given = asked(r#"{"num_neighbors": [5, 1, 0, 3], "num_hops": 2}"#); assert_eq!( neighbors(&given, RunMode::Fast).unwrap(), - vec![Hop::Keep(5), Hop::Keep(1), Hop::Stop, Hop::Keep(3)] + Hops::from(vec![Hop::Keep(5), Hop::Keep(1), Hop::Stop, Hop::Keep(3)]) ); // Numeric bounds keep their sign; caller text retains its origin // for capability-driven warehouse lookback arithmetic. @@ -1986,19 +1998,19 @@ mod tests { RunMode::Fast ) .unwrap(), - vec![Hop::Keep(4), wanted], + Hops::from(vec![Hop::Keep(4), wanted]), ); } assert_eq!(hops_ignored(&given), None); let wide = asked(r#"{"num_neighbors": [100000000000000000000000000000]}"#); assert_eq!( neighbors(&wide, RunMode::Fast).unwrap(), - vec![Hop::Keep(u64::MAX)] + Hops::from(vec![Hop::Keep(u64::MAX)]) ); let odd = asked(r#"{"num_neighbors": [2.0, 2.5, true, false, 0.5, -0.0, 0.0, 1e30]}"#); assert_eq!( neighbors(&odd, RunMode::Fast).unwrap(), - vec![ + Hops::from(vec![ Hop::Keep(2), Hop::Keep(2), Hop::Keep(1), @@ -2007,17 +2019,17 @@ mod tests { Hop::Stop, Hop::Stop, Hop::Keep(u64::MAX) - ] + ]) ); let spelled = asked(r#"{"num_neighbors": ["3", "-0", "0", "2.5"]}"#); assert_eq!( neighbors(&spelled, RunMode::Fast).unwrap(), - vec![Hop::Keep(3), Hop::Keep(0), Hop::Keep(0), Hop::Keep(2)] + Hops::from(vec![Hop::Keep(3), Hop::Keep(0), Hop::Keep(0), Hop::Keep(2)]) ); let spaced = asked(r#"{"num_neighbors": [" 3 ", "1e1", ".5"]}"#); assert_eq!( neighbors(&spaced, RunMode::Fast).unwrap(), - vec![Hop::Keep(3), Hop::Keep(10), Hop::Keep(0)] + Hops::from(vec![Hop::Keep(3), Hop::Keep(10), Hop::Keep(0)]) ); for size in ["null", "[1]"] { let e = neighbors( @@ -2037,7 +2049,7 @@ mod tests { ); assert_eq!( neighbors(&values, RunMode::Fast).unwrap(), - vec![ + Hops::from(vec![ Hop::Negative(sdm_contract::data::NegativeNeighborBound::new("-1".into()).unwrap()), Hop::Negative( sdm_contract::data::NegativeNeighborBound::new("-0.5".into()).unwrap() @@ -2046,7 +2058,7 @@ mod tests { Hop::Expression("3 OR TRUE".into()), Hop::Expression("nonsense".into()), Hop::Keep(u64::MAX), - ] + ]) ); } diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index 31df574d7..716373a7c 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -79,11 +79,6 @@ pub const RESERVED: [&str; 2] = [INSTANCE_ID, ANCHOR_TIME]; const NO_ANCHOR: i64 = -9_223_372_036_854_775_807; /// The largest integer a JSON number carries exactly in every reader. const JSON_SAFE: i128 = 9_007_199_254_740_991; -/// The largest payload the endpoint takes, in bytes (`_MAX_SIZE`), counted -/// as the bytes the payload is written as and sent, UTF-8 (register PD-2), -/// not the reference's `json.dumps` count, which escapes every character -/// past ASCII. -pub const PAYLOAD_BYTES: usize = 30 * 1024 * 1024; /// One column of a sampled table: as sampled, or a text feature as the /// reference's sampler sends it, each value its words @@ -219,11 +214,6 @@ pub struct Sampled { /// How many candidates the call asked be kept (`top_k`), written into /// the task as the engine writes it. pub top_k: Option, - /// The most rows any call of the request this payload is one call of - /// scores (`_the_largest_batch`, `main`'s multi-call `predict`): the - /// size a refusal of a payload past the endpoint's limit names as the - /// one to ask below. - pub rows_per_call: usize, /// The lagged target columns (`lag_timesteps`), in order; none /// where the call asks for none. pub lags: Vec, @@ -1493,17 +1483,6 @@ fn build_mode( num_prediction_examples: sampled.num_test as u64, }; let model = "kumo-relational".to_owned(); - // Only this small state-only member is counted separately; never copy or - // reparse the potentially 30MiB table data merely to measure its wire size. - let client_only_bytes = if client_generated_keys.is_empty() { - 0 - } else { - serde_json::to_vec(&client_generated_keys) - .expect("string map serializes") - .len() - + br#""client_generated_keys":"#.len() - + 1 // comma separating the member from model-facing members - }; let bytes = if link.is_some() { StatePayload { client_analysis_delta: client_analysis_delta.clone(), @@ -1564,49 +1543,6 @@ fn build_mode( } .to_bytes() }; - // The limit counts the bytes the request sends, as written (PD-2). - // The refusal names what to change, as the reference's client words - // its engine's (`_said_in_this_client_s_terms`, `main`'s 1fc5a686): a - // call scoring several rows can be asked as smaller calls, below the - // most rows a call of the request scores (`smaller_calls`); a call of - // one row cannot, and only a narrower context helps - // (`no_smaller_call`). - let analysis_bytes = client_analysis_delta.as_ref().map_or(0, |delta| { - serde_json::to_vec(delta) - .expect("typed delta serializes") - .len() - + br#""client_analysis_delta":"#.len() - + 1 - }); - let model_bytes = bytes.len() - client_only_bytes - analysis_bytes; - if model_bytes > PAYLOAD_BYTES { - let count = |n: usize| Detail::Scalar(sdm_contract::Scalar::UInt64(n as u64)); - return Err(CoreError::new( - if sampled.num_test > 1 { - Sentence::PayloadSmallerCalls - } else { - Sentence::PayloadNoSmallerCall - }, - "the request is larger than the endpoint takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("bytes", count(model_bytes)) - .with_field("limit", count(PAYLOAD_BYTES)) - .with_field( - "batch_too_large_to_send", - Detail::Scalar(sdm_contract::Scalar::Bool(true)), - ) - .with_field( - "remedy", - Detail::text(if sampled.num_test > 1 { - "smaller_calls" - } else { - "no_smaller_call" - }), - ) - .with_field("rows", count(sampled.num_test)) - .with_field("rows_per_call", count(sampled.rows_per_call))); - } Ok(Built { bytes, answering: answered.len(), diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index a4477af21..0810c1345 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -93,9 +93,6 @@ pub const INFERENCE_OPTIONS: [&str; 6] = [ /// The most classes `kumo-relational` classifies (`classes_max`). pub const CLASSES_MAX: usize = 10; -/// The most context rows `kumo-relational` reads (`context_rows_max`). -pub const CONTEXT_ROWS_MAX: usize = 10_000; - /// The context the mode that explains takes at most (`context_rows_fast`). pub const CONTEXT_ROWS_FAST: usize = 1_000; @@ -306,21 +303,7 @@ fn classifies(kind: &str) -> bool { /// One half's instance table: its column names and its rows. fn instance_table<'a>(payload: &'a Value, half: &str) -> Option<(Vec<&'a str>, &'a [Value])> { - let table = payload.get(half)?.get("instance_table")?; - let Some(Value::Array(columns)) = table.get("columns") else { - return None; - }; - let Some(Value::Array(rows)) = table.get("rows") else { - return None; - }; - let names: Vec<&str> = columns.iter().filter_map(Value::as_text).collect(); - (names.len() == columns.len()).then_some((names, rows.as_slice())) -} - -/// The rows of one half, counted as `Request._rows` counts them: none -/// where the half cannot be read. -fn rows_of(payload: &Value, half: &str) -> usize { - instance_table(payload, half).map_or(0, |(_, rows)| rows.len()) + crate::batching::table_rows(payload.get(half)?.get("instance_table")?) } /// The rows one call scores (`_envelope_rows(payload, 'predict')`): its @@ -451,12 +434,6 @@ pub fn prepare( }); check_shapes(spec, kind, anchored, linked, tables)?; check_task_outputs(spec, kind, &outputs, true)?; - let rows = payloads - .iter() - .map(|p| rows_of(p, "context")) - .max() - .unwrap_or(0); - check_capacity(spec, rows, false)?; if spec.id == TABULAR { // The tabular model takes no relational request this build sends. return Err(not_yet("model")); @@ -857,27 +834,6 @@ fn is_chosen(value: &Value) -> bool { } } -/// The bytes of the call `body` writes, as the host sends them, unchanged: -/// refused past the endpoint's limit (`PAYLOAD_BYTES`), counted on these -/// bytes, the envelope included, which are the bytes sent (register PD-2, -/// fix round 1), before the model is called. -pub fn call_bytes(call: &Value) -> Result, CoreError> { - let bytes = call.to_bytes(); - if bytes.len() > crate::payload::PAYLOAD_BYTES { - return Err(CoreError::new( - Sentence::CallPastEndpoint, - "the call is larger than the endpoint takes", - ) - .with_field("field", Detail::text("call")) - .with_field("bytes", Detail::Scalar(Scalar::UInt64(bytes.len() as u64))) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(crate::payload::PAYLOAD_BYTES as u64)), - )); - } - Ok(bytes) -} - /// The entity a row is about, as the result gives it: as the reference's /// `as_declared` reads its value back as the key's dtype, or as written /// where it does not read or no dtype is known. @@ -2328,23 +2284,6 @@ fn check_task_outputs( } Ok(()) } -fn check_capacity(spec: &Spec, rows: usize, direct: bool) -> Result<(), CoreError> { - if (direct || spec.id == RELATIONAL) && rows > CONTEXT_ROWS_MAX { - return Err(unsupported( - if direct { - Sentence::DirectContextLimit - } else { - Sentence::ContextPastModel - }, - spec.id, - "the request's context is larger than the model reads", - ) - .with_field("context_rows", count(rows)) - .with_field("context_rows_max", count(CONTEXT_ROWS_MAX))); - } - Ok(()) -} - /// Model preflight for live direct request facts, before conversion or empty handling. pub fn bind_direct( input: sdm_contract::direct::DirectCallFacts, @@ -2381,7 +2320,6 @@ pub fn bind_direct( input.tables, )?; check_task_outputs(spec, &input.kind, &outputs, false)?; - check_capacity(spec, input.context_rows, true)?; Ok(sdm_contract::direct::BoundDirectCall { facts, model: input.model, diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index f868a0a49..1c9e61e13 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -57,8 +57,10 @@ const STATUS_BY_CODE: &[(&str, u16)] = &[ ]; /// The HTTP statuses whose SDK error class the old client reports as the -/// status itself; any other it reports as 503, which callers retry. -const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 429, 504]; +/// status itself; any other it reports as 503, which callers retry. A 413 or +/// a 504 is kept, so the core asks the call's rows in halves; a 502, so it +/// is named as the gateway's. +const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 413, 429, 502, 504]; /// The call `call` (the core's own JSON) as the endpoint's envelope, the /// bytes a host posts, or `ServingPastLimit` when it is more than @@ -71,7 +73,18 @@ pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result max_request_bytes { + return Err(past_limit(endpoint, max_request_bytes, size)); + } + Ok(out) +} + +/// The envelope's bytes of `call`, as [`envelope`] writes them, refused +/// where a value has no JSON form. +pub fn wrapped(call: &Value) -> Result, CoreError> { + if !call.all_finite() { return Err(CoreError::new( Sentence::ServingValueNotJson, "the call carries a value with no JSON form", @@ -80,31 +93,41 @@ pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result max_request_bytes { - return Err(CoreError::new( - Sentence::ServingPastLimit, - "the call is larger than the serving endpoint takes", - ) - .with_field("endpoint", Detail::text(endpoint)) - .with_field("bytes", count(size)) - .with_field("limit", count(max_request_bytes))); - } - Ok(out) + .to_bytes()) +} + +/// The refusal of a call whose envelope is `size` bytes, past the +/// `max_request_bytes` the endpoint takes. +pub fn past_limit(endpoint: &str, max_request_bytes: u64, size: u64) -> CoreError { + CoreError::new( + Sentence::ServingPastLimit, + "the call is larger than the serving endpoint takes", + ) + .with_field("endpoint", Detail::text(endpoint)) + .with_field("bytes", count(size)) + .with_field("limit", count(max_request_bytes)) +} + +/// What the gateway's refusal of a call too large says, a 400 whose message +/// is this (`Request size cannot exceed 16MB`), matched case-insensitively. +const OVERSIZED: &str = "request size cannot exceed"; + +/// Whether a failure's text is the gateway's refusal of a call too large. +pub fn oversized(text: &str) -> bool { + text.to_ascii_lowercase().contains(OVERSIZED) } /// The served model's answer in the endpoint's reply `body` (a 2xx), a JSON @@ -203,7 +226,13 @@ pub fn failure(endpoint: &str, failed: &Failure<'_>) -> CoreError { rest_body(&detail, failed.status) }; let raw_code = raw_code.map(|code| canonical_code(&code)); - let status = reported_status(failed.status, raw_code.as_deref()); + // The gateway refuses a call too large as a 400 that says so, which the + // core asks again in halves as it does a 413. + let status = if failed.status == 400 && oversized(&text) { + 413 + } else { + reported_status(failed.status, raw_code.as_deref()) + }; let provider_code = raw_code.as_deref().filter(|c| code_shaped(c)); let code = model_diagnosis(&text, provider_code); if status >= 500 @@ -571,7 +600,9 @@ mod tests { #[test] fn a_status_is_reported_as_the_sdk_s_class_gives_it() { assert_eq!(reported_status(500, None), 503); - assert_eq!(reported_status(502, None), 503); + // Kept, so a call too large for the gateway is asked in halves. + assert_eq!(reported_status(502, None), 502); + assert_eq!(reported_status(413, None), 413); assert_eq!(reported_status(429, None), 429); assert_eq!(reported_status(400, Some("RESOURCE_DOES_NOT_EXIST")), 404); assert_eq!(reported_status(400, Some("BAD_REQUEST")), 400); diff --git a/rust/crates/sdm-request/src/tabular.rs b/rust/crates/sdm-request/src/tabular.rs index f05c38be9..8bcdddafb 100644 --- a/rust/crates/sdm-request/src/tabular.rs +++ b/rust/crates/sdm-request/src/tabular.rs @@ -1243,13 +1243,3 @@ pub fn decide_materialized_task( }, } } - -/// Select the cap refusal from a reached native comparison, without coercing -/// its original count owner or reading the candidate frame population. -pub fn check_materialized_cap( - input: sdm_contract::tabular::CheckTabularCap, -) -> sdm_contract::tabular::CheckedTabularCap { - sdm_contract::tabular::CheckedTabularCap { - accepted: !input.over_limit, - } -} diff --git a/rust/crates/sdm-request/tests/batch_limits.rs b/rust/crates/sdm-request/tests/batch_limits.rs index 4c32f6645..cbaa053b8 100644 --- a/rust/crates/sdm-request/tests/batch_limits.rs +++ b/rust/crates/sdm-request/tests/batch_limits.rs @@ -5,36 +5,16 @@ use sdm_request::options::{BatchTask, rows_one_call_scores, rows_one_task_call_scores}; #[test] -fn temporal_links_cannot_raise_the_model_limit() { - for (asked, expected) in [ - (None, 200), - (Some(100), 100), - (Some(199), 199), - (Some(200), 200), - (Some(201), 200), - (Some(1000), 200), - (Some(u64::MAX), 200), +fn each_task_batches_by_its_default_and_the_size_asked_is_taken_as_is() { + for (task, default) in [ + (BatchTask::TemporalLink, 200), + (BatchTask::Ordinary, 10_000), ] { - assert_eq!( - rows_one_task_call_scores(asked, BatchTask::TemporalLink), - expected - ); - } -} - -#[test] -fn ordinary_and_decomposed_tasks_keep_their_existing_limit() { - for (asked, expected) in [ - (None, 10_000), - (Some(100), 100), - (Some(201), 201), - (Some(1000), 1000), - (Some(u64::MAX), 10_000), - ] { - assert_eq!( - rows_one_task_call_scores(asked, BatchTask::Ordinary), - expected - ); - assert_eq!(rows_one_call_scores(asked), expected); + assert_eq!(rows_one_task_call_scores(None, task), default); + for asked in [1, 100, 20_000, u64::MAX] { + assert_eq!(rows_one_task_call_scores(Some(asked), task), asked); + } } + assert_eq!(rows_one_call_scores(None), 10_000); + assert_eq!(rows_one_call_scores(Some(50_000)), 50_000); } diff --git a/rust/crates/sdm-request/tests/payload_bytes.rs b/rust/crates/sdm-request/tests/payload_bytes.rs deleted file mode 100644 index 3b34a3390..000000000 --- a/rust/crates/sdm-request/tests/payload_bytes.rs +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! The payload limit counts the bytes the request sends (register PD-2, -//! ): the builder's UTF-8 bytes, as written, so a payload of exactly -//! the limit is taken and one byte more refused, where the reference -//! counted `json.dumps`' text, which writes each character past ASCII as a -//! six-byte escape. Held at the limit with ASCII text and with text of two-, -//! three- and four-byte characters, which the reference's count would -//! have taken for three to twelve times its size. - -use std::sync::Arc; - -use arrow_array::{ArrayRef, Int64Array, StringArray}; -use sdm_contract::ErrorCode; -use sdm_contract::ops::Stype; -use sdm_engine::labels::Labels; -use sdm_request::options::{Kind, Options, RunMode}; -use sdm_request::payload::{PAYLOAD_BYTES, Sampled, SampledColumn, SampledTable, build}; - -const ANCHOR: i64 = 1_709_251_200_000_000_000; - -/// One entity, one context example and one to predict, each carrying -/// `note` as a categorical text feature. -fn sampled(note: &str) -> Sampled { - let column = |a: ArrayRef| SampledColumn::Arrow(a); - Sampled { - num_train: 1, - num_test: 1, - anchors: vec![ANCHOR, ANCHOR], - kind: Kind::Regression, - labels: Labels::Float(vec![1.0]), - windowed: true, - tables: vec![SampledTable { - name: "users".into(), - primary_key: Some("user_id".into()), - batch: vec![0, 1], - columns: vec![ - ( - "user_id".into(), - column(Arc::new(Int64Array::from(vec![0, 1]))), - ), - ( - "note".into(), - column(Arc::new(StringArray::from(vec![note, note]))), - ), - ], - stypes: vec![("note".into(), Stype::Categorical)], - }], - links: vec![], - entity_table: "users".into(), - source_query: String::new(), - run_mode: RunMode::Fast, - use_prediction_time: false, - return_embeddings: false, - options: Options::default(), - statistic: None, - top_k: None, - rows_per_call: 1, - lags: vec![], - float_text: |x| format!("{x:?}"), - } -} - -/// The payload's length with `n` copies of `unit` in each note. -fn length(unit: &str, n: usize) -> Result { - build(&sampled(&unit.repeat(n))) - .map(|built| built.bytes.len()) - .map_err(|e| e.code) -} - -#[test] -fn the_limit_counts_the_bytes_sent_at_the_limit() { - for unit in ["a", "\u{e9}", "\u{20ac}", "\u{1f600}"] { - let width = unit.len(); - // Each note is written twice (the context's row and the predicted - // row), so the payload grows by twice a unit's bytes per unit. - let base = length(unit, 0).unwrap(); - let step = length(unit, 1).unwrap() - base; - assert_eq!(step, 2 * width, "{unit:?}"); - // The most units whose payload is within the limit, and the slack - // left, which ASCII padding of the bytes that remain fills. - let n = (PAYLOAD_BYTES - base) / step; - let slack = PAYLOAD_BYTES - base - n * step; - let at = format!("{}{}", unit.repeat(n), "a".repeat(slack / 2)); - let taken = build(&sampled(&at)).expect("a payload within the limit is taken"); - let exact = base + n * step + 2 * (slack / 2); - assert_eq!(taken.bytes.len(), exact, "{unit:?}"); - assert!(exact <= PAYLOAD_BYTES && PAYLOAD_BYTES - exact <= 1); - // The bytes counted are the bytes written: UTF-8, each character - // its own bytes, not an escape. - let text = String::from_utf8(taken.bytes.clone()).unwrap(); - assert!(!text.contains("\\u"), "{unit:?}"); - // Two units more (four bytes or more past it) are refused, as the - // reference refuses a payload past its limit. - let past = format!("{}{}", unit.repeat(n + 2), "a".repeat(slack / 2)); - let refused = build(&sampled(&past)).expect_err("a payload past the limit"); - assert_eq!(refused.code, ErrorCode::InvalidRequest, "{unit:?}"); - } - // With ASCII the payload reaches the limit exactly, one byte step past - // it refused: the count is of bytes, to the byte. - let base = length("a", 0).unwrap(); - let fill = |extra: usize| { - let half = (PAYLOAD_BYTES - base + extra) / 2; - build(&sampled(&"a".repeat(half))) - }; - let even = (PAYLOAD_BYTES - base).is_multiple_of(2); - let at = fill(0).expect("at the limit"); - assert_eq!( - at.bytes.len(), - if even { - PAYLOAD_BYTES - } else { - PAYLOAD_BYTES - 1 - } - ); - assert_eq!(fill(2).unwrap_err().code, ErrorCode::InvalidRequest); -} diff --git a/rust/crates/sdm-request/tests/payload_lock.rs b/rust/crates/sdm-request/tests/payload_lock.rs index 9be1743fa..9546fecbf 100644 --- a/rust/crates/sdm-request/tests/payload_lock.rs +++ b/rust/crates/sdm-request/tests/payload_lock.rs @@ -56,7 +56,6 @@ fn shop() -> Sampled { options: Options::default(), statistic: Some("mean".into()), top_k: None, - rows_per_call: 1, lags: vec![], float_text: |x| format!("{x:?}"), tables: vec![ @@ -140,7 +139,6 @@ fn family() -> Sampled { options: Options::default(), statistic: None, top_k: None, - rows_per_call: 1, lags: vec![], float_text: |x| format!("{x:?}"), tables: vec![ @@ -191,7 +189,6 @@ fn classified(kind: Kind, labels: Labels) -> Sampled { windowed: false, statistic: None, top_k: None, - rows_per_call: 1, lags: vec![], ..shop() } diff --git a/rust/crates/sdm-request/tests/serving.rs b/rust/crates/sdm-request/tests/serving.rs index 6f86a0013..c4795efad 100644 --- a/rust/crates/sdm-request/tests/serving.rs +++ b/rust/crates/sdm-request/tests/serving.rs @@ -696,7 +696,7 @@ fn an_http_failure_reads_as_the_sdk_reads_the_same_response() { ( 502, "Bad Gateway".to_owned(), - (Sentence::ServingFailed, 503, None), + (Sentence::ServingFailed, 502, None), ), ( 500, @@ -735,7 +735,7 @@ fn an_http_failure_reads_as_the_sdk_reads_the_same_response() { ]; for (status, body, reference) in cases { let expected = if status == 502 && body.starts_with("") { - (Sentence::ServingFailedCoded, 503, Some("BAD_GATEWAY")) + (Sentence::ServingFailedCoded, 502, Some("BAD_GATEWAY")) } else { reference }; @@ -758,3 +758,25 @@ fn an_invalid_status_is_a_malformed_response_not_a_clamped_status() { assert_eq!(error.code, ErrorCode::InvalidResponse); assert_eq!(error.fields["reason"], text("invalid_serving_status")); } + +/// The gateway refuses a call too large as a 400 saying so: read as a 413, +/// so the core asks the call's rows in halves, whichever host carried it. +#[test] +fn the_gateway_s_oversize_400_is_read_as_a_413() { + for body in [ + r#"{"error_code":"BAD_REQUEST","message":"Request size cannot exceed 16MB"}"#, + "BAD_REQUEST: Request size cannot exceed 16777216 bytes", + ] { + let error = rest(400, body); + assert_eq!( + error.fields[REFUSED_STATUS], + Detail::Scalar(Scalar::UInt16(413)), + "{body}" + ); + } + let other = rest(400, r#"{"error_code":"BAD_REQUEST","message":"bad row"}"#); + assert_eq!( + other.fields[REFUSED_STATUS], + Detail::Scalar(Scalar::UInt16(400)) + ); +} diff --git a/rust/crates/sdm-testkit/src/host.rs b/rust/crates/sdm-testkit/src/host.rs index 722c21124..72afe4f7d 100644 --- a/rust/crates/sdm-testkit/src/host.rs +++ b/rust/crates/sdm-testkit/src/host.rs @@ -1794,8 +1794,6 @@ fn start_materialize( values: Some(AttachmentId(6)), options: vec![], most_parts: None, - most_entities: None, - most_calls: None, sampler_state: sampler .as_ref() .and_then(|held| held.as_ref()) diff --git a/rust/crates/sdm-testkit/tests/payload_rules.rs b/rust/crates/sdm-testkit/tests/payload_rules.rs index cc90a3b3f..3dfa5b642 100644 --- a/rust/crates/sdm-testkit/tests/payload_rules.rs +++ b/rust/crates/sdm-testkit/tests/payload_rules.rs @@ -190,12 +190,12 @@ fn base_multiclass_s_classification_payload_and_leakage_are_the_reference_s() { } #[test] -fn more_entities_than_one_materialize_holds_are_refused_by_its_limit() { - // The entity count, on base.regression. the core takes up to - // 100,000 entities, in calls of at most 10,000, as the reference at - // 6298913c (main's 1fc5a686): 1,001 entities are refused INVALID_REQUEST - // for the reference's reason (the base holds 8 users), never by a limit - // (B-3, retired). +fn no_entity_count_is_refused_by_a_limit() { + // The entity count, on base.regression: the core holds a materialize + // to no count, gathering as many calls as the entities need. 1,001 + // entities are refused INVALID_REQUEST for the reference's reason (the + // base holds 8 users), never by a limit (B-3, retired), and so are + // 100,001. let want = expected("base.regression.entities-1001"); assert_eq!(want["outcome"], "refused"); assert_eq!(want["code"], "INVALID_REQUEST"); @@ -211,26 +211,12 @@ fn more_entities_than_one_materialize_holds_are_refused_by_its_limit() { .collect(); host::run(&case, &bases).unwrap() }; - let observed = with(1001); - let Answer::Error { code, fields, .. } = &observed.answer else { - panic!("not refused: {:?}", observed.answer); - }; - assert_eq!(code, "INVALID_REQUEST"); - assert_eq!(fields.get("field").map(String::as_str), Some("indices")); - // Past 100,000 the core's own limit refuses them before anything is - // read, naming it (`most_entities`), not a rule it does not hold yet. - let observed = with(100_001); - assert!( - observed.materialized.is_none(), - "100,001 entities were taken" - ); - let Answer::Error { code, fields, .. } = &observed.answer else { - panic!("not refused: {:?}", observed.answer); - }; - assert_eq!(code, "UNSUPPORTED_REQUEST"); - assert_eq!( - fields.get("rule").map(String::as_str), - Some("most_entities") - ); - assert_eq!(fields.get("field").map(String::as_str), Some("indices")); + for n in [1001, 100_001] { + let observed = with(n); + let Answer::Error { code, fields, .. } = &observed.answer else { + panic!("not refused: {:?}", observed.answer); + }; + assert_eq!(code, "INVALID_REQUEST", "{n}"); + assert_eq!(fields.get("field").map(String::as_str), Some("indices")); + } } diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 1ab1a346e..ef34daace 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -152,38 +152,30 @@ views, and a notebook's own session are in ## Which rows get scored -`materialize(indices=[...])` names the entities. It gathers each one's -neighbourhood, checks every limit before sending, and refuses with the limit it -would break rather than letting the endpoint return a 500. +`materialize(indices=[...])` names the entities and gathers each one's +neighbourhood. The client holds a request to no size of its own: it sends its +best attempt, and a call the endpoint refuses as too large, or times out on, +is asked again in halves, each with the whole context. A refusal of one row, +or a `422`, is the endpoint's, shown as it said it. | Limit | Value | |---|---| | context rows, by default | `1,000` | -| context rows, with `num_context_samples` | `10,000` | -| rows scored at once | `10,000` | -| rows scored for a link prediction | `200` | -| tables in the gathered graph | `15` | -| hops gathered | `6` | | classes | `10` | -**More rows than one call scores are not refused.** They are gathered in -batches of at most that many, and `predict()` sends one call per batch and -returns one result, whose rows are numbered across the whole request, so -`to_pandas()` gives the indices back in the order they were asked for. The -10,000 is the deployment's `TFM_MAX_TEST_ROWS`, not a property of the -protocol; pass `rows_per_call=` to ask in smaller batches, which is what a -graph too wide for a full one needs. It only goes down. - -Every batch is built before the first is sent, so one `materialize()` gathers -at most 100,000 entities, in at most 100 calls. Both are this client's own: -the endpoint is never asked for more than one call holds. A question asked -in parts is not batched at all: a ranking is already a call per candidate and +**Rows are scored in batches.** They are gathered in batches of 10,000 (200 +for a link prediction), and `predict()` sends one call per batch and returns +one result, whose rows are numbered across the whole request, so `to_pandas()` +gives the indices back in the order they were asked for. Pass +`rows_per_call=` to ask in batches of another size. + +Every batch is built before the first is sent, each with its own context, so +the whole population is held in memory at once; the client sets no most on +it. A question asked in parts is not batched at all: a ranking is already a call per candidate and a forecast one per step, and batching a part would send that many times as many calls. -**The table limit is on the graph a request carries, not on the schema.** A -schema may describe far more than fifteen tables; what counts is how many the -query actually reaches. +The gathered graph may carry any number of tables and hops. Gradient explanations follow the model endpoint's supported outputs; the client does not impose a separate one-row or fast-context limit. ## Asking for more than one answer each @@ -271,8 +263,6 @@ change. The ones worth knowing: |---|---| | every candidate's context holds one class | each gathered entity reached it, or none did, so there is nothing to tell apart and nothing is sent; widen with `num_context_samples` or ask over a table of pairs. One such candidate among several is dropped with a warning instead, and named on `result.unscored_candidates` | | no entity existed at the anchor | the entity table's time column starts after the anchor, so nothing existed to ask about | -| more than 100,000 entities at once | every batch is built before any is sent; ask about fewer at a time | -| more than 100 calls at once | a context is gathered per call; ask about fewer entities, or raise `rows_per_call` if you lowered it | | more entities than one call scores, on a ranking or a forecast | each candidate or step is already a call, and batching one would send that many times as many; ask about fewer at a time | | an output this question cannot give | `rankings` on a regression, `explanation` on a client-assembled ranking | | `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation` | diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 93f5f2d5b..e9d284042 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -34,7 +34,6 @@ with SDMClient(url='http://localhost:8000') as client: | Relationships | no | | Explanations | yes | | Sessions | yes | -| Context rows | 10,000 | Anything outside that list belongs to the relational model. Asking for `rankings` or a forecast here is refused before a request @@ -110,8 +109,9 @@ guesses that way and names the argument that settles it. ## Which rows get scored `Request(context=..., predict=...)` takes two frames and sends exactly those -rows. It trims neither. A context above 10,000 rows is refused with the count -and the cap rather than quietly dropping rows you named. +rows. It trims neither, and holds the context to no size of its own: a +context larger than the endpoint takes is the endpoint's refusal, shown as it +said it. The rows to score need not appear in the labelled half, and usually should not: they are the rows whose answer you do not have. @@ -157,8 +157,8 @@ print(answer.to_pandas()) ``` Twenty rows in, three named, and `request.context_rows` is 17: the scored rows -are held out of their own context. `materialize()` also samples down to the -10,000-row cap. +are held out of their own context. `materialize()` samples the context only +to a `num_context_samples` you pass. On this model the language is cut down to a column of one table as the target: no aggregation, no window, no comparison, no forecast, no ranking, no entity @@ -215,7 +215,6 @@ names what to change. |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | | an output it does not produce | `rankings`, `embeddings`, `scores` | -| a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | Catch `StructuredDataError` to catch all of them. diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index b795ebb9a..fdc3e15b9 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -3338,7 +3338,8 @@ "packages/structured-data-models-client/tests/test_databricks_serving.py::test_the_ambient_identity_is_used_when_no_client_is_given", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_a_workspace_that_cannot_authenticate_says_how_to_fix_it", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_unusable_timeout_is_refused_at_construction", - "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not" + "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not", + "packages/structured-data-models-client/tests/test_databricks_splitting.py" ], "java": [ "java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java#officialSdkAuthenticationRefreshesEveryHeaderWithoutRewritingTheCoreBytes", @@ -4748,7 +4749,7 @@ } } ], - "public_api_snapshot_sha256": "652bd2757ca1d8ed12360b795a5c7f80ce63978210306cee1319532503f9d031", + "public_api_snapshot_sha256": "5f1f001f45522d17f2a3527668d3313d5cd0d439ba7d21acedfdddf5b073a6d0", "api_inventory": { "python": { "sdm_api.Endpoint": { @@ -5262,7 +5263,17 @@ "capability": "sessions" }, "sdm_client.SDMClient.for_databricks_serving": { - "capability": "public-serving-refusals" + "capability": "public-serving-refusals", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_databricks_splitting.py::test_max_request_bytes_is_passed_through", + "behavior": "max_request_bytes reaches the Databricks transport." + }, + { + "test": "packages/structured-data-models-client/tests/test_databricks_splitting.py::test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent", + "behavior": "A call past max_request_bytes is split before it is sent, each part within it." + } + ] }, "sdm_client.SDMClient.health_ready": { "capability": "readiness-discovery" @@ -8233,6 +8244,21 @@ "base_commit": "ebc5746391d977fd8d9d779423db33ae547d64ee", "record": "type_contracts/arrow-admission-and-inference", "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "public_api_snapshot_sha256", + "reason": "SDMClient.for_databricks_serving takes an optional max_request_bytes, passed to the Databricks transport, whose calls past it are split before they are sent." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_inventory", + "reason": "SDMClient.for_databricks_serving gains max_request_bytes, asserted by the Databricks splitting tests." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/public-serving-refusals", + "reason": "Adds the Databricks splitting tests, which assert max_request_bytes and the split before a call is sent." } ], "capabilities": [ diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 5c81e999e..8908ca3ba 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -476,7 +476,7 @@ "capabilities": "(self, model: 'str | Model') -> 'ModelCapabilities'", "close": "(self) -> 'None'", "fit": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None, holdout: 'Iterable[Any] | None' = None) -> 'FittedModel'", - "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None) -> 'SDMClient'", + "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None, max_request_bytes: 'int | None' = None) -> 'SDMClient'", "health_ready": "(self) -> 'bool'", "in_process": "(*models: 'Any', device: 'Any' = None) -> 'SDMClient'", "models": "(self) -> 'list[str]'", diff --git a/tests/test_skills.py b/tests/test_skills.py index 4e9695f35..19a45436b 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2231,10 +2231,7 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'Sessions': capabilities.sessions, } if name == 'tabular': - from sdm_client.models import TABULAR_LIMITS - said['Explanations'] = 'explanation' in capabilities.outputs - said['Context rows'] = TABULAR_LIMITS['context_rows_max'] assert set(shown) == set(said), ( f'{name} shows rows nothing checks: {sorted(set(shown) ^ set(said))}' ) @@ -2264,13 +2261,6 @@ def test_the_capability_table_is_the_clients_own(name, model_name): #: and a row naming no limit is a number nobody is holding to anything. LIMIT_ROWS = { 'context rows, by default': 'context_rows_fast', - 'context rows, with `num_context_samples`': 'context_rows_max', - 'rows scored at once': 'predict_rows_max', - 'rows scored for a link prediction': ( - 'predict_rows_temporal_link_prediction' - ), - 'tables in the gathered graph': 'tables_max', - 'hops gathered': 'hops_max', 'classes': 'classes_max', }