From aea4372dcad1a71f95d427581866c9172dbff9b8 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Fri, 6 Nov 2026 21:57:48 +0000 Subject: [PATCH 1/8] Make the client's size limits configurable and split calls by bytes A deployment can raise the serving layer's limits (TFM_MAX_TRAIN_ROWS, TFM_MAX_TEST_ROWS, TFM_MAX_PAYLOAD_BYTES), so the client's copies of them become settings with today's values as defaults, defined once in Rust: - max_relational_context_rows (10,000), max_relational_predict_rows (10,000), max_temporal_link_predict_rows (200), max_tabular_context_rows (10,000) and max_request_bytes (30 MiB), each read from an argument, else SDM_CLIENT_, else the default. The core resolves them on every run start (contract v72: StartRun.limits, resolve_limits), so Python and Java share one rule and one table (native LIMITS / info "limits"). - Python: sdm_client.Limits, SDMClient(limits=), in_process(limits=), for_databricks_serving(limits=), materialize(limits=). Java: Limits, ClientOptions.limits, DatabricksServing.limits, MaterializeOptions.limits, SdmClient.limits(). - A refusal at a limit names the setting and its environment variable. - The instance-table cap is the relational context setting. - The 15-table and 6-hop caps of an earlier endpoint are removed; hops past the mode's six sizes keep its last, and a walk ends at the first hop that reaches no row, so any count of at least one is safe. - A predicted call past max_request_bytes, or past a Databricks serving endpoint's 16 MB once wrapped, is split into smaller calls by its rows, each with the whole context; only a context too large on its own is refused, naming its size and the limit. materialize() makes the same check before anything is sent. Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 9 + contract/expected/README.md | 3 + contract/expected/duckdb-python.json | 4 +- contract/expected/postgres-python.json | 4 +- contract/expected/sqlite-python.json | 4 +- contract/support/error_wording_scenarios.py | 2 - contract/tests/test_prediction_batching.py | 22 +- contract/tests/test_public_error_messages.py | 2 - docs/reference/limits.md | 191 ++++----- .../java/com/nvidia/sdm/ClientOptions.java | 21 +- .../com/nvidia/sdm/DatabricksServing.java | 24 +- java/src/main/java/com/nvidia/sdm/Limits.java | 129 ++++++ .../com/nvidia/sdm/MaterializeOptions.java | 16 + .../java/com/nvidia/sdm/PredictiveQuery.java | 9 +- .../nvidia/sdm/RelationalMaterializer.java | 3 +- .../main/java/com/nvidia/sdm/SdmClient.java | 23 +- .../com/nvidia/sdm/internal/ArrowTabular.java | 11 +- .../java/com/nvidia/sdm/internal/Host.java | 40 +- .../nvidia/sdm/internal/LimitSettings.java | 62 +++ .../java/com/nvidia/sdm/internal/Wording.java | 21 +- .../com/nvidia/sdm/DatabricksServingTest.java | 15 +- .../test/java/com/nvidia/sdm/LimitsTest.java | 66 +++ .../src/sdm_client/__init__.py | 2 + .../src/sdm_client/_native.pyi | 4 + .../src/sdm_client/_rust/errors.py | 2 +- .../src/sdm_client/_rust/run.py | 2 + .../sdm_client/_rust/tabular_materialize.py | 12 +- .../src/sdm_client/client.py | 76 +++- .../src/sdm_client/limits.py | 185 +++++++++ .../src/sdm_client/models.py | 40 +- .../src/sdm_client/query.py | 37 +- .../tests/tabular_pql_count_cases.py | 6 +- .../test_asked_in_the_questions_it_serves.py | 5 +- .../tests/test_configurable_limits.py | 238 +++++++++++ .../tests/test_context_samples.py | 5 +- .../tests/test_limits_and_absent_outputs.py | 18 +- .../tests/test_limits_and_sessions.py | 22 +- .../tests/test_live_relational.py | 6 +- .../tests/test_materialize_options.py | 2 +- .../tests/test_materialize_state.py | 4 +- .../tests/test_payload_rules.py | 50 ++- .../tests/test_predict_result.py | 12 +- .../tests/test_query_engine_boundary.py | 18 +- .../tests/test_serving_route.py | 33 +- .../tests/test_the_predict_route.py | 41 +- .../tests/test_wording.py | 6 +- rust/bindings/sdm-jni/src/lib.rs | 11 + rust/bindings/sdm-py/src/lib.rs | 13 + rust/crates/sdm-contract/schema/contract.lock | 2 +- ...-contract-71.json => sdm-contract-72.json} | 118 +++++- rust/crates/sdm-contract/src/error.rs | 7 +- rust/crates/sdm-contract/src/lib.rs | 3 +- rust/crates/sdm-contract/src/limits.rs | 224 +++++++++++ rust/crates/sdm-contract/src/ops.rs | 11 +- rust/crates/sdm-contract/src/run.rs | 10 + rust/crates/sdm-contract/src/wording.rs | 45 +-- rust/crates/sdm-contract/tests/corpus/mod.rs | 1 + rust/crates/sdm-contract/tests/host_facts.rs | 1 + .../tests/provider_view_contract.rs | 9 +- rust/crates/sdm-contract/tests/schema_lock.rs | 5 + rust/crates/sdm-core/src/direct.rs | 7 +- rust/crates/sdm-core/src/dispatch.rs | 34 +- rust/crates/sdm-core/src/frame_session.rs | 16 +- rust/crates/sdm-core/src/lag.rs | 30 +- rust/crates/sdm-core/src/lib.rs | 1 + rust/crates/sdm-core/src/limits.rs | 172 ++++++++ rust/crates/sdm-core/src/materialize.rs | 65 +-- .../sdm-core/src/materialize/examples.rs | 36 +- .../crates/sdm-core/src/materialize/frames.rs | 9 +- rust/crates/sdm-core/src/predict.rs | 125 +++++- rust/crates/sdm-core/src/ranking.rs | 8 +- rust/crates/sdm-core/src/session.rs | 3 +- rust/crates/sdm-core/src/subgraph.rs | 50 ++- rust/crates/sdm-core/src/tabular_arrow.rs | 84 ++-- rust/crates/sdm-core/src/tabular_selection.rs | 23 +- rust/crates/sdm-core/src/test_ops.rs | 2 + rust/crates/sdm-core/tests/answer_encoding.rs | 1 + rust/crates/sdm-core/tests/attachments.rs | 1 + rust/crates/sdm-core/tests/dispatch.rs | 80 ++++ rust/crates/sdm-core/tests/edit_schema.rs | 1 + .../sdm-core/tests/frame_arrow_schema.rs | 1 + rust/crates/sdm-core/tests/held.rs | 1 + rust/crates/sdm-core/tests/held_schema.rs | 1 + rust/crates/sdm-core/tests/materialize.rs | 11 +- rust/crates/sdm-core/tests/multi_call.rs | 1 + rust/crates/sdm-core/tests/predict.rs | 28 +- rust/crates/sdm-core/tests/serving.rs | 36 +- .../crates/sdm-core/tests/tabular_sessions.rs | 24 +- rust/crates/sdm-core/tests/view_captures.rs | 2 + rust/crates/sdm-core/tests/view_semantic.rs | 1 + rust/crates/sdm-engine/src/frames.rs | 7 + rust/crates/sdm-host/tests/codec.rs | 1 + rust/crates/sdm-host/tests/contract_v3.rs | 1 + rust/crates/sdm-host/tests/data_port.rs | 1 + rust/crates/sdm-host/tests/error_table.rs | 1 + rust/crates/sdm-host/tests/faults.rs | 2 + rust/crates/sdm-host/tests/runs.rs | 1 + rust/crates/sdm-host/tests/type_set.rs | 1 + rust/crates/sdm-request/src/batching.rs | 299 ++++++++++++++ rust/crates/sdm-request/src/direct.rs | 9 +- rust/crates/sdm-request/src/lib.rs | 1 + rust/crates/sdm-request/src/options.rs | 212 ++++++---- rust/crates/sdm-request/src/payload.rs | 99 ++--- rust/crates/sdm-request/src/predict.rs | 110 +++-- rust/crates/sdm-request/src/serving.rs | 59 ++- rust/crates/sdm-request/tests/batch_limits.rs | 34 +- .../crates/sdm-request/tests/payload_bytes.rs | 49 ++- rust/crates/sdm-request/tests/payload_lock.rs | 3 - rust/crates/sdm-testkit/src/host.rs | 5 + .../src/host_replay_validation_tests.rs | 1 + .../sdm-testkit/tests/duckdb_provider.rs | 2 + rust/crates/sdm-testkit/tests/neighbors.rs | 1 + .../sdm-testkit/tests/warehouse_replay.rs | 1 + skills/relational/SKILL.md | 13 +- skills/tabular/SKILL.md | 5 +- tests/compatibility/manifest.json | 376 +++++++++++++++++- tests/compatibility/test_contract.py | 1 + tests/golden/public_api.json | 25 +- tests/test_skills.py | 42 +- 119 files changed, 3437 insertions(+), 746 deletions(-) create mode 100644 java/src/main/java/com/nvidia/sdm/Limits.java create mode 100644 java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java create mode 100644 java/src/test/java/com/nvidia/sdm/LimitsTest.java create mode 100644 packages/structured-data-models-client/src/sdm_client/limits.py create mode 100644 packages/structured-data-models-client/tests/test_configurable_limits.py rename rust/crates/sdm-contract/schema/{sdm-contract-71.json => sdm-contract-72.json} (99%) create mode 100644 rust/crates/sdm-contract/src/limits.rs create mode 100644 rust/crates/sdm-core/src/limits.rs create mode 100644 rust/crates/sdm-request/src/batching.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f651b43ea..00443b97e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ The public API snapshot records this version update; publication remains pending ### Added +- The size limits the client holds a request to are settings rather than constants, for a deployment configured to take more than the serving layer's defaults. + Python `Limits` and Java `Limits` name each one (`max_relational_context_rows`, `max_relational_predict_rows`, `max_temporal_link_predict_rows`, `max_tabular_context_rows`, `max_request_bytes`), taken by `SDMClient(limits=...)`, `materialize(limits=...)`, Java `ClientOptions`, `DatabricksServing` and `MaterializeOptions`. + A setting not given takes its environment variable (`SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS` and the like), else today's default. + A refusal at one of them names the setting and its environment variable. +- A call past the most bytes one call takes, or past a Databricks serving endpoint's 16 MB once wrapped, is split into smaller calls, each with the whole context, rather than refused; only a context too large on its own is refused, naming its size and the limit. - Python `Schema.from_relbench` and Java `Schema.fromRelbench` load the same pinned RelBench datasets with verified caches, offline mode and declared schema metadata. - Python and Java requests and fitted handles expose immutable sanitization reports for in-memory relational preparation, with per-table row exclusions and explicit unavailable/not-applicable states. @@ -64,6 +69,10 @@ The public API snapshot records this version update; publication remains pending Endpoint capabilities and ordinary request limits still apply. The singular raw `result.explanation` view remains supported. +### Removed + +- The relational limits of 15 tables in a request's graph and 6 hops, which an earlier endpoint enforced and the current model does not: a request may carry any number of tables and hops of at least one. + ### Fixed - Explicit empty outputs remain invalid with `EXPLAIN`; unsupported local-runtime outputs are refused before model weights load. diff --git a/contract/expected/README.md b/contract/expected/README.md index c3422e18f..576cac550 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -29,6 +29,9 @@ Their differences from DuckDB follow the fixture loader's explicit mappings in ` | SQLite / Postgres | Malformed text-time values are refused; the US-date fixture completes under these backends' existing read behavior | | Postgres | NaN-key fixtures refuse at materialize; a negative neighbor bound retains the existing `PROVIDER_ERROR` outcome | +`sample.num-hops.seven` and `sample.num-neighbors.seven-hops` now reach predict on every backend: the client sets no most on hops, since the model reads as many as a request carries, where the reference refused past six. +The `multicall.body-30mib` cases keep their refusal at materialize, now naming the context's size and the `max_request_bytes` setting, since a call past the limit is split and only a context too large on its own is refused. + These expectations describe the declared fixture mappings; they do not claim that every backend supports identical physical types. `explain.two-entities` now succeeds on all three backends: The client supports explanations for multiple prediction rows. `contract/tests/test_query_options.py::test_each_explained_entity_retains_its_own_raw_response` independently checks both rows' distinct explanation values, the request batch size and the public success outcome. diff --git a/contract/expected/duckdb-python.json b/contract/expected/duckdb-python.json index 2a5376ff1..3a6ebf933 100644 --- a/contract/expected/duckdb-python.json +++ b/contract/expected/duckdb-python.json @@ -247,7 +247,7 @@ "sample.num-hops-3": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -256,7 +256,7 @@ "sample.num-neighbors.minus-one": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.number": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/postgres-python.json b/contract/expected/postgres-python.json index de5fe0707..cd2200315 100644 --- a/contract/expected/postgres-python.json +++ b/contract/expected/postgres-python.json @@ -245,7 +245,7 @@ "sample.num-hops-3@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -254,7 +254,7 @@ "sample.num-neighbors.minus-one@postgres": {"code": "PROVIDER_ERROR", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.number@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.relbench-avito@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/sqlite-python.json b/contract/expected/sqlite-python.json index 209fbf7ab..e186dd9ee 100644 --- a/contract/expected/sqlite-python.json +++ b/contract/expected/sqlite-python.json @@ -229,7 +229,7 @@ "sample.num-hops-3@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.bool@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.fraction@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "sample.num-hops.seven@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-hops.seven@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-hops.text@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-hops.zero@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors-2-2@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -238,7 +238,7 @@ "sample.num-neighbors.minus-one@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.number@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "sample.num-neighbors.over-num-hops@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "sample.num-neighbors.seven-hops@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, + "sample.num-neighbors.seven-hops@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.num-neighbors.zero@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.decimal-keys@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "sample.parent-table.relbench-avito@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/support/error_wording_scenarios.py b/contract/support/error_wording_scenarios.py index b0b407189..67e5ff548 100644 --- a/contract/support/error_wording_scenarios.py +++ b/contract/support/error_wording_scenarios.py @@ -68,12 +68,10 @@ 'sample.max-pq-iterations.zero', 'sample.num-hops.bool', 'sample.num-hops.fraction', - 'sample.num-hops.seven', 'sample.num-hops.text', 'sample.num-hops.zero', 'sample.num-neighbors.empty', 'sample.num-neighbors.number', - 'sample.num-neighbors.seven-hops', 'sample.seed.negative', 'shape.star.customers', 'target.filter.sum', diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 7105edb25..6b60579a7 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -680,14 +680,22 @@ def check() -> None: f'{case_key}: details[{name!r}] is {details.get(name)!r} ' f'where python-ref has {value!r}' ) - assert got['remedy'] == want['remedy'], ( - f'{case_key}: the remedy is {got["remedy"]!r:.200} where ' - f'python-ref names {want["remedy"]!r:.200}' - ) if case_id in BODY: - assert got['opening'] == want['opening'], ( - f'{case_key}: {got["opening"]!r:.200} where python-ref ' - f'opens {want["opening"]!r}' + # Both contexts are past the 30 MiB on their own. This client + # splits a call past the limit into smaller ones, so it names + # what does not split, the context, its size and the limit, + # where the reference named a smaller call or a narrower context. + assert want['remedy'].startswith(BODY[case_id]) + assert (got['opening'] or '').startswith( + '[INVALID_REQUEST] the context alone is too large to send: a ' + 'call carrying it ' + 'and one row to score is ' + ), f'{case_key}: {got["opening"]!r:.300}' + assert got['remedy'] is None + else: + assert got['remedy'] == want['remedy'], ( + f'{case_key}: the remedy is {got["remedy"]!r:.200} where ' + f'python-ref names {want["remedy"]!r:.200}' ) if case_id in INVALID: assert 'rows_per_call' in (got['opening'] or ''), ( diff --git a/contract/tests/test_public_error_messages.py b/contract/tests/test_public_error_messages.py index c63b65eda..28869d3dc 100644 --- a/contract/tests/test_public_error_messages.py +++ b/contract/tests/test_public_error_messages.py @@ -111,12 +111,10 @@ def runner() -> ModuleType: 'sample.max-pq-iterations.zero': ['max_pq_iterations', '0'], 'sample.num-hops.bool': ['num_hops', 'True'], 'sample.num-hops.fraction': ['num_hops', '1.5'], - 'sample.num-hops.seven': ['7 hops', '6'], 'sample.num-hops.text': ['num_hops', "'2'"], 'sample.num-hops.zero': ['num_hops', 'num_neighbors=[8, 8]'], 'sample.num-neighbors.empty': ['num_neighbors', 'no hop'], 'sample.num-neighbors.number': ['num_neighbors', 'sequence'], - 'sample.num-neighbors.seven-hops': ['7 hops', '6'], 'sample.seed.negative': ['random_seed', '-1'], 'wording.rank-text-candidate': ['buys.item_id', "'1'"], 'wording.float32-classification': ['non-finite', 'inf'], diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 4424d8c2d..11c8f6492 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -1,18 +1,17 @@ # Limits -The endpoint enforces several size limits. `materialize()` checks them before -sending, then refuses or samples while naming the limit. Otherwise, a rejected -request returns a 500 or 422 that names neither the limit nor how to meet it. +The endpoint enforces several size limits. +`materialize()` checks them before sending, then refuses, samples or splits while naming the limit. +Otherwise, a rejected request returns a 500 or 422 that names neither the limit nor how to meet it. -`Request(context=..., predict=...)` receives exact frames and does not trim -them. It refuses a context above the model's cap instead of discarding named -rows, and reports the count and cap. +`Request(context=..., predict=...)` receives exact frames and does not trim them. +It refuses a context above the model's cap instead of discarding named rows, and reports the count and cap. -`materialize()` keeps the context under each model's cap. On `kumo-tabular`, it -samples to the cap and reports the row count. On `kumo-relational`, it gathers -the context. It refuses `num_context_samples` above the cap. Omitting the -argument gathers and reports the 1,000-row budget; it never samples down to -10,000. +`materialize()` keeps the context under each model's cap. +On `kumo-tabular`, it samples to the cap and reports the row count. +On `kumo-relational`, it gathers the context. +It refuses `num_context_samples` above the cap. +Omitting the argument gathers and reports the 1,000-row budget; it never samples down to 10,000. ## Sizes @@ -25,101 +24,103 @@ Both endpoints enforce sizes, and most of these are the relational one's. | rows scored per call | relational | 10,000, or 200 for link prediction | more are gathered as several calls at `materialize()` | | entities one question gathers | relational | 100,000, in at most 100 calls | at `materialize()`, and this client's own: the endpoint is never asked for more than one call's worth | | calls a question asked in parts makes | relational | 200, one per candidate or step | at `materialize()`, as `sdm_client.decompose.MOST_PARTS` | -| request body | relational | 30 MiB per call | at `materialize()` | -| request body | tabular or relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | before sending, on the wrapped envelope | +| request body | tabular or relational | 30 MiB per call | a call past it is split into smaller calls when it is predicted; a context past it on its own is refused at `materialize()` | +| request body | tabular or relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | on the wrapped envelope, split as above | | gradient explanations | relational and tabular | endpoint-dependent; no separate client row/context limit | endpoint validates support | -| hops | relational | 6 | at `materialize()` | -| tables | relational | 15 | against the graph the request carries, not the schema it was built from | -A client opened with `SDMClient.in_process()` is held to this table as well, -since this client checks it. A NIM also has limits of its own, which each -deployment sets, such as the longest string a cell may hold or the most -distinct values a column may have. Those protect a shared server, and are -not applied when the model runs in your own process. +The relational model reads as many tables and hops as a request carries. +This client sets no most on either: a schema may describe any number of tables, and `num_hops` or `num_neighbors` may ask for any number of hops of at least one. +A walk that reaches no new row ends there, so a hop count past the data's depth costs nothing. + +A client opened with `SDMClient.in_process()` is held to this table as well, since this client checks it. +A NIM also has limits of its own, which each deployment sets, such as the longest string a cell may hold or the most distinct values a column may have. +Those protect a shared server, and are not applied when the model runs in your own process. + +## Settings + +The context rows, the rows scored per call and the request body are the serving layer's defaults (`TFM_MAX_TRAIN_ROWS`, `TFM_MAX_TEST_ROWS` and `TFM_MAX_PAYLOAD_BYTES`), and a deployment may be configured to take more or less. +Each is a setting of this client with that default: + +| Setting | Environment variable | Default | What it holds | +|---|---|---|---| +| `max_relational_context_rows` | `SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS` | 10,000 | the most context rows a relational request carries; `num_context_samples` asks for at most this many | +| `max_relational_predict_rows` | `SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS` | 10,000 | the most rows one relational call scores; more are asked in several calls | +| `max_temporal_link_predict_rows` | `SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS` | 200 | the most rows one temporal link prediction call scores | +| `max_tabular_context_rows` | `SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS` | 10,000 | the most context rows a single-table request carries; `materialize()` samples down to it | +| `max_request_bytes` | `SDM_CLIENT_MAX_REQUEST_BYTES` | 31,457,280 (30 MiB) | the most bytes one call's body takes | + +Set one with an argument or with its environment variable. +An argument comes before the environment variable, which comes before the default. +In Python, pass `limits=` to the client and to `materialize()`: + +```python +from sdm_client import Limits, SDMClient + +limits = Limits(max_tabular_context_rows=200_000, max_request_bytes=512 * 1024 * 1024) +client = SDMClient(url, limits=limits) +request = query.materialize(indices=keys, limits=limits) +result = client.predict(request, model=KumoTabular()) +``` + +`client.limits.resolved()` reports what each setting comes to. +In Java, pass `Limits.builder()...build()` to `ClientOptions.builder().limits(...)`, `DatabricksServing.limits(...)` and `MaterializeOptions.builder().limits(...)`. +Both read the same environment variables, and an environment variable that is not a whole number of at least 1 is refused with code `INVALID_CONFIGURATION`, naming it. +A refusal at one of these limits names the setting and its environment variable. + +Raise a setting only to what the deployment is configured to take. +This client then holds requests to the raised limit, and the endpoint answers them; a setting above the deployment's limit only moves the refusal to the endpoint. + +## Batches A relational request for more entities than one call scores is not refused. -`materialize()` gathers it in batches of at most that many, each with a -context of its own, sampled around that batch's rows rather than copied from -the first, and `predict()` sends one call per batch and returns one result. -Its `row_index` counts across the whole request rather than from zero per -call, so `to_pandas()`, which orders by it, gives the indices back in the -order they were asked for. A session scores those batches against the context -it pinned, so the context is uploaded once rather than once per call. `fit()` -itself takes a request of one call, since a session pins one context: fit a -small request and score the batched one against the handle. - -**The 10,000 is the deployment's, not this client's.** It is the serving -layer's `TFM_MAX_TEST_ROWS`, whose default it is, so a deployment may hold to -less. Measured at the boundary rather than read from a document: 10,000 rows -to score are answered and 10,001 come back as +`materialize()` gathers it in batches of at most that many, each with a context of its own, sampled around that batch's rows rather than copied from the first, and `predict()` sends one call per batch and returns one result. +Its `row_index` counts across the whole request rather than from zero per call, so `to_pandas()`, which orders by it, gives the indices back in the order they were asked for. +A session scores those batches against the context it pinned, so the context is uploaded once rather than once per call. +`fit()` itself takes a request of one call, since a session pins one context: fit a small request and score the batched one against the handle. + +**The 10,000 is the deployment's, not this client's.** +It is the serving layer's `TFM_MAX_TEST_ROWS`, whose default it is, so a deployment may hold to more or less. +Measured at the boundary rather than read from a document: 10,000 rows to score are answered and 10,001 come back as ``` 422 UNSUPPORTED_SHAPE N_test 10001 exceeds limit 10000 ``` -The earlier `kumo-relational` container holds to 1,000 instead, through a -model-specific admission check of its own, so a request gathered at this -default is refused there on its first call. +The earlier `kumo-relational` container holds to 1,000 instead, through a model-specific admission check of its own, so a request gathered at this default is refused there on its first call. Pass `rows_per_call` to `materialize()` to ask in smaller batches than that. -It only ever goes down, since a larger call buys a 422 rather than a bigger -call. Two reasons to reach for it: a graph wide enough that a full batch -outruns the 30 MiB body, and a deployment whose cap is lower than the one -above. - -It is not the answer to every 30 MiB refusal. A batch carries its context as -well as its rows, so where the context is itself over the limit, no batch is -small enough: a wide graph at 10,000 context rows was refused at every batch -size tried, and the same graph at the default context was gathered as 15 -calls of 200. `num_context_samples`, `num_neighbors` and the columns the graph -carries are what come down in the first case. - -**That 100,000 is this client's, not the endpoint's.** The endpoint is never -asked for more than one call holds. Every batch is built before the first is -sent, each with its own context, so the whole population is held in memory at -once, and 100,000 entities is about a gigabyte of it. One `materialize()` -gathers at most that many, in at most 100 calls, and the refusal names -whichever of the two it met. The second is what a small `rows_per_call` -reaches, and what a temporal link prediction reaches at 20,001 entities, since -it scores 200 a call. - -Both govern the batches of one question, and neither counts the parts of a -question asked in several. A ranking is one call per candidate and a forecast -one per step, and what bounds those is `sdm_client.decompose.MOST_PARTS`, 200, -which is why a 101-candidate ranking makes 101 calls without meeting the 100 -above. What the two ceilings hold for such a question is the population each -part gathers, counted across all of them. - -A call that fails ends the request. The answers already paid for are carried -on the error as `details['answered']`, and the rows they cover as -`details['rows_answered']`, so the rest can be asked for again without buying -the whole population twice. A forecast keeps its answered steps the same way, -under `details['answered']`, numbered by step, with how many of them under -`details['parts_answered']`; a ranking keeps only the count, because an order -over some of its candidates is a different answer rather than part of the one -that was asked for. - -**Batch size is chosen from the row count, not from the bytes.** The 30 MiB -body is checked per batch at `materialize()`, before anything is sent. -Databricks model serving weighs its own wrapped envelope against a smaller -16 MB, and only at send time, so a wide graph can pass `materialize()` and be -refused on the first call. Ask in smaller calls there, with `rows_per_call`, -unless the context alone is over the gateway's limit, which a single-entity -request can be: then it is `num_context_samples`, `num_neighbors` or the -columns the graph carries, as it is for the 30 MiB body. - -**A question asked in parts is not batched.** A ranking is asked as one call -per candidate and a forecast as one per step, and `MOST_PARTS` counts those -calls, so a batched part would send the parts times the batches: a ranking -over 200 candidates and 50,000 entities would pass as 200 calls and make -1,000. More entities than one call scores is refused for both, before -anything is gathered. A ranking could not be batched in any case, since a -candidate is reported unscored for the whole ranking rather than per batch, so -one batch whose context held a single class would drop it from every row. - -The tabular cap was measured at the boundary rather than read from a document: -10,000 context rows are answered and 10,001 come back as -`422 UNSUPPORTED_SHAPE N_train 10001 exceeds limit 10000`. +It only ever goes down, since a larger call buys a 422 rather than a bigger call. +Reach for it for a deployment whose cap is lower than the one above, or set `max_relational_predict_rows` to that cap. + +**A call past the request body's limit is split by its bytes as well as its rows.** +When a call's body would pass `max_request_bytes`, or a Databricks serving endpoint's 16 MB once wrapped, `predict()` sends its rows as several smaller calls, halving until each fits, and returns one result numbered as before. +Every one of those calls carries the batch's whole context, which is not split. +Where the context alone, with one row to score, is past the limit, no call is small enough, and the request is refused with the context's size and the limit: a wide graph at 10,000 context rows can be. +`num_context_samples`, `num_neighbors` and the columns the graph carries are what come down then, or the limit goes up where the deployment takes more. +`materialize()` makes the same check before anything is sent. +A single-table request is sent as one call and is not split: past the limit, it is refused with its size, and the rows scored at once or the context come down. + +**That 100,000 is this client's, not the endpoint's.** +The endpoint is never asked for more than one call holds. +Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once, and 100,000 entities is about a gigabyte of it. +One `materialize()` gathers at most that many, in at most 100 calls, and the refusal names whichever of the two it met. +The second is what a small `rows_per_call` reaches, and what a temporal link prediction reaches at 20,001 entities, since it scores 200 a call. + +Both govern the batches of one question, and neither counts the parts of a question asked in several. +A ranking is one call per candidate and a forecast one per step, and what bounds those is `sdm_client.decompose.MOST_PARTS`, 200, which is why a 101-candidate ranking makes 101 calls without meeting the 100 above. +What the two ceilings hold for such a question is the population each part gathers, counted across all of them. + +A call that fails ends the request. +The answers already paid for are carried on the error as `details['answered']`, and the rows they cover as `details['rows_answered']`, so the rest can be asked for again without buying the whole population twice. +A forecast keeps its answered steps the same way, under `details['answered']`, numbered by step, with how many of them under `details['parts_answered']`; a ranking keeps only the count, because an order over some of its candidates is a different answer rather than part of the one that was asked for. + +**A question asked in parts is not batched by rows.** +A ranking is asked as one call per candidate and a forecast as one per step, and `MOST_PARTS` counts those calls, so a batched part would send the parts times the batches: a ranking over 200 candidates and 50,000 entities would pass as 200 calls and make 1,000. +More entities than one call scores is refused for both, before anything is gathered. +A ranking could not be batched in any case, since a candidate is reported unscored for the whole ranking rather than per batch, so one batch whose context held a single class would drop it from every row. +A part past the request body's limit is still split by its bytes, each smaller call carrying the part's one context. + +The tabular cap was measured at the boundary rather than read from a document: 10,000 context rows are answered and 10,001 come back as `422 UNSUPPORTED_SHAPE N_train 10001 exceeds limit 10000`. ## Aggregation windows open after the anchor and close on their last day diff --git a/java/src/main/java/com/nvidia/sdm/ClientOptions.java b/java/src/main/java/com/nvidia/sdm/ClientOptions.java index 292fee843..1e942530f 100644 --- a/java/src/main/java/com/nvidia/sdm/ClientOptions.java +++ b/java/src/main/java/com/nvidia/sdm/ClientOptions.java @@ -9,19 +9,23 @@ /** * How {@link SdmClient#connect(String, ClientOptions)} reaches the endpoint, as Python's {@code - * SDMClient(url, api_key, timeout=, max_retries=)}: an API key sent as {@code X-API-Key} (refused - * over plaintext {@code http://} to a host other than this one), how long each attempt may take - * (60 seconds), and how many times a transient failure is tried again (3). + * SDMClient(url, api_key, timeout=, max_retries=, limits=)}: an API key sent as {@code X-API-Key} + * (refused over plaintext {@code http://} to a host other than this one), how long each attempt + * may take (60 seconds), how many times a transient failure is tried again (3), and the size + * limits requests are held to ({@link Limits}, each by default its environment variable's, else + * the serving layer's default). */ public final class ClientOptions { private final String apiKey; private final Duration timeout; private final int maxRetries; + private final Limits limits; private ClientOptions(Builder builder) { this.apiKey = builder.apiKey; this.timeout = builder.timeout; this.maxRetries = builder.maxRetries; + this.limits = builder.limits; } public static ClientOptions defaults() { @@ -44,11 +48,16 @@ int maxRetries() { return maxRetries; } + Limits limits() { + return limits; + } + /** Builds {@link ClientOptions}. */ public static final class Builder { private String apiKey; private Duration timeout = Duration.ofSeconds(60); private int maxRetries = 3; + private Limits limits = Limits.defaults(); private Builder() {} @@ -70,6 +79,12 @@ public Builder maxRetries(int maxRetries) { return this; } + /** The size limits requests to the endpoint are held to. */ + public Builder limits(Limits limits) { + this.limits = Objects.requireNonNull(limits, "limits"); + return this; + } + public ClientOptions build() { return new ClientOptions(this); } diff --git a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java index edfa4e823..b05ddada1 100644 --- a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java +++ b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java @@ -64,15 +64,17 @@ public final class DatabricksServing { private final AuthenticationSource authentication; private final long maxRequestBytes; private final Duration timeout; + private final Limits limits; private final HttpClient client; private DatabricksServing( String endpoint, AuthenticationSource authentication, - long maxRequestBytes, Duration timeout) { + long maxRequestBytes, Duration timeout, Limits limits) { this.endpoint = endpoint; this.authentication = authentication; this.maxRequestBytes = maxRequestBytes; this.timeout = timeout; + this.limits = limits; this.client = HttpClient.newBuilder().connectTimeout(timeout).build(); } @@ -110,7 +112,7 @@ public static DatabricksServing of( Map headers = Map.copyOf(config.authenticate()); return new Authentication(config.getHost(), headers); }); - return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT); + return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT, Limits.defaults()); } /** The endpoint {@code endpoint} in the workspace at {@code host}, with the gateway's limit. */ @@ -138,7 +140,7 @@ public static DatabricksServing of( positiveLimit(maxRequestBytes); AuthenticationSource source = new AuthenticationSource(() -> new Authentication( base, Map.of("Authorization", "Bearer " + token.get()))); - return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT); + return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT, Limits.defaults()); } /** This endpoint, each attempt and its retries bounded by {@code timeout}. */ @@ -146,7 +148,16 @@ public DatabricksServing timeout(Duration timeout) { if (timeout == null || timeout.isNegative() || timeout.isZero()) { throw configuration("timeout must be a positive duration"); } - return new DatabricksServing(endpoint, authentication, maxRequestBytes, timeout); + return new DatabricksServing(endpoint, authentication, maxRequestBytes, timeout, limits); + } + + /** + * This endpoint, its requests held to {@code limits} as a client's are ({@link Limits}); the + * gateway's own limit on each wrapped call, {@link #maxRequestBytes}, holds as well. + */ + public DatabricksServing limits(Limits limits) { + return new DatabricksServing( + endpoint, authentication, maxRequestBytes, timeout, Objects.requireNonNull(limits, "limits")); } /** The serving endpoint's name. */ @@ -164,6 +175,11 @@ public Duration timeout() { return timeout; } + /** The size limits requests to this endpoint are held to. */ + public Limits limits() { + return limits; + } + /** Where requests go: the endpoint's name, since it has no URL of its own. */ String url() { return "databricks-serving:" + endpoint; diff --git a/java/src/main/java/com/nvidia/sdm/Limits.java b/java/src/main/java/com/nvidia/sdm/Limits.java new file mode 100644 index 000000000..5a961462a --- /dev/null +++ b/java/src/main/java/com/nvidia/sdm/Limits.java @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.nvidia.sdm; + +import com.nvidia.sdm.internal.LimitSettings; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The size limits a client holds a request to before it is sent, as Python's {@code + * sdm_client.Limits}. Each mirrors a limit the serving layer enforces ({@code TFM_MAX_TRAIN_ROWS}, + * {@code TFM_MAX_TEST_ROWS}, {@code TFM_MAX_PAYLOAD_BYTES}), which a deployment sets, so each is a + * setting with the serving layer's default. + * + *
{@code
+ * Limits limits = Limits.builder()
+ *     .maxTabularContextRows(200_000)
+ *     .maxRequestBytes(512L * 1024 * 1024)
+ *     .build();
+ * SdmClient client = SdmClient.connect(url, ClientOptions.builder().limits(limits).build());
+ * Request request = query.materialize(MaterializeOptions.builder().indices(keys).limits(limits).build());
+ * }
+ * + *

A setting not set here takes its environment variable, {@code SDM_CLIENT_} and its name in + * capitals (such as {@code SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS}), else its default: 10,000 context + * rows for either model, 10,000 rows scored per relational call, 200 per temporal link prediction + * call, and 31,457,280 bytes (30 MiB) per call. The core resolves them, as it does for every host. + */ +public final class Limits { + private final Map values; + + private Limits(Map values) { + this.values = Collections.unmodifiableMap(new LinkedHashMap<>(values)); + } + + /** No setting set: each takes its environment variable, else its default. */ + public static Limits defaults() { + return new Limits(Map.of()); + } + + public static Builder builder() { + return new Builder(); + } + + /** The settings set here, by name, such as {@code max_request_bytes}; none for a default. */ + public Map values() { + return values; + } + + /** + * What these limits come to, each setting by name: the value set here, else its environment + * variable's, else its default, as the core resolves them for every request. + * + * @throws StructuredDataException {@code INVALID_CONFIGURATION} when an environment variable is + * not a whole number of at least 1 + */ + public Map resolved() { + return LimitSettings.resolved(values); + } + + @Override + public boolean equals(Object other) { + return other instanceof Limits limits && limits.values.equals(values); + } + + @Override + public int hashCode() { + return values.hashCode(); + } + + @Override + public String toString() { + return "Limits" + values; + } + + /** Builds {@link Limits}; each value is a whole number of at least 1. */ + public static final class Builder { + private final Map values = new LinkedHashMap<>(); + + private Builder() {} + + /** The most context rows a relational request carries ({@code num_context_samples}' most). */ + public Builder maxRelationalContextRows(long rows) { + return set("max_relational_context_rows", rows); + } + + /** The most rows one relational call scores; more are asked in several calls. */ + public Builder maxRelationalPredictRows(long rows) { + return set("max_relational_predict_rows", rows); + } + + /** The most rows one temporal link prediction call scores. */ + public Builder maxTemporalLinkPredictRows(long rows) { + return set("max_temporal_link_predict_rows", rows); + } + + /** The most context rows a single-table request carries; materialize samples down to it. */ + public Builder maxTabularContextRows(long rows) { + return set("max_tabular_context_rows", rows); + } + + /** + * The most bytes one call's body takes. A call past it that scores several rows is split into + * smaller calls; one whose context alone is past it is refused. + */ + public Builder maxRequestBytes(long bytes) { + return set("max_request_bytes", bytes); + } + + private Builder set(String name, long value) { + if (value < 1) { + throw new StructuredDataException( + name + " is " + value + ", and it takes a whole number of at least 1", + "INVALID_CONFIGURATION", + Map.of("setting", name), + null); + } + values.put(name, value); + return this; + } + + public Limits build() { + return new Limits(values); + } + } +} diff --git a/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java b/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java index 0a87aa6b6..04bfd91bf 100644 --- a/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java +++ b/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java @@ -48,6 +48,7 @@ public final class MaterializeOptions { private final Object contextAnchorTime; private final boolean contextAnchorGiven; private final Map options; + private final Limits limits; private MaterializeOptions(Builder builder) { this.model = builder.model; @@ -59,6 +60,7 @@ private MaterializeOptions(Builder builder) { this.contextAnchorTime = builder.contextAnchorTime; this.contextAnchorGiven = builder.contextAnchorGiven; this.options = Collections.unmodifiableMap(new LinkedHashMap<>(builder.options)); + this.limits = builder.limits; } public static Builder builder() { @@ -93,6 +95,10 @@ Map options() { return options; } + Limits limits() { + return limits; + } + /** Builds {@link MaterializeOptions}. */ public static final class Builder { private String model; @@ -104,6 +110,7 @@ public static final class Builder { private Object contextAnchorTime; private boolean contextAnchorGiven; private final Map options = new LinkedHashMap<>(); + private Limits limits = Limits.defaults(); private Builder() {} @@ -207,6 +214,15 @@ public Builder rowsPerCall(long rows) { return option("rows_per_call", rows); } + /** + * The size limits the request is gathered for: pass the {@link SdmClient#limits} of the + * client it will be sent with ({@link Limits}). + */ + public Builder limits(Limits limits) { + this.limits = Objects.requireNonNull(limits, "limits"); + return this; + } + public MaterializeOptions build() { return new MaterializeOptions(this); } diff --git a/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java b/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java index 186891d9d..dedd66b8f 100644 --- a/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java +++ b/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java @@ -130,7 +130,8 @@ public Request materialize(Schema schema, org.apache.arrow.vector.table.Table pr input.put("task", options.task()); try (Host.Finished finished = com.nvidia.sdm.internal.ArrowTabular.run("materialize_predict", input, Map.of("table", schema.data().values().iterator().next(), "predict", - com.nvidia.sdm.internal.ArrowTabular.snapshot(predict)), null)) { + com.nvidia.sdm.internal.ArrowTabular.snapshot(predict)), null, + options.limits().values())) { Map result = com.nvidia.sdm.internal.ArrowTabular.value(finished); @SuppressWarnings("unchecked") Map description = (Map) result.get("description"); Map request = com.nvidia.sdm.internal.ArrowTabular.request( @@ -185,7 +186,8 @@ public Request materialize(Schema schema, MaterializeOptions options) { throw new StructuredDataException("warehouse sampling options do not apply to Tabular", "INVALID_REQUEST", Map.of(), null); try (Host.Finished finished = com.nvidia.sdm.internal.ArrowTabular.run("materialize", input, - Map.of("table", schema.data().values().iterator().next()), null)) { + Map.of("table", schema.data().values().iterator().next()), null, + options.limits().values())) { Map result = com.nvidia.sdm.internal.ArrowTabular.value(finished); @SuppressWarnings("unchecked") Map description = (Map) result.get("description"); Map request = com.nvidia.sdm.internal.ArrowTabular.request( @@ -222,7 +224,8 @@ public Request materialize(Schema schema, MaterializeOptions options) { new LinkedHashMap<>(Providers.held(schema.source())); held.put("query", fields -> text); try (Host.Finished finished = - Host.run(operation, attachments, new Host.Call(chosen.provider(), null, held))) { + Host.run(operation, attachments, + new Host.Call(chosen.provider(), null, held, options.limits().values()))) { Results.Gathered gathered = (Results.Gathered) Results.expect(finished.result(), "materialize"); if (gathered.query() == null) { diff --git a/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java b/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java index ccf5e24cf..b3ac96c37 100644 --- a/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java +++ b/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java @@ -75,7 +75,8 @@ synchronized Request materialize(PredictiveQuery query, MaterializeOptions optio attached.put(2, Host.attachment(input)); attached.put(3, Indices.column(options.indices(), ArrowData.allocator())); attached.put(4, Host.optionAttachment(PredictiveQuery.values(options))); - try (Host.Finished finished = Host.run(op, attached)) { + try (Host.Finished finished = + Host.run(op, attached, Host.Call.limited(options.limits().values()))) { Results.Gathered gathered = (Results.Gathered) Results.expect(finished.result(), "materialize"); @SuppressWarnings("unchecked") Map state = (Map) finished.result().get("state"); MaterializedRequest retained = new MaterializedRequest(state, finished); diff --git a/java/src/main/java/com/nvidia/sdm/SdmClient.java b/java/src/main/java/com/nvidia/sdm/SdmClient.java index 6606e187d..5689326ba 100644 --- a/java/src/main/java/com/nvidia/sdm/SdmClient.java +++ b/java/src/main/java/com/nvidia/sdm/SdmClient.java @@ -39,12 +39,14 @@ public final class SdmClient implements AutoCloseable { private final HttpTransport transport; private final DatabricksServing serving; + private final Limits limits; private volatile boolean closed; private final java.util.Set sessions = java.util.concurrent.ConcurrentHashMap.newKeySet(); - private SdmClient(HttpTransport transport, DatabricksServing serving) { + private SdmClient(HttpTransport transport, DatabricksServing serving, Limits limits) { this.transport = transport; this.serving = serving; + this.limits = limits; } /** @@ -62,7 +64,9 @@ public static SdmClient connect(String url) { public static SdmClient connect(String url, ClientOptions options) { Objects.requireNonNull(options, "options"); return new SdmClient( - new HttpTransport(url, options.apiKey(), options.timeout(), options.maxRetries()), null); + new HttpTransport(url, options.apiKey(), options.timeout(), options.maxRetries()), + null, + options.limits()); } /** @@ -71,7 +75,16 @@ public static SdmClient connect(String url, ClientOptions options) { * endpoint's envelope and reads its every answer and failure. */ public static SdmClient forDatabricksServing(DatabricksServing serving) { - return new SdmClient(null, Objects.requireNonNull(serving, "serving")); + Objects.requireNonNull(serving, "serving"); + return new SdmClient(null, serving, serving.limits()); + } + + /** + * The size limits this client holds requests to, as it was opened with them; a setting not set + * takes its environment variable, else its default, which {@link Limits#resolved} reports. + */ + public Limits limits() { + return limits; } /** Where this client sends requests: a URL, or {@code databricks-serving:}. */ @@ -197,6 +210,10 @@ void requireOpen() { Host.Model tabularTransport() { return new Host.Model() { + public Map limits() { + return limits.values(); + } + public byte[] call(byte[] body) throws Host.TransportFailed, Host.Refused, Host.EndpointFailed { return SdmClient.this.call(body); } diff --git a/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java b/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java index 3d7db9d50..6cdf598ea 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java +++ b/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java @@ -54,6 +54,15 @@ public static byte[] snapshot(Table table) { public static Host.Finished run(String action, Object input, Map tables, Host.Model model) { + return run(action, input, tables, model, null); + } + + /** + * Runs {@code action} as {@link #run(String, Object, Map, Host.Model)}, held to the size limits + * {@code limits} names (null: those of {@code model}'s client, else none). + */ + public static Host.Finished run(String action, Object input, Map tables, + Host.Model model, Map limits) { Map operation = new LinkedHashMap<>(); operation.put("op", "tabular_arrow"); operation.put("action", action); @@ -72,7 +81,7 @@ public static Host.Finished run(String action, Object input, Map operation.put(entry.getKey(), id); attachments.put(id, stream); } - return Host.run(operation, attachments, new Host.Call(null, model, Map.of())); + return Host.run(operation, attachments, new Host.Call(null, model, Map.of(), limits)); } finally { for (ArrowArrayStream stream : streams) stream.close(); for (ArrowReader reader : readers) { diff --git a/java/src/main/java/com/nvidia/sdm/internal/Host.java b/java/src/main/java/com/nvidia/sdm/internal/Host.java index 1807bc2db..3a8d4e7da 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Host.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Host.java @@ -87,6 +87,14 @@ default Map failure(Exception failure) { public interface Model { byte[] call(byte[] body) throws TransportFailed, Refused, EndpointFailed; + /** + * The size limits the caller set on the client this model is reached through, by setting + * name; null where it set none. + */ + default Map limits() { + return null; + } + /** Session-aware transports override this; existing model callbacks remain prediction-only. */ default byte[] call(Object endpoint, byte[] body) throws TransportFailed, Refused, EndpointFailed { if (!"predict".equals(endpoint)) throw new StructuredDataException( @@ -155,19 +163,41 @@ public Integer status() { /** * What one run is given beside its operation: its provider and model, or null for an operation - * that reads no data and calls no model, and the values only this host holds that a sentence - * may name ({@code {host.}}), each read from the error's or warning's fields. + * that reads no data and calls no model, the values only this host holds that a sentence may + * name ({@code {host.}}), each read from the error's or warning's fields, and the size + * limits the caller set, by setting name ({@code com.nvidia.sdm.Limits}), null for those of the + * model's client, else none. */ public record Call( - Provider provider, Model model, Map, Object>> held) { + Provider provider, + Model model, + Map, Object>> held, + Map limits) { public Call { held = held == null ? Map.of() : Map.copyOf(held); + limits = limits == null ? null : Map.copyOf(limits); + } + + public Call( + Provider provider, Model model, Map, Object>> held) { + this(provider, model, held, null); } /** A run with nothing beside its operation. */ public static Call none() { return new Call(null, null, Map.of()); } + + /** A run with nothing beside its operation but the size limits the caller set. */ + public static Call limited(Map limits) { + return new Call(null, null, Map.of(), limits); + } + + /** The size limits the run is held to: those given, else those of the model's client. */ + Map effectiveLimits() { + if (limits != null) return limits; + return model == null ? null : model.limits(); + } } /** A finished run: its result, the warnings it gave in order, and the run, to take from. */ @@ -308,6 +338,10 @@ public static Finished run( start.put("message", "start"); start.put("contract", contract()); start.put("facts", HostFacts.now()); + Map limits = LimitSettings.given(call.effectiveLimits()); + if (!limits.isEmpty()) { + start.put("limits", limits); + } start.put("operation", operation); byte[] message = control(start); NativeCore.setLogLevel(NativeLog.level()); diff --git a/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java b/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java new file mode 100644 index 000000000..190abcbcf --- /dev/null +++ b/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.nvidia.sdm.internal; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The size limits a run's start carries ({@code com.nvidia.sdm.Limits}): the values the caller set + * and the text of each setting's environment variable that is set, read when the run starts, which + * the core resolves by the one rule every host reads them with. The settings' names and variables + * are the core's ({@code sdm_contract::limits::SETTINGS}), read from the library's description. + */ +public final class LimitSettings { + private LimitSettings() {} + + /** Each setting the core names, in order: its {@code name}, {@code env} and {@code default}. */ + @SuppressWarnings("unchecked") + public static List> settings() { + return (List>) NativeCore.load().get("limits"); + } + + /** What a run's start carries of {@code values}, the settings the caller set by name. */ + public static Map given(Map values) { + Map environment = new LinkedHashMap<>(); + for (Map setting : settings()) { + String name = (String) setting.get("env"); + String text = System.getenv(name); + if (text != null) { + environment.put(name, text); + } + } + Map given = new LinkedHashMap<>(); + if (values != null && !values.isEmpty()) { + given.put("given", new LinkedHashMap<>(values)); + } + if (!environment.isEmpty()) { + given.put("environment", environment); + } + return given; + } + + /** What {@code values} come to, each setting by name, as the core resolves them. */ + public static Map resolved(Map values) { + Map operation = new LinkedHashMap<>(); + operation.put("op", "resolve_limits"); + operation.put("input", 1); + try (Host.Finished finished = + Host.run(operation, Map.of(1, Host.attachment(Map.of())), Host.Call.limited(values))) { + Map answer = ArrowTabular.value(finished); + Map resolved = new LinkedHashMap<>(); + for (Map setting : settings()) { + String name = (String) setting.get("name"); + resolved.put(name, ((Number) answer.get(name)).longValue()); + } + return resolved; + } + } +} diff --git a/java/src/main/java/com/nvidia/sdm/internal/Wording.java b/java/src/main/java/com/nvidia/sdm/internal/Wording.java index 50ae80ad6..0b3f69c7b 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Wording.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Wording.java @@ -118,6 +118,9 @@ private static Map, String>> advice() { spelled.put("nothing", given -> "null"); spelled.put("optimize_indices", given -> "an index on the key column"); spelled.put("predict_call", given -> "predict()"); + spelled.put( + "raise_limit", + given -> "Limits.builder()." + camel(plainText(given.get(0))) + "(...)"); spelled.put("raise_most_parts", given -> "a higher limit on the parts one question is asked as"); spelled.put("result_frame", given -> "PredictionResult.read()"); spelled.put( @@ -135,9 +138,6 @@ private static Map, String>> advice() { given -> "Name another key with FromSourceOptions.builder().primaryKey(...), or pass" + " checkKeys(false) to skip (one full scan per table)."); - spelled.put( - "smaller_calls", - given -> "MaterializeOptions.builder().option(\"rows_per_call\", N)"); spelled.put("trust_expressions", given -> "semanticModel(name, true)"); return Map.copyOf(spelled); } @@ -712,6 +712,21 @@ private static String plain(Object value) throws Unfilled { return value instanceof String text ? text : literal(value); } + /** A setting's name as its builder method spells it: {@code max_request_bytes} as {@code maxRequestBytes}. */ + static String camel(String name) { + StringBuilder spelled = new StringBuilder(name.length()); + boolean upper = false; + for (char c : name.toCharArray()) { + if (c == '_') { + upper = true; + } else { + spelled.append(upper ? Character.toUpperCase(c) : c); + upper = false; + } + } + return spelled.toString(); + } + private static String plainText(Object value) { try { return plain(value); diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index ac1fc4ed6..66cb384e9 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -404,13 +404,14 @@ void utf8FeatureValuesUseTheSharedEnvelopeAndActualByteLimit() throws Exception client.predict(request, SdmClient.KUMO_RELATIONAL); byte[] body = workspace.bodies.get(0); assertTrue(new String(body, StandardCharsets.UTF_8).contains("é😀")); + // One byte under the call: it is split into smaller calls, each within the limit and + // each carrying the whole context, its rows answered as one. try (Workspace limited = new Workspace(); SdmClient capped = SdmClient.forDatabricksServing(limited.serving(body.length - 1))) { - StructuredDataException error = assertThrows(StructuredDataException.class, - () -> capped.predict(request, SdmClient.KUMO_RELATIONAL)); - assertEquals("INVALID_REQUEST", error.code()); - assertEquals(body.length, ((Number) error.details().get("bytes")).intValue()); - assertTrue(limited.bodies.isEmpty()); + assertEquals(client.predict(request, SdmClient.KUMO_RELATIONAL).rows(), + capped.predict(request, SdmClient.KUMO_RELATIONAL).rows()); + assertTrue(limited.bodies.size() > 1); + for (byte[] part : limited.bodies) assertTrue(part.length <= body.length - 1); } } } @@ -551,7 +552,9 @@ void aCallPastTheCallersLimitIsRefusedBeforeItIsSent() throws Exception { () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); assertEquals("INVALID_REQUEST", error.code()); assertEquals(100L, ((Number) error.details().get("limit")).longValue()); - assertTrue(error.getMessage().contains("above the 100 byte limit it accepts"), error.getMessage()); + assertTrue(error.getMessage().contains("above the 100 byte limit the endpoint accepts"), + error.getMessage()); + assertTrue(error.getMessage().contains("of them the context"), error.getMessage()); assertEquals(List.of(), workspace.bodies); } } diff --git a/java/src/test/java/com/nvidia/sdm/LimitsTest.java b/java/src/test/java/com/nvidia/sdm/LimitsTest.java new file mode 100644 index 000000000..eaac68985 --- /dev/null +++ b/java/src/test/java/com/nvidia/sdm/LimitsTest.java @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.nvidia.sdm; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.Map; +import java.util.stream.LongStream; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.table.Table; +import org.junit.jupiter.api.Test; + +/** The size limits a client holds requests to: their defaults and a raised one taking effect. */ +class LimitsTest { + @Test void theDefaultsAreTheServingLayers() { + assertEquals( + Map.of( + "max_relational_context_rows", 10_000L, + "max_relational_predict_rows", 10_000L, + "max_temporal_link_predict_rows", 200L, + "max_tabular_context_rows", 10_000L, + "max_request_bytes", 31_457_280L), + Limits.defaults().resolved()); + } + + @Test void aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne() { + Limits limits = Limits.builder().maxTabularContextRows(200_000).build(); + assertEquals(Map.of("max_tabular_context_rows", 200_000L), limits.values()); + assertEquals(200_000L, limits.resolved().get("max_tabular_context_rows")); + assertEquals(10_000L, limits.resolved().get("max_relational_context_rows")); + StructuredDataException refused = assertThrows( + StructuredDataException.class, () -> Limits.builder().maxRequestBytes(0)); + assertEquals("INVALID_CONFIGURATION", refused.code()); + } + + @Test void aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised() throws Exception { + long[] ids = LongStream.range(0, 10_001).toArray(); + try (RootAllocator allocator = new RootAllocator(); + Table context = TabularTest.table(allocator, true, ids); + Table scoring = TabularTest.table(allocator, false, 20_000); + TabularTest.Endpoint endpoint = new TabularTest.Endpoint()) { + Request request = new Request(context, scoring, "y", "regression"); + try (SdmClient client = SdmClient.connect(endpoint.url())) { + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> client.predict(request, SdmClient.KUMO_TABULAR)); + assertTrue(refused.getMessage().contains("SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS"), + refused.getMessage()); + assertTrue(endpoint.calls.isEmpty()); + } + Limits raised = Limits.builder().maxTabularContextRows(20_000).build(); + try (SdmClient client = SdmClient.connect(endpoint.url(), + ClientOptions.builder().limits(raised).build())) { + assertEquals(raised, client.limits()); + assertEquals(1, client.predict(request, SdmClient.KUMO_TABULAR).rows()); + @SuppressWarnings("unchecked") Map sent = + (Map) endpoint.calls.get(0).get("context"); + @SuppressWarnings("unchecked") Map table = + (Map) sent.get("instance_table"); + assertEquals(10_001, ((List) table.get("rows")).size()); + } + } + } +} diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index 361605049..40ea19ac1 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -36,6 +36,7 @@ GradientExplanations, ) from sdm_client.inference import InferenceConfig +from sdm_client.limits import Limits from sdm_client.models import KumoRelational, KumoTabular, Model from sdm_client.query import PredictiveQuery from sdm_client.relational_task import RelationalTask @@ -81,6 +82,7 @@ 'FittedModel', 'KumoRelational', 'KumoTabular', + 'Limits', 'MissingExtraError', 'Model', 'ModelCapabilities', diff --git a/packages/structured-data-models-client/src/sdm_client/_native.pyi b/packages/structured-data-models-client/src/sdm_client/_native.pyi index 3a91a6fa6..2ade6e7ce 100644 --- a/packages/structured-data-models-client/src/sdm_client/_native.pyi +++ b/packages/structured-data-models-client/src/sdm_client/_native.pyi @@ -70,6 +70,10 @@ MEANINGS: tuple[tuple[str, str], ...] #: Each data type name a host may give a column, and the contract's name #: of the type it names: the one table of them. DTYPE_NAMES: tuple[tuple[str, str], ...] +#: Each size limit a caller may set, ``(name, env, default, about)``: its +#: name, the environment variable a host reads it from, its default and +#: what it holds a request to; the one table of them every host reads. +LIMITS: tuple[tuple[str, str, int, str], ...] class _ArrowStreamExportable(Protocol): def __arrow_c_stream__( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py index 9da5854b0..10098594a 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py @@ -154,6 +154,7 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name 'nothing': lambda: 'None', 'optimize_indices': lambda: 'KumoRelational(graph, optimize=True)', 'predict_call': lambda: 'predict()', + 'raise_limit': lambda setting: f'limits=Limits({setting}=...)', 'raise_most_parts': lambda: 'raise sdm_client.decompose.MOST_PARTS', 'semantic_model': lambda name: f'source.semantic_model({name!r})', 'set_primary_key': lambda table, column: ( @@ -165,7 +166,6 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name ), 'missing_time': lambda: "'NaT'", 'result_frame': lambda: 'to_pandas()', - 'smaller_calls': lambda: 'materialize(rows_per_call=N)', 'trust_expressions': lambda: 'trust_expressions=True', } diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index 570bcb51e..062e1e38d 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -48,6 +48,7 @@ # dependency, and a core route over frames needs none of it. from sdm_connectors.providers import Provider +from sdm_client import limits from sdm_client._rust import NativeBoundaryError, codec, errors, facts from sdm_client.core.transport import EndpointFailed from sdm_client.errors import StructuredDataError @@ -224,6 +225,7 @@ def run( 'message': 'start', 'contract': core.CONTRACT_VERSION, 'facts': facts.host_facts(core), + 'limits': limits.in_effect()._given(), 'operation': operation, } ) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py index 9b15ae10f..6e28db207 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py @@ -215,9 +215,11 @@ def _context( has_custom_sorter, observe, ) - from sdm_client.models import TABULAR_LIMITS + from sdm_client.limits import in_effect - limit = TABULAR_LIMITS['context_rows_max'] + # The most context rows a single-table request carries, as the core + # resolves the setting for the runs below. + limit = in_effect().resolved()['max_tabular_context_rows'] if decision is None: over_limit = wanted is not None and bool(wanted > limit) accepted = _call('check_tabular_cap', {'over_limit': over_limit})[ @@ -236,8 +238,10 @@ def _context( ) if decision['kind'] == 'refused': raise StructuredDataError( - f'num_context_samples is {wanted:,}, and the ' - f'endpoint accepts at most {limit:,} context rows', + f'num_context_samples is {wanted:,}, and a request carries at ' + f'most {limit:,} context rows; if the endpoint takes more, raise ' + f'it with limits=Limits(max_tabular_context_rows=...) or the ' + f'environment variable SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', code='UNSUPPORTED_REQUEST', ) if not decision['reduced']: diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 64e2d41e9..35eae7bfd 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -37,6 +37,8 @@ StructuredDataError, UnknownModelError, ) +from sdm_client.limits import Limits, limits_in_effect +from sdm_client.limits import check as check_limits from sdm_client.models import MODELS, Model, spec_for from sdm_client.request import ( Question, @@ -189,14 +191,15 @@ def predict(self, request: Request) -> PredictionResult: f'question that many times. Ask it with predict()', code='UNSUPPORTED_REQUEST', ) - if getattr(self, '_relational_session', None) is not None: - from sdm_client._rust import frame_sessions + with limits_in_effect(self._client._limits): + if getattr(self, '_relational_session', None) is not None: + from sdm_client._rust import frame_sessions - return frame_sessions.predict(self, request) - if getattr(self, '_tabular_session', None) is not None: - from sdm_client._rust import sessions + return frame_sessions.predict(self, request) + if getattr(self, '_tabular_session', None) is not None: + from sdm_client._rust import sessions - return sessions.predict(self, request) + return sessions.predict(self, request) raise StructuredDataError( 'this fitted model has no native session state; fit again', code='INVALID_CONFIGURATION', @@ -271,6 +274,7 @@ def __init__( verify_ssl: bool = True, timeout: float = 60.0, max_retries: int = 3, + limits: Limits | None = None, ) -> None: r"""Open a client against one NIM. @@ -291,6 +295,11 @@ def __init__( the same on every attempt. ``POST /v1/sessions`` is excluded: a re-sent create would orphan a pinned context on the endpoint. + limits: The size limits requests to this endpoint are held to, + for a deployment configured to take more or less than the + serving layer's defaults. A setting left out takes its + environment variable, else its default; see + :class:`~sdm_client.Limits`. """ self._start( Transport( @@ -299,10 +308,13 @@ def __init__( verify_ssl=verify_ssl, timeout=timeout, max_retries=max_retries, - ) + ), + limits, ) - def _start(self, transport: WireTransport) -> None: + def _start( + self, transport: WireTransport, limits: Limits | None = None + ) -> None: r"""Set everything a client holds, given what carries its requests. The one place a client's state is set.:meth:`for_databricks_serving` @@ -311,22 +323,27 @@ def _start(self, transport: WireTransport) -> None: new state here, not to ``__init__``. """ self._transport: WireTransport = transport + self._limits: Limits | None = check_limits(limits) self._adapters: dict[str, ModelAdapter] = {} self._pinned: set[FittedModel] = set() @classmethod - def in_process(cls, *models: Any, device: Any = None) -> SDMClient: + def in_process( + cls, *models: Any, device: Any = None, limits: Limits | None = None + ) -> SDMClient: """Run models locally using the same Rust preparation and result path. Pass loaded structured-data-models models, or omit them to load weights on first use. The optional ``in-process`` extra requires Python 3.11+. ``device`` controls models loaded by this client; supplied models keep - their placement. Local refusals carry no HTTP status. + their placement. Local refusals carry no HTTP status. ``limits`` + holds requests to the sizes it names, as for an endpoint; see + :class:`~sdm_client.Limits`. """ from sdm_client.in_process import InProcessBackend client = cls.__new__(cls) - client._start(InProcessBackend(models, device)) + client._start(InProcessBackend(models, device), limits) return client @classmethod @@ -336,6 +353,7 @@ def for_databricks_serving( *, workspace_client: Any | None = None, timeout: float | None = None, + limits: Limits | None = None, ) -> SDMClient: r"""Open a client against a Databricks Model Serving endpoint. @@ -367,6 +385,10 @@ def for_databricks_serving( timeouts. Given with a ``WorkspaceClient``, it is set on the copy, never on the caller's client; with a subclass or test double, which cannot be copied, it is refused. + limits: The size limits requests are held to, as for a NIM; see + :class:`~sdm_client.Limits`. The gateway's own 16 MB limit on + each call, once wrapped, holds as well: a call past either is + split into smaller calls where it scores several rows. Raises: MissingExtraError: If ``databricks-sdk`` is not installed; install @@ -385,10 +407,18 @@ def for_databricks_serving( client._start( DatabricksServingTransport( endpoint, workspace_client, timeout=timeout - ) + ), + limits, ) return client + @property + def limits(self) -> Limits: + r"""The size limits this client was opened with; a setting left + ``None`` takes its environment variable, else its default, which + :meth:`Limits.resolved` reports.""" + return self._limits or Limits() + @property def url(self) -> str: r"""Where this client sends requests. @@ -449,11 +479,12 @@ def predict( """ _require_a_request(request) _require_output_names(outputs) - if request.parts is not None: - return self._answer_in_parts(request, model, outputs) - return _prediction_handler(request).predict( - self, request, model, outputs - ) + with limits_in_effect(self._limits): + if request.parts is not None: + return self._answer_in_parts(request, model, outputs) + return _prediction_handler(request).predict( + self, request, model, outputs + ) def _answer_in_parts( self, @@ -575,13 +606,14 @@ def fit( _require_output_names(outputs) _refuse_to_pin_a_rewritten_question(request) spec = self._spec(model) - if spec.model_id == 'kumo-tabular': - from sdm_client._rust import sessions + with limits_in_effect(self._limits): + if spec.model_id == 'kumo-tabular': + from sdm_client._rust import sessions - return sessions.fit(self, request, model, outputs, holdout) - from sdm_client._rust import frame_sessions + return sessions.fit(self, request, model, outputs, holdout) + from sdm_client._rust import frame_sessions - return frame_sessions.fit(self, request, model, outputs, holdout) + return frame_sessions.fit(self, request, model, outputs, holdout) @staticmethod def _spec(model: str | Model) -> ModelSpec: diff --git a/packages/structured-data-models-client/src/sdm_client/limits.py b/packages/structured-data-models-client/src/sdm_client/limits.py new file mode 100644 index 000000000..1e25ff0d6 --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/limits.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""The size limits this client holds a request to before it is sent. + +Each mirrors a limit the serving layer enforces, so the client refuses or +batches first and says which limit, rather than sending a request the +endpoint refuses without saying how to meet it. A deployment sets its own +(the serving layer's ``TFM_MAX_TRAIN_ROWS``, ``TFM_MAX_TEST_ROWS`` and +``TFM_MAX_PAYLOAD_BYTES``), so each is a setting with the serving layer's +default. Raise one to match a deployment configured to take more: + +>>> limits = Limits(max_tabular_context_rows=200_000, +... max_request_bytes=512 * 1024 * 1024) +>>> client = SDMClient(url, limits=limits) +>>> request = query.materialize(indices=keys, limits=limits) + +or set its environment variable, ``SDM_CLIENT_`` and the setting's name in +capitals, such as ``SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS=200000``. A value +passed to :class:`Limits` comes before the environment's, which comes before +the default. The names, variables and defaults are the core's, the one table +every host reads (:data:`SETTINGS`), and the core resolves them. +""" + +from __future__ import annotations + +import contextvars +import os +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, fields +from typing import Any, NamedTuple + +from sdm_client.errors import StructuredDataError + + +class Setting(NamedTuple): + r"""One limit a caller may set: its name, the environment variable it is + read from, its default and what it holds a request to.""" + + name: str + env: str + default: int + about: str + + +def _settings() -> tuple[Setting, ...]: + from sdm_client._rust.run import native + + return tuple(Setting(*setting) for setting in native().LIMITS) + + +@dataclass(frozen=True) +class Limits: + r"""The size limits a client holds requests to, each ``None`` to take + its environment variable, else its default. + + Attributes: + max_relational_context_rows: The most context rows a relational + request carries; ``num_context_samples`` asks for at most this + many. ``SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS``, 10,000. + max_relational_predict_rows: The most rows one relational call + scores; more are asked in several calls. + ``SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS``, 10,000. + max_temporal_link_predict_rows: The most rows one temporal link + prediction call scores. + ``SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS``, 200. + max_tabular_context_rows: The most context rows a single-table + request carries; ``materialize()`` samples down to it. + ``SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS``, 10,000. + max_request_bytes: The most bytes one call's body takes. A call + past it that scores several rows is split into smaller calls; one + whose context alone is past it is refused. + ``SDM_CLIENT_MAX_REQUEST_BYTES``, 31,457,280 (30 MiB). A + Databricks serving endpoint also holds each call to its own + ``max_request_bytes``. + + Raises: + StructuredDataError: With code ``INVALID_CONFIGURATION``, if a value + is not a whole number of at least 1. + """ + + max_relational_context_rows: int | None = None + max_relational_predict_rows: int | None = None + max_temporal_link_predict_rows: int | None = None + max_tabular_context_rows: int | None = None + max_request_bytes: int | None = None + + def __post_init__(self) -> None: + for field in fields(self): + value = getattr(self, field.name) + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, int): + raise StructuredDataError( + f'{field.name} takes a whole number of at least 1, ' + f'got {type(value).__name__} {value!r}', + code='INVALID_CONFIGURATION', + ) + if value < 1: + raise StructuredDataError( + f'{field.name} is {value:,}, and it takes a whole number ' + f'of at least 1', + code='INVALID_CONFIGURATION', + ) + + def _given(self) -> dict[str, Any]: + r"""What a run's start carries of these limits: the values set here, + and the text of each setting's environment variable that is set, + read now, which the core resolves (contract ``GivenLimits``).""" + given = { + field.name: getattr(self, field.name) + for field in fields(self) + if getattr(self, field.name) is not None + } + environment = { + setting.env: os.environ[setting.env] + for setting in _settings() + if setting.env in os.environ + } + start: dict[str, Any] = {} + if given: + start['given'] = given + if environment: + start['environment'] = environment + return start + + def resolved(self) -> dict[str, int]: + r"""What these limits come to, each setting by its name: the value + set here, else its environment variable's, else its default, as the + core resolves them for every request. + + Raises: + StructuredDataError: With code ``INVALID_CONFIGURATION``, if an + environment variable is not a whole number of at least 1. + """ + from sdm_client._rust import direct + + with limits_in_effect(self): + answer = direct._call('resolve_limits', {}) + return { + setting.name: int(answer[setting.name]) for setting in _settings() + } + + +def settings() -> tuple[Setting, ...]: + r"""Every limit a caller may set, in order: the core's one table.""" + return _settings() + + +_IN_EFFECT: contextvars.ContextVar[Limits | None] = contextvars.ContextVar( + 'sdm_client_limits', default=None +) + + +@contextmanager +def limits_in_effect(limits: Limits | None) -> Iterator[None]: + r"""Hold every run started inside to ``limits``; ``None`` keeps the ones + already in effect. A client's calls and ``materialize(limits=...)`` set + them for their own runs only.""" + if limits is None: + yield + return + token = _IN_EFFECT.set(limits) + try: + yield + finally: + _IN_EFFECT.reset(token) + + +def in_effect() -> Limits: + r"""The limits a run started now is held to: those set by the call it is + part of, else the environment's and the defaults.""" + return _IN_EFFECT.get() or Limits() + + +def check(limits: Any) -> Limits | None: + r"""``limits`` as an argument takes it: a :class:`Limits`, or ``None``.""" + if limits is None or isinstance(limits, Limits): + return limits + raise StructuredDataError( + f'limits takes an sdm_client.Limits, got {type(limits).__name__}', + code='INVALID_CONFIGURATION', + ) diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 5a8a36717..5933afb90 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -37,28 +37,16 @@ from sdm_client.base import ModelSpec, RuntimeContract from sdm_client.errors import StructuredDataError, UnknownModelError -# Sizes the relational endpoint enforces. `predict_rows_max` is the serving -# layer's own `TFM_MAX_TEST_ROWS`, measured at the boundary rather than read -# from a document: 10,000 rows to score are answered, and 10,001 come back +# Fixed properties of the relational model. The sizes a deployment sets (the +# most context rows, rows one call scores and bytes one call takes) are not +# here: they are settings, the core's one table of them, which a caller +# raises or lowers to match the deployment it sends to (`sdm_client.Limits`). # -# [422 UNSUPPORTED_SHAPE] N_test 10001 exceeds limit 10000. -# -# A deployment may be configured to hold to less, which is what the -# `rows_per_call` option on materialize() is for. -# -# The endpoint refuses past these, so the client refuses first with a message -# that names the limit. See the architecture note on inference knobs: -# `run_mode` is derived from these rather than being set. -# -# Prediction row limits also bound the batches requested from the Rust core. -# Keep endpoint limits and batching consistent so each batch is accepted. +# `context_rows_fast` is the context gathered when `num_context_samples` is +# not given, a default rather than a cap. The relational model reads as many +# tables and hops as a request carries, so neither is limited here. RELATIONAL_LIMITS = { 'context_rows_fast': 1_000, - 'context_rows_max': 10_000, - 'predict_rows_max': 10_000, - 'predict_rows_temporal_link_prediction': 200, - 'hops_max': 6, - 'tables_max': 15, 'classes_max': 10, } @@ -69,17 +57,9 @@ PREDICTION_STATISTICS = frozenset({'mean', 'median'}) SUMMARISED_TASK_KINDS = frozenset({'regression', 'forecasting'}) -# The tabular endpoint refuses a context past this, exactly: -# -# [422 UNSUPPORTED_SHAPE] N_train 10001 exceeds limit 10000. -# -# Measured against the deployment rather than read from a document, at the -# boundary: 10,000 rows answer, 10,001 refuse. Declared so the client refuses -# first, because the alternative is encoding and transmitting an arbitrarily -# large context to be told the limit at the far end. -TABULAR_LIMITS = { - 'context_rows_max': 10_000, -} +# The tabular model's context is held to a setting, not a fixed size +# (`sdm_client.Limits.max_tabular_context_rows`). +TABULAR_LIMITS: dict[str, int] = {} KUMO_TABULAR = ModelSpec( model_id=TFM_MODEL_KUMO_TABULAR, diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index d3e97a2c8..ad428144b 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -41,6 +41,8 @@ said_as_parts_are, ) from sdm_client.errors import MissingExtraError, StructuredDataError +from sdm_client.limits import Limits, limits_in_effect +from sdm_client.limits import check as check_limits from sdm_client.materializers import RelationalMaterializer from sdm_client.models import RELATIONAL_LIMITS, Model, spec_for from sdm_client.request import Request @@ -147,6 +149,7 @@ def materialize( model: str | Model | None = None, candidates: Sequence[Any] | None = None, task: str | None = None, + limits: Limits | None = None, **options: Any, ) -> Request: r"""Build a request with the data needed to answer this query. @@ -196,9 +199,14 @@ def materialize( ``rows_per_call``: how many entities one call scores, which defaults to what the endpoint takes, 10,000 or 200 for a temporal link prediction, and may only be lowered. Lower it - for a graph wide enough that a full batch outruns the 30 MiB - body, or for a deployment holding to less than this client - knows of. + for a deployment holding to less than this client knows of. A + call past the most bytes one takes is asked as smaller calls + when it is predicted. + limits: The size limits the request is gathered for: the most + context rows, rows one call scores and bytes one call takes. + Pass the :attr:`~sdm_client.SDMClient.limits` of the client it + will be sent with. A setting left out takes its environment + variable, else its default; see :class:`~sdm_client.Limits`. Returns: A:class:`~sdm_client.Request` carrying the data. @@ -210,6 +218,29 @@ def materialize( given, if both are, if an option is not one this accepts, or if ``model`` cannot answer a schema of this shape. """ + with limits_in_effect(check_limits(limits)): + return self._materialize( + indices=indices, + predict=predict, + num_context_samples=num_context_samples, + model=model, + candidates=candidates, + task=task, + **options, + ) + + def _materialize( + self, + *, + indices: Sequence[Any] | None, + predict: pd.DataFrame | None, + num_context_samples: int | None, + model: str | Model | None, + candidates: Sequence[Any] | None, + task: str | None, + **options: Any, + ) -> Request: + r""":meth:`materialize`, with the limits it was given in effect.""" spec = spec_for(model) if model is not None else None in_place = self.schema.source is not None relational = ( diff --git a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py index 85baf1abd..f4c7d426e 100644 --- a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py +++ b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py @@ -34,10 +34,12 @@ class StopWarning(Exception): def observe_case(name: str) -> dict[str, Any]: from sdm_client import KumoTabular, PredictiveQuery, SDMClient - from sdm_client.models import TABULAR_LIMITS + from sdm_client.limits import settings assert name in CASES - cap = TABULAR_LIMITS['context_rows_max'] + cap = {setting.name: setting.default for setting in settings()}[ + 'max_tabular_context_rows' + ] events: list[Any] = [] recorded: list[dict[str, str]] = [] outcome: dict[str, Any] = {} diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index 641cc7926..b067465fe 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -97,9 +97,10 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( ): """A part is one call, which is what makes the fan-out countable. The refusal reads the count it was given, so it costs no gathering.""" - from sdm_client.models import RELATIONAL_LIMITS + from sdm_client.limits import settings - count = RELATIONAL_LIMITS['predict_rows_max'] + 1 + defaults = {setting.name: setting.default for setting in settings()} + count = defaults['max_relational_predict_rows'] + 1 with pytest.raises(StructuredDataError) as refused: built(query, indices=[1] * count, **how) assert refused.value.code == 'UNSUPPORTED_REQUEST' diff --git a/packages/structured-data-models-client/tests/test_configurable_limits.py b/packages/structured-data-models-client/tests/test_configurable_limits.py new file mode 100644 index 000000000..9e2176e43 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_configurable_limits.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The size limits a client holds requests to are settings. + +Each defaults to the serving layer's own, and a deployment configured to take +more is matched by raising it: with an argument, else its environment +variable. A call past the most bytes one takes is split rather than refused, +and only a context too large on its own is refused, naming the setting. +""" + +from __future__ import annotations + +import json +import warnings + +import numpy as np +import pandas as pd +import pytest +from test_limits_and_absent_outputs import gathered, query, table +from test_tabular_direct_core import Endpoint, client + +from sdm_client import KumoTabular, Limits, Request, StructuredDataError +from sdm_client.limits import settings + +DEFAULTS = { + 'max_relational_context_rows': 10_000, + 'max_relational_predict_rows': 10_000, + 'max_temporal_link_predict_rows': 200, + 'max_tabular_context_rows': 10_000, + 'max_request_bytes': 30 * 1024 * 1024, +} + + +@pytest.fixture(autouse=True) +def _no_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for setting in settings(): + monkeypatch.delenv(setting.env, raising=False) + + +def test_the_defaults_are_the_serving_layers() -> None: + assert {s.name: s.default for s in settings()} == DEFAULTS + assert {s.name: s.env for s in settings()} == { + name: f'SDM_CLIENT_{name.upper()}' for name in DEFAULTS + } + assert Limits().resolved() == DEFAULTS + + +def test_an_argument_comes_before_the_environment_before_the_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '200000') + monkeypatch.setenv('SDM_CLIENT_MAX_REQUEST_BYTES', '64_000_000') + from_env = Limits().resolved() + assert from_env['max_tabular_context_rows'] == 200_000 + assert from_env['max_request_bytes'] == 64_000_000 + assert from_env['max_relational_context_rows'] == 10_000 + given = Limits(max_tabular_context_rows=50_000).resolved() + assert given['max_tabular_context_rows'] == 50_000 + + +@pytest.mark.parametrize('text', ['0', '-1', '2.5', 'lots', '1e6']) +def test_an_environment_variable_that_is_not_a_whole_number_is_refused( + monkeypatch: pytest.MonkeyPatch, text: str +) -> None: + monkeypatch.setenv('SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS', text) + with pytest.raises(StructuredDataError) as refused: + Limits().resolved() + assert refused.value.code == 'INVALID_CONFIGURATION' + assert 'SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS' in str(refused.value) + + +@pytest.mark.parametrize('value', [0, -5, True, 2.5, '10']) +def test_a_setting_given_that_is_not_a_whole_number_is_refused(value) -> None: + with pytest.raises(StructuredDataError) as refused: + Limits(max_request_bytes=value) + assert refused.value.code == 'INVALID_CONFIGURATION' + + +def _direct(context_rows: int) -> Request: + context = pd.DataFrame( + { + 'row_id': np.arange(context_rows), + 'feature': np.arange(context_rows) % 7, + 'target': np.arange(context_rows) % 2, + } + ) + scoring = pd.DataFrame({'row_id': [90, 91], 'feature': [1, 2]}) + return Request(context, scoring, target='target', task='classification') + + +def test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised() -> ( + None +): + request = _direct(200_000) + endpoint = Endpoint() + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model=KumoTabular()) + assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) + assert endpoint.calls == [] + + raised = client(endpoint) + raised._limits = Limits(max_tabular_context_rows=200_000) + raised.predict(request, model=KumoTabular()) + (sent,) = endpoint.calls + assert len(sent['context']['instance_table']['rows']) == 200_000 + + +def test_materialize_samples_to_a_raised_tabular_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + schema = table(12_000) + with pytest.raises(StructuredDataError): + query(schema).materialize(indices=[0, 1], num_context_samples=11_000) + request = query(schema).materialize( + indices=[0, 1], + num_context_samples=11_000, + limits=Limits(max_tabular_context_rows=20_000), + ) + assert request.context_rows == 11_000 + monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '20000') + assert ( + query(schema) + .materialize(indices=[0, 1], num_context_samples=11_000) + .context_rows + == 11_000 + ) + + +def test_a_raised_relational_predict_limit_and_a_lowered_one_set_the_calls() -> ( + None +): + from sdm_client import KumoRelational, PredictiveQuery, Schema + + frame = pd.DataFrame( + {'uid': range(40), 'label': [*range(30), *[None] * 10]} + ) + schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + schema['users']['label'].stype = 'numerical' + asked = PredictiveQuery('PREDICT users.label FOR EACH users.uid', schema) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + lowered = asked.materialize( + indices=list(range(30, 40)), + model=KumoRelational(), + limits=Limits(max_relational_predict_rows=4), + ) + assert [ + len(p['predict']['instance_table']['rows']) for p in lowered.payloads + ] == [ + 4, + 4, + 2, + ] + + +def _relational(predict_rows: int): + return gathered(context_rows=30, run_mode='fast', predict_rows=predict_rows) + + +def _sent(request, limits: Limits) -> list[dict]: + class RelationalEndpoint(Endpoint): + def predict_body(self, body): + payload = json.loads(body) + self.calls.append(payload) + self.sizes.append(len(body)) + table = payload['predict']['instance_table'] + key = table['columns'].index('instance_id') + return json.dumps( + { + 'id': 'answer', + 'model': 'kumo-relational', + 'metadata': {}, + 'predictions': [ + {'id': str(row[key]), 'row_index': i, 'prediction': 1.0} + for i, row in enumerate(table['rows']) + ], + } + ).encode() + + endpoint = RelationalEndpoint() + endpoint.sizes = [] + sending = client(endpoint) + sending._limits = limits + result = sending.predict(request, model='kumo-relational') + assert list(result.to_pandas()['row_index']) == list(range(8)) + return endpoint + + +def test_a_call_past_the_byte_limit_is_split_and_each_part_fits() -> None: + request = _relational(8) + whole = _sent(request, Limits()) + (size,) = whole.sizes + split = _sent(request, Limits(max_request_bytes=size - 1)) + assert len(split.calls) > 1 + assert all(sent <= size - 1 for sent in split.sizes) + assert ( + sum(len(c['predict']['instance_table']['rows']) for c in split.calls) + == 8 + ) + # Every call carries the whole context. + contexts = {json.dumps(c['context'], sort_keys=True) for c in split.calls} + assert len(contexts) == 1 + # At the limit exactly, one call. + assert len(_sent(request, Limits(max_request_bytes=size)).calls) == 1 + + +def test_a_context_too_large_on_its_own_is_refused_naming_the_setting() -> None: + request = _relational(8) + with pytest.raises(StructuredDataError) as refused: + _sent(request, Limits(max_request_bytes=1_000)) + said = str(refused.value) + assert 'context alone' in said + assert 'SDM_CLIENT_MAX_REQUEST_BYTES' in said + assert 'max_request_bytes' in said + + +def test_more_than_six_hops_and_fifteen_tables_are_taken() -> None: + from test_query_engine_boundary import gathered_from, star + + from sdm_client import KumoRelational + + assert len(gathered_from(star(satellites=20, joining=20))) == 21 + request = _relational(1) + assert request.payloads + frame = pd.DataFrame({'uid': range(12), 'label': [*range(10), None, None]}) + from sdm_client import PredictiveQuery, Schema + + schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + schema['users']['label'].stype = 'numerical' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for options in ({'num_hops': 8}, {'num_neighbors': [4] * 9}): + built = PredictiveQuery( + 'PREDICT users.label FOR EACH users.uid', schema + ).materialize(indices=[10, 11], model=KumoRelational(), **options) + assert built.payloads diff --git a/packages/structured-data-models-client/tests/test_context_samples.py b/packages/structured-data-models-client/tests/test_context_samples.py index d41d87c62..89de07d51 100644 --- a/packages/structured-data-models-client/tests/test_context_samples.py +++ b/packages/structured-data-models-client/tests/test_context_samples.py @@ -21,11 +21,14 @@ from test_materialize_options import _materialize from sdm_client.errors import StructuredDataError +from sdm_client.limits import settings from sdm_client.materializers.relational import RelationalMaterializer from sdm_client.models import RELATIONAL_LIMITS FAST = RELATIONAL_LIMITS['context_rows_fast'] -MAX = RELATIONAL_LIMITS['context_rows_max'] +MAX = {setting.name: setting.default for setting in settings()}[ + 'max_relational_context_rows' +] # --- which value goes on the wire ----------------------------------------- diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 33e837fe8..322c3297d 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -24,10 +24,9 @@ import pandas as pd import pytest -from sdm_client import PredictiveQuery, Schema +from sdm_client import Limits, PredictiveQuery, Schema from sdm_client.client import _report_absent from sdm_client.errors import StructuredDataError -from sdm_client.models import KUMO_TABULAR from sdm_client.result import PredictionResult @@ -60,15 +59,21 @@ def query(schema: Schema) -> PredictiveQuery: def test_the_declared_cap_is_the_one_the_endpoint_enforces(): - """Measured at the boundary: 10,000 answer, 10,001 refuse.""" - assert KUMO_TABULAR.runtime.limits['context_rows_max'] == 10_000 + """Measured at the boundary: 10,000 answer, 10,001 refuse; the default + of a setting a deployment may raise.""" + from sdm_client.limits import settings + + defaults = {setting.name: setting.default for setting in settings()} + assert defaults['max_tabular_context_rows'] == 10_000 + assert Limits().resolved()['max_tabular_context_rows'] == 10_000 def test_a_context_above_the_cap_is_refused_before_it_is_encoded(): schema = table(200) with pytest.raises(StructuredDataError) as raised: query(schema).materialize(indices=[0, 1], num_context_samples=50_000) - assert 'accepts at most 10,000 context rows' in str(raised.value) + assert 'carries at most 10,000 context rows' in str(raised.value) + assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(raised.value) def test_the_cap_itself_is_allowed(): @@ -226,7 +231,8 @@ def test_a_materialized_context_above_the_cap_is_refused_not_merely_the_number() schema = table(12_000) with pytest.raises(StructuredDataError) as raised: query(schema).materialize(indices=[0, 1], num_context_samples=10_001) - assert 'accepts at most 10,000 context rows' in str(raised.value) + assert 'carries at most 10,000 context rows' in str(raised.value) + assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(raised.value) def test_the_predict_route_advises_something_that_route_allows(): diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 9940e7c6f..2001f5196 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -57,16 +57,18 @@ def _Gathered(*payloads): def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): """A pin, not a probe. - These numbers were read from the relational NIM. This asserts they have - not drifted in the client, which is all it can do offline. The live suites - check that the endpoint still agrees. + These numbers were read from the relational NIM, as the defaults of the + settings a deployment may raise. This asserts they have not drifted in + the client, which is all it can do offline. The live suites check that + the endpoint still agrees. No most is set on tables or hops. """ - assert RELATIONAL_LIMITS['context_rows_fast'] == 1_000 - assert RELATIONAL_LIMITS['context_rows_max'] == 10_000 - assert RELATIONAL_LIMITS['predict_rows_max'] == 10_000 - assert RELATIONAL_LIMITS['predict_rows_temporal_link_prediction'] == 200 - assert RELATIONAL_LIMITS['hops_max'] == 6 - assert RELATIONAL_LIMITS['tables_max'] == 15 + from sdm_client.limits import settings + + defaults = {setting.name: setting.default for setting in settings()} + assert RELATIONAL_LIMITS == {'context_rows_fast': 1_000, 'classes_max': 10} + assert defaults['max_relational_context_rows'] == 10_000 + assert defaults['max_relational_predict_rows'] == 10_000 + assert defaults['max_temporal_link_predict_rows'] == 200 @pytest.mark.parametrize('bad', ['2', True]) @@ -247,7 +249,7 @@ def test_the_context_cap_is_held_to_each_call_and_not_their_sum(over): """Both sides of the cap, because only the allowed one was covered: a per-call count that always read nought would have passed that alone.""" - cap = RELATIONAL_LIMITS['context_rows_max'] + cap = 10_000 each = cap + 1 if over else cap // 2 + 1 request = gathered_in_calls(calls=2, context_rows=each) if not over: diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index deccb4296..23836abf8 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -193,9 +193,11 @@ def test_the_endpoint_scores_what_this_client_says_it_does(client, query): """ import copy - from sdm_client.models import RELATIONAL_LIMITS + from sdm_client.limits import settings - cap = RELATIONAL_LIMITS['predict_rows_max'] + cap = {setting.name: setting.default for setting in settings()}[ + 'max_relational_predict_rows' + ] payload = copy.deepcopy(query.materialize(indices=[0, 1]).payloads[0]) rows = payload['predict']['instance_table']['rows'] diff --git a/packages/structured-data-models-client/tests/test_materialize_options.py b/packages/structured-data-models-client/tests/test_materialize_options.py index 0c2e91ec5..72698572f 100644 --- a/packages/structured-data-models-client/tests/test_materialize_options.py +++ b/packages/structured-data-models-client/tests/test_materialize_options.py @@ -372,7 +372,7 @@ def _refusal_of(**options: Any) -> StructuredDataError: def test_a_limit_is_raised_as_the_subclass_on_the_core_register_p1b_20() -> ( None ): - for options in ({'num_context_samples': 10001}, {'num_hops': 7}): + for options in ({'num_context_samples': 10001},): core = _refusal_of(**options) assert core.code == 'UNSUPPORTED_REQUEST', options assert type(core).__name__ == 'UnsupportedRequestError', options diff --git a/packages/structured-data-models-client/tests/test_materialize_state.py b/packages/structured-data-models-client/tests/test_materialize_state.py index fc0588434..20f372cbe 100644 --- a/packages/structured-data-models-client/tests/test_materialize_state.py +++ b/packages/structured-data-models-client/tests/test_materialize_state.py @@ -602,8 +602,8 @@ def test_a_call_the_core_refuses_before_its_check_leaves_it_to_the_next() -> ( [query], connection, indices=[0, 1], - num_hops=7, - expected_error='UNSUPPORTED_REQUEST', + num_hops=0, + expected_error='INVALID_REQUEST', ) == [(0, 0)] assert _calls([query], connection, indices=[0, 1]) == [(6, 1)] other = PredictiveQuery(QUERY, schema) diff --git a/packages/structured-data-models-client/tests/test_payload_rules.py b/packages/structured-data-models-client/tests/test_payload_rules.py index 8e1fac88c..efb815def 100644 --- a/packages/structured-data-models-client/tests/test_payload_rules.py +++ b/packages/structured-data-models-client/tests/test_payload_rules.py @@ -189,6 +189,19 @@ def test_vectors_cover_required_success_and_failure_cases() -> None: CASES = RECORDED['cases'] +#: The recorded cases this client departs from the reference on, on purpose, +#: and why: it gathers each of them where the reference refused. +DEPARTED = { + # An earlier endpoint read at most 15 tables; the model reads as many as + # a request carries, so the client sets no most. + 'star.tables-16': 'no table limit', + # A call past the byte limit that scores several rows is split into + # smaller calls when it is predicted; only a context too large on its + # own is refused at materialize. + 'size.one-past-the-limit': 'split by bytes', +} + + @pytest.mark.parametrize('spec', CASES, ids=[c['name'] for c in CASES]) def test_the_core_s_payload_rules_are_the_reference_s( spec: dict[str, Any], monkeypatch: pytest.MonkeyPatch @@ -197,6 +210,11 @@ def test_the_core_s_payload_rules_are_the_reference_s( got, attachments = _on_the_core(spec, monkeypatch) covers = spec['covers'] + if spec['name'] in DEPARTED: + assert expected['outcome'] == 'refused' + assert got['outcome'] == 'payload', got + return + if covers == R.AVG_NULL: # The reference crashes (INTERNAL_ERROR); the core refuses it by # name and does not accept it. @@ -279,22 +297,20 @@ def test_the_core_s_payload_rules_are_the_reference_s( pytest.fail(f'no exit named: {covers}') -def test_a_payload_past_the_byte_limit_is_refused_by_name( +def test_a_payload_past_the_byte_limit_is_gathered_to_be_split( monkeypatch: pytest.MonkeyPatch, ) -> None: - # PD-2, past the limit: the core's refusal is named (a rule or a code - # the reference gives), never a crash, and the case before it is taken. + # PD-2, past the limit: a payload of two rows whose context fits with + # one is gathered, to be asked as smaller calls when it is predicted. past = next(c for c in CASES if c['name'] == 'size.one-past-the-limit') got, attachments = _on_the_core(past, monkeypatch) - assert got['outcome'] == 'refused' - assert not _not_held(got), got - assert got['code'] == past['expected']['code'] - assert not attachments + assert got['outcome'] == 'payload', got + assert len(attachments) == 1 -@pytest.mark.parametrize('padding, accepted', [(5, True), (8, False)]) +@pytest.mark.parametrize('padding, within', [(5, True), (8, False)]) def test_client_only_provenance_does_not_consume_model_payload_limit( - padding, accepted, monkeypatch + padding, within, monkeypatch ): import copy @@ -305,13 +321,13 @@ def test_client_only_provenance_does_not_consume_model_payload_limit( f"UPDATE users SET note=note||'{'a' * padding}' WHERE user_id=0" ) got, attachments = _on_the_core(spec, monkeypatch) - if accepted: - assert got['outcome'] == 'payload', got - state = bytes(attachments[0]) - prefix = b'{"client_generated_keys":{"orders":"__node_id"},' - assert state.startswith(prefix) - model_bytes = b'{' + state[len(prefix) :] + # Past the limit, the two rows are gathered to be split when predicted. + assert got['outcome'] == 'payload', got + state = bytes(attachments[0]) + prefix = b'{"client_generated_keys":{"orders":"__node_id"},' + assert state.startswith(prefix) + model_bytes = b'{' + state[len(prefix) :] + if within: assert len(model_bytes) <= R.PAYLOAD_LIMIT < len(state) else: - assert got['outcome'] == 'refused' - assert '30MB' in got['message'] + assert R.PAYLOAD_LIMIT < len(model_bytes) diff --git a/packages/structured-data-models-client/tests/test_predict_result.py b/packages/structured-data-models-client/tests/test_predict_result.py index 3cbdfc196..6a53d1c76 100644 --- a/packages/structured-data-models-client/tests/test_predict_result.py +++ b/packages/structured-data-models-client/tests/test_predict_result.py @@ -285,14 +285,12 @@ def test_the_core_s_predict_is_the_reference_s( ) return if spec['covers'] == R.SIZE: - # The reference's code, and a message that explains the limit as - # the old route's does (it names 30MB). - assert got['outcome'] == 'refused', 'past the limit, taken' - assert got['code'] == expected['code'], got['message'] + # The reference refused a payload of two rows past 30MB; this client + # gathers it, since its context fits with one row, and holds each + # call it sends to the limit, split where it is past it, by design. + assert expected['outcome'] == 'refused' assert '30MB' in expected['message'] - assert re.search(r'30 ?MB|30 ?MiB|31,?457,?280', got['message']), ( - f'the size refusal does not explain the limit: {got["message"]!r}' - ) + assert got['outcome'] == 'result', got.get('message') return if spec['name'] in { 'prepare.regression.outputs-explanation-two-rows', diff --git a/packages/structured-data-models-client/tests/test_query_engine_boundary.py b/packages/structured-data-models-client/tests/test_query_engine_boundary.py index 46b3393df..365e2bb0f 100644 --- a/packages/structured-data-models-client/tests/test_query_engine_boundary.py +++ b/packages/structured-data-models-client/tests/test_query_engine_boundary.py @@ -731,22 +731,18 @@ def gathered_from(built: Schema) -> set[str]: return set(request.payloads[0]['schema']['related_tables']) -def test_a_schema_wider_than_the_cap_is_gathered_when_its_subgraph_is_not(): - """21 declared tables, 6 gathered, and the endpoint reads 15.""" - from sdm_client.models import RELATIONAL_LIMITS - +def test_a_schema_wider_than_fifteen_tables_is_gathered_as_its_subgraph(): + """21 declared tables, 6 gathered: what is sent is what the query reaches.""" built = star(satellites=20, joining=5) - assert len(built) > RELATIONAL_LIMITS['tables_max'] + assert len(built) > 15 assert len(gathered_from(built)) == 6 -def test_a_subgraph_wider_than_the_cap_is_still_refused(): - from sdm_client.models import RELATIONAL_LIMITS - - with pytest.raises(StructuredDataError) as error: - gathered_from(star(satellites=20, joining=20)) - assert str(RELATIONAL_LIMITS['tables_max']) in str(error.value) +def test_a_subgraph_wider_than_fifteen_tables_is_gathered_whole(): + """An earlier endpoint read at most 15 tables; the model reads as many as + a request carries, so a subgraph of 21 is gathered rather than refused.""" + assert len(gathered_from(star(satellites=20, joining=20))) == 21 # --- a table the request leaves out is said to be left out ------------------ diff --git a/packages/structured-data-models-client/tests/test_serving_route.py b/packages/structured-data-models-client/tests/test_serving_route.py index a8bc2287f..39257d628 100644 --- a/packages/structured-data-models-client/tests/test_serving_route.py +++ b/packages/structured-data-models-client/tests/test_serving_route.py @@ -395,16 +395,29 @@ def test_malformed_serving_response_is_a_transport_error( assert len(received) == 1 -def test_a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() -> None: +def test_a_call_past_the_caller_s_limit_is_split_and_a_context_past_it_refused() -> ( + None +): request = _materialized() _, received = _predict(request) [(_, body, _)] = received - # The limit applies to the actual UTF-8 envelope posted to the SDK. - core, on_core = _predict(request, max_request_bytes=len(body) - 1) + # The limit applies to the actual UTF-8 envelope posted to the SDK: one + # byte under the call, its two rows are asked as two calls, each with + # the whole context and within the limit. + split, on_core = _predict(request, max_request_bytes=len(body) - 1) + assert not isinstance(split, Exception), split + assert len(on_core) == 2 + assert all(len(sent) <= len(body) - 1 for _, sent, _ in on_core) + assert {json.dumps(_call(sent)['context']) for _, sent, _ in on_core} == { + json.dumps(_call(body)['context']) + } + # A limit no call of one row fits: refused, naming the context's size, + # and nothing sent. + core, on_core = _predict(request, max_request_bytes=100) assert on_core == [] assert isinstance(core, StructuredDataError), core - assert core.details['limit'] == len(body) - 1 - assert core.details['bytes'] == len(body) + assert core.details['limit'] == 100 + assert core.details['context_bytes'] < core.details['bytes'] < len(body) # The size itself is admitted. core, received = _predict(request, max_request_bytes=len(body)) assert not isinstance(core, Exception), core @@ -426,11 +439,11 @@ def test_utf8_serving_preserves_public_payload_and_measures_the_sent_body() -> ( assert call[field] == request.payloads[0][field], field assert result.to_pandas()['id'].tolist() == [0, 1] assert result.to_pandas()['prediction'].tolist() == [1.0, 1.0] - refused, sent = _predict(request, max_request_bytes=len(body) - 1) - assert isinstance(refused, StructuredDataError) - assert sent == [] - assert refused.details['bytes'] == len(body) - assert refused.details['limit'] == len(body) - 1 + split, sent = _predict(request, max_request_bytes=len(body) - 1) + assert not isinstance(split, Exception), split + assert len(sent) == 2 + assert all('é😀'.encode() in part for _, part, _ in sent) + assert split.to_pandas()['id'].tolist() == [0, 1] @pytest.mark.parametrize( diff --git a/packages/structured-data-models-client/tests/test_the_predict_route.py b/packages/structured-data-models-client/tests/test_the_predict_route.py index ad1bb939b..95aa9fc7a 100644 --- a/packages/structured-data-models-client/tests/test_the_predict_route.py +++ b/packages/structured-data-models-client/tests/test_the_predict_route.py @@ -768,14 +768,11 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( # core's call as the core wrote it, so the bytes the endpoint receives # are the bytes the core counted against the limit, characters past # ASCII as their own bytes, never json.dumps' escapes. At the limit's - # edge the call is sent, at most the limit; one character past it, - # nothing is sent. Since the reference's 26ecdd28 the call - # leaves out the engine's inference config, which the payload holds, - # so the payload is the larger and meets the limit first: the edge is - # the payload's, which materialize names in its refusal, and one past - # it materialize refuses (the call's own refusal, past the limit with - # the payload within it, is sdm-core's - # a_call_past_the_limit_is_refused_before_it_is_sent). + # edge the call is sent whole, at most the limit; one character past + # it, the call's two rows are asked as smaller calls, each within it. + # The payload, which holds the engine's inference config the call + # leaves out, may pass the limit where the call does not: materialize + # gathers it all the same, since a call of one row fits. limit = 30 * 1024 * 1024 lengths = [] for emoji, ascii in ((0, 0), (1, 0), (0, 1)): @@ -792,31 +789,23 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( assert wide % 4 == 0 and narrow == 1 emoji = (limit - base) // wide ascii = (limit - base - emoji * wide) // narrow - # The call's edge is past the payload's: materialize refuses it, naming - # the payload's size, and the payload's edge is that much nearer. - with pytest.raises(StructuredDataError) as over: - _emoji_request(monkeypatch, emoji, ascii) - details = over.value.details - assert over.value.code == 'INVALID_REQUEST' - assert details['field'] == 'payload' and details['limit'] == limit - excess = details['bytes'] - limit - assert 0 < excess < wide * 64 - room = emoji * wide + ascii * narrow - excess - emoji = room // wide - ascii = (room - emoji * wide) // narrow written, received, refused = _sent( monkeypatch, _emoji_request(monkeypatch, emoji, ascii) ) assert refused.code == 'RECORDED' assert written == received - assert len(received[0]) == base + emoji * wide + ascii * narrow <= limit + assert len(received[0]) == base + emoji * wide + ascii * narrow == limit assert b'\\ud83d' not in received[0] assert '\U0001f600'.encode() in received[0] - with pytest.raises(StructuredDataError) as past: - _emoji_request(monkeypatch, emoji, ascii + 1) - assert past.value.code == 'INVALID_REQUEST' - assert past.value.details['field'] == 'payload' - assert past.value.details['bytes'] == limit + 1 + # One byte past: split into calls of one row each, which share their + # context, so the first post pins that context as a session for them; + # the endpoint refuses it before anything else is sent. + _, received, refused = _sent( + monkeypatch, _emoji_request(monkeypatch, emoji, ascii + 1) + ) + assert refused.code == 'RECORDED' + [first] = received + assert json.loads(first)['metadata']['num_prediction_examples'] == 1 def test_embeddings_are_the_old_route_s_on_the_core( diff --git a/packages/structured-data-models-client/tests/test_wording.py b/packages/structured-data-models-client/tests/test_wording.py index 12ac5eef7..9191dbb1f 100644 --- a/packages/structured-data-models-client/tests/test_wording.py +++ b/packages/structured-data-models-client/tests/test_wording.py @@ -522,8 +522,10 @@ def test_the_context_size_refused_is_named_as_the_caller_gave_it() -> None: with pytest.raises(StructuredDataError) as past: query.materialize(indices=[1], num_context_samples=10_001) assert str(past.value).split('] ', 1)[1] == ( - 'num_context_samples is 10,001, and the endpoint accepts at most ' - '10,000 context rows' + 'num_context_samples is 10,001, and a request carries at most 10,000 ' + 'context rows; if the endpoint takes more, raise it with ' + 'limits=Limits(max_relational_context_rows=...) or the environment ' + 'variable SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS' ) with pytest.raises(StructuredDataError) as below: query.materialize(indices=[1], num_context_samples=-(10**100)) diff --git a/rust/bindings/sdm-jni/src/lib.rs b/rust/bindings/sdm-jni/src/lib.rs index cdda3cde4..8e0dd1b13 100644 --- a/rust/bindings/sdm-jni/src/lib.rs +++ b/rust/bindings/sdm-jni/src/lib.rs @@ -456,6 +456,17 @@ pub extern "system" fn Java_com_nvidia_sdm_internal_NativeCore_info0<'caller>( "wire_names": sdm_host::wire_names() .into_iter() .collect::>(), + // Each size limit a caller may set, in order: its name, the + // environment variable a host reads it from, and its default. + "limits": sdm_contract::limits::SETTINGS + .iter() + .map(|setting| serde_json::json!({ + "name": setting.name, + "env": setting.env, + "default": setting.default, + "about": setting.about, + })) + .collect::>(), }); let bytes = serde_json::to_vec(&info).map_err(|_| Failure::Usage("the library's description"))?; diff --git a/rust/bindings/sdm-py/src/lib.rs b/rust/bindings/sdm-py/src/lib.rs index b1bfabb91..ee702d2b9 100644 --- a/rust/bindings/sdm-py/src/lib.rs +++ b/rust/bindings/sdm-py/src/lib.rs @@ -651,6 +651,19 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { .collect::>>()?, )?, )?; + // Each size limit a caller may set (`sdm_contract::limits::SETTINGS`): + // its name, the environment variable a host reads it from, and its + // default, the one table of them every host reads. + module.add( + "LIMITS", + PyTuple::new( + py, + sdm_contract::limits::SETTINGS + .iter() + .map(|setting| (setting.name, setting.env, setting.default, setting.about)) + .collect::>(), + )?, + )?; // Each data type name a host may give a column, and the contract's name // of the type it names: the one table of them. module.add( diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 73a67e69f..a3aaa6624 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -71 sha256:6041dbc10ec757561b56eb6cc0decdedf1c9e7f7e1268e641e03401b492ea8be sdm-contract-71.json +72 sha256:96fd2d685c95ceee80d653d48e3dd022e64530ec3257e2a0406233394f001faf sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-71.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json similarity index 99% rename from rust/crates/sdm-contract/schema/sdm-contract-71.json rename to rust/crates/sdm-contract/schema/sdm-contract-72.json index dcd05e49d..851c4ab3e 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-71.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -1,5 +1,5 @@ { - "$id": "sdm-contract/71", + "$id": "sdm-contract/72", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ControlMessage", "description": "Every control message that crosses between a host and the core.", @@ -17,6 +17,10 @@ "facts": { "$ref": "#/$defs/HostFacts" }, + "limits": { + "description": "The size limits the caller set and the environment names, which the\ncore resolves before the run starts (contract v72); left out, each\nlimit is its default ([`crate::limits`]).", + "$ref": "#/$defs/GivenLimits" + }, "operation": { "$ref": "#/$defs/Operation" }, @@ -312,6 +316,12 @@ "DirectValue.check_tabular_cap": { "$ref": "#/$defs/CheckedTabularCap" }, + "ResolveLimits.input": { + "$ref": "#/$defs/ResolveLimits" + }, + "DirectValue.resolve_limits": { + "$ref": "#/$defs/Limits" + }, "ObserveTabularMembership.input": { "$ref": "#/$defs/ObserveTabularMembership" }, @@ -701,6 +711,8 @@ "DirectValue.select_tabular_rows", "CheckTabularCap.input", "DirectValue.check_tabular_cap", + "ResolveLimits.input", + "DirectValue.resolve_limits", "ObserveTabularMembership.input", "DirectValue.observe_tabular_membership", "DecideTabularTask.input", @@ -3831,6 +3843,50 @@ "accepted" ] }, + "ResolveLimits": { + "description": "The input of `resolve_limits`, which reads nothing but the run's\nlimits: none.", + "type": "object", + "additionalProperties": false + }, + "Limits": { + "description": "The limits a run holds a request to, resolved; `resolve_limits` answers\nthem, so a host can say what its caller's settings came to.", + "type": "object", + "properties": { + "max_relational_context_rows": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_relational_predict_rows": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_temporal_link_predict_rows": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_tabular_context_rows": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_request_bytes": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_relational_context_rows", + "max_relational_predict_rows", + "max_temporal_link_predict_rows", + "max_tabular_context_rows", + "max_request_bytes" + ] + }, "ObserveTabularMembership": { "type": "object", "properties": { @@ -7954,7 +8010,7 @@ ] }, "GivenCandidate": { - "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\nits own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\nholds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\nfixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\ntext the reference writes into its question (in Python, `str` of it);\nit is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\nno scalar holds, written into its question by its text as `spelled`\nis, with the name of the kind of value it is (`date`, `datetime` for a\ntimestamp with no zone, `datetime_zoned` for one with one, `time`).\nTwo of one kind and one text are one candidate; a host whose language\nholds others equal (two zones of one instant) says so with `repeat`\n.\n- `repeat`: a value the host's own language holds equal to one given\nbefore it, left out as the reference's `dict.fromkeys` leaves it out.\nA host whose candidates are all scalars sends none, the core keeping\neach distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\none it cannot hash), which names no row and which the core refuses by\nname, as the reference fails on it.", + "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\nits own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\nholds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\nfixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\ntext the reference writes into its question (in Python, `str` of it);\nit is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\nno scalar holds, written into its question by its text as `spelled`\nis, with the name of the kind of value it is (`date`, `datetime` for a\ntimestamp with no zone, `datetime_zoned` for one with one, `time`).\nTwo of one kind and one text are one candidate; a host whose language\nholds others equal (two zones of one instant) says so with `repeat`\n .\n- `repeat`: a value the host's own language holds equal to one given\nbefore it, left out as the reference's `dict.fromkeys` leaves it out.\nA host whose candidates are all scalars sends none, the core keeping\neach distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\none it cannot hash), which names no row and which the core refuses by\nname, as the reference fails on it.", "oneOf": [ { "type": "string", @@ -10915,7 +10971,7 @@ ] }, "NamedTable": { - "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads (design choice 12), which a host naming a table\nby a string gives as one part.", + "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads, which a host naming a table\nby a string gives as one part.", "type": "object", "properties": { "name": { @@ -10993,7 +11049,7 @@ "type": "string" }, "source": { - "description": "The relation of the source this table reads (design choice 12): its\nown name, one part, unless the caller named it otherwise (decision\nE59).", + "description": "The relation of the source this table reads: its\nown name, one part, unless the caller named it otherwise (decision\nE59).", "$ref": "#/$defs/Relation" }, "columns": { @@ -12693,6 +12749,29 @@ "max_ns" ] }, + "GivenLimits": { + "description": "What a host passes with every run (contract v72): the values its caller\nset, by [`Setting::name`], and the text of each setting's environment\nvariable that is set, by [`Setting::env`], each as the host read it.\nLeft out, every limit is its default.", + "type": "object", + "properties": { + "given": { + "description": "The values the caller set, each a whole number of at least one.", + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "environment": { + "description": "The environment variables set, each as its text.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + }, "Operation": { "description": "One operation's request.", "oneOf": [ @@ -13707,6 +13786,24 @@ "input" ] }, + { + "description": "The size limits the run holds requests to, as resolved from its\nstart's (contract v72).", + "type": "object", + "properties": { + "input": { + "$ref": "#/$defs/AttachmentId" + }, + "op": { + "type": "string", + "const": "resolve_limits" + } + }, + "additionalProperties": false, + "required": [ + "op", + "input" + ] + }, { "description": "An input-sized direct value, carried as typed JSON bytes.", "type": "object", @@ -14740,7 +14837,7 @@ "$ref": "#/$defs/Determinism" }, "max_seed": { - "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n(design choice 1). The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", + "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n. The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", "type": "integer", "format": "uint64", "minimum": 0 @@ -14779,7 +14876,7 @@ ] }, "fallback": { - "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads (design choice 1, the in-memory fallback); left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", + "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads; left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", "anyOf": [ { "$ref": "#/$defs/FallbackLimits" @@ -17682,9 +17779,9 @@ "explanation_fast_mode", "options_unread", "options_unread_statistic", - "payload_smaller_calls", "payload_no_smaller_call", "call_past_endpoint", + "direct_call_past_bytes", "outputs_none", "not_finite", "past_int64", @@ -17720,8 +17817,6 @@ "statistic_not_a_regression", "no_hop_num_hops", "no_hop_num_neighbors", - "hops_past_endpoint", - "hops_past_endpoint_uncounted", "inference_run_mode_invalid", "inference_config_not_an_object", "inference_option_unknown", @@ -17769,7 +17864,6 @@ "frame_key_missing_parts", "frame_no_valid_rows", "not_in_table", - "subgraph_past_tables", "unresolved", "anchor_not_a_time", "anchor_before_data", @@ -17829,6 +17923,8 @@ "large_text_materialize", "admitted_past_type", "invalid_configuration", + "limit_environment_invalid", + "limit_given_below_one", "part_forecast_step", "part_forecast_step_outran", "part_ranking_candidate", @@ -18010,7 +18106,7 @@ } }, "answered": { - "description": "What a request asked in several model calls had been answered when\na later call failed (design choice 1, `main`'s answered parts,\n): the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", + "description": "What a request asked in several model calls had been answered when\na later call failed: the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", "anyOf": [ { "$ref": "#/$defs/AnsweredParts" diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index e2a198142..d30489c27 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -1013,9 +1013,9 @@ sentences! { ExplanationFastMode = "explanation_fast_mode", E::UnsupportedRequest; OptionsUnread = "options_unread", E::UnsupportedRequest; OptionsUnreadStatistic = "options_unread_statistic", E::UnsupportedRequest; - PayloadSmallerCalls = "payload_smaller_calls", E::InvalidRequest; PayloadNoSmallerCall = "payload_no_smaller_call", E::InvalidRequest; CallPastEndpoint = "call_past_endpoint", E::InvalidRequest; + DirectCallPastBytes = "direct_call_past_bytes", E::InvalidRequest; OutputsNone = "outputs_none", E::InvalidRequest; NotFinite = "not_finite", E::InvalidRequest; PastInt64 = "past_int64", E::InvalidRequest; @@ -1051,8 +1051,6 @@ sentences! { StatisticNotARegression = "statistic_not_a_regression", E::InvalidRequest; NoHopNumHops = "no_hop_num_hops", E::InvalidRequest; NoHopNumNeighbors = "no_hop_num_neighbors", E::InvalidRequest; - HopsPastEndpoint = "hops_past_endpoint", E::UnsupportedRequest; - HopsPastEndpointUncounted = "hops_past_endpoint_uncounted", E::UnsupportedRequest; InferenceRunModeInvalid = "inference_run_mode_invalid", E::InvalidRequest; InferenceConfigNotAnObject = "inference_config_not_an_object", E::InvalidRequest; InferenceOptionUnknown = "inference_option_unknown", E::InvalidRequest; @@ -1100,7 +1098,6 @@ sentences! { FrameKeyMissingParts = "frame_key_missing_parts", E::InvalidRequest; FrameNoValidRows = "frame_no_valid_rows", E::InvalidRequest; NotInTable = "not_in_table", E::InvalidRequest; - SubgraphPastTables = "subgraph_past_tables", E::InvalidRequest; Unresolved = "unresolved", E::InvalidRequest; AnchorNotATime = "anchor_not_a_time", E::InvalidRequest; AnchorBeforeData = "anchor_before_data", E::InvalidRequest; @@ -1160,6 +1157,8 @@ sentences! { LargeTextMaterialize = "large_text_materialize", E::UnsupportedType; AdmittedPastType = "admitted_past_type", E::ValueOutOfRange; InvalidConfiguration = "invalid_configuration", E::InvalidConfiguration; + LimitEnvironmentInvalid = "limit_environment_invalid", E::InvalidConfiguration; + LimitGivenBelowOne = "limit_given_below_one", E::InvalidConfiguration; PartForecastStep = "part_forecast_step", Code::Part; PartForecastStepOutran = "part_forecast_step_outran", Code::Part; PartRankingCandidate = "part_ranking_candidate", Code::Part; diff --git a/rust/crates/sdm-contract/src/lib.rs b/rust/crates/sdm-contract/src/lib.rs index 024697026..8151ff5b0 100644 --- a/rust/crates/sdm-contract/src/lib.rs +++ b/rust/crates/sdm-contract/src/lib.rs @@ -27,6 +27,7 @@ pub mod examples; pub mod facts; pub mod frames; pub mod labels; +pub mod limits; pub mod ops; pub mod pql; pub mod rebind; @@ -56,7 +57,7 @@ pub use scalar::Scalar; pub use types::{ColumnType, Field, Unit}; /// The contract version this build speaks. -pub const CONTRACT_VERSION: u32 = 71; +pub const CONTRACT_VERSION: u32 = 72; /// The JSON Schema of [`ControlMessage`], as committed: pretty-printed, with /// a trailing newline. Its `$defs` hold [`ops::Attachments`] too, the JSON diff --git a/rust/crates/sdm-contract/src/limits.rs b/rust/crates/sdm-contract/src/limits.rs new file mode 100644 index 000000000..adbfcbe69 --- /dev/null +++ b/rust/crates/sdm-contract/src/limits.rs @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The size limits a client holds a request to before it is sent, each a +//! setting a caller may raise or lower to match the deployment it sends to. +//! +//! Each limit mirrors one the serving layer enforces, so the client refuses +//! or batches first and names the limit, rather than sending a request the +//! endpoint refuses without saying how to meet it. A deployment configures +//! its own (the serving layer's `TFM_MAX_TRAIN_ROWS`, `TFM_MAX_TEST_ROWS` +//! and `TFM_MAX_PAYLOAD_BYTES`), so these are settings with the serving +//! layer's defaults, not constants. +//! +//! [`SETTINGS`] is the one table of them: each setting's name, the +//! environment variable a host reads it from, and its default. A host +//! passes what its caller gave and what its environment holds with every +//! run ([`GivenLimits`]), reading nothing else; the core resolves them, +//! the caller's value before the environment's before the default, and +//! refuses a value that is not a whole number of at least one by name. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::error::{CoreError, Detail}; + +/// The prefix of every setting's environment variable. +pub const ENVIRONMENT_PREFIX: &str = "SDM_CLIENT_"; + +/// One limit a caller may set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Setting { + /// The setting's name, as a host's argument names it. + pub name: &'static str, + /// The environment variable a host reads it from, the name uppercased + /// after [`ENVIRONMENT_PREFIX`]. + pub env: &'static str, + /// The value where neither the caller nor the environment names one. + pub default: u64, + /// What the limit holds a request to. + pub about: &'static str, +} + +/// The most context rows `kumo-relational` reads: the serving layer's +/// `TFM_MAX_TRAIN_ROWS` default. +pub const MAX_RELATIONAL_CONTEXT_ROWS: Setting = Setting { + name: "max_relational_context_rows", + env: "SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS", + default: 10_000, + about: "the most context rows a relational request carries; num_context_samples asks for at most this many", +}; + +/// The most rows one `kumo-relational` call scores: the serving layer's +/// `TFM_MAX_TEST_ROWS` default. A request of more is asked in several calls. +pub const MAX_RELATIONAL_PREDICT_ROWS: Setting = Setting { + name: "max_relational_predict_rows", + env: "SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS", + default: 10_000, + about: "the most rows one relational call scores; more are asked in several calls", +}; + +/// The most rows one temporal link prediction call scores. +pub const MAX_TEMPORAL_LINK_PREDICT_ROWS: Setting = Setting { + name: "max_temporal_link_predict_rows", + env: "SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS", + default: 200, + about: "the most rows one temporal link prediction call scores; more are asked in several calls", +}; + +/// The most context rows `kumo-tabular` reads: the serving layer's +/// `TFM_MAX_TRAIN_ROWS` default. +pub const MAX_TABULAR_CONTEXT_ROWS: Setting = Setting { + name: "max_tabular_context_rows", + env: "SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS", + default: 10_000, + about: "the most context rows a single-table request carries; materialize() samples down to it", +}; + +/// The most bytes one call's body takes, as written and sent: the serving +/// layer's `TFM_MAX_PAYLOAD_BYTES` default, 30 MiB. A batch of calls past it +/// is asked as smaller calls; a context past it on its own is refused. +pub const MAX_REQUEST_BYTES: Setting = Setting { + name: "max_request_bytes", + env: "SDM_CLIENT_MAX_REQUEST_BYTES", + default: 30 * 1024 * 1024, + about: "the most bytes one call's body takes; a call past it is split into smaller calls", +}; + +impl Setting { + /// `error`, naming this setting and its environment variable, which + /// its sentence tells the caller to raise (`setting`, `env`). + pub fn named(&self, error: CoreError) -> CoreError { + error + .with_field("setting", Detail::text(self.name)) + .with_field("env", Detail::text(self.env)) + } +} + +/// Every setting, in the order a host lists them. +pub const SETTINGS: [Setting; 5] = [ + MAX_RELATIONAL_CONTEXT_ROWS, + MAX_RELATIONAL_PREDICT_ROWS, + MAX_TEMPORAL_LINK_PREDICT_ROWS, + MAX_TABULAR_CONTEXT_ROWS, + MAX_REQUEST_BYTES, +]; + +/// The setting named `name`, if there is one. +pub fn setting(name: &str) -> Option<&'static Setting> { + SETTINGS.iter().find(|setting| setting.name == name) +} + +/// What a host passes with every run (contract v72): the values its caller +/// set, by [`Setting::name`], and the text of each setting's environment +/// variable that is set, by [`Setting::env`], each as the host read it. +/// Left out, every limit is its default. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GivenLimits { + /// The values the caller set, each a whole number of at least one. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub given: BTreeMap, + /// The environment variables set, each as its text. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub environment: BTreeMap, +} + +/// The input of `resolve_limits`, which reads nothing but the run's +/// limits: none. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ResolveLimits {} + +/// The limits a run holds a request to, resolved; `resolve_limits` answers +/// them, so a host can say what its caller's settings came to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Limits { + pub max_relational_context_rows: u64, + pub max_relational_predict_rows: u64, + pub max_temporal_link_predict_rows: u64, + pub max_tabular_context_rows: u64, + pub max_request_bytes: u64, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + max_relational_context_rows: MAX_RELATIONAL_CONTEXT_ROWS.default, + max_relational_predict_rows: MAX_RELATIONAL_PREDICT_ROWS.default, + max_temporal_link_predict_rows: MAX_TEMPORAL_LINK_PREDICT_ROWS.default, + max_tabular_context_rows: MAX_TABULAR_CONTEXT_ROWS.default, + max_request_bytes: MAX_REQUEST_BYTES.default, + } + } +} + +impl Limits { + /// The value of the setting named `name`, if there is one. + pub fn get(&self, name: &str) -> Option { + Some(match name { + n if n == MAX_RELATIONAL_CONTEXT_ROWS.name => self.max_relational_context_rows, + n if n == MAX_RELATIONAL_PREDICT_ROWS.name => self.max_relational_predict_rows, + n if n == MAX_TEMPORAL_LINK_PREDICT_ROWS.name => self.max_temporal_link_predict_rows, + n if n == MAX_TABULAR_CONTEXT_ROWS.name => self.max_tabular_context_rows, + n if n == MAX_REQUEST_BYTES.name => self.max_request_bytes, + _ => return None, + }) + } + + /// The setting named `name` set to `value`; false where there is none. + pub fn set(&mut self, name: &str, value: u64) -> bool { + let slot = match name { + n if n == MAX_RELATIONAL_CONTEXT_ROWS.name => &mut self.max_relational_context_rows, + n if n == MAX_RELATIONAL_PREDICT_ROWS.name => &mut self.max_relational_predict_rows, + n if n == MAX_TEMPORAL_LINK_PREDICT_ROWS.name => { + &mut self.max_temporal_link_predict_rows + } + n if n == MAX_TABULAR_CONTEXT_ROWS.name => &mut self.max_tabular_context_rows, + n if n == MAX_REQUEST_BYTES.name => &mut self.max_request_bytes, + _ => return false, + }; + *slot = value; + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_setting_is_named_once_and_read_from_its_own_variable() { + let mut names: Vec<&str> = SETTINGS.iter().map(|s| s.name).collect(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), SETTINGS.len()); + for setting in SETTINGS { + assert_eq!( + setting.env, + format!("{ENVIRONMENT_PREFIX}{}", setting.name.to_ascii_uppercase()) + ); + assert!(setting.default >= 1); + assert_eq!(Limits::default().get(setting.name), Some(setting.default)); + let mut limits = Limits::default(); + assert!(limits.set(setting.name, setting.default + 1)); + assert_eq!(limits.get(setting.name), Some(setting.default + 1)); + } + assert_eq!(Limits::default().get("max_tables"), None); + assert!(!Limits::default().set("max_tables", 1)); + } + + #[test] + fn the_defaults_are_the_serving_layers() { + let limits = Limits::default(); + assert_eq!(limits.max_relational_context_rows, 10_000); + assert_eq!(limits.max_relational_predict_rows, 10_000); + assert_eq!(limits.max_temporal_link_predict_rows, 200); + assert_eq!(limits.max_tabular_context_rows, 10_000); + assert_eq!(limits.max_request_bytes, 31_457_280); + } +} diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 6c9daf634..288b90dcd 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -69,6 +69,9 @@ pub enum Operation { OrderSampledTabularRows(crate::direct::DirectInput), SelectTabularRows(crate::direct::DirectInput), CheckTabularCap(crate::direct::DirectInput), + /// The size limits the run holds requests to, as resolved from its + /// start's (contract v72). + ResolveLimits(crate::direct::DirectInput), CheckDirectShape(crate::direct::DirectInput), CheckDirectCell(crate::direct::DirectInput), CheckColumn(crate::direct::DirectInput), @@ -161,6 +164,7 @@ impl Operation { Operation::OrderSampledTabularRows(_) => "order_sampled_tabular_rows", Operation::SelectTabularRows(_) => "select_tabular_rows", Operation::CheckTabularCap(_) => "check_tabular_cap", + Operation::ResolveLimits(_) => "resolve_limits", Operation::CheckDirectShape(_) => "check_direct_shape", Operation::CheckDirectCell(_) => "check_direct_cell", Operation::CheckColumn(_) => "check_column", @@ -333,6 +337,7 @@ impl Operation { | Operation::OrderSampledTabularRows(v) | Operation::SelectTabularRows(v) | Operation::CheckTabularCap(v) + | Operation::ResolveLimits(v) | Operation::CheckDirectShape(v) | Operation::CheckDirectCell(v) | Operation::FeatureImportance(v) @@ -1198,7 +1203,7 @@ impl Given { /// timestamp with no zone, `datetime_zoned` for one with one, `time`). /// Two of one kind and one text are one candidate; a host whose language /// holds others equal (two zones of one instant) says so with `repeat` -///. +/// . /// - `repeat`: a value the host's own language holds equal to one given /// before it, left out as the reference's `dict.fromkeys` leaves it out. /// A host whose candidates are all scalars sends none, the core keeping @@ -1948,6 +1953,10 @@ pub struct Attachments { pub check_tabular_cap_input: crate::tabular::CheckTabularCap, #[schemars(rename = "DirectValue.check_tabular_cap")] pub check_tabular_cap_output: crate::tabular::CheckedTabularCap, + #[schemars(rename = "ResolveLimits.input")] + pub resolve_limits_input: crate::limits::ResolveLimits, + #[schemars(rename = "DirectValue.resolve_limits")] + pub resolve_limits_output: crate::limits::Limits, #[schemars(rename = "ObserveTabularMembership.input")] pub observe_tabular_membership_input: crate::tabular::ObserveTabularMembership, #[schemars(rename = "DirectValue.observe_tabular_membership")] diff --git a/rust/crates/sdm-contract/src/run.rs b/rust/crates/sdm-contract/src/run.rs index 1ca392fca..e1874e8d8 100644 --- a/rust/crates/sdm-contract/src/run.rs +++ b/rust/crates/sdm-contract/src/run.rs @@ -38,6 +38,7 @@ use serde::{Deserialize, Serialize}; use crate::data::{AttachmentId, DataRequest, FetchPlan, ProviderError}; use crate::error::{CoreError, CoreWarning, ErrorKind}; use crate::facts::HostFacts; +use crate::limits::GivenLimits; use crate::ops::{Operation, OperationResult}; /// Names one fetch or model call within a run, so its reply can be matched. @@ -194,9 +195,18 @@ pub struct StartRun { /// The contract version the host speaks; see `CONTRACT_VERSION`. pub contract: u32, pub facts: HostFacts, + /// The size limits the caller set and the environment names, which the + /// core resolves before the run starts (contract v72); left out, each + /// limit is its default ([`crate::limits`]). + #[serde(default, skip_serializing_if = "is_unset")] + pub limits: GivenLimits, pub operation: Operation, } +fn is_unset(limits: &GivenLimits) -> bool { + limits == &GivenLimits::default() +} + /// Every control message that crosses between a host and the core. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(tag = "message", rename_all = "snake_case", deny_unknown_fields)] diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 7970f1dc6..2cdc4ef19 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -209,12 +209,12 @@ advice! { Nothing = "nothing", []; OptimizeIndices = "optimize_indices", []; PredictCall = "predict_call", []; + RaiseLimit = "raise_limit", ["setting"]; RaiseMostParts = "raise_most_parts", []; SemanticModel = "semantic_model", ["metric_view"]; ResultFrame = "result_frame", []; SetPrimaryKey = "set_primary_key", ["target_table", "target_column"]; SkipKeyCheck = "skip_key_check", []; - SmallerCalls = "smaller_calls", []; TrustExpressions = "trust_expressions", []; } @@ -341,7 +341,7 @@ pub fn template(sentence: Sentence) -> Template { r#"predict carries the target column {target!r}. Those are the rows whose answer is being asked for, so the column has to be absent; drop it with predict.drop(columns=[{target!r}])"#, ), S::DirectContextLimit => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with materialize(), which keeps the context under the cap and says how"#, + r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with materialize(), which keeps the context under the cap and says how; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::TabularMissingRowIndex => said( r#"kumo-tabular returned a response that does not match the contract: PredictionItem response is missing required field 'row_index'"#, @@ -745,7 +745,7 @@ pub fn template(sentence: Sentence) -> Template { r#"column {column!r} holds a value that {admitted_type}, the type it was described as, cannot hold"#, ), S::LagPastPayload => said( - r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes the endpoint takes; ask for fewer steps"#, + r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes a call takes; ask for fewer steps, or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::DateLiteralPacked => said( r#"{query!r} is not a query a relational model can answer: {literal} is not a valid instance of time"#, @@ -883,7 +883,7 @@ pub fn template(sentence: Sentence) -> Template { r#"{model} does not produce {unsupported_outputs!r}; it produces {supported_outputs!r}"#, ), S::ContextPastModel => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with {advice:materialize_call()}, which keeps the context under the cap and says how"#, + r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with {advice:materialize_call()}, which keeps the context under the cap and says how; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::NoContextRows => said( r#"no context rows are left: every row that could be context is also being scored, and a row carrying the answer being asked for is not sent as context. Score fewer rows, so that the rest are there to learn from"#, @@ -936,14 +936,14 @@ pub fn template(sentence: Sentence) -> Template { S::OptionsUnreadStatistic => said( r#"{model} does not read inference option(s) {unsupported!r}; drop them from inference_config to send the request without them. The keys this client forwards are {forwarded!r}, and the endpoint decides for itself whether it serves the value given for one. A regression's statistic is chosen with {advice:choose_statistic()} instead: 'median', the default, or 'mean'"#, ), - S::PayloadSmallerCalls => said( - r#"Context size exceeds the 30MB limit. This call is {bytes:,} bytes, and the endpoint takes at most {limit:,}. What comes down is one of three. For an ordinary question it is the call: {advice:smaller_calls()} below the {rows_per_call:,} this one holds asks for the same entities in more and smaller calls. Where the context is itself over the limit, no call is small enough, and it is num_context_samples, num_neighbors or the columns the graph carries. For a ranking or a forecast, which is one call per candidate or step and is never batched, it is the entities."#, - ), S::PayloadNoSmallerCall => said( - r#"Context size exceeds the 30MB limit. This call is {bytes:,} bytes, and the endpoint takes at most {limit:,}. A call already scores one row, so there is no smaller one to ask for: what comes down is num_context_samples, num_neighbors or the columns the graph carries."#, + r#"the context alone is too large to send: a call carrying it and one row to score is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. Calls past the limit are split into smaller ones, but the context is sent whole with every call, so what comes down is num_context_samples, num_neighbors or the columns the graph carries; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::CallPastEndpoint => said( - r#"the request is {bytes:,} bytes and the endpoint takes at most {limit:,} bytes; reduce the number of tables in the graph, their columns, the neighbourhood sampled or the context gathered (num_context_samples)"#, + r#"the context alone is too large to send: a call carrying it is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. Calls past the limit are split into smaller ones, but the context is sent whole with every call, so reduce the context gathered (num_context_samples), the neighbourhood sampled, or the tables and columns in the graph; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, + ), + S::DirectCallPastBytes => said( + r#"this request is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. A single-table request is sent as one call, so score fewer rows at a time or carry a smaller context; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::OutputsNone => said(r#"outputs must name at least one field"#), S::NotFinite => said( @@ -974,7 +974,7 @@ pub fn template(sentence: Sentence) -> Template { r#"num_context_samples must be at least 1; a model predicts from labelled examples, and {given!S} leaves it none"#, ), S::NumContextSamplesPastMost => said( - r#"num_context_samples is {given!I:,}, and the endpoint accepts at most {limit:,} context rows"#, + r#"num_context_samples is {given!I:,}, and a request carries at most {limit:,} context rows; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::NumHopsNotWhole => said(r#"num_hops must be a whole number, got {given!N} {given!G}"#), S::NumHopsNone => said(r#"num_hops must be a whole number, got {advice:nothing()}"#), @@ -1030,17 +1030,11 @@ pub fn template(sentence: Sentence) -> Template { r#"prediction_statistic applies only to a regression, and this question is a {task}; ask it without prediction_statistic"#, ), S::NoHopNumHops => said( - r#"num_hops is {hops}, which asks for no hop at all, and the endpoint reads between 1 and {limit}. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, + r#"num_hops is {hops}, which asks for no hop at all, and a request samples at least 1. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, ), S::NoHopNumNeighbors => said( - r#"num_neighbors is empty, which asks for no hop at all, and the endpoint reads between 1 and {limit}. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, + r#"num_neighbors is empty, which asks for no hop at all, and a request samples at least 1. A request that crosses no relationship is a single-table one: give a hop, such as {advice:give_a_hop()}, or ask it of kumo-tabular"#, ), - S::HopsPastEndpoint => { - said(r#"this request samples {hops} hops and the endpoint reads at most {limit}"#) - } - S::HopsPastEndpointUncounted => { - said(r#"this request samples more hops than the endpoint reads, at most {limit}"#) - } S::InferenceRunModeInvalid => { said("inference run_mode is text of at most 1024 characters, or null") } @@ -1147,7 +1141,9 @@ pub fn template(sentence: Sentence) -> Template { S::TargetNotAFeature => { said(r#"the column the query predicts is not one of the features of the entity table"#) } - S::ContextPastRows => said(r#"the context holds more rows than a request takes"#), + S::ContextPastRows => said( + r#"the context holds {context_rows:,} rows, and a request carries at most {limit:,}; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, + ), S::HeldBySeveralRowsOne => said( r#"Cannot score {keys[0]!r}: more than one row of table '{table}' holds this value of '{key}', so which row is meant would be a guess. The source does not enforce the key; keep one row per value, or key the table on columns that identify a row."#, ), @@ -1167,9 +1163,6 @@ pub fn template(sentence: Sentence) -> Template { said("Found no valid rows in table '{table}'; supply valid non-missing {requirements}.") } S::NotInTable => said(r#""The primary keys {keys!r} do not exist in the '{table}' table""#), - S::SubgraphPastTables => said( - r#"the request's subgraph carries {tables:,} tables, and the endpoint reads at most {limit:,}"#, - ), S::Unresolved => { said(r#"{query!r} is not a query a relational model can answer: {reasons!R}"#) } @@ -1340,6 +1333,12 @@ pub fn template(sentence: Sentence) -> Template { S::InvalidConfiguration => { said(r#"the configuration of this call is not one the core accepts"#) } + S::LimitEnvironmentInvalid => said( + r#"the environment variable {env} is {given!r}, and {setting} takes a whole number of at least 1"#, + ), + S::LimitGivenBelowOne => { + said(r#"{setting} is {given:,}, and it takes a whole number of at least 1"#) + } S::PartForecastStep => { said(r#"This is forecast step {forecast_step} of {forecast_steps}."#) } @@ -1556,7 +1555,7 @@ pub fn template(sentence: Sentence) -> Template { said(r#"this request carries a value that cannot be sent as JSON: {reason}"#) } S::ServingPastLimit => said( - r#"this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller num_context_samples, or fewer entities to score"#, + r#"this request is too large for serving endpoint {endpoint!r}: a call carrying it is {bytes:,} bytes once wrapped, {context_bytes:,} of them the context, above the {limit:,} byte limit the endpoint accepts. Calls past the limit are split into smaller ones where they score several rows, but the context is sent whole with every call, so gather a smaller context (num_context_samples, num_neighbors or the columns the graph carries), or score fewer rows at a time"#, ), S::ServingNotJson => { said(r#"serving endpoint {endpoint!r} answered with a response that is not JSON"#) diff --git a/rust/crates/sdm-contract/tests/corpus/mod.rs b/rust/crates/sdm-contract/tests/corpus/mod.rs index 79e7f9793..4c5ab3c32 100644 --- a/rust/crates/sdm-contract/tests/corpus/mod.rs +++ b/rust/crates/sdm-contract/tests/corpus/mod.rs @@ -103,6 +103,7 @@ pub fn messages() -> Vec { facts.local_zone = Some(LocalZone::Posix("EST5EDT,M3.2.0,M11.1.0".into())); } ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts, operation, diff --git a/rust/crates/sdm-contract/tests/host_facts.rs b/rust/crates/sdm-contract/tests/host_facts.rs index 13648accd..cc6618189 100644 --- a/rust/crates/sdm-contract/tests/host_facts.rs +++ b/rust/crates/sdm-contract/tests/host_facts.rs @@ -41,6 +41,7 @@ fn pandas_2() -> InstantRange { fn start(facts: HostFacts) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts, operation: Operation::ValidateQuery(ValidateQuery { diff --git a/rust/crates/sdm-contract/tests/provider_view_contract.rs b/rust/crates/sdm-contract/tests/provider_view_contract.rs index 708f44fb1..4d077465c 100644 --- a/rust/crates/sdm-contract/tests/provider_view_contract.rs +++ b/rust/crates/sdm-contract/tests/provider_view_contract.rs @@ -781,9 +781,12 @@ fn view_errors_explain_trust_and_recovery() { ); assert_eq!( said(Sentence::ServingPastLimit), - "this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above \ - the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller \ - num_context_samples, or fewer entities to score" + "this request is too large for serving endpoint {endpoint!r}: a call carrying it is \ + {bytes:,} bytes once wrapped, {context_bytes:,} of them the context, above the {limit:,} \ + byte limit the endpoint accepts. Calls past the limit are split into smaller ones where \ + they score several rows, but the context is sent whole with every call, so gather a \ + smaller context (num_context_samples, num_neighbors or the columns the graph carries), \ + or score fewer rows at a time" ); // The way out of an untrusted expression is the host's to spell. assert_eq!( diff --git a/rust/crates/sdm-contract/tests/schema_lock.rs b/rust/crates/sdm-contract/tests/schema_lock.rs index bbe065843..92abff4a9 100644 --- a/rust/crates/sdm-contract/tests/schema_lock.rs +++ b/rust/crates/sdm-contract/tests/schema_lock.rs @@ -557,6 +557,11 @@ const JSON: &[(&str, &str)] = &[ "OperationResult/direct_value.value", "DirectValue.check_tabular_cap", ), + ("Operation/resolve_limits.input", "ResolveLimits.input"), + ( + "OperationResult/direct_value.value", + "DirectValue.resolve_limits", + ), ( "Operation/observe_tabular_membership.input", "ObserveTabularMembership.input", diff --git a/rust/crates/sdm-core/src/direct.rs b/rust/crates/sdm-core/src/direct.rs index e7ab051b0..9e30f0c4e 100644 --- a/rust/crates/sdm-core/src/direct.rs +++ b/rust/crates/sdm-core/src/direct.rs @@ -715,6 +715,7 @@ pub(crate) async fn predict( asker: Asker, request: Predict, mut attached: Attachments, + limits: sdm_contract::limits::Limits, ) -> Result { if request.state.is_some() || request.held.is_some() { return Err(CoreError::protocol( @@ -823,11 +824,14 @@ pub(crate) async fn predict( None }; let analysis = crate::columns::expose(&asker, analysis.as_deref(), &bound.outputs)?; + // The context and the call held to the run's limits, as they are sent. + predict::check_direct_context(&bound.model, bound.facts.context_rows, &limits)?; if predict::nothing_to_score(&payload) { let mut part = tabular::response::empty(&bound.model, &payload)?; part.columns = analysis; return crate::predict::finish(&asker, part, bound.outputs); } + let bytes = crate::predict::one_call_bytes(&payload, &limits)?; if deferred_options { use sdm_contract::direct::native_response::{NativeDirectCall, PreparedDirectCall}; let payload = asker.attach(Attachment::Bytes(payload.to_bytes()))?; @@ -844,8 +848,7 @@ pub(crate) async fn predict( })); } let answer = - crate::predict::call_answer(&asker, payload.to_bytes(), request.model_endpoint.as_ref()) - .await?; + crate::predict::call_answer(&asker, bytes, request.model_endpoint.as_ref()).await?; let mut payload = payload; if let Some(analysis) = analysis { payload.set("client_column_analysis", Value::Text(analysis)); diff --git a/rust/crates/sdm-core/src/dispatch.rs b/rust/crates/sdm-core/src/dispatch.rs index 04aeffe69..4a0c30cb8 100644 --- a/rust/crates/sdm-core/src/dispatch.rs +++ b/rust/crates/sdm-core/src/dispatch.rs @@ -6,6 +6,7 @@ use std::sync::Arc; +use sdm_contract::limits::Limits; use sdm_contract::ops::{ GivenCandidate, HeldSchema, Operation, OperationResult, QuerySchema, REQUEST_STATE_VERSION, RequestState, SAMPLER_STATE_VERSION, SamplerMemo, SchemaEdit, SourceSpec, @@ -49,6 +50,9 @@ pub fn start( zones: Arc, ) -> Result { check_contract(start.contract)?; + // The size limits the run holds requests to, the caller's before the + // environment's before the defaults, resolved before anything runs. + let limits = crate::limits::resolve(&start.limits)?; // The clock is an instant in UTC, spelled as the contract spells it. if !matches!(&start.facts.now, Scalar::Timestamp { zone: Some(zone), .. } if zone == "UTC") { return Err(CoreError::new( @@ -68,7 +72,7 @@ pub fn start( crate::materialize::frames::labels(asker, request, attachments, facts, zones).await }), Operation::MaterializeExamples(request) => { - crate::materialize::examples::start(root, request, attachments, facts)? + crate::materialize::examples::start(root, request, attachments, facts, limits)? } Operation::RebindSource(request) => { crate::rebind::start(root, request, attachments, facts)? @@ -111,10 +115,10 @@ pub fn start( Run::new(move |asker| async move { crate::result::merge(&asker, request, attachments) }) } Operation::TabularArrow(request) => Run::new(move |asker| async move { - crate::tabular_arrow::run(asker, request, attachments, facts, zones).await + crate::tabular_arrow::run(asker, request, attachments, facts, zones, limits).await }), Operation::FrameSession(input) => Run::new(move |asker| async move { - crate::frame_session::run(asker, input, attachments).await + crate::frame_session::run(asker, input, attachments, limits).await }), Operation::DescribeFrameTables(input) => Run::new(move |asker| async move { crate::frame_schema::describe(&asker, input, attachments) @@ -123,7 +127,8 @@ pub fn start( crate::materialize::frames::prepare(&asker, input, attachments, &facts) }), Operation::MaterializeFrames(input) => Run::new(move |asker| async move { - crate::materialize::frames::materialize(asker, input, attachments, facts, zones).await + crate::materialize::frames::materialize(asker, input, attachments, facts, zones, limits) + .await }), Operation::CountTabularSessionOverlap(request) => Run::new(move |asker| async move { crate::session::count_overlap(&asker, request, attachments) @@ -132,7 +137,7 @@ pub fn start( crate::session::send(&asker, request, attachments).await }), Operation::PrepareTabularScore(request) => Run::new(move |asker| async move { - crate::session::prepare_score(&asker, request, attachments) + crate::session::prepare_score(&asker, request, attachments, &limits) }), Operation::CheckTabularSessionReply(request) => Run::new(move |asker| async move { crate::session::check_reply(&asker, request, attachments) @@ -283,7 +288,7 @@ pub fn start( let input: sdm_contract::tabular::DecideTabularContext = take_json(&mut attachments, request.input)?; Run::new(move |asker| async move { - crate::direct::value(&asker, &crate::tabular_selection::decide(input)) + crate::direct::value(&asker, &crate::tabular_selection::decide(input, &limits)) }) } Operation::SampleTabularRows(request) => { @@ -321,9 +326,14 @@ pub fn start( let input: sdm_contract::tabular::SelectTabularRows = take_json(&mut attachments, request.input)?; Run::new(move |asker| async move { - crate::direct::value(&asker, &crate::tabular_selection::select(input)?) + crate::direct::value(&asker, &crate::tabular_selection::select(input, &limits)?) }) } + Operation::ResolveLimits(request) => { + let _: sdm_contract::limits::ResolveLimits = + take_json(&mut attachments, request.input)?; + Run::new(move |asker| async move { crate::direct::value(&asker, &limits) }) + } Operation::CheckTabularCap(request) => { let input: sdm_contract::tabular::CheckTabularCap = take_json(&mut attachments, request.input)?; @@ -353,7 +363,7 @@ pub fn start( &input.kind, input.outputs, ); - crate::direct::value(&asker, &sdm_request::direct::bind(input)?) + crate::direct::value(&asker, &sdm_request::direct::bind(input, &limits)?) }) } Operation::InspectDirectPair(request) => Run::new(move |asker| async move { @@ -481,6 +491,7 @@ pub fn start( operation, attachments, facts, + limits, Arc::clone(&zones), #[cfg(feature = "test-ops")] false, @@ -488,7 +499,7 @@ pub fn start( Operation::Predict(request) => { if request.direct.is_some() { return Ok(Run::new(move |asker| { - crate::direct::predict(asker, request, attachments) + crate::direct::predict(asker, request, attachments, limits) })); } // What predict asks: the state `materialize` answered, or a @@ -519,12 +530,13 @@ pub fn start( ranking: asked.ranking, batches: asked.batches, endpoint: request.model_endpoint, + limits, }; Run::new(move |asker| crate::predict::run(asker, inputs)) } #[cfg(feature = "test-ops")] Operation::Test(operation) => { - crate::test_ops::run(root, operation, &mut attachments, facts, zones)? + crate::test_ops::run(root, operation, &mut attachments, facts, limits, zones)? } }) } @@ -680,6 +692,7 @@ pub(crate) fn materialize( operation: Operation, mut attachments: Attachments, facts: HostFacts, + limits: Limits, zones: Arc, #[cfg(feature = "test-ops")] labels_only: bool, ) -> Result { @@ -789,6 +802,7 @@ pub(crate) fn materialize( most_entities: request.most_entities, most_calls: request.most_calls, facts, + limits, zones: Arc::clone(&zones), #[cfg(feature = "test-ops")] labels_only, diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 7650900c7..992a3f836 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -64,11 +64,7 @@ pub(crate) async fn call( body: Value, ) -> Result { match asker - .call_model( - endpoint, - "application/json".into(), - predict::call_bytes(&body)?, - ) + .call_model(endpoint, "application/json".into(), body.to_bytes()) .await { Reply::Model(ModelResult::Response { status, body, .. }) @@ -94,6 +90,7 @@ pub(crate) async fn run( asker: Asker, operation: FrameSessionOperation, mut attached: Attachments, + limits: sdm_contract::limits::Limits, ) -> Result { let (input_id, payload_ids, pinned_id, action) = match operation { FrameSessionOperation::Fit { input, payloads } => (input, payloads, None, "fit"), @@ -151,6 +148,7 @@ pub(crate) async fn run( &model, outputs.as_deref().unwrap_or(&[]), default_outputs, + &limits, )?; let query = question .as_ref() @@ -182,6 +180,8 @@ pub(crate) async fn run( let state_keys = row_keys(&body, "context", windowed)?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&body))?); crate::columns::expose(&asker, column_analysis.as_deref(), &fields)?; + // The pinned context is one call, which no split makes smaller. + predict::call_bytes(&body, limits.max_request_bytes)?; let answer = call(&asker, Endpoint::CreateSession, body).await?; let id = session_id(&answer)?; let state = FrameSessionState { @@ -274,6 +274,7 @@ pub(crate) async fn run( &state.model, &state.outputs, false, + &limits, )?; let scored = row_keys(&layout, "predict", state.windowed)?; shown += keys::overlap_count(&state.keys.rows, &scored.rows)?; @@ -293,6 +294,11 @@ pub(crate) async fn run( } let analysis = crate::columns::expose(&asker, state.column_analysis.as_deref(), &state.outputs)?; + // Every scoring call held to the most bytes a call takes before + // the first is sent. + for (body, _) in &plans { + crate::predict::one_call_bytes(body, &limits)?; + } let mut parts = Vec::new(); let mut start = 0; for (body, layout) in plans { diff --git a/rust/crates/sdm-core/src/lag.rs b/rust/crates/sdm-core/src/lag.rs index 55812a9e5..792056b65 100644 --- a/rust/crates/sdm-core/src/lag.rs +++ b/rust/crates/sdm-core/src/lag.rs @@ -34,11 +34,12 @@ //! than their own; the core puts each on its own row ( fix 1, a //! registered difference decided under ). +use sdm_contract::limits::MAX_REQUEST_BYTES; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_engine::labels::{self, Labels, Target}; use sdm_engine::sample::out_of_range; use sdm_engine::time::offset::Offset; -use sdm_request::payload::{LagColumn, LagValues, PAYLOAD_BYTES}; +use sdm_request::payload::{LagColumn, LagValues}; /// The field a lag's refusal names. const FIELD: &str = "lag_timesteps"; @@ -144,9 +145,9 @@ pub(crate) struct Lag { impl Lag { /// Refuses lags that would make a payload of `rows` instance rows - /// larger than the endpoint takes, before any column is computed, as - /// the payload's limit refuses it. - pub fn fits(&self, rows: usize) -> Result<(), CoreError> { + /// larger than one call takes (`limit`, `max_request_bytes`), before + /// any column is computed, as the payload's limit refuses it. + pub fn fits(&self, rows: usize, limit: u64) -> Result<(), CoreError> { let least = usize::try_from(self.steps) .ok() .and_then(|steps| { @@ -155,16 +156,15 @@ impl Lag { .checked_mul(steps) }) .unwrap_or(usize::MAX); - if least > PAYLOAD_BYTES { - return Err(CoreError::new( - Sentence::LagPastPayload, - "the request is larger than the endpoint takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(PAYLOAD_BYTES as u64)), + if least as u64 > limit { + return Err(MAX_REQUEST_BYTES.named( + CoreError::new( + Sentence::LagPastPayload, + "the request is larger than one call takes", + ) + .with_field("field", Detail::text("payload")) + .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) + .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), )); } Ok(()) @@ -542,7 +542,7 @@ mod tests { times: vec![], }, ); - let e = lag.fits(2).err().unwrap(); + let e = lag.fits(2, MAX_REQUEST_BYTES.default).err().unwrap(); assert_eq!(e.code, ErrorCode::InvalidRequest); assert_eq!(e.fields["field"], Detail::text("payload")); } diff --git a/rust/crates/sdm-core/src/lib.rs b/rust/crates/sdm-core/src/lib.rs index d1159c432..78d45d0bd 100644 --- a/rust/crates/sdm-core/src/lib.rs +++ b/rust/crates/sdm-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod exact_number; mod fallback; mod held; mod lag; +pub mod limits; pub mod materialize; mod memory; #[cfg(test)] diff --git a/rust/crates/sdm-core/src/limits.rs b/rust/crates/sdm-core/src/limits.rs new file mode 100644 index 000000000..f6fe3ee2c --- /dev/null +++ b/rust/crates/sdm-core/src/limits.rs @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The size limits a run holds a request to, resolved from what the host +//! passed with its start ([`GivenLimits`]): the caller's value, else the +//! environment's, else the default ([`sdm_contract::limits::SETTINGS`]). +//! Every host reads its limits through this one rule. + +use sdm_contract::limits::{GivenLimits, Limits, SETTINGS}; +use sdm_contract::{CoreError, Detail, Scalar, Sentence}; + +/// The limits `given` names, or the refusal of a value that is not a whole +/// number of at least one, naming the setting and where it was set. A name +/// that is no setting's is the host's fault, `INVALID_MESSAGE`. +pub fn resolve(given: &GivenLimits) -> Result { + for name in given.given.keys() { + if !SETTINGS.iter().any(|setting| setting.name == name) { + return Err(unknown("limits.given", name)); + } + } + for env in given.environment.keys() { + if !SETTINGS.iter().any(|setting| setting.env == env) { + return Err(unknown("limits.environment", env)); + } + } + let mut limits = Limits::default(); + for setting in &SETTINGS { + let value = if let Some(&value) = given.given.get(setting.name) { + if value == 0 { + return Err(setting.named( + CoreError::new( + Sentence::LimitGivenBelowOne, + "a limit the caller set is not a whole number of at least one", + ) + .with_field("given", Detail::Scalar(Scalar::UInt64(value))), + )); + } + value + } else if let Some(text) = given.environment.get(setting.env) { + whole(text).ok_or_else(|| { + setting.named( + CoreError::new( + Sentence::LimitEnvironmentInvalid, + "a limit's environment variable is not a whole number of at least one", + ) + .with_field("given", Detail::text(text)), + ) + })? + } else { + setting.default + }; + limits.set(setting.name, value); + } + Ok(limits) +} + +/// `text` as a whole number of at least one: decimal digits, with `_` +/// allowed between them as Python and Java write a long number, and white +/// space around them; anything else, zero, or a number past `u64`, none. +fn whole(text: &str) -> Option { + let digits = text.trim(); + let well_formed = digits.as_bytes().first().is_some_and(u8::is_ascii_digit) + && digits.as_bytes().last().is_some_and(u8::is_ascii_digit) + && digits.bytes().all(|b| b.is_ascii_digit() || b == b'_') + && !digits.contains("__"); + if !well_formed { + return None; + } + digits + .replace('_', "") + .parse::() + .ok() + .filter(|&value| value >= 1) +} + +fn unknown(field: &str, name: &str) -> CoreError { + CoreError::protocol( + Sentence::InvalidMessage, + "the start names a limit that is no setting's", + ) + .with_field("field", Detail::text(field)) + .with_field("name", Detail::text(name)) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use sdm_contract::ErrorCode; + use sdm_contract::limits::{MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS}; + + use super::*; + + fn given(given: &[(&str, u64)], environment: &[(&str, &str)]) -> GivenLimits { + GivenLimits { + given: given.iter().map(|(k, v)| ((*k).to_owned(), *v)).collect(), + environment: environment + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect::>(), + } + } + + #[test] + fn nothing_given_is_every_default() { + assert_eq!(resolve(&GivenLimits::default()).unwrap(), Limits::default()); + } + + #[test] + fn the_caller_comes_before_the_environment_before_the_default() { + let env = MAX_TABULAR_CONTEXT_ROWS.env; + let from_env = resolve(&given(&[], &[(env, "200000")])).unwrap(); + assert_eq!(from_env.max_tabular_context_rows, 200_000); + assert_eq!(from_env.max_request_bytes, MAX_REQUEST_BYTES.default); + let both = resolve(&given( + &[(MAX_TABULAR_CONTEXT_ROWS.name, 50_000)], + &[(env, "200000")], + )) + .unwrap(); + assert_eq!(both.max_tabular_context_rows, 50_000); + } + + #[test] + fn environment_text_is_a_whole_number_of_at_least_one() { + let env = MAX_REQUEST_BYTES.env; + for (text, expected) in [ + ("1", Some(1)), + (" 64000000 ", Some(64_000_000)), + ("200_000", Some(200_000)), + ("0", None), + ("", None), + ("-1", None), + ("1e6", None), + ("2.5", None), + ("_1", None), + ("1__0", None), + ("ten", None), + ("18446744073709551616", None), + ] { + let resolved = resolve(&given(&[], &[(env, text)])); + match expected { + Some(value) => assert_eq!(resolved.unwrap().max_request_bytes, value, "{text:?}"), + None => { + let error = resolved.unwrap_err(); + assert_eq!( + error.sentence, + Sentence::LimitEnvironmentInvalid, + "{text:?}" + ); + assert_eq!(error.code, ErrorCode::InvalidConfiguration); + assert_eq!(error.fields["env"], Detail::text(env)); + assert_eq!( + error.fields["setting"], + Detail::text(MAX_REQUEST_BYTES.name) + ); + assert_eq!(error.fields["given"], Detail::text(text)); + } + } + } + } + + #[test] + fn a_given_zero_and_an_unknown_name_are_refused() { + let error = resolve(&given(&[(MAX_REQUEST_BYTES.name, 0)], &[])).unwrap_err(); + assert_eq!(error.sentence, Sentence::LimitGivenBelowOne); + let error = resolve(&given(&[("max_tables", 20)], &[])).unwrap_err(); + assert_eq!(error.sentence, Sentence::InvalidMessage); + let error = resolve(&given(&[], &[("SDM_CLIENT_MAX_HOPS", "8")])).unwrap_err(); + assert_eq!(error.sentence, Sentence::InvalidMessage); + } +} diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index d171e3ebd..731c35f6d 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -79,6 +79,7 @@ use sdm_contract::data::{ MATCHES_COLUMN, NullOrder, OptionalRequest, ProviderChoice, SEED_STORED_TIME, TableColumn, TextTimes, }; +use sdm_contract::limits::{Limits, MAX_RELATIONAL_CONTEXT_ROWS}; use sdm_contract::ops::{ ColumnRef, ForecastStep, Materialized, MemoBounds, MemoRows, MemoStoredTime, OperationResult, ProfileName, ProfileSpec, QueryColumn, QueryFacts, QuerySchema, QueryTable, RequestBatch, @@ -139,8 +140,6 @@ const DEFAULT_SEED: u128 = 42; /// How many times as many candidates as examples a run tries /// (`max_pq_iterations`' default). const ITERATIONS: usize = 10; -/// The most tables a request's subgraph carries (`_MAX_SUBGRAPH_TABLES`). -const SUBGRAPH_TABLES: usize = 15; /// The most entities one `materialize` gathers where the host names no /// limit (`main`'s `MOST_ENTITIES`): every call's payload is built, with /// its own context, before any is sent, so the whole population is held at @@ -149,8 +148,6 @@ pub const MOST_ENTITIES: u64 = 100_000; /// The most model calls one request is gathered as where the host names no /// limit (`main`'s `MOST_CALLS`): a context is gathered for every call. pub const MOST_CALLS: u32 = 100; -/// The most rows an instance table may hold (`MAX_TABLE_ROWS`). -const TABLE_ROWS: usize = 10_000; /// How many examples the reference labels at once, stopping once it has /// the run mode's (`_sample_target_set`'s `batch_size`): the first batch /// holds every example a context takes whenever all of it is labelled. @@ -190,6 +187,8 @@ pub(crate) struct Inputs { /// (`Materialize.most_calls`). pub most_calls: Option, pub facts: HostFacts, + /// The size limits the run holds the request to. + pub limits: Limits, /// The zones the run resolves names with: the host's local zone, which /// places a date literal's two-digit year (`pql::local_year`). pub zones: Arc, @@ -2880,7 +2879,8 @@ pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result Result Result Result Result mode.context_examples(), + options::RunMode::Best => mode.context_examples().max(asked_examples.unwrap_or(0)), + }; // `max_pq_iterations` times as many candidates as examples; a // float's count is no slice bound for a temporal query's walk, the // engine's `TypeError`, the caller's error. @@ -3790,7 +3797,7 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result TABLE_ROWS { - return Err(invalid( - Sentence::ContextPastRows, - "the context holds more rows than a request takes", - "query", + // The instance table holds the context's examples: at most as many as + // a request carries (`MAX_TABLE_ROWS`, `max_relational_context_rows`). + if num_train as u64 > limits.max_relational_context_rows { + return Err(MAX_RELATIONAL_CONTEXT_ROWS.named( + invalid( + Sentence::ContextPastRows, + "the context holds more rows than a request takes", + "query", + ) + .with_field("context_rows", count(num_train)) + .with_field( + "limit", + Detail::Scalar(Scalar::UInt64(limits.max_relational_context_rows)), + ), )); } - // The rows the largest call scores (`_the_largest_batch`), which a - // payload past the endpoint's body names as the size to come down from. + // The rows the largest call scores (`_the_largest_batch`), which the + // lagged columns are held to. let largest = per_call.min(num_test); let train_anchors = anchors.len(); let entity_names = crate::subgraph::sample_columns(&graph, entity, &excluded); @@ -4115,18 +4132,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result SUBGRAPH_TABLES { - return Err(invalid( - Sentence::SubgraphPastTables, - "the request's subgraph carries more tables than the endpoint reads", - "tables", - ) - .with_field("tables", count(walked.tables.len())) - .with_field("limit", count(SUBGRAPH_TABLES))); - } sent.extend(walked.tables.iter().map(|t| t.name.clone())); // The lagged columns of this call's examples: the narrowed // context's, then this call's own entities, each its own row's, @@ -4138,7 +4143,7 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result Result { match &request.source { ExampleSource::Sql { provider, .. } => { let data = root.data_access(provider)?; Ok(Run::with_data(data, move |asker| async move { - run(asker, request, attachments, facts).await + run(asker, request, attachments, facts, limits).await })) } ExampleSource::Frames { .. } => Ok(Run::new(move |asker| async move { - run(asker, request, attachments, facts).await + run(asker, request, attachments, facts, limits).await })), } } @@ -111,6 +112,7 @@ async fn run( request: MaterializeExamples, mut attachments: Attachments, facts: HostFacts, + limits: Limits, ) -> Result { let call: ExampleCall = crate::attached::take_json(&mut attachments, request.input)?; if call.rows_per_call == Some(0) { @@ -151,8 +153,20 @@ async fn run( } else { None }; - if context.num_rows() > TABLE_ROWS { - return Err(refuse("context exceeds the model row limit")); + // The instance table holds the context's examples: at most as many as + // a request carries (`max_relational_context_rows`). + if context.num_rows() as u64 > limits.max_relational_context_rows { + return Err(MAX_RELATIONAL_CONTEXT_ROWS.named( + CoreError::new( + Sentence::ContextPastRows, + "the context holds more rows than a request takes", + ) + .with_field("context_rows", count(context.num_rows())) + .with_field( + "limit", + Detail::Scalar(Scalar::UInt64(limits.max_relational_context_rows)), + ), + )); } refuse_entities(predict.num_rows() as u64, 1, None)?; let seed = call @@ -301,6 +315,7 @@ async fn run( magnitude: Some(context.num_rows() as u128), }), None, + limits.max_relational_context_rows, )?; options::hops( None, @@ -331,8 +346,12 @@ async fn run( } else { options::BatchTask::Ordinary }; - usize::try_from(options::rows_one_task_call_scores(call.rows_per_call, task)) - .map_err(|_| internal("batch size overflow"))? + usize::try_from(options::rows_one_task_call_scores( + call.rows_per_call, + task, + &limits, + )) + .map_err(|_| internal("batch size overflow"))? }; let mut calls = Vec::new(); for start in (0..predict.num_rows()).step_by(per_call) { @@ -420,9 +439,6 @@ async fn run( Some(seed), ) .await?; - if walked.tables.len() > SUBGRAPH_TABLES { - return Err(refuse("sampled graph exceeds the model table limit")); - } if let Some(destination) = destination && !walked.tables.iter().any(|t| t.name == destination.name) { @@ -447,7 +463,6 @@ async fn run( options: options.clone(), statistic: None, top_k: None, - rows_per_call: per_call, lags: vec![], float_text: crate::pql::float_text, }; @@ -459,6 +474,7 @@ async fn run( forecast, }, )?; + payload::check_bytes(&built, limits.max_request_bytes)?; calls.push((built.bytes, count as u64)); } let mut calls = calls.into_iter(); diff --git a/rust/crates/sdm-core/src/materialize/frames.rs b/rust/crates/sdm-core/src/materialize/frames.rs index cf826199c..ff1c9c074 100644 --- a/rust/crates/sdm-core/src/materialize/frames.rs +++ b/rust/crates/sdm-core/src/materialize/frames.rs @@ -133,7 +133,7 @@ impl FrameGraph { &self, graph: &Graph, seeds: SeedRows<'_>, - hops: &[options::Hop], + hops: &options::Hops, excluded: &[(String, String)], seed: Option, ) -> Result { @@ -144,7 +144,10 @@ impl FrameGraph { } = seeds; let rows = self.0.snapshot.entity_rows(&entity.name, keys)?; let mut counts = Vec::new(); - for hop in hops { + // Hops past as many as the snapshot holds rows reach nothing more, + // so a count of hops past it is walked as that many. + let most = self.0.snapshot.rows().saturating_add(1); + for hop in (0..most).map_while(|at| hops.get(at)) { match hop { options::Hop::Keep(n) => { counts.push(i64::try_from(*n).map_err(|_| not_yet("frame_neighbor_count"))?) @@ -331,6 +334,7 @@ pub(crate) async fn materialize( mut attachments: crate::run::Attachments, facts: HostFacts, zones: Arc, + limits: sdm_contract::limits::Limits, ) -> Result { let Some(Attachment::FrameGraph(owner)) = attachments.remove(&request.graph) else { return Err(internal("frame graph owner is absent")); @@ -368,6 +372,7 @@ pub(crate) async fn materialize( most_entities: call.most_entities, most_calls: call.most_calls, facts, + limits, zones, memo: None, source: "in_memory".into(), diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 9b4d03f2e..f075a34e3 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -32,11 +32,12 @@ //! the answer. use sdm_contract::error::AnsweredParts; +use sdm_contract::limits::Limits; use sdm_contract::ops::{ModelEndpoint, OperationResult, Predicted}; use sdm_contract::run::Endpoint; use sdm_contract::{CoreError, CoreWarning, Detail, Scalar, Sentence}; use sdm_request::predict::{ - Notice, Part, absent, absent_notice, bad_response, body, by_step, call_bytes, call_starts, + Notice, Part, absent, absent_notice, bad_response, body, by_step, call_past_bytes, call_starts, empty, layout, left_out, merge, nothing_to_score, numbered_from, predict_rows, preflight, prepare, refused, steps_unexplained, table, }; @@ -103,6 +104,8 @@ pub(crate) struct Inputs { /// core writes and whose answers it reads; `None` for the /// client's own endpoint. pub endpoint: Option, + /// The size limits the run holds its calls to. + pub limits: Limits, } /// The JSON a request state's payload holds, or a refusal naming where in @@ -407,6 +410,7 @@ pub(crate) async fn one( &inputs.model, &inputs.outputs, inputs.default_outputs, + &inputs.limits, )?; let computed = if provided_analysis.is_none() && crate::columns::requested(&outputs) { Some(crate::columns::compute(&payloads[..1])?) @@ -429,18 +433,20 @@ pub(crate) async fn one( warn(asker, notice); } // Every call written before the first is sent, as the client's - // plan is listed whole (`_numbered_across_calls`), then each call's - // rows counted. + // plan is listed whole (`_numbered_across_calls`), each split into + // smaller calls where it is past the most bytes a call takes, then + // each call's rows counted. + let mut sent = Vec::with_capacity(payloads.len()); let mut bodies = Vec::with_capacity(payloads.len()); + let mut sizes = Vec::with_capacity(payloads.len()); for (payload, request_id) in payloads.iter().zip(&ids) { - bodies.push(body( - payload, - &inputs.model, - request_id, - &outputs, - &inputs.inference_asked, - )?); + for (call, body, bytes) in fitted(inputs, &outputs, payload, request_id)? { + sent.push(call); + bodies.push(body); + sizes.push(bytes); + } } + let payloads = sent; let mut session = crate::batch_session::Session::eligible( &bodies, inputs.supports_sessions && inputs.endpoint.is_none(), @@ -449,8 +455,8 @@ pub(crate) async fn one( // Retain only the small scoring half alongside encoded fallback bytes. let plans = bodies .into_iter() - .map(|body| { - let bytes = call_bytes(&body)?; + .zip(sizes) + .map(|(body, bytes)| { let scoring = if session.is_some() { sdm_request::frame_session::subset( &body, @@ -459,9 +465,9 @@ pub(crate) async fn one( } else { Value::Null }; - Ok((scoring, bytes)) + (scoring, bytes) }) - .collect::, CoreError>>()?; + .collect::>(); let starts = call_starts(&payloads)?; let mut parts = Vec::with_capacity(payloads.len()); for ((payload, (body, bytes)), start) in payloads.iter().zip(plans).zip(starts) { @@ -507,6 +513,97 @@ pub(crate) async fn one( Ok((merged, outputs, scored)) } +/// `payload`'s calls, each with the body it sends and that body's bytes: +/// the one call where its body fits the most bytes a call takes +/// (`max_request_bytes`) and, to a serving endpoint, the most its envelope +/// takes; else as many calls as it is split into so that each fits +/// ([`sdm_request::batching::fitted`]), each scoring a run of its rows with +/// the whole context. Every call of a split carries the payload's request +/// id, so the bytes weighed are the bytes sent. A call of one row that is +/// still past a limit is refused, naming the context's bytes. +fn fitted( + inputs: &Inputs, + outputs: &[String], + payload: &Value, + request_id: &str, +) -> Result)>, CoreError> { + let write = |call: &Value| { + body( + call, + &inputs.model, + request_id, + outputs, + &inputs.inference_asked, + ) + }; + let limit = inputs.limits.max_request_bytes; + let serving = inputs.endpoint.as_ref().map( + |ModelEndpoint::ServingEndpoint { + name, + max_request_bytes, + }| (name.as_str(), *max_request_bytes), + ); + let mut fits = |call: &Value| -> Result { + let body = write(call)?; + if body.to_bytes().len() as u64 > limit { + return Ok(false); + } + Ok(match serving { + Some((_, most)) => serving::envelope_len(&body)? <= most, + None => true, + }) + }; + let mut refuse = |call: &Value| -> CoreError { + let body = match write(call) { + Ok(body) => body, + Err(error) => return error, + }; + let bytes = body.to_bytes(); + if bytes.len() as u64 > limit { + return call_past_bytes(&body, bytes.len() as u64, limit); + } + match serving { + Some((name, most)) => match serving::envelope(name, most, &bytes) { + Err(error) => error, + Ok(_) => internal("a call that fits was refused"), + }, + None => internal("a call that fits was refused"), + } + }; + sdm_request::batching::fitted(payload, &mut fits, &mut refuse)? + .into_iter() + .map(|(_, call)| { + let body = write(&call)?; + let bytes = body.to_bytes(); + Ok((call, body, bytes)) + }) + .collect() +} + +/// The bytes of a call sent as it is, never split (a request of a single +/// table, sent as one call): refused past the most bytes a call takes +/// (`max_request_bytes`), naming the bytes of its context. +pub(crate) fn one_call_bytes(call: &Value, limits: &Limits) -> Result, CoreError> { + let bytes = call.to_bytes(); + let limit = limits.max_request_bytes; + if bytes.len() as u64 > limit { + let context = sdm_request::batching::predict_slice(call, 0..0) + .map_or(bytes.len() as u64, |context| { + context.to_bytes().len() as u64 + }); + return Err(sdm_contract::limits::MAX_REQUEST_BYTES.named( + CoreError::new( + Sentence::DirectCallPastBytes, + "a call sent whole is larger than one call takes", + ) + .with_field("bytes", count(bytes.len())) + .with_field("context_bytes", Detail::Scalar(Scalar::UInt64(context))) + .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), + )); + } + Ok(bytes) +} + /// One payload's call, its body as written, sent and its answer laid out; /// to a serving endpoint, in its envelope, its answer read out of its row. async fn call( diff --git a/rust/crates/sdm-core/src/ranking.rs b/rust/crates/sdm-core/src/ranking.rs index 97d311259..ec16a08af 100644 --- a/rust/crates/sdm-core/src/ranking.rs +++ b/rust/crates/sdm-core/src/ranking.rs @@ -176,8 +176,10 @@ pub(crate) async fn materialize( // each question gathers whole, held as many times over as there are // questions (`refuse_more_entities_than_fit_in_memory`), past the // host's limit, 100,000 where it names none. - let per_call = - sdm_request::options::rows_one_call_scores(asked_rows_per_call(inputs.values.as_ref())?); + let per_call = sdm_request::options::rows_one_call_scores( + asked_rows_per_call(inputs.values.as_ref())?, + &inputs.limits, + ); let entities = inputs.indices.as_ref().map_or(0, |data| { data.batches.iter().map(|b| b.num_rows()).sum::() }); @@ -347,7 +349,7 @@ pub(crate) async fn predict( let checked = (|| -> Result<(), CoreError> { let payload = crate::predict::payload(payload)?; let payloads = [payload]; - let outputs = prepare(&payloads, &inputs.model, &asked, false)?; + let outputs = prepare(&payloads, &inputs.model, &asked, false, &inputs.limits)?; preflight(&payloads, &outputs, query, &inputs.model)?; body( &payloads[0], diff --git a/rust/crates/sdm-core/src/session.rs b/rust/crates/sdm-core/src/session.rs index 4da783244..473489b99 100644 --- a/rust/crates/sdm-core/src/session.rs +++ b/rust/crates/sdm-core/src/session.rs @@ -175,6 +175,7 @@ pub(crate) fn prepare_score( asker: &Asker, input: SessionScoreInput, mut attached: Attachments, + limits: &sdm_contract::limits::Limits, ) -> Result { let mut request: PrepareTabularScore = take_json(&mut attached, input.input)?; if request.predict != input.predict { @@ -197,7 +198,7 @@ pub(crate) fn prepare_score( &facts.kind, Some(request.state.bound.outputs.clone()), ); - request.state.bound = sdm_request::direct::bind(facts)?; + request.state.bound = sdm_request::direct::bind(facts, limits)?; } let table = take_value(&mut attached, request.predict)?; let mut payload = object([ diff --git a/rust/crates/sdm-core/src/subgraph.rs b/rust/crates/sdm-core/src/subgraph.rs index 4d9751860..6199311ae 100644 --- a/rust/crates/sdm-core/src/subgraph.rs +++ b/rust/crates/sdm-core/src/subgraph.rs @@ -48,7 +48,7 @@ use sdm_contract::{CoreError, CoreWarning, Detail, Scalar, Sentence}; use sdm_engine::graph::{Graph, Table}; use sdm_engine::keys::{self, Key}; use sdm_engine::sample::{self, SEED_TIME}; -use sdm_request::options::Hop; +use sdm_request::options::{Hop, Hops}; use sdm_request::payload::{SampledColumn, SampledLink, SampledTable}; use crate::describe::{ask, ask_all_vec}; @@ -64,24 +64,38 @@ struct Visit { /// Each row's number, by its key and example (the reference's /// `Mapper`), for a table with a key. nodes: BTreeMap<(Key, usize), usize>, - /// The rows reached at each hop, the entity's first. + /// The rows reached at each hop, the entity's first; a hop past the + /// end reached none. counts: Vec, /// The frames visited so far. offset: usize, } impl Visit { - fn new(name: &str, hops: usize) -> Visit { + fn new(name: &str) -> Visit { Visit { name: name.to_owned(), frames: Vec::new(), batches: Vec::new(), nodes: BTreeMap::new(), - counts: vec![0; hops + 1], + counts: Vec::new(), offset: 0, } } + /// The rows reached at hop `hop`. + fn reached(&self, hop: usize) -> usize { + self.counts.get(hop).copied().unwrap_or(0) + } + + /// Counts `rows` more reached at hop `hop`. + fn add_reached(&mut self, hop: usize, rows: usize) { + if self.counts.len() <= hop { + self.counts.resize(hop + 1, 0); + } + self.counts[hop] += rows; + } + fn rows(&self) -> usize { self.counts.iter().sum() } @@ -119,10 +133,11 @@ pub(crate) struct Walk<'a> { /// The hops walked, each with the neighbors it keeps of each row: the /// run mode's two by default, or as the sampling options ask /// (`num_hops`, `num_neighbors`, [`options::neighbors`]); the walk ends - /// at a hop of size zero. + /// at a hop of size zero, after the last, or at a hop no row is walked + /// from, past which nothing is reached. /// /// [`options::neighbors`]: sdm_request::options::neighbors - pub neighbors: &'a [Hop], + pub neighbors: &'a Hops, /// The features left out, by table: a static target's columns, which /// the reference's sampler neither reads nor sends /// (`get_exclude_cols_dict`, `exclude_cols_dict`). @@ -293,7 +308,7 @@ impl Walk<'_> { /// [`sample_columns`]. pub(crate) async fn run(&self, entity_rows: RecordBatch) -> Result { let examples = entity_rows.num_rows(); - let mut visits = vec![Visit::new(&self.entity.name, self.neighbors.len())]; + let mut visits = vec![Visit::new(&self.entity.name)]; let mut edges: Vec = Vec::new(); { let entity = &mut visits[0]; @@ -311,9 +326,17 @@ impl Walk<'_> { } entity.frames.push(entity_rows); entity.batches.push((0..examples).collect()); - entity.counts[0] = examples; + entity.add_reached(0, examples); } - for (hop, neighbors) in self.neighbors.iter().enumerate() { + for hop in 0.. { + let Some(neighbors) = self.neighbors.get(hop) else { + break; + }; + // A hop no table has rows to walk from reaches none, nor does + // any after it: the walk ends, as it would hop by hop. + if visits.iter().all(|visit| visit.reached(hop) == 0) { + break; + } // A hop of size zero ends the walk (`neighbors == 0`). let neighbors = match neighbors { Hop::Stop => break, @@ -352,7 +375,7 @@ impl Walk<'_> { neighbors: &sdm_contract::data::NeighborLimit, ) -> Result<(), CoreError> { let at = position(visits, name)?; - if visits[at].counts[hop] == 0 { + if visits[at].reached(hop) == 0 { return Ok(()); } let table = table_named(self.graph, name)?; @@ -479,7 +502,7 @@ impl Walk<'_> { return Err(internal("a row looked up was already reached")); } } - dst_visit.counts[hop + 1] += examples.len(); + dst_visit.add_reached(hop + 1, examples.len()); dst_visit.frames.push(added); dst_visit.batches.push(examples); for &i in &missing { @@ -641,7 +664,7 @@ impl Walk<'_> { ), }; let kept_examples: Vec = kept.iter().map(|&i| examples[i]).collect(); - src_visit.counts[hop + 1] += kept.len(); + src_visit.add_reached(hop + 1, kept.len()); src_visit.frames.push(take(&fetched, &kept)?); src_visit.batches.push(kept_examples); edge(edges, src, fkey, name, false).extend(rows, cols); @@ -840,8 +863,7 @@ fn reach(visits: &mut Vec, name: &str) -> usize { match visits.iter().position(|v| v.name == name) { Some(at) => at, None => { - let hops = visits[0].counts.len() - 1; - visits.push(Visit::new(name, hops)); + visits.push(Visit::new(name)); visits.len() - 1 } } diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 4ff4d8346..9f117eaae 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -12,6 +12,7 @@ use sdm_contract::direct::storage::{ DIRECT_STORAGE_KEY, DirectStorage, StorageOwner, TimestampUnit, }; use sdm_contract::direct::*; +use sdm_contract::limits::Limits; use sdm_contract::ops::{HeldSchema, OperationResult}; use sdm_contract::tabular_arrow::*; use sdm_contract::{CoreError, Detail, HostFacts, Sentence}; @@ -292,29 +293,33 @@ fn bound( input: &ArrowPrediction, context: &ArrowData, scoring: &ArrowData, + limits: &Limits, ) -> Result { let description = prepare(asker, &input.request, context, scoring)?; - let bound = direct::bind(DirectCallFacts { - query: None, - model: input.model.clone(), - outputs: crate::pql::explanation_outputs( - input.request.query.as_deref().unwrap_or(""), - &input.request.task, - input.outputs.clone(), - ), - kind: input.request.task.clone(), - anchored: input - .request - .task_options - .entries - .iter() - .rev() - .find(|(name, _)| name == "anchor_time_column") - .is_some_and(|(_, value)| option_truth(value)), - relationships: !description.schema.links.is_empty(), - tables: description.schema.tables.len(), - context_rows: header(context).rows, - })?; + let bound = direct::bind( + DirectCallFacts { + query: None, + model: input.model.clone(), + outputs: crate::pql::explanation_outputs( + input.request.query.as_deref().unwrap_or(""), + &input.request.task, + input.outputs.clone(), + ), + kind: input.request.task.clone(), + anchored: input + .request + .task_options + .entries + .iter() + .rev() + .find(|(name, _)| name == "anchor_time_column") + .is_some_and(|(_, value)| option_truth(value)), + relationships: !description.schema.links.is_empty(), + tables: description.schema.tables.len(), + context_rows: header(context).rows, + }, + limits, + )?; if bound.model != sdm_request::predict::TABULAR { return Err(direct::invalid(Sentence::DirectRelationalFrames) .with_field("model", Detail::text(&bound.model))); @@ -371,8 +376,9 @@ async fn predict( input: ArrowPrediction, context: ArrowData, scoring: ArrowData, + limits: &Limits, ) -> Result { - let prepared = bound(asker, &input, &context, &scoring)?; + let prepared = bound(asker, &input, &context, &scoring, limits)?; let payload = tabular::body( &prepared.columns, &context.batches, @@ -396,8 +402,8 @@ async fn predict( part.columns = analysis; return crate::predict::finish(asker, part, prepared.bound.outputs); } - let answer = - crate::predict::call_answer(asker, payload.to_bytes(), input.endpoint.as_ref()).await?; + let bytes = crate::predict::one_call_bytes(&payload, limits)?; + let answer = crate::predict::call_answer(asker, bytes, input.endpoint.as_ref()).await?; finish(asker, payload, answer, prepared.bound.outputs, analysis) } fn finish( @@ -420,6 +426,7 @@ pub(crate) async fn run( mut attached: Attachments, facts: HostFacts, zones: Arc, + limits: Limits, ) -> Result { match operation { ArrowTabular::Describe { input, table } => { @@ -445,7 +452,7 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let context = take_table(&mut attached, context)?; let scoring = take_table(&mut attached, scoring)?; - predict(&asker, input, context, scoring).await + predict(&asker, input, context, scoring, &limits).await } ArrowTabular::Fit { input, @@ -455,12 +462,12 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let context = take_table(&mut attached, context)?; let scoring = take_table(&mut attached, predict)?; - fit(&asker, input, context, scoring).await + fit(&asker, input, context, scoring, &limits).await } ArrowTabular::Score { input, predict } => { let input = take_json(&mut attached, input)?; let scoring = take_table(&mut attached, predict)?; - score(&asker, input, scoring).await + score(&asker, input, scoring, &limits).await } ArrowTabular::Close { input } => close(&asker, take_json(&mut attached, input)?).await, ArrowTabular::MaterializePredict { @@ -471,12 +478,20 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let table = take_table(&mut attached, table)?; let predict = take_table(&mut attached, predict)?; - materialize(&asker, input, table, Some(predict), &facts, zones.as_ref()) + materialize( + &asker, + input, + table, + Some(predict), + &facts, + zones.as_ref(), + &limits, + ) } ArrowTabular::Materialize { input, table } => { let input = take_json(&mut attached, input)?; let table = take_table(&mut attached, table)?; - materialize(&asker, input, table, None, &facts, zones.as_ref()) + materialize(&asker, input, table, None, &facts, zones.as_ref(), &limits) } } } @@ -519,13 +534,14 @@ async fn fit( input: ArrowFit, context: ArrowData, scoring: ArrowData, + limits: &Limits, ) -> Result { use sdm_contract::{run::Endpoint, session::*}; use tabular::session as policy; if input.prediction.endpoint.is_some() { return Err(invalid("this transport does not support sessions")); } - let prepared = bound(asker, &input.prediction, &context, &scoring)?; + let prepared = bound(asker, &input.prediction, &context, &scoring, limits)?; let encoded = tabular::encode_table(&prepared.columns, &context.batches, crate::pql::float_text)?; let keys = policy::encoded_keys(&encoded, &prepared.keys)?; @@ -549,6 +565,8 @@ async fn fit( tabular::typed_inference(input.prediction.request.inference.clone())?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&payload))?); crate::columns::expose(asker, column_analysis.as_deref(), &prepared.bound.outputs)?; + // The pinned context is one call, which no split makes smaller. + sdm_request::predict::call_bytes(&payload, limits.max_request_bytes)?; let answer = session_call(asker, Endpoint::CreateSession, payload).await?; let session_id = answer .get("session_id") @@ -578,6 +596,7 @@ async fn score( asker: &Asker, mut input: ArrowScore, scoring: ArrowData, + limits: &Limits, ) -> Result { use tabular::session as policy; policy::validate_state(&input.session.state)?; @@ -589,7 +608,7 @@ async fn score( &input.request.task, Some(input.session.state.bound.outputs.clone()), ); - input.session.state.bound = direct::bind(facts)?; + input.session.state.bound = direct::bind(facts, limits)?; } let state = &input.session.state; if state.closed { @@ -780,6 +799,7 @@ fn materialize( supplied: Option, facts: &HostFacts, zones: &dyn Zones, + limits: &Limits, ) -> Result { use sdm_contract::ops::{ProfileName, QueryColumn, QuerySchema, QueryTable}; use sdm_contract::pql::{NodeKind, ProblemType}; @@ -873,7 +893,7 @@ fn materialize( let decision = sdm_engine::tabular::context_decision( candidates.len(), input.num_context_samples, - sdm_request::predict::CONTEXT_ROWS_MAX, + usize::try_from(limits.max_tabular_context_rows).unwrap_or(usize::MAX), ) .map_err(|_| invalid("context sample count is invalid"))?; let context_rows = if decision.reduced { diff --git a/rust/crates/sdm-core/src/tabular_selection.rs b/rust/crates/sdm-core/src/tabular_selection.rs index c11020253..e9ad44ca8 100644 --- a/rust/crates/sdm-core/src/tabular_selection.rs +++ b/rust/crates/sdm-core/src/tabular_selection.rs @@ -35,11 +35,14 @@ fn count_error(error: tabular::CountError) -> TabularCountError { } } -pub(crate) fn decide(input: DecideTabularContext) -> TabularContextDecision { +pub(crate) fn decide( + input: DecideTabularContext, + limits: &sdm_contract::limits::Limits, +) -> TabularContextDecision { match tabular::context_decision( input.population, input.requested, - sdm_request::predict::CONTEXT_ROWS_MAX, + usize::try_from(limits.max_tabular_context_rows).unwrap_or(usize::MAX), ) { Ok(value) => TabularContextDecision::Selected { wanted: value.wanted, @@ -80,7 +83,10 @@ pub(crate) fn order( /// Fuse ordinary expansion, exclusion, target filtering and capacity policy. /// Sampling remains after native warning delivery and uses its reached count. -pub(crate) fn select(input: SelectTabularRows) -> Result { +pub(crate) fn select( + input: SelectTabularRows, + limits: &sdm_contract::limits::Limits, +) -> Result { let (scored, excluded, population) = match input.mode { TabularSelectionMode::Indices { source, requested } => { let expanded = tabular::expand_keys(&source, &requested); @@ -108,10 +114,13 @@ pub(crate) fn select(input: SelectTabularRows) -> Result, ) -> Result { let keyed = match &operation { @@ -106,6 +107,7 @@ pub(crate) fn run( sdm_contract::ops::Operation::Materialize(*materialize), std::mem::take(attachments), facts, + limits, Arc::clone(&zones), true, )?, diff --git a/rust/crates/sdm-core/tests/answer_encoding.rs b/rust/crates/sdm-core/tests/answer_encoding.rs index 6e8654e71..6aad4be56 100644 --- a/rust/crates/sdm-core/tests/answer_encoding.rs +++ b/rust/crates/sdm-core/tests/answer_encoding.rs @@ -247,6 +247,7 @@ fn predict_run() -> Run { start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation, diff --git a/rust/crates/sdm-core/tests/attachments.rs b/rust/crates/sdm-core/tests/attachments.rs index 64d0d3bf0..7b238d100 100644 --- a/rust/crates/sdm-core/tests/attachments.rs +++ b/rust/crates/sdm-core/tests/attachments.rs @@ -50,6 +50,7 @@ const READ: &[&str] = &[ "OrderSampledTabularRows.input", "SelectTabularRows.input", "CheckTabularCap.input", + "ResolveLimits.input", "CheckDirectShape.input", "BindDirect.input", "CheckDirectRows.input", diff --git a/rust/crates/sdm-core/tests/dispatch.rs b/rust/crates/sdm-core/tests/dispatch.rs index 63acae7a6..a33ca719d 100644 --- a/rust/crates/sdm-core/tests/dispatch.rs +++ b/rust/crates/sdm-core/tests/dispatch.rs @@ -136,6 +136,7 @@ fn run_of(root: &CompositionRoot, operation: Operation) -> Result Result Result { start( &root(), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation, @@ -786,6 +791,7 @@ fn validate_run(query: &str, facts: HostFacts) -> Result Result { + use sdm_contract::direct::DirectInput; + let input = AttachmentId(1); + let mut run = start( + &root(), + StartRun { + limits, + contract: CONTRACT_VERSION, + facts: facts(), + operation: Operation::ResolveLimits(DirectInput { input }), + }, + Attachments::from([(input, json(&serde_json::json!({})))]), + )?; + let Step::Done { + result: OperationResult::DirectValue(value), + } = run.next()? + else { + panic!("the run answers a value"); + }; + let Some(Attachment::Bytes(bytes)) = run.take_attachment(value.value) else { + panic!("the value is attached"); + }; + Ok(serde_json::from_slice(&bytes).unwrap()) +} + +#[test] +fn a_run_resolves_its_limits_the_caller_s_before_the_environment_s_before_the_default() { + use sdm_contract::limits::GivenLimits; + let defaults = resolved(GivenLimits::default()).unwrap(); + assert_eq!( + defaults, + serde_json::json!({ + "max_relational_context_rows": 10_000, + "max_relational_predict_rows": 10_000, + "max_temporal_link_predict_rows": 200, + "max_tabular_context_rows": 10_000, + "max_request_bytes": 31_457_280, + }) + ); + let raised = resolved(GivenLimits { + given: BTreeMap::from([("max_relational_predict_rows".to_owned(), 50_000)]), + environment: BTreeMap::from([ + ( + "SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS".to_owned(), + "200000".to_owned(), + ), + ( + "SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS".to_owned(), + "20000".to_owned(), + ), + ]), + }) + .unwrap(); + assert_eq!(raised["max_tabular_context_rows"], 200_000); + assert_eq!(raised["max_relational_predict_rows"], 50_000); + assert_eq!(raised["max_request_bytes"], 31_457_280); + // A variable that is not a whole number refuses the start, by name. + let refused = resolved(GivenLimits { + given: BTreeMap::new(), + environment: BTreeMap::from([( + "SDM_CLIENT_MAX_REQUEST_BYTES".to_owned(), + "lots".to_owned(), + )]), + }) + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::InvalidConfiguration); + assert_eq!( + refused.fields["env"], + sdm_contract::Detail::text("SDM_CLIENT_MAX_REQUEST_BYTES") + ); +} diff --git a/rust/crates/sdm-core/tests/edit_schema.rs b/rust/crates/sdm-core/tests/edit_schema.rs index bfb974cdd..5fb93bc37 100644 --- a/rust/crates/sdm-core/tests/edit_schema.rs +++ b/rust/crates/sdm-core/tests/edit_schema.rs @@ -54,6 +54,7 @@ fn start(operation: Operation, attachments: Attachments) -> Result Run { sdm_core::start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-core/tests/held.rs b/rust/crates/sdm-core/tests/held.rs index 991851b13..1bc8ee891 100644 --- a/rust/crates/sdm-core/tests/held.rs +++ b/rust/crates/sdm-core/tests/held.rs @@ -173,6 +173,7 @@ fn try_run( start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Predict(Predict { diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 75a4e644a..05d9e1531 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -214,6 +214,7 @@ fn run_of(operation: Operation, attachments: Attachments) -> Result Result panic!("not a call: {other:?}"), } }; - let limit = sdm_request::payload::PAYLOAD_BYTES; + let limit = sdm_contract::limits::MAX_REQUEST_BYTES.default as usize; let base = called(padded("")).unwrap(); - // ASCII to the limit, exactly: taken; one byte more: refused. + // ASCII to the limit, exactly: taken whole; one byte more: split, the + // first call smaller. assert_eq!(called(padded(&"a".repeat(limit - base))).unwrap(), limit); - let past = called(padded(&"a".repeat(limit - base + 1))).unwrap_err(); + let split = called(padded(&"a".repeat(limit - base + 1))).unwrap(); + assert!(split < limit, "{split}"); + // Padding past the limit on its own: no call is small enough. + let past = called(padded(&"a".repeat(limit))).unwrap_err(); assert_eq!(past.code, ErrorCode::InvalidRequest); + assert_eq!(past.sentence, Sentence::CallPastEndpoint); assert_eq!(past.fields["field"], text("call")); + assert_eq!(past.fields["setting"], text("max_request_bytes")); + assert_eq!(past.fields["env"], text("SDM_CLIENT_MAX_REQUEST_BYTES")); // Three-byte characters, each counted as its three bytes written. let euros = (limit - base) / 3; let rest = limit - base - 3 * euros; let at = format!("{}{}", "\u{20ac}".repeat(euros), "a".repeat(rest)); assert_eq!(called(padded(&at)).unwrap(), limit); let over = format!("{}{}", "\u{20ac}".repeat(euros + 1), "a".repeat(rest)); - assert!(called(padded(&over)).is_err()); + assert!(called(padded(&over)).unwrap() < limit); } diff --git a/rust/crates/sdm-core/tests/serving.rs b/rust/crates/sdm-core/tests/serving.rs index 92f611d81..dc5ceaf4f 100644 --- a/rust/crates/sdm-core/tests/serving.rs +++ b/rust/crates/sdm-core/tests/serving.rs @@ -183,6 +183,7 @@ fn predict(calls: &[&[i64]], endpoint: Option) -> Run { start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Predict(Predict { @@ -349,10 +350,9 @@ fn each_call_is_wrapped_as_the_endpoint_reads_it_and_answered_from_its_row() { } #[test] -fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { - // The first call fits; the second, larger, does not: it is refused - // when it would be sent, carrying the first call's answers, as the old - // transport weighs each call as it sends it. +fn a_call_past_the_caller_s_limit_is_split_and_one_that_cannot_be_is_refused() { + // The first call fits; the second, larger, does not: it is asked as + // smaller calls, each with the whole context, each within the limit. let mut probe = predict(&[&[10]], serving(16_000_000)); let Ok(Step::CallModel(call)) = probe.next() else { panic!("a call"); @@ -367,18 +367,30 @@ fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { retryable: false, }; let (end, bodies) = drive(&mut run, None, &failure); - assert_eq!(bodies.len(), 1); + assert_eq!(bodies.len(), 4); assert_eq!(bodies[0], one); - let error = end.unwrap_err(); - assert_eq!(error.sentence, Sentence::ServingPastLimit); - assert_eq!(error.code, ErrorCode::InvalidRequest); - assert_eq!(error.fields["limit"], Detail::Scalar(Scalar::UInt64(limit))); - assert_eq!(count(&error, "calls_answered"), Some(1)); - // A limit the first call misses: nothing is sent. + assert!(bodies.iter().all(|body| body.len() as u64 <= limit)); + let Ok(Step::Done { + result: OperationResult::Predict(predicted), + }) = end + else { + panic!("the run predicts: {end:?}"); + }; + let result = arrow(&mut run, predicted.result); + assert_eq!(int64s(&result, "row_index"), (0..4).collect::>()); + // A limit one row's call misses: no call is small enough, and nothing + // is sent. let mut run = predict(&[&[10]], serving(limit - 1)); let (end, bodies) = drive(&mut run, None, &failure); assert!(bodies.is_empty()); - assert_eq!(end.unwrap_err().sentence, Sentence::ServingPastLimit); + let error = end.unwrap_err(); + assert_eq!(error.sentence, Sentence::ServingPastLimit); + assert_eq!(error.code, ErrorCode::InvalidRequest); + assert_eq!( + error.fields["limit"], + Detail::Scalar(Scalar::UInt64(limit - 1)) + ); + assert!(count(&error, "context_bytes").is_some_and(|context| context < limit)); } #[test] diff --git a/rust/crates/sdm-core/tests/tabular_sessions.rs b/rust/crates/sdm-core/tests/tabular_sessions.rs index bd2f6bcbf..dc9eab19c 100644 --- a/rust/crates/sdm-core/tests/tabular_sessions.rs +++ b/rust/crates/sdm-core/tests/tabular_sessions.rs @@ -34,6 +34,7 @@ fn run( sdm_core::start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { @@ -116,16 +117,19 @@ fn send(endpoint: Endpoint, payload: Vec, response: Value) -> Value { #[test] fn fit_score_twice_and_retry_close_with_remapped_attachments() { - let bound = sdm_request::direct::bind(DirectCallFacts { - query: None, - model: "kumo-tabular".into(), - outputs: Some(vec!["prediction".into()]), - kind: "regression".into(), - anchored: false, - relationships: false, - tables: 1, - context_rows: 3, - }) + let bound = sdm_request::direct::bind( + DirectCallFacts { + query: None, + model: "kumo-tabular".into(), + outputs: Some(vec!["prediction".into()]), + kind: "regression".into(), + anchored: false, + relationships: false, + tables: 1, + context_rows: 3, + }, + &Default::default(), + ) .unwrap(); let columns = TabularColumns { dtypes: vec![("id".into(), Dtype::Int64), ("y".into(), Dtype::Float64)], diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index 23fa7ebe7..562c77402 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -337,6 +337,7 @@ fn replay_spec( let mut run = start( &root, StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { @@ -890,6 +891,7 @@ fn materialize_query( let mut run = start( &root, StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Materialize(Materialize { diff --git a/rust/crates/sdm-core/tests/view_semantic.rs b/rust/crates/sdm-core/tests/view_semantic.rs index d2d49f4bc..5a7293174 100644 --- a/rust/crates/sdm-core/tests/view_semantic.rs +++ b/rust/crates/sdm-core/tests/view_semantic.rs @@ -241,6 +241,7 @@ fn semantic_run(definition: &str, spec: SourceSpec) -> sdm_core::Run { start( &CompositionRoot::new(Arc::new(fake)), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-engine/src/frames.rs b/rust/crates/sdm-engine/src/frames.rs index 1c392bd1b..f0b4dd395 100644 --- a/rust/crates/sdm-engine/src/frames.rs +++ b/rust/crates/sdm-engine/src/frames.rs @@ -237,6 +237,13 @@ pub enum FrameNeighbor { } impl Snapshot { + /// How many rows the snapshot admits across its tables: a walk reaches + /// no row it has not reached at any hop past this many, since each hop + /// that reaches anything reaches at least one row anew. + pub fn rows(&self) -> usize { + self.tables.values().map(|table| table.valid.len()).sum() + } + /// Physical storage of one admitted column, without scanning or sampling rows. pub fn column_type(&self, table: &str, column: &str) -> Result { let rows = &self diff --git a/rust/crates/sdm-host/tests/codec.rs b/rust/crates/sdm-host/tests/codec.rs index 46369e4b8..b86b4e00f 100644 --- a/rust/crates/sdm-host/tests/codec.rs +++ b/rust/crates/sdm-host/tests/codec.rs @@ -27,6 +27,7 @@ use sdm_host::{CONTROL_CAP, Outgoing, decode, encode, read, write, write_error}; /// envelope. fn start(model: String) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/contract_v3.rs b/rust/crates/sdm-host/tests/contract_v3.rs index eb7e14f95..1f7fa0952 100644 --- a/rust/crates/sdm-host/tests/contract_v3.rs +++ b/rust/crates/sdm-host/tests/contract_v3.rs @@ -53,6 +53,7 @@ fn facts(local_zone: Option) -> HostFacts { fn start_message(operation: Operation, local_zone: Option) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(local_zone), operation, diff --git a/rust/crates/sdm-host/tests/data_port.rs b/rust/crates/sdm-host/tests/data_port.rs index a1437f60b..e2722523e 100644 --- a/rust/crates/sdm-host/tests/data_port.rs +++ b/rust/crates/sdm-host/tests/data_port.rs @@ -56,6 +56,7 @@ fn fixture(files: &[(&str, &str)]) -> PathBuf { fn start_message(operation: Operation, roots: &[&Path]) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/error_table.rs b/rust/crates/sdm-host/tests/error_table.rs index 46b10fc76..10a55ed59 100644 --- a/rust/crates/sdm-host/tests/error_table.rs +++ b/rust/crates/sdm-host/tests/error_table.rs @@ -26,6 +26,7 @@ use sdm_host::{Attachments, HostRun, Outgoing, ZoneResolver, encode, read}; fn start_message(operation: TestOperation) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/faults.rs b/rust/crates/sdm-host/tests/faults.rs index 2490c35ef..3bf855e58 100644 --- a/rust/crates/sdm-host/tests/faults.rs +++ b/rust/crates/sdm-host/tests/faults.rs @@ -43,6 +43,7 @@ use sdm_host::{HostRun, Outgoing, Unreadable, Unreadables, encode, read}; fn start_message(operation: Operation) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { @@ -833,6 +834,7 @@ fn a_test_only_panic_s_payload_is_the_canary() { let mut run = sdm_core::start( &sdm_host::default_root(), StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/runs.rs b/rust/crates/sdm-host/tests/runs.rs index 94393bf4e..8e1b81de3 100644 --- a/rust/crates/sdm-host/tests/runs.rs +++ b/rust/crates/sdm-host/tests/runs.rs @@ -53,6 +53,7 @@ fn facts(zone_roots: Vec) -> HostFacts { fn start_message(operation: Operation, zone_roots: Vec) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(zone_roots), operation, diff --git a/rust/crates/sdm-host/tests/type_set.rs b/rust/crates/sdm-host/tests/type_set.rs index b55a1f42d..9d2882035 100644 --- a/rust/crates/sdm-host/tests/type_set.rs +++ b/rust/crates/sdm-host/tests/type_set.rs @@ -828,6 +828,7 @@ fn infer_column(received: &DataType) -> Value { "time_columns": {}, }); let start = StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-request/src/batching.rs b/rust/crates/sdm-request/src/batching.rs new file mode 100644 index 000000000..d6e04475a --- /dev/null +++ b/rust/crates/sdm-request/src/batching.rs @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A call's predict rows taken apart, so a call past the most bytes one call +//! takes is asked as several smaller ones. +//! +//! A payload carries its context whole and the rows it scores beside it, +//! each row of its predict half's tables belonging to one of the rows +//! scored by its `instance_id`. [`predict_slice`] keeps the context as it +//! is and a run of the rows scored, with the rows of every related table +//! that belong to them, numbered as a payload of only those rows would be. +//! The context is sent with every call, so it is never split. + +use std::collections::BTreeSet; +use std::ops::Range; + +use sdm_contract::{CoreError, Sentence}; +use sdm_wire::json::Value; + +use crate::payload::INSTANCE_ID; + +fn bad(reason: &str) -> CoreError { + CoreError::new( + Sentence::Internal, + format!("a payload cannot be split: {reason}"), + ) +} + +/// The rows a table of the payload holds: its column names and rows. +fn table_rows(table: &Value) -> Option<(Vec<&str>, &[Value])> { + let Some(Value::Array(columns)) = table.get("columns") else { + return None; + }; + let Some(Value::Array(rows)) = table.get("rows") else { + return None; + }; + let names: Vec<&str> = columns.iter().filter_map(Value::as_text).collect(); + (names.len() == columns.len()).then_some((names, rows.as_slice())) +} + +/// How many rows the payload scores: its predict half's instance rows. +pub fn predict_rows(payload: &Value) -> Option { + let table = payload.get("predict")?.get("instance_table")?; + table_rows(table).map(|(_, rows)| rows.len()) +} + +/// An instance id as a whole number, where it is one. +fn id_of(value: &Value) -> Option { + match value { + Value::Number(token) => token.parse::().ok(), + _ => None, + } +} + +/// `table` with only the rows `keep` takes, each row's `instance_id` moved +/// `shift` places down where it is a whole number. +fn kept( + table: &Value, + keep: impl Fn(usize, &[Value]) -> bool, + id_at: Option, + shift: i64, +) -> Result { + let (_, rows) = table_rows(table).ok_or_else(|| bad("a table has no rows to take"))?; + let rows = rows + .iter() + .enumerate() + .filter_map(|(at, row)| { + let Value::Array(cells) = row else { + return Some(Err(bad("a row is not a list"))); + }; + if !keep(at, cells) { + return None; + } + let mut cells = cells.clone(); + if shift != 0 + && let Some(at) = id_at + && let Some(id) = cells.get(at).and_then(id_of) + { + cells[at] = Value::Number((id - shift).to_string()); + } + Some(Ok(Value::Array(cells))) + }) + .collect::, _>>()?; + let mut table = table.clone(); + table.set("rows", Value::Array(rows)); + Ok(table) +} + +/// `payload` scoring only its predict rows `range`, in order, with the rows +/// of each related table of its predict half that belong to them, and its +/// context whole. The rows kept are numbered from where the payload's own +/// were (`instance_id`), as a payload built of only those rows numbers +/// them, and its metadata counts them (`num_prediction_examples`). +pub fn predict_slice(payload: &Value, range: Range) -> Result { + let predict = payload + .get("predict") + .ok_or_else(|| bad("there is no predict half"))?; + let instance = predict + .get("instance_table") + .ok_or_else(|| bad("there is no instance table"))?; + let (names, rows) = table_rows(instance).ok_or_else(|| bad("the instance table is unread"))?; + if range.end > rows.len() || range.start > range.end { + return Err(bad("the rows asked for are not there")); + } + let id_at = names.iter().position(|name| *name == INSTANCE_ID); + // The ids of the rows kept, and how far they move down: a run of rows + // numbered on from the first row of the half is numbered from it again. + let ids = |row: &Value| match (row, id_at) { + (Value::Array(cells), Some(at)) => cells.get(at).and_then(id_of), + _ => None, + }; + let kept_ids: BTreeSet = rows[range.clone()].iter().filter_map(ids).collect(); + let shift = match ( + rows.first().and_then(ids), + rows.get(range.start).and_then(ids), + ) { + (Some(first), Some(start)) if kept_ids.len() == range.len() => start - first, + _ => 0, + }; + let mut half = predict.clone(); + half.set( + "instance_table", + kept(instance, |at, _| range.contains(&at), id_at, shift)?, + ); + if let Some(Value::Object(related)) = predict.get("related_tables") { + let mut tables = Vec::with_capacity(related.len()); + for (name, table) in related { + let (columns, _) = table_rows(table).ok_or_else(|| bad("a related table is unread"))?; + let at = columns.iter().position(|column| *column == INSTANCE_ID); + let Some(at) = at.filter(|_| id_at.is_some()) else { + // A related table whose rows name no instance belongs to no + // one row scored: kept whole. + tables.push((name.clone(), table.clone())); + continue; + }; + let belongs = |_: usize, cells: &[Value]| { + cells + .get(at) + .and_then(id_of) + .is_some_and(|id| kept_ids.contains(&id)) + }; + tables.push((name.clone(), kept(table, belongs, Some(at), shift)?)); + } + half.set("related_tables", Value::Object(tables)); + } + let mut sliced = payload.clone(); + sliced.set("predict", half); + if let Some(metadata @ Value::Object(_)) = payload.get("metadata") + && metadata.get("num_prediction_examples").is_some() + { + let mut metadata = metadata.clone(); + metadata.set( + "num_prediction_examples", + Value::Number(range.len().to_string()), + ); + sliced.set("metadata", metadata); + } + Ok(sliced) +} + +/// The calls `payload` is asked as so that each fits, in order, each its +/// rows' range: the whole where it fits (`fits`, given the payload a call +/// would carry), else halves, each split again until it fits. A call of one +/// row that does not fit cannot be split, and is refused by `refuse`, given +/// that one row's payload: the context is sent with every call. +pub fn fitted( + payload: &Value, + fits: &mut dyn FnMut(&Value) -> Result, + refuse: &mut dyn FnMut(&Value) -> CoreError, +) -> Result, Value)>, CoreError> { + if fits(payload)? { + let rows = predict_rows(payload).unwrap_or(0); + return Ok(vec![(0..rows, payload.clone())]); + } + let rows = predict_rows(payload).ok_or_else(|| bad("the rows scored are unread"))?; + let mut done = Vec::new(); + // Ranges still to fit, the first next. + let mut pending: Vec> = Vec::new(); + pending.push(0..rows); + while let Some(range) = pending.pop() { + let sliced = predict_slice(payload, range.clone())?; + if range.len() != rows && fits(&sliced)? { + done.push((range, sliced)); + continue; + } + if range.len() <= 1 { + return Err(refuse(&sliced)); + } + let middle = range.start + range.len() / 2; + pending.push(middle..range.end); + pending.push(range.start..middle); + } + Ok(done) +} + +#[cfg(test)] +mod tests { + use super::*; + use sdm_wire::json; + + fn payload() -> Value { + json::read( + br#"{ + "model": "kumo-relational", + "context": {"instance_table": {"format": "rows", "columns": ["instance_id", "TARGET"], "rows": [[0, 1], [1, 2]]}, + "related_tables": {"users": {"format": "rows", "columns": ["instance_id", "user_id"], "rows": [[0, 10], [1, 11]]}}}, + "predict": {"instance_table": {"format": "rows", "columns": ["instance_id", "__kumo_entity_ref_0"], "rows": [[2, 20], [3, 21], [4, 22], [5, 23]]}, + "related_tables": {"users": {"format": "rows", "columns": ["instance_id", "user_id"], "rows": [[2, 20], [3, 21], [4, 22], [5, 23], [3, 99]]}, + "plain": {"format": "rows", "columns": ["x"], "rows": [[7]]}}}, + "metadata": {"num_context_examples": 2, "num_prediction_examples": 4} +}"#, + ) + .unwrap() + } + + fn rows(value: &Value, path: &[&str]) -> String { + let mut at = value; + for key in path { + at = at.get(key).unwrap(); + } + String::from_utf8(at.get("rows").unwrap().to_bytes()).unwrap() + } + + #[test] + fn a_slice_keeps_the_context_and_the_rows_of_its_instances_renumbered() { + let whole = payload(); + let sliced = predict_slice(&whole, 1..3).unwrap(); + assert_eq!(sliced.get("context"), whole.get("context")); + assert_eq!( + rows(&sliced, &["predict", "instance_table"]), + "[[2,21],[3,22]]" + ); + assert_eq!( + rows(&sliced, &["predict", "related_tables", "users"]), + "[[2,21],[3,22],[2,99]]" + ); + assert_eq!( + rows(&sliced, &["predict", "related_tables", "plain"]), + "[[7]]" + ); + assert_eq!( + sliced + .get("metadata") + .unwrap() + .get("num_prediction_examples"), + Some(&Value::Number("2".into())) + ); + // The first run is numbered as it was. + let first = predict_slice(&whole, 0..1).unwrap(); + assert_eq!(rows(&first, &["predict", "instance_table"]), "[[2,20]]"); + assert_eq!(predict_slice(&whole, 0..4).unwrap(), whole); + assert!(predict_slice(&whole, 2..5).is_err()); + } + + fn size(value: &Value) -> usize { + value.to_bytes().len() + } + + #[test] + fn a_call_that_fits_is_one_call_and_one_past_is_halved_until_each_fits() { + let whole = payload(); + let one = size(&predict_slice(&whole, 0..1).unwrap()); + let two = size(&predict_slice(&whole, 0..2).unwrap()); + let all = size(&whole); + let mut refuse = |_: &Value| CoreError::new(Sentence::Internal, "refused"); + let calls = fitted(&whole, &mut |p| Ok(size(p) <= all), &mut refuse).unwrap(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, 0..4); + // One byte under the whole: two calls of two rows. + let calls = fitted(&whole, &mut |p| Ok(size(p) < all), &mut refuse).unwrap(); + let ranges: Vec<_> = calls.iter().map(|(r, _)| r.clone()).collect(); + assert_eq!(ranges, [0..2, 2..4]); + // Exactly two rows' worth: still two calls; one byte under it: four. + let ranges: Vec<_> = fitted(&whole, &mut |p| Ok(size(p) <= two), &mut refuse) + .unwrap() + .into_iter() + .map(|(r, _)| r) + .collect(); + assert_eq!(ranges, [0..2, 2..4]); + let ranges: Vec<_> = fitted(&whole, &mut |p| Ok(size(p) < two), &mut refuse) + .unwrap() + .into_iter() + .map(|(r, _)| r) + .collect(); + // The last two rows carry fewer related rows than the first two, + // so together they fit where the first two do not. + assert_eq!(ranges, [0..1, 1..2, 2..4]); + // Under one row's worth, the context alone does not fit. + let mut refused_rows = None; + let error = fitted(&whole, &mut |p| Ok(size(p) < one), &mut |p| { + refused_rows = predict_rows(p); + CoreError::new(Sentence::Internal, "refused") + }) + .unwrap_err(); + assert_eq!(error.message, "refused"); + assert_eq!(refused_rows, Some(1)); + } +} diff --git a/rust/crates/sdm-request/src/direct.rs b/rust/crates/sdm-request/src/direct.rs index 440e6240d..9dde43da4 100644 --- a/rust/crates/sdm-request/src/direct.rs +++ b/rust/crates/sdm-request/src/direct.rs @@ -5,6 +5,7 @@ //! Model-neutral direct shape guards and shared model preflight facts. use sdm_contract::direct::{BoundDirectCall, ColumnFlags, DirectCallFacts, DirectShape, FrameKind}; +use sdm_contract::limits::Limits; use sdm_contract::ops::{DescribedTable, Dtype, Stype}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use std::collections::BTreeSet; @@ -106,13 +107,15 @@ pub fn column_names(columns: &[String], table: &str) -> Result<(), CoreError> { } /// Resolve model/output/capability/task/capacity before inspecting scoring values. -pub fn bind(input: DirectCallFacts) -> Result { - crate::predict::bind_direct(input) +pub fn bind(input: DirectCallFacts, limits: &Limits) -> Result { + crate::predict::bind_direct(input, Some(limits)) } /// Validate the producer's typed decision against its original consumed facts. +/// The capacity was checked when it was bound, and is checked again where +/// the request is sent. pub fn bound(value: &BoundDirectCall) -> Result<(), CoreError> { - if bind(value.facts.clone())? != *value { + if crate::predict::bind_direct(value.facts.clone(), None)? != *value { return Err(CoreError::protocol( Sentence::InvalidMessage, "bound direct call differs from its consumed facts", diff --git a/rust/crates/sdm-request/src/lib.rs b/rust/crates/sdm-request/src/lib.rs index 41f0b89cb..7371cebbf 100644 --- a/rust/crates/sdm-request/src/lib.rs +++ b/rust/crates/sdm-request/src/lib.rs @@ -14,6 +14,7 @@ #![forbid(unsafe_code)] +pub mod batching; pub mod inference; pub mod options; pub mod payload; diff --git a/rust/crates/sdm-request/src/options.rs b/rust/crates/sdm-request/src/options.rs index ffa61d7e8..930935fd6 100644 --- a/rust/crates/sdm-request/src/options.rs +++ b/rust/crates/sdm-request/src/options.rs @@ -8,6 +8,7 @@ //! Errors identify the option and reason without exposing the supplied value. use sdm_contract::data::NegativeNeighborBound; +use sdm_contract::limits::{Limits, MAX_RELATIONAL_CONTEXT_ROWS}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_wire::json::{Value, is_integer_token, whole}; use sdm_wire::payload::{ClassificationConfig, InferenceConfig}; @@ -591,14 +592,50 @@ pub enum Hop { Stop, } +/// The hops a walk takes, each with the neighbors it keeps of each row: the +/// sizes listed, then the last of them again for as many hops more as were +/// asked. A walk reads them one hop at a time ([`Hops::get`]), so a count +/// of hops far past what the data holds costs nothing: the walk ends at +/// the first hop that reaches no row. +#[derive(Debug, Clone, PartialEq)] +pub struct Hops { + listed: Vec, + /// How many hops follow the listed ones, each the last listed size. + more: usize, +} + +impl Hops { + /// The size of hop `hop`, counted from zero; none past the last. + pub fn get(&self, hop: usize) -> Option<&Hop> { + match self.listed.get(hop) { + Some(size) => Some(size), + None if hop - self.listed.len() < self.more => self.listed.last(), + None => None, + } + } +} + +impl PartialEq> for Hops { + fn eq(&self, other: &Vec) -> bool { + self.more == 0 && &self.listed == other + } +} + +impl From> for Hops { + fn from(listed: Vec) -> Self { + Hops { listed, more: 0 } + } +} + /// The hops the walk takes, each with its size, as the engine resolves /// them (`_resolve_options`): `num_neighbors` where it is given, each size /// read as [`hop`] reads it; else the first `num_hops` of the run mode's sizes -/// (`_DEFAULT_NUM_NEIGHBORS`), two where it is not named, and a `num_hops` +/// (`_DEFAULT_NUM_NEIGHBORS`), two where it is not named, each hop past the +/// sizes the mode lists keeping as many as its last; and a `num_hops` /// named as none, which the engine cannot compare with its bounds, /// `INVALID_REQUEST` (`not_a_whole_number`), as the engine's `TypeError` -/// is the caller's. Call it after [`hops`] has bounded the count. -pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { +/// is the caller's. Call it after [`hops`] has checked the count. +pub fn neighbors(asked: &Asked, mode: RunMode) -> Result { let Some(given) = &asked.neighbors else { if asked.num_hops_none { return Err(mistyped( @@ -609,14 +646,13 @@ pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { } let hops = asked .num_hops - .and_then(|n| n.magnitude) - .map_or(2, |n| usize::try_from(n).unwrap_or(usize::MAX)); - return Ok(mode - .all_neighbors() - .into_iter() - .take(hops) - .map(Hop::Keep) - .collect()); + .map_or(Some(2), |n| n.magnitude) + .map_or(usize::MAX, |n| usize::try_from(n).unwrap_or(usize::MAX)); + let sizes = mode.all_neighbors(); + return Ok(Hops { + listed: sizes.into_iter().take(hops).map(Hop::Keep).collect(), + more: hops.saturating_sub(sizes.len()), + }); }; given .iter() @@ -624,7 +660,8 @@ pub fn neighbors(asked: &Asked, mode: RunMode) -> Result, CoreError> { .map(|(at, size)| { hop(size).map_err(|e| e.with_field("hop", Detail::Scalar(Scalar::UInt64(at as u64)))) }) - .collect() + .collect::, _>>() + .map(Hops::from) } /// One hop's size, as the reference's DuckDB sampler reads the value it @@ -790,16 +827,11 @@ pub fn rows_per_call(given: &Value) -> Result, CoreError> { } } -/// The most rows one model call scores (`predict_rows_max`, the serving -/// layer's `TFM_MAX_TEST_ROWS`, which the reference's client and engine -/// hold at 10,000 from `main`'s 1fc5a686). -pub const PREDICT_ROWS: u64 = 10_000; - /// How many entities one call scores, as the caller asked or as the -/// endpoint allows (`rows_one_call_scores`): the caller's size only ever -/// lowers the endpoint's. -pub fn rows_one_call_scores(asked: Option) -> u64 { - rows_one_task_call_scores(asked, BatchTask::Ordinary) +/// limits allow (`rows_one_call_scores`): the caller's size only ever +/// lowers the limit's. +pub fn rows_one_call_scores(asked: Option, limits: &Limits) -> u64 { + rows_one_task_call_scores(asked, BatchTask::Ordinary, limits) } /// The task distinction that determines an entity batch's model limit. @@ -810,12 +842,15 @@ pub enum BatchTask { TemporalLink, } -/// Apply the task's serving limit after validating the requested positive size. +/// Apply the task's serving limit after validating the requested positive +/// size: the most rows one call scores (`max_relational_predict_rows`, the +/// serving layer's `TFM_MAX_TEST_ROWS`), or one temporal link prediction +/// call (`max_temporal_link_predict_rows`). /// PQL and supplied examples use the same policy; hosts do not split rows. -pub fn rows_one_task_call_scores(asked: Option, task: BatchTask) -> u64 { +pub fn rows_one_task_call_scores(asked: Option, task: BatchTask, limits: &Limits) -> u64 { let limit = match task { - BatchTask::Ordinary => PREDICT_ROWS, - BatchTask::TemporalLink => 200, + BatchTask::Ordinary => limits.max_relational_predict_rows, + BatchTask::TemporalLink => limits.max_temporal_link_predict_rows, }; asked.map_or(limit, |n| n.min(limit)) } @@ -860,7 +895,8 @@ impl RunMode { } } - /// The most context examples the mode gathers. + /// The most context examples the mode gathers where the caller asks + /// for no more: a best run gathers as many as are asked past it. pub fn context_examples(self) -> usize { match self { RunMode::Fast => 1_000, @@ -881,10 +917,14 @@ impl RunMode { /// The run mode a context size asks for, as the client chooses it /// (`_run_mode_for`, with `num_context_samples`' bound of /// `_validate_materialize_args`): none or up to a fast run's examples, a -/// fast run; up to a best run's, a best one. Below 1 is `INVALID_REQUEST`; -/// past a best run's examples, `UNSUPPORTED_REQUEST`, as the endpoint takes -/// no more. -pub fn run_mode(asked: Option, given: Option<&str>) -> Result { +/// fast run; up to `most`, a best one. Below 1 is `INVALID_REQUEST`; +/// past `most` (`max_relational_context_rows`), `UNSUPPORTED_REQUEST`, +/// naming the setting that raises it. +pub fn run_mode( + asked: Option, + given: Option<&str>, + most: u64, +) -> Result { // A refusal names the size as the caller gave it, its JSON text. let named = |refused: CoreError| match given { Some(given) => refused.with_field("given", Detail::text(given)), @@ -908,29 +948,24 @@ pub fn run_mode(asked: Option, given: Option<&str>) -> Result Ok(RunMode::Fast), - Some(n) if n <= RunMode::Best.context_examples() as u128 => Ok(RunMode::Best), - _ => Err(named( + Some(n) if n <= u128::from(most) => Ok(RunMode::Best), + _ => Err(MAX_RELATIONAL_CONTEXT_ROWS.named(named( CoreError::new( Sentence::NumContextSamplesPastMost, - "num_context_samples is past the most context the endpoint takes", + "num_context_samples is past the most context a request carries", ) .with_field("field", Detail::text("num_context_samples")) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(RunMode::Best.context_examples() as u64)), - ), - )), + .with_field("limit", Detail::Scalar(Scalar::UInt64(most))), + ))), } } -/// The most hops the endpoint reads (`hops_max`). -pub const HOPS_MAX: u128 = 6; - -/// The hops the sampling options ask for, as the client bounds them +/// The hops the sampling options ask for, as the client checks them /// (`_check_shape`): `num_neighbors`' length where it is given, else /// `num_hops`; none, or fewer than one, is `INVALID_REQUEST` (a request -/// that crosses no relationship), more than [`HOPS_MAX`] -/// `UNSUPPORTED_REQUEST`. +/// that crosses no relationship). The client sets no most: the model reads +/// as many hops as the request carries, and a walk ends at the first hop +/// that reaches no row. pub fn hops(num_hops: Option, num_neighbors: Option) -> Result<(), CoreError> { let ( field, @@ -943,34 +978,19 @@ pub fn hops(num_hops: Option, num_neighbors: Option) -> Result (None, Some(n)) => ("num_hops", n), (None, None) => return Ok(()), }; - // How many hops were asked, as the host's words name them. - let asked = value.map(|n| format!("{}{n}", if negative && n > 0 { "-" } else { "" })); - let named = |error: CoreError| match &asked { - Some(n) => error.with_field("hops", Detail::text(n)), - None => error, - }; if negative || value == Some(0) { + // How many hops were asked, as the host's words name them. + let asked = value.map(|n| format!("{}{n}", if negative && n > 0 { "-" } else { "" })); let sentence = if field == "num_hops" { Sentence::NoHopNumHops } else { Sentence::NoHopNumNeighbors }; - return Err(named(invalid(sentence, field, "no_hop")) - .with_field("limit", Detail::Scalar(Scalar::UInt64(HOPS_MAX as u64)))); - } - if value.is_none_or(|n| n > HOPS_MAX) { - return Err(named( - CoreError::new( - if asked.is_some() { - Sentence::HopsPastEndpoint - } else { - Sentence::HopsPastEndpointUncounted - }, - "the sampling options ask more hops than the endpoint reads", - ) - .with_field("field", Detail::text(field)) - .with_field("limit", Detail::Scalar(Scalar::UInt64(HOPS_MAX as u64))), - )); + let refused = invalid(sentence, field, "no_hop"); + return Err(match asked { + Some(n) => refused.with_field("hops", Detail::text(n)), + None => refused, + }); } Ok(()) } @@ -1369,18 +1389,22 @@ mod tests { #[test] fn the_run_mode_is_the_client_s() { - assert_eq!(run_mode(None, None).unwrap(), RunMode::Fast); - assert_eq!(run_mode(int(false, Some(1)), None).unwrap(), RunMode::Fast); + let most = MAX_RELATIONAL_CONTEXT_ROWS.default; + assert_eq!(run_mode(None, None, most).unwrap(), RunMode::Fast); + assert_eq!( + run_mode(int(false, Some(1)), None, most).unwrap(), + RunMode::Fast + ); assert_eq!( - run_mode(int(false, Some(1_000)), None).unwrap(), + run_mode(int(false, Some(1_000)), None, most).unwrap(), RunMode::Fast ); assert_eq!( - run_mode(int(false, Some(1_001)), None).unwrap(), + run_mode(int(false, Some(1_001)), None, most).unwrap(), RunMode::Best ); assert_eq!( - run_mode(int(false, Some(10_000)), None).unwrap(), + run_mode(int(false, Some(10_000)), None, most).unwrap(), RunMode::Best ); for (asked, code) in [ @@ -1389,8 +1413,27 @@ mod tests { (int(false, Some(10_001)), ErrorCode::UnsupportedRequest), (int(false, None), ErrorCode::UnsupportedRequest), ] { - assert_eq!(run_mode(asked, None).unwrap_err().code, code, "{asked:?}"); + assert_eq!( + run_mode(asked, None, most).unwrap_err().code, + code, + "{asked:?}" + ); } + let past = run_mode(int(false, Some(10_001)), None, most).unwrap_err(); + assert_eq!( + past.fields["setting"], + Detail::text("max_relational_context_rows") + ); + assert_eq!( + past.fields["env"], + Detail::text("SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS") + ); + // A raised limit admits more, as a best run. + assert_eq!( + run_mode(int(false, Some(200_000)), None, 200_000).unwrap(), + RunMode::Best + ); + assert!(run_mode(int(false, Some(200_001)), None, 200_000).is_err()); } #[test] @@ -1398,6 +1441,11 @@ mod tests { let some = |n: u128| int(false, Some(n)); assert!(hops(None, None).is_ok()); assert!(hops(some(6), None).is_ok()); + // No most: past the six an earlier endpoint read is taken. + assert!(hops(some(7), None).is_ok()); + assert!(hops(some(1_000), None).is_ok()); + assert!(hops(None, some(12)).is_ok()); + assert!(hops(int(false, None), None).is_ok()); assert!( hops(some(9), some(2)).is_ok(), "the neighbors' length counts" @@ -1414,14 +1462,6 @@ mod tests { hops(int(true, Some(1)), None).unwrap_err().code, ErrorCode::InvalidRequest ); - assert_eq!( - hops(some(7), None).unwrap_err().code, - ErrorCode::UnsupportedRequest - ); - assert_eq!( - hops(int(false, None), None).unwrap_err().code, - ErrorCode::UnsupportedRequest - ); } #[test] @@ -1951,6 +1991,18 @@ mod tests { neighbors(&asked(r#"{"num_hops": 3}"#), RunMode::Best).unwrap(), vec![Hop::Keep(64), Hop::Keep(64), Hop::Keep(8)] ); + // Past the six sizes a mode lists, each hop keeps the last. + let eight = fast(r#"{"num_hops": 8}"#); + let sizes: Vec> = (0..9).map(|hop| eight.get(hop).cloned()).collect(); + let mut expected: Vec> = [32, 32, 8, 8, 4, 4, 4, 4] + .iter() + .map(|&n| Some(Hop::Keep(n))) + .collect(); + expected.push(None); + assert_eq!(sizes, expected); + // A count far past any graph is read lazily, never laid out. + let far = fast(r#"{"num_hops": 100000000000000000000000000000000000000000}"#); + assert_eq!(far.get(1_000_000), Some(&Hop::Keep(4))); // num_hops named as none, with no num_neighbors, is the engine's // TypeError: the caller's. let e = neighbors(&asked(r#"{"num_hops": null}"#), RunMode::Fast).unwrap_err(); diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index 31df574d7..bbfbc6e2b 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -79,12 +79,6 @@ pub const RESERVED: [&str; 2] = [INSTANCE_ID, ANCHOR_TIME]; const NO_ANCHOR: i64 = -9_223_372_036_854_775_807; /// The largest integer a JSON number carries exactly in every reader. const JSON_SAFE: i128 = 9_007_199_254_740_991; -/// The largest payload the endpoint takes, in bytes (`_MAX_SIZE`), counted -/// as the bytes the payload is written as and sent, UTF-8 (register PD-2), -/// not the reference's `json.dumps` count, which escapes every character -/// past ASCII. -pub const PAYLOAD_BYTES: usize = 30 * 1024 * 1024; - /// One column of a sampled table: as sampled, or a text feature as the /// reference's sampler sends it, each value its words /// (`sdm_engine::words`), written as a `stringlist` column. @@ -219,11 +213,6 @@ pub struct Sampled { /// How many candidates the call asked be kept (`top_k`), written into /// the task as the engine writes it. pub top_k: Option, - /// The most rows any call of the request this payload is one call of - /// scores (`_the_largest_batch`, `main`'s multi-call `predict`): the - /// size a refusal of a payload past the endpoint's limit names as the - /// one to ask below. - pub rows_per_call: usize, /// The lagged target columns (`lag_timesteps`), in order; none /// where the call asks for none. pub lags: Vec, @@ -234,14 +223,60 @@ pub struct Sampled { pub float_text: fn(f64) -> String, } -/// A payload built: its JSON text, the bytes the request sends, and how -/// many context examples answered the question and were taken out. +/// A payload built: its JSON text, how many of its bytes a call sends (the +/// members only this client reads left out), and how many context examples +/// answered the question and were taken out. #[derive(Debug, Clone)] pub struct Built { pub bytes: Vec, + pub model_bytes: usize, pub answering: usize, } +/// A built payload held to the most bytes one call takes (`limit`, the +/// `max_request_bytes` setting), counted as the bytes the payload is +/// written as and sent, UTF-8 (register PD-2), not the reference's +/// `json.dumps` count, which escapes every character past ASCII. A payload +/// past it is asked as smaller calls when it is predicted +/// ([`crate::batching`]), so it is refused here only where no call is small +/// enough: where the context, sent with every call, makes a call of one row +/// past the limit on its own. +pub fn check_bytes(built: &Built, limit: u64) -> Result<(), CoreError> { + if built.model_bytes as u64 <= limit { + return Ok(()); + } + let payload = sdm_wire::json::read(&built.bytes) + .map_err(|_| internal("a built payload does not read back"))?; + let rows = crate::batching::predict_rows(&payload).unwrap_or(0); + // The bytes a call carrying the first `n` rows sends. + let sent = |n: usize| -> Result { + let mut call = crate::batching::predict_slice(&payload, 0..n)?; + call.remove("client_generated_keys"); + call.remove("client_analysis_delta"); + Ok(call.to_bytes().len() as u64) + }; + let one = sent(rows.min(1))?; + if rows > 1 && one <= limit { + return Ok(()); + } + let count = |n: u64| Detail::Scalar(sdm_contract::Scalar::UInt64(n)); + Err(sdm_contract::limits::MAX_REQUEST_BYTES.named( + CoreError::new( + Sentence::PayloadNoSmallerCall, + "the context alone makes a call larger than one call takes", + ) + .with_field("field", Detail::text("payload")) + .with_field("bytes", count(one)) + .with_field("context_bytes", count(sent(0)?)) + .with_field("limit", count(limit)) + .with_field( + "batch_too_large_to_send", + Detail::Scalar(sdm_contract::Scalar::Bool(true)), + ) + .with_field("rows", count(rows as u64)), + )) +} + fn invalid(sentence: Sentence, message: &str, column: &str) -> CoreError { CoreError::new(sentence, message).with_field("column", Detail::text(column)) } @@ -1564,13 +1599,8 @@ fn build_mode( } .to_bytes() }; - // The limit counts the bytes the request sends, as written (PD-2). - // The refusal names what to change, as the reference's client words - // its engine's (`_said_in_this_client_s_terms`, `main`'s 1fc5a686): a - // call scoring several rows can be asked as smaller calls, below the - // most rows a call of the request scores (`smaller_calls`); a call of - // one row cannot, and only a narrower context helps - // (`no_smaller_call`). + // The bytes a call sends of it, which the limit counts (PD-2): all but + // the members only this client reads. let analysis_bytes = client_analysis_delta.as_ref().map_or(0, |delta| { serde_json::to_vec(delta) .expect("typed delta serializes") @@ -1579,36 +1609,9 @@ fn build_mode( + 1 }); let model_bytes = bytes.len() - client_only_bytes - analysis_bytes; - if model_bytes > PAYLOAD_BYTES { - let count = |n: usize| Detail::Scalar(sdm_contract::Scalar::UInt64(n as u64)); - return Err(CoreError::new( - if sampled.num_test > 1 { - Sentence::PayloadSmallerCalls - } else { - Sentence::PayloadNoSmallerCall - }, - "the request is larger than the endpoint takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("bytes", count(model_bytes)) - .with_field("limit", count(PAYLOAD_BYTES)) - .with_field( - "batch_too_large_to_send", - Detail::Scalar(sdm_contract::Scalar::Bool(true)), - ) - .with_field( - "remedy", - Detail::text(if sampled.num_test > 1 { - "smaller_calls" - } else { - "no_smaller_call" - }), - ) - .with_field("rows", count(sampled.num_test)) - .with_field("rows_per_call", count(sampled.rows_per_call))); - } Ok(Built { bytes, + model_bytes, answering: answered.len(), }) } diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index a4477af21..22e9fabd1 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -49,6 +49,9 @@ use arrow_array::{ Array, ArrayRef, BooleanArray, Float64Array, Int64Array, RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema}; +use sdm_contract::limits::{ + Limits, MAX_RELATIONAL_CONTEXT_ROWS, MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS, +}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_wire::json::{Value, is_integer_token}; use sdm_wire::response::{PredictionResponse, Refused, integer_text, numeric_text}; @@ -93,9 +96,6 @@ pub const INFERENCE_OPTIONS: [&str; 6] = [ /// The most classes `kumo-relational` classifies (`classes_max`). pub const CLASSES_MAX: usize = 10; -/// The most context rows `kumo-relational` reads (`context_rows_max`). -pub const CONTEXT_ROWS_MAX: usize = 10_000; - /// The context the mode that explains takes at most (`context_rows_fast`). pub const CONTEXT_ROWS_FAST: usize = 1_000; @@ -391,6 +391,7 @@ pub fn prepare( model: &str, asked: &[String], defaulted: bool, + limits: &Limits, ) -> Result, CoreError> { let Some(payload) = payloads.first() else { return Err(CoreError::new(Sentence::Internal, "a request of no call")); @@ -456,7 +457,7 @@ pub fn prepare( .map(|p| rows_of(p, "context")) .max() .unwrap_or(0); - check_capacity(spec, rows, false)?; + check_capacity(spec, rows, false, limits)?; if spec.id == TABULAR { // The tabular model takes no relational request this build sends. return Err(not_yet("model")); @@ -857,25 +858,36 @@ fn is_chosen(value: &Value) -> bool { } } -/// The bytes of the call `body` writes, as the host sends them, unchanged: -/// refused past the endpoint's limit (`PAYLOAD_BYTES`), counted on these -/// bytes, the envelope included, which are the bytes sent (register PD-2, -/// fix round 1), before the model is called. -pub fn call_bytes(call: &Value) -> Result, CoreError> { +/// The bytes of the call `body` writes, as the host sends them, unchanged, +/// counted on these bytes, the envelope included, which are the bytes sent +/// (register PD-2, fix round 1): refused past `limit` (`max_request_bytes`) +/// before the model is called ([`call_past_bytes`]). A call that scores rows +/// is split to fit before it gets here ([`crate::batching`]), so this +/// refuses a call that cannot be made smaller. +pub fn call_bytes(call: &Value, limit: u64) -> Result, CoreError> { let bytes = call.to_bytes(); - if bytes.len() > crate::payload::PAYLOAD_BYTES { - return Err(CoreError::new( + if bytes.len() as u64 > limit { + return Err(call_past_bytes(call, bytes.len() as u64, limit)); + } + Ok(bytes) +} + +/// The refusal of `call`, `bytes` long, past `limit`, the most bytes one call +/// takes (`max_request_bytes`): it names the call's bytes, those of its +/// context, which every call carries, and the setting that raises the limit. +pub fn call_past_bytes(call: &Value, bytes: u64, limit: u64) -> CoreError { + let context = crate::batching::predict_slice(call, 0..0) + .map_or(bytes, |context| context.to_bytes().len() as u64); + MAX_REQUEST_BYTES.named( + CoreError::new( Sentence::CallPastEndpoint, - "the call is larger than the endpoint takes", + "the call is larger than one call takes", ) .with_field("field", Detail::text("call")) - .with_field("bytes", Detail::Scalar(Scalar::UInt64(bytes.len() as u64))) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(crate::payload::PAYLOAD_BYTES as u64)), - )); - } - Ok(bytes) + .with_field("bytes", Detail::Scalar(Scalar::UInt64(bytes))) + .with_field("context_bytes", Detail::Scalar(Scalar::UInt64(context))) + .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), + ) } /// The entity a row is about, as the result gives it: as the reference's @@ -2328,26 +2340,58 @@ fn check_task_outputs( } Ok(()) } -fn check_capacity(spec: &Spec, rows: usize, direct: bool) -> Result<(), CoreError> { - if (direct || spec.id == RELATIONAL) && rows > CONTEXT_ROWS_MAX { - return Err(unsupported( - if direct { - Sentence::DirectContextLimit - } else { - Sentence::ContextPastModel - }, - spec.id, - "the request's context is larger than the model reads", +/// A direct request's context, as it is sent to `model`, held to the most +/// that model reads ([`check_capacity`]); a model this build does not +/// carry is checked where it is bound. +pub fn check_direct_context(model: &str, rows: usize, limits: &Limits) -> Result<(), CoreError> { + match MODELS.iter().find(|spec| spec.id == model) { + Some(spec) => check_capacity(spec, rows, true, limits), + None => Ok(()), + } +} + +/// The context a request carries held to the most its model reads +/// (`max_relational_context_rows` or `max_tabular_context_rows`), a +/// refusal naming the setting that raises it. +fn check_capacity( + spec: &Spec, + rows: usize, + direct: bool, + limits: &Limits, +) -> Result<(), CoreError> { + let (setting, most) = if spec.id == TABULAR { + (&MAX_TABULAR_CONTEXT_ROWS, limits.max_tabular_context_rows) + } else { + ( + &MAX_RELATIONAL_CONTEXT_ROWS, + limits.max_relational_context_rows, ) - .with_field("context_rows", count(rows)) - .with_field("context_rows_max", count(CONTEXT_ROWS_MAX))); + }; + if (direct || spec.id == RELATIONAL) && rows as u64 > most { + return Err(setting.named( + unsupported( + if direct { + Sentence::DirectContextLimit + } else { + Sentence::ContextPastModel + }, + spec.id, + "the request's context is larger than the model reads", + ) + .with_field("context_rows", count(rows)) + .with_field("context_rows_max", Detail::Scalar(Scalar::UInt64(most))), + )); } Ok(()) } /// Model preflight for live direct request facts, before conversion or empty handling. +/// The context is held to `limits` where they are given; a decision checked +/// again later is the same decision whatever the limits, which the call that +/// sends the request holds the context to once more. pub fn bind_direct( input: sdm_contract::direct::DirectCallFacts, + limits: Option<&Limits>, ) -> Result { let facts = input.clone(); let spec = MODELS @@ -2381,7 +2425,9 @@ pub fn bind_direct( input.tables, )?; check_task_outputs(spec, &input.kind, &outputs, false)?; - check_capacity(spec, input.context_rows, true)?; + if let Some(limits) = limits { + check_capacity(spec, input.context_rows, true, limits)?; + } Ok(sdm_contract::direct::BoundDirectCall { facts, model: input.model, diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index f868a0a49..5331ca5fe 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -62,8 +62,45 @@ const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 429, 504]; /// The call `call` (the core's own JSON) as the endpoint's envelope, the /// bytes a host posts, or `ServingPastLimit` when it is more than -/// `max_request_bytes`, counted on the actual UTF-8 body. +/// `max_request_bytes`, counted on the actual UTF-8 body. A call that scores +/// rows is split to fit before it gets here ([`crate::batching`]), so the +/// refusal names the context every call carries. pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result, CoreError> { + let read = readable(call)?; + let out = wrapped(&read)?; + let size = out.len() as u64; + if size > max_request_bytes { + let context = crate::batching::predict_slice(&read, 0..0) + .and_then(|context| wrapped(&context)) + .map_or(size, |context| context.len() as u64); + return Err(CoreError::new( + Sentence::ServingPastLimit, + "the call is larger than the serving endpoint takes", + ) + .with_field("endpoint", Detail::text(endpoint)) + .with_field("bytes", count(size)) + .with_field("context_bytes", count(context)) + .with_field("limit", count(max_request_bytes))); + } + Ok(out) +} + +/// How many bytes the endpoint's envelope of `call` is, as [`envelope`] +/// writes and weighs it. +pub fn envelope_len(call: &Value) -> Result { + if !call.all_finite() { + return Err(CoreError::new( + Sentence::ServingValueNotJson, + "the call carries a value with no JSON form", + ) + .with_field("reason", Detail::text("out_of_range_float"))); + } + Ok(wrapped(call)?.len() as u64) +} + +/// The call read back as the envelope carries it, refused where a value +/// has no JSON form. +fn readable(call: &[u8]) -> Result { let read = json::read(call).map_err(|e| { CoreError::new( Sentence::ServingValueNotJson, @@ -78,6 +115,11 @@ pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result Result, CoreError> { // Value's writer always produces UTF-8; this conversion performs no // number decoding, integer narrowing or host-dependent formatting. let payload = String::from_utf8(read.to_bytes()).map_err(|_| { @@ -86,25 +128,14 @@ pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result max_request_bytes { - return Err(CoreError::new( - Sentence::ServingPastLimit, - "the call is larger than the serving endpoint takes", - ) - .with_field("endpoint", Detail::text(endpoint)) - .with_field("bytes", count(size)) - .with_field("limit", count(max_request_bytes))); - } - Ok(out) + .to_bytes()) } /// The served model's answer in the endpoint's reply `body` (a 2xx), a JSON diff --git a/rust/crates/sdm-request/tests/batch_limits.rs b/rust/crates/sdm-request/tests/batch_limits.rs index 4c32f6645..209e4025a 100644 --- a/rust/crates/sdm-request/tests/batch_limits.rs +++ b/rust/crates/sdm-request/tests/batch_limits.rs @@ -2,10 +2,12 @@ // All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use sdm_contract::limits::Limits; use sdm_request::options::{BatchTask, rows_one_call_scores, rows_one_task_call_scores}; #[test] fn temporal_links_cannot_raise_the_model_limit() { + let limits = Limits::default(); for (asked, expected) in [ (None, 200), (Some(100), 100), @@ -16,7 +18,7 @@ fn temporal_links_cannot_raise_the_model_limit() { (Some(u64::MAX), 200), ] { assert_eq!( - rows_one_task_call_scores(asked, BatchTask::TemporalLink), + rows_one_task_call_scores(asked, BatchTask::TemporalLink, &limits), expected ); } @@ -24,6 +26,7 @@ fn temporal_links_cannot_raise_the_model_limit() { #[test] fn ordinary_and_decomposed_tasks_keep_their_existing_limit() { + let limits = Limits::default(); for (asked, expected) in [ (None, 10_000), (Some(100), 100), @@ -32,9 +35,34 @@ fn ordinary_and_decomposed_tasks_keep_their_existing_limit() { (Some(u64::MAX), 10_000), ] { assert_eq!( - rows_one_task_call_scores(asked, BatchTask::Ordinary), + rows_one_task_call_scores(asked, BatchTask::Ordinary, &limits), expected ); - assert_eq!(rows_one_call_scores(asked), expected); + assert_eq!(rows_one_call_scores(asked, &limits), expected); } } + +#[test] +fn a_raised_limit_raises_each_task_s_call_and_rows_per_call_still_lowers_it() { + let limits = Limits { + max_relational_predict_rows: 50_000, + max_temporal_link_predict_rows: 1_000, + ..Limits::default() + }; + assert_eq!( + rows_one_task_call_scores(None, BatchTask::Ordinary, &limits), + 50_000 + ); + assert_eq!( + rows_one_task_call_scores(Some(20_000), BatchTask::Ordinary, &limits), + 20_000 + ); + assert_eq!( + rows_one_task_call_scores(None, BatchTask::TemporalLink, &limits), + 1_000 + ); + assert_eq!( + rows_one_task_call_scores(Some(u64::MAX), BatchTask::TemporalLink, &limits), + 1_000 + ); +} diff --git a/rust/crates/sdm-request/tests/payload_bytes.rs b/rust/crates/sdm-request/tests/payload_bytes.rs index 3b34a3390..bd014be60 100644 --- a/rust/crates/sdm-request/tests/payload_bytes.rs +++ b/rust/crates/sdm-request/tests/payload_bytes.rs @@ -9,15 +9,28 @@ //! six-byte escape. Held at the limit with ASCII text and with text of two-, //! three- and four-byte characters, which the reference's count would //! have taken for three to twelve times its size. +//! +//! The payloads here score one row, so a payload past the limit cannot be +//! split into smaller calls, and the context alone is past it. use std::sync::Arc; use arrow_array::{ArrayRef, Int64Array, StringArray}; -use sdm_contract::ErrorCode; +use sdm_contract::limits::MAX_REQUEST_BYTES; use sdm_contract::ops::Stype; +use sdm_contract::{ErrorCode, Sentence}; use sdm_engine::labels::Labels; use sdm_request::options::{Kind, Options, RunMode}; -use sdm_request::payload::{PAYLOAD_BYTES, Sampled, SampledColumn, SampledTable, build}; +use sdm_request::payload::{Built, Sampled, SampledColumn, SampledTable, build, check_bytes}; + +const PAYLOAD_BYTES: usize = MAX_REQUEST_BYTES.default as usize; + +/// The payload built and held to the default limit. +fn checked(sampled: &Sampled) -> Result { + let built = build(sampled)?; + check_bytes(&built, MAX_REQUEST_BYTES.default)?; + Ok(built) +} const ANCHOR: i64 = 1_709_251_200_000_000_000; @@ -57,7 +70,6 @@ fn sampled(note: &str) -> Sampled { options: Options::default(), statistic: None, top_k: None, - rows_per_call: 1, lags: vec![], float_text: |x| format!("{x:?}"), } @@ -65,7 +77,7 @@ fn sampled(note: &str) -> Sampled { /// The payload's length with `n` copies of `unit` in each note. fn length(unit: &str, n: usize) -> Result { - build(&sampled(&unit.repeat(n))) + checked(&sampled(&unit.repeat(n))) .map(|built| built.bytes.len()) .map_err(|e| e.code) } @@ -84,7 +96,7 @@ fn the_limit_counts_the_bytes_sent_at_the_limit() { let n = (PAYLOAD_BYTES - base) / step; let slack = PAYLOAD_BYTES - base - n * step; let at = format!("{}{}", unit.repeat(n), "a".repeat(slack / 2)); - let taken = build(&sampled(&at)).expect("a payload within the limit is taken"); + let taken = checked(&sampled(&at)).expect("a payload within the limit is taken"); let exact = base + n * step + 2 * (slack / 2); assert_eq!(taken.bytes.len(), exact, "{unit:?}"); assert!(exact <= PAYLOAD_BYTES && PAYLOAD_BYTES - exact <= 1); @@ -95,15 +107,16 @@ fn the_limit_counts_the_bytes_sent_at_the_limit() { // Two units more (four bytes or more past it) are refused, as the // reference refuses a payload past its limit. let past = format!("{}{}", unit.repeat(n + 2), "a".repeat(slack / 2)); - let refused = build(&sampled(&past)).expect_err("a payload past the limit"); + let refused = checked(&sampled(&past)).expect_err("a payload past the limit"); assert_eq!(refused.code, ErrorCode::InvalidRequest, "{unit:?}"); + assert_eq!(refused.sentence, Sentence::PayloadNoSmallerCall, "{unit:?}"); } // With ASCII the payload reaches the limit exactly, one byte step past // it refused: the count is of bytes, to the byte. let base = length("a", 0).unwrap(); let fill = |extra: usize| { let half = (PAYLOAD_BYTES - base + extra) / 2; - build(&sampled(&"a".repeat(half))) + checked(&sampled(&"a".repeat(half))) }; let even = (PAYLOAD_BYTES - base).is_multiple_of(2); let at = fill(0).expect("at the limit"); @@ -117,3 +130,25 @@ fn the_limit_counts_the_bytes_sent_at_the_limit() { ); assert_eq!(fill(2).unwrap_err().code, ErrorCode::InvalidRequest); } + +#[test] +fn a_raised_limit_takes_a_payload_past_the_default_and_names_its_setting_past_it() { + let base = length("a", 0).unwrap(); + let past = sampled(&"a".repeat((PAYLOAD_BYTES - base) / 2 + 8)); + let built = build(&past).unwrap(); + let refused = check_bytes(&built, MAX_REQUEST_BYTES.default).unwrap_err(); + assert_eq!( + refused.fields["setting"], + sdm_contract::Detail::text("max_request_bytes") + ); + assert_eq!( + refused.fields["env"], + sdm_contract::Detail::text("SDM_CLIENT_MAX_REQUEST_BYTES") + ); + let context = match &refused.fields["context_bytes"] { + sdm_contract::Detail::Scalar(sdm_contract::Scalar::UInt64(n)) => *n, + other => panic!("{other:?}"), + }; + assert!(context < built.model_bytes as u64); + check_bytes(&built, 2 * MAX_REQUEST_BYTES.default).expect("a raised limit takes it"); +} diff --git a/rust/crates/sdm-request/tests/payload_lock.rs b/rust/crates/sdm-request/tests/payload_lock.rs index 9be1743fa..9546fecbf 100644 --- a/rust/crates/sdm-request/tests/payload_lock.rs +++ b/rust/crates/sdm-request/tests/payload_lock.rs @@ -56,7 +56,6 @@ fn shop() -> Sampled { options: Options::default(), statistic: Some("mean".into()), top_k: None, - rows_per_call: 1, lags: vec![], float_text: |x| format!("{x:?}"), tables: vec![ @@ -140,7 +139,6 @@ fn family() -> Sampled { options: Options::default(), statistic: None, top_k: None, - rows_per_call: 1, lags: vec![], float_text: |x| format!("{x:?}"), tables: vec![ @@ -191,7 +189,6 @@ fn classified(kind: Kind, labels: Labels) -> Sampled { windowed: false, statistic: None, top_k: None, - rows_per_call: 1, lags: vec![], ..shop() } diff --git a/rust/crates/sdm-testkit/src/host.rs b/rust/crates/sdm-testkit/src/host.rs index 722c21124..3f4362876 100644 --- a/rust/crates/sdm-testkit/src/host.rs +++ b/rust/crates/sdm-testkit/src/host.rs @@ -587,6 +587,7 @@ fn validate( "schema": serde_json::from_slice::(&schema_bytes)?, }); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::ValidateQuery(ValidateQuery { @@ -1106,6 +1107,7 @@ fn describe_on( "check_keys": spec.check_keys, }); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::DescribeSource(DescribeSource { @@ -1263,6 +1265,7 @@ fn edit_schema( "edit": serde_json::from_slice::(&edit_bytes)?, })); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::EditSchema(EditSchema { @@ -1779,6 +1782,7 @@ fn start_materialize( "seed": serde_json::to_value(&seed)?, })); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: as_op(Materialize { @@ -2124,6 +2128,7 @@ fn predict( }, })); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::Predict(Predict { diff --git a/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs b/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs index f8e6b597d..d8b6cb191 100644 --- a/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs +++ b/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs @@ -44,6 +44,7 @@ fn description_recording(case: &Case, extra_fetch: bool) -> Recording { .as_ref() .unwrap(); let start = ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: facts().unwrap(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index ebe290fc2..fbaffb33d 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -72,6 +72,7 @@ fn start(request: &Value) -> HostRun { fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Fetch { @@ -571,6 +572,7 @@ fn keyed(connection: &Connection, request: &Value, keys: RecordBatch) -> (Record fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Keyed { diff --git a/rust/crates/sdm-testkit/tests/neighbors.rs b/rust/crates/sdm-testkit/tests/neighbors.rs index 8e0819524..7849db5db 100644 --- a/rust/crates/sdm-testkit/tests/neighbors.rs +++ b/rust/crates/sdm-testkit/tests/neighbors.rs @@ -170,6 +170,7 @@ fn answered(connection: &Connection, request: DataRequest, seeds: RecordBatch) - fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Keyed { diff --git a/rust/crates/sdm-testkit/tests/warehouse_replay.rs b/rust/crates/sdm-testkit/tests/warehouse_replay.rs index 9729083e7..c635a37dd 100644 --- a/rust/crates/sdm-testkit/tests/warehouse_replay.rs +++ b/rust/crates/sdm-testkit/tests/warehouse_replay.rs @@ -75,6 +75,7 @@ fn core(provider: &Value) -> Value { fn recording(leg: &str, core: &Value) -> Value { let spec = &core["describe_source"]; let message = encode(&ControlMessage::Start(Box::new(StartRun { + limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::DescribeSource(DescribeSource { diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 1ab1a346e..b1edeb94f 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -162,10 +162,13 @@ would break rather than letting the endpoint return a 500. | context rows, with `num_context_samples` | `10,000` | | rows scored at once | `10,000` | | rows scored for a link prediction | `200` | -| tables in the gathered graph | `15` | -| hops gathered | `6` | +| bytes one call takes | `31,457,280` | | classes | `10` | +The context rows with `num_context_samples`, the rows scored at once, the rows scored for a link prediction and the bytes one call takes are the serving layer's defaults, and a deployment may be configured to take more. +Each is a setting: pass `limits=Limits(...)` to `SDMClient` and to `materialize()`, or set its environment variable, such as `SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS`. +The gathered graph may carry any number of tables and hops. + **More rows than one call scores are not refused.** They are gathered in batches of at most that many, and `predict()` sends one call per batch and returns one result, whose rows are numbered across the whole request, so @@ -181,9 +184,9 @@ in parts is not batched at all: a ranking is already a call per candidate and a forecast one per step, and batching a part would send that many times as many calls. -**The table limit is on the graph a request carries, not on the schema.** A -schema may describe far more than fifteen tables; what counts is how many the -query actually reaches. +**A call past the most bytes one takes is split, not refused.** It is sent as +several smaller calls, each with the whole context, so only a context too +large on its own is refused, naming its size and the limit. Gradient explanations follow the model endpoint's supported outputs; the client does not impose a separate one-row or fast-context limit. ## Asking for more than one answer each diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 93f5f2d5b..43cb20021 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -111,7 +111,10 @@ guesses that way and names the argument that settles it. `Request(context=..., predict=...)` takes two frames and sends exactly those rows. It trims neither. A context above 10,000 rows is refused with the count -and the cap rather than quietly dropping rows you named. +and the cap rather than quietly dropping rows you named. The cap is a setting +with the serving layer's default: for a deployment configured to take more, +pass `limits=Limits(max_tabular_context_rows=...)` to `SDMClient` and to +`materialize()`, or set `SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS`. The rows to score need not appear in the labelled half, and usually should not: they are the rows whose answer you do not have. diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index b795ebb9a..48a9fcc9b 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -2832,7 +2832,8 @@ "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_custom_nullable_producer_never_takes_arrow_snapshot", "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_float_refusal_keeps_specific_diagnostic", "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_integer_range_after_earlier_column_mutation", - "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_conversion_order_and_nested_options" + "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_conversion_order_and_nested_options", + "packages/structured-data-models-client/tests/test_configurable_limits.py" ], "java": [ "java/src/test/java/com/nvidia/sdm/DirectOptionsTest.java#directSchemaAndOptionsAreOwnedAndInferenceTravelsOnPredictAndScore", @@ -3338,7 +3339,8 @@ "packages/structured-data-models-client/tests/test_databricks_serving.py::test_the_ambient_identity_is_used_when_no_client_is_given", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_a_workspace_that_cannot_authenticate_says_how_to_fix_it", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_unusable_timeout_is_refused_at_construction", - "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not" + "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not", + "packages/structured-data-models-client/tests/test_configurable_limits.py" ], "java": [ "java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java#officialSdkAuthenticationRefreshesEveryHeaderWithoutRewritingTheCoreBytes", @@ -3868,7 +3870,8 @@ "python": [ "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_without_the_extra_the_extra_is_named", "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_tabular_answer_is_the_one_the_model_gives_when_called_directly", - "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_session_keeps_the_question_it_pinned" + "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_session_keeps_the_question_it_pinned", + "packages/structured-data-models-client/tests/test_configurable_limits.py" ], "java": [] }, @@ -4746,9 +4749,60 @@ "reason": "Exact class-label identity and presentation are tested here; this does not expand general model dtype admission." } } + }, + { + "id": "configurable-limits", + "apis": { + "python": [ + "Limits", + "Limits.resolved", + "SDMClient.limits" + ], + "java": [ + "Limits", + "Limits.resolved", + "SdmClient.limits" + ] + }, + "tests": { + "python": [ + "packages/structured-data-models-client/tests/test_configurable_limits.py" + ], + "java": [ + "java/src/test/java/com/nvidia/sdm/LimitsTest.java" + ] + }, + "gate": "local", + "note": "Size limits as settings: defaults, argument and environment precedence, a raised tabular context taken, byte-aware splitting and the refusal of a context too large on its own, through Rust and controlled HTTP exchanges; no live endpoint validation claimed.", + "ownership": { + "paths": [ + "rust/crates/sdm-core/src/limits.rs", + "rust/crates/sdm-request/src/batching.rs", + "rust/crates/sdm-core/src/predict.rs" + ], + "operations": [ + "resolve_limits", + "predict", + "materialize" + ] + }, + "scope": { + "scenarios": { + "refs": [], + "reason": "Focused limit tests define the settings' boundary; no scenario linkage is declared." + }, + "input_paths": { + "refs": [], + "reason": "This group exposes no input-route entry point; its focused tests define the boundary." + }, + "type_contracts": { + "refs": [], + "reason": "This group does not define dtype or semantic-type admission; its tests cover capability behavior." + } + } } ], - "public_api_snapshot_sha256": "652bd2757ca1d8ed12360b795a5c7f80ce63978210306cee1319532503f9d031", + "public_api_snapshot_sha256": "c6a02b69061a6f8f3a8703232962044d1ec5d79d67c4e8749374f57efc2ccee0", "api_inventory": { "python": { "sdm_api.Endpoint": { @@ -5169,7 +5223,13 @@ ] }, "sdm_client.PredictiveQuery.materialize": { - "capability": "direct-dtypes-options" + "capability": "direct-dtypes-options", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", + "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." + } + ] }, "sdm_client.Relationship": { "capability": "schema-edits" @@ -5250,7 +5310,13 @@ "capability": "direct-dtypes-options" }, "sdm_client.SDMClient": { - "capability": "direct-dtypes-options" + "capability": "direct-dtypes-options", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", + "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." + } + ] }, "sdm_client.SDMClient.capabilities": { "capability": "readiness-discovery" @@ -5262,13 +5328,25 @@ "capability": "sessions" }, "sdm_client.SDMClient.for_databricks_serving": { - "capability": "public-serving-refusals" + "capability": "public-serving-refusals", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", + "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." + } + ] }, "sdm_client.SDMClient.health_ready": { "capability": "readiness-discovery" }, "sdm_client.SDMClient.in_process": { - "capability": "in-process-models" + "capability": "in-process-models", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", + "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." + } + ] }, "sdm_client.SDMClient.models": { "capability": "readiness-discovery" @@ -6154,6 +6232,78 @@ "behavior": "Existing public informational policy constants remain available with the engine cohort budgets and reserved labels; assigning them is not a tuning API." } ] + }, + "sdm_client.Limits": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_an_argument_comes_before_the_environment_before_the_default", + "behavior": "A setting given comes before its environment variable, which comes before the default, as resolved by the core." + } + ] + }, + "sdm_client.Limits.max_relational_context_rows": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + } + ] + }, + "sdm_client.Limits.max_relational_predict_rows": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + } + ] + }, + "sdm_client.Limits.max_temporal_link_predict_rows": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + } + ] + }, + "sdm_client.Limits.max_tabular_context_rows": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + } + ] + }, + "sdm_client.Limits.max_request_bytes": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + } + ] + }, + "sdm_client.Limits.resolved": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", + "behavior": "Resolved limits equal the serving layer defaults when nothing is set." + } + ] + }, + "sdm_client.SDMClient.limits": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", + "behavior": "A client raised to 200,000 tabular context rows sends them; the default refuses naming the setting." + } + ] } }, "java": { @@ -8052,6 +8202,186 @@ "behavior": "Java exposes the same informational cohort budgets and reserved names as Python; these constants do not tune engine policy." } ] + }, + "com.nvidia.sdm.ClientOptions$Builder#public com.nvidia.sdm.ClientOptions$Builder limits(com.nvidia.sdm.Limits);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.DatabricksServing#public com.nvidia.sdm.DatabricksServing limits(com.nvidia.sdm.Limits);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.DatabricksServing#public com.nvidia.sdm.Limits limits();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public boolean equals(java.lang.Object);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public int hashCode();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public java.lang.String toString();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public java.util.Map resolved();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#theDefaultsAreTheServingLayers", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public java.util.Map values();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public static com.nvidia.sdm.Limits defaults();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#theDefaultsAreTheServingLayers", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits#public static com.nvidia.sdm.Limits$Builder builder();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits build();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRelationalContextRows(long);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRelationalPredictRows(long);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRequestBytes(long);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxTabularContextRows(long);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxTemporalLinkPredictRows(long);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.MaterializeOptions$Builder#public com.nvidia.sdm.MaterializeOptions$Builder limits(com.nvidia.sdm.Limits);": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] + }, + "com.nvidia.sdm.SdmClient#public com.nvidia.sdm.Limits limits();": { + "capability": "configurable-limits", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised", + "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." + } + ] } } }, @@ -8233,6 +8563,36 @@ "base_commit": "ebc5746391d977fd8d9d779423db33ae547d64ee", "record": "type_contracts/arrow-admission-and-inference", "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/configurable-limits", + "reason": "Adds the configurable size limits: Python Limits and SDMClient.limits, Java Limits, SdmClient.limits and the limits options of ClientOptions, DatabricksServing and MaterializeOptions, each with focused behavior tests." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_inventory", + "reason": "Adds the public declarations of the configurable size limits, each mapped to the configurable-limits group with a direct assertion." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "public_api_snapshot_sha256", + "reason": "The public snapshot records sdm_client.Limits, SDMClient.limits and the limits arguments of SDMClient, SDMClient.in_process, SDMClient.for_databricks_serving and PredictiveQuery.materialize." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/public-serving-refusals", + "reason": "Links the configurable-limits tests that assert the new limits argument of SDMClient.for_databricks_serving; the group's existing tests and scope are unchanged." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/in-process-models", + "reason": "Links the configurable-limits tests that assert the new limits argument of this group's entry points; the group's existing tests and scope are unchanged." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/direct-dtypes-options", + "reason": "Links the configurable-limits tests that assert the new limits argument of this group's entry points; the group's existing tests and scope are unchanged." } ], "capabilities": [ diff --git a/tests/compatibility/test_contract.py b/tests/compatibility/test_contract.py index 4652c845b..aa0c933b0 100644 --- a/tests/compatibility/test_contract.py +++ b/tests/compatibility/test_contract.py @@ -251,6 +251,7 @@ def test_helper_api_registry_has_real_behavior_tests_and_declared_scope(): 'explanation-summaries', 'gradient-explanations', 'column-explanations', + 'configurable-limits', } diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 5c81e999e..832327d39 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -125,6 +125,7 @@ "InferenceConfig", "KumoRelational", "KumoTabular", + "Limits", "LinkMetadata", "LinkReport", "MetadataReport", @@ -318,6 +319,21 @@ "public_methods": {}, "signature": "()" }, + "Limits": { + "bases": [], + "kind": "class", + "public_attributes": [ + "max_relational_context_rows", + "max_relational_predict_rows", + "max_request_bytes", + "max_tabular_context_rows", + "max_temporal_link_predict_rows" + ], + "public_methods": { + "resolved": "(self) -> 'dict[str, int]'" + }, + "signature": "(max_relational_context_rows: 'int | None' = None, max_relational_predict_rows: 'int | None' = None, max_temporal_link_predict_rows: 'int | None' = None, max_tabular_context_rows: 'int | None' = None, max_request_bytes: 'int | None' = None) -> None" + }, "LinkMetadata": { "bases": [], "kind": "class", @@ -410,7 +426,7 @@ "public_attributes": [], "public_methods": { "labels": "(self, size: 'int', *, anchor_time: 'pd.Timestamp | str | None' = None, random_seed: 'int | None' = 42, max_iterations: 'int' = 10) -> 'pd.DataFrame'", - "materialize": "(self, *, indices: 'Sequence[Any] | None' = None, predict: 'pd.DataFrame | None' = None, num_context_samples: 'int | None' = None, model: 'str | Model | None' = None, candidates: 'Sequence[Any] | None' = None, task: 'str | None' = None, **options: 'Any') -> 'Request'" + "materialize": "(self, *, indices: 'Sequence[Any] | None' = None, predict: 'pd.DataFrame | None' = None, num_context_samples: 'int | None' = None, model: 'str | Model | None' = None, candidates: 'Sequence[Any] | None' = None, task: 'str | None' = None, limits: 'Limits | None' = None, **options: 'Any') -> 'Request'" }, "signature": "(query: 'str', data: 'Schema | Mapping[str, pd.DataFrame]') -> 'None'" }, @@ -470,19 +486,20 @@ "bases": [], "kind": "class", "public_attributes": [ + "limits", "url" ], "public_methods": { "capabilities": "(self, model: 'str | Model') -> 'ModelCapabilities'", "close": "(self) -> 'None'", "fit": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None, holdout: 'Iterable[Any] | None' = None) -> 'FittedModel'", - "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None) -> 'SDMClient'", + "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None, limits: 'Limits | None' = None) -> 'SDMClient'", "health_ready": "(self) -> 'bool'", - "in_process": "(*models: 'Any', device: 'Any' = None) -> 'SDMClient'", + "in_process": "(*models: 'Any', device: 'Any' = None, limits: 'Limits | None' = None) -> 'SDMClient'", "models": "(self) -> 'list[str]'", "predict": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None) -> 'PredictionResult'" }, - "signature": "(url: 'str', api_key: 'str | None' = None, *, verify_ssl: 'bool' = True, timeout: 'float' = 60.0, max_retries: 'int' = 3) -> 'None'" + "signature": "(url: 'str', api_key: 'str | None' = None, *, verify_ssl: 'bool' = True, timeout: 'float' = 60.0, max_retries: 'int' = 3, limits: 'Limits | None' = None) -> 'None'" }, "SDMWarning": { "bases": [ diff --git a/tests/test_skills.py b/tests/test_skills.py index 4e9695f35..4fec39f88 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2231,10 +2231,8 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'Sessions': capabilities.sessions, } if name == 'tabular': - from sdm_client.models import TABULAR_LIMITS - said['Explanations'] = 'explanation' in capabilities.outputs - said['Context rows'] = TABULAR_LIMITS['context_rows_max'] + said['Context rows'] = _defaults()['max_tabular_context_rows'] assert set(shown) == set(said), ( f'{name} shows rows nothing checks: {sorted(set(shown) ^ set(said))}' ) @@ -2259,26 +2257,37 @@ def test_the_capability_table_is_the_clients_own(name, model_name): #: Each row of the limits table in ``skills/relational/SKILL.md``, against the -#: constant it reports. Every key is spoken for in both directions: a limit the -#: client enforces and the skill omits is a refusal an agent cannot see coming, -#: and a row naming no limit is a number nobody is holding to anything. +#: constant or setting it reports. Every key is spoken for in both +#: directions: a limit the client enforces and the skill omits is a refusal +#: an agent cannot see coming, and a row naming no limit is a number nobody is +#: holding to anything. LIMIT_ROWS = { 'context rows, by default': 'context_rows_fast', - 'context rows, with `num_context_samples`': 'context_rows_max', - 'rows scored at once': 'predict_rows_max', - 'rows scored for a link prediction': ( - 'predict_rows_temporal_link_prediction' - ), - 'tables in the gathered graph': 'tables_max', - 'hops gathered': 'hops_max', + 'context rows, with `num_context_samples`': 'max_relational_context_rows', + 'rows scored at once': 'max_relational_predict_rows', + 'rows scored for a link prediction': 'max_temporal_link_predict_rows', + 'bytes one call takes': 'max_request_bytes', 'classes': 'classes_max', } +def _defaults() -> dict[str, int]: + r"""Each setting's default, by name: the core's table.""" + from sdm_client.limits import settings + + return {setting.name: setting.default for setting in settings()} + + def test_the_limits_table_is_the_shipped_limits(): from sdm_client.models import RELATIONAL_LIMITS - assert set(LIMIT_ROWS.values()) == set(RELATIONAL_LIMITS), ( + relational = { + name: default + for name, default in _defaults().items() + if name != 'max_tabular_context_rows' + } + enforced = {**RELATIONAL_LIMITS, **relational} + assert set(LIMIT_ROWS.values()) == set(enforced), ( 'the client enforces a limit this table does not account for' ) path = SKILLS / 'relational' / 'SKILL.md' @@ -2291,9 +2300,8 @@ def test_the_limits_table_is_the_shipped_limits(): ) for label, key in LIMIT_ROWS.items(): value = int(only(shown[label][0]).replace(',', '')) - assert value == RELATIONAL_LIMITS[key], ( - f'{label!r} says {value}, the client enforces ' - f'{RELATIONAL_LIMITS[key]}' + assert value == enforced[key], ( + f'{label!r} says {value}, the client enforces {enforced[key]}' ) From 88a54fe036cbc2e55dbdab5e4f24c2e6f25390cc Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Sat, 7 Nov 2026 05:36:06 +0000 Subject: [PATCH 2/8] Trim the configurable limits' plumbing and duplicate tests - Drop the resolve_limits operation and the public Limits.resolved() in Python and Java. check_tabular_cap now answers the resolved tabular limit, still compared natively by the host, and the core refuses a count past it with its own sentence, naming the setting. - One refusal for a call past max_request_bytes: one_call_bytes and DirectCallPastBytes are folded into call_bytes and CallPastEndpoint. batching::fitted takes a single closure that answers a fit or the refusal of a call too large, so a refusal is no longer rebuilt and the body is written once per slice. serving::weigh replaces envelope_len. The duplicate predict_rows and table reader are gone. - Limits are resolved field by field rather than through string-keyed get/set; an unknown name or a value below one stays the host's fault (INVALID_MESSAGE), since both hosts validate the values first. - Remove the unused Setting.about, ENVIRONMENT_PREFIX, setting(), the empty TABULAR_LIMITS and the test-only PartialEq> for Hops. - Tests: drop the dispatch-level resolve test and the repeated default checks; Python covers argument-before-environment precedence through materialize. The materialize-time context check is kept. - Regenerate the contract schema and lock and the public API snapshot, and update the compatibility ledger to match. Signed-off-by: Adi Wadaskar --- docs/reference/limits.md | 1 - java/src/main/java/com/nvidia/sdm/Limits.java | 12 -- .../main/java/com/nvidia/sdm/SdmClient.java | 2 +- .../nvidia/sdm/internal/LimitSettings.java | 27 +--- .../test/java/com/nvidia/sdm/LimitsTest.java | 18 +-- .../src/sdm_client/_native.pyi | 8 +- .../sdm_client/_rust/tabular_materialize.py | 22 ++-- .../src/sdm_client/client.py | 3 +- .../src/sdm_client/limits.py | 51 ++------ .../src/sdm_client/models.py | 5 - .../tests/tabular_pql_count_cases.py | 6 +- .../test_asked_in_the_questions_it_serves.py | 4 +- .../tests/test_configurable_limits.py | 65 ++++------ .../tests/test_context_samples.py | 6 +- .../tests/test_limits_and_absent_outputs.py | 12 +- .../tests/test_limits_and_sessions.py | 9 +- .../tests/test_live_relational.py | 6 +- rust/bindings/sdm-jni/src/lib.rs | 1 - rust/bindings/sdm-py/src/lib.rs | 2 +- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 89 ++----------- rust/crates/sdm-contract/src/error.rs | 2 - rust/crates/sdm-contract/src/limits.rs | 92 +++----------- rust/crates/sdm-contract/src/ops.rs | 9 -- rust/crates/sdm-contract/src/tabular.rs | 11 +- rust/crates/sdm-contract/src/wording.rs | 8 +- rust/crates/sdm-contract/tests/schema_lock.rs | 5 - rust/crates/sdm-core/src/direct.rs | 2 +- rust/crates/sdm-core/src/dispatch.rs | 8 +- rust/crates/sdm-core/src/frame_session.rs | 2 +- rust/crates/sdm-core/src/limits.rs | 82 ++++++------ rust/crates/sdm-core/src/predict.rs | 80 +++--------- rust/crates/sdm-core/src/tabular_arrow.rs | 2 +- rust/crates/sdm-core/tests/attachments.rs | 1 - rust/crates/sdm-core/tests/dispatch.rs | 74 ----------- rust/crates/sdm-request/src/batching.rs | 120 +++++++++--------- rust/crates/sdm-request/src/options.rs | 35 +++-- rust/crates/sdm-request/src/payload.rs | 2 +- rust/crates/sdm-request/src/predict.rs | 10 +- rust/crates/sdm-request/src/serving.rs | 67 +++++----- rust/crates/sdm-request/src/tabular.rs | 30 ++++- tests/compatibility/manifest.json | 79 +++++------- tests/golden/public_api.json | 4 +- tests/test_skills.py | 4 +- 44 files changed, 326 insertions(+), 754 deletions(-) diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 11c8f6492..ff963b060 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -62,7 +62,6 @@ request = query.materialize(indices=keys, limits=limits) result = client.predict(request, model=KumoTabular()) ``` -`client.limits.resolved()` reports what each setting comes to. In Java, pass `Limits.builder()...build()` to `ClientOptions.builder().limits(...)`, `DatabricksServing.limits(...)` and `MaterializeOptions.builder().limits(...)`. Both read the same environment variables, and an environment variable that is not a whole number of at least 1 is refused with code `INVALID_CONFIGURATION`, naming it. A refusal at one of these limits names the setting and its environment variable. diff --git a/java/src/main/java/com/nvidia/sdm/Limits.java b/java/src/main/java/com/nvidia/sdm/Limits.java index 5a961462a..5fc7d995c 100644 --- a/java/src/main/java/com/nvidia/sdm/Limits.java +++ b/java/src/main/java/com/nvidia/sdm/Limits.java @@ -4,7 +4,6 @@ package com.nvidia.sdm; -import com.nvidia.sdm.internal.LimitSettings; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -50,17 +49,6 @@ public Map values() { return values; } - /** - * What these limits come to, each setting by name: the value set here, else its environment - * variable's, else its default, as the core resolves them for every request. - * - * @throws StructuredDataException {@code INVALID_CONFIGURATION} when an environment variable is - * not a whole number of at least 1 - */ - public Map resolved() { - return LimitSettings.resolved(values); - } - @Override public boolean equals(Object other) { return other instanceof Limits limits && limits.values.equals(values); diff --git a/java/src/main/java/com/nvidia/sdm/SdmClient.java b/java/src/main/java/com/nvidia/sdm/SdmClient.java index 5689326ba..810fe4a49 100644 --- a/java/src/main/java/com/nvidia/sdm/SdmClient.java +++ b/java/src/main/java/com/nvidia/sdm/SdmClient.java @@ -81,7 +81,7 @@ public static SdmClient forDatabricksServing(DatabricksServing serving) { /** * The size limits this client holds requests to, as it was opened with them; a setting not set - * takes its environment variable, else its default, which {@link Limits#resolved} reports. + * takes its environment variable, else its default. */ public Limits limits() { return limits; diff --git a/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java b/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java index 190abcbcf..b991a4257 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java +++ b/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java @@ -17,16 +17,12 @@ public final class LimitSettings { private LimitSettings() {} - /** Each setting the core names, in order: its {@code name}, {@code env} and {@code default}. */ - @SuppressWarnings("unchecked") - public static List> settings() { - return (List>) NativeCore.load().get("limits"); - } - /** What a run's start carries of {@code values}, the settings the caller set by name. */ + @SuppressWarnings("unchecked") public static Map given(Map values) { Map environment = new LinkedHashMap<>(); - for (Map setting : settings()) { + for (Map setting : + (List>) NativeCore.load().get("limits")) { String name = (String) setting.get("env"); String text = System.getenv(name); if (text != null) { @@ -42,21 +38,4 @@ public static Map given(Map values) { } return given; } - - /** What {@code values} come to, each setting by name, as the core resolves them. */ - public static Map resolved(Map values) { - Map operation = new LinkedHashMap<>(); - operation.put("op", "resolve_limits"); - operation.put("input", 1); - try (Host.Finished finished = - Host.run(operation, Map.of(1, Host.attachment(Map.of())), Host.Call.limited(values))) { - Map answer = ArrowTabular.value(finished); - Map resolved = new LinkedHashMap<>(); - for (Map setting : settings()) { - String name = (String) setting.get("name"); - resolved.put(name, ((Number) answer.get(name)).longValue()); - } - return resolved; - } - } } diff --git a/java/src/test/java/com/nvidia/sdm/LimitsTest.java b/java/src/test/java/com/nvidia/sdm/LimitsTest.java index eaac68985..ffeff5f5e 100644 --- a/java/src/test/java/com/nvidia/sdm/LimitsTest.java +++ b/java/src/test/java/com/nvidia/sdm/LimitsTest.java @@ -13,24 +13,12 @@ import org.apache.arrow.vector.table.Table; import org.junit.jupiter.api.Test; -/** The size limits a client holds requests to: their defaults and a raised one taking effect. */ +/** The size limits a client holds requests to: a raised one taking effect. */ class LimitsTest { - @Test void theDefaultsAreTheServingLayers() { - assertEquals( - Map.of( - "max_relational_context_rows", 10_000L, - "max_relational_predict_rows", 10_000L, - "max_temporal_link_predict_rows", 200L, - "max_tabular_context_rows", 10_000L, - "max_request_bytes", 31_457_280L), - Limits.defaults().resolved()); - } - - @Test void aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne() { + @Test void aSettingGivenIsKeptByNameAndMustBeAtLeastOne() { Limits limits = Limits.builder().maxTabularContextRows(200_000).build(); assertEquals(Map.of("max_tabular_context_rows", 200_000L), limits.values()); - assertEquals(200_000L, limits.resolved().get("max_tabular_context_rows")); - assertEquals(10_000L, limits.resolved().get("max_relational_context_rows")); + assertEquals(Map.of(), Limits.defaults().values()); StructuredDataException refused = assertThrows( StructuredDataException.class, () -> Limits.builder().maxRequestBytes(0)); assertEquals("INVALID_CONFIGURATION", refused.code()); diff --git a/packages/structured-data-models-client/src/sdm_client/_native.pyi b/packages/structured-data-models-client/src/sdm_client/_native.pyi index 2ade6e7ce..3fb994333 100644 --- a/packages/structured-data-models-client/src/sdm_client/_native.pyi +++ b/packages/structured-data-models-client/src/sdm_client/_native.pyi @@ -70,10 +70,10 @@ MEANINGS: tuple[tuple[str, str], ...] #: Each data type name a host may give a column, and the contract's name #: of the type it names: the one table of them. DTYPE_NAMES: tuple[tuple[str, str], ...] -#: Each size limit a caller may set, ``(name, env, default, about)``: its -#: name, the environment variable a host reads it from, its default and -#: what it holds a request to; the one table of them every host reads. -LIMITS: tuple[tuple[str, str, int, str], ...] +#: Each size limit a caller may set, ``(name, env, default)``: its name, +#: the environment variable a host reads it from and its default; the one +#: table of them every host reads. +LIMITS: tuple[tuple[str, str, int], ...] class _ArrowStreamExportable(Protocol): def __arrow_c_stream__( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py index 6e28db207..0604b73d8 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py @@ -215,17 +215,13 @@ def _context( has_custom_sorter, observe, ) - from sdm_client.limits import in_effect - # The most context rows a single-table request carries, as the core - # resolves the setting for the runs below. - limit = in_effect().resolved()['max_tabular_context_rows'] + # The most context rows a request carries (`max_tabular_context_rows`), + # the count compared with it natively before any frame length is read; + # past it, the core refuses, naming the setting. + limit = _call('check_tabular_cap', {'requested': None})['limit'] if decision is None: - over_limit = wanted is not None and bool(wanted > limit) - accepted = _call('check_tabular_cap', {'over_limit': over_limit})[ - 'accepted' - ] - if not accepted: + if wanted is not None and bool(wanted > limit): decision = {'kind': 'refused'} else: count = int(wanted) if isinstance(wanted, np.integer) else wanted @@ -237,12 +233,10 @@ def _context( }, ) if decision['kind'] == 'refused': + _call('check_tabular_cap', {'requested': str(int(wanted))}) raise StructuredDataError( - f'num_context_samples is {wanted:,}, and a request carries at ' - f'most {limit:,} context rows; if the endpoint takes more, raise ' - f'it with limits=Limits(max_tabular_context_rows=...) or the ' - f'environment variable SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', - code='UNSUPPORTED_REQUEST', + 'the core refused a context count it then accepted', + code='INTERNAL_ERROR', ) if not decision['reduced']: return candidates diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 35eae7bfd..c9c7a91cc 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -415,8 +415,7 @@ def for_databricks_serving( @property def limits(self) -> Limits: r"""The size limits this client was opened with; a setting left - ``None`` takes its environment variable, else its default, which - :meth:`Limits.resolved` reports.""" + ``None`` takes its environment variable, else its default.""" return self._limits or Limits() @property diff --git a/packages/structured-data-models-client/src/sdm_client/limits.py b/packages/structured-data-models-client/src/sdm_client/limits.py index 1e25ff0d6..b2d547f9d 100644 --- a/packages/structured-data-models-client/src/sdm_client/limits.py +++ b/packages/structured-data-models-client/src/sdm_client/limits.py @@ -20,7 +20,7 @@ capitals, such as ``SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS=200000``. A value passed to :class:`Limits` comes before the environment's, which comes before the default. The names, variables and defaults are the core's, the one table -every host reads (:data:`SETTINGS`), and the core resolves them. +every host reads, and the core resolves them. """ from __future__ import annotations @@ -30,27 +30,11 @@ from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass, fields -from typing import Any, NamedTuple +from typing import Any from sdm_client.errors import StructuredDataError -class Setting(NamedTuple): - r"""One limit a caller may set: its name, the environment variable it is - read from, its default and what it holds a request to.""" - - name: str - env: str - default: int - about: str - - -def _settings() -> tuple[Setting, ...]: - from sdm_client._rust.run import native - - return tuple(Setting(*setting) for setting in native().LIMITS) - - @dataclass(frozen=True) class Limits: r"""The size limits a client holds requests to, each ``None`` to take @@ -109,15 +93,17 @@ def _given(self) -> dict[str, Any]: r"""What a run's start carries of these limits: the values set here, and the text of each setting's environment variable that is set, read now, which the core resolves (contract ``GivenLimits``).""" + from sdm_client._rust.run import native + given = { field.name: getattr(self, field.name) for field in fields(self) if getattr(self, field.name) is not None } environment = { - setting.env: os.environ[setting.env] - for setting in _settings() - if setting.env in os.environ + env: os.environ[env] + for _, env, _ in native().LIMITS + if env in os.environ } start: dict[str, Any] = {} if given: @@ -126,27 +112,12 @@ def _given(self) -> dict[str, Any]: start['environment'] = environment return start - def resolved(self) -> dict[str, int]: - r"""What these limits come to, each setting by its name: the value - set here, else its environment variable's, else its default, as the - core resolves them for every request. - - Raises: - StructuredDataError: With code ``INVALID_CONFIGURATION``, if an - environment variable is not a whole number of at least 1. - """ - from sdm_client._rust import direct - - with limits_in_effect(self): - answer = direct._call('resolve_limits', {}) - return { - setting.name: int(answer[setting.name]) for setting in _settings() - } +def defaults() -> dict[str, int]: + r"""Each setting's default, by name: the core's one table of them.""" + from sdm_client._rust.run import native -def settings() -> tuple[Setting, ...]: - r"""Every limit a caller may set, in order: the core's one table.""" - return _settings() + return {name: default for name, _, default in native().LIMITS} _IN_EFFECT: contextvars.ContextVar[Limits | None] = contextvars.ContextVar( diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 5933afb90..b6be3bd55 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -57,10 +57,6 @@ PREDICTION_STATISTICS = frozenset({'mean', 'median'}) SUMMARISED_TASK_KINDS = frozenset({'regression', 'forecasting'}) -# The tabular model's context is held to a setting, not a fixed size -# (`sdm_client.Limits.max_tabular_context_rows`). -TABULAR_LIMITS: dict[str, int] = {} - KUMO_TABULAR = ModelSpec( model_id=TFM_MODEL_KUMO_TABULAR, profile='TABULAR', @@ -71,7 +67,6 @@ outputs=frozenset( {'prediction', 'probabilities', 'quantiles', 'explanation'} ), - limits=TABULAR_LIMITS, explanation_formats=('sdm_gradient_v1',), ), ) diff --git a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py index f4c7d426e..94cd242d1 100644 --- a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py +++ b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py @@ -34,12 +34,10 @@ class StopWarning(Exception): def observe_case(name: str) -> dict[str, Any]: from sdm_client import KumoTabular, PredictiveQuery, SDMClient - from sdm_client.limits import settings + from sdm_client.limits import defaults assert name in CASES - cap = {setting.name: setting.default for setting in settings()}[ - 'max_tabular_context_rows' - ] + cap = defaults()['max_tabular_context_rows'] events: list[Any] = [] recorded: list[dict[str, str]] = [] outcome: dict[str, Any] = {} diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index b067465fe..c310e3f8d 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -97,9 +97,9 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( ): """A part is one call, which is what makes the fan-out countable. The refusal reads the count it was given, so it costs no gathering.""" - from sdm_client.limits import settings + from sdm_client.limits import defaults - defaults = {setting.name: setting.default for setting in settings()} + defaults = defaults() count = defaults['max_relational_predict_rows'] + 1 with pytest.raises(StructuredDataError) as refused: built(query, indices=[1] * count, **how) diff --git a/packages/structured-data-models-client/tests/test_configurable_limits.py b/packages/structured-data-models-client/tests/test_configurable_limits.py index 9e2176e43..7ea4a2bc6 100644 --- a/packages/structured-data-models-client/tests/test_configurable_limits.py +++ b/packages/structured-data-models-client/tests/test_configurable_limits.py @@ -22,53 +22,31 @@ from test_tabular_direct_core import Endpoint, client from sdm_client import KumoTabular, Limits, Request, StructuredDataError -from sdm_client.limits import settings -DEFAULTS = { - 'max_relational_context_rows': 10_000, - 'max_relational_predict_rows': 10_000, - 'max_temporal_link_predict_rows': 200, - 'max_tabular_context_rows': 10_000, - 'max_request_bytes': 30 * 1024 * 1024, -} +SETTINGS = ( + 'max_relational_context_rows', + 'max_relational_predict_rows', + 'max_temporal_link_predict_rows', + 'max_tabular_context_rows', + 'max_request_bytes', +) @pytest.fixture(autouse=True) def _no_environment(monkeypatch: pytest.MonkeyPatch) -> None: - for setting in settings(): - monkeypatch.delenv(setting.env, raising=False) - - -def test_the_defaults_are_the_serving_layers() -> None: - assert {s.name: s.default for s in settings()} == DEFAULTS - assert {s.name: s.env for s in settings()} == { - name: f'SDM_CLIENT_{name.upper()}' for name in DEFAULTS - } - assert Limits().resolved() == DEFAULTS - - -def test_an_argument_comes_before_the_environment_before_the_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '200000') - monkeypatch.setenv('SDM_CLIENT_MAX_REQUEST_BYTES', '64_000_000') - from_env = Limits().resolved() - assert from_env['max_tabular_context_rows'] == 200_000 - assert from_env['max_request_bytes'] == 64_000_000 - assert from_env['max_relational_context_rows'] == 10_000 - given = Limits(max_tabular_context_rows=50_000).resolved() - assert given['max_tabular_context_rows'] == 50_000 + for name in SETTINGS: + monkeypatch.delenv(f'SDM_CLIENT_{name.upper()}', raising=False) @pytest.mark.parametrize('text', ['0', '-1', '2.5', 'lots', '1e6']) def test_an_environment_variable_that_is_not_a_whole_number_is_refused( monkeypatch: pytest.MonkeyPatch, text: str ) -> None: - monkeypatch.setenv('SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS', text) + monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', text) with pytest.raises(StructuredDataError) as refused: - Limits().resolved() + query(table(20)).materialize(indices=[0, 1]) assert refused.value.code == 'INVALID_CONFIGURATION' - assert 'SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS' in str(refused.value) + assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) @pytest.mark.parametrize('value', [0, -5, True, 2.5, '10']) @@ -113,19 +91,22 @@ def test_materialize_samples_to_a_raised_tabular_limit( schema = table(12_000) with pytest.raises(StructuredDataError): query(schema).materialize(indices=[0, 1], num_context_samples=11_000) - request = query(schema).materialize( - indices=[0, 1], - num_context_samples=11_000, - limits=Limits(max_tabular_context_rows=20_000), - ) - assert request.context_rows == 11_000 - monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '20000') + monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '20_000') assert ( query(schema) .materialize(indices=[0, 1], num_context_samples=11_000) .context_rows == 11_000 ) + # An argument comes before the environment variable. + with pytest.raises(StructuredDataError) as refused: + query(schema).materialize( + indices=[0, 1], + num_context_samples=11_000, + limits=Limits(max_tabular_context_rows=10_500), + ) + assert 'at most 10,500 context rows' in str(refused.value) + assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) def test_a_raised_relational_predict_limit_and_a_lowered_one_set_the_calls() -> ( @@ -211,7 +192,7 @@ def test_a_context_too_large_on_its_own_is_refused_naming_the_setting() -> None: with pytest.raises(StructuredDataError) as refused: _sent(request, Limits(max_request_bytes=1_000)) said = str(refused.value) - assert 'context alone' in said + assert 'of them the context' in said assert 'SDM_CLIENT_MAX_REQUEST_BYTES' in said assert 'max_request_bytes' in said diff --git a/packages/structured-data-models-client/tests/test_context_samples.py b/packages/structured-data-models-client/tests/test_context_samples.py index 89de07d51..0a04fed8d 100644 --- a/packages/structured-data-models-client/tests/test_context_samples.py +++ b/packages/structured-data-models-client/tests/test_context_samples.py @@ -21,14 +21,12 @@ from test_materialize_options import _materialize from sdm_client.errors import StructuredDataError -from sdm_client.limits import settings +from sdm_client.limits import defaults from sdm_client.materializers.relational import RelationalMaterializer from sdm_client.models import RELATIONAL_LIMITS FAST = RELATIONAL_LIMITS['context_rows_fast'] -MAX = {setting.name: setting.default for setting in settings()}[ - 'max_relational_context_rows' -] +MAX = defaults()['max_relational_context_rows'] # --- which value goes on the wire ----------------------------------------- diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 322c3297d..6ff8c7663 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -24,7 +24,7 @@ import pandas as pd import pytest -from sdm_client import Limits, PredictiveQuery, Schema +from sdm_client import PredictiveQuery, Schema from sdm_client.client import _report_absent from sdm_client.errors import StructuredDataError from sdm_client.result import PredictionResult @@ -58,16 +58,6 @@ def query(schema: Schema) -> PredictiveQuery: # --- the context the endpoint will take ------------------------------------ -def test_the_declared_cap_is_the_one_the_endpoint_enforces(): - """Measured at the boundary: 10,000 answer, 10,001 refuse; the default - of a setting a deployment may raise.""" - from sdm_client.limits import settings - - defaults = {setting.name: setting.default for setting in settings()} - assert defaults['max_tabular_context_rows'] == 10_000 - assert Limits().resolved()['max_tabular_context_rows'] == 10_000 - - def test_a_context_above_the_cap_is_refused_before_it_is_encoded(): schema = table(200) with pytest.raises(StructuredDataError) as raised: diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 2001f5196..3f3d0d084 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -60,15 +60,10 @@ def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): These numbers were read from the relational NIM, as the defaults of the settings a deployment may raise. This asserts they have not drifted in the client, which is all it can do offline. The live suites check that - the endpoint still agrees. No most is set on tables or hops. + the endpoint still agrees. The sizes a deployment sets are settings, + whose defaults the core pins; no most is set on tables or hops. """ - from sdm_client.limits import settings - - defaults = {setting.name: setting.default for setting in settings()} assert RELATIONAL_LIMITS == {'context_rows_fast': 1_000, 'classes_max': 10} - assert defaults['max_relational_context_rows'] == 10_000 - assert defaults['max_relational_predict_rows'] == 10_000 - assert defaults['max_temporal_link_predict_rows'] == 200 @pytest.mark.parametrize('bad', ['2', True]) diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index 23836abf8..20e469f07 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -193,11 +193,9 @@ def test_the_endpoint_scores_what_this_client_says_it_does(client, query): """ import copy - from sdm_client.limits import settings + from sdm_client.limits import defaults - cap = {setting.name: setting.default for setting in settings()}[ - 'max_relational_predict_rows' - ] + cap = defaults()['max_relational_predict_rows'] payload = copy.deepcopy(query.materialize(indices=[0, 1]).payloads[0]) rows = payload['predict']['instance_table']['rows'] diff --git a/rust/bindings/sdm-jni/src/lib.rs b/rust/bindings/sdm-jni/src/lib.rs index 8e0dd1b13..7cb067fa8 100644 --- a/rust/bindings/sdm-jni/src/lib.rs +++ b/rust/bindings/sdm-jni/src/lib.rs @@ -464,7 +464,6 @@ pub extern "system" fn Java_com_nvidia_sdm_internal_NativeCore_info0<'caller>( "name": setting.name, "env": setting.env, "default": setting.default, - "about": setting.about, })) .collect::>(), }); diff --git a/rust/bindings/sdm-py/src/lib.rs b/rust/bindings/sdm-py/src/lib.rs index ee702d2b9..4cc9c5fa0 100644 --- a/rust/bindings/sdm-py/src/lib.rs +++ b/rust/bindings/sdm-py/src/lib.rs @@ -660,7 +660,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { py, sdm_contract::limits::SETTINGS .iter() - .map(|setting| (setting.name, setting.env, setting.default, setting.about)) + .map(|setting| (setting.name, setting.env, setting.default)) .collect::>(), )?, )?; diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index a3aaa6624..be5c113c5 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:96fd2d685c95ceee80d653d48e3dd022e64530ec3257e2a0406233394f001faf sdm-contract-72.json +72 sha256:c4e98a97ae3920e095fbdd83b4d2df343e1107494366bed7e021210abf9a276e sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index 851c4ab3e..9a3c87363 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -316,12 +316,6 @@ "DirectValue.check_tabular_cap": { "$ref": "#/$defs/CheckedTabularCap" }, - "ResolveLimits.input": { - "$ref": "#/$defs/ResolveLimits" - }, - "DirectValue.resolve_limits": { - "$ref": "#/$defs/Limits" - }, "ObserveTabularMembership.input": { "$ref": "#/$defs/ObserveTabularMembership" }, @@ -711,8 +705,6 @@ "DirectValue.select_tabular_rows", "CheckTabularCap.input", "DirectValue.check_tabular_cap", - "ResolveLimits.input", - "DirectValue.resolve_limits", "ObserveTabularMembership.input", "DirectValue.observe_tabular_membership", "DecideTabularTask.input", @@ -3819,60 +3811,23 @@ ] }, "CheckTabularCap": { - "description": "Reached native comparison against the model cap, before frame length reads.", - "type": "object", - "properties": { - "over_limit": { - "type": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "over_limit" - ] - }, - "CheckedTabularCap": { + "description": "The context a materialized single-table request asks for, held to the\nmost context rows a request carries before frame length reads: the\n`num_context_samples` given, as its decimal text however large, or none.", "type": "object", "properties": { - "accepted": { - "type": "boolean" + "requested": { + "type": [ + "string", + "null" + ] } }, - "additionalProperties": false, - "required": [ - "accepted" - ] - }, - "ResolveLimits": { - "description": "The input of `resolve_limits`, which reads nothing but the run's\nlimits: none.", - "type": "object", "additionalProperties": false }, - "Limits": { - "description": "The limits a run holds a request to, resolved; `resolve_limits` answers\nthem, so a host can say what its caller's settings came to.", + "CheckedTabularCap": { + "description": "The context asked for is within the limit, which the host compares its\nown count with, natively, before it reads any frame length; past it,\nthe run is refused.", "type": "object", "properties": { - "max_relational_context_rows": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "max_relational_predict_rows": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "max_temporal_link_predict_rows": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "max_tabular_context_rows": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "max_request_bytes": { + "limit": { "type": "integer", "format": "uint64", "minimum": 0 @@ -3880,11 +3835,7 @@ }, "additionalProperties": false, "required": [ - "max_relational_context_rows", - "max_relational_predict_rows", - "max_temporal_link_predict_rows", - "max_tabular_context_rows", - "max_request_bytes" + "limit" ] }, "ObserveTabularMembership": { @@ -13786,24 +13737,6 @@ "input" ] }, - { - "description": "The size limits the run holds requests to, as resolved from its\nstart's (contract v72).", - "type": "object", - "properties": { - "input": { - "$ref": "#/$defs/AttachmentId" - }, - "op": { - "type": "string", - "const": "resolve_limits" - } - }, - "additionalProperties": false, - "required": [ - "op", - "input" - ] - }, { "description": "An input-sized direct value, carried as typed JSON bytes.", "type": "object", @@ -17781,7 +17714,6 @@ "options_unread_statistic", "payload_no_smaller_call", "call_past_endpoint", - "direct_call_past_bytes", "outputs_none", "not_finite", "past_int64", @@ -17924,7 +17856,6 @@ "admitted_past_type", "invalid_configuration", "limit_environment_invalid", - "limit_given_below_one", "part_forecast_step", "part_forecast_step_outran", "part_ranking_candidate", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index d30489c27..e31dac620 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -1015,7 +1015,6 @@ sentences! { OptionsUnreadStatistic = "options_unread_statistic", E::UnsupportedRequest; PayloadNoSmallerCall = "payload_no_smaller_call", E::InvalidRequest; CallPastEndpoint = "call_past_endpoint", E::InvalidRequest; - DirectCallPastBytes = "direct_call_past_bytes", E::InvalidRequest; OutputsNone = "outputs_none", E::InvalidRequest; NotFinite = "not_finite", E::InvalidRequest; PastInt64 = "past_int64", E::InvalidRequest; @@ -1158,7 +1157,6 @@ sentences! { AdmittedPastType = "admitted_past_type", E::ValueOutOfRange; InvalidConfiguration = "invalid_configuration", E::InvalidConfiguration; LimitEnvironmentInvalid = "limit_environment_invalid", E::InvalidConfiguration; - LimitGivenBelowOne = "limit_given_below_one", E::InvalidConfiguration; PartForecastStep = "part_forecast_step", Code::Part; PartForecastStepOutran = "part_forecast_step_outran", Code::Part; PartRankingCandidate = "part_ranking_candidate", Code::Part; diff --git a/rust/crates/sdm-contract/src/limits.rs b/rust/crates/sdm-contract/src/limits.rs index adbfcbe69..40fef7d5d 100644 --- a/rust/crates/sdm-contract/src/limits.rs +++ b/rust/crates/sdm-contract/src/limits.rs @@ -26,21 +26,16 @@ use serde::{Deserialize, Serialize}; use crate::error::{CoreError, Detail}; -/// The prefix of every setting's environment variable. -pub const ENVIRONMENT_PREFIX: &str = "SDM_CLIENT_"; - /// One limit a caller may set. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Setting { /// The setting's name, as a host's argument names it. pub name: &'static str, - /// The environment variable a host reads it from, the name uppercased - /// after [`ENVIRONMENT_PREFIX`]. + /// The environment variable a host reads it from: `SDM_CLIENT_` and the + /// name in capitals. pub env: &'static str, /// The value where neither the caller nor the environment names one. pub default: u64, - /// What the limit holds a request to. - pub about: &'static str, } /// The most context rows `kumo-relational` reads: the serving layer's @@ -49,7 +44,6 @@ pub const MAX_RELATIONAL_CONTEXT_ROWS: Setting = Setting { name: "max_relational_context_rows", env: "SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS", default: 10_000, - about: "the most context rows a relational request carries; num_context_samples asks for at most this many", }; /// The most rows one `kumo-relational` call scores: the serving layer's @@ -58,7 +52,6 @@ pub const MAX_RELATIONAL_PREDICT_ROWS: Setting = Setting { name: "max_relational_predict_rows", env: "SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS", default: 10_000, - about: "the most rows one relational call scores; more are asked in several calls", }; /// The most rows one temporal link prediction call scores. @@ -66,7 +59,6 @@ pub const MAX_TEMPORAL_LINK_PREDICT_ROWS: Setting = Setting { name: "max_temporal_link_predict_rows", env: "SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS", default: 200, - about: "the most rows one temporal link prediction call scores; more are asked in several calls", }; /// The most context rows `kumo-tabular` reads: the serving layer's @@ -75,7 +67,6 @@ pub const MAX_TABULAR_CONTEXT_ROWS: Setting = Setting { name: "max_tabular_context_rows", env: "SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS", default: 10_000, - about: "the most context rows a single-table request carries; materialize() samples down to it", }; /// The most bytes one call's body takes, as written and sent: the serving @@ -85,7 +76,6 @@ pub const MAX_REQUEST_BYTES: Setting = Setting { name: "max_request_bytes", env: "SDM_CLIENT_MAX_REQUEST_BYTES", default: 30 * 1024 * 1024, - about: "the most bytes one call's body takes; a call past it is split into smaller calls", }; impl Setting { @@ -107,11 +97,6 @@ pub const SETTINGS: [Setting; 5] = [ MAX_REQUEST_BYTES, ]; -/// The setting named `name`, if there is one. -pub fn setting(name: &str) -> Option<&'static Setting> { - SETTINGS.iter().find(|setting| setting.name == name) -} - /// What a host passes with every run (contract v72): the values its caller /// set, by [`Setting::name`], and the text of each setting's environment /// variable that is set, by [`Setting::env`], each as the host read it. @@ -127,16 +112,8 @@ pub struct GivenLimits { pub environment: BTreeMap, } -/// The input of `resolve_limits`, which reads nothing but the run's -/// limits: none. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct ResolveLimits {} - -/// The limits a run holds a request to, resolved; `resolve_limits` answers -/// them, so a host can say what its caller's settings came to. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +/// The limits a run holds a request to, resolved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Limits { pub max_relational_context_rows: u64, pub max_relational_predict_rows: u64, @@ -157,42 +134,12 @@ impl Default for Limits { } } -impl Limits { - /// The value of the setting named `name`, if there is one. - pub fn get(&self, name: &str) -> Option { - Some(match name { - n if n == MAX_RELATIONAL_CONTEXT_ROWS.name => self.max_relational_context_rows, - n if n == MAX_RELATIONAL_PREDICT_ROWS.name => self.max_relational_predict_rows, - n if n == MAX_TEMPORAL_LINK_PREDICT_ROWS.name => self.max_temporal_link_predict_rows, - n if n == MAX_TABULAR_CONTEXT_ROWS.name => self.max_tabular_context_rows, - n if n == MAX_REQUEST_BYTES.name => self.max_request_bytes, - _ => return None, - }) - } - - /// The setting named `name` set to `value`; false where there is none. - pub fn set(&mut self, name: &str, value: u64) -> bool { - let slot = match name { - n if n == MAX_RELATIONAL_CONTEXT_ROWS.name => &mut self.max_relational_context_rows, - n if n == MAX_RELATIONAL_PREDICT_ROWS.name => &mut self.max_relational_predict_rows, - n if n == MAX_TEMPORAL_LINK_PREDICT_ROWS.name => { - &mut self.max_temporal_link_predict_rows - } - n if n == MAX_TABULAR_CONTEXT_ROWS.name => &mut self.max_tabular_context_rows, - n if n == MAX_REQUEST_BYTES.name => &mut self.max_request_bytes, - _ => return false, - }; - *slot = value; - true - } -} - #[cfg(test)] mod tests { use super::*; #[test] - fn every_setting_is_named_once_and_read_from_its_own_variable() { + fn each_setting_has_its_own_name_variable_and_serving_layer_default() { let mut names: Vec<&str> = SETTINGS.iter().map(|s| s.name).collect(); names.sort_unstable(); names.dedup(); @@ -200,25 +147,18 @@ mod tests { for setting in SETTINGS { assert_eq!( setting.env, - format!("{ENVIRONMENT_PREFIX}{}", setting.name.to_ascii_uppercase()) + format!("SDM_CLIENT_{}", setting.name.to_ascii_uppercase()) ); - assert!(setting.default >= 1); - assert_eq!(Limits::default().get(setting.name), Some(setting.default)); - let mut limits = Limits::default(); - assert!(limits.set(setting.name, setting.default + 1)); - assert_eq!(limits.get(setting.name), Some(setting.default + 1)); } - assert_eq!(Limits::default().get("max_tables"), None); - assert!(!Limits::default().set("max_tables", 1)); - } - - #[test] - fn the_defaults_are_the_serving_layers() { - let limits = Limits::default(); - assert_eq!(limits.max_relational_context_rows, 10_000); - assert_eq!(limits.max_relational_predict_rows, 10_000); - assert_eq!(limits.max_temporal_link_predict_rows, 200); - assert_eq!(limits.max_tabular_context_rows, 10_000); - assert_eq!(limits.max_request_bytes, 31_457_280); + assert_eq!( + Limits::default(), + Limits { + max_relational_context_rows: 10_000, + max_relational_predict_rows: 10_000, + max_temporal_link_predict_rows: 200, + max_tabular_context_rows: 10_000, + max_request_bytes: 31_457_280, + } + ); } } diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 288b90dcd..569afddb4 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -69,9 +69,6 @@ pub enum Operation { OrderSampledTabularRows(crate::direct::DirectInput), SelectTabularRows(crate::direct::DirectInput), CheckTabularCap(crate::direct::DirectInput), - /// The size limits the run holds requests to, as resolved from its - /// start's (contract v72). - ResolveLimits(crate::direct::DirectInput), CheckDirectShape(crate::direct::DirectInput), CheckDirectCell(crate::direct::DirectInput), CheckColumn(crate::direct::DirectInput), @@ -164,7 +161,6 @@ impl Operation { Operation::OrderSampledTabularRows(_) => "order_sampled_tabular_rows", Operation::SelectTabularRows(_) => "select_tabular_rows", Operation::CheckTabularCap(_) => "check_tabular_cap", - Operation::ResolveLimits(_) => "resolve_limits", Operation::CheckDirectShape(_) => "check_direct_shape", Operation::CheckDirectCell(_) => "check_direct_cell", Operation::CheckColumn(_) => "check_column", @@ -337,7 +333,6 @@ impl Operation { | Operation::OrderSampledTabularRows(v) | Operation::SelectTabularRows(v) | Operation::CheckTabularCap(v) - | Operation::ResolveLimits(v) | Operation::CheckDirectShape(v) | Operation::CheckDirectCell(v) | Operation::FeatureImportance(v) @@ -1953,10 +1948,6 @@ pub struct Attachments { pub check_tabular_cap_input: crate::tabular::CheckTabularCap, #[schemars(rename = "DirectValue.check_tabular_cap")] pub check_tabular_cap_output: crate::tabular::CheckedTabularCap, - #[schemars(rename = "ResolveLimits.input")] - pub resolve_limits_input: crate::limits::ResolveLimits, - #[schemars(rename = "DirectValue.resolve_limits")] - pub resolve_limits_output: crate::limits::Limits, #[schemars(rename = "ObserveTabularMembership.input")] pub observe_tabular_membership_input: crate::tabular::ObserveTabularMembership, #[schemars(rename = "DirectValue.observe_tabular_membership")] diff --git a/rust/crates/sdm-contract/src/tabular.rs b/rust/crates/sdm-contract/src/tabular.rs index 443c1afad..24f473c0b 100644 --- a/rust/crates/sdm-contract/src/tabular.rs +++ b/rust/crates/sdm-contract/src/tabular.rs @@ -472,17 +472,22 @@ pub enum TabularOrderError { TemporalYearOutOfRange, } -/// Reached native comparison against the model cap, before frame length reads. +/// The context a materialized single-table request asks for, held to the +/// most context rows a request carries before frame length reads: the +/// `num_context_samples` given, as its decimal text however large, or none. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CheckTabularCap { - pub over_limit: bool, + pub requested: Option, } +/// The context asked for is within the limit, which the host compares its +/// own count with, natively, before it reads any frame length; past it, +/// the run is refused. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CheckedTabularCap { - pub accepted: bool, + pub limit: u64, } /// One ordinary selection composition. Only proven-passive native producers diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 2cdc4ef19..77521a45e 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -940,10 +940,7 @@ pub fn template(sentence: Sentence) -> Template { r#"the context alone is too large to send: a call carrying it and one row to score is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. Calls past the limit are split into smaller ones, but the context is sent whole with every call, so what comes down is num_context_samples, num_neighbors or the columns the graph carries; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::CallPastEndpoint => said( - r#"the context alone is too large to send: a call carrying it is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. Calls past the limit are split into smaller ones, but the context is sent whole with every call, so reduce the context gathered (num_context_samples), the neighbourhood sampled, or the tables and columns in the graph; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), - S::DirectCallPastBytes => said( - r#"this request is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. A single-table request is sent as one call, so score fewer rows at a time or carry a smaller context; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, + r#"this call is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. A relational call scoring several rows is split into smaller ones, but the context is sent whole with every call, and a single-table request or a pinned context is sent as one call; so carry a smaller context (num_context_samples, the neighbourhood sampled, or the tables and columns in the graph), or score fewer rows at a time; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, ), S::OutputsNone => said(r#"outputs must name at least one field"#), S::NotFinite => said( @@ -1336,9 +1333,6 @@ pub fn template(sentence: Sentence) -> Template { S::LimitEnvironmentInvalid => said( r#"the environment variable {env} is {given!r}, and {setting} takes a whole number of at least 1"#, ), - S::LimitGivenBelowOne => { - said(r#"{setting} is {given:,}, and it takes a whole number of at least 1"#) - } S::PartForecastStep => { said(r#"This is forecast step {forecast_step} of {forecast_steps}."#) } diff --git a/rust/crates/sdm-contract/tests/schema_lock.rs b/rust/crates/sdm-contract/tests/schema_lock.rs index 92abff4a9..bbe065843 100644 --- a/rust/crates/sdm-contract/tests/schema_lock.rs +++ b/rust/crates/sdm-contract/tests/schema_lock.rs @@ -557,11 +557,6 @@ const JSON: &[(&str, &str)] = &[ "OperationResult/direct_value.value", "DirectValue.check_tabular_cap", ), - ("Operation/resolve_limits.input", "ResolveLimits.input"), - ( - "OperationResult/direct_value.value", - "DirectValue.resolve_limits", - ), ( "Operation/observe_tabular_membership.input", "ObserveTabularMembership.input", diff --git a/rust/crates/sdm-core/src/direct.rs b/rust/crates/sdm-core/src/direct.rs index 9e30f0c4e..899e14758 100644 --- a/rust/crates/sdm-core/src/direct.rs +++ b/rust/crates/sdm-core/src/direct.rs @@ -831,7 +831,7 @@ pub(crate) async fn predict( part.columns = analysis; return crate::predict::finish(&asker, part, bound.outputs); } - let bytes = crate::predict::one_call_bytes(&payload, &limits)?; + let bytes = predict::call_bytes(&payload, limits.max_request_bytes)?; if deferred_options { use sdm_contract::direct::native_response::{NativeDirectCall, PreparedDirectCall}; let payload = asker.attach(Attachment::Bytes(payload.to_bytes()))?; diff --git a/rust/crates/sdm-core/src/dispatch.rs b/rust/crates/sdm-core/src/dispatch.rs index 4a0c30cb8..07296b66e 100644 --- a/rust/crates/sdm-core/src/dispatch.rs +++ b/rust/crates/sdm-core/src/dispatch.rs @@ -329,16 +329,12 @@ pub fn start( crate::direct::value(&asker, &crate::tabular_selection::select(input, &limits)?) }) } - Operation::ResolveLimits(request) => { - let _: sdm_contract::limits::ResolveLimits = - take_json(&mut attachments, request.input)?; - Run::new(move |asker| async move { crate::direct::value(&asker, &limits) }) - } Operation::CheckTabularCap(request) => { let input: sdm_contract::tabular::CheckTabularCap = take_json(&mut attachments, request.input)?; Run::new(move |asker| async move { - crate::direct::value(&asker, &sdm_request::tabular::check_materialized_cap(input)) + let checked = sdm_request::tabular::check_materialized_cap(input, &limits)?; + crate::direct::value(&asker, &checked) }) } Operation::CheckDirectShape(request) => { diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 992a3f836..3da7df268 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -297,7 +297,7 @@ pub(crate) async fn run( // Every scoring call held to the most bytes a call takes before // the first is sent. for (body, _) in &plans { - crate::predict::one_call_bytes(body, &limits)?; + predict::call_bytes(body, limits.max_request_bytes)?; } let mut parts = Vec::new(); let mut start = 0; diff --git a/rust/crates/sdm-core/src/limits.rs b/rust/crates/sdm-core/src/limits.rs index f6fe3ee2c..edf219780 100644 --- a/rust/crates/sdm-core/src/limits.rs +++ b/rust/crates/sdm-core/src/limits.rs @@ -7,52 +7,51 @@ //! environment's, else the default ([`sdm_contract::limits::SETTINGS`]). //! Every host reads its limits through this one rule. -use sdm_contract::limits::{GivenLimits, Limits, SETTINGS}; -use sdm_contract::{CoreError, Detail, Scalar, Sentence}; +use sdm_contract::limits::{ + GivenLimits, Limits, MAX_RELATIONAL_CONTEXT_ROWS, MAX_RELATIONAL_PREDICT_ROWS, + MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS, MAX_TEMPORAL_LINK_PREDICT_ROWS, SETTINGS, Setting, +}; +use sdm_contract::{CoreError, Detail, Sentence}; -/// The limits `given` names, or the refusal of a value that is not a whole -/// number of at least one, naming the setting and where it was set. A name -/// that is no setting's is the host's fault, `INVALID_MESSAGE`. +/// The limits `given` names, or the refusal of an environment variable that +/// is not a whole number of at least one, naming the setting and the +/// variable. A name that is no setting's, or a value given below one, is +/// the host's fault, `INVALID_MESSAGE`: a host checks its caller's values. pub fn resolve(given: &GivenLimits) -> Result { - for name in given.given.keys() { - if !SETTINGS.iter().any(|setting| setting.name == name) { - return Err(unknown("limits.given", name)); + for (name, &value) in &given.given { + if value == 0 || !SETTINGS.iter().any(|setting| setting.name == name) { + return Err(malformed("limits.given", name)); } } for env in given.environment.keys() { if !SETTINGS.iter().any(|setting| setting.env == env) { - return Err(unknown("limits.environment", env)); + return Err(malformed("limits.environment", env)); } } - let mut limits = Limits::default(); - for setting in &SETTINGS { - let value = if let Some(&value) = given.given.get(setting.name) { - if value == 0 { - return Err(setting.named( - CoreError::new( - Sentence::LimitGivenBelowOne, - "a limit the caller set is not a whole number of at least one", - ) - .with_field("given", Detail::Scalar(Scalar::UInt64(value))), - )); - } - value - } else if let Some(text) = given.environment.get(setting.env) { - whole(text).ok_or_else(|| { - setting.named( - CoreError::new( - Sentence::LimitEnvironmentInvalid, - "a limit's environment variable is not a whole number of at least one", - ) - .with_field("given", Detail::text(text)), - ) - })? - } else { - setting.default + let pick = |setting: &Setting| -> Result { + if let Some(&value) = given.given.get(setting.name) { + return Ok(value); + } + let Some(text) = given.environment.get(setting.env) else { + return Ok(setting.default); }; - limits.set(setting.name, value); - } - Ok(limits) + whole(text).ok_or_else(|| { + setting.named( + CoreError::new( + Sentence::LimitEnvironmentInvalid, + "a limit's environment variable is not a whole number of at least one", + ) + .with_field("given", Detail::text(text)), + ) + }) + }; + Ok(Limits { + max_relational_context_rows: pick(&MAX_RELATIONAL_CONTEXT_ROWS)?, + max_relational_predict_rows: pick(&MAX_RELATIONAL_PREDICT_ROWS)?, + max_temporal_link_predict_rows: pick(&MAX_TEMPORAL_LINK_PREDICT_ROWS)?, + max_tabular_context_rows: pick(&MAX_TABULAR_CONTEXT_ROWS)?, + max_request_bytes: pick(&MAX_REQUEST_BYTES)?, + }) } /// `text` as a whole number of at least one: decimal digits, with `_` @@ -74,10 +73,10 @@ fn whole(text: &str) -> Option { .filter(|&value| value >= 1) } -fn unknown(field: &str, name: &str) -> CoreError { +fn malformed(field: &str, name: &str) -> CoreError { CoreError::protocol( Sentence::InvalidMessage, - "the start names a limit that is no setting's", + "the start names a limit that is no setting's, or one below one", ) .with_field("field", Detail::text(field)) .with_field("name", Detail::text(name)) @@ -88,7 +87,6 @@ mod tests { use std::collections::BTreeMap; use sdm_contract::ErrorCode; - use sdm_contract::limits::{MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS}; use super::*; @@ -161,9 +159,9 @@ mod tests { } #[test] - fn a_given_zero_and_an_unknown_name_are_refused() { + fn a_given_zero_and_an_unknown_name_are_the_host_s_fault() { let error = resolve(&given(&[(MAX_REQUEST_BYTES.name, 0)], &[])).unwrap_err(); - assert_eq!(error.sentence, Sentence::LimitGivenBelowOne); + assert_eq!(error.sentence, Sentence::InvalidMessage); let error = resolve(&given(&[("max_tables", 20)], &[])).unwrap_err(); assert_eq!(error.sentence, Sentence::InvalidMessage); let error = resolve(&given(&[], &[("SDM_CLIENT_MAX_HOPS", "8")])).unwrap_err(); diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index f075a34e3..25b46f22c 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -527,81 +527,33 @@ fn fitted( payload: &Value, request_id: &str, ) -> Result)>, CoreError> { - let write = |call: &Value| { - body( + let limit = inputs.limits.max_request_bytes; + let mut fit = |call: &Value| -> sdm_request::batching::Fit<(Value, Vec)> { + let body = body( call, &inputs.model, request_id, outputs, &inputs.inference_asked, - ) - }; - let limit = inputs.limits.max_request_bytes; - let serving = inputs.endpoint.as_ref().map( - |ModelEndpoint::ServingEndpoint { - name, - max_request_bytes, - }| (name.as_str(), *max_request_bytes), - ); - let mut fits = |call: &Value| -> Result { - let body = write(call)?; - if body.to_bytes().len() as u64 > limit { - return Ok(false); - } - Ok(match serving { - Some((_, most)) => serving::envelope_len(&body)? <= most, - None => true, - }) - }; - let mut refuse = |call: &Value| -> CoreError { - let body = match write(call) { - Ok(body) => body, - Err(error) => return error, - }; + )?; let bytes = body.to_bytes(); if bytes.len() as u64 > limit { - return call_past_bytes(&body, bytes.len() as u64, limit); + return Ok(Err(call_past_bytes(&body, bytes.len() as u64, limit))); } - match serving { - Some((name, most)) => match serving::envelope(name, most, &bytes) { - Err(error) => error, - Ok(_) => internal("a call that fits was refused"), - }, - None => internal("a call that fits was refused"), + if let Some(ModelEndpoint::ServingEndpoint { + name, + max_request_bytes, + }) = &inputs.endpoint + && let (_, Some(past)) = serving::weigh(name, *max_request_bytes, &body)? + { + return Ok(Err(past)); } + Ok(Ok((body, bytes))) }; - sdm_request::batching::fitted(payload, &mut fits, &mut refuse)? + Ok(sdm_request::batching::fitted(payload, &mut fit)? .into_iter() - .map(|(_, call)| { - let body = write(&call)?; - let bytes = body.to_bytes(); - Ok((call, body, bytes)) - }) - .collect() -} - -/// The bytes of a call sent as it is, never split (a request of a single -/// table, sent as one call): refused past the most bytes a call takes -/// (`max_request_bytes`), naming the bytes of its context. -pub(crate) fn one_call_bytes(call: &Value, limits: &Limits) -> Result, CoreError> { - let bytes = call.to_bytes(); - let limit = limits.max_request_bytes; - if bytes.len() as u64 > limit { - let context = sdm_request::batching::predict_slice(call, 0..0) - .map_or(bytes.len() as u64, |context| { - context.to_bytes().len() as u64 - }); - return Err(sdm_contract::limits::MAX_REQUEST_BYTES.named( - CoreError::new( - Sentence::DirectCallPastBytes, - "a call sent whole is larger than one call takes", - ) - .with_field("bytes", count(bytes.len())) - .with_field("context_bytes", Detail::Scalar(Scalar::UInt64(context))) - .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), - )); - } - Ok(bytes) + .map(|(call, (body, bytes))| (call, body, bytes)) + .collect()) } /// One payload's call, its body as written, sent and its answer laid out; diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 9f117eaae..3558274ea 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -402,7 +402,7 @@ async fn predict( part.columns = analysis; return crate::predict::finish(asker, part, prepared.bound.outputs); } - let bytes = crate::predict::one_call_bytes(&payload, limits)?; + let bytes = sdm_request::predict::call_bytes(&payload, limits.max_request_bytes)?; let answer = crate::predict::call_answer(asker, bytes, input.endpoint.as_ref()).await?; finish(asker, payload, answer, prepared.bound.outputs, analysis) } diff --git a/rust/crates/sdm-core/tests/attachments.rs b/rust/crates/sdm-core/tests/attachments.rs index 7b238d100..64d0d3bf0 100644 --- a/rust/crates/sdm-core/tests/attachments.rs +++ b/rust/crates/sdm-core/tests/attachments.rs @@ -50,7 +50,6 @@ const READ: &[&str] = &[ "OrderSampledTabularRows.input", "SelectTabularRows.input", "CheckTabularCap.input", - "ResolveLimits.input", "CheckDirectShape.input", "BindDirect.input", "CheckDirectRows.input", diff --git a/rust/crates/sdm-core/tests/dispatch.rs b/rust/crates/sdm-core/tests/dispatch.rs index a33ca719d..e000ecd48 100644 --- a/rust/crates/sdm-core/tests/dispatch.rs +++ b/rust/crates/sdm-core/tests/dispatch.rs @@ -975,77 +975,3 @@ fn a_packed_date_literal_ends_validate_query_with_the_core_s_error() { assert_eq!(codes, want, "{literal}"); } } - -/// A `resolve_limits` run started with `limits`: its answer, or the start's -/// refusal. -fn resolved(limits: sdm_contract::limits::GivenLimits) -> Result { - use sdm_contract::direct::DirectInput; - let input = AttachmentId(1); - let mut run = start( - &root(), - StartRun { - limits, - contract: CONTRACT_VERSION, - facts: facts(), - operation: Operation::ResolveLimits(DirectInput { input }), - }, - Attachments::from([(input, json(&serde_json::json!({})))]), - )?; - let Step::Done { - result: OperationResult::DirectValue(value), - } = run.next()? - else { - panic!("the run answers a value"); - }; - let Some(Attachment::Bytes(bytes)) = run.take_attachment(value.value) else { - panic!("the value is attached"); - }; - Ok(serde_json::from_slice(&bytes).unwrap()) -} - -#[test] -fn a_run_resolves_its_limits_the_caller_s_before_the_environment_s_before_the_default() { - use sdm_contract::limits::GivenLimits; - let defaults = resolved(GivenLimits::default()).unwrap(); - assert_eq!( - defaults, - serde_json::json!({ - "max_relational_context_rows": 10_000, - "max_relational_predict_rows": 10_000, - "max_temporal_link_predict_rows": 200, - "max_tabular_context_rows": 10_000, - "max_request_bytes": 31_457_280, - }) - ); - let raised = resolved(GivenLimits { - given: BTreeMap::from([("max_relational_predict_rows".to_owned(), 50_000)]), - environment: BTreeMap::from([ - ( - "SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS".to_owned(), - "200000".to_owned(), - ), - ( - "SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS".to_owned(), - "20000".to_owned(), - ), - ]), - }) - .unwrap(); - assert_eq!(raised["max_tabular_context_rows"], 200_000); - assert_eq!(raised["max_relational_predict_rows"], 50_000); - assert_eq!(raised["max_request_bytes"], 31_457_280); - // A variable that is not a whole number refuses the start, by name. - let refused = resolved(GivenLimits { - given: BTreeMap::new(), - environment: BTreeMap::from([( - "SDM_CLIENT_MAX_REQUEST_BYTES".to_owned(), - "lots".to_owned(), - )]), - }) - .unwrap_err(); - assert_eq!(refused.code, ErrorCode::InvalidConfiguration); - assert_eq!( - refused.fields["env"], - sdm_contract::Detail::text("SDM_CLIENT_MAX_REQUEST_BYTES") - ); -} diff --git a/rust/crates/sdm-request/src/batching.rs b/rust/crates/sdm-request/src/batching.rs index d6e04475a..103e4ea49 100644 --- a/rust/crates/sdm-request/src/batching.rs +++ b/rust/crates/sdm-request/src/batching.rs @@ -19,6 +19,7 @@ use sdm_contract::{CoreError, Sentence}; use sdm_wire::json::Value; use crate::payload::INSTANCE_ID; +use crate::predict::predict_rows; fn bad(reason: &str) -> CoreError { CoreError::new( @@ -28,7 +29,7 @@ fn bad(reason: &str) -> CoreError { } /// The rows a table of the payload holds: its column names and rows. -fn table_rows(table: &Value) -> Option<(Vec<&str>, &[Value])> { +pub(crate) fn table_rows(table: &Value) -> Option<(Vec<&str>, &[Value])> { let Some(Value::Array(columns)) = table.get("columns") else { return None; }; @@ -39,12 +40,6 @@ fn table_rows(table: &Value) -> Option<(Vec<&str>, &[Value])> { (names.len() == columns.len()).then_some((names, rows.as_slice())) } -/// How many rows the payload scores: its predict half's instance rows. -pub fn predict_rows(payload: &Value) -> Option { - let table = payload.get("predict")?.get("instance_table")?; - table_rows(table).map(|(_, rows)| rows.len()) -} - /// An instance id as a whole number, where it is one. fn id_of(value: &Value) -> Option { match value { @@ -159,37 +154,43 @@ pub fn predict_slice(payload: &Value, range: Range) -> Result = Result, CoreError>; + +/// The calls `payload` is asked as so that each fits, in order, each with +/// what `fit` made of it: the whole where it fits, else halves, each split +/// again until it fits. `fit` answers what a call carries, or the refusal +/// of a call too large (the inner error); any other error ends the split. A +/// call of one row too large cannot be split, and its refusal is returned: +/// the context is sent with every call. +pub fn fitted( payload: &Value, - fits: &mut dyn FnMut(&Value) -> Result, - refuse: &mut dyn FnMut(&Value) -> CoreError, -) -> Result, Value)>, CoreError> { - if fits(payload)? { - let rows = predict_rows(payload).unwrap_or(0); - return Ok(vec![(0..rows, payload.clone())]); - } - let rows = predict_rows(payload).ok_or_else(|| bad("the rows scored are unread"))?; + fit: &mut dyn FnMut(&Value) -> Fit, +) -> Result, CoreError> { + let past = match fit(payload)? { + Ok(fitted) => return Ok(vec![(payload.clone(), fitted)]), + Err(past) => past, + }; + let rows = match predict_rows(payload) { + Some(rows) if rows > 1 => rows, + _ => return Err(past), + }; let mut done = Vec::new(); - // Ranges still to fit, the first next. - let mut pending: Vec> = Vec::new(); - pending.push(0..rows); + // Ranges still to fit, the first next; the whole is known not to. + let middle = rows / 2; + let mut pending: Vec> = vec![middle..rows, 0..middle]; while let Some(range) = pending.pop() { let sliced = predict_slice(payload, range.clone())?; - if range.len() != rows && fits(&sliced)? { - done.push((range, sliced)); - continue; - } - if range.len() <= 1 { - return Err(refuse(&sliced)); + match fit(&sliced)? { + Ok(fitted) => done.push((sliced, fitted)), + Err(past) if range.len() <= 1 => return Err(past), + Err(_) => { + let middle = range.start + range.len() / 2; + pending.push(middle..range.end); + pending.push(range.start..middle); + } } - let middle = range.start + range.len() / 2; - pending.push(middle..range.end); - pending.push(range.start..middle); } Ok(done) } @@ -257,43 +258,36 @@ mod tests { value.to_bytes().len() } + /// The rows each call of `whole` scores where a call takes `most` bytes. + fn calls(whole: &Value, most: usize) -> Result, CoreError> { + let mut fit = |p: &Value| { + Ok(if size(p) <= most { + Ok(()) + } else { + Err(CoreError::new(Sentence::Internal, "refused")) + }) + }; + Ok(fitted(whole, &mut fit)? + .iter() + .map(|(call, ())| predict_rows(call).unwrap()) + .collect()) + } + #[test] fn a_call_that_fits_is_one_call_and_one_past_is_halved_until_each_fits() { let whole = payload(); let one = size(&predict_slice(&whole, 0..1).unwrap()); let two = size(&predict_slice(&whole, 0..2).unwrap()); let all = size(&whole); - let mut refuse = |_: &Value| CoreError::new(Sentence::Internal, "refused"); - let calls = fitted(&whole, &mut |p| Ok(size(p) <= all), &mut refuse).unwrap(); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].0, 0..4); + assert_eq!(calls(&whole, all).unwrap(), [4]); // One byte under the whole: two calls of two rows. - let calls = fitted(&whole, &mut |p| Ok(size(p) < all), &mut refuse).unwrap(); - let ranges: Vec<_> = calls.iter().map(|(r, _)| r.clone()).collect(); - assert_eq!(ranges, [0..2, 2..4]); - // Exactly two rows' worth: still two calls; one byte under it: four. - let ranges: Vec<_> = fitted(&whole, &mut |p| Ok(size(p) <= two), &mut refuse) - .unwrap() - .into_iter() - .map(|(r, _)| r) - .collect(); - assert_eq!(ranges, [0..2, 2..4]); - let ranges: Vec<_> = fitted(&whole, &mut |p| Ok(size(p) < two), &mut refuse) - .unwrap() - .into_iter() - .map(|(r, _)| r) - .collect(); - // The last two rows carry fewer related rows than the first two, - // so together they fit where the first two do not. - assert_eq!(ranges, [0..1, 1..2, 2..4]); + assert_eq!(calls(&whole, all - 1).unwrap(), [2, 2]); + // Exactly two rows' worth: still two calls. One byte under it: the + // last two rows carry fewer related rows than the first two, so + // together they fit where the first two do not. + assert_eq!(calls(&whole, two).unwrap(), [2, 2]); + assert_eq!(calls(&whole, two - 1).unwrap(), [1, 1, 2]); // Under one row's worth, the context alone does not fit. - let mut refused_rows = None; - let error = fitted(&whole, &mut |p| Ok(size(p) < one), &mut |p| { - refused_rows = predict_rows(p); - CoreError::new(Sentence::Internal, "refused") - }) - .unwrap_err(); - assert_eq!(error.message, "refused"); - assert_eq!(refused_rows, Some(1)); + assert_eq!(calls(&whole, one - 1).unwrap_err().message, "refused"); } } diff --git a/rust/crates/sdm-request/src/options.rs b/rust/crates/sdm-request/src/options.rs index 930935fd6..fef3b18f1 100644 --- a/rust/crates/sdm-request/src/options.rs +++ b/rust/crates/sdm-request/src/options.rs @@ -615,12 +615,6 @@ impl Hops { } } -impl PartialEq> for Hops { - fn eq(&self, other: &Vec) -> bool { - self.more == 0 && &self.listed == other - } -} - impl From> for Hops { fn from(listed: Vec) -> Self { Hops { listed, more: 0 } @@ -1982,14 +1976,19 @@ mod tests { #[test] fn the_hops_are_the_run_mode_s_first_num_hops_two_by_default() { let fast = |text: &str| neighbors(&asked(text), RunMode::Fast).unwrap(); - assert_eq!(fast("{}"), vec![Hop::Keep(32), Hop::Keep(32)]); + assert_eq!(fast("{}"), Hops::from(vec![Hop::Keep(32), Hop::Keep(32)])); assert_eq!( fast(r#"{"num_hops": 4}"#), - vec![Hop::Keep(32), Hop::Keep(32), Hop::Keep(8), Hop::Keep(8)] + Hops::from(vec![ + Hop::Keep(32), + Hop::Keep(32), + Hop::Keep(8), + Hop::Keep(8) + ]) ); assert_eq!( neighbors(&asked(r#"{"num_hops": 3}"#), RunMode::Best).unwrap(), - vec![Hop::Keep(64), Hop::Keep(64), Hop::Keep(8)] + Hops::from(vec![Hop::Keep(64), Hop::Keep(64), Hop::Keep(8)]) ); // Past the six sizes a mode lists, each hop keeps the last. let eight = fast(r#"{"num_hops": 8}"#); @@ -2015,7 +2014,7 @@ mod tests { let given = asked(r#"{"num_neighbors": [5, 1, 0, 3], "num_hops": 2}"#); assert_eq!( neighbors(&given, RunMode::Fast).unwrap(), - vec![Hop::Keep(5), Hop::Keep(1), Hop::Stop, Hop::Keep(3)] + Hops::from(vec![Hop::Keep(5), Hop::Keep(1), Hop::Stop, Hop::Keep(3)]) ); // Numeric bounds keep their sign; caller text retains its origin // for capability-driven warehouse lookback arithmetic. @@ -2038,19 +2037,19 @@ mod tests { RunMode::Fast ) .unwrap(), - vec![Hop::Keep(4), wanted], + Hops::from(vec![Hop::Keep(4), wanted]), ); } assert_eq!(hops_ignored(&given), None); let wide = asked(r#"{"num_neighbors": [100000000000000000000000000000]}"#); assert_eq!( neighbors(&wide, RunMode::Fast).unwrap(), - vec![Hop::Keep(u64::MAX)] + Hops::from(vec![Hop::Keep(u64::MAX)]) ); let odd = asked(r#"{"num_neighbors": [2.0, 2.5, true, false, 0.5, -0.0, 0.0, 1e30]}"#); assert_eq!( neighbors(&odd, RunMode::Fast).unwrap(), - vec![ + Hops::from(vec![ Hop::Keep(2), Hop::Keep(2), Hop::Keep(1), @@ -2059,17 +2058,17 @@ mod tests { Hop::Stop, Hop::Stop, Hop::Keep(u64::MAX) - ] + ]) ); let spelled = asked(r#"{"num_neighbors": ["3", "-0", "0", "2.5"]}"#); assert_eq!( neighbors(&spelled, RunMode::Fast).unwrap(), - vec![Hop::Keep(3), Hop::Keep(0), Hop::Keep(0), Hop::Keep(2)] + Hops::from(vec![Hop::Keep(3), Hop::Keep(0), Hop::Keep(0), Hop::Keep(2)]) ); let spaced = asked(r#"{"num_neighbors": [" 3 ", "1e1", ".5"]}"#); assert_eq!( neighbors(&spaced, RunMode::Fast).unwrap(), - vec![Hop::Keep(3), Hop::Keep(10), Hop::Keep(0)] + Hops::from(vec![Hop::Keep(3), Hop::Keep(10), Hop::Keep(0)]) ); for size in ["null", "[1]"] { let e = neighbors( @@ -2089,7 +2088,7 @@ mod tests { ); assert_eq!( neighbors(&values, RunMode::Fast).unwrap(), - vec![ + Hops::from(vec![ Hop::Negative(sdm_contract::data::NegativeNeighborBound::new("-1".into()).unwrap()), Hop::Negative( sdm_contract::data::NegativeNeighborBound::new("-0.5".into()).unwrap() @@ -2098,7 +2097,7 @@ mod tests { Hop::Expression("3 OR TRUE".into()), Hop::Expression("nonsense".into()), Hop::Keep(u64::MAX), - ] + ]) ); } diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index bbfbc6e2b..5b55b8982 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -247,7 +247,7 @@ pub fn check_bytes(built: &Built, limit: u64) -> Result<(), CoreError> { } let payload = sdm_wire::json::read(&built.bytes) .map_err(|_| internal("a built payload does not read back"))?; - let rows = crate::batching::predict_rows(&payload).unwrap_or(0); + let rows = crate::predict::predict_rows(&payload).unwrap_or(0); // The bytes a call carrying the first `n` rows sends. let sent = |n: usize| -> Result { let mut call = crate::batching::predict_slice(&payload, 0..n)?; diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index 22e9fabd1..4b57b9a04 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -306,15 +306,7 @@ fn classifies(kind: &str) -> bool { /// One half's instance table: its column names and its rows. fn instance_table<'a>(payload: &'a Value, half: &str) -> Option<(Vec<&'a str>, &'a [Value])> { - let table = payload.get(half)?.get("instance_table")?; - let Some(Value::Array(columns)) = table.get("columns") else { - return None; - }; - let Some(Value::Array(rows)) = table.get("rows") else { - return None; - }; - let names: Vec<&str> = columns.iter().filter_map(Value::as_text).collect(); - (names.len() == columns.len()).then_some((names, rows.as_slice())) + crate::batching::table_rows(payload.get(half)?.get("instance_table")?) } /// The rows of one half, counted as `Request._rows` counts them: none diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index 5331ca5fe..e37935d5c 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -66,41 +66,6 @@ const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 429, 504]; /// rows is split to fit before it gets here ([`crate::batching`]), so the /// refusal names the context every call carries. pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result, CoreError> { - let read = readable(call)?; - let out = wrapped(&read)?; - let size = out.len() as u64; - if size > max_request_bytes { - let context = crate::batching::predict_slice(&read, 0..0) - .and_then(|context| wrapped(&context)) - .map_or(size, |context| context.len() as u64); - return Err(CoreError::new( - Sentence::ServingPastLimit, - "the call is larger than the serving endpoint takes", - ) - .with_field("endpoint", Detail::text(endpoint)) - .with_field("bytes", count(size)) - .with_field("context_bytes", count(context)) - .with_field("limit", count(max_request_bytes))); - } - Ok(out) -} - -/// How many bytes the endpoint's envelope of `call` is, as [`envelope`] -/// writes and weighs it. -pub fn envelope_len(call: &Value) -> Result { - if !call.all_finite() { - return Err(CoreError::new( - Sentence::ServingValueNotJson, - "the call carries a value with no JSON form", - ) - .with_field("reason", Detail::text("out_of_range_float"))); - } - Ok(wrapped(call)?.len() as u64) -} - -/// The call read back as the envelope carries it, refused where a value -/// has no JSON form. -fn readable(call: &[u8]) -> Result { let read = json::read(call).map_err(|e| { CoreError::new( Sentence::ServingValueNotJson, @@ -108,14 +73,42 @@ fn readable(call: &[u8]) -> Result { ) .with_field("reason", Detail::text(e.category)) })?; - if !read.all_finite() { + let (out, past) = weigh(endpoint, max_request_bytes, &read)?; + past.map_or(Ok(out), Err) +} + +/// The envelope of `call`, as [`envelope`] writes it, and where it is more +/// than `max_request_bytes`, its refusal (`ServingPastLimit`), naming the +/// context's bytes too; a value with no JSON form is refused outright. +pub fn weigh( + endpoint: &str, + max_request_bytes: u64, + call: &Value, +) -> Result<(Vec, Option), CoreError> { + if !call.all_finite() { return Err(CoreError::new( Sentence::ServingValueNotJson, "the call carries a value with no JSON form", ) .with_field("reason", Detail::text("out_of_range_float"))); } - Ok(read) + let out = wrapped(call)?; + let size = out.len() as u64; + if size <= max_request_bytes { + return Ok((out, None)); + } + let context = crate::batching::predict_slice(call, 0..0) + .and_then(|context| wrapped(&context)) + .map_or(size, |context| context.len() as u64); + let past = CoreError::new( + Sentence::ServingPastLimit, + "the call is larger than the serving endpoint takes", + ) + .with_field("endpoint", Detail::text(endpoint)) + .with_field("bytes", count(size)) + .with_field("context_bytes", count(context)) + .with_field("limit", count(max_request_bytes)); + Ok((out, Some(past))) } /// The envelope's bytes of a readable call. diff --git a/rust/crates/sdm-request/src/tabular.rs b/rust/crates/sdm-request/src/tabular.rs index f05c38be9..6f4a93df3 100644 --- a/rust/crates/sdm-request/src/tabular.rs +++ b/rust/crates/sdm-request/src/tabular.rs @@ -1244,12 +1244,32 @@ pub fn decide_materialized_task( } } -/// Select the cap refusal from a reached native comparison, without coercing -/// its original count owner or reading the candidate frame population. +/// The context a materialized request asks for held to the most a request +/// carries (`max_tabular_context_rows`), without reading the candidate +/// frame population: past it, `UNSUPPORTED_REQUEST` naming the count as it +/// was given and the setting that raises the limit. A count past `u128` is +/// past any limit, and one below zero is under it. pub fn check_materialized_cap( input: sdm_contract::tabular::CheckTabularCap, -) -> sdm_contract::tabular::CheckedTabularCap { - sdm_contract::tabular::CheckedTabularCap { - accepted: !input.over_limit, + limits: &sdm_contract::limits::Limits, +) -> Result { + let most = limits.max_tabular_context_rows; + if let Some(given) = input.requested { + let past = match given.parse::() { + Ok(count) => count > u128::from(most), + Err(_) => !given.starts_with('-'), + }; + if past { + return Err(sdm_contract::limits::MAX_TABULAR_CONTEXT_ROWS.named( + CoreError::new( + Sentence::NumContextSamplesPastMost, + "num_context_samples is past the most context a request carries", + ) + .with_field("field", Detail::text("num_context_samples")) + .with_field("given", Detail::text(given)) + .with_field("limit", Detail::Scalar(Scalar::UInt64(most))), + )); + } } + Ok(sdm_contract::tabular::CheckedTabularCap { limit: most }) } diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index 48a9fcc9b..3d21f086c 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -4755,12 +4755,10 @@ "apis": { "python": [ "Limits", - "Limits.resolved", "SDMClient.limits" ], "java": [ "Limits", - "Limits.resolved", "SdmClient.limits" ] }, @@ -4781,7 +4779,6 @@ "rust/crates/sdm-core/src/predict.rs" ], "operations": [ - "resolve_limits", "predict", "materialize" ] @@ -4802,7 +4799,7 @@ } } ], - "public_api_snapshot_sha256": "c6a02b69061a6f8f3a8703232962044d1ec5d79d67c4e8749374f57efc2ccee0", + "public_api_snapshot_sha256": "9d490ce2c68265be9c2bb679a7ed7215d594bd63fada342929eaa9833f4ea37b", "api_inventory": { "python": { "sdm_api.Endpoint": { @@ -6237,7 +6234,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_an_argument_comes_before_the_environment_before_the_default", + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", "behavior": "A setting given comes before its environment variable, which comes before the default, as resolved by the core." } ] @@ -6246,8 +6243,8 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_setting_given_that_is_not_a_whole_number_is_refused", + "behavior": "Each setting is a keyword of Limits taking a whole number of at least 1, read otherwise from SDM_CLIENT_ and its name in capitals." } ] }, @@ -6255,8 +6252,8 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_raised_relational_predict_limit_and_a_lowered_one_set_the_calls", + "behavior": "A lowered relational predict limit sets the rows each call scores." } ] }, @@ -6264,8 +6261,8 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_setting_given_that_is_not_a_whole_number_is_refused", + "behavior": "Each setting is a keyword of Limits taking a whole number of at least 1, read otherwise from SDM_CLIENT_ and its name in capitals." } ] }, @@ -6273,8 +6270,8 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", + "behavior": "A raised tabular context limit is taken, from the argument before the environment variable." } ] }, @@ -6282,17 +6279,8 @@ "capability": "configurable-limits", "assertions": [ { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Each setting is named, read from SDM_CLIENT_ and its name in capitals, and defaults to the serving layer's value." - } - ] - }, - "sdm_client.Limits.resolved": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_the_defaults_are_the_serving_layers", - "behavior": "Resolved limits equal the serving layer defaults when nothing is set." + "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_call_past_the_byte_limit_is_split_and_each_part_fits", + "behavior": "A call past the byte limit is split and each part fits." } ] }, @@ -8216,7 +8204,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8225,7 +8213,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8234,7 +8222,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8243,7 +8231,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8252,7 +8240,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8261,16 +8249,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public java.util.Map resolved();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#theDefaultsAreTheServingLayers", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8279,7 +8258,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8288,7 +8267,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#theDefaultsAreTheServingLayers", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8297,7 +8276,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8306,7 +8285,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8315,7 +8294,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8324,7 +8303,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8333,7 +8312,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8342,7 +8321,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8351,7 +8330,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8360,7 +8339,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] @@ -8369,7 +8348,7 @@ "capability": "configurable-limits", "assertions": [ { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenComesBeforeTheDefaultAndMustBeAtLeastOne", + "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." } ] diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 832327d39..e5e04b5f1 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -329,9 +329,7 @@ "max_tabular_context_rows", "max_temporal_link_predict_rows" ], - "public_methods": { - "resolved": "(self) -> 'dict[str, int]'" - }, + "public_methods": {}, "signature": "(max_relational_context_rows: 'int | None' = None, max_relational_predict_rows: 'int | None' = None, max_temporal_link_predict_rows: 'int | None' = None, max_tabular_context_rows: 'int | None' = None, max_request_bytes: 'int | None' = None) -> None" }, "LinkMetadata": { diff --git a/tests/test_skills.py b/tests/test_skills.py index 4fec39f88..3d56bc8c8 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2273,9 +2273,9 @@ def test_the_capability_table_is_the_clients_own(name, model_name): def _defaults() -> dict[str, int]: r"""Each setting's default, by name: the core's table.""" - from sdm_client.limits import settings + from sdm_client.limits import defaults - return {setting.name: setting.default for setting in settings()} + return defaults() def test_the_limits_table_is_the_shipped_limits(): From 368c51731b06b5e7b232eff0dc0d82a240da3518 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Sun, 8 Nov 2026 19:47:20 +0000 Subject: [PATCH 3/8] Hold requests to no size of the client's own, and recover instead Replace the configurable limits with a client that enforces none and is resilient to what the endpoint answers: - Remove the Limits settings, their SDM_CLIENT_MAX_* variables, the StartRun field, resolve_limits and the Python and Java plumbing, along with the base caps they configured: the 10,000 context rows (relational refusal and tabular default sampling), the 30 MiB body at materialize and at send, and the check_tabular_cap operation. The context is sampled only to a num_context_samples the caller passes; rows_per_call is taken as asked, 10,000 (200 for link prediction) being only the default batch. - On a 413, a gateway's 502 or 504, or a timeout (told to the core as a 408 by both hosts), predict asks the call's rows again in halves, each with the whole context, and keeps that size for the request's later calls. A refusal of one row, or a 422, is the endpoint's and shown as it said it. Both transports retry a 500 as they do a 503. - Databricks serving still splits a call up front to fit its max_request_bytes, reusing batching::fitted, since its oversize error is a dropped connection. - Keep the removal of the 15-table and 6-hop caps. - Regenerate the contract schema and lock (v72). Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 10 +- contract/expected/README.md | 1 - contract/tests/test_prediction_batching.py | 22 +- docs/reference/limits.md | 135 ++----- .../java/com/nvidia/sdm/ClientOptions.java | 21 +- .../com/nvidia/sdm/DatabricksServing.java | 24 +- java/src/main/java/com/nvidia/sdm/Limits.java | 117 ------ .../com/nvidia/sdm/MaterializeOptions.java | 16 - .../java/com/nvidia/sdm/PredictiveQuery.java | 9 +- .../nvidia/sdm/RelationalMaterializer.java | 3 +- .../main/java/com/nvidia/sdm/SdmClient.java | 23 +- .../com/nvidia/sdm/internal/ArrowTabular.java | 11 +- .../java/com/nvidia/sdm/internal/Host.java | 40 +- .../nvidia/sdm/internal/HttpTransport.java | 8 +- .../nvidia/sdm/internal/LimitSettings.java | 41 -- .../java/com/nvidia/sdm/internal/Wording.java | 18 - .../com/nvidia/sdm/DatabricksServingTest.java | 15 +- .../test/java/com/nvidia/sdm/LimitsTest.java | 54 --- .../src/sdm_client/__init__.py | 2 - .../src/sdm_client/_native.pyi | 4 - .../src/sdm_client/_rust/errors.py | 1 - .../src/sdm_client/_rust/predict.py | 11 +- .../src/sdm_client/_rust/run.py | 2 - .../sdm_client/_rust/tabular_materialize.py | 25 +- .../src/sdm_client/client.py | 75 ++-- .../src/sdm_client/core/transport.py | 2 +- .../src/sdm_client/limits.py | 156 -------- .../src/sdm_client/models.py | 11 +- .../src/sdm_client/query.py | 37 +- .../tests/tabular_pql_count_cases.py | 4 +- .../test_asked_in_the_questions_it_serves.py | 5 +- .../tests/test_configurable_limits.py | 219 ----------- .../tests/test_context_samples.py | 3 +- .../tests/test_limits_and_absent_outputs.py | 12 +- .../tests/test_limits_and_sessions.py | 9 +- .../tests/test_live_relational.py | 4 +- .../tests/test_payload_rules.py | 7 +- .../tests/test_predict_result.py | 12 +- .../tests/test_serving_route.py | 33 +- .../tests/test_the_predict_route.py | 41 +- .../tests/test_wording.py | 6 +- rust/bindings/sdm-jni/src/lib.rs | 10 - rust/bindings/sdm-py/src/lib.rs | 13 - rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 90 +---- rust/crates/sdm-contract/src/error.rs | 7 - rust/crates/sdm-contract/src/lib.rs | 1 - rust/crates/sdm-contract/src/limits.rs | 164 -------- rust/crates/sdm-contract/src/ops.rs | 9 +- rust/crates/sdm-contract/src/run.rs | 10 - rust/crates/sdm-contract/src/tabular.rs | 18 - rust/crates/sdm-contract/src/wording.rs | 26 +- rust/crates/sdm-contract/tests/corpus/mod.rs | 1 - rust/crates/sdm-contract/tests/host_facts.rs | 1 - .../tests/provider_view_contract.rs | 9 +- rust/crates/sdm-contract/tests/schema_lock.rs | 5 - rust/crates/sdm-core/src/direct.rs | 7 +- rust/crates/sdm-core/src/dispatch.rs | 37 +- rust/crates/sdm-core/src/frame_session.rs | 10 - rust/crates/sdm-core/src/lag.rs | 30 +- rust/crates/sdm-core/src/lib.rs | 1 - rust/crates/sdm-core/src/limits.rs | 170 --------- rust/crates/sdm-core/src/materialize.rs | 40 +- .../sdm-core/src/materialize/examples.rs | 31 +- .../crates/sdm-core/src/materialize/frames.rs | 2 - rust/crates/sdm-core/src/predict.rs | 156 +++++--- rust/crates/sdm-core/src/ranking.rs | 8 +- rust/crates/sdm-core/src/session.rs | 3 +- rust/crates/sdm-core/src/tabular_arrow.rs | 85 ++--- rust/crates/sdm-core/src/tabular_selection.rs | 24 +- rust/crates/sdm-core/src/test_ops.rs | 2 - rust/crates/sdm-core/tests/answer_encoding.rs | 1 - rust/crates/sdm-core/tests/attachments.rs | 1 - rust/crates/sdm-core/tests/dispatch.rs | 6 - rust/crates/sdm-core/tests/edit_schema.rs | 1 - .../sdm-core/tests/frame_arrow_schema.rs | 1 - rust/crates/sdm-core/tests/held.rs | 1 - rust/crates/sdm-core/tests/held_schema.rs | 1 - rust/crates/sdm-core/tests/materialize.rs | 1 - rust/crates/sdm-core/tests/multi_call.rs | 1 - rust/crates/sdm-core/tests/predict.rs | 28 +- rust/crates/sdm-core/tests/serving.rs | 36 +- .../crates/sdm-core/tests/tabular_sessions.rs | 24 +- rust/crates/sdm-core/tests/view_captures.rs | 2 - rust/crates/sdm-core/tests/view_semantic.rs | 1 - rust/crates/sdm-host/tests/codec.rs | 1 - rust/crates/sdm-host/tests/contract_v3.rs | 1 - rust/crates/sdm-host/tests/data_port.rs | 1 - rust/crates/sdm-host/tests/error_table.rs | 1 - rust/crates/sdm-host/tests/faults.rs | 2 - rust/crates/sdm-host/tests/runs.rs | 1 - rust/crates/sdm-host/tests/type_set.rs | 1 - rust/crates/sdm-request/src/batching.rs | 5 +- rust/crates/sdm-request/src/direct.rs | 9 +- rust/crates/sdm-request/src/options.rs | 105 ++---- rust/crates/sdm-request/src/payload.rs | 78 +--- rust/crates/sdm-request/src/predict.rs | 100 ----- rust/crates/sdm-request/src/serving.rs | 58 ++- rust/crates/sdm-request/src/tabular.rs | 30 -- rust/crates/sdm-request/tests/batch_limits.rs | 68 +--- .../crates/sdm-request/tests/payload_bytes.rs | 154 -------- rust/crates/sdm-testkit/src/host.rs | 5 - .../src/host_replay_validation_tests.rs | 1 - .../sdm-testkit/tests/duckdb_provider.rs | 2 - rust/crates/sdm-testkit/tests/neighbors.rs | 1 - .../sdm-testkit/tests/warehouse_replay.rs | 1 - skills/relational/SKILL.md | 32 +- skills/tabular/SKILL.md | 14 +- tests/compatibility/manifest.json | 355 +----------------- tests/compatibility/test_contract.py | 1 - tests/golden/public_api.json | 23 +- tests/test_skills.py | 32 +- 112 files changed, 518 insertions(+), 2890 deletions(-) delete mode 100644 java/src/main/java/com/nvidia/sdm/Limits.java delete mode 100644 java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java delete mode 100644 java/src/test/java/com/nvidia/sdm/LimitsTest.java delete mode 100644 packages/structured-data-models-client/src/sdm_client/limits.py delete mode 100644 packages/structured-data-models-client/tests/test_configurable_limits.py delete mode 100644 rust/crates/sdm-contract/src/limits.rs delete mode 100644 rust/crates/sdm-core/src/limits.rs delete mode 100644 rust/crates/sdm-request/tests/payload_bytes.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 00443b97e..1a6e1ad0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,6 @@ The public API snapshot records this version update; publication remains pending ### Added -- The size limits the client holds a request to are settings rather than constants, for a deployment configured to take more than the serving layer's defaults. - Python `Limits` and Java `Limits` name each one (`max_relational_context_rows`, `max_relational_predict_rows`, `max_temporal_link_predict_rows`, `max_tabular_context_rows`, `max_request_bytes`), taken by `SDMClient(limits=...)`, `materialize(limits=...)`, Java `ClientOptions`, `DatabricksServing` and `MaterializeOptions`. - A setting not given takes its environment variable (`SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS` and the like), else today's default. - A refusal at one of them names the setting and its environment variable. -- A call past the most bytes one call takes, or past a Databricks serving endpoint's 16 MB once wrapped, is split into smaller calls, each with the whole context, rather than refused; only a context too large on its own is refused, naming its size and the limit. - Python `Schema.from_relbench` and Java `Schema.fromRelbench` load the same pinned RelBench datasets with verified caches, offline mode and declared schema metadata. - Python and Java requests and fitted handles expose immutable sanitization reports for in-memory relational preparation, with per-table row exclusions and explicit unavailable/not-applicable states. @@ -71,7 +66,10 @@ The public API snapshot records this version update; publication remains pending ### Removed -- The relational limits of 15 tables in a request's graph and 6 hops, which an earlier endpoint enforced and the current model does not: a request may carry any number of tables and hops of at least one. +- The client's own size limits: the 10,000 context rows, the 30 MiB request body and the 15 tables and 6 hops of a relational request. + The client sends its best attempt: a call the endpoint refuses as too large (`413`, a gateway's `502` or `504`) or times out on is asked again in halves, each with the whole context, and a `500` is retried as a `503` is. + A `422`, or a refusal of a call of one row, is shown as the endpoint said it. + The context is sampled only to a `num_context_samples` the caller passes; through Databricks model serving, a call is still split up front to fit its 16 MB. ### Fixed diff --git a/contract/expected/README.md b/contract/expected/README.md index 576cac550..240a550c0 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -30,7 +30,6 @@ Their differences from DuckDB follow the fixture loader's explicit mappings in ` | Postgres | NaN-key fixtures refuse at materialize; a negative neighbor bound retains the existing `PROVIDER_ERROR` outcome | `sample.num-hops.seven` and `sample.num-neighbors.seven-hops` now reach predict on every backend: the client sets no most on hops, since the model reads as many as a request carries, where the reference refused past six. -The `multicall.body-30mib` cases keep their refusal at materialize, now naming the context's size and the `max_request_bytes` setting, since a call past the limit is split and only a context too large on its own is refused. These expectations describe the declared fixture mappings; they do not claim that every backend supports identical physical types. `explain.two-entities` now succeeds on all three backends: The client supports explanations for multiple prediction rows. diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 6b60579a7..7105edb25 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -680,22 +680,14 @@ def check() -> None: f'{case_key}: details[{name!r}] is {details.get(name)!r} ' f'where python-ref has {value!r}' ) + assert got['remedy'] == want['remedy'], ( + f'{case_key}: the remedy is {got["remedy"]!r:.200} where ' + f'python-ref names {want["remedy"]!r:.200}' + ) if case_id in BODY: - # Both contexts are past the 30 MiB on their own. This client - # splits a call past the limit into smaller ones, so it names - # what does not split, the context, its size and the limit, - # where the reference named a smaller call or a narrower context. - assert want['remedy'].startswith(BODY[case_id]) - assert (got['opening'] or '').startswith( - '[INVALID_REQUEST] the context alone is too large to send: a ' - 'call carrying it ' - 'and one row to score is ' - ), f'{case_key}: {got["opening"]!r:.300}' - assert got['remedy'] is None - else: - assert got['remedy'] == want['remedy'], ( - f'{case_key}: the remedy is {got["remedy"]!r:.200} where ' - f'python-ref names {want["remedy"]!r:.200}' + assert got['opening'] == want['opening'], ( + f'{case_key}: {got["opening"]!r:.200} where python-ref ' + f'opens {want["opening"]!r}' ) if case_id in INVALID: assert 'rows_per_call' in (got['opening'] or ''), ( diff --git a/docs/reference/limits.md b/docs/reference/limits.md index ff963b060..756ff3070 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -1,125 +1,42 @@ # Limits -The endpoint enforces several size limits. -`materialize()` checks them before sending, then refuses, samples or splits while naming the limit. -Otherwise, a rejected request returns a 500 or 422 that names neither the limit nor how to meet it. - -`Request(context=..., predict=...)` receives exact frames and does not trim them. -It refuses a context above the model's cap instead of discarding named rows, and reports the count and cap. - -`materialize()` keeps the context under each model's cap. -On `kumo-tabular`, it samples to the cap and reports the row count. -On `kumo-relational`, it gathers the context. -It refuses `num_context_samples` above the cap. -Omitting the argument gathers and reports the 1,000-row budget; it never samples down to 10,000. - -## Sizes - -Both endpoints enforce sizes, and most of these are the relational one's. - -| Limit | Model | Value | Checked | -|---|---|---|---| -| context rows | relational | 1,000 by default, up to 10,000 with `num_context_samples` | at `materialize()` | -| context rows | tabular | 10,000 | sampled at `materialize()`, refused before sending on a direct `Request` | -| rows scored per call | relational | 10,000, or 200 for link prediction | more are gathered as several calls at `materialize()` | -| entities one question gathers | relational | 100,000, in at most 100 calls | at `materialize()`, and this client's own: the endpoint is never asked for more than one call's worth | -| calls a question asked in parts makes | relational | 200, one per candidate or step | at `materialize()`, as `sdm_client.decompose.MOST_PARTS` | -| request body | tabular or relational | 30 MiB per call | a call past it is split into smaller calls when it is predicted; a context past it on its own is refused at `materialize()` | -| request body | tabular or relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | on the wrapped envelope, split as above | -| gradient explanations | relational and tabular | endpoint-dependent; no separate client row/context limit | endpoint validates support | +The client holds a request to no size of its own. +It sends its best attempt and recovers from what the endpoint says: -The relational model reads as many tables and hops as a request carries. -This client sets no most on either: a schema may describe any number of tables, and `num_hops` or `num_neighbors` may ask for any number of hops of at least one. -A walk that reaches no new row ends there, so a hop count past the data's depth costs nothing. - -A client opened with `SDMClient.in_process()` is held to this table as well, since this client checks it. -A NIM also has limits of its own, which each deployment sets, such as the longest string a cell may hold or the most distinct values a column may have. -Those protect a shared server, and are not applied when the model runs in your own process. - -## Settings - -The context rows, the rows scored per call and the request body are the serving layer's defaults (`TFM_MAX_TRAIN_ROWS`, `TFM_MAX_TEST_ROWS` and `TFM_MAX_PAYLOAD_BYTES`), and a deployment may be configured to take more or less. -Each is a setting of this client with that default: - -| Setting | Environment variable | Default | What it holds | -|---|---|---|---| -| `max_relational_context_rows` | `SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS` | 10,000 | the most context rows a relational request carries; `num_context_samples` asks for at most this many | -| `max_relational_predict_rows` | `SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS` | 10,000 | the most rows one relational call scores; more are asked in several calls | -| `max_temporal_link_predict_rows` | `SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS` | 200 | the most rows one temporal link prediction call scores | -| `max_tabular_context_rows` | `SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS` | 10,000 | the most context rows a single-table request carries; `materialize()` samples down to it | -| `max_request_bytes` | `SDM_CLIENT_MAX_REQUEST_BYTES` | 31,457,280 (30 MiB) | the most bytes one call's body takes | - -Set one with an argument or with its environment variable. -An argument comes before the environment variable, which comes before the default. -In Python, pass `limits=` to the client and to `materialize()`: - -```python -from sdm_client import Limits, SDMClient - -limits = Limits(max_tabular_context_rows=200_000, max_request_bytes=512 * 1024 * 1024) -client = SDMClient(url, limits=limits) -request = query.materialize(indices=keys, limits=limits) -result = client.predict(request, model=KumoTabular()) -``` +| The endpoint answers | The client | +|---|---| +| `413`, a gateway's `502` or `504`, or the call times out | asks the call's rows again in two halves, each with the whole context, and keeps that smaller size for the request's later calls | +| `503` | waits as `Retry-After` says and tries again | +| a connection error or another `5xx` | tries again with backoff, up to `max_retries` (3 by default) | +| `422`, or a refusal of a call of one row | does not try again, and shows the endpoint's message | -In Java, pass `Limits.builder()...build()` to `ClientOptions.builder().limits(...)`, `DatabricksServing.limits(...)` and `MaterializeOptions.builder().limits(...)`. -Both read the same environment variables, and an environment variable that is not a whole number of at least 1 is refused with code `INVALID_CONFIGURATION`, naming it. -A refusal at one of these limits names the setting and its environment variable. +The context is never split, so a context too large on its own is the endpoint's refusal, shown as it said it. +The context is sampled only to a `num_context_samples` you pass. +`timeout` and `max_retries` are optional settings of the client. -Raise a setting only to what the deployment is configured to take. -This client then holds requests to the raised limit, and the endpoint answers them; a setting above the deployment's limit only moves the refusal to the endpoint. +Through Databricks model serving, an oversized call is dropped as an opaque connection error, so `predict()` splits a call whose wrapped envelope is past the endpoint's `max_request_bytes` (16 MB, the gateway's own decimal limit) before sending it. +A call of one row past it is refused, since only a smaller context helps. ## Batches -A relational request for more entities than one call scores is not refused. -`materialize()` gathers it in batches of at most that many, each with a context of its own, sampled around that batch's rows rather than copied from the first, and `predict()` sends one call per batch and returns one result. +A relational request is gathered in batches of 10,000 rows to score, or 200 for a link prediction: a default, not a cap. +Pass `rows_per_call` to `materialize()` to ask in batches of another size. +Each batch has a context of its own, sampled around that batch's rows rather than copied from the first, and `predict()` sends one call per batch and returns one result. Its `row_index` counts across the whole request rather than from zero per call, so `to_pandas()`, which orders by it, gives the indices back in the order they were asked for. A session scores those batches against the context it pinned, so the context is uploaded once rather than once per call. `fit()` itself takes a request of one call, since a session pins one context: fit a small request and score the batched one against the handle. -**The 10,000 is the deployment's, not this client's.** -It is the serving layer's `TFM_MAX_TEST_ROWS`, whose default it is, so a deployment may hold to more or less. -Measured at the boundary rather than read from a document: 10,000 rows to score are answered and 10,001 come back as - -``` -422 UNSUPPORTED_SHAPE N_test 10001 exceeds limit 10000 -``` - -The earlier `kumo-relational` container holds to 1,000 instead, through a model-specific admission check of its own, so a request gathered at this default is refused there on its first call. - -Pass `rows_per_call` to `materialize()` to ask in smaller batches than that. -It only ever goes down, since a larger call buys a 422 rather than a bigger call. -Reach for it for a deployment whose cap is lower than the one above, or set `max_relational_predict_rows` to that cap. - -**A call past the request body's limit is split by its bytes as well as its rows.** -When a call's body would pass `max_request_bytes`, or a Databricks serving endpoint's 16 MB once wrapped, `predict()` sends its rows as several smaller calls, halving until each fits, and returns one result numbered as before. -Every one of those calls carries the batch's whole context, which is not split. -Where the context alone, with one row to score, is past the limit, no call is small enough, and the request is refused with the context's size and the limit: a wide graph at 10,000 context rows can be. -`num_context_samples`, `num_neighbors` and the columns the graph carries are what come down then, or the limit goes up where the deployment takes more. -`materialize()` makes the same check before anything is sent. -A single-table request is sent as one call and is not split: past the limit, it is refused with its size, and the rows scored at once or the context come down. - -**That 100,000 is this client's, not the endpoint's.** -The endpoint is never asked for more than one call holds. -Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once, and 100,000 entities is about a gigabyte of it. -One `materialize()` gathers at most that many, in at most 100 calls, and the refusal names whichever of the two it met. -The second is what a small `rows_per_call` reaches, and what a temporal link prediction reaches at 20,001 entities, since it scores 200 a call. - -Both govern the batches of one question, and neither counts the parts of a question asked in several. -A ranking is one call per candidate and a forecast one per step, and what bounds those is `sdm_client.decompose.MOST_PARTS`, 200, which is why a 101-candidate ranking makes 101 calls without meeting the 100 above. -What the two ceilings hold for such a question is the population each part gathers, counted across all of them. +Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once. +One `materialize()` gathers at most 100,000 entities, in at most 100 calls, and the refusal names whichever of the two it met. +Neither counts the parts of a question asked in several: a ranking is one call per candidate and a forecast one per step, bounded by `sdm_client.decompose.MOST_PARTS`, 200. +A question asked in parts is not batched by rows, since a batched part would send the parts times the batches. A call that fails ends the request. The answers already paid for are carried on the error as `details['answered']`, and the rows they cover as `details['rows_answered']`, so the rest can be asked for again without buying the whole population twice. A forecast keeps its answered steps the same way, under `details['answered']`, numbered by step, with how many of them under `details['parts_answered']`; a ranking keeps only the count, because an order over some of its candidates is a different answer rather than part of the one that was asked for. -**A question asked in parts is not batched by rows.** -A ranking is asked as one call per candidate and a forecast as one per step, and `MOST_PARTS` counts those calls, so a batched part would send the parts times the batches: a ranking over 200 candidates and 50,000 entities would pass as 200 calls and make 1,000. -More entities than one call scores is refused for both, before anything is gathered. -A ranking could not be batched in any case, since a candidate is reported unscored for the whole ranking rather than per batch, so one batch whose context held a single class would drop it from every row. -A part past the request body's limit is still split by its bytes, each smaller call carrying the part's one context. - -The tabular cap was measured at the boundary rather than read from a document: 10,000 context rows are answered and 10,001 come back as `422 UNSUPPORTED_SHAPE N_train 10001 exceeds limit 10000`. +The relational model reads as many tables and hops as a request carries. +A walk that reaches no new row ends there, so a hop count past the data's depth costs nothing. ## Aggregation windows open after the anchor and close on their last day @@ -151,13 +68,11 @@ values. Omit it on a relational request to use the smaller budget, up to 1,000 examples. This is the endpoint's long-standing default. Omit it on a single-table `materialize()` with either `indices=` or `predict=` to use every -labelled row, up to the endpoint's 10,000-row limit. Above that, it samples -10,000 rows and reports doing so. Use the argument to request more. +labelled row. Ask for more than exists and you get what exists; there are only so many -labelled examples, and that is not an error. Ask for more than 10,000 and you -are told the ceiling rather than silently given fewer. Zero, a negative, a -fraction or a string are refused by name. +labelled examples, and that is not an error. Zero, a negative, a fraction or a +string are refused by name. **Which examples you get** when the count narrows: the first `n` of what was gathered, and for a temporal question that is **not a sample across time**. diff --git a/java/src/main/java/com/nvidia/sdm/ClientOptions.java b/java/src/main/java/com/nvidia/sdm/ClientOptions.java index 1e942530f..292fee843 100644 --- a/java/src/main/java/com/nvidia/sdm/ClientOptions.java +++ b/java/src/main/java/com/nvidia/sdm/ClientOptions.java @@ -9,23 +9,19 @@ /** * How {@link SdmClient#connect(String, ClientOptions)} reaches the endpoint, as Python's {@code - * SDMClient(url, api_key, timeout=, max_retries=, limits=)}: an API key sent as {@code X-API-Key} - * (refused over plaintext {@code http://} to a host other than this one), how long each attempt - * may take (60 seconds), how many times a transient failure is tried again (3), and the size - * limits requests are held to ({@link Limits}, each by default its environment variable's, else - * the serving layer's default). + * SDMClient(url, api_key, timeout=, max_retries=)}: an API key sent as {@code X-API-Key} (refused + * over plaintext {@code http://} to a host other than this one), how long each attempt may take + * (60 seconds), and how many times a transient failure is tried again (3). */ public final class ClientOptions { private final String apiKey; private final Duration timeout; private final int maxRetries; - private final Limits limits; private ClientOptions(Builder builder) { this.apiKey = builder.apiKey; this.timeout = builder.timeout; this.maxRetries = builder.maxRetries; - this.limits = builder.limits; } public static ClientOptions defaults() { @@ -48,16 +44,11 @@ int maxRetries() { return maxRetries; } - Limits limits() { - return limits; - } - /** Builds {@link ClientOptions}. */ public static final class Builder { private String apiKey; private Duration timeout = Duration.ofSeconds(60); private int maxRetries = 3; - private Limits limits = Limits.defaults(); private Builder() {} @@ -79,12 +70,6 @@ public Builder maxRetries(int maxRetries) { return this; } - /** The size limits requests to the endpoint are held to. */ - public Builder limits(Limits limits) { - this.limits = Objects.requireNonNull(limits, "limits"); - return this; - } - public ClientOptions build() { return new ClientOptions(this); } diff --git a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java index b05ddada1..edfa4e823 100644 --- a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java +++ b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java @@ -64,17 +64,15 @@ public final class DatabricksServing { private final AuthenticationSource authentication; private final long maxRequestBytes; private final Duration timeout; - private final Limits limits; private final HttpClient client; private DatabricksServing( String endpoint, AuthenticationSource authentication, - long maxRequestBytes, Duration timeout, Limits limits) { + long maxRequestBytes, Duration timeout) { this.endpoint = endpoint; this.authentication = authentication; this.maxRequestBytes = maxRequestBytes; this.timeout = timeout; - this.limits = limits; this.client = HttpClient.newBuilder().connectTimeout(timeout).build(); } @@ -112,7 +110,7 @@ public static DatabricksServing of( Map headers = Map.copyOf(config.authenticate()); return new Authentication(config.getHost(), headers); }); - return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT, Limits.defaults()); + return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT); } /** The endpoint {@code endpoint} in the workspace at {@code host}, with the gateway's limit. */ @@ -140,7 +138,7 @@ public static DatabricksServing of( positiveLimit(maxRequestBytes); AuthenticationSource source = new AuthenticationSource(() -> new Authentication( base, Map.of("Authorization", "Bearer " + token.get()))); - return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT, Limits.defaults()); + return new DatabricksServing(name, source, maxRequestBytes, DEFAULT_TIMEOUT); } /** This endpoint, each attempt and its retries bounded by {@code timeout}. */ @@ -148,16 +146,7 @@ public DatabricksServing timeout(Duration timeout) { if (timeout == null || timeout.isNegative() || timeout.isZero()) { throw configuration("timeout must be a positive duration"); } - return new DatabricksServing(endpoint, authentication, maxRequestBytes, timeout, limits); - } - - /** - * This endpoint, its requests held to {@code limits} as a client's are ({@link Limits}); the - * gateway's own limit on each wrapped call, {@link #maxRequestBytes}, holds as well. - */ - public DatabricksServing limits(Limits limits) { - return new DatabricksServing( - endpoint, authentication, maxRequestBytes, timeout, Objects.requireNonNull(limits, "limits")); + return new DatabricksServing(endpoint, authentication, maxRequestBytes, timeout); } /** The serving endpoint's name. */ @@ -175,11 +164,6 @@ public Duration timeout() { return timeout; } - /** The size limits requests to this endpoint are held to. */ - public Limits limits() { - return limits; - } - /** Where requests go: the endpoint's name, since it has no URL of its own. */ String url() { return "databricks-serving:" + endpoint; diff --git a/java/src/main/java/com/nvidia/sdm/Limits.java b/java/src/main/java/com/nvidia/sdm/Limits.java deleted file mode 100644 index 5fc7d995c..000000000 --- a/java/src/main/java/com/nvidia/sdm/Limits.java +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.nvidia.sdm; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * The size limits a client holds a request to before it is sent, as Python's {@code - * sdm_client.Limits}. Each mirrors a limit the serving layer enforces ({@code TFM_MAX_TRAIN_ROWS}, - * {@code TFM_MAX_TEST_ROWS}, {@code TFM_MAX_PAYLOAD_BYTES}), which a deployment sets, so each is a - * setting with the serving layer's default. - * - *

{@code
- * Limits limits = Limits.builder()
- *     .maxTabularContextRows(200_000)
- *     .maxRequestBytes(512L * 1024 * 1024)
- *     .build();
- * SdmClient client = SdmClient.connect(url, ClientOptions.builder().limits(limits).build());
- * Request request = query.materialize(MaterializeOptions.builder().indices(keys).limits(limits).build());
- * }
- * - *

A setting not set here takes its environment variable, {@code SDM_CLIENT_} and its name in - * capitals (such as {@code SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS}), else its default: 10,000 context - * rows for either model, 10,000 rows scored per relational call, 200 per temporal link prediction - * call, and 31,457,280 bytes (30 MiB) per call. The core resolves them, as it does for every host. - */ -public final class Limits { - private final Map values; - - private Limits(Map values) { - this.values = Collections.unmodifiableMap(new LinkedHashMap<>(values)); - } - - /** No setting set: each takes its environment variable, else its default. */ - public static Limits defaults() { - return new Limits(Map.of()); - } - - public static Builder builder() { - return new Builder(); - } - - /** The settings set here, by name, such as {@code max_request_bytes}; none for a default. */ - public Map values() { - return values; - } - - @Override - public boolean equals(Object other) { - return other instanceof Limits limits && limits.values.equals(values); - } - - @Override - public int hashCode() { - return values.hashCode(); - } - - @Override - public String toString() { - return "Limits" + values; - } - - /** Builds {@link Limits}; each value is a whole number of at least 1. */ - public static final class Builder { - private final Map values = new LinkedHashMap<>(); - - private Builder() {} - - /** The most context rows a relational request carries ({@code num_context_samples}' most). */ - public Builder maxRelationalContextRows(long rows) { - return set("max_relational_context_rows", rows); - } - - /** The most rows one relational call scores; more are asked in several calls. */ - public Builder maxRelationalPredictRows(long rows) { - return set("max_relational_predict_rows", rows); - } - - /** The most rows one temporal link prediction call scores. */ - public Builder maxTemporalLinkPredictRows(long rows) { - return set("max_temporal_link_predict_rows", rows); - } - - /** The most context rows a single-table request carries; materialize samples down to it. */ - public Builder maxTabularContextRows(long rows) { - return set("max_tabular_context_rows", rows); - } - - /** - * The most bytes one call's body takes. A call past it that scores several rows is split into - * smaller calls; one whose context alone is past it is refused. - */ - public Builder maxRequestBytes(long bytes) { - return set("max_request_bytes", bytes); - } - - private Builder set(String name, long value) { - if (value < 1) { - throw new StructuredDataException( - name + " is " + value + ", and it takes a whole number of at least 1", - "INVALID_CONFIGURATION", - Map.of("setting", name), - null); - } - values.put(name, value); - return this; - } - - public Limits build() { - return new Limits(values); - } - } -} diff --git a/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java b/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java index 04bfd91bf..0a87aa6b6 100644 --- a/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java +++ b/java/src/main/java/com/nvidia/sdm/MaterializeOptions.java @@ -48,7 +48,6 @@ public final class MaterializeOptions { private final Object contextAnchorTime; private final boolean contextAnchorGiven; private final Map options; - private final Limits limits; private MaterializeOptions(Builder builder) { this.model = builder.model; @@ -60,7 +59,6 @@ private MaterializeOptions(Builder builder) { this.contextAnchorTime = builder.contextAnchorTime; this.contextAnchorGiven = builder.contextAnchorGiven; this.options = Collections.unmodifiableMap(new LinkedHashMap<>(builder.options)); - this.limits = builder.limits; } public static Builder builder() { @@ -95,10 +93,6 @@ Map options() { return options; } - Limits limits() { - return limits; - } - /** Builds {@link MaterializeOptions}. */ public static final class Builder { private String model; @@ -110,7 +104,6 @@ public static final class Builder { private Object contextAnchorTime; private boolean contextAnchorGiven; private final Map options = new LinkedHashMap<>(); - private Limits limits = Limits.defaults(); private Builder() {} @@ -214,15 +207,6 @@ public Builder rowsPerCall(long rows) { return option("rows_per_call", rows); } - /** - * The size limits the request is gathered for: pass the {@link SdmClient#limits} of the - * client it will be sent with ({@link Limits}). - */ - public Builder limits(Limits limits) { - this.limits = Objects.requireNonNull(limits, "limits"); - return this; - } - public MaterializeOptions build() { return new MaterializeOptions(this); } diff --git a/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java b/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java index dedd66b8f..186891d9d 100644 --- a/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java +++ b/java/src/main/java/com/nvidia/sdm/PredictiveQuery.java @@ -130,8 +130,7 @@ public Request materialize(Schema schema, org.apache.arrow.vector.table.Table pr input.put("task", options.task()); try (Host.Finished finished = com.nvidia.sdm.internal.ArrowTabular.run("materialize_predict", input, Map.of("table", schema.data().values().iterator().next(), "predict", - com.nvidia.sdm.internal.ArrowTabular.snapshot(predict)), null, - options.limits().values())) { + com.nvidia.sdm.internal.ArrowTabular.snapshot(predict)), null)) { Map result = com.nvidia.sdm.internal.ArrowTabular.value(finished); @SuppressWarnings("unchecked") Map description = (Map) result.get("description"); Map request = com.nvidia.sdm.internal.ArrowTabular.request( @@ -186,8 +185,7 @@ public Request materialize(Schema schema, MaterializeOptions options) { throw new StructuredDataException("warehouse sampling options do not apply to Tabular", "INVALID_REQUEST", Map.of(), null); try (Host.Finished finished = com.nvidia.sdm.internal.ArrowTabular.run("materialize", input, - Map.of("table", schema.data().values().iterator().next()), null, - options.limits().values())) { + Map.of("table", schema.data().values().iterator().next()), null)) { Map result = com.nvidia.sdm.internal.ArrowTabular.value(finished); @SuppressWarnings("unchecked") Map description = (Map) result.get("description"); Map request = com.nvidia.sdm.internal.ArrowTabular.request( @@ -224,8 +222,7 @@ public Request materialize(Schema schema, MaterializeOptions options) { new LinkedHashMap<>(Providers.held(schema.source())); held.put("query", fields -> text); try (Host.Finished finished = - Host.run(operation, attachments, - new Host.Call(chosen.provider(), null, held, options.limits().values()))) { + Host.run(operation, attachments, new Host.Call(chosen.provider(), null, held))) { Results.Gathered gathered = (Results.Gathered) Results.expect(finished.result(), "materialize"); if (gathered.query() == null) { diff --git a/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java b/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java index b3ac96c37..ccf5e24cf 100644 --- a/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java +++ b/java/src/main/java/com/nvidia/sdm/RelationalMaterializer.java @@ -75,8 +75,7 @@ synchronized Request materialize(PredictiveQuery query, MaterializeOptions optio attached.put(2, Host.attachment(input)); attached.put(3, Indices.column(options.indices(), ArrowData.allocator())); attached.put(4, Host.optionAttachment(PredictiveQuery.values(options))); - try (Host.Finished finished = - Host.run(op, attached, Host.Call.limited(options.limits().values()))) { + try (Host.Finished finished = Host.run(op, attached)) { Results.Gathered gathered = (Results.Gathered) Results.expect(finished.result(), "materialize"); @SuppressWarnings("unchecked") Map state = (Map) finished.result().get("state"); MaterializedRequest retained = new MaterializedRequest(state, finished); diff --git a/java/src/main/java/com/nvidia/sdm/SdmClient.java b/java/src/main/java/com/nvidia/sdm/SdmClient.java index 810fe4a49..6606e187d 100644 --- a/java/src/main/java/com/nvidia/sdm/SdmClient.java +++ b/java/src/main/java/com/nvidia/sdm/SdmClient.java @@ -39,14 +39,12 @@ public final class SdmClient implements AutoCloseable { private final HttpTransport transport; private final DatabricksServing serving; - private final Limits limits; private volatile boolean closed; private final java.util.Set sessions = java.util.concurrent.ConcurrentHashMap.newKeySet(); - private SdmClient(HttpTransport transport, DatabricksServing serving, Limits limits) { + private SdmClient(HttpTransport transport, DatabricksServing serving) { this.transport = transport; this.serving = serving; - this.limits = limits; } /** @@ -64,9 +62,7 @@ public static SdmClient connect(String url) { public static SdmClient connect(String url, ClientOptions options) { Objects.requireNonNull(options, "options"); return new SdmClient( - new HttpTransport(url, options.apiKey(), options.timeout(), options.maxRetries()), - null, - options.limits()); + new HttpTransport(url, options.apiKey(), options.timeout(), options.maxRetries()), null); } /** @@ -75,16 +71,7 @@ public static SdmClient connect(String url, ClientOptions options) { * endpoint's envelope and reads its every answer and failure. */ public static SdmClient forDatabricksServing(DatabricksServing serving) { - Objects.requireNonNull(serving, "serving"); - return new SdmClient(null, serving, serving.limits()); - } - - /** - * The size limits this client holds requests to, as it was opened with them; a setting not set - * takes its environment variable, else its default. - */ - public Limits limits() { - return limits; + return new SdmClient(null, Objects.requireNonNull(serving, "serving")); } /** Where this client sends requests: a URL, or {@code databricks-serving:}. */ @@ -210,10 +197,6 @@ void requireOpen() { Host.Model tabularTransport() { return new Host.Model() { - public Map limits() { - return limits.values(); - } - public byte[] call(byte[] body) throws Host.TransportFailed, Host.Refused, Host.EndpointFailed { return SdmClient.this.call(body); } diff --git a/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java b/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java index 6cdf598ea..3d7db9d50 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java +++ b/java/src/main/java/com/nvidia/sdm/internal/ArrowTabular.java @@ -54,15 +54,6 @@ public static byte[] snapshot(Table table) { public static Host.Finished run(String action, Object input, Map tables, Host.Model model) { - return run(action, input, tables, model, null); - } - - /** - * Runs {@code action} as {@link #run(String, Object, Map, Host.Model)}, held to the size limits - * {@code limits} names (null: those of {@code model}'s client, else none). - */ - public static Host.Finished run(String action, Object input, Map tables, - Host.Model model, Map limits) { Map operation = new LinkedHashMap<>(); operation.put("op", "tabular_arrow"); operation.put("action", action); @@ -81,7 +72,7 @@ public static Host.Finished run(String action, Object input, Map operation.put(entry.getKey(), id); attachments.put(id, stream); } - return Host.run(operation, attachments, new Host.Call(null, model, Map.of(), limits)); + return Host.run(operation, attachments, new Host.Call(null, model, Map.of())); } finally { for (ArrowArrayStream stream : streams) stream.close(); for (ArrowReader reader : readers) { diff --git a/java/src/main/java/com/nvidia/sdm/internal/Host.java b/java/src/main/java/com/nvidia/sdm/internal/Host.java index 3a8d4e7da..1807bc2db 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Host.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Host.java @@ -87,14 +87,6 @@ default Map failure(Exception failure) { public interface Model { byte[] call(byte[] body) throws TransportFailed, Refused, EndpointFailed; - /** - * The size limits the caller set on the client this model is reached through, by setting - * name; null where it set none. - */ - default Map limits() { - return null; - } - /** Session-aware transports override this; existing model callbacks remain prediction-only. */ default byte[] call(Object endpoint, byte[] body) throws TransportFailed, Refused, EndpointFailed { if (!"predict".equals(endpoint)) throw new StructuredDataException( @@ -163,41 +155,19 @@ public Integer status() { /** * What one run is given beside its operation: its provider and model, or null for an operation - * that reads no data and calls no model, the values only this host holds that a sentence may - * name ({@code {host.}}), each read from the error's or warning's fields, and the size - * limits the caller set, by setting name ({@code com.nvidia.sdm.Limits}), null for those of the - * model's client, else none. + * that reads no data and calls no model, and the values only this host holds that a sentence + * may name ({@code {host.}}), each read from the error's or warning's fields. */ public record Call( - Provider provider, - Model model, - Map, Object>> held, - Map limits) { + Provider provider, Model model, Map, Object>> held) { public Call { held = held == null ? Map.of() : Map.copyOf(held); - limits = limits == null ? null : Map.copyOf(limits); - } - - public Call( - Provider provider, Model model, Map, Object>> held) { - this(provider, model, held, null); } /** A run with nothing beside its operation. */ public static Call none() { return new Call(null, null, Map.of()); } - - /** A run with nothing beside its operation but the size limits the caller set. */ - public static Call limited(Map limits) { - return new Call(null, null, Map.of(), limits); - } - - /** The size limits the run is held to: those given, else those of the model's client. */ - Map effectiveLimits() { - if (limits != null) return limits; - return model == null ? null : model.limits(); - } } /** A finished run: its result, the warnings it gave in order, and the run, to take from. */ @@ -338,10 +308,6 @@ public static Finished run( start.put("message", "start"); start.put("contract", contract()); start.put("facts", HostFacts.now()); - Map limits = LimitSettings.given(call.effectiveLimits()); - if (!limits.isEmpty()) { - start.put("limits", limits); - } start.put("operation", operation); byte[] message = control(start); NativeCore.setLogLevel(NativeLog.level()); diff --git a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java index 2677af03e..6e79d1950 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java +++ b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java @@ -31,7 +31,7 @@ * Transport} ({@code sdm_client/core/transport.py}): it posts the core's call to {@code * /v1/predictions} as the bytes the core wrote, never parsing or re-encoding them (design choice * 10, "Model calls"), and gives back the endpoint's answer as the bytes it sent, which the core - * reads. A transient failure (a status of 429, 502, 503 or 504, or a connection that failed) is + * reads. A transient failure (a status of 429, 500, 502, 503 or 504, or a connection that failed) is * tried again with exponential backoff, a {@code Retry-After} the endpoint sends honoured up to a * minute; the timeout bounds each attempt whole, its answer's body included; a status of 400 or more is the endpoint's refusal ({@link ModelEndpointException}); an * endpoint that cannot be reached, or an answer past 64 MiB or that is not a JSON object, is a @@ -42,7 +42,7 @@ public final class HttpTransport { public static final String PREDICTIONS = "/v1/predictions"; /** The statuses tried again. */ - static final Set RETRY_STATUS = Set.of(408, 429, 502, 503, 504); + static final Set RETRY_STATUS = Set.of(408, 429, 500, 502, 503, 504); /** The largest answer read. */ static final int MAX_RESPONSE_BYTES = 64 * 1024 * 1024; @@ -216,7 +216,9 @@ else if (endpoint instanceof java.util.Map map) { backoff(attempt, null); continue; } - throw new Failed("the request timed out after " + timeout + ", the timeout given"); + // As the endpoint's 408, so the core asks the rows again in halves. + throw ModelEndpointException.of(408, ("the request timed out after " + timeout + + ", the timeout given").getBytes(java.nio.charset.StandardCharsets.UTF_8)); } catch (IOException e) { if (attempt < retries) { backoff(attempt, null); diff --git a/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java b/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java deleted file mode 100644 index b991a4257..000000000 --- a/java/src/main/java/com/nvidia/sdm/internal/LimitSettings.java +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.nvidia.sdm.internal; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * The size limits a run's start carries ({@code com.nvidia.sdm.Limits}): the values the caller set - * and the text of each setting's environment variable that is set, read when the run starts, which - * the core resolves by the one rule every host reads them with. The settings' names and variables - * are the core's ({@code sdm_contract::limits::SETTINGS}), read from the library's description. - */ -public final class LimitSettings { - private LimitSettings() {} - - /** What a run's start carries of {@code values}, the settings the caller set by name. */ - @SuppressWarnings("unchecked") - public static Map given(Map values) { - Map environment = new LinkedHashMap<>(); - for (Map setting : - (List>) NativeCore.load().get("limits")) { - String name = (String) setting.get("env"); - String text = System.getenv(name); - if (text != null) { - environment.put(name, text); - } - } - Map given = new LinkedHashMap<>(); - if (values != null && !values.isEmpty()) { - given.put("given", new LinkedHashMap<>(values)); - } - if (!environment.isEmpty()) { - given.put("environment", environment); - } - return given; - } -} diff --git a/java/src/main/java/com/nvidia/sdm/internal/Wording.java b/java/src/main/java/com/nvidia/sdm/internal/Wording.java index 0b3f69c7b..ecd7f2dcb 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Wording.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Wording.java @@ -118,9 +118,6 @@ private static Map, String>> advice() { spelled.put("nothing", given -> "null"); spelled.put("optimize_indices", given -> "an index on the key column"); spelled.put("predict_call", given -> "predict()"); - spelled.put( - "raise_limit", - given -> "Limits.builder()." + camel(plainText(given.get(0))) + "(...)"); spelled.put("raise_most_parts", given -> "a higher limit on the parts one question is asked as"); spelled.put("result_frame", given -> "PredictionResult.read()"); spelled.put( @@ -712,21 +709,6 @@ private static String plain(Object value) throws Unfilled { return value instanceof String text ? text : literal(value); } - /** A setting's name as its builder method spells it: {@code max_request_bytes} as {@code maxRequestBytes}. */ - static String camel(String name) { - StringBuilder spelled = new StringBuilder(name.length()); - boolean upper = false; - for (char c : name.toCharArray()) { - if (c == '_') { - upper = true; - } else { - spelled.append(upper ? Character.toUpperCase(c) : c); - upper = false; - } - } - return spelled.toString(); - } - private static String plainText(Object value) { try { return plain(value); diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index 66cb384e9..ac1fc4ed6 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -404,14 +404,13 @@ void utf8FeatureValuesUseTheSharedEnvelopeAndActualByteLimit() throws Exception client.predict(request, SdmClient.KUMO_RELATIONAL); byte[] body = workspace.bodies.get(0); assertTrue(new String(body, StandardCharsets.UTF_8).contains("é😀")); - // One byte under the call: it is split into smaller calls, each within the limit and - // each carrying the whole context, its rows answered as one. try (Workspace limited = new Workspace(); SdmClient capped = SdmClient.forDatabricksServing(limited.serving(body.length - 1))) { - assertEquals(client.predict(request, SdmClient.KUMO_RELATIONAL).rows(), - capped.predict(request, SdmClient.KUMO_RELATIONAL).rows()); - assertTrue(limited.bodies.size() > 1); - for (byte[] part : limited.bodies) assertTrue(part.length <= body.length - 1); + StructuredDataException error = assertThrows(StructuredDataException.class, + () -> capped.predict(request, SdmClient.KUMO_RELATIONAL)); + assertEquals("INVALID_REQUEST", error.code()); + assertEquals(body.length, ((Number) error.details().get("bytes")).intValue()); + assertTrue(limited.bodies.isEmpty()); } } } @@ -552,9 +551,7 @@ void aCallPastTheCallersLimitIsRefusedBeforeItIsSent() throws Exception { () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); assertEquals("INVALID_REQUEST", error.code()); assertEquals(100L, ((Number) error.details().get("limit")).longValue()); - assertTrue(error.getMessage().contains("above the 100 byte limit the endpoint accepts"), - error.getMessage()); - assertTrue(error.getMessage().contains("of them the context"), error.getMessage()); + assertTrue(error.getMessage().contains("above the 100 byte limit it accepts"), error.getMessage()); assertEquals(List.of(), workspace.bodies); } } diff --git a/java/src/test/java/com/nvidia/sdm/LimitsTest.java b/java/src/test/java/com/nvidia/sdm/LimitsTest.java deleted file mode 100644 index ffeff5f5e..000000000 --- a/java/src/test/java/com/nvidia/sdm/LimitsTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.nvidia.sdm; - -import static org.junit.jupiter.api.Assertions.*; - -import java.util.List; -import java.util.Map; -import java.util.stream.LongStream; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.table.Table; -import org.junit.jupiter.api.Test; - -/** The size limits a client holds requests to: a raised one taking effect. */ -class LimitsTest { - @Test void aSettingGivenIsKeptByNameAndMustBeAtLeastOne() { - Limits limits = Limits.builder().maxTabularContextRows(200_000).build(); - assertEquals(Map.of("max_tabular_context_rows", 200_000L), limits.values()); - assertEquals(Map.of(), Limits.defaults().values()); - StructuredDataException refused = assertThrows( - StructuredDataException.class, () -> Limits.builder().maxRequestBytes(0)); - assertEquals("INVALID_CONFIGURATION", refused.code()); - } - - @Test void aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised() throws Exception { - long[] ids = LongStream.range(0, 10_001).toArray(); - try (RootAllocator allocator = new RootAllocator(); - Table context = TabularTest.table(allocator, true, ids); - Table scoring = TabularTest.table(allocator, false, 20_000); - TabularTest.Endpoint endpoint = new TabularTest.Endpoint()) { - Request request = new Request(context, scoring, "y", "regression"); - try (SdmClient client = SdmClient.connect(endpoint.url())) { - StructuredDataException refused = assertThrows(StructuredDataException.class, - () -> client.predict(request, SdmClient.KUMO_TABULAR)); - assertTrue(refused.getMessage().contains("SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS"), - refused.getMessage()); - assertTrue(endpoint.calls.isEmpty()); - } - Limits raised = Limits.builder().maxTabularContextRows(20_000).build(); - try (SdmClient client = SdmClient.connect(endpoint.url(), - ClientOptions.builder().limits(raised).build())) { - assertEquals(raised, client.limits()); - assertEquals(1, client.predict(request, SdmClient.KUMO_TABULAR).rows()); - @SuppressWarnings("unchecked") Map sent = - (Map) endpoint.calls.get(0).get("context"); - @SuppressWarnings("unchecked") Map table = - (Map) sent.get("instance_table"); - assertEquals(10_001, ((List) table.get("rows")).size()); - } - } - } -} diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index 40ea19ac1..361605049 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -36,7 +36,6 @@ GradientExplanations, ) from sdm_client.inference import InferenceConfig -from sdm_client.limits import Limits from sdm_client.models import KumoRelational, KumoTabular, Model from sdm_client.query import PredictiveQuery from sdm_client.relational_task import RelationalTask @@ -82,7 +81,6 @@ 'FittedModel', 'KumoRelational', 'KumoTabular', - 'Limits', 'MissingExtraError', 'Model', 'ModelCapabilities', diff --git a/packages/structured-data-models-client/src/sdm_client/_native.pyi b/packages/structured-data-models-client/src/sdm_client/_native.pyi index 3fb994333..3a91a6fa6 100644 --- a/packages/structured-data-models-client/src/sdm_client/_native.pyi +++ b/packages/structured-data-models-client/src/sdm_client/_native.pyi @@ -70,10 +70,6 @@ MEANINGS: tuple[tuple[str, str], ...] #: Each data type name a host may give a column, and the contract's name #: of the type it names: the one table of them. DTYPE_NAMES: tuple[tuple[str, str], ...] -#: Each size limit a caller may set, ``(name, env, default)``: its name, -#: the environment variable a host reads it from and its default; the one -#: table of them every host reads. -LIMITS: tuple[tuple[str, str, int], ...] class _ArrowStreamExportable(Protocol): def __arrow_c_stream__( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py index 10098594a..8046723f0 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py @@ -154,7 +154,6 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name 'nothing': lambda: 'None', 'optimize_indices': lambda: 'KumoRelational(graph, optimize=True)', 'predict_call': lambda: 'predict()', - 'raise_limit': lambda setting: f'limits=Limits({setting}=...)', 'raise_most_parts': lambda: 'raise sdm_client.decompose.MOST_PARTS', 'semantic_model': lambda name: f'source.semantic_model({name!r})', 'set_primary_key': lambda table, column: ( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index c4feccaa3..4aa0ed787 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -606,6 +606,7 @@ def model_call(body: bytes) -> bytes: failure: StructuredDataError | None = None unexpected: Exception | None = None transport_failed = False + timed_out: StructuredDataError | None = None refused: NimRequestError | None = None try: # The core's call, its bytes as written (register PD-2), @@ -619,7 +620,13 @@ def model_call(body: bytes) -> bytes: except NimRequestError as refusal: refused = refusal except StructuredDataError as failed: - if failed.code == 'TRANSPORT_ERROR': + if failed.code == 'TRANSPORT_ERROR' and 'timeout' in failed.details: + # Told to the core as a 408, so it asks the rows again in + # halves; the timeout is what is raised if no half answers. + timed_out = StructuredDataError( + failed.message, code=failed.code, details=failed.details + ) + elif failed.code == 'TRANSPORT_ERROR': transport_failed = True else: from sdm_client.in_process.backend import InProcessBackend @@ -641,6 +648,8 @@ def model_call(body: bytes) -> bytes: unexpected = other if transport_failed: raise TransportFailed + if timed_out is not None: + raise Refused(timed_out, 408) raised: StructuredDataError | None = failure if refused is not None: # Built outside the handler, so nothing failing in the copy is diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index 062e1e38d..570bcb51e 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -48,7 +48,6 @@ # dependency, and a core route over frames needs none of it. from sdm_connectors.providers import Provider -from sdm_client import limits from sdm_client._rust import NativeBoundaryError, codec, errors, facts from sdm_client.core.transport import EndpointFailed from sdm_client.errors import StructuredDataError @@ -225,7 +224,6 @@ def run( 'message': 'start', 'contract': core.CONTRACT_VERSION, 'facts': facts.host_facts(core), - 'limits': limits.in_effect()._given(), 'operation': operation, } ) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py index 0604b73d8..8e37a6295 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/tabular_materialize.py @@ -216,27 +216,12 @@ def _context( observe, ) - # The most context rows a request carries (`max_tabular_context_rows`), - # the count compared with it natively before any frame length is read; - # past it, the core refuses, naming the setting. - limit = _call('check_tabular_cap', {'requested': None})['limit'] + # The context is sampled only to a count the caller asked for. if decision is None: - if wanted is not None and bool(wanted > limit): - decision = {'kind': 'refused'} - else: - count = int(wanted) if isinstance(wanted, np.integer) else wanted - decision = _call( - 'decide_tabular_context', - { - 'population': len(candidates), - 'requested': count, - }, - ) - if decision['kind'] == 'refused': - _call('check_tabular_cap', {'requested': str(int(wanted))}) - raise StructuredDataError( - 'the core refused a context count it then accepted', - code='INTERNAL_ERROR', + count = int(wanted) if isinstance(wanted, np.integer) else wanted + decision = _call( + 'decide_tabular_context', + {'population': len(candidates), 'requested': count}, ) if not decision['reduced']: return candidates diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index c9c7a91cc..64e2d41e9 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -37,8 +37,6 @@ StructuredDataError, UnknownModelError, ) -from sdm_client.limits import Limits, limits_in_effect -from sdm_client.limits import check as check_limits from sdm_client.models import MODELS, Model, spec_for from sdm_client.request import ( Question, @@ -191,15 +189,14 @@ def predict(self, request: Request) -> PredictionResult: f'question that many times. Ask it with predict()', code='UNSUPPORTED_REQUEST', ) - with limits_in_effect(self._client._limits): - if getattr(self, '_relational_session', None) is not None: - from sdm_client._rust import frame_sessions + if getattr(self, '_relational_session', None) is not None: + from sdm_client._rust import frame_sessions - return frame_sessions.predict(self, request) - if getattr(self, '_tabular_session', None) is not None: - from sdm_client._rust import sessions + return frame_sessions.predict(self, request) + if getattr(self, '_tabular_session', None) is not None: + from sdm_client._rust import sessions - return sessions.predict(self, request) + return sessions.predict(self, request) raise StructuredDataError( 'this fitted model has no native session state; fit again', code='INVALID_CONFIGURATION', @@ -274,7 +271,6 @@ def __init__( verify_ssl: bool = True, timeout: float = 60.0, max_retries: int = 3, - limits: Limits | None = None, ) -> None: r"""Open a client against one NIM. @@ -295,11 +291,6 @@ def __init__( the same on every attempt. ``POST /v1/sessions`` is excluded: a re-sent create would orphan a pinned context on the endpoint. - limits: The size limits requests to this endpoint are held to, - for a deployment configured to take more or less than the - serving layer's defaults. A setting left out takes its - environment variable, else its default; see - :class:`~sdm_client.Limits`. """ self._start( Transport( @@ -308,13 +299,10 @@ def __init__( verify_ssl=verify_ssl, timeout=timeout, max_retries=max_retries, - ), - limits, + ) ) - def _start( - self, transport: WireTransport, limits: Limits | None = None - ) -> None: + def _start(self, transport: WireTransport) -> None: r"""Set everything a client holds, given what carries its requests. The one place a client's state is set.:meth:`for_databricks_serving` @@ -323,27 +311,22 @@ def _start( new state here, not to ``__init__``. """ self._transport: WireTransport = transport - self._limits: Limits | None = check_limits(limits) self._adapters: dict[str, ModelAdapter] = {} self._pinned: set[FittedModel] = set() @classmethod - def in_process( - cls, *models: Any, device: Any = None, limits: Limits | None = None - ) -> SDMClient: + def in_process(cls, *models: Any, device: Any = None) -> SDMClient: """Run models locally using the same Rust preparation and result path. Pass loaded structured-data-models models, or omit them to load weights on first use. The optional ``in-process`` extra requires Python 3.11+. ``device`` controls models loaded by this client; supplied models keep - their placement. Local refusals carry no HTTP status. ``limits`` - holds requests to the sizes it names, as for an endpoint; see - :class:`~sdm_client.Limits`. + their placement. Local refusals carry no HTTP status. """ from sdm_client.in_process import InProcessBackend client = cls.__new__(cls) - client._start(InProcessBackend(models, device), limits) + client._start(InProcessBackend(models, device)) return client @classmethod @@ -353,7 +336,6 @@ def for_databricks_serving( *, workspace_client: Any | None = None, timeout: float | None = None, - limits: Limits | None = None, ) -> SDMClient: r"""Open a client against a Databricks Model Serving endpoint. @@ -385,10 +367,6 @@ def for_databricks_serving( timeouts. Given with a ``WorkspaceClient``, it is set on the copy, never on the caller's client; with a subclass or test double, which cannot be copied, it is refused. - limits: The size limits requests are held to, as for a NIM; see - :class:`~sdm_client.Limits`. The gateway's own 16 MB limit on - each call, once wrapped, holds as well: a call past either is - split into smaller calls where it scores several rows. Raises: MissingExtraError: If ``databricks-sdk`` is not installed; install @@ -407,17 +385,10 @@ def for_databricks_serving( client._start( DatabricksServingTransport( endpoint, workspace_client, timeout=timeout - ), - limits, + ) ) return client - @property - def limits(self) -> Limits: - r"""The size limits this client was opened with; a setting left - ``None`` takes its environment variable, else its default.""" - return self._limits or Limits() - @property def url(self) -> str: r"""Where this client sends requests. @@ -478,12 +449,11 @@ def predict( """ _require_a_request(request) _require_output_names(outputs) - with limits_in_effect(self._limits): - if request.parts is not None: - return self._answer_in_parts(request, model, outputs) - return _prediction_handler(request).predict( - self, request, model, outputs - ) + if request.parts is not None: + return self._answer_in_parts(request, model, outputs) + return _prediction_handler(request).predict( + self, request, model, outputs + ) def _answer_in_parts( self, @@ -605,14 +575,13 @@ def fit( _require_output_names(outputs) _refuse_to_pin_a_rewritten_question(request) spec = self._spec(model) - with limits_in_effect(self._limits): - if spec.model_id == 'kumo-tabular': - from sdm_client._rust import sessions + if spec.model_id == 'kumo-tabular': + from sdm_client._rust import sessions - return sessions.fit(self, request, model, outputs, holdout) - from sdm_client._rust import frame_sessions + return sessions.fit(self, request, model, outputs, holdout) + from sdm_client._rust import frame_sessions - return frame_sessions.fit(self, request, model, outputs, holdout) + return frame_sessions.fit(self, request, model, outputs, holdout) @staticmethod def _spec(model: str | Model) -> ModelSpec: diff --git a/packages/structured-data-models-client/src/sdm_client/core/transport.py b/packages/structured-data-models-client/src/sdm_client/core/transport.py index 0b1a13300..888c23b7d 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/transport.py +++ b/packages/structured-data-models-client/src/sdm_client/core/transport.py @@ -35,7 +35,7 @@ #: the same on every attempt, so a server fault reproduces, and the caller #: mistakes this endpoint reports as a 500 are reclassified from the body as #: the caller's after three more uploads of the whole gathered context. -_RETRY_STATUS = (408, 429, 502, 503, 504) +_RETRY_STATUS = (408, 429, 500, 502, 503, 504) _RETRY_AFTER_MAX_SECONDS = 60 _MAX_BODY_SNIPPET = 512 _LOCAL_HOSTS = frozenset({'localhost', '127.0.0.1', '::1'}) diff --git a/packages/structured-data-models-client/src/sdm_client/limits.py b/packages/structured-data-models-client/src/sdm_client/limits.py deleted file mode 100644 index b2d547f9d..000000000 --- a/packages/structured-data-models-client/src/sdm_client/limits.py +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""The size limits this client holds a request to before it is sent. - -Each mirrors a limit the serving layer enforces, so the client refuses or -batches first and says which limit, rather than sending a request the -endpoint refuses without saying how to meet it. A deployment sets its own -(the serving layer's ``TFM_MAX_TRAIN_ROWS``, ``TFM_MAX_TEST_ROWS`` and -``TFM_MAX_PAYLOAD_BYTES``), so each is a setting with the serving layer's -default. Raise one to match a deployment configured to take more: - ->>> limits = Limits(max_tabular_context_rows=200_000, -... max_request_bytes=512 * 1024 * 1024) ->>> client = SDMClient(url, limits=limits) ->>> request = query.materialize(indices=keys, limits=limits) - -or set its environment variable, ``SDM_CLIENT_`` and the setting's name in -capitals, such as ``SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS=200000``. A value -passed to :class:`Limits` comes before the environment's, which comes before -the default. The names, variables and defaults are the core's, the one table -every host reads, and the core resolves them. -""" - -from __future__ import annotations - -import contextvars -import os -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass, fields -from typing import Any - -from sdm_client.errors import StructuredDataError - - -@dataclass(frozen=True) -class Limits: - r"""The size limits a client holds requests to, each ``None`` to take - its environment variable, else its default. - - Attributes: - max_relational_context_rows: The most context rows a relational - request carries; ``num_context_samples`` asks for at most this - many. ``SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS``, 10,000. - max_relational_predict_rows: The most rows one relational call - scores; more are asked in several calls. - ``SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS``, 10,000. - max_temporal_link_predict_rows: The most rows one temporal link - prediction call scores. - ``SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS``, 200. - max_tabular_context_rows: The most context rows a single-table - request carries; ``materialize()`` samples down to it. - ``SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS``, 10,000. - max_request_bytes: The most bytes one call's body takes. A call - past it that scores several rows is split into smaller calls; one - whose context alone is past it is refused. - ``SDM_CLIENT_MAX_REQUEST_BYTES``, 31,457,280 (30 MiB). A - Databricks serving endpoint also holds each call to its own - ``max_request_bytes``. - - Raises: - StructuredDataError: With code ``INVALID_CONFIGURATION``, if a value - is not a whole number of at least 1. - """ - - max_relational_context_rows: int | None = None - max_relational_predict_rows: int | None = None - max_temporal_link_predict_rows: int | None = None - max_tabular_context_rows: int | None = None - max_request_bytes: int | None = None - - def __post_init__(self) -> None: - for field in fields(self): - value = getattr(self, field.name) - if value is None: - continue - if isinstance(value, bool) or not isinstance(value, int): - raise StructuredDataError( - f'{field.name} takes a whole number of at least 1, ' - f'got {type(value).__name__} {value!r}', - code='INVALID_CONFIGURATION', - ) - if value < 1: - raise StructuredDataError( - f'{field.name} is {value:,}, and it takes a whole number ' - f'of at least 1', - code='INVALID_CONFIGURATION', - ) - - def _given(self) -> dict[str, Any]: - r"""What a run's start carries of these limits: the values set here, - and the text of each setting's environment variable that is set, - read now, which the core resolves (contract ``GivenLimits``).""" - from sdm_client._rust.run import native - - given = { - field.name: getattr(self, field.name) - for field in fields(self) - if getattr(self, field.name) is not None - } - environment = { - env: os.environ[env] - for _, env, _ in native().LIMITS - if env in os.environ - } - start: dict[str, Any] = {} - if given: - start['given'] = given - if environment: - start['environment'] = environment - return start - - -def defaults() -> dict[str, int]: - r"""Each setting's default, by name: the core's one table of them.""" - from sdm_client._rust.run import native - - return {name: default for name, _, default in native().LIMITS} - - -_IN_EFFECT: contextvars.ContextVar[Limits | None] = contextvars.ContextVar( - 'sdm_client_limits', default=None -) - - -@contextmanager -def limits_in_effect(limits: Limits | None) -> Iterator[None]: - r"""Hold every run started inside to ``limits``; ``None`` keeps the ones - already in effect. A client's calls and ``materialize(limits=...)`` set - them for their own runs only.""" - if limits is None: - yield - return - token = _IN_EFFECT.set(limits) - try: - yield - finally: - _IN_EFFECT.reset(token) - - -def in_effect() -> Limits: - r"""The limits a run started now is held to: those set by the call it is - part of, else the environment's and the defaults.""" - return _IN_EFFECT.get() or Limits() - - -def check(limits: Any) -> Limits | None: - r"""``limits`` as an argument takes it: a :class:`Limits`, or ``None``.""" - if limits is None or isinstance(limits, Limits): - return limits - raise StructuredDataError( - f'limits takes an sdm_client.Limits, got {type(limits).__name__}', - code='INVALID_CONFIGURATION', - ) diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index b6be3bd55..eb48a78b4 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -37,14 +37,9 @@ from sdm_client.base import ModelSpec, RuntimeContract from sdm_client.errors import StructuredDataError, UnknownModelError -# Fixed properties of the relational model. The sizes a deployment sets (the -# most context rows, rows one call scores and bytes one call takes) are not -# here: they are settings, the core's one table of them, which a caller -# raises or lowers to match the deployment it sends to (`sdm_client.Limits`). -# -# `context_rows_fast` is the context gathered when `num_context_samples` is -# not given, a default rather than a cap. The relational model reads as many -# tables and hops as a request carries, so neither is limited here. +# Fixed properties of the relational model. The client holds a request to no +# size of its own: it sends its best attempt, and a call the endpoint refuses +# as too large, or times out on, is asked again in halves. RELATIONAL_LIMITS = { 'context_rows_fast': 1_000, 'classes_max': 10, diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index ad428144b..d3e97a2c8 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -41,8 +41,6 @@ said_as_parts_are, ) from sdm_client.errors import MissingExtraError, StructuredDataError -from sdm_client.limits import Limits, limits_in_effect -from sdm_client.limits import check as check_limits from sdm_client.materializers import RelationalMaterializer from sdm_client.models import RELATIONAL_LIMITS, Model, spec_for from sdm_client.request import Request @@ -149,7 +147,6 @@ def materialize( model: str | Model | None = None, candidates: Sequence[Any] | None = None, task: str | None = None, - limits: Limits | None = None, **options: Any, ) -> Request: r"""Build a request with the data needed to answer this query. @@ -199,14 +196,9 @@ def materialize( ``rows_per_call``: how many entities one call scores, which defaults to what the endpoint takes, 10,000 or 200 for a temporal link prediction, and may only be lowered. Lower it - for a deployment holding to less than this client knows of. A - call past the most bytes one takes is asked as smaller calls - when it is predicted. - limits: The size limits the request is gathered for: the most - context rows, rows one call scores and bytes one call takes. - Pass the :attr:`~sdm_client.SDMClient.limits` of the client it - will be sent with. A setting left out takes its environment - variable, else its default; see :class:`~sdm_client.Limits`. + for a graph wide enough that a full batch outruns the 30 MiB + body, or for a deployment holding to less than this client + knows of. Returns: A:class:`~sdm_client.Request` carrying the data. @@ -218,29 +210,6 @@ def materialize( given, if both are, if an option is not one this accepts, or if ``model`` cannot answer a schema of this shape. """ - with limits_in_effect(check_limits(limits)): - return self._materialize( - indices=indices, - predict=predict, - num_context_samples=num_context_samples, - model=model, - candidates=candidates, - task=task, - **options, - ) - - def _materialize( - self, - *, - indices: Sequence[Any] | None, - predict: pd.DataFrame | None, - num_context_samples: int | None, - model: str | Model | None, - candidates: Sequence[Any] | None, - task: str | None, - **options: Any, - ) -> Request: - r""":meth:`materialize`, with the limits it was given in effect.""" spec = spec_for(model) if model is not None else None in_place = self.schema.source is not None relational = ( diff --git a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py index 94cd242d1..85baf1abd 100644 --- a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py +++ b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py @@ -34,10 +34,10 @@ class StopWarning(Exception): def observe_case(name: str) -> dict[str, Any]: from sdm_client import KumoTabular, PredictiveQuery, SDMClient - from sdm_client.limits import defaults + from sdm_client.models import TABULAR_LIMITS assert name in CASES - cap = defaults()['max_tabular_context_rows'] + cap = TABULAR_LIMITS['context_rows_max'] events: list[Any] = [] recorded: list[dict[str, str]] = [] outcome: dict[str, Any] = {} diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index c310e3f8d..641cc7926 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -97,10 +97,9 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( ): """A part is one call, which is what makes the fan-out countable. The refusal reads the count it was given, so it costs no gathering.""" - from sdm_client.limits import defaults + from sdm_client.models import RELATIONAL_LIMITS - defaults = defaults() - count = defaults['max_relational_predict_rows'] + 1 + count = RELATIONAL_LIMITS['predict_rows_max'] + 1 with pytest.raises(StructuredDataError) as refused: built(query, indices=[1] * count, **how) assert refused.value.code == 'UNSUPPORTED_REQUEST' diff --git a/packages/structured-data-models-client/tests/test_configurable_limits.py b/packages/structured-data-models-client/tests/test_configurable_limits.py deleted file mode 100644 index 7ea4a2bc6..000000000 --- a/packages/structured-data-models-client/tests/test_configurable_limits.py +++ /dev/null @@ -1,219 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""The size limits a client holds requests to are settings. - -Each defaults to the serving layer's own, and a deployment configured to take -more is matched by raising it: with an argument, else its environment -variable. A call past the most bytes one takes is split rather than refused, -and only a context too large on its own is refused, naming the setting. -""" - -from __future__ import annotations - -import json -import warnings - -import numpy as np -import pandas as pd -import pytest -from test_limits_and_absent_outputs import gathered, query, table -from test_tabular_direct_core import Endpoint, client - -from sdm_client import KumoTabular, Limits, Request, StructuredDataError - -SETTINGS = ( - 'max_relational_context_rows', - 'max_relational_predict_rows', - 'max_temporal_link_predict_rows', - 'max_tabular_context_rows', - 'max_request_bytes', -) - - -@pytest.fixture(autouse=True) -def _no_environment(monkeypatch: pytest.MonkeyPatch) -> None: - for name in SETTINGS: - monkeypatch.delenv(f'SDM_CLIENT_{name.upper()}', raising=False) - - -@pytest.mark.parametrize('text', ['0', '-1', '2.5', 'lots', '1e6']) -def test_an_environment_variable_that_is_not_a_whole_number_is_refused( - monkeypatch: pytest.MonkeyPatch, text: str -) -> None: - monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', text) - with pytest.raises(StructuredDataError) as refused: - query(table(20)).materialize(indices=[0, 1]) - assert refused.value.code == 'INVALID_CONFIGURATION' - assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) - - -@pytest.mark.parametrize('value', [0, -5, True, 2.5, '10']) -def test_a_setting_given_that_is_not_a_whole_number_is_refused(value) -> None: - with pytest.raises(StructuredDataError) as refused: - Limits(max_request_bytes=value) - assert refused.value.code == 'INVALID_CONFIGURATION' - - -def _direct(context_rows: int) -> Request: - context = pd.DataFrame( - { - 'row_id': np.arange(context_rows), - 'feature': np.arange(context_rows) % 7, - 'target': np.arange(context_rows) % 2, - } - ) - scoring = pd.DataFrame({'row_id': [90, 91], 'feature': [1, 2]}) - return Request(context, scoring, target='target', task='classification') - - -def test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised() -> ( - None -): - request = _direct(200_000) - endpoint = Endpoint() - with pytest.raises(StructuredDataError) as refused: - client(endpoint).predict(request, model=KumoTabular()) - assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) - assert endpoint.calls == [] - - raised = client(endpoint) - raised._limits = Limits(max_tabular_context_rows=200_000) - raised.predict(request, model=KumoTabular()) - (sent,) = endpoint.calls - assert len(sent['context']['instance_table']['rows']) == 200_000 - - -def test_materialize_samples_to_a_raised_tabular_limit( - monkeypatch: pytest.MonkeyPatch, -) -> None: - schema = table(12_000) - with pytest.raises(StructuredDataError): - query(schema).materialize(indices=[0, 1], num_context_samples=11_000) - monkeypatch.setenv('SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS', '20_000') - assert ( - query(schema) - .materialize(indices=[0, 1], num_context_samples=11_000) - .context_rows - == 11_000 - ) - # An argument comes before the environment variable. - with pytest.raises(StructuredDataError) as refused: - query(schema).materialize( - indices=[0, 1], - num_context_samples=11_000, - limits=Limits(max_tabular_context_rows=10_500), - ) - assert 'at most 10,500 context rows' in str(refused.value) - assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(refused.value) - - -def test_a_raised_relational_predict_limit_and_a_lowered_one_set_the_calls() -> ( - None -): - from sdm_client import KumoRelational, PredictiveQuery, Schema - - frame = pd.DataFrame( - {'uid': range(40), 'label': [*range(30), *[None] * 10]} - ) - schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) - schema['users']['label'].stype = 'numerical' - asked = PredictiveQuery('PREDICT users.label FOR EACH users.uid', schema) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - lowered = asked.materialize( - indices=list(range(30, 40)), - model=KumoRelational(), - limits=Limits(max_relational_predict_rows=4), - ) - assert [ - len(p['predict']['instance_table']['rows']) for p in lowered.payloads - ] == [ - 4, - 4, - 2, - ] - - -def _relational(predict_rows: int): - return gathered(context_rows=30, run_mode='fast', predict_rows=predict_rows) - - -def _sent(request, limits: Limits) -> list[dict]: - class RelationalEndpoint(Endpoint): - def predict_body(self, body): - payload = json.loads(body) - self.calls.append(payload) - self.sizes.append(len(body)) - table = payload['predict']['instance_table'] - key = table['columns'].index('instance_id') - return json.dumps( - { - 'id': 'answer', - 'model': 'kumo-relational', - 'metadata': {}, - 'predictions': [ - {'id': str(row[key]), 'row_index': i, 'prediction': 1.0} - for i, row in enumerate(table['rows']) - ], - } - ).encode() - - endpoint = RelationalEndpoint() - endpoint.sizes = [] - sending = client(endpoint) - sending._limits = limits - result = sending.predict(request, model='kumo-relational') - assert list(result.to_pandas()['row_index']) == list(range(8)) - return endpoint - - -def test_a_call_past_the_byte_limit_is_split_and_each_part_fits() -> None: - request = _relational(8) - whole = _sent(request, Limits()) - (size,) = whole.sizes - split = _sent(request, Limits(max_request_bytes=size - 1)) - assert len(split.calls) > 1 - assert all(sent <= size - 1 for sent in split.sizes) - assert ( - sum(len(c['predict']['instance_table']['rows']) for c in split.calls) - == 8 - ) - # Every call carries the whole context. - contexts = {json.dumps(c['context'], sort_keys=True) for c in split.calls} - assert len(contexts) == 1 - # At the limit exactly, one call. - assert len(_sent(request, Limits(max_request_bytes=size)).calls) == 1 - - -def test_a_context_too_large_on_its_own_is_refused_naming_the_setting() -> None: - request = _relational(8) - with pytest.raises(StructuredDataError) as refused: - _sent(request, Limits(max_request_bytes=1_000)) - said = str(refused.value) - assert 'of them the context' in said - assert 'SDM_CLIENT_MAX_REQUEST_BYTES' in said - assert 'max_request_bytes' in said - - -def test_more_than_six_hops_and_fifteen_tables_are_taken() -> None: - from test_query_engine_boundary import gathered_from, star - - from sdm_client import KumoRelational - - assert len(gathered_from(star(satellites=20, joining=20))) == 21 - request = _relational(1) - assert request.payloads - frame = pd.DataFrame({'uid': range(12), 'label': [*range(10), None, None]}) - from sdm_client import PredictiveQuery, Schema - - schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) - schema['users']['label'].stype = 'numerical' - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - for options in ({'num_hops': 8}, {'num_neighbors': [4] * 9}): - built = PredictiveQuery( - 'PREDICT users.label FOR EACH users.uid', schema - ).materialize(indices=[10, 11], model=KumoRelational(), **options) - assert built.payloads diff --git a/packages/structured-data-models-client/tests/test_context_samples.py b/packages/structured-data-models-client/tests/test_context_samples.py index 0a04fed8d..d41d87c62 100644 --- a/packages/structured-data-models-client/tests/test_context_samples.py +++ b/packages/structured-data-models-client/tests/test_context_samples.py @@ -21,12 +21,11 @@ from test_materialize_options import _materialize from sdm_client.errors import StructuredDataError -from sdm_client.limits import defaults from sdm_client.materializers.relational import RelationalMaterializer from sdm_client.models import RELATIONAL_LIMITS FAST = RELATIONAL_LIMITS['context_rows_fast'] -MAX = defaults()['max_relational_context_rows'] +MAX = RELATIONAL_LIMITS['context_rows_max'] # --- which value goes on the wire ----------------------------------------- diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 6ff8c7663..33e837fe8 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -27,6 +27,7 @@ from sdm_client import PredictiveQuery, Schema from sdm_client.client import _report_absent from sdm_client.errors import StructuredDataError +from sdm_client.models import KUMO_TABULAR from sdm_client.result import PredictionResult @@ -58,12 +59,16 @@ def query(schema: Schema) -> PredictiveQuery: # --- the context the endpoint will take ------------------------------------ +def test_the_declared_cap_is_the_one_the_endpoint_enforces(): + """Measured at the boundary: 10,000 answer, 10,001 refuse.""" + assert KUMO_TABULAR.runtime.limits['context_rows_max'] == 10_000 + + def test_a_context_above_the_cap_is_refused_before_it_is_encoded(): schema = table(200) with pytest.raises(StructuredDataError) as raised: query(schema).materialize(indices=[0, 1], num_context_samples=50_000) - assert 'carries at most 10,000 context rows' in str(raised.value) - assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(raised.value) + assert 'accepts at most 10,000 context rows' in str(raised.value) def test_the_cap_itself_is_allowed(): @@ -221,8 +226,7 @@ def test_a_materialized_context_above_the_cap_is_refused_not_merely_the_number() schema = table(12_000) with pytest.raises(StructuredDataError) as raised: query(schema).materialize(indices=[0, 1], num_context_samples=10_001) - assert 'carries at most 10,000 context rows' in str(raised.value) - assert 'SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS' in str(raised.value) + assert 'accepts at most 10,000 context rows' in str(raised.value) def test_the_predict_route_advises_something_that_route_allows(): diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 3f3d0d084..a3085a0f8 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -57,11 +57,10 @@ def _Gathered(*payloads): def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): """A pin, not a probe. - These numbers were read from the relational NIM, as the defaults of the - settings a deployment may raise. This asserts they have not drifted in - the client, which is all it can do offline. The live suites check that - the endpoint still agrees. The sizes a deployment sets are settings, - whose defaults the core pins; no most is set on tables or hops. + These numbers were read from the relational NIM. This asserts they have + not drifted in the client, which is all it can do offline. The client + holds a request to no size of its own, so no most is set on context, + rows scored, bytes, tables or hops. """ assert RELATIONAL_LIMITS == {'context_rows_fast': 1_000, 'classes_max': 10} diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index 20e469f07..deccb4296 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -193,9 +193,9 @@ def test_the_endpoint_scores_what_this_client_says_it_does(client, query): """ import copy - from sdm_client.limits import defaults + from sdm_client.models import RELATIONAL_LIMITS - cap = defaults()['max_relational_predict_rows'] + cap = RELATIONAL_LIMITS['predict_rows_max'] payload = copy.deepcopy(query.materialize(indices=[0, 1]).payloads[0]) rows = payload['predict']['instance_table']['rows'] diff --git a/packages/structured-data-models-client/tests/test_payload_rules.py b/packages/structured-data-models-client/tests/test_payload_rules.py index efb815def..3b8d97d34 100644 --- a/packages/structured-data-models-client/tests/test_payload_rules.py +++ b/packages/structured-data-models-client/tests/test_payload_rules.py @@ -195,10 +195,9 @@ def test_vectors_cover_required_success_and_failure_cases() -> None: # An earlier endpoint read at most 15 tables; the model reads as many as # a request carries, so the client sets no most. 'star.tables-16': 'no table limit', - # A call past the byte limit that scores several rows is split into - # smaller calls when it is predicted; only a context too large on its - # own is refused at materialize. - 'size.one-past-the-limit': 'split by bytes', + # The client holds a payload to no size of its own: a call the endpoint + # refuses as too large is asked again in halves when it is predicted. + 'size.one-past-the-limit': 'no byte limit', } diff --git a/packages/structured-data-models-client/tests/test_predict_result.py b/packages/structured-data-models-client/tests/test_predict_result.py index 6a53d1c76..3cbdfc196 100644 --- a/packages/structured-data-models-client/tests/test_predict_result.py +++ b/packages/structured-data-models-client/tests/test_predict_result.py @@ -285,12 +285,14 @@ def test_the_core_s_predict_is_the_reference_s( ) return if spec['covers'] == R.SIZE: - # The reference refused a payload of two rows past 30MB; this client - # gathers it, since its context fits with one row, and holds each - # call it sends to the limit, split where it is past it, by design. - assert expected['outcome'] == 'refused' + # The reference's code, and a message that explains the limit as + # the old route's does (it names 30MB). + assert got['outcome'] == 'refused', 'past the limit, taken' + assert got['code'] == expected['code'], got['message'] assert '30MB' in expected['message'] - assert got['outcome'] == 'result', got.get('message') + assert re.search(r'30 ?MB|30 ?MiB|31,?457,?280', got['message']), ( + f'the size refusal does not explain the limit: {got["message"]!r}' + ) return if spec['name'] in { 'prepare.regression.outputs-explanation-two-rows', diff --git a/packages/structured-data-models-client/tests/test_serving_route.py b/packages/structured-data-models-client/tests/test_serving_route.py index 39257d628..a8bc2287f 100644 --- a/packages/structured-data-models-client/tests/test_serving_route.py +++ b/packages/structured-data-models-client/tests/test_serving_route.py @@ -395,29 +395,16 @@ def test_malformed_serving_response_is_a_transport_error( assert len(received) == 1 -def test_a_call_past_the_caller_s_limit_is_split_and_a_context_past_it_refused() -> ( - None -): +def test_a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() -> None: request = _materialized() _, received = _predict(request) [(_, body, _)] = received - # The limit applies to the actual UTF-8 envelope posted to the SDK: one - # byte under the call, its two rows are asked as two calls, each with - # the whole context and within the limit. - split, on_core = _predict(request, max_request_bytes=len(body) - 1) - assert not isinstance(split, Exception), split - assert len(on_core) == 2 - assert all(len(sent) <= len(body) - 1 for _, sent, _ in on_core) - assert {json.dumps(_call(sent)['context']) for _, sent, _ in on_core} == { - json.dumps(_call(body)['context']) - } - # A limit no call of one row fits: refused, naming the context's size, - # and nothing sent. - core, on_core = _predict(request, max_request_bytes=100) + # The limit applies to the actual UTF-8 envelope posted to the SDK. + core, on_core = _predict(request, max_request_bytes=len(body) - 1) assert on_core == [] assert isinstance(core, StructuredDataError), core - assert core.details['limit'] == 100 - assert core.details['context_bytes'] < core.details['bytes'] < len(body) + assert core.details['limit'] == len(body) - 1 + assert core.details['bytes'] == len(body) # The size itself is admitted. core, received = _predict(request, max_request_bytes=len(body)) assert not isinstance(core, Exception), core @@ -439,11 +426,11 @@ def test_utf8_serving_preserves_public_payload_and_measures_the_sent_body() -> ( assert call[field] == request.payloads[0][field], field assert result.to_pandas()['id'].tolist() == [0, 1] assert result.to_pandas()['prediction'].tolist() == [1.0, 1.0] - split, sent = _predict(request, max_request_bytes=len(body) - 1) - assert not isinstance(split, Exception), split - assert len(sent) == 2 - assert all('é😀'.encode() in part for _, part, _ in sent) - assert split.to_pandas()['id'].tolist() == [0, 1] + refused, sent = _predict(request, max_request_bytes=len(body) - 1) + assert isinstance(refused, StructuredDataError) + assert sent == [] + assert refused.details['bytes'] == len(body) + assert refused.details['limit'] == len(body) - 1 @pytest.mark.parametrize( diff --git a/packages/structured-data-models-client/tests/test_the_predict_route.py b/packages/structured-data-models-client/tests/test_the_predict_route.py index 95aa9fc7a..ad1bb939b 100644 --- a/packages/structured-data-models-client/tests/test_the_predict_route.py +++ b/packages/structured-data-models-client/tests/test_the_predict_route.py @@ -768,11 +768,14 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( # core's call as the core wrote it, so the bytes the endpoint receives # are the bytes the core counted against the limit, characters past # ASCII as their own bytes, never json.dumps' escapes. At the limit's - # edge the call is sent whole, at most the limit; one character past - # it, the call's two rows are asked as smaller calls, each within it. - # The payload, which holds the engine's inference config the call - # leaves out, may pass the limit where the call does not: materialize - # gathers it all the same, since a call of one row fits. + # edge the call is sent, at most the limit; one character past it, + # nothing is sent. Since the reference's 26ecdd28 the call + # leaves out the engine's inference config, which the payload holds, + # so the payload is the larger and meets the limit first: the edge is + # the payload's, which materialize names in its refusal, and one past + # it materialize refuses (the call's own refusal, past the limit with + # the payload within it, is sdm-core's + # a_call_past_the_limit_is_refused_before_it_is_sent). limit = 30 * 1024 * 1024 lengths = [] for emoji, ascii in ((0, 0), (1, 0), (0, 1)): @@ -789,23 +792,31 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( assert wide % 4 == 0 and narrow == 1 emoji = (limit - base) // wide ascii = (limit - base - emoji * wide) // narrow + # The call's edge is past the payload's: materialize refuses it, naming + # the payload's size, and the payload's edge is that much nearer. + with pytest.raises(StructuredDataError) as over: + _emoji_request(monkeypatch, emoji, ascii) + details = over.value.details + assert over.value.code == 'INVALID_REQUEST' + assert details['field'] == 'payload' and details['limit'] == limit + excess = details['bytes'] - limit + assert 0 < excess < wide * 64 + room = emoji * wide + ascii * narrow - excess + emoji = room // wide + ascii = (room - emoji * wide) // narrow written, received, refused = _sent( monkeypatch, _emoji_request(monkeypatch, emoji, ascii) ) assert refused.code == 'RECORDED' assert written == received - assert len(received[0]) == base + emoji * wide + ascii * narrow == limit + assert len(received[0]) == base + emoji * wide + ascii * narrow <= limit assert b'\\ud83d' not in received[0] assert '\U0001f600'.encode() in received[0] - # One byte past: split into calls of one row each, which share their - # context, so the first post pins that context as a session for them; - # the endpoint refuses it before anything else is sent. - _, received, refused = _sent( - monkeypatch, _emoji_request(monkeypatch, emoji, ascii + 1) - ) - assert refused.code == 'RECORDED' - [first] = received - assert json.loads(first)['metadata']['num_prediction_examples'] == 1 + with pytest.raises(StructuredDataError) as past: + _emoji_request(monkeypatch, emoji, ascii + 1) + assert past.value.code == 'INVALID_REQUEST' + assert past.value.details['field'] == 'payload' + assert past.value.details['bytes'] == limit + 1 def test_embeddings_are_the_old_route_s_on_the_core( diff --git a/packages/structured-data-models-client/tests/test_wording.py b/packages/structured-data-models-client/tests/test_wording.py index 9191dbb1f..12ac5eef7 100644 --- a/packages/structured-data-models-client/tests/test_wording.py +++ b/packages/structured-data-models-client/tests/test_wording.py @@ -522,10 +522,8 @@ def test_the_context_size_refused_is_named_as_the_caller_gave_it() -> None: with pytest.raises(StructuredDataError) as past: query.materialize(indices=[1], num_context_samples=10_001) assert str(past.value).split('] ', 1)[1] == ( - 'num_context_samples is 10,001, and a request carries at most 10,000 ' - 'context rows; if the endpoint takes more, raise it with ' - 'limits=Limits(max_relational_context_rows=...) or the environment ' - 'variable SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS' + 'num_context_samples is 10,001, and the endpoint accepts at most ' + '10,000 context rows' ) with pytest.raises(StructuredDataError) as below: query.materialize(indices=[1], num_context_samples=-(10**100)) diff --git a/rust/bindings/sdm-jni/src/lib.rs b/rust/bindings/sdm-jni/src/lib.rs index 7cb067fa8..cdda3cde4 100644 --- a/rust/bindings/sdm-jni/src/lib.rs +++ b/rust/bindings/sdm-jni/src/lib.rs @@ -456,16 +456,6 @@ pub extern "system" fn Java_com_nvidia_sdm_internal_NativeCore_info0<'caller>( "wire_names": sdm_host::wire_names() .into_iter() .collect::>(), - // Each size limit a caller may set, in order: its name, the - // environment variable a host reads it from, and its default. - "limits": sdm_contract::limits::SETTINGS - .iter() - .map(|setting| serde_json::json!({ - "name": setting.name, - "env": setting.env, - "default": setting.default, - })) - .collect::>(), }); let bytes = serde_json::to_vec(&info).map_err(|_| Failure::Usage("the library's description"))?; diff --git a/rust/bindings/sdm-py/src/lib.rs b/rust/bindings/sdm-py/src/lib.rs index 4cc9c5fa0..b1bfabb91 100644 --- a/rust/bindings/sdm-py/src/lib.rs +++ b/rust/bindings/sdm-py/src/lib.rs @@ -651,19 +651,6 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { .collect::>>()?, )?, )?; - // Each size limit a caller may set (`sdm_contract::limits::SETTINGS`): - // its name, the environment variable a host reads it from, and its - // default, the one table of them every host reads. - module.add( - "LIMITS", - PyTuple::new( - py, - sdm_contract::limits::SETTINGS - .iter() - .map(|setting| (setting.name, setting.env, setting.default)) - .collect::>(), - )?, - )?; // Each data type name a host may give a column, and the contract's name // of the type it names: the one table of them. module.add( diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index be5c113c5..44f475790 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:c4e98a97ae3920e095fbdd83b4d2df343e1107494366bed7e021210abf9a276e sdm-contract-72.json +72 sha256:d6ca0faa0176c2901562e9c734ebefee43d8d874c4b22276ea8e7ab260e507d6 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index 9a3c87363..e59e371ab 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -17,10 +17,6 @@ "facts": { "$ref": "#/$defs/HostFacts" }, - "limits": { - "description": "The size limits the caller set and the environment names, which the\ncore resolves before the run starts (contract v72); left out, each\nlimit is its default ([`crate::limits`]).", - "$ref": "#/$defs/GivenLimits" - }, "operation": { "$ref": "#/$defs/Operation" }, @@ -310,12 +306,6 @@ "DirectValue.select_tabular_rows": { "$ref": "#/$defs/SelectedTabularRows" }, - "CheckTabularCap.input": { - "$ref": "#/$defs/CheckTabularCap" - }, - "DirectValue.check_tabular_cap": { - "$ref": "#/$defs/CheckedTabularCap" - }, "ObserveTabularMembership.input": { "$ref": "#/$defs/ObserveTabularMembership" }, @@ -703,8 +693,6 @@ "tabular_arrow_session", "SelectTabularRows.input", "DirectValue.select_tabular_rows", - "CheckTabularCap.input", - "DirectValue.check_tabular_cap", "ObserveTabularMembership.input", "DirectValue.observe_tabular_membership", "DecideTabularTask.input", @@ -3810,34 +3798,6 @@ "above_population" ] }, - "CheckTabularCap": { - "description": "The context a materialized single-table request asks for, held to the\nmost context rows a request carries before frame length reads: the\n`num_context_samples` given, as its decimal text however large, or none.", - "type": "object", - "properties": { - "requested": { - "type": [ - "string", - "null" - ] - } - }, - "additionalProperties": false - }, - "CheckedTabularCap": { - "description": "The context asked for is within the limit, which the host compares its\nown count with, natively, before it reads any frame length; past it,\nthe run is refused.", - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "additionalProperties": false, - "required": [ - "limit" - ] - }, "ObserveTabularMembership": { "type": "object", "properties": { @@ -7961,7 +7921,7 @@ ] }, "GivenCandidate": { - "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\nits own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\nholds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\nfixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\ntext the reference writes into its question (in Python, `str` of it);\nit is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\nno scalar holds, written into its question by its text as `spelled`\nis, with the name of the kind of value it is (`date`, `datetime` for a\ntimestamp with no zone, `datetime_zoned` for one with one, `time`).\nTwo of one kind and one text are one candidate; a host whose language\nholds others equal (two zones of one instant) says so with `repeat`\n .\n- `repeat`: a value the host's own language holds equal to one given\nbefore it, left out as the reference's `dict.fromkeys` leaves it out.\nA host whose candidates are all scalars sends none, the core keeping\neach distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\none it cannot hash), which names no row and which the core refuses by\nname, as the reference fails on it.", + "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\nits own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\nholds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\nfixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\ntext the reference writes into its question (in Python, `str` of it);\nit is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\nno scalar holds, written into its question by its text as `spelled`\nis, with the name of the kind of value it is (`date`, `datetime` for a\ntimestamp with no zone, `datetime_zoned` for one with one, `time`).\nTwo of one kind and one text are one candidate; a host whose language\nholds others equal (two zones of one instant) says so with `repeat`\n.\n- `repeat`: a value the host's own language holds equal to one given\nbefore it, left out as the reference's `dict.fromkeys` leaves it out.\nA host whose candidates are all scalars sends none, the core keeping\neach distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\none it cannot hash), which names no row and which the core refuses by\nname, as the reference fails on it.", "oneOf": [ { "type": "string", @@ -12700,29 +12660,6 @@ "max_ns" ] }, - "GivenLimits": { - "description": "What a host passes with every run (contract v72): the values its caller\nset, by [`Setting::name`], and the text of each setting's environment\nvariable that is set, by [`Setting::env`], each as the host read it.\nLeft out, every limit is its default.", - "type": "object", - "properties": { - "given": { - "description": "The values the caller set, each a whole number of at least one.", - "type": "object", - "additionalProperties": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "environment": { - "description": "The environment variables set, each as its text.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "additionalProperties": false - }, "Operation": { "description": "One operation's request.", "oneOf": [ @@ -13719,24 +13656,6 @@ "input" ] }, - { - "description": "An input-sized direct value, carried as typed JSON bytes.", - "type": "object", - "properties": { - "input": { - "$ref": "#/$defs/AttachmentId" - }, - "op": { - "type": "string", - "const": "check_tabular_cap" - } - }, - "additionalProperties": false, - "required": [ - "op", - "input" - ] - }, { "description": "An input-sized direct value, carried as typed JSON bytes.", "type": "object", @@ -17486,7 +17405,6 @@ "direct_scored", "direct_target_missing", "direct_target_scored", - "direct_context_limit", "tabular_missing_row_index", "direct_multilabel", "direct_relational_frames", @@ -17693,7 +17611,6 @@ "model_lacks_shape", "model_lacks_task", "model_lacks_outputs", - "context_past_model", "no_context_rows", "too_many_classes", "one_class_column", @@ -17712,8 +17629,6 @@ "explanation_fast_mode", "options_unread", "options_unread_statistic", - "payload_no_smaller_call", - "call_past_endpoint", "outputs_none", "not_finite", "past_int64", @@ -17724,7 +17639,6 @@ "lag_past_times", "num_context_samples_not_whole", "num_context_samples_below_one", - "num_context_samples_past_most", "num_hops_not_whole", "num_hops_none", "random_seed_not_whole", @@ -17787,7 +17701,6 @@ "entity_table_empty", "entity_key_missing", "target_not_a_feature", - "context_past_rows", "held_by_several_rows_one", "held_by_several_rows", "held_by_several_rows_many", @@ -17855,7 +17768,6 @@ "large_text_materialize", "admitted_past_type", "invalid_configuration", - "limit_environment_invalid", "part_forecast_step", "part_forecast_step_outran", "part_ranking_candidate", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index e31dac620..bfba5b9e8 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -787,7 +787,6 @@ sentences! { DirectScored = "direct_scored", E::InvalidRequest; DirectTargetMissing = "direct_target_missing", E::InvalidRequest; DirectTargetScored = "direct_target_scored", E::InvalidRequest; - DirectContextLimit = "direct_context_limit", E::UnsupportedRequest; TabularMissingRowIndex = "tabular_missing_row_index", E::InvalidResponse; DirectMultilabel = "direct_multilabel", E::UnsupportedRequest; DirectRelationalFrames = "direct_relational_frames", E::UnsupportedRequest; @@ -994,7 +993,6 @@ sentences! { ModelLacksShape = "model_lacks_shape", E::UnsupportedRequest; ModelLacksTask = "model_lacks_task", E::UnsupportedRequest; ModelLacksOutputs = "model_lacks_outputs", E::UnsupportedRequest; - ContextPastModel = "context_past_model", E::UnsupportedRequest; NoContextRows = "no_context_rows", E::UnsupportedRequest; TooManyClasses = "too_many_classes", E::UnsupportedRequest; OneClassColumn = "one_class_column", E::UnsupportedRequest; @@ -1013,8 +1011,6 @@ sentences! { ExplanationFastMode = "explanation_fast_mode", E::UnsupportedRequest; OptionsUnread = "options_unread", E::UnsupportedRequest; OptionsUnreadStatistic = "options_unread_statistic", E::UnsupportedRequest; - PayloadNoSmallerCall = "payload_no_smaller_call", E::InvalidRequest; - CallPastEndpoint = "call_past_endpoint", E::InvalidRequest; OutputsNone = "outputs_none", E::InvalidRequest; NotFinite = "not_finite", E::InvalidRequest; PastInt64 = "past_int64", E::InvalidRequest; @@ -1025,7 +1021,6 @@ sentences! { LagPastTimes = "lag_past_times", E::ValueOutOfRange; NumContextSamplesNotWhole = "num_context_samples_not_whole", E::InvalidRequest; NumContextSamplesBelowOne = "num_context_samples_below_one", E::InvalidRequest; - NumContextSamplesPastMost = "num_context_samples_past_most", E::UnsupportedRequest; NumHopsNotWhole = "num_hops_not_whole", E::InvalidRequest; NumHopsNone = "num_hops_none", E::InvalidRequest; RandomSeedNotWhole = "random_seed_not_whole", E::InvalidRequest; @@ -1088,7 +1083,6 @@ sentences! { EntityTableEmpty = "entity_table_empty", E::InvalidRequest; EntityKeyMissing = "entity_key_missing", E::InvalidRequest; TargetNotAFeature = "target_not_a_feature", E::InvalidRequest; - ContextPastRows = "context_past_rows", E::InvalidRequest; HeldBySeveralRowsOne = "held_by_several_rows_one", E::InvalidRequest; HeldBySeveralRows = "held_by_several_rows", E::InvalidRequest; HeldBySeveralRowsMany = "held_by_several_rows_many", E::InvalidRequest; @@ -1156,7 +1150,6 @@ sentences! { LargeTextMaterialize = "large_text_materialize", E::UnsupportedType; AdmittedPastType = "admitted_past_type", E::ValueOutOfRange; InvalidConfiguration = "invalid_configuration", E::InvalidConfiguration; - LimitEnvironmentInvalid = "limit_environment_invalid", E::InvalidConfiguration; PartForecastStep = "part_forecast_step", Code::Part; PartForecastStepOutran = "part_forecast_step_outran", Code::Part; PartRankingCandidate = "part_ranking_candidate", Code::Part; diff --git a/rust/crates/sdm-contract/src/lib.rs b/rust/crates/sdm-contract/src/lib.rs index 8151ff5b0..47893f2cb 100644 --- a/rust/crates/sdm-contract/src/lib.rs +++ b/rust/crates/sdm-contract/src/lib.rs @@ -27,7 +27,6 @@ pub mod examples; pub mod facts; pub mod frames; pub mod labels; -pub mod limits; pub mod ops; pub mod pql; pub mod rebind; diff --git a/rust/crates/sdm-contract/src/limits.rs b/rust/crates/sdm-contract/src/limits.rs deleted file mode 100644 index 40fef7d5d..000000000 --- a/rust/crates/sdm-contract/src/limits.rs +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! The size limits a client holds a request to before it is sent, each a -//! setting a caller may raise or lower to match the deployment it sends to. -//! -//! Each limit mirrors one the serving layer enforces, so the client refuses -//! or batches first and names the limit, rather than sending a request the -//! endpoint refuses without saying how to meet it. A deployment configures -//! its own (the serving layer's `TFM_MAX_TRAIN_ROWS`, `TFM_MAX_TEST_ROWS` -//! and `TFM_MAX_PAYLOAD_BYTES`), so these are settings with the serving -//! layer's defaults, not constants. -//! -//! [`SETTINGS`] is the one table of them: each setting's name, the -//! environment variable a host reads it from, and its default. A host -//! passes what its caller gave and what its environment holds with every -//! run ([`GivenLimits`]), reading nothing else; the core resolves them, -//! the caller's value before the environment's before the default, and -//! refuses a value that is not a whole number of at least one by name. - -use std::collections::BTreeMap; - -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::error::{CoreError, Detail}; - -/// One limit a caller may set. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Setting { - /// The setting's name, as a host's argument names it. - pub name: &'static str, - /// The environment variable a host reads it from: `SDM_CLIENT_` and the - /// name in capitals. - pub env: &'static str, - /// The value where neither the caller nor the environment names one. - pub default: u64, -} - -/// The most context rows `kumo-relational` reads: the serving layer's -/// `TFM_MAX_TRAIN_ROWS` default. -pub const MAX_RELATIONAL_CONTEXT_ROWS: Setting = Setting { - name: "max_relational_context_rows", - env: "SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS", - default: 10_000, -}; - -/// The most rows one `kumo-relational` call scores: the serving layer's -/// `TFM_MAX_TEST_ROWS` default. A request of more is asked in several calls. -pub const MAX_RELATIONAL_PREDICT_ROWS: Setting = Setting { - name: "max_relational_predict_rows", - env: "SDM_CLIENT_MAX_RELATIONAL_PREDICT_ROWS", - default: 10_000, -}; - -/// The most rows one temporal link prediction call scores. -pub const MAX_TEMPORAL_LINK_PREDICT_ROWS: Setting = Setting { - name: "max_temporal_link_predict_rows", - env: "SDM_CLIENT_MAX_TEMPORAL_LINK_PREDICT_ROWS", - default: 200, -}; - -/// The most context rows `kumo-tabular` reads: the serving layer's -/// `TFM_MAX_TRAIN_ROWS` default. -pub const MAX_TABULAR_CONTEXT_ROWS: Setting = Setting { - name: "max_tabular_context_rows", - env: "SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS", - default: 10_000, -}; - -/// The most bytes one call's body takes, as written and sent: the serving -/// layer's `TFM_MAX_PAYLOAD_BYTES` default, 30 MiB. A batch of calls past it -/// is asked as smaller calls; a context past it on its own is refused. -pub const MAX_REQUEST_BYTES: Setting = Setting { - name: "max_request_bytes", - env: "SDM_CLIENT_MAX_REQUEST_BYTES", - default: 30 * 1024 * 1024, -}; - -impl Setting { - /// `error`, naming this setting and its environment variable, which - /// its sentence tells the caller to raise (`setting`, `env`). - pub fn named(&self, error: CoreError) -> CoreError { - error - .with_field("setting", Detail::text(self.name)) - .with_field("env", Detail::text(self.env)) - } -} - -/// Every setting, in the order a host lists them. -pub const SETTINGS: [Setting; 5] = [ - MAX_RELATIONAL_CONTEXT_ROWS, - MAX_RELATIONAL_PREDICT_ROWS, - MAX_TEMPORAL_LINK_PREDICT_ROWS, - MAX_TABULAR_CONTEXT_ROWS, - MAX_REQUEST_BYTES, -]; - -/// What a host passes with every run (contract v72): the values its caller -/// set, by [`Setting::name`], and the text of each setting's environment -/// variable that is set, by [`Setting::env`], each as the host read it. -/// Left out, every limit is its default. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct GivenLimits { - /// The values the caller set, each a whole number of at least one. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub given: BTreeMap, - /// The environment variables set, each as its text. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub environment: BTreeMap, -} - -/// The limits a run holds a request to, resolved. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Limits { - pub max_relational_context_rows: u64, - pub max_relational_predict_rows: u64, - pub max_temporal_link_predict_rows: u64, - pub max_tabular_context_rows: u64, - pub max_request_bytes: u64, -} - -impl Default for Limits { - fn default() -> Self { - Limits { - max_relational_context_rows: MAX_RELATIONAL_CONTEXT_ROWS.default, - max_relational_predict_rows: MAX_RELATIONAL_PREDICT_ROWS.default, - max_temporal_link_predict_rows: MAX_TEMPORAL_LINK_PREDICT_ROWS.default, - max_tabular_context_rows: MAX_TABULAR_CONTEXT_ROWS.default, - max_request_bytes: MAX_REQUEST_BYTES.default, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn each_setting_has_its_own_name_variable_and_serving_layer_default() { - let mut names: Vec<&str> = SETTINGS.iter().map(|s| s.name).collect(); - names.sort_unstable(); - names.dedup(); - assert_eq!(names.len(), SETTINGS.len()); - for setting in SETTINGS { - assert_eq!( - setting.env, - format!("SDM_CLIENT_{}", setting.name.to_ascii_uppercase()) - ); - } - assert_eq!( - Limits::default(), - Limits { - max_relational_context_rows: 10_000, - max_relational_predict_rows: 10_000, - max_temporal_link_predict_rows: 200, - max_tabular_context_rows: 10_000, - max_request_bytes: 31_457_280, - } - ); - } -} diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 569afddb4..99cc93197 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -68,7 +68,6 @@ pub enum Operation { DecideTabularTask(crate::direct::DirectInput), OrderSampledTabularRows(crate::direct::DirectInput), SelectTabularRows(crate::direct::DirectInput), - CheckTabularCap(crate::direct::DirectInput), CheckDirectShape(crate::direct::DirectInput), CheckDirectCell(crate::direct::DirectInput), CheckColumn(crate::direct::DirectInput), @@ -160,7 +159,6 @@ impl Operation { Operation::DecideTabularTask(_) => "decide_tabular_task", Operation::OrderSampledTabularRows(_) => "order_sampled_tabular_rows", Operation::SelectTabularRows(_) => "select_tabular_rows", - Operation::CheckTabularCap(_) => "check_tabular_cap", Operation::CheckDirectShape(_) => "check_direct_shape", Operation::CheckDirectCell(_) => "check_direct_cell", Operation::CheckColumn(_) => "check_column", @@ -332,7 +330,6 @@ impl Operation { | Operation::DecideTabularTask(v) | Operation::OrderSampledTabularRows(v) | Operation::SelectTabularRows(v) - | Operation::CheckTabularCap(v) | Operation::CheckDirectShape(v) | Operation::CheckDirectCell(v) | Operation::FeatureImportance(v) @@ -1198,7 +1195,7 @@ impl Given { /// timestamp with no zone, `datetime_zoned` for one with one, `time`). /// Two of one kind and one text are one candidate; a host whose language /// holds others equal (two zones of one instant) says so with `repeat` -/// . +///. /// - `repeat`: a value the host's own language holds equal to one given /// before it, left out as the reference's `dict.fromkeys` leaves it out. /// A host whose candidates are all scalars sends none, the core keeping @@ -1944,10 +1941,6 @@ pub struct Attachments { pub select_tabular_rows_input: crate::tabular::SelectTabularRows, #[schemars(rename = "DirectValue.select_tabular_rows")] pub select_tabular_rows_output: crate::tabular::SelectedTabularRows, - #[schemars(rename = "CheckTabularCap.input")] - pub check_tabular_cap_input: crate::tabular::CheckTabularCap, - #[schemars(rename = "DirectValue.check_tabular_cap")] - pub check_tabular_cap_output: crate::tabular::CheckedTabularCap, #[schemars(rename = "ObserveTabularMembership.input")] pub observe_tabular_membership_input: crate::tabular::ObserveTabularMembership, #[schemars(rename = "DirectValue.observe_tabular_membership")] diff --git a/rust/crates/sdm-contract/src/run.rs b/rust/crates/sdm-contract/src/run.rs index e1874e8d8..1ca392fca 100644 --- a/rust/crates/sdm-contract/src/run.rs +++ b/rust/crates/sdm-contract/src/run.rs @@ -38,7 +38,6 @@ use serde::{Deserialize, Serialize}; use crate::data::{AttachmentId, DataRequest, FetchPlan, ProviderError}; use crate::error::{CoreError, CoreWarning, ErrorKind}; use crate::facts::HostFacts; -use crate::limits::GivenLimits; use crate::ops::{Operation, OperationResult}; /// Names one fetch or model call within a run, so its reply can be matched. @@ -195,18 +194,9 @@ pub struct StartRun { /// The contract version the host speaks; see `CONTRACT_VERSION`. pub contract: u32, pub facts: HostFacts, - /// The size limits the caller set and the environment names, which the - /// core resolves before the run starts (contract v72); left out, each - /// limit is its default ([`crate::limits`]). - #[serde(default, skip_serializing_if = "is_unset")] - pub limits: GivenLimits, pub operation: Operation, } -fn is_unset(limits: &GivenLimits) -> bool { - limits == &GivenLimits::default() -} - /// Every control message that crosses between a host and the core. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(tag = "message", rename_all = "snake_case", deny_unknown_fields)] diff --git a/rust/crates/sdm-contract/src/tabular.rs b/rust/crates/sdm-contract/src/tabular.rs index 24f473c0b..1babdc910 100644 --- a/rust/crates/sdm-contract/src/tabular.rs +++ b/rust/crates/sdm-contract/src/tabular.rs @@ -472,24 +472,6 @@ pub enum TabularOrderError { TemporalYearOutOfRange, } -/// The context a materialized single-table request asks for, held to the -/// most context rows a request carries before frame length reads: the -/// `num_context_samples` given, as its decimal text however large, or none. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct CheckTabularCap { - pub requested: Option, -} - -/// The context asked for is within the limit, which the host compares its -/// own count with, natively, before it reads any frame length; past it, -/// the run is refused. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct CheckedTabularCap { - pub limit: u64, -} - /// One ordinary selection composition. Only proven-passive native producers /// may supply the complete population before earlier semantic guards finish. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 77521a45e..a3b2000af 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -209,7 +209,6 @@ advice! { Nothing = "nothing", []; OptimizeIndices = "optimize_indices", []; PredictCall = "predict_call", []; - RaiseLimit = "raise_limit", ["setting"]; RaiseMostParts = "raise_most_parts", []; SemanticModel = "semantic_model", ["metric_view"]; ResultFrame = "result_frame", []; @@ -340,9 +339,6 @@ pub fn template(sentence: Sentence) -> Template { S::DirectTargetScored => said( r#"predict carries the target column {target!r}. Those are the rows whose answer is being asked for, so the column has to be absent; drop it with predict.drop(columns=[{target!r}])"#, ), - S::DirectContextLimit => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with materialize(), which keeps the context under the cap and says how; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), S::TabularMissingRowIndex => said( r#"kumo-tabular returned a response that does not match the contract: PredictionItem response is missing required field 'row_index'"#, ), @@ -745,7 +741,7 @@ pub fn template(sentence: Sentence) -> Template { r#"column {column!r} holds a value that {admitted_type}, the type it was described as, cannot hold"#, ), S::LagPastPayload => said( - r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes a call takes; ask for fewer steps, or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, + r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes the endpoint takes; ask for fewer steps"#, ), S::DateLiteralPacked => said( r#"{query!r} is not a query a relational model can answer: {literal} is not a valid instance of time"#, @@ -882,9 +878,6 @@ pub fn template(sentence: Sentence) -> Template { S::ModelLacksOutputs => said( r#"{model} does not produce {unsupported_outputs!r}; it produces {supported_outputs!r}"#, ), - S::ContextPastModel => said( - r#"this request carries {context_rows:,} context rows and {model} reads at most {context_rows_max:,}. A request built directly is sent as it was given, so choose the rows you want as context, or gather it with {advice:materialize_call()}, which keeps the context under the cap and says how; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), S::NoContextRows => said( r#"no context rows are left: every row that could be context is also being scored, and a row carrying the answer being asked for is not sent as context. Score fewer rows, so that the rest are there to learn from"#, ), @@ -936,12 +929,6 @@ pub fn template(sentence: Sentence) -> Template { S::OptionsUnreadStatistic => said( r#"{model} does not read inference option(s) {unsupported!r}; drop them from inference_config to send the request without them. The keys this client forwards are {forwarded!r}, and the endpoint decides for itself whether it serves the value given for one. A regression's statistic is chosen with {advice:choose_statistic()} instead: 'median', the default, or 'mean'"#, ), - S::PayloadNoSmallerCall => said( - r#"the context alone is too large to send: a call carrying it and one row to score is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. Calls past the limit are split into smaller ones, but the context is sent whole with every call, so what comes down is num_context_samples, num_neighbors or the columns the graph carries; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), - S::CallPastEndpoint => said( - r#"this call is {bytes:,} bytes, {context_bytes:,} of them the context, and a call takes at most {limit:,}. A relational call scoring several rows is split into smaller ones, but the context is sent whole with every call, and a single-table request or a pinned context is sent as one call; so carry a smaller context (num_context_samples, the neighbourhood sampled, or the tables and columns in the graph), or score fewer rows at a time; or, if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), S::OutputsNone => said(r#"outputs must name at least one field"#), S::NotFinite => said( r#"encountered a non-finite value ({value}); NIM requests must contain finite numbers or nulls"#, @@ -970,9 +957,6 @@ pub fn template(sentence: Sentence) -> Template { S::NumContextSamplesBelowOne => said( r#"num_context_samples must be at least 1; a model predicts from labelled examples, and {given!S} leaves it none"#, ), - S::NumContextSamplesPastMost => said( - r#"num_context_samples is {given!I:,}, and a request carries at most {limit:,} context rows; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), S::NumHopsNotWhole => said(r#"num_hops must be a whole number, got {given!N} {given!G}"#), S::NumHopsNone => said(r#"num_hops must be a whole number, got {advice:nothing()}"#), S::RandomSeedNotWhole => said(r#"random_seed must be a whole number"#), @@ -1138,9 +1122,6 @@ pub fn template(sentence: Sentence) -> Template { S::TargetNotAFeature => { said(r#"the column the query predicts is not one of the features of the entity table"#) } - S::ContextPastRows => said( - r#"the context holds {context_rows:,} rows, and a request carries at most {limit:,}; if the endpoint takes more, raise it with {advice:raise_limit(setting)} or the environment variable {env}"#, - ), S::HeldBySeveralRowsOne => said( r#"Cannot score {keys[0]!r}: more than one row of table '{table}' holds this value of '{key}', so which row is meant would be a guess. The source does not enforce the key; keep one row per value, or key the table on columns that identify a row."#, ), @@ -1330,9 +1311,6 @@ pub fn template(sentence: Sentence) -> Template { S::InvalidConfiguration => { said(r#"the configuration of this call is not one the core accepts"#) } - S::LimitEnvironmentInvalid => said( - r#"the environment variable {env} is {given!r}, and {setting} takes a whole number of at least 1"#, - ), S::PartForecastStep => { said(r#"This is forecast step {forecast_step} of {forecast_steps}."#) } @@ -1549,7 +1527,7 @@ pub fn template(sentence: Sentence) -> Template { said(r#"this request carries a value that cannot be sent as JSON: {reason}"#) } S::ServingPastLimit => said( - r#"this request is too large for serving endpoint {endpoint!r}: a call carrying it is {bytes:,} bytes once wrapped, {context_bytes:,} of them the context, above the {limit:,} byte limit the endpoint accepts. Calls past the limit are split into smaller ones where they score several rows, but the context is sent whole with every call, so gather a smaller context (num_context_samples, num_neighbors or the columns the graph carries), or score fewer rows at a time"#, + r#"this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller num_context_samples, or fewer entities to score"#, ), S::ServingNotJson => { said(r#"serving endpoint {endpoint!r} answered with a response that is not JSON"#) diff --git a/rust/crates/sdm-contract/tests/corpus/mod.rs b/rust/crates/sdm-contract/tests/corpus/mod.rs index 4c5ab3c32..79e7f9793 100644 --- a/rust/crates/sdm-contract/tests/corpus/mod.rs +++ b/rust/crates/sdm-contract/tests/corpus/mod.rs @@ -103,7 +103,6 @@ pub fn messages() -> Vec { facts.local_zone = Some(LocalZone::Posix("EST5EDT,M3.2.0,M11.1.0".into())); } ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts, operation, diff --git a/rust/crates/sdm-contract/tests/host_facts.rs b/rust/crates/sdm-contract/tests/host_facts.rs index cc6618189..13648accd 100644 --- a/rust/crates/sdm-contract/tests/host_facts.rs +++ b/rust/crates/sdm-contract/tests/host_facts.rs @@ -41,7 +41,6 @@ fn pandas_2() -> InstantRange { fn start(facts: HostFacts) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts, operation: Operation::ValidateQuery(ValidateQuery { diff --git a/rust/crates/sdm-contract/tests/provider_view_contract.rs b/rust/crates/sdm-contract/tests/provider_view_contract.rs index 4d077465c..708f44fb1 100644 --- a/rust/crates/sdm-contract/tests/provider_view_contract.rs +++ b/rust/crates/sdm-contract/tests/provider_view_contract.rs @@ -781,12 +781,9 @@ fn view_errors_explain_trust_and_recovery() { ); assert_eq!( said(Sentence::ServingPastLimit), - "this request is too large for serving endpoint {endpoint!r}: a call carrying it is \ - {bytes:,} bytes once wrapped, {context_bytes:,} of them the context, above the {limit:,} \ - byte limit the endpoint accepts. Calls past the limit are split into smaller ones where \ - they score several rows, but the context is sent whole with every call, so gather a \ - smaller context (num_context_samples, num_neighbors or the columns the graph carries), \ - or score fewer rows at a time" + "this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above \ + the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller \ + num_context_samples, or fewer entities to score" ); // The way out of an untrusted expression is the host's to spell. assert_eq!( diff --git a/rust/crates/sdm-contract/tests/schema_lock.rs b/rust/crates/sdm-contract/tests/schema_lock.rs index bbe065843..4daaa96af 100644 --- a/rust/crates/sdm-contract/tests/schema_lock.rs +++ b/rust/crates/sdm-contract/tests/schema_lock.rs @@ -552,11 +552,6 @@ const JSON: &[(&str, &str)] = &[ "OperationResult/direct_value.value", "DirectValue.select_tabular_rows", ), - ("Operation/check_tabular_cap.input", "CheckTabularCap.input"), - ( - "OperationResult/direct_value.value", - "DirectValue.check_tabular_cap", - ), ( "Operation/observe_tabular_membership.input", "ObserveTabularMembership.input", diff --git a/rust/crates/sdm-core/src/direct.rs b/rust/crates/sdm-core/src/direct.rs index 899e14758..e7ab051b0 100644 --- a/rust/crates/sdm-core/src/direct.rs +++ b/rust/crates/sdm-core/src/direct.rs @@ -715,7 +715,6 @@ pub(crate) async fn predict( asker: Asker, request: Predict, mut attached: Attachments, - limits: sdm_contract::limits::Limits, ) -> Result { if request.state.is_some() || request.held.is_some() { return Err(CoreError::protocol( @@ -824,14 +823,11 @@ pub(crate) async fn predict( None }; let analysis = crate::columns::expose(&asker, analysis.as_deref(), &bound.outputs)?; - // The context and the call held to the run's limits, as they are sent. - predict::check_direct_context(&bound.model, bound.facts.context_rows, &limits)?; if predict::nothing_to_score(&payload) { let mut part = tabular::response::empty(&bound.model, &payload)?; part.columns = analysis; return crate::predict::finish(&asker, part, bound.outputs); } - let bytes = predict::call_bytes(&payload, limits.max_request_bytes)?; if deferred_options { use sdm_contract::direct::native_response::{NativeDirectCall, PreparedDirectCall}; let payload = asker.attach(Attachment::Bytes(payload.to_bytes()))?; @@ -848,7 +844,8 @@ pub(crate) async fn predict( })); } let answer = - crate::predict::call_answer(&asker, bytes, request.model_endpoint.as_ref()).await?; + crate::predict::call_answer(&asker, payload.to_bytes(), request.model_endpoint.as_ref()) + .await?; let mut payload = payload; if let Some(analysis) = analysis { payload.set("client_column_analysis", Value::Text(analysis)); diff --git a/rust/crates/sdm-core/src/dispatch.rs b/rust/crates/sdm-core/src/dispatch.rs index 07296b66e..6f5f52a00 100644 --- a/rust/crates/sdm-core/src/dispatch.rs +++ b/rust/crates/sdm-core/src/dispatch.rs @@ -6,7 +6,6 @@ use std::sync::Arc; -use sdm_contract::limits::Limits; use sdm_contract::ops::{ GivenCandidate, HeldSchema, Operation, OperationResult, QuerySchema, REQUEST_STATE_VERSION, RequestState, SAMPLER_STATE_VERSION, SamplerMemo, SchemaEdit, SourceSpec, @@ -50,9 +49,6 @@ pub fn start( zones: Arc, ) -> Result { check_contract(start.contract)?; - // The size limits the run holds requests to, the caller's before the - // environment's before the defaults, resolved before anything runs. - let limits = crate::limits::resolve(&start.limits)?; // The clock is an instant in UTC, spelled as the contract spells it. if !matches!(&start.facts.now, Scalar::Timestamp { zone: Some(zone), .. } if zone == "UTC") { return Err(CoreError::new( @@ -72,7 +68,7 @@ pub fn start( crate::materialize::frames::labels(asker, request, attachments, facts, zones).await }), Operation::MaterializeExamples(request) => { - crate::materialize::examples::start(root, request, attachments, facts, limits)? + crate::materialize::examples::start(root, request, attachments, facts)? } Operation::RebindSource(request) => { crate::rebind::start(root, request, attachments, facts)? @@ -115,10 +111,10 @@ pub fn start( Run::new(move |asker| async move { crate::result::merge(&asker, request, attachments) }) } Operation::TabularArrow(request) => Run::new(move |asker| async move { - crate::tabular_arrow::run(asker, request, attachments, facts, zones, limits).await + crate::tabular_arrow::run(asker, request, attachments, facts, zones).await }), Operation::FrameSession(input) => Run::new(move |asker| async move { - crate::frame_session::run(asker, input, attachments, limits).await + crate::frame_session::run(asker, input, attachments).await }), Operation::DescribeFrameTables(input) => Run::new(move |asker| async move { crate::frame_schema::describe(&asker, input, attachments) @@ -127,8 +123,7 @@ pub fn start( crate::materialize::frames::prepare(&asker, input, attachments, &facts) }), Operation::MaterializeFrames(input) => Run::new(move |asker| async move { - crate::materialize::frames::materialize(asker, input, attachments, facts, zones, limits) - .await + crate::materialize::frames::materialize(asker, input, attachments, facts, zones).await }), Operation::CountTabularSessionOverlap(request) => Run::new(move |asker| async move { crate::session::count_overlap(&asker, request, attachments) @@ -137,7 +132,7 @@ pub fn start( crate::session::send(&asker, request, attachments).await }), Operation::PrepareTabularScore(request) => Run::new(move |asker| async move { - crate::session::prepare_score(&asker, request, attachments, &limits) + crate::session::prepare_score(&asker, request, attachments) }), Operation::CheckTabularSessionReply(request) => Run::new(move |asker| async move { crate::session::check_reply(&asker, request, attachments) @@ -288,7 +283,7 @@ pub fn start( let input: sdm_contract::tabular::DecideTabularContext = take_json(&mut attachments, request.input)?; Run::new(move |asker| async move { - crate::direct::value(&asker, &crate::tabular_selection::decide(input, &limits)) + crate::direct::value(&asker, &crate::tabular_selection::decide(input)) }) } Operation::SampleTabularRows(request) => { @@ -326,15 +321,7 @@ pub fn start( let input: sdm_contract::tabular::SelectTabularRows = take_json(&mut attachments, request.input)?; Run::new(move |asker| async move { - crate::direct::value(&asker, &crate::tabular_selection::select(input, &limits)?) - }) - } - Operation::CheckTabularCap(request) => { - let input: sdm_contract::tabular::CheckTabularCap = - take_json(&mut attachments, request.input)?; - Run::new(move |asker| async move { - let checked = sdm_request::tabular::check_materialized_cap(input, &limits)?; - crate::direct::value(&asker, &checked) + crate::direct::value(&asker, &crate::tabular_selection::select(input)?) }) } Operation::CheckDirectShape(request) => { @@ -359,7 +346,7 @@ pub fn start( &input.kind, input.outputs, ); - crate::direct::value(&asker, &sdm_request::direct::bind(input, &limits)?) + crate::direct::value(&asker, &sdm_request::direct::bind(input)?) }) } Operation::InspectDirectPair(request) => Run::new(move |asker| async move { @@ -487,7 +474,6 @@ pub fn start( operation, attachments, facts, - limits, Arc::clone(&zones), #[cfg(feature = "test-ops")] false, @@ -495,7 +481,7 @@ pub fn start( Operation::Predict(request) => { if request.direct.is_some() { return Ok(Run::new(move |asker| { - crate::direct::predict(asker, request, attachments, limits) + crate::direct::predict(asker, request, attachments) })); } // What predict asks: the state `materialize` answered, or a @@ -526,13 +512,12 @@ pub fn start( ranking: asked.ranking, batches: asked.batches, endpoint: request.model_endpoint, - limits, }; Run::new(move |asker| crate::predict::run(asker, inputs)) } #[cfg(feature = "test-ops")] Operation::Test(operation) => { - crate::test_ops::run(root, operation, &mut attachments, facts, limits, zones)? + crate::test_ops::run(root, operation, &mut attachments, facts, zones)? } }) } @@ -688,7 +673,6 @@ pub(crate) fn materialize( operation: Operation, mut attachments: Attachments, facts: HostFacts, - limits: Limits, zones: Arc, #[cfg(feature = "test-ops")] labels_only: bool, ) -> Result { @@ -798,7 +782,6 @@ pub(crate) fn materialize( most_entities: request.most_entities, most_calls: request.most_calls, facts, - limits, zones: Arc::clone(&zones), #[cfg(feature = "test-ops")] labels_only, diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 3da7df268..744261d3d 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -90,7 +90,6 @@ pub(crate) async fn run( asker: Asker, operation: FrameSessionOperation, mut attached: Attachments, - limits: sdm_contract::limits::Limits, ) -> Result { let (input_id, payload_ids, pinned_id, action) = match operation { FrameSessionOperation::Fit { input, payloads } => (input, payloads, None, "fit"), @@ -148,7 +147,6 @@ pub(crate) async fn run( &model, outputs.as_deref().unwrap_or(&[]), default_outputs, - &limits, )?; let query = question .as_ref() @@ -180,8 +178,6 @@ pub(crate) async fn run( let state_keys = row_keys(&body, "context", windowed)?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&body))?); crate::columns::expose(&asker, column_analysis.as_deref(), &fields)?; - // The pinned context is one call, which no split makes smaller. - predict::call_bytes(&body, limits.max_request_bytes)?; let answer = call(&asker, Endpoint::CreateSession, body).await?; let id = session_id(&answer)?; let state = FrameSessionState { @@ -274,7 +270,6 @@ pub(crate) async fn run( &state.model, &state.outputs, false, - &limits, )?; let scored = row_keys(&layout, "predict", state.windowed)?; shown += keys::overlap_count(&state.keys.rows, &scored.rows)?; @@ -294,11 +289,6 @@ pub(crate) async fn run( } let analysis = crate::columns::expose(&asker, state.column_analysis.as_deref(), &state.outputs)?; - // Every scoring call held to the most bytes a call takes before - // the first is sent. - for (body, _) in &plans { - predict::call_bytes(body, limits.max_request_bytes)?; - } let mut parts = Vec::new(); let mut start = 0; for (body, layout) in plans { diff --git a/rust/crates/sdm-core/src/lag.rs b/rust/crates/sdm-core/src/lag.rs index 792056b65..55812a9e5 100644 --- a/rust/crates/sdm-core/src/lag.rs +++ b/rust/crates/sdm-core/src/lag.rs @@ -34,12 +34,11 @@ //! than their own; the core puts each on its own row ( fix 1, a //! registered difference decided under ). -use sdm_contract::limits::MAX_REQUEST_BYTES; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_engine::labels::{self, Labels, Target}; use sdm_engine::sample::out_of_range; use sdm_engine::time::offset::Offset; -use sdm_request::payload::{LagColumn, LagValues}; +use sdm_request::payload::{LagColumn, LagValues, PAYLOAD_BYTES}; /// The field a lag's refusal names. const FIELD: &str = "lag_timesteps"; @@ -145,9 +144,9 @@ pub(crate) struct Lag { impl Lag { /// Refuses lags that would make a payload of `rows` instance rows - /// larger than one call takes (`limit`, `max_request_bytes`), before - /// any column is computed, as the payload's limit refuses it. - pub fn fits(&self, rows: usize, limit: u64) -> Result<(), CoreError> { + /// larger than the endpoint takes, before any column is computed, as + /// the payload's limit refuses it. + pub fn fits(&self, rows: usize) -> Result<(), CoreError> { let least = usize::try_from(self.steps) .ok() .and_then(|steps| { @@ -156,15 +155,16 @@ impl Lag { .checked_mul(steps) }) .unwrap_or(usize::MAX); - if least as u64 > limit { - return Err(MAX_REQUEST_BYTES.named( - CoreError::new( - Sentence::LagPastPayload, - "the request is larger than one call takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) - .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), + if least > PAYLOAD_BYTES { + return Err(CoreError::new( + Sentence::LagPastPayload, + "the request is larger than the endpoint takes", + ) + .with_field("field", Detail::text("payload")) + .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) + .with_field( + "limit", + Detail::Scalar(Scalar::UInt64(PAYLOAD_BYTES as u64)), )); } Ok(()) @@ -542,7 +542,7 @@ mod tests { times: vec![], }, ); - let e = lag.fits(2, MAX_REQUEST_BYTES.default).err().unwrap(); + let e = lag.fits(2).err().unwrap(); assert_eq!(e.code, ErrorCode::InvalidRequest); assert_eq!(e.fields["field"], Detail::text("payload")); } diff --git a/rust/crates/sdm-core/src/lib.rs b/rust/crates/sdm-core/src/lib.rs index 78d45d0bd..d1159c432 100644 --- a/rust/crates/sdm-core/src/lib.rs +++ b/rust/crates/sdm-core/src/lib.rs @@ -21,7 +21,6 @@ pub mod exact_number; mod fallback; mod held; mod lag; -pub mod limits; pub mod materialize; mod memory; #[cfg(test)] diff --git a/rust/crates/sdm-core/src/limits.rs b/rust/crates/sdm-core/src/limits.rs deleted file mode 100644 index edf219780..000000000 --- a/rust/crates/sdm-core/src/limits.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! The size limits a run holds a request to, resolved from what the host -//! passed with its start ([`GivenLimits`]): the caller's value, else the -//! environment's, else the default ([`sdm_contract::limits::SETTINGS`]). -//! Every host reads its limits through this one rule. - -use sdm_contract::limits::{ - GivenLimits, Limits, MAX_RELATIONAL_CONTEXT_ROWS, MAX_RELATIONAL_PREDICT_ROWS, - MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS, MAX_TEMPORAL_LINK_PREDICT_ROWS, SETTINGS, Setting, -}; -use sdm_contract::{CoreError, Detail, Sentence}; - -/// The limits `given` names, or the refusal of an environment variable that -/// is not a whole number of at least one, naming the setting and the -/// variable. A name that is no setting's, or a value given below one, is -/// the host's fault, `INVALID_MESSAGE`: a host checks its caller's values. -pub fn resolve(given: &GivenLimits) -> Result { - for (name, &value) in &given.given { - if value == 0 || !SETTINGS.iter().any(|setting| setting.name == name) { - return Err(malformed("limits.given", name)); - } - } - for env in given.environment.keys() { - if !SETTINGS.iter().any(|setting| setting.env == env) { - return Err(malformed("limits.environment", env)); - } - } - let pick = |setting: &Setting| -> Result { - if let Some(&value) = given.given.get(setting.name) { - return Ok(value); - } - let Some(text) = given.environment.get(setting.env) else { - return Ok(setting.default); - }; - whole(text).ok_or_else(|| { - setting.named( - CoreError::new( - Sentence::LimitEnvironmentInvalid, - "a limit's environment variable is not a whole number of at least one", - ) - .with_field("given", Detail::text(text)), - ) - }) - }; - Ok(Limits { - max_relational_context_rows: pick(&MAX_RELATIONAL_CONTEXT_ROWS)?, - max_relational_predict_rows: pick(&MAX_RELATIONAL_PREDICT_ROWS)?, - max_temporal_link_predict_rows: pick(&MAX_TEMPORAL_LINK_PREDICT_ROWS)?, - max_tabular_context_rows: pick(&MAX_TABULAR_CONTEXT_ROWS)?, - max_request_bytes: pick(&MAX_REQUEST_BYTES)?, - }) -} - -/// `text` as a whole number of at least one: decimal digits, with `_` -/// allowed between them as Python and Java write a long number, and white -/// space around them; anything else, zero, or a number past `u64`, none. -fn whole(text: &str) -> Option { - let digits = text.trim(); - let well_formed = digits.as_bytes().first().is_some_and(u8::is_ascii_digit) - && digits.as_bytes().last().is_some_and(u8::is_ascii_digit) - && digits.bytes().all(|b| b.is_ascii_digit() || b == b'_') - && !digits.contains("__"); - if !well_formed { - return None; - } - digits - .replace('_', "") - .parse::() - .ok() - .filter(|&value| value >= 1) -} - -fn malformed(field: &str, name: &str) -> CoreError { - CoreError::protocol( - Sentence::InvalidMessage, - "the start names a limit that is no setting's, or one below one", - ) - .with_field("field", Detail::text(field)) - .with_field("name", Detail::text(name)) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use sdm_contract::ErrorCode; - - use super::*; - - fn given(given: &[(&str, u64)], environment: &[(&str, &str)]) -> GivenLimits { - GivenLimits { - given: given.iter().map(|(k, v)| ((*k).to_owned(), *v)).collect(), - environment: environment - .iter() - .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) - .collect::>(), - } - } - - #[test] - fn nothing_given_is_every_default() { - assert_eq!(resolve(&GivenLimits::default()).unwrap(), Limits::default()); - } - - #[test] - fn the_caller_comes_before_the_environment_before_the_default() { - let env = MAX_TABULAR_CONTEXT_ROWS.env; - let from_env = resolve(&given(&[], &[(env, "200000")])).unwrap(); - assert_eq!(from_env.max_tabular_context_rows, 200_000); - assert_eq!(from_env.max_request_bytes, MAX_REQUEST_BYTES.default); - let both = resolve(&given( - &[(MAX_TABULAR_CONTEXT_ROWS.name, 50_000)], - &[(env, "200000")], - )) - .unwrap(); - assert_eq!(both.max_tabular_context_rows, 50_000); - } - - #[test] - fn environment_text_is_a_whole_number_of_at_least_one() { - let env = MAX_REQUEST_BYTES.env; - for (text, expected) in [ - ("1", Some(1)), - (" 64000000 ", Some(64_000_000)), - ("200_000", Some(200_000)), - ("0", None), - ("", None), - ("-1", None), - ("1e6", None), - ("2.5", None), - ("_1", None), - ("1__0", None), - ("ten", None), - ("18446744073709551616", None), - ] { - let resolved = resolve(&given(&[], &[(env, text)])); - match expected { - Some(value) => assert_eq!(resolved.unwrap().max_request_bytes, value, "{text:?}"), - None => { - let error = resolved.unwrap_err(); - assert_eq!( - error.sentence, - Sentence::LimitEnvironmentInvalid, - "{text:?}" - ); - assert_eq!(error.code, ErrorCode::InvalidConfiguration); - assert_eq!(error.fields["env"], Detail::text(env)); - assert_eq!( - error.fields["setting"], - Detail::text(MAX_REQUEST_BYTES.name) - ); - assert_eq!(error.fields["given"], Detail::text(text)); - } - } - } - } - - #[test] - fn a_given_zero_and_an_unknown_name_are_the_host_s_fault() { - let error = resolve(&given(&[(MAX_REQUEST_BYTES.name, 0)], &[])).unwrap_err(); - assert_eq!(error.sentence, Sentence::InvalidMessage); - let error = resolve(&given(&[("max_tables", 20)], &[])).unwrap_err(); - assert_eq!(error.sentence, Sentence::InvalidMessage); - let error = resolve(&given(&[], &[("SDM_CLIENT_MAX_HOPS", "8")])).unwrap_err(); - assert_eq!(error.sentence, Sentence::InvalidMessage); - } -} diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index 731c35f6d..7e1912f94 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -79,7 +79,6 @@ use sdm_contract::data::{ MATCHES_COLUMN, NullOrder, OptionalRequest, ProviderChoice, SEED_STORED_TIME, TableColumn, TextTimes, }; -use sdm_contract::limits::{Limits, MAX_RELATIONAL_CONTEXT_ROWS}; use sdm_contract::ops::{ ColumnRef, ForecastStep, Materialized, MemoBounds, MemoRows, MemoStoredTime, OperationResult, ProfileName, ProfileSpec, QueryColumn, QueryFacts, QuerySchema, QueryTable, RequestBatch, @@ -187,8 +186,6 @@ pub(crate) struct Inputs { /// (`Materialize.most_calls`). pub most_calls: Option, pub facts: HostFacts, - /// The size limits the run holds the request to. - pub limits: Limits, /// The zones the run resolves names with: the host's local zone, which /// places a date literal's two-digit year (`pql::local_year`). pub zones: Arc, @@ -2879,8 +2876,7 @@ pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result Result Result Result Result mode.context_examples(), options::RunMode::Best => mode.context_examples().max(asked_examples.unwrap_or(0)), @@ -3797,7 +3791,7 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result limits.max_relational_context_rows { - return Err(MAX_RELATIONAL_CONTEXT_ROWS.named( - invalid( - Sentence::ContextPastRows, - "the context holds more rows than a request takes", - "query", - ) - .with_field("context_rows", count(num_train)) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(limits.max_relational_context_rows)), - ), - )); - } - // The rows the largest call scores (`_the_largest_batch`), which the - // lagged columns are held to. + // The rows the largest call scores (`_the_largest_batch`), which a + // payload past the endpoint's body names as the size to come down from. let largest = per_call.min(num_test); let train_anchors = anchors.len(); let entity_names = crate::subgraph::sample_columns(&graph, entity, &excluded); @@ -4143,7 +4120,7 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result { match &request.source { ExampleSource::Sql { provider, .. } => { let data = root.data_access(provider)?; Ok(Run::with_data(data, move |asker| async move { - run(asker, request, attachments, facts, limits).await + run(asker, request, attachments, facts).await })) } ExampleSource::Frames { .. } => Ok(Run::new(move |asker| async move { - run(asker, request, attachments, facts, limits).await + run(asker, request, attachments, facts).await })), } } @@ -112,7 +111,6 @@ async fn run( request: MaterializeExamples, mut attachments: Attachments, facts: HostFacts, - limits: Limits, ) -> Result { let call: ExampleCall = crate::attached::take_json(&mut attachments, request.input)?; if call.rows_per_call == Some(0) { @@ -153,21 +151,6 @@ async fn run( } else { None }; - // The instance table holds the context's examples: at most as many as - // a request carries (`max_relational_context_rows`). - if context.num_rows() as u64 > limits.max_relational_context_rows { - return Err(MAX_RELATIONAL_CONTEXT_ROWS.named( - CoreError::new( - Sentence::ContextPastRows, - "the context holds more rows than a request takes", - ) - .with_field("context_rows", count(context.num_rows())) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(limits.max_relational_context_rows)), - ), - )); - } refuse_entities(predict.num_rows() as u64, 1, None)?; let seed = call .random_seed @@ -315,7 +298,6 @@ async fn run( magnitude: Some(context.num_rows() as u128), }), None, - limits.max_relational_context_rows, )?; options::hops( None, @@ -346,12 +328,8 @@ async fn run( } else { options::BatchTask::Ordinary }; - usize::try_from(options::rows_one_task_call_scores( - call.rows_per_call, - task, - &limits, - )) - .map_err(|_| internal("batch size overflow"))? + usize::try_from(options::rows_one_task_call_scores(call.rows_per_call, task)) + .map_err(|_| internal("batch size overflow"))? }; let mut calls = Vec::new(); for start in (0..predict.num_rows()).step_by(per_call) { @@ -474,7 +452,6 @@ async fn run( forecast, }, )?; - payload::check_bytes(&built, limits.max_request_bytes)?; calls.push((built.bytes, count as u64)); } let mut calls = calls.into_iter(); diff --git a/rust/crates/sdm-core/src/materialize/frames.rs b/rust/crates/sdm-core/src/materialize/frames.rs index ff1c9c074..ffd8a16a4 100644 --- a/rust/crates/sdm-core/src/materialize/frames.rs +++ b/rust/crates/sdm-core/src/materialize/frames.rs @@ -334,7 +334,6 @@ pub(crate) async fn materialize( mut attachments: crate::run::Attachments, facts: HostFacts, zones: Arc, - limits: sdm_contract::limits::Limits, ) -> Result { let Some(Attachment::FrameGraph(owner)) = attachments.remove(&request.graph) else { return Err(internal("frame graph owner is absent")); @@ -372,7 +371,6 @@ pub(crate) async fn materialize( most_entities: call.most_entities, most_calls: call.most_calls, facts, - limits, zones, memo: None, source: "in_memory".into(), diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 25b46f22c..11bf19e11 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -32,14 +32,13 @@ //! the answer. use sdm_contract::error::AnsweredParts; -use sdm_contract::limits::Limits; use sdm_contract::ops::{ModelEndpoint, OperationResult, Predicted}; use sdm_contract::run::Endpoint; use sdm_contract::{CoreError, CoreWarning, Detail, Scalar, Sentence}; use sdm_request::predict::{ - Notice, Part, absent, absent_notice, bad_response, body, by_step, call_past_bytes, call_starts, - empty, layout, left_out, merge, nothing_to_score, numbered_from, predict_rows, preflight, - prepare, refused, steps_unexplained, table, + Notice, Part, absent, absent_notice, bad_response, body, by_step, call_starts, empty, layout, + left_out, merge, nothing_to_score, numbered_from, predict_rows, preflight, prepare, refused, + steps_unexplained, table, }; use sdm_request::serving::{self, Failure}; use sdm_wire::json::{self, Value}; @@ -104,8 +103,6 @@ pub(crate) struct Inputs { /// core writes and whose answers it reads; `None` for the /// client's own endpoint. pub endpoint: Option, - /// The size limits the run holds its calls to. - pub limits: Limits, } /// The JSON a request state's payload holds, or a refusal naming where in @@ -410,7 +407,6 @@ pub(crate) async fn one( &inputs.model, &inputs.outputs, inputs.default_outputs, - &inputs.limits, )?; let computed = if provided_analysis.is_none() && crate::columns::requested(&outputs) { Some(crate::columns::compute(&payloads[..1])?) @@ -433,30 +429,28 @@ pub(crate) async fn one( warn(asker, notice); } // Every call written before the first is sent, as the client's - // plan is listed whole (`_numbered_across_calls`), each split into - // smaller calls where it is past the most bytes a call takes, then - // each call's rows counted. - let mut sent = Vec::with_capacity(payloads.len()); + // plan is listed whole (`_numbered_across_calls`), then each call's + // rows counted. To a serving endpoint, whose refusal of a call too + // large is a dropped connection, a call past its limit is split up + // front into calls that fit. + let mut calls = Vec::with_capacity(payloads.len()); let mut bodies = Vec::with_capacity(payloads.len()); - let mut sizes = Vec::with_capacity(payloads.len()); for (payload, request_id) in payloads.iter().zip(&ids) { - for (call, body, bytes) in fitted(inputs, &outputs, payload, request_id)? { - sent.push(call); + for (call, body) in fitted(inputs, &outputs, payload, request_id)? { + calls.push((call, request_id.as_str())); bodies.push(body); - sizes.push(bytes); } } - let payloads = sent; + let payloads: Vec = calls.iter().map(|(call, _)| call.clone()).collect(); let mut session = crate::batch_session::Session::eligible( &bodies, inputs.supports_sessions && inputs.endpoint.is_none(), ); - // Validate all full calls before any network effect, including their size. // Retain only the small scoring half alongside encoded fallback bytes. let plans = bodies .into_iter() - .zip(sizes) - .map(|(body, bytes)| { + .map(|body| { + let bytes = body.to_bytes(); let scoring = if session.is_some() { sdm_request::frame_session::subset( &body, @@ -470,7 +464,10 @@ pub(crate) async fn one( .collect::>(); let starts = call_starts(&payloads)?; let mut parts = Vec::with_capacity(payloads.len()); - for ((payload, (body, bytes)), start) in payloads.iter().zip(plans).zip(starts) { + // The rows a call the endpoint took after a smaller one was asked, + // kept for the calls after it. + let mut most = None; + for (((payload, request_id), (body, bytes)), start) in calls.iter().zip(plans).zip(starts) { let response = async { if let Some(session) = session.as_mut() && let Some(answer) = session.score(asker, &body).await? @@ -478,7 +475,10 @@ pub(crate) async fn one( let response = PredictionResponse::read(&answer).map_err(refused)?; return layout(payload, &response, crate::pql::float_text); } - call(asker, payload, bytes, inputs.endpoint.as_ref()).await + answered( + asker, inputs, &outputs, request_id, payload, bytes, &mut most, + ) + .await } .await; let answered = match response { @@ -513,47 +513,103 @@ pub(crate) async fn one( Ok((merged, outputs, scored)) } -/// `payload`'s calls, each with the body it sends and that body's bytes: -/// the one call where its body fits the most bytes a call takes -/// (`max_request_bytes`) and, to a serving endpoint, the most its envelope -/// takes; else as many calls as it is split into so that each fits +/// `payload` as the calls it is sent in, each with its body: one call, or, +/// to a serving endpoint, as many as it is split into so that each one's +/// envelope fits the endpoint's `max_request_bytes` /// ([`sdm_request::batching::fitted`]), each scoring a run of its rows with -/// the whole context. Every call of a split carries the payload's request -/// id, so the bytes weighed are the bytes sent. A call of one row that is -/// still past a limit is refused, naming the context's bytes. +/// the whole context. A call of one row past it is refused. fn fitted( inputs: &Inputs, outputs: &[String], payload: &Value, request_id: &str, -) -> Result)>, CoreError> { - let limit = inputs.limits.max_request_bytes; - let mut fit = |call: &Value| -> sdm_request::batching::Fit<(Value, Vec)> { - let body = body( +) -> Result, CoreError> { + let write = |call: &Value| { + body( call, &inputs.model, request_id, outputs, &inputs.inference_asked, - )?; - let bytes = body.to_bytes(); - if bytes.len() as u64 > limit { - return Ok(Err(call_past_bytes(&body, bytes.len() as u64, limit))); - } - if let Some(ModelEndpoint::ServingEndpoint { - name, - max_request_bytes, - }) = &inputs.endpoint - && let (_, Some(past)) = serving::weigh(name, *max_request_bytes, &body)? - { - return Ok(Err(past)); - } - Ok(Ok((body, bytes))) + ) + }; + let Some(ModelEndpoint::ServingEndpoint { + name, + max_request_bytes, + }) = &inputs.endpoint + else { + return Ok(vec![(payload.clone(), write(payload)?)]); + }; + let mut fit = |call: &Value| -> sdm_request::batching::Fit { + let body = write(call)?; + let size = serving::wrapped(&body)?.len() as u64; + Ok(if size > *max_request_bytes { + Err(serving::past_limit(name, *max_request_bytes, size)) + } else { + Ok(body) + }) + }; + sdm_request::batching::fitted(payload, &mut fit) +} + +/// The statuses that say a call was too large or too slow to answer: the +/// host's timeout (told as 408), the endpoint's 413, and a gateway's 502 or +/// 504, which a call it gave up on comes back as. +const TOO_LARGE: [u16; 4] = [408, 413, 502, 504]; + +/// `payload`'s answer, its rows numbered from zero. Where the endpoint +/// refuses the call as too large or too slow ([`TOO_LARGE`]), its rows are +/// asked again in calls of half as many, each with the whole context, and +/// that size is kept in `most` for the calls after it. A call of one row is +/// not split: its refusal, the endpoint's own, stands, as does any other. +async fn answered( + asker: &Asker, + inputs: &Inputs, + outputs: &[String], + request_id: &str, + payload: &Value, + bytes: Vec, + most: &mut Option, +) -> Result { + let rows = predict_rows(payload).unwrap_or(0); + let size = match *most { + Some(most) if rows > most => most, + _ => match call(asker, payload, bytes, inputs.endpoint.as_ref()).await { + Err(error) if rows > 1 && too_large(&error) => { + let half = rows.div_ceil(2); + *most = Some(half); + half + } + answer => return answer, + }, }; - Ok(sdm_request::batching::fitted(payload, &mut fit)? - .into_iter() - .map(|(call, (body, bytes))| (call, body, bytes)) - .collect()) + let mut parts = Vec::new(); + for start in (0..rows).step_by(size) { + let range = start..(start + size).min(rows); + let call = sdm_request::batching::predict_slice(payload, range)?; + let bytes = body( + &call, + &inputs.model, + request_id, + outputs, + &inputs.inference_asked, + )? + .to_bytes(); + let part = Box::pin(answered( + asker, inputs, outputs, request_id, &call, bytes, most, + )) + .await?; + parts.push(numbered_from(part, start)?); + } + merge(parts) +} + +/// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. +fn too_large(error: &CoreError) -> bool { + matches!( + error.fields.get(sdm_contract::error::REFUSED_STATUS), + Some(Detail::Scalar(Scalar::UInt16(status))) if TOO_LARGE.contains(status) + ) } /// One payload's call, its body as written, sent and its answer laid out; diff --git a/rust/crates/sdm-core/src/ranking.rs b/rust/crates/sdm-core/src/ranking.rs index ec16a08af..97d311259 100644 --- a/rust/crates/sdm-core/src/ranking.rs +++ b/rust/crates/sdm-core/src/ranking.rs @@ -176,10 +176,8 @@ pub(crate) async fn materialize( // each question gathers whole, held as many times over as there are // questions (`refuse_more_entities_than_fit_in_memory`), past the // host's limit, 100,000 where it names none. - let per_call = sdm_request::options::rows_one_call_scores( - asked_rows_per_call(inputs.values.as_ref())?, - &inputs.limits, - ); + let per_call = + sdm_request::options::rows_one_call_scores(asked_rows_per_call(inputs.values.as_ref())?); let entities = inputs.indices.as_ref().map_or(0, |data| { data.batches.iter().map(|b| b.num_rows()).sum::() }); @@ -349,7 +347,7 @@ pub(crate) async fn predict( let checked = (|| -> Result<(), CoreError> { let payload = crate::predict::payload(payload)?; let payloads = [payload]; - let outputs = prepare(&payloads, &inputs.model, &asked, false, &inputs.limits)?; + let outputs = prepare(&payloads, &inputs.model, &asked, false)?; preflight(&payloads, &outputs, query, &inputs.model)?; body( &payloads[0], diff --git a/rust/crates/sdm-core/src/session.rs b/rust/crates/sdm-core/src/session.rs index 473489b99..4da783244 100644 --- a/rust/crates/sdm-core/src/session.rs +++ b/rust/crates/sdm-core/src/session.rs @@ -175,7 +175,6 @@ pub(crate) fn prepare_score( asker: &Asker, input: SessionScoreInput, mut attached: Attachments, - limits: &sdm_contract::limits::Limits, ) -> Result { let mut request: PrepareTabularScore = take_json(&mut attached, input.input)?; if request.predict != input.predict { @@ -198,7 +197,7 @@ pub(crate) fn prepare_score( &facts.kind, Some(request.state.bound.outputs.clone()), ); - request.state.bound = sdm_request::direct::bind(facts, limits)?; + request.state.bound = sdm_request::direct::bind(facts)?; } let table = take_value(&mut attached, request.predict)?; let mut payload = object([ diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 3558274ea..4c092180e 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -12,7 +12,6 @@ use sdm_contract::direct::storage::{ DIRECT_STORAGE_KEY, DirectStorage, StorageOwner, TimestampUnit, }; use sdm_contract::direct::*; -use sdm_contract::limits::Limits; use sdm_contract::ops::{HeldSchema, OperationResult}; use sdm_contract::tabular_arrow::*; use sdm_contract::{CoreError, Detail, HostFacts, Sentence}; @@ -293,33 +292,29 @@ fn bound( input: &ArrowPrediction, context: &ArrowData, scoring: &ArrowData, - limits: &Limits, ) -> Result { let description = prepare(asker, &input.request, context, scoring)?; - let bound = direct::bind( - DirectCallFacts { - query: None, - model: input.model.clone(), - outputs: crate::pql::explanation_outputs( - input.request.query.as_deref().unwrap_or(""), - &input.request.task, - input.outputs.clone(), - ), - kind: input.request.task.clone(), - anchored: input - .request - .task_options - .entries - .iter() - .rev() - .find(|(name, _)| name == "anchor_time_column") - .is_some_and(|(_, value)| option_truth(value)), - relationships: !description.schema.links.is_empty(), - tables: description.schema.tables.len(), - context_rows: header(context).rows, - }, - limits, - )?; + let bound = direct::bind(DirectCallFacts { + query: None, + model: input.model.clone(), + outputs: crate::pql::explanation_outputs( + input.request.query.as_deref().unwrap_or(""), + &input.request.task, + input.outputs.clone(), + ), + kind: input.request.task.clone(), + anchored: input + .request + .task_options + .entries + .iter() + .rev() + .find(|(name, _)| name == "anchor_time_column") + .is_some_and(|(_, value)| option_truth(value)), + relationships: !description.schema.links.is_empty(), + tables: description.schema.tables.len(), + context_rows: header(context).rows, + })?; if bound.model != sdm_request::predict::TABULAR { return Err(direct::invalid(Sentence::DirectRelationalFrames) .with_field("model", Detail::text(&bound.model))); @@ -376,9 +371,8 @@ async fn predict( input: ArrowPrediction, context: ArrowData, scoring: ArrowData, - limits: &Limits, ) -> Result { - let prepared = bound(asker, &input, &context, &scoring, limits)?; + let prepared = bound(asker, &input, &context, &scoring)?; let payload = tabular::body( &prepared.columns, &context.batches, @@ -402,8 +396,8 @@ async fn predict( part.columns = analysis; return crate::predict::finish(asker, part, prepared.bound.outputs); } - let bytes = sdm_request::predict::call_bytes(&payload, limits.max_request_bytes)?; - let answer = crate::predict::call_answer(asker, bytes, input.endpoint.as_ref()).await?; + let answer = + crate::predict::call_answer(asker, payload.to_bytes(), input.endpoint.as_ref()).await?; finish(asker, payload, answer, prepared.bound.outputs, analysis) } fn finish( @@ -426,7 +420,6 @@ pub(crate) async fn run( mut attached: Attachments, facts: HostFacts, zones: Arc, - limits: Limits, ) -> Result { match operation { ArrowTabular::Describe { input, table } => { @@ -452,7 +445,7 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let context = take_table(&mut attached, context)?; let scoring = take_table(&mut attached, scoring)?; - predict(&asker, input, context, scoring, &limits).await + predict(&asker, input, context, scoring).await } ArrowTabular::Fit { input, @@ -462,12 +455,12 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let context = take_table(&mut attached, context)?; let scoring = take_table(&mut attached, predict)?; - fit(&asker, input, context, scoring, &limits).await + fit(&asker, input, context, scoring).await } ArrowTabular::Score { input, predict } => { let input = take_json(&mut attached, input)?; let scoring = take_table(&mut attached, predict)?; - score(&asker, input, scoring, &limits).await + score(&asker, input, scoring).await } ArrowTabular::Close { input } => close(&asker, take_json(&mut attached, input)?).await, ArrowTabular::MaterializePredict { @@ -478,20 +471,12 @@ pub(crate) async fn run( let input = take_json(&mut attached, input)?; let table = take_table(&mut attached, table)?; let predict = take_table(&mut attached, predict)?; - materialize( - &asker, - input, - table, - Some(predict), - &facts, - zones.as_ref(), - &limits, - ) + materialize(&asker, input, table, Some(predict), &facts, zones.as_ref()) } ArrowTabular::Materialize { input, table } => { let input = take_json(&mut attached, input)?; let table = take_table(&mut attached, table)?; - materialize(&asker, input, table, None, &facts, zones.as_ref(), &limits) + materialize(&asker, input, table, None, &facts, zones.as_ref()) } } } @@ -534,14 +519,13 @@ async fn fit( input: ArrowFit, context: ArrowData, scoring: ArrowData, - limits: &Limits, ) -> Result { use sdm_contract::{run::Endpoint, session::*}; use tabular::session as policy; if input.prediction.endpoint.is_some() { return Err(invalid("this transport does not support sessions")); } - let prepared = bound(asker, &input.prediction, &context, &scoring, limits)?; + let prepared = bound(asker, &input.prediction, &context, &scoring)?; let encoded = tabular::encode_table(&prepared.columns, &context.batches, crate::pql::float_text)?; let keys = policy::encoded_keys(&encoded, &prepared.keys)?; @@ -565,8 +549,6 @@ async fn fit( tabular::typed_inference(input.prediction.request.inference.clone())?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&payload))?); crate::columns::expose(asker, column_analysis.as_deref(), &prepared.bound.outputs)?; - // The pinned context is one call, which no split makes smaller. - sdm_request::predict::call_bytes(&payload, limits.max_request_bytes)?; let answer = session_call(asker, Endpoint::CreateSession, payload).await?; let session_id = answer .get("session_id") @@ -596,7 +578,6 @@ async fn score( asker: &Asker, mut input: ArrowScore, scoring: ArrowData, - limits: &Limits, ) -> Result { use tabular::session as policy; policy::validate_state(&input.session.state)?; @@ -608,7 +589,7 @@ async fn score( &input.request.task, Some(input.session.state.bound.outputs.clone()), ); - input.session.state.bound = direct::bind(facts, limits)?; + input.session.state.bound = direct::bind(facts)?; } let state = &input.session.state; if state.closed { @@ -799,7 +780,6 @@ fn materialize( supplied: Option, facts: &HostFacts, zones: &dyn Zones, - limits: &Limits, ) -> Result { use sdm_contract::ops::{ProfileName, QueryColumn, QuerySchema, QueryTable}; use sdm_contract::pql::{NodeKind, ProblemType}; @@ -893,7 +873,8 @@ fn materialize( let decision = sdm_engine::tabular::context_decision( candidates.len(), input.num_context_samples, - usize::try_from(limits.max_tabular_context_rows).unwrap_or(usize::MAX), + // No most: the context is sampled only to the count asked for. + usize::MAX, ) .map_err(|_| invalid("context sample count is invalid"))?; let context_rows = if decision.reduced { diff --git a/rust/crates/sdm-core/src/tabular_selection.rs b/rust/crates/sdm-core/src/tabular_selection.rs index e9ad44ca8..6dae80d39 100644 --- a/rust/crates/sdm-core/src/tabular_selection.rs +++ b/rust/crates/sdm-core/src/tabular_selection.rs @@ -35,14 +35,12 @@ fn count_error(error: tabular::CountError) -> TabularCountError { } } -pub(crate) fn decide( - input: DecideTabularContext, - limits: &sdm_contract::limits::Limits, -) -> TabularContextDecision { +pub(crate) fn decide(input: DecideTabularContext) -> TabularContextDecision { match tabular::context_decision( input.population, input.requested, - usize::try_from(limits.max_tabular_context_rows).unwrap_or(usize::MAX), + // No most: the context is sampled only to the count asked for. + usize::MAX, ) { Ok(value) => TabularContextDecision::Selected { wanted: value.wanted, @@ -83,10 +81,7 @@ pub(crate) fn order( /// Fuse ordinary expansion, exclusion, target filtering and capacity policy. /// Sampling remains after native warning delivery and uses its reached count. -pub(crate) fn select( - input: SelectTabularRows, - limits: &sdm_contract::limits::Limits, -) -> Result { +pub(crate) fn select(input: SelectTabularRows) -> Result { let (scored, excluded, population) = match input.mode { TabularSelectionMode::Indices { source, requested } => { let expanded = tabular::expand_keys(&source, &requested); @@ -114,13 +109,10 @@ pub(crate) fn select( excluded, })? .positions; - let context = decide( - DecideTabularContext { - population: candidates.len(), - requested: input.requested_context, - }, - limits, - ); + let context = decide(DecideTabularContext { + population: candidates.len(), + requested: input.requested_context, + }); Ok(SelectedTabularRows::Selected { scored, candidates, diff --git a/rust/crates/sdm-core/src/test_ops.rs b/rust/crates/sdm-core/src/test_ops.rs index f927138c9..dcf78c1f3 100644 --- a/rust/crates/sdm-core/src/test_ops.rs +++ b/rust/crates/sdm-core/src/test_ops.rs @@ -25,7 +25,6 @@ pub(crate) fn run( operation: TestOperation, attachments: &mut Attachments, facts: HostFacts, - limits: sdm_contract::limits::Limits, zones: Arc, ) -> Result { let keyed = match &operation { @@ -107,7 +106,6 @@ pub(crate) fn run( sdm_contract::ops::Operation::Materialize(*materialize), std::mem::take(attachments), facts, - limits, Arc::clone(&zones), true, )?, diff --git a/rust/crates/sdm-core/tests/answer_encoding.rs b/rust/crates/sdm-core/tests/answer_encoding.rs index 6aad4be56..6e8654e71 100644 --- a/rust/crates/sdm-core/tests/answer_encoding.rs +++ b/rust/crates/sdm-core/tests/answer_encoding.rs @@ -247,7 +247,6 @@ fn predict_run() -> Run { start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation, diff --git a/rust/crates/sdm-core/tests/attachments.rs b/rust/crates/sdm-core/tests/attachments.rs index 64d0d3bf0..57dd02b97 100644 --- a/rust/crates/sdm-core/tests/attachments.rs +++ b/rust/crates/sdm-core/tests/attachments.rs @@ -49,7 +49,6 @@ const READ: &[&str] = &[ "DecideTabularTask.input", "OrderSampledTabularRows.input", "SelectTabularRows.input", - "CheckTabularCap.input", "CheckDirectShape.input", "BindDirect.input", "CheckDirectRows.input", diff --git a/rust/crates/sdm-core/tests/dispatch.rs b/rust/crates/sdm-core/tests/dispatch.rs index e000ecd48..63acae7a6 100644 --- a/rust/crates/sdm-core/tests/dispatch.rs +++ b/rust/crates/sdm-core/tests/dispatch.rs @@ -136,7 +136,6 @@ fn run_of(root: &CompositionRoot, operation: Operation) -> Result Result Result { start( &root(), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation, @@ -791,7 +786,6 @@ fn validate_run(query: &str, facts: HostFacts) -> Result Result Run { sdm_core::start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-core/tests/held.rs b/rust/crates/sdm-core/tests/held.rs index 1bc8ee891..991851b13 100644 --- a/rust/crates/sdm-core/tests/held.rs +++ b/rust/crates/sdm-core/tests/held.rs @@ -173,7 +173,6 @@ fn try_run( start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Predict(Predict { diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 05d9e1531..75a4e644a 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -214,7 +214,6 @@ fn run_of(operation: Operation, attachments: Attachments) -> Result Result panic!("not a call: {other:?}"), } }; - let limit = sdm_contract::limits::MAX_REQUEST_BYTES.default as usize; + let limit = sdm_request::payload::PAYLOAD_BYTES; let base = called(padded("")).unwrap(); - // ASCII to the limit, exactly: taken whole; one byte more: split, the - // first call smaller. + // ASCII to the limit, exactly: taken; one byte more: refused. assert_eq!(called(padded(&"a".repeat(limit - base))).unwrap(), limit); - let split = called(padded(&"a".repeat(limit - base + 1))).unwrap(); - assert!(split < limit, "{split}"); - // Padding past the limit on its own: no call is small enough. - let past = called(padded(&"a".repeat(limit))).unwrap_err(); + let past = called(padded(&"a".repeat(limit - base + 1))).unwrap_err(); assert_eq!(past.code, ErrorCode::InvalidRequest); - assert_eq!(past.sentence, Sentence::CallPastEndpoint); assert_eq!(past.fields["field"], text("call")); - assert_eq!(past.fields["setting"], text("max_request_bytes")); - assert_eq!(past.fields["env"], text("SDM_CLIENT_MAX_REQUEST_BYTES")); // Three-byte characters, each counted as its three bytes written. let euros = (limit - base) / 3; let rest = limit - base - 3 * euros; let at = format!("{}{}", "\u{20ac}".repeat(euros), "a".repeat(rest)); assert_eq!(called(padded(&at)).unwrap(), limit); let over = format!("{}{}", "\u{20ac}".repeat(euros + 1), "a".repeat(rest)); - assert!(called(padded(&over)).unwrap() < limit); + assert!(called(padded(&over)).is_err()); } diff --git a/rust/crates/sdm-core/tests/serving.rs b/rust/crates/sdm-core/tests/serving.rs index dc5ceaf4f..92f611d81 100644 --- a/rust/crates/sdm-core/tests/serving.rs +++ b/rust/crates/sdm-core/tests/serving.rs @@ -183,7 +183,6 @@ fn predict(calls: &[&[i64]], endpoint: Option) -> Run { start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Predict(Predict { @@ -350,9 +349,10 @@ fn each_call_is_wrapped_as_the_endpoint_reads_it_and_answered_from_its_row() { } #[test] -fn a_call_past_the_caller_s_limit_is_split_and_one_that_cannot_be_is_refused() { - // The first call fits; the second, larger, does not: it is asked as - // smaller calls, each with the whole context, each within the limit. +fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { + // The first call fits; the second, larger, does not: it is refused + // when it would be sent, carrying the first call's answers, as the old + // transport weighs each call as it sends it. let mut probe = predict(&[&[10]], serving(16_000_000)); let Ok(Step::CallModel(call)) = probe.next() else { panic!("a call"); @@ -367,30 +367,18 @@ fn a_call_past_the_caller_s_limit_is_split_and_one_that_cannot_be_is_refused() { retryable: false, }; let (end, bodies) = drive(&mut run, None, &failure); - assert_eq!(bodies.len(), 4); + assert_eq!(bodies.len(), 1); assert_eq!(bodies[0], one); - assert!(bodies.iter().all(|body| body.len() as u64 <= limit)); - let Ok(Step::Done { - result: OperationResult::Predict(predicted), - }) = end - else { - panic!("the run predicts: {end:?}"); - }; - let result = arrow(&mut run, predicted.result); - assert_eq!(int64s(&result, "row_index"), (0..4).collect::>()); - // A limit one row's call misses: no call is small enough, and nothing - // is sent. - let mut run = predict(&[&[10]], serving(limit - 1)); - let (end, bodies) = drive(&mut run, None, &failure); - assert!(bodies.is_empty()); let error = end.unwrap_err(); assert_eq!(error.sentence, Sentence::ServingPastLimit); assert_eq!(error.code, ErrorCode::InvalidRequest); - assert_eq!( - error.fields["limit"], - Detail::Scalar(Scalar::UInt64(limit - 1)) - ); - assert!(count(&error, "context_bytes").is_some_and(|context| context < limit)); + assert_eq!(error.fields["limit"], Detail::Scalar(Scalar::UInt64(limit))); + assert_eq!(count(&error, "calls_answered"), Some(1)); + // A limit the first call misses: nothing is sent. + let mut run = predict(&[&[10]], serving(limit - 1)); + let (end, bodies) = drive(&mut run, None, &failure); + assert!(bodies.is_empty()); + assert_eq!(end.unwrap_err().sentence, Sentence::ServingPastLimit); } #[test] diff --git a/rust/crates/sdm-core/tests/tabular_sessions.rs b/rust/crates/sdm-core/tests/tabular_sessions.rs index dc9eab19c..bd2f6bcbf 100644 --- a/rust/crates/sdm-core/tests/tabular_sessions.rs +++ b/rust/crates/sdm-core/tests/tabular_sessions.rs @@ -34,7 +34,6 @@ fn run( sdm_core::start( &CompositionRoot::new(Arc::new(sdm_data::Data::new())), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { @@ -117,19 +116,16 @@ fn send(endpoint: Endpoint, payload: Vec, response: Value) -> Value { #[test] fn fit_score_twice_and_retry_close_with_remapped_attachments() { - let bound = sdm_request::direct::bind( - DirectCallFacts { - query: None, - model: "kumo-tabular".into(), - outputs: Some(vec!["prediction".into()]), - kind: "regression".into(), - anchored: false, - relationships: false, - tables: 1, - context_rows: 3, - }, - &Default::default(), - ) + let bound = sdm_request::direct::bind(DirectCallFacts { + query: None, + model: "kumo-tabular".into(), + outputs: Some(vec!["prediction".into()]), + kind: "regression".into(), + anchored: false, + relationships: false, + tables: 1, + context_rows: 3, + }) .unwrap(); let columns = TabularColumns { dtypes: vec![("id".into(), Dtype::Int64), ("y".into(), Dtype::Float64)], diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index 562c77402..23fa7ebe7 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -337,7 +337,6 @@ fn replay_spec( let mut run = start( &root, StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { @@ -891,7 +890,6 @@ fn materialize_query( let mut run = start( &root, StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::Materialize(Materialize { diff --git a/rust/crates/sdm-core/tests/view_semantic.rs b/rust/crates/sdm-core/tests/view_semantic.rs index 5a7293174..d2d49f4bc 100644 --- a/rust/crates/sdm-core/tests/view_semantic.rs +++ b/rust/crates/sdm-core/tests/view_semantic.rs @@ -241,7 +241,6 @@ fn semantic_run(definition: &str, spec: SourceSpec) -> sdm_core::Run { start( &CompositionRoot::new(Arc::new(fake)), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-host/tests/codec.rs b/rust/crates/sdm-host/tests/codec.rs index b86b4e00f..46369e4b8 100644 --- a/rust/crates/sdm-host/tests/codec.rs +++ b/rust/crates/sdm-host/tests/codec.rs @@ -27,7 +27,6 @@ use sdm_host::{CONTROL_CAP, Outgoing, decode, encode, read, write, write_error}; /// envelope. fn start(model: String) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/contract_v3.rs b/rust/crates/sdm-host/tests/contract_v3.rs index 1f7fa0952..eb7e14f95 100644 --- a/rust/crates/sdm-host/tests/contract_v3.rs +++ b/rust/crates/sdm-host/tests/contract_v3.rs @@ -53,7 +53,6 @@ fn facts(local_zone: Option) -> HostFacts { fn start_message(operation: Operation, local_zone: Option) -> ControlMessage { ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(local_zone), operation, diff --git a/rust/crates/sdm-host/tests/data_port.rs b/rust/crates/sdm-host/tests/data_port.rs index e2722523e..a1437f60b 100644 --- a/rust/crates/sdm-host/tests/data_port.rs +++ b/rust/crates/sdm-host/tests/data_port.rs @@ -56,7 +56,6 @@ fn fixture(files: &[(&str, &str)]) -> PathBuf { fn start_message(operation: Operation, roots: &[&Path]) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/error_table.rs b/rust/crates/sdm-host/tests/error_table.rs index 10a55ed59..46b10fc76 100644 --- a/rust/crates/sdm-host/tests/error_table.rs +++ b/rust/crates/sdm-host/tests/error_table.rs @@ -26,7 +26,6 @@ use sdm_host::{Attachments, HostRun, Outgoing, ZoneResolver, encode, read}; fn start_message(operation: TestOperation) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/faults.rs b/rust/crates/sdm-host/tests/faults.rs index 3bf855e58..2490c35ef 100644 --- a/rust/crates/sdm-host/tests/faults.rs +++ b/rust/crates/sdm-host/tests/faults.rs @@ -43,7 +43,6 @@ use sdm_host::{HostRun, Outgoing, Unreadable, Unreadables, encode, read}; fn start_message(operation: Operation) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { @@ -834,7 +833,6 @@ fn a_test_only_panic_s_payload_is_the_canary() { let mut run = sdm_core::start( &sdm_host::default_root(), StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: HostFacts { now: Scalar::Timestamp { diff --git a/rust/crates/sdm-host/tests/runs.rs b/rust/crates/sdm-host/tests/runs.rs index 8e1b81de3..94393bf4e 100644 --- a/rust/crates/sdm-host/tests/runs.rs +++ b/rust/crates/sdm-host/tests/runs.rs @@ -53,7 +53,6 @@ fn facts(zone_roots: Vec) -> HostFacts { fn start_message(operation: Operation, zone_roots: Vec) -> Vec { encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(zone_roots), operation, diff --git a/rust/crates/sdm-host/tests/type_set.rs b/rust/crates/sdm-host/tests/type_set.rs index 9d2882035..b55a1f42d 100644 --- a/rust/crates/sdm-host/tests/type_set.rs +++ b/rust/crates/sdm-host/tests/type_set.rs @@ -828,7 +828,6 @@ fn infer_column(received: &DataType) -> Value { "time_columns": {}, }); let start = StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-request/src/batching.rs b/rust/crates/sdm-request/src/batching.rs index 103e4ea49..36195b830 100644 --- a/rust/crates/sdm-request/src/batching.rs +++ b/rust/crates/sdm-request/src/batching.rs @@ -2,8 +2,9 @@ // All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! A call's predict rows taken apart, so a call past the most bytes one call -//! takes is asked as several smaller ones. +//! A call's predict rows taken apart, so a call too large is asked as +//! several smaller ones: up front where a serving endpoint's byte limit is +//! known ([`fitted`]), or in halves where the endpoint refuses it. //! //! A payload carries its context whole and the rows it scores beside it, //! each row of its predict half's tables belonging to one of the rows diff --git a/rust/crates/sdm-request/src/direct.rs b/rust/crates/sdm-request/src/direct.rs index 9dde43da4..440e6240d 100644 --- a/rust/crates/sdm-request/src/direct.rs +++ b/rust/crates/sdm-request/src/direct.rs @@ -5,7 +5,6 @@ //! Model-neutral direct shape guards and shared model preflight facts. use sdm_contract::direct::{BoundDirectCall, ColumnFlags, DirectCallFacts, DirectShape, FrameKind}; -use sdm_contract::limits::Limits; use sdm_contract::ops::{DescribedTable, Dtype, Stype}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use std::collections::BTreeSet; @@ -107,15 +106,13 @@ pub fn column_names(columns: &[String], table: &str) -> Result<(), CoreError> { } /// Resolve model/output/capability/task/capacity before inspecting scoring values. -pub fn bind(input: DirectCallFacts, limits: &Limits) -> Result { - crate::predict::bind_direct(input, Some(limits)) +pub fn bind(input: DirectCallFacts) -> Result { + crate::predict::bind_direct(input) } /// Validate the producer's typed decision against its original consumed facts. -/// The capacity was checked when it was bound, and is checked again where -/// the request is sent. pub fn bound(value: &BoundDirectCall) -> Result<(), CoreError> { - if crate::predict::bind_direct(value.facts.clone(), None)? != *value { + if bind(value.facts.clone())? != *value { return Err(CoreError::protocol( Sentence::InvalidMessage, "bound direct call differs from its consumed facts", diff --git a/rust/crates/sdm-request/src/options.rs b/rust/crates/sdm-request/src/options.rs index fef3b18f1..b233aa83b 100644 --- a/rust/crates/sdm-request/src/options.rs +++ b/rust/crates/sdm-request/src/options.rs @@ -8,7 +8,6 @@ //! Errors identify the option and reason without exposing the supplied value. use sdm_contract::data::NegativeNeighborBound; -use sdm_contract::limits::{Limits, MAX_RELATIONAL_CONTEXT_ROWS}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_wire::json::{Value, is_integer_token, whole}; use sdm_wire::payload::{ClassificationConfig, InferenceConfig}; @@ -821,11 +820,16 @@ pub fn rows_per_call(given: &Value) -> Result, CoreError> { } } -/// How many entities one call scores, as the caller asked or as the -/// limits allow (`rows_one_call_scores`): the caller's size only ever -/// lowers the limit's. -pub fn rows_one_call_scores(asked: Option, limits: &Limits) -> u64 { - rows_one_task_call_scores(asked, BatchTask::Ordinary, limits) +/// How many rows one model call scores where the caller names no size +/// (`predict_rows_max`, the serving layer's `TFM_MAX_TEST_ROWS` default): +/// a batching default, not a cap; a call the endpoint refuses as too large +/// is asked again in halves. +pub const PREDICT_ROWS: u64 = 10_000; + +/// How many entities one call scores, as the caller asked, else the +/// default (`rows_one_call_scores`). +pub fn rows_one_call_scores(asked: Option) -> u64 { + rows_one_task_call_scores(asked, BatchTask::Ordinary) } /// The task distinction that determines an entity batch's model limit. @@ -836,17 +840,13 @@ pub enum BatchTask { TemporalLink, } -/// Apply the task's serving limit after validating the requested positive -/// size: the most rows one call scores (`max_relational_predict_rows`, the -/// serving layer's `TFM_MAX_TEST_ROWS`), or one temporal link prediction -/// call (`max_temporal_link_predict_rows`). -/// PQL and supplied examples use the same policy; hosts do not split rows. -pub fn rows_one_task_call_scores(asked: Option, task: BatchTask, limits: &Limits) -> u64 { - let limit = match task { - BatchTask::Ordinary => limits.max_relational_predict_rows, - BatchTask::TemporalLink => limits.max_temporal_link_predict_rows, - }; - asked.map_or(limit, |n| n.min(limit)) +/// The rows one call scores: the requested positive size, else the task's +/// default batch. PQL and supplied examples use the same policy. +pub fn rows_one_task_call_scores(asked: Option, task: BatchTask) -> u64 { + asked.unwrap_or(match task { + BatchTask::Ordinary => PREDICT_ROWS, + BatchTask::TemporalLink => 200, + }) } /// A task's kind, as the engine names it. @@ -911,14 +911,9 @@ impl RunMode { /// The run mode a context size asks for, as the client chooses it /// (`_run_mode_for`, with `num_context_samples`' bound of /// `_validate_materialize_args`): none or up to a fast run's examples, a -/// fast run; up to `most`, a best one. Below 1 is `INVALID_REQUEST`; -/// past `most` (`max_relational_context_rows`), `UNSUPPORTED_REQUEST`, -/// naming the setting that raises it. -pub fn run_mode( - asked: Option, - given: Option<&str>, - most: u64, -) -> Result { +/// fast run; past it, a best one, gathering as many as are asked. Below 1 +/// is `INVALID_REQUEST`. +pub fn run_mode(asked: Option, given: Option<&str>) -> Result { // A refusal names the size as the caller gave it, its JSON text. let named = |refused: CoreError| match given { Some(given) => refused.with_field("given", Detail::text(given)), @@ -942,15 +937,7 @@ pub fn run_mode( } match value { Some(n) if n <= RunMode::Fast.context_examples() as u128 => Ok(RunMode::Fast), - Some(n) if n <= u128::from(most) => Ok(RunMode::Best), - _ => Err(MAX_RELATIONAL_CONTEXT_ROWS.named(named( - CoreError::new( - Sentence::NumContextSamplesPastMost, - "num_context_samples is past the most context a request carries", - ) - .with_field("field", Detail::text("num_context_samples")) - .with_field("limit", Detail::Scalar(Scalar::UInt64(most))), - ))), + _ => Ok(RunMode::Best), } } @@ -1383,51 +1370,25 @@ mod tests { #[test] fn the_run_mode_is_the_client_s() { - let most = MAX_RELATIONAL_CONTEXT_ROWS.default; - assert_eq!(run_mode(None, None, most).unwrap(), RunMode::Fast); + assert_eq!(run_mode(None, None).unwrap(), RunMode::Fast); assert_eq!( - run_mode(int(false, Some(1)), None, most).unwrap(), + run_mode(int(false, Some(1_000)), None).unwrap(), RunMode::Fast ); - assert_eq!( - run_mode(int(false, Some(1_000)), None, most).unwrap(), - RunMode::Fast - ); - assert_eq!( - run_mode(int(false, Some(1_001)), None, most).unwrap(), - RunMode::Best - ); - assert_eq!( - run_mode(int(false, Some(10_000)), None, most).unwrap(), - RunMode::Best - ); - for (asked, code) in [ - (int(false, Some(0)), ErrorCode::InvalidRequest), - (int(true, Some(5)), ErrorCode::InvalidRequest), - (int(false, Some(10_001)), ErrorCode::UnsupportedRequest), - (int(false, None), ErrorCode::UnsupportedRequest), + // Past a fast run's examples, a best run, however many are asked. + for asked in [ + int(false, Some(1_001)), + int(false, Some(200_000)), + int(false, None), ] { + assert_eq!(run_mode(asked, None).unwrap(), RunMode::Best); + } + for asked in [int(false, Some(0)), int(true, Some(5))] { assert_eq!( - run_mode(asked, None, most).unwrap_err().code, - code, - "{asked:?}" + run_mode(asked, None).unwrap_err().code, + ErrorCode::InvalidRequest ); } - let past = run_mode(int(false, Some(10_001)), None, most).unwrap_err(); - assert_eq!( - past.fields["setting"], - Detail::text("max_relational_context_rows") - ); - assert_eq!( - past.fields["env"], - Detail::text("SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS") - ); - // A raised limit admits more, as a best run. - assert_eq!( - run_mode(int(false, Some(200_000)), None, 200_000).unwrap(), - RunMode::Best - ); - assert!(run_mode(int(false, Some(200_001)), None, 200_000).is_err()); } #[test] diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index 5b55b8982..80afdb7b0 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -79,6 +79,12 @@ pub const RESERVED: [&str; 2] = [INSTANCE_ID, ANCHOR_TIME]; const NO_ANCHOR: i64 = -9_223_372_036_854_775_807; /// The largest integer a JSON number carries exactly in every reader. const JSON_SAFE: i128 = 9_007_199_254_740_991; +/// The bytes a payload's lagged columns are held to before any is computed +/// (`_MAX_SIZE`, 30 MiB), a bound on the work `lag_timesteps` asks for: a +/// call itself is held to no size, and is asked in halves where the +/// endpoint refuses it as too large. +pub const PAYLOAD_BYTES: usize = 30 * 1024 * 1024; + /// One column of a sampled table: as sampled, or a text feature as the /// reference's sampler sends it, each value its words /// (`sdm_engine::words`), written as a `stringlist` column. @@ -223,60 +229,14 @@ pub struct Sampled { pub float_text: fn(f64) -> String, } -/// A payload built: its JSON text, how many of its bytes a call sends (the -/// members only this client reads left out), and how many context examples -/// answered the question and were taken out. +/// A payload built: its JSON text, the bytes the request sends, and how +/// many context examples answered the question and were taken out. #[derive(Debug, Clone)] pub struct Built { pub bytes: Vec, - pub model_bytes: usize, pub answering: usize, } -/// A built payload held to the most bytes one call takes (`limit`, the -/// `max_request_bytes` setting), counted as the bytes the payload is -/// written as and sent, UTF-8 (register PD-2), not the reference's -/// `json.dumps` count, which escapes every character past ASCII. A payload -/// past it is asked as smaller calls when it is predicted -/// ([`crate::batching`]), so it is refused here only where no call is small -/// enough: where the context, sent with every call, makes a call of one row -/// past the limit on its own. -pub fn check_bytes(built: &Built, limit: u64) -> Result<(), CoreError> { - if built.model_bytes as u64 <= limit { - return Ok(()); - } - let payload = sdm_wire::json::read(&built.bytes) - .map_err(|_| internal("a built payload does not read back"))?; - let rows = crate::predict::predict_rows(&payload).unwrap_or(0); - // The bytes a call carrying the first `n` rows sends. - let sent = |n: usize| -> Result { - let mut call = crate::batching::predict_slice(&payload, 0..n)?; - call.remove("client_generated_keys"); - call.remove("client_analysis_delta"); - Ok(call.to_bytes().len() as u64) - }; - let one = sent(rows.min(1))?; - if rows > 1 && one <= limit { - return Ok(()); - } - let count = |n: u64| Detail::Scalar(sdm_contract::Scalar::UInt64(n)); - Err(sdm_contract::limits::MAX_REQUEST_BYTES.named( - CoreError::new( - Sentence::PayloadNoSmallerCall, - "the context alone makes a call larger than one call takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("bytes", count(one)) - .with_field("context_bytes", count(sent(0)?)) - .with_field("limit", count(limit)) - .with_field( - "batch_too_large_to_send", - Detail::Scalar(sdm_contract::Scalar::Bool(true)), - ) - .with_field("rows", count(rows as u64)), - )) -} - fn invalid(sentence: Sentence, message: &str, column: &str) -> CoreError { CoreError::new(sentence, message).with_field("column", Detail::text(column)) } @@ -1528,17 +1488,6 @@ fn build_mode( num_prediction_examples: sampled.num_test as u64, }; let model = "kumo-relational".to_owned(); - // Only this small state-only member is counted separately; never copy or - // reparse the potentially 30MiB table data merely to measure its wire size. - let client_only_bytes = if client_generated_keys.is_empty() { - 0 - } else { - serde_json::to_vec(&client_generated_keys) - .expect("string map serializes") - .len() - + br#""client_generated_keys":"#.len() - + 1 // comma separating the member from model-facing members - }; let bytes = if link.is_some() { StatePayload { client_analysis_delta: client_analysis_delta.clone(), @@ -1599,19 +1548,8 @@ fn build_mode( } .to_bytes() }; - // The bytes a call sends of it, which the limit counts (PD-2): all but - // the members only this client reads. - let analysis_bytes = client_analysis_delta.as_ref().map_or(0, |delta| { - serde_json::to_vec(delta) - .expect("typed delta serializes") - .len() - + br#""client_analysis_delta":"#.len() - + 1 - }); - let model_bytes = bytes.len() - client_only_bytes - analysis_bytes; Ok(Built { bytes, - model_bytes, answering: answered.len(), }) } diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index 4b57b9a04..0810c1345 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -49,9 +49,6 @@ use arrow_array::{ Array, ArrayRef, BooleanArray, Float64Array, Int64Array, RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema}; -use sdm_contract::limits::{ - Limits, MAX_RELATIONAL_CONTEXT_ROWS, MAX_REQUEST_BYTES, MAX_TABULAR_CONTEXT_ROWS, -}; use sdm_contract::{CoreError, Detail, Scalar, Sentence}; use sdm_wire::json::{Value, is_integer_token}; use sdm_wire::response::{PredictionResponse, Refused, integer_text, numeric_text}; @@ -309,12 +306,6 @@ fn instance_table<'a>(payload: &'a Value, half: &str) -> Option<(Vec<&'a str>, & crate::batching::table_rows(payload.get(half)?.get("instance_table")?) } -/// The rows of one half, counted as `Request._rows` counts them: none -/// where the half cannot be read. -fn rows_of(payload: &Value, half: &str) -> usize { - instance_table(payload, half).map_or(0, |(_, rows)| rows.len()) -} - /// The rows one call scores (`_envelope_rows(payload, 'predict')`): its /// predict half's rows, none where the half cannot be read. pub fn predict_rows(payload: &Value) -> Option { @@ -383,7 +374,6 @@ pub fn prepare( model: &str, asked: &[String], defaulted: bool, - limits: &Limits, ) -> Result, CoreError> { let Some(payload) = payloads.first() else { return Err(CoreError::new(Sentence::Internal, "a request of no call")); @@ -444,12 +434,6 @@ pub fn prepare( }); check_shapes(spec, kind, anchored, linked, tables)?; check_task_outputs(spec, kind, &outputs, true)?; - let rows = payloads - .iter() - .map(|p| rows_of(p, "context")) - .max() - .unwrap_or(0); - check_capacity(spec, rows, false, limits)?; if spec.id == TABULAR { // The tabular model takes no relational request this build sends. return Err(not_yet("model")); @@ -850,38 +834,6 @@ fn is_chosen(value: &Value) -> bool { } } -/// The bytes of the call `body` writes, as the host sends them, unchanged, -/// counted on these bytes, the envelope included, which are the bytes sent -/// (register PD-2, fix round 1): refused past `limit` (`max_request_bytes`) -/// before the model is called ([`call_past_bytes`]). A call that scores rows -/// is split to fit before it gets here ([`crate::batching`]), so this -/// refuses a call that cannot be made smaller. -pub fn call_bytes(call: &Value, limit: u64) -> Result, CoreError> { - let bytes = call.to_bytes(); - if bytes.len() as u64 > limit { - return Err(call_past_bytes(call, bytes.len() as u64, limit)); - } - Ok(bytes) -} - -/// The refusal of `call`, `bytes` long, past `limit`, the most bytes one call -/// takes (`max_request_bytes`): it names the call's bytes, those of its -/// context, which every call carries, and the setting that raises the limit. -pub fn call_past_bytes(call: &Value, bytes: u64, limit: u64) -> CoreError { - let context = crate::batching::predict_slice(call, 0..0) - .map_or(bytes, |context| context.to_bytes().len() as u64); - MAX_REQUEST_BYTES.named( - CoreError::new( - Sentence::CallPastEndpoint, - "the call is larger than one call takes", - ) - .with_field("field", Detail::text("call")) - .with_field("bytes", Detail::Scalar(Scalar::UInt64(bytes))) - .with_field("context_bytes", Detail::Scalar(Scalar::UInt64(context))) - .with_field("limit", Detail::Scalar(Scalar::UInt64(limit))), - ) -} - /// The entity a row is about, as the result gives it: as the reference's /// `as_declared` reads its value back as the key's dtype, or as written /// where it does not read or no dtype is known. @@ -2332,58 +2284,9 @@ fn check_task_outputs( } Ok(()) } -/// A direct request's context, as it is sent to `model`, held to the most -/// that model reads ([`check_capacity`]); a model this build does not -/// carry is checked where it is bound. -pub fn check_direct_context(model: &str, rows: usize, limits: &Limits) -> Result<(), CoreError> { - match MODELS.iter().find(|spec| spec.id == model) { - Some(spec) => check_capacity(spec, rows, true, limits), - None => Ok(()), - } -} - -/// The context a request carries held to the most its model reads -/// (`max_relational_context_rows` or `max_tabular_context_rows`), a -/// refusal naming the setting that raises it. -fn check_capacity( - spec: &Spec, - rows: usize, - direct: bool, - limits: &Limits, -) -> Result<(), CoreError> { - let (setting, most) = if spec.id == TABULAR { - (&MAX_TABULAR_CONTEXT_ROWS, limits.max_tabular_context_rows) - } else { - ( - &MAX_RELATIONAL_CONTEXT_ROWS, - limits.max_relational_context_rows, - ) - }; - if (direct || spec.id == RELATIONAL) && rows as u64 > most { - return Err(setting.named( - unsupported( - if direct { - Sentence::DirectContextLimit - } else { - Sentence::ContextPastModel - }, - spec.id, - "the request's context is larger than the model reads", - ) - .with_field("context_rows", count(rows)) - .with_field("context_rows_max", Detail::Scalar(Scalar::UInt64(most))), - )); - } - Ok(()) -} - /// Model preflight for live direct request facts, before conversion or empty handling. -/// The context is held to `limits` where they are given; a decision checked -/// again later is the same decision whatever the limits, which the call that -/// sends the request holds the context to once more. pub fn bind_direct( input: sdm_contract::direct::DirectCallFacts, - limits: Option<&Limits>, ) -> Result { let facts = input.clone(); let spec = MODELS @@ -2417,9 +2320,6 @@ pub fn bind_direct( input.tables, )?; check_task_outputs(spec, &input.kind, &outputs, false)?; - if let Some(limits) = limits { - check_capacity(spec, input.context_rows, true, limits)?; - } Ok(sdm_contract::direct::BoundDirectCall { facts, model: input.model, diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index e37935d5c..cb1f1c17c 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -62,9 +62,7 @@ const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 429, 504]; /// The call `call` (the core's own JSON) as the endpoint's envelope, the /// bytes a host posts, or `ServingPastLimit` when it is more than -/// `max_request_bytes`, counted on the actual UTF-8 body. A call that scores -/// rows is split to fit before it gets here ([`crate::batching`]), so the -/// refusal names the context every call carries. +/// `max_request_bytes`, counted on the actual UTF-8 body. pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result, CoreError> { let read = json::read(call).map_err(|e| { CoreError::new( @@ -73,18 +71,17 @@ pub fn envelope(endpoint: &str, max_request_bytes: u64, call: &[u8]) -> Result max_request_bytes { + return Err(past_limit(endpoint, max_request_bytes, size)); + } + Ok(out) } -/// The envelope of `call`, as [`envelope`] writes it, and where it is more -/// than `max_request_bytes`, its refusal (`ServingPastLimit`), naming the -/// context's bytes too; a value with no JSON form is refused outright. -pub fn weigh( - endpoint: &str, - max_request_bytes: u64, - call: &Value, -) -> Result<(Vec, Option), CoreError> { +/// The envelope's bytes of `call`, as [`envelope`] writes them, refused +/// where a value has no JSON form. +pub fn wrapped(call: &Value) -> Result, CoreError> { if !call.all_finite() { return Err(CoreError::new( Sentence::ServingValueNotJson, @@ -92,30 +89,9 @@ pub fn weigh( ) .with_field("reason", Detail::text("out_of_range_float"))); } - let out = wrapped(call)?; - let size = out.len() as u64; - if size <= max_request_bytes { - return Ok((out, None)); - } - let context = crate::batching::predict_slice(call, 0..0) - .and_then(|context| wrapped(&context)) - .map_or(size, |context| context.len() as u64); - let past = CoreError::new( - Sentence::ServingPastLimit, - "the call is larger than the serving endpoint takes", - ) - .with_field("endpoint", Detail::text(endpoint)) - .with_field("bytes", count(size)) - .with_field("context_bytes", count(context)) - .with_field("limit", count(max_request_bytes)); - Ok((out, Some(past))) -} - -/// The envelope's bytes of a readable call. -fn wrapped(read: &Value) -> Result, CoreError> { // Value's writer always produces UTF-8; this conversion performs no // number decoding, integer narrowing or host-dependent formatting. - let payload = String::from_utf8(read.to_bytes()).map_err(|_| { + let payload = String::from_utf8(call.to_bytes()).map_err(|_| { CoreError::new( Sentence::Internal, "the wire JSON writer returned invalid UTF-8", @@ -131,6 +107,18 @@ fn wrapped(read: &Value) -> Result, CoreError> { .to_bytes()) } +/// The refusal of a call whose envelope is `size` bytes, past the +/// `max_request_bytes` the endpoint takes. +pub fn past_limit(endpoint: &str, max_request_bytes: u64, size: u64) -> CoreError { + CoreError::new( + Sentence::ServingPastLimit, + "the call is larger than the serving endpoint takes", + ) + .with_field("endpoint", Detail::text(endpoint)) + .with_field("bytes", count(size)) + .with_field("limit", count(max_request_bytes)) +} + /// The served model's answer in the endpoint's reply `body` (a 2xx), a JSON /// object, or the error the old client raises for it. pub fn answer(endpoint: &str, body: &[u8]) -> Result { diff --git a/rust/crates/sdm-request/src/tabular.rs b/rust/crates/sdm-request/src/tabular.rs index 6f4a93df3..8bcdddafb 100644 --- a/rust/crates/sdm-request/src/tabular.rs +++ b/rust/crates/sdm-request/src/tabular.rs @@ -1243,33 +1243,3 @@ pub fn decide_materialized_task( }, } } - -/// The context a materialized request asks for held to the most a request -/// carries (`max_tabular_context_rows`), without reading the candidate -/// frame population: past it, `UNSUPPORTED_REQUEST` naming the count as it -/// was given and the setting that raises the limit. A count past `u128` is -/// past any limit, and one below zero is under it. -pub fn check_materialized_cap( - input: sdm_contract::tabular::CheckTabularCap, - limits: &sdm_contract::limits::Limits, -) -> Result { - let most = limits.max_tabular_context_rows; - if let Some(given) = input.requested { - let past = match given.parse::() { - Ok(count) => count > u128::from(most), - Err(_) => !given.starts_with('-'), - }; - if past { - return Err(sdm_contract::limits::MAX_TABULAR_CONTEXT_ROWS.named( - CoreError::new( - Sentence::NumContextSamplesPastMost, - "num_context_samples is past the most context a request carries", - ) - .with_field("field", Detail::text("num_context_samples")) - .with_field("given", Detail::text(given)) - .with_field("limit", Detail::Scalar(Scalar::UInt64(most))), - )); - } - } - Ok(sdm_contract::tabular::CheckedTabularCap { limit: most }) -} diff --git a/rust/crates/sdm-request/tests/batch_limits.rs b/rust/crates/sdm-request/tests/batch_limits.rs index 209e4025a..cbaa053b8 100644 --- a/rust/crates/sdm-request/tests/batch_limits.rs +++ b/rust/crates/sdm-request/tests/batch_limits.rs @@ -2,67 +2,19 @@ // All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use sdm_contract::limits::Limits; use sdm_request::options::{BatchTask, rows_one_call_scores, rows_one_task_call_scores}; #[test] -fn temporal_links_cannot_raise_the_model_limit() { - let limits = Limits::default(); - for (asked, expected) in [ - (None, 200), - (Some(100), 100), - (Some(199), 199), - (Some(200), 200), - (Some(201), 200), - (Some(1000), 200), - (Some(u64::MAX), 200), +fn each_task_batches_by_its_default_and_the_size_asked_is_taken_as_is() { + for (task, default) in [ + (BatchTask::TemporalLink, 200), + (BatchTask::Ordinary, 10_000), ] { - assert_eq!( - rows_one_task_call_scores(asked, BatchTask::TemporalLink, &limits), - expected - ); + assert_eq!(rows_one_task_call_scores(None, task), default); + for asked in [1, 100, 20_000, u64::MAX] { + assert_eq!(rows_one_task_call_scores(Some(asked), task), asked); + } } -} - -#[test] -fn ordinary_and_decomposed_tasks_keep_their_existing_limit() { - let limits = Limits::default(); - for (asked, expected) in [ - (None, 10_000), - (Some(100), 100), - (Some(201), 201), - (Some(1000), 1000), - (Some(u64::MAX), 10_000), - ] { - assert_eq!( - rows_one_task_call_scores(asked, BatchTask::Ordinary, &limits), - expected - ); - assert_eq!(rows_one_call_scores(asked, &limits), expected); - } -} - -#[test] -fn a_raised_limit_raises_each_task_s_call_and_rows_per_call_still_lowers_it() { - let limits = Limits { - max_relational_predict_rows: 50_000, - max_temporal_link_predict_rows: 1_000, - ..Limits::default() - }; - assert_eq!( - rows_one_task_call_scores(None, BatchTask::Ordinary, &limits), - 50_000 - ); - assert_eq!( - rows_one_task_call_scores(Some(20_000), BatchTask::Ordinary, &limits), - 20_000 - ); - assert_eq!( - rows_one_task_call_scores(None, BatchTask::TemporalLink, &limits), - 1_000 - ); - assert_eq!( - rows_one_task_call_scores(Some(u64::MAX), BatchTask::TemporalLink, &limits), - 1_000 - ); + assert_eq!(rows_one_call_scores(None), 10_000); + assert_eq!(rows_one_call_scores(Some(50_000)), 50_000); } diff --git a/rust/crates/sdm-request/tests/payload_bytes.rs b/rust/crates/sdm-request/tests/payload_bytes.rs deleted file mode 100644 index bd014be60..000000000 --- a/rust/crates/sdm-request/tests/payload_bytes.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -// All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! The payload limit counts the bytes the request sends (register PD-2, -//! ): the builder's UTF-8 bytes, as written, so a payload of exactly -//! the limit is taken and one byte more refused, where the reference -//! counted `json.dumps`' text, which writes each character past ASCII as a -//! six-byte escape. Held at the limit with ASCII text and with text of two-, -//! three- and four-byte characters, which the reference's count would -//! have taken for three to twelve times its size. -//! -//! The payloads here score one row, so a payload past the limit cannot be -//! split into smaller calls, and the context alone is past it. - -use std::sync::Arc; - -use arrow_array::{ArrayRef, Int64Array, StringArray}; -use sdm_contract::limits::MAX_REQUEST_BYTES; -use sdm_contract::ops::Stype; -use sdm_contract::{ErrorCode, Sentence}; -use sdm_engine::labels::Labels; -use sdm_request::options::{Kind, Options, RunMode}; -use sdm_request::payload::{Built, Sampled, SampledColumn, SampledTable, build, check_bytes}; - -const PAYLOAD_BYTES: usize = MAX_REQUEST_BYTES.default as usize; - -/// The payload built and held to the default limit. -fn checked(sampled: &Sampled) -> Result { - let built = build(sampled)?; - check_bytes(&built, MAX_REQUEST_BYTES.default)?; - Ok(built) -} - -const ANCHOR: i64 = 1_709_251_200_000_000_000; - -/// One entity, one context example and one to predict, each carrying -/// `note` as a categorical text feature. -fn sampled(note: &str) -> Sampled { - let column = |a: ArrayRef| SampledColumn::Arrow(a); - Sampled { - num_train: 1, - num_test: 1, - anchors: vec![ANCHOR, ANCHOR], - kind: Kind::Regression, - labels: Labels::Float(vec![1.0]), - windowed: true, - tables: vec![SampledTable { - name: "users".into(), - primary_key: Some("user_id".into()), - batch: vec![0, 1], - columns: vec![ - ( - "user_id".into(), - column(Arc::new(Int64Array::from(vec![0, 1]))), - ), - ( - "note".into(), - column(Arc::new(StringArray::from(vec![note, note]))), - ), - ], - stypes: vec![("note".into(), Stype::Categorical)], - }], - links: vec![], - entity_table: "users".into(), - source_query: String::new(), - run_mode: RunMode::Fast, - use_prediction_time: false, - return_embeddings: false, - options: Options::default(), - statistic: None, - top_k: None, - lags: vec![], - float_text: |x| format!("{x:?}"), - } -} - -/// The payload's length with `n` copies of `unit` in each note. -fn length(unit: &str, n: usize) -> Result { - checked(&sampled(&unit.repeat(n))) - .map(|built| built.bytes.len()) - .map_err(|e| e.code) -} - -#[test] -fn the_limit_counts_the_bytes_sent_at_the_limit() { - for unit in ["a", "\u{e9}", "\u{20ac}", "\u{1f600}"] { - let width = unit.len(); - // Each note is written twice (the context's row and the predicted - // row), so the payload grows by twice a unit's bytes per unit. - let base = length(unit, 0).unwrap(); - let step = length(unit, 1).unwrap() - base; - assert_eq!(step, 2 * width, "{unit:?}"); - // The most units whose payload is within the limit, and the slack - // left, which ASCII padding of the bytes that remain fills. - let n = (PAYLOAD_BYTES - base) / step; - let slack = PAYLOAD_BYTES - base - n * step; - let at = format!("{}{}", unit.repeat(n), "a".repeat(slack / 2)); - let taken = checked(&sampled(&at)).expect("a payload within the limit is taken"); - let exact = base + n * step + 2 * (slack / 2); - assert_eq!(taken.bytes.len(), exact, "{unit:?}"); - assert!(exact <= PAYLOAD_BYTES && PAYLOAD_BYTES - exact <= 1); - // The bytes counted are the bytes written: UTF-8, each character - // its own bytes, not an escape. - let text = String::from_utf8(taken.bytes.clone()).unwrap(); - assert!(!text.contains("\\u"), "{unit:?}"); - // Two units more (four bytes or more past it) are refused, as the - // reference refuses a payload past its limit. - let past = format!("{}{}", unit.repeat(n + 2), "a".repeat(slack / 2)); - let refused = checked(&sampled(&past)).expect_err("a payload past the limit"); - assert_eq!(refused.code, ErrorCode::InvalidRequest, "{unit:?}"); - assert_eq!(refused.sentence, Sentence::PayloadNoSmallerCall, "{unit:?}"); - } - // With ASCII the payload reaches the limit exactly, one byte step past - // it refused: the count is of bytes, to the byte. - let base = length("a", 0).unwrap(); - let fill = |extra: usize| { - let half = (PAYLOAD_BYTES - base + extra) / 2; - checked(&sampled(&"a".repeat(half))) - }; - let even = (PAYLOAD_BYTES - base).is_multiple_of(2); - let at = fill(0).expect("at the limit"); - assert_eq!( - at.bytes.len(), - if even { - PAYLOAD_BYTES - } else { - PAYLOAD_BYTES - 1 - } - ); - assert_eq!(fill(2).unwrap_err().code, ErrorCode::InvalidRequest); -} - -#[test] -fn a_raised_limit_takes_a_payload_past_the_default_and_names_its_setting_past_it() { - let base = length("a", 0).unwrap(); - let past = sampled(&"a".repeat((PAYLOAD_BYTES - base) / 2 + 8)); - let built = build(&past).unwrap(); - let refused = check_bytes(&built, MAX_REQUEST_BYTES.default).unwrap_err(); - assert_eq!( - refused.fields["setting"], - sdm_contract::Detail::text("max_request_bytes") - ); - assert_eq!( - refused.fields["env"], - sdm_contract::Detail::text("SDM_CLIENT_MAX_REQUEST_BYTES") - ); - let context = match &refused.fields["context_bytes"] { - sdm_contract::Detail::Scalar(sdm_contract::Scalar::UInt64(n)) => *n, - other => panic!("{other:?}"), - }; - assert!(context < built.model_bytes as u64); - check_bytes(&built, 2 * MAX_REQUEST_BYTES.default).expect("a raised limit takes it"); -} diff --git a/rust/crates/sdm-testkit/src/host.rs b/rust/crates/sdm-testkit/src/host.rs index 3f4362876..722c21124 100644 --- a/rust/crates/sdm-testkit/src/host.rs +++ b/rust/crates/sdm-testkit/src/host.rs @@ -587,7 +587,6 @@ fn validate( "schema": serde_json::from_slice::(&schema_bytes)?, }); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::ValidateQuery(ValidateQuery { @@ -1107,7 +1106,6 @@ fn describe_on( "check_keys": spec.check_keys, }); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::DescribeSource(DescribeSource { @@ -1265,7 +1263,6 @@ fn edit_schema( "edit": serde_json::from_slice::(&edit_bytes)?, })); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::EditSchema(EditSchema { @@ -1782,7 +1779,6 @@ fn start_materialize( "seed": serde_json::to_value(&seed)?, })); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: as_op(Materialize { @@ -2128,7 +2124,6 @@ fn predict( }, })); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts()?, operation: Operation::Predict(Predict { diff --git a/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs b/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs index d8b6cb191..f8e6b597d 100644 --- a/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs +++ b/rust/crates/sdm-testkit/src/host_replay_validation_tests.rs @@ -44,7 +44,6 @@ fn description_recording(case: &Case, extra_fetch: bool) -> Recording { .as_ref() .unwrap(); let start = ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: facts().unwrap(), operation: Operation::DescribeSource(DescribeSource { diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index fbaffb33d..ebe290fc2 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -72,7 +72,6 @@ fn start(request: &Value) -> HostRun { fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Fetch { @@ -572,7 +571,6 @@ fn keyed(connection: &Connection, request: &Value, keys: RecordBatch) -> (Record fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Keyed { diff --git a/rust/crates/sdm-testkit/tests/neighbors.rs b/rust/crates/sdm-testkit/tests/neighbors.rs index 7849db5db..8e0819524 100644 --- a/rust/crates/sdm-testkit/tests/neighbors.rs +++ b/rust/crates/sdm-testkit/tests/neighbors.rs @@ -170,7 +170,6 @@ fn answered(connection: &Connection, request: DataRequest, seeds: RecordBatch) - fallback: None, }; let message = encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::Test(TestOperation::Keyed { diff --git a/rust/crates/sdm-testkit/tests/warehouse_replay.rs b/rust/crates/sdm-testkit/tests/warehouse_replay.rs index c635a37dd..9729083e7 100644 --- a/rust/crates/sdm-testkit/tests/warehouse_replay.rs +++ b/rust/crates/sdm-testkit/tests/warehouse_replay.rs @@ -75,7 +75,6 @@ fn core(provider: &Value) -> Value { fn recording(leg: &str, core: &Value) -> Value { let spec = &core["describe_source"]; let message = encode(&ControlMessage::Start(Box::new(StartRun { - limits: Default::default(), contract: CONTRACT_VERSION, facts: host::facts().unwrap(), operation: Operation::DescribeSource(DescribeSource { diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index b1edeb94f..eb2d67eb4 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -152,30 +152,22 @@ views, and a notebook's own session are in ## Which rows get scored -`materialize(indices=[...])` names the entities. It gathers each one's -neighbourhood, checks every limit before sending, and refuses with the limit it -would break rather than letting the endpoint return a 500. +`materialize(indices=[...])` names the entities and gathers each one's +neighbourhood. The client holds a request to no size of its own: it sends its +best attempt, and a call the endpoint refuses as too large, or times out on, +is asked again in halves, each with the whole context. A refusal of one row, +or a `422`, is the endpoint's, shown as it said it. | Limit | Value | |---|---| | context rows, by default | `1,000` | -| context rows, with `num_context_samples` | `10,000` | -| rows scored at once | `10,000` | -| rows scored for a link prediction | `200` | -| bytes one call takes | `31,457,280` | | classes | `10` | -The context rows with `num_context_samples`, the rows scored at once, the rows scored for a link prediction and the bytes one call takes are the serving layer's defaults, and a deployment may be configured to take more. -Each is a setting: pass `limits=Limits(...)` to `SDMClient` and to `materialize()`, or set its environment variable, such as `SDM_CLIENT_MAX_RELATIONAL_CONTEXT_ROWS`. -The gathered graph may carry any number of tables and hops. - -**More rows than one call scores are not refused.** They are gathered in -batches of at most that many, and `predict()` sends one call per batch and -returns one result, whose rows are numbered across the whole request, so -`to_pandas()` gives the indices back in the order they were asked for. The -10,000 is the deployment's `TFM_MAX_TEST_ROWS`, not a property of the -protocol; pass `rows_per_call=` to ask in smaller batches, which is what a -graph too wide for a full one needs. It only goes down. +**Rows are scored in batches.** They are gathered in batches of 10,000 (200 +for a link prediction), and `predict()` sends one call per batch and returns +one result, whose rows are numbered across the whole request, so `to_pandas()` +gives the indices back in the order they were asked for. Pass +`rows_per_call=` to ask in batches of another size. Every batch is built before the first is sent, so one `materialize()` gathers at most 100,000 entities, in at most 100 calls. Both are this client's own: @@ -184,9 +176,7 @@ in parts is not batched at all: a ranking is already a call per candidate and a forecast one per step, and batching a part would send that many times as many calls. -**A call past the most bytes one takes is split, not refused.** It is sent as -several smaller calls, each with the whole context, so only a context too -large on its own is refused, naming its size and the limit. +The gathered graph may carry any number of tables and hops. Gradient explanations follow the model endpoint's supported outputs; the client does not impose a separate one-row or fast-context limit. ## Asking for more than one answer each diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 43cb20021..e9d284042 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -34,7 +34,6 @@ with SDMClient(url='http://localhost:8000') as client: | Relationships | no | | Explanations | yes | | Sessions | yes | -| Context rows | 10,000 | Anything outside that list belongs to the relational model. Asking for `rankings` or a forecast here is refused before a request @@ -110,11 +109,9 @@ guesses that way and names the argument that settles it. ## Which rows get scored `Request(context=..., predict=...)` takes two frames and sends exactly those -rows. It trims neither. A context above 10,000 rows is refused with the count -and the cap rather than quietly dropping rows you named. The cap is a setting -with the serving layer's default: for a deployment configured to take more, -pass `limits=Limits(max_tabular_context_rows=...)` to `SDMClient` and to -`materialize()`, or set `SDM_CLIENT_MAX_TABULAR_CONTEXT_ROWS`. +rows. It trims neither, and holds the context to no size of its own: a +context larger than the endpoint takes is the endpoint's refusal, shown as it +said it. The rows to score need not appear in the labelled half, and usually should not: they are the rows whose answer you do not have. @@ -160,8 +157,8 @@ print(answer.to_pandas()) ``` Twenty rows in, three named, and `request.context_rows` is 17: the scored rows -are held out of their own context. `materialize()` also samples down to the -10,000-row cap. +are held out of their own context. `materialize()` samples the context only +to a `num_context_samples` you pass. On this model the language is cut down to a column of one table as the target: no aggregation, no window, no comparison, no forecast, no ranking, no entity @@ -218,7 +215,6 @@ names what to change. |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | | an output it does not produce | `rankings`, `embeddings`, `scores` | -| a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | Catch `StructuredDataError` to catch all of them. diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index 3d21f086c..b795ebb9a 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -2832,8 +2832,7 @@ "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_custom_nullable_producer_never_takes_arrow_snapshot", "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_float_refusal_keeps_specific_diagnostic", "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_integer_range_after_earlier_column_mutation", - "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_conversion_order_and_nested_options", - "packages/structured-data-models-client/tests/test_configurable_limits.py" + "packages/structured-data-models-client/tests/test_tabular_input_families.py::test_native_conversion_order_and_nested_options" ], "java": [ "java/src/test/java/com/nvidia/sdm/DirectOptionsTest.java#directSchemaAndOptionsAreOwnedAndInferenceTravelsOnPredictAndScore", @@ -3339,8 +3338,7 @@ "packages/structured-data-models-client/tests/test_databricks_serving.py::test_the_ambient_identity_is_used_when_no_client_is_given", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_a_workspace_that_cannot_authenticate_says_how_to_fix_it", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_unusable_timeout_is_refused_at_construction", - "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not", - "packages/structured-data-models-client/tests/test_configurable_limits.py" + "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not" ], "java": [ "java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java#officialSdkAuthenticationRefreshesEveryHeaderWithoutRewritingTheCoreBytes", @@ -3870,8 +3868,7 @@ "python": [ "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_without_the_extra_the_extra_is_named", "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_tabular_answer_is_the_one_the_model_gives_when_called_directly", - "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_session_keeps_the_question_it_pinned", - "packages/structured-data-models-client/tests/test_configurable_limits.py" + "packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py::test_a_session_keeps_the_question_it_pinned" ], "java": [] }, @@ -4749,57 +4746,9 @@ "reason": "Exact class-label identity and presentation are tested here; this does not expand general model dtype admission." } } - }, - { - "id": "configurable-limits", - "apis": { - "python": [ - "Limits", - "SDMClient.limits" - ], - "java": [ - "Limits", - "SdmClient.limits" - ] - }, - "tests": { - "python": [ - "packages/structured-data-models-client/tests/test_configurable_limits.py" - ], - "java": [ - "java/src/test/java/com/nvidia/sdm/LimitsTest.java" - ] - }, - "gate": "local", - "note": "Size limits as settings: defaults, argument and environment precedence, a raised tabular context taken, byte-aware splitting and the refusal of a context too large on its own, through Rust and controlled HTTP exchanges; no live endpoint validation claimed.", - "ownership": { - "paths": [ - "rust/crates/sdm-core/src/limits.rs", - "rust/crates/sdm-request/src/batching.rs", - "rust/crates/sdm-core/src/predict.rs" - ], - "operations": [ - "predict", - "materialize" - ] - }, - "scope": { - "scenarios": { - "refs": [], - "reason": "Focused limit tests define the settings' boundary; no scenario linkage is declared." - }, - "input_paths": { - "refs": [], - "reason": "This group exposes no input-route entry point; its focused tests define the boundary." - }, - "type_contracts": { - "refs": [], - "reason": "This group does not define dtype or semantic-type admission; its tests cover capability behavior." - } - } } ], - "public_api_snapshot_sha256": "9d490ce2c68265be9c2bb679a7ed7215d594bd63fada342929eaa9833f4ea37b", + "public_api_snapshot_sha256": "652bd2757ca1d8ed12360b795a5c7f80ce63978210306cee1319532503f9d031", "api_inventory": { "python": { "sdm_api.Endpoint": { @@ -5220,13 +5169,7 @@ ] }, "sdm_client.PredictiveQuery.materialize": { - "capability": "direct-dtypes-options", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", - "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." - } - ] + "capability": "direct-dtypes-options" }, "sdm_client.Relationship": { "capability": "schema-edits" @@ -5307,13 +5250,7 @@ "capability": "direct-dtypes-options" }, "sdm_client.SDMClient": { - "capability": "direct-dtypes-options", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", - "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." - } - ] + "capability": "direct-dtypes-options" }, "sdm_client.SDMClient.capabilities": { "capability": "readiness-discovery" @@ -5325,25 +5262,13 @@ "capability": "sessions" }, "sdm_client.SDMClient.for_databricks_serving": { - "capability": "public-serving-refusals", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", - "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." - } - ] + "capability": "public-serving-refusals" }, "sdm_client.SDMClient.health_ready": { "capability": "readiness-discovery" }, "sdm_client.SDMClient.in_process": { - "capability": "in-process-models", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", - "behavior": "The limits argument holds the requests made through it to the settings it names; a raised tabular context limit is taken where the default refuses, naming the setting." - } - ] + "capability": "in-process-models" }, "sdm_client.SDMClient.models": { "capability": "readiness-discovery" @@ -6229,69 +6154,6 @@ "behavior": "Existing public informational policy constants remain available with the engine cohort budgets and reserved labels; assigning them is not a tuning API." } ] - }, - "sdm_client.Limits": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", - "behavior": "A setting given comes before its environment variable, which comes before the default, as resolved by the core." - } - ] - }, - "sdm_client.Limits.max_relational_context_rows": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_setting_given_that_is_not_a_whole_number_is_refused", - "behavior": "Each setting is a keyword of Limits taking a whole number of at least 1, read otherwise from SDM_CLIENT_ and its name in capitals." - } - ] - }, - "sdm_client.Limits.max_relational_predict_rows": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_raised_relational_predict_limit_and_a_lowered_one_set_the_calls", - "behavior": "A lowered relational predict limit sets the rows each call scores." - } - ] - }, - "sdm_client.Limits.max_temporal_link_predict_rows": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_setting_given_that_is_not_a_whole_number_is_refused", - "behavior": "Each setting is a keyword of Limits taking a whole number of at least 1, read otherwise from SDM_CLIENT_ and its name in capitals." - } - ] - }, - "sdm_client.Limits.max_tabular_context_rows": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_materialize_samples_to_a_raised_tabular_limit", - "behavior": "A raised tabular context limit is taken, from the argument before the environment variable." - } - ] - }, - "sdm_client.Limits.max_request_bytes": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_call_past_the_byte_limit_is_split_and_each_part_fits", - "behavior": "A call past the byte limit is split and each part fits." - } - ] - }, - "sdm_client.SDMClient.limits": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "packages/structured-data-models-client/tests/test_configurable_limits.py::test_a_tabular_context_of_200_000_rows_is_sent_when_the_limit_is_raised", - "behavior": "A client raised to 200,000 tabular context rows sends them; the default refuses naming the setting." - } - ] } }, "java": { @@ -8190,177 +8052,6 @@ "behavior": "Java exposes the same informational cohort budgets and reserved names as Python; these constants do not tune engine policy." } ] - }, - "com.nvidia.sdm.ClientOptions$Builder#public com.nvidia.sdm.ClientOptions$Builder limits(com.nvidia.sdm.Limits);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.DatabricksServing#public com.nvidia.sdm.DatabricksServing limits(com.nvidia.sdm.Limits);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.DatabricksServing#public com.nvidia.sdm.Limits limits();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public boolean equals(java.lang.Object);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public int hashCode();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public java.lang.String toString();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public java.util.Map values();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public static com.nvidia.sdm.Limits defaults();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits#public static com.nvidia.sdm.Limits$Builder builder();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits build();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRelationalContextRows(long);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRelationalPredictRows(long);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxRequestBytes(long);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxTabularContextRows(long);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.Limits$Builder#public com.nvidia.sdm.Limits$Builder maxTemporalLinkPredictRows(long);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.MaterializeOptions$Builder#public com.nvidia.sdm.MaterializeOptions$Builder limits(com.nvidia.sdm.Limits);": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aSettingGivenIsKeptByNameAndMustBeAtLeastOne", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] - }, - "com.nvidia.sdm.SdmClient#public com.nvidia.sdm.Limits limits();": { - "capability": "configurable-limits", - "assertions": [ - { - "test": "java/src/test/java/com/nvidia/sdm/LimitsTest.java#aTabularContextPastTheDefaultIsSentWhenTheLimitIsRaised", - "behavior": "Limits are set through the public builders and options, resolved by the core, and held by the client's requests." - } - ] } } }, @@ -8542,36 +8233,6 @@ "base_commit": "ebc5746391d977fd8d9d779423db33ae547d64ee", "record": "type_contracts/arrow-admission-and-inference", "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "api_contracts/configurable-limits", - "reason": "Adds the configurable size limits: Python Limits and SDMClient.limits, Java Limits, SdmClient.limits and the limits options of ClientOptions, DatabricksServing and MaterializeOptions, each with focused behavior tests." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "api_inventory", - "reason": "Adds the public declarations of the configurable size limits, each mapped to the configurable-limits group with a direct assertion." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "public_api_snapshot_sha256", - "reason": "The public snapshot records sdm_client.Limits, SDMClient.limits and the limits arguments of SDMClient, SDMClient.in_process, SDMClient.for_databricks_serving and PredictiveQuery.materialize." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "api_contracts/public-serving-refusals", - "reason": "Links the configurable-limits tests that assert the new limits argument of SDMClient.for_databricks_serving; the group's existing tests and scope are unchanged." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "api_contracts/in-process-models", - "reason": "Links the configurable-limits tests that assert the new limits argument of this group's entry points; the group's existing tests and scope are unchanged." - }, - { - "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", - "record": "api_contracts/direct-dtypes-options", - "reason": "Links the configurable-limits tests that assert the new limits argument of this group's entry points; the group's existing tests and scope are unchanged." } ], "capabilities": [ diff --git a/tests/compatibility/test_contract.py b/tests/compatibility/test_contract.py index aa0c933b0..4652c845b 100644 --- a/tests/compatibility/test_contract.py +++ b/tests/compatibility/test_contract.py @@ -251,7 +251,6 @@ def test_helper_api_registry_has_real_behavior_tests_and_declared_scope(): 'explanation-summaries', 'gradient-explanations', 'column-explanations', - 'configurable-limits', } diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index e5e04b5f1..5c81e999e 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -125,7 +125,6 @@ "InferenceConfig", "KumoRelational", "KumoTabular", - "Limits", "LinkMetadata", "LinkReport", "MetadataReport", @@ -319,19 +318,6 @@ "public_methods": {}, "signature": "()" }, - "Limits": { - "bases": [], - "kind": "class", - "public_attributes": [ - "max_relational_context_rows", - "max_relational_predict_rows", - "max_request_bytes", - "max_tabular_context_rows", - "max_temporal_link_predict_rows" - ], - "public_methods": {}, - "signature": "(max_relational_context_rows: 'int | None' = None, max_relational_predict_rows: 'int | None' = None, max_temporal_link_predict_rows: 'int | None' = None, max_tabular_context_rows: 'int | None' = None, max_request_bytes: 'int | None' = None) -> None" - }, "LinkMetadata": { "bases": [], "kind": "class", @@ -424,7 +410,7 @@ "public_attributes": [], "public_methods": { "labels": "(self, size: 'int', *, anchor_time: 'pd.Timestamp | str | None' = None, random_seed: 'int | None' = 42, max_iterations: 'int' = 10) -> 'pd.DataFrame'", - "materialize": "(self, *, indices: 'Sequence[Any] | None' = None, predict: 'pd.DataFrame | None' = None, num_context_samples: 'int | None' = None, model: 'str | Model | None' = None, candidates: 'Sequence[Any] | None' = None, task: 'str | None' = None, limits: 'Limits | None' = None, **options: 'Any') -> 'Request'" + "materialize": "(self, *, indices: 'Sequence[Any] | None' = None, predict: 'pd.DataFrame | None' = None, num_context_samples: 'int | None' = None, model: 'str | Model | None' = None, candidates: 'Sequence[Any] | None' = None, task: 'str | None' = None, **options: 'Any') -> 'Request'" }, "signature": "(query: 'str', data: 'Schema | Mapping[str, pd.DataFrame]') -> 'None'" }, @@ -484,20 +470,19 @@ "bases": [], "kind": "class", "public_attributes": [ - "limits", "url" ], "public_methods": { "capabilities": "(self, model: 'str | Model') -> 'ModelCapabilities'", "close": "(self) -> 'None'", "fit": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None, holdout: 'Iterable[Any] | None' = None) -> 'FittedModel'", - "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None, limits: 'Limits | None' = None) -> 'SDMClient'", + "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None) -> 'SDMClient'", "health_ready": "(self) -> 'bool'", - "in_process": "(*models: 'Any', device: 'Any' = None, limits: 'Limits | None' = None) -> 'SDMClient'", + "in_process": "(*models: 'Any', device: 'Any' = None) -> 'SDMClient'", "models": "(self) -> 'list[str]'", "predict": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None) -> 'PredictionResult'" }, - "signature": "(url: 'str', api_key: 'str | None' = None, *, verify_ssl: 'bool' = True, timeout: 'float' = 60.0, max_retries: 'int' = 3, limits: 'Limits | None' = None) -> 'None'" + "signature": "(url: 'str', api_key: 'str | None' = None, *, verify_ssl: 'bool' = True, timeout: 'float' = 60.0, max_retries: 'int' = 3) -> 'None'" }, "SDMWarning": { "bases": [ diff --git a/tests/test_skills.py b/tests/test_skills.py index 3d56bc8c8..19a45436b 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2232,7 +2232,6 @@ def test_the_capability_table_is_the_clients_own(name, model_name): } if name == 'tabular': said['Explanations'] = 'explanation' in capabilities.outputs - said['Context rows'] = _defaults()['max_tabular_context_rows'] assert set(shown) == set(said), ( f'{name} shows rows nothing checks: {sorted(set(shown) ^ set(said))}' ) @@ -2257,37 +2256,19 @@ def test_the_capability_table_is_the_clients_own(name, model_name): #: Each row of the limits table in ``skills/relational/SKILL.md``, against the -#: constant or setting it reports. Every key is spoken for in both -#: directions: a limit the client enforces and the skill omits is a refusal -#: an agent cannot see coming, and a row naming no limit is a number nobody is -#: holding to anything. +#: constant it reports. Every key is spoken for in both directions: a limit the +#: client enforces and the skill omits is a refusal an agent cannot see coming, +#: and a row naming no limit is a number nobody is holding to anything. LIMIT_ROWS = { 'context rows, by default': 'context_rows_fast', - 'context rows, with `num_context_samples`': 'max_relational_context_rows', - 'rows scored at once': 'max_relational_predict_rows', - 'rows scored for a link prediction': 'max_temporal_link_predict_rows', - 'bytes one call takes': 'max_request_bytes', 'classes': 'classes_max', } -def _defaults() -> dict[str, int]: - r"""Each setting's default, by name: the core's table.""" - from sdm_client.limits import defaults - - return defaults() - - def test_the_limits_table_is_the_shipped_limits(): from sdm_client.models import RELATIONAL_LIMITS - relational = { - name: default - for name, default in _defaults().items() - if name != 'max_tabular_context_rows' - } - enforced = {**RELATIONAL_LIMITS, **relational} - assert set(LIMIT_ROWS.values()) == set(enforced), ( + assert set(LIMIT_ROWS.values()) == set(RELATIONAL_LIMITS), ( 'the client enforces a limit this table does not account for' ) path = SKILLS / 'relational' / 'SKILL.md' @@ -2300,8 +2281,9 @@ def test_the_limits_table_is_the_shipped_limits(): ) for label, key in LIMIT_ROWS.items(): value = int(only(shown[label][0]).replace(',', '')) - assert value == enforced[key], ( - f'{label!r} says {value}, the client enforces {enforced[key]}' + assert value == RELATIONAL_LIMITS[key], ( + f'{label!r} says {value}, the client enforces ' + f'{RELATIONAL_LIMITS[key]}' ) From 699b56f0fa83d8a14d6e5469840bd850e38ae211 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Sun, 8 Nov 2026 21:05:01 +0000 Subject: [PATCH 4/8] Drop the materialize guards and test the resilient client - Remove the 100,000-entity and 100-call ceilings of one materialize (MOST_ENTITIES, MOST_CALLS, the contract's most_entities and most_calls, their sentences and the Python plumbing). MOST_PARTS, a bound on the calls a question in parts makes, stays. - Remove the lag-size guard: its bound was the 30 MiB body this client no longer holds a call to, so it guarded size, not correctness; a lag count past what times hold is still refused by name. - Split on a timeout only when the answer was slow (a read timeout), not when the connection could not be made. - Tests: split-and-retry on 413, 502, 504 and a timeout in the core and through the Python host; a 422 and a one-row refusal not retried; Databricks still splitting up front at its byte limit; more than 100,000 entities gathered; the stale cap tests and the body-30mib, entity, call and retried-500 contract cases updated to the new behaviour. A 500 is now retried as the other 5xx are. Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 2 +- contract/expected/README.md | 1 + contract/expected/duckdb-python.json | 4 +- contract/expected/postgres-python.json | 4 +- contract/expected/sqlite-python.json | 4 +- contract/tests/test_prediction_batching.py | 25 ++++ docs/reference/limits.md | 5 +- .../java/com/nvidia/sdm/internal/Wording.java | 1 - .../nvidia/sdm/AutomaticBatchSessionTest.java | 4 +- .../com/nvidia/sdm/DatabricksServingTest.java | 10 +- .../nvidia/sdm/TemporalLinkBatchingTest.java | 5 +- .../nvidia/sdm/internal/TransportTest.java | 17 +-- .../src/sdm_client/_rust/errors.py | 6 +- .../src/sdm_client/_rust/frame_materialize.py | 3 - .../src/sdm_client/_rust/materialize.py | 12 +- .../src/sdm_client/_rust/predict.py | 4 +- .../src/sdm_client/core/transport.py | 15 +++ .../sdm_client/materializers/relational.py | 5 - .../tests/tabular_pql_count_cases.py | 3 +- .../test_a_remedy_is_followed_to_the_wire.py | 2 +- .../test_asked_in_the_questions_it_serves.py | 20 +--- .../tests/test_automatic_batch_sessions.py | 6 +- .../tests/test_context_samples.py | 17 +-- .../tests/test_custom_examples.py | 3 +- .../tests/test_databricks_from_source.py | 19 --- .../tests/test_limits_and_absent_outputs.py | 80 +++++-------- .../tests/test_limits_and_sessions.py | 18 +-- .../tests/test_live_relational.py | 4 +- .../tests/test_materialize_options.py | 17 +-- .../tests/test_payload_rules.py | 2 + .../tests/test_pql_for_tabular.py | 8 +- .../tests/test_predict_result.py | 10 +- .../tests/test_query_engine_boundary.py | 74 +++++------- .../tests/test_resilient_calls.py | 108 +++++++++++++++++ .../tests/test_serving_route.py | 30 +++-- .../tests/test_tabular_arrow_indexes.py | 9 +- .../tests/test_tabular_direct_core.py | 7 +- .../tests/test_tabular_ordinary_indexes.py | 18 ++- .../tests/test_tabular_pql_counts.py | 25 +--- .../tests/test_tabular_sorting_scope.py | 4 +- .../tests/test_temporal_link_batching.py | 18 ++- .../tests/test_the_predict_route.py | 50 ++------ .../tests/test_transport_socket.py | 10 +- .../tests/test_wording.py | 8 +- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 38 ------ rust/crates/sdm-contract/src/error.rs | 4 - rust/crates/sdm-contract/src/frames.rs | 2 - rust/crates/sdm-contract/src/ops.rs | 13 -- rust/crates/sdm-contract/src/wording.rs | 13 -- rust/crates/sdm-contract/tests/corpus/mod.rs | 2 - rust/crates/sdm-contract/tests/wording.rs | 1 - rust/crates/sdm-core/src/dispatch.rs | 2 - rust/crates/sdm-core/src/lag.rs | 59 +--------- rust/crates/sdm-core/src/materialize.rs | 77 +----------- .../sdm-core/src/materialize/examples.rs | 1 - .../crates/sdm-core/src/materialize/frames.rs | 2 - rust/crates/sdm-core/src/ranking.rs | 3 +- rust/crates/sdm-core/tests/dispatch.rs | 2 - rust/crates/sdm-core/tests/held_schema.rs | 2 - rust/crates/sdm-core/tests/materialize.rs | 7 -- rust/crates/sdm-core/tests/predict.rs | 111 ++++++++++++------ rust/crates/sdm-core/tests/serving.rs | 33 ++++-- rust/crates/sdm-core/tests/view_captures.rs | 2 - rust/crates/sdm-host/tests/runs.rs | 2 - rust/crates/sdm-request/src/payload.rs | 5 - rust/crates/sdm-testkit/src/host.rs | 2 - .../crates/sdm-testkit/tests/payload_rules.rs | 42 +++---- skills/relational/SKILL.md | 9 +- 69 files changed, 465 insertions(+), 668 deletions(-) create mode 100644 packages/structured-data-models-client/tests/test_resilient_calls.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a6e1ad0a..4d5a4eaa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ The public API snapshot records this version update; publication remains pending ### Removed -- The client's own size limits: the 10,000 context rows, the 30 MiB request body and the 15 tables and 6 hops of a relational request. +- The client's own size limits: the 10,000 context rows, the 30 MiB request body, the 100,000 entities and 100 calls of one `materialize()`, and the 15 tables and 6 hops of a relational request. The client sends its best attempt: a call the endpoint refuses as too large (`413`, a gateway's `502` or `504`) or times out on is asked again in halves, each with the whole context, and a `500` is retried as a `503` is. A `422`, or a refusal of a call of one row, is shown as the endpoint said it. The context is sampled only to a `num_context_samples` the caller passes; through Databricks model serving, a call is still split up front to fit its 16 MB. diff --git a/contract/expected/README.md b/contract/expected/README.md index 240a550c0..af4795e20 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -30,6 +30,7 @@ Their differences from DuckDB follow the fixture loader's explicit mappings in ` | Postgres | NaN-key fixtures refuse at materialize; a negative neighbor bound retains the existing `PROVIDER_ERROR` outcome | `sample.num-hops.seven` and `sample.num-neighbors.seven-hops` now reach predict on every backend: the client sets no most on hops, since the model reads as many as a request carries, where the reference refused past six. +The `multicall.body-30mib` cases now reach predict on every backend: the client holds a call to no size of its own, and a call the endpoint refuses as too large is asked again in halves. These expectations describe the declared fixture mappings; they do not claim that every backend supports identical physical types. `explain.two-entities` now succeeds on all three backends: The client supports explanations for multiple prediction rows. diff --git a/contract/expected/duckdb-python.json b/contract/expected/duckdb-python.json index 3a6ebf933..44f56c90d 100644 --- a/contract/expected/duckdb-python.json +++ b/contract/expected/duckdb-python.json @@ -108,8 +108,8 @@ "metadata.items": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-across-calls": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.101": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "multicall.entities-per-call.10001": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/postgres-python.json b/contract/expected/postgres-python.json index cd2200315..bbbb4baed 100644 --- a/contract/expected/postgres-python.json +++ b/contract/expected/postgres-python.json @@ -108,8 +108,8 @@ "metadata.items-across-calls@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.101@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "multicall.entities-per-call.10001@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/sqlite-python.json b/contract/expected/sqlite-python.json index e186dd9ee..8365c4b16 100644 --- a/contract/expected/sqlite-python.json +++ b/contract/expected/sqlite-python.json @@ -108,8 +108,8 @@ "metadata.items-across-calls@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items-explained@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "metadata.items@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.body-30mib.one-row@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.body-30mib.several-rows@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.body-30mib.one-row@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.body-30mib.several-rows@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.101@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "multicall.entities-per-call.10001@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 7105edb25..dc7706477 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -102,6 +102,17 @@ 'the call: materialize(rows_per_call=N) below the 3 this one holds' ), } +#: The cases this client departs from the reference on, on purpose: it holds a +#: request to no size of its own (no entity, call or body ceiling), so each is +#: gathered where the reference refused it at materialize; and it tries a call +#: the endpoint failed with a 500 again, so a scripted one-off failure is +#: answered where the reference ended the request. +DEPARTED = ( + 'multicall.entities.100001', + 'multicall.calls.101', + *BODY, + *FAILED, +) CASE_IDS = ( 'multicall.rows-per-call-3', 'multicall.fail-call-2', @@ -609,6 +620,8 @@ def test_python_core_answers_each_case_as_the_reference_does( database = case_key.partition('@')[2] or 'duckdb' if database == 'postgres' and not _server(): return + if case_key.split('@')[0] in DEPARTED: + return def check() -> None: run = core_run(database) @@ -656,6 +669,8 @@ def test_python_core_carries_what_the_reference_carries(case_key: str) -> None: case_id, _, database = case_key.partition('@') if database == 'postgres' and not _server(): return + if case_id in DEPARTED: + return def check() -> None: want = vectors()['vectors'][case_key] @@ -780,3 +795,13 @@ def check() -> None: assert not found, f'{case_key}: ' + '; '.join(found) check() + + +@pytest.mark.parametrize('case_id', DEPARTED) +def test_python_core_gathers_and_predicts_what_the_reference_refused( + case_id: str, +) -> None: + r"""The client sets no most on the entities, the calls or the body, and + tries a call failed with a 500 again: each case is answered.""" + got = core_carried(case_id) + assert got['outcome']['result'] == 'success', got['outcome'] diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 756ff3070..1d824aebb 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -26,9 +26,8 @@ Its `row_index` counts across the whole request rather than from zero per call, A session scores those batches against the context it pinned, so the context is uploaded once rather than once per call. `fit()` itself takes a request of one call, since a session pins one context: fit a small request and score the batched one against the handle. -Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once. -One `materialize()` gathers at most 100,000 entities, in at most 100 calls, and the refusal names whichever of the two it met. -Neither counts the parts of a question asked in several: a ranking is one call per candidate and a forecast one per step, bounded by `sdm_client.decompose.MOST_PARTS`, 200. +Every batch is built before the first is sent, each with its own context, so the whole population is held in memory at once; the client sets no most on the entities or the calls. +A question asked in parts is different: a ranking is one call per candidate and a forecast one per step, bounded by `sdm_client.decompose.MOST_PARTS`, 200, a limit on the wait rather than on the size. A question asked in parts is not batched by rows, since a batched part would send the parts times the batches. A call that fails ends the request. diff --git a/java/src/main/java/com/nvidia/sdm/internal/Wording.java b/java/src/main/java/com/nvidia/sdm/internal/Wording.java index ecd7f2dcb..4a1158a6e 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Wording.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Wording.java @@ -99,7 +99,6 @@ private static Map, String>> advice() { + ") && List.of(" + String.join(", ", quotedAll(given.get(2))) + ").contains(link.sourceColumn()))).toList())"); - spelled.put("materialize_call", given -> "materialize()"); spelled.put("missing_time", given -> "null"); spelled.put( "name_candidates", diff --git a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java index 0f83ddb7f..5af2861fa 100644 --- a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java +++ b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java @@ -49,7 +49,7 @@ static final class Endpoint implements AutoCloseable { if(mode.equals("transport") && scores>=2) { exchange.close();return; } if(mode.equals("expired") && scores==2) status=404; else if(mode.equals("retry") && scores==1) status=408; - else if(mode.equals("fail") && scores==2) status=500; + else if(mode.equals("fail") && scores==2) status=422; else if(mode.equals("malformed") && scores==2) answer=Map.of("predictions","invalid"); else { var table=object(object(payload.get("predict")).get("instance_table")); @@ -134,7 +134,7 @@ void livePreparedBatchPrediction() throws Exception { @Test void failedBatchKeepsPaidAnswersAndCloses() throws Exception { try(Endpoint endpoint=new Endpoint("fail"); SdmClient client=endpoint.client()) { var error=assertThrows(ModelEndpointException.class,()->client.predict(request(),SdmClient.KUMO_RELATIONAL)); - assertEquals(500,error.status()); assertEquals(2,endpoint.scores);assertEquals(1,endpoint.deletes); + assertEquals(422,error.status()); assertEquals(2,endpoint.scores);assertEquals(1,endpoint.deletes); assertNotNull(error.details().get("answered")); } } diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index ac1fc4ed6..f4de61970 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -406,11 +406,11 @@ void utf8FeatureValuesUseTheSharedEnvelopeAndActualByteLimit() throws Exception assertTrue(new String(body, StandardCharsets.UTF_8).contains("é😀")); try (Workspace limited = new Workspace(); SdmClient capped = SdmClient.forDatabricksServing(limited.serving(body.length - 1))) { - StructuredDataException error = assertThrows(StructuredDataException.class, - () -> capped.predict(request, SdmClient.KUMO_RELATIONAL)); - assertEquals("INVALID_REQUEST", error.code()); - assertEquals(body.length, ((Number) error.details().get("bytes")).intValue()); - assertTrue(limited.bodies.isEmpty()); + // Past the endpoint's limit, the call is split before it is sent, + // each part within it. + capped.predict(request, SdmClient.KUMO_RELATIONAL); + assertTrue(limited.bodies.size() > 1); + for (byte[] part : limited.bodies) assertTrue(part.length <= body.length - 1); } } } diff --git a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java index 172317164..a24e5e1b2 100644 --- a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java +++ b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java @@ -96,7 +96,8 @@ static List sizes(Request request) throws Exception { assertEquals(java.util.stream.IntStream.rangeClosed(1, count).boxed().toList(), ids); } assertEquals(List.of(100, 100, 1), sizes(materialize(schema, QUERY, 201, 100))); - assertEquals(List.of(200, 1), sizes(materialize(schema, QUERY, 201, 1000))); + // A size asked is taken as asked; 200 is only the default. + assertEquals(List.of(201), sizes(materialize(schema, QUERY, 201, 1000))); assertEquals(List.of(201), sizes(materialize(schema, "PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id", 201, null))); } @@ -113,7 +114,7 @@ static List sizes(Request request) throws Exception { Table predict = arrow(allocator, predicted, "temporal_link_prediction")) { Request request = memory(fixture, false).materializeExamples(task(test), context, predict, ExamplesOptions.builder().rowsPerCall(1000).build()); - assertEquals(List.of(200, 1), sizes(request)); + assertEquals(List.of(201), sizes(request)); } } diff --git a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java index db617ab5b..46fc651d3 100644 --- a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java +++ b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java @@ -27,7 +27,8 @@ /** * The model transport over {@code java.net.http}: the call is sent as the bytes given, - * one past 16 MB included; a transient status is tried again; a refusal is the endpoint's; an + * one past 16 MB included; a transient status or a 500 is tried again; a refusal is the endpoint's; + * a timeout is told as a 408; an * answer that is not JSON, or one past 64 MiB, is a failed transport. */ class TransportTest { @@ -83,20 +84,20 @@ void aCallPast16MiBIsSentAsTheBytesGiven() throws Exception { @Test void aTransientStatusIsTriedAgain() throws Exception { - try (Endpoint endpoint = new Endpoint(OBJECT, 503, 429)) { + try (Endpoint endpoint = new Endpoint(OBJECT, 503, 429, 500)) { assertArrayEquals(OBJECT, endpoint.transport(3).predict("{}".getBytes(StandardCharsets.UTF_8))); - assertEquals(3, endpoint.received.size()); + assertEquals(4, endpoint.received.size()); } } @Test void aRefusalIsTheEndpointsAndIsNotTriedAgain() throws Exception { - try (Endpoint endpoint = new Endpoint(OBJECT, 500)) { + try (Endpoint endpoint = new Endpoint(OBJECT, 422)) { ModelEndpointException refused = assertThrows( ModelEndpointException.class, () -> endpoint.transport(3).predict("{}".getBytes(StandardCharsets.UTF_8))); - assertEquals(500, refused.status()); + assertEquals(422, refused.status()); assertEquals("later", refused.getMessage()); assertEquals(1, endpoint.received.size()); } @@ -161,10 +162,12 @@ void anAnswerWhoseBodyIsHeldBackTimesOut() throws Exception { new HttpTransport( "http://127.0.0.1:" + server.getAddress().getPort(), null, Duration.ofMillis(300), 0); long started = System.nanoTime(); - HttpTransport.Failed failed = + // Told as the endpoint's 408, so the core asks the rows again in halves. + ModelEndpointException failed = assertThrows( - HttpTransport.Failed.class, + ModelEndpointException.class, () -> transport.predict("{}".getBytes(StandardCharsets.UTF_8))); + assertEquals(408, failed.status()); long millis = (System.nanoTime() - started) / 1_000_000; assertTrue(millis < 5_000, "the call took " + millis + " ms"); assertTrue(failed.getMessage().contains("timed out"), failed.getMessage()); diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py index 8046723f0..6f7efb39b 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py @@ -68,15 +68,12 @@ } #: The refusals the reference raises as its base class under their code, -#: by code and rule: ``main``'s 1fc5a686 ceilings on what one -#: ``materialize`` gathers and its refusal of a batched forecast, each a +#: by code and rule: ``main``'s 1fc5a686 refusal of a batched forecast, a #: ``StructuredDataError(code='UNSUPPORTED_REQUEST')`` in the reference's #: client, so a caller's ``except`` sees the class it would there ( fix #: 2). Every other refusal takes its code's class (data:`ERRORS`). AS_THE_BASE_CLASS: frozenset[tuple[str, str]] = frozenset( { - ('UNSUPPORTED_REQUEST', 'most_entities'), - ('UNSUPPORTED_REQUEST', 'most_calls'), ('UNSUPPORTED_REQUEST', 'batched_part'), } ) @@ -143,7 +140,6 @@ class UserWarning(builtins.UserWarning): # noqa: A001 - the reference's name f'not (link.source_table == {source!r} and link.target_table == ' f'{target!r} and link.source_column in {others!r})]' ), - 'materialize_call': lambda: 'materialize()', 'name_candidates': lambda column: ( f'materialize(..., candidates=frame[{column!r}])' ), diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py index 0582c9574..6052fdbef 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/frame_materialize.py @@ -104,7 +104,6 @@ def _gather_locked( candidates: Sequence[Any] | None, ) -> tuple[tuple[dict[str, Any], ...], results.QueryFacts, results.CoreRequest]: from sdm_client import decompose - from sdm_client.materializers import relational anchor = options.get('anchor_time') call = { @@ -116,8 +115,6 @@ def _gather_locked( 'unseeded': 'random_seed' in options and options['random_seed'] is None, 'options': sorted(set(options) - TAKEN), 'most_parts': max(0, min(int(decompose.MOST_PARTS), 2**32 - 1)), - 'most_entities': max(0, min(int(relational.MOST_ENTITIES), 2**64 - 1)), - 'most_calls': max(0, min(int(relational.MOST_CALLS), 2**32 - 1)), } attachments = { 1: _graph(materializer), diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py b/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py index 4c62bd3f9..b226618ac 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/materialize.py @@ -289,9 +289,7 @@ def gather( A request of more entities than one call scores is gathered by the core as several calls (``main``'s 1fc5a686): its payloads are each call's, in order, and the state keeps how many rows each later call - scores. The most entities and calls one ``materialize`` holds are this - client's (``MOST_ENTITIES``, ``MOST_CALLS``), which the core holds the - call to. + scores, however many entities there are. Raises: StructuredDataError: The core's refusal, as:mod:`.errors` maps it, @@ -344,14 +342,6 @@ def gather( }, ), } - # The ceilings on what one materialize holds, which the caller may set - # as it sets the most steps of a forecast. - from sdm_client.materializers import relational - - operation['most_entities'] = max( - 0, min(int(relational.MOST_ENTITIES), 2**64 - 1) - ) - operation['most_calls'] = max(0, min(int(relational.MOST_CALLS), 2**32 - 1)) if candidates is not None: operation['candidates'] = 6 attachments[6] = codec.attachment(_candidates(candidates)) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index 4aa0ed787..460dcad58 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -19,7 +19,7 @@ from sdm_client._rust import codec, errors, results from sdm_client._rust.run import Refused, TransportFailed, run -from sdm_client.core.transport import EndpointFailed +from sdm_client.core.transport import EndpointFailed, read_timed_out from sdm_client.decompose import RANKING from sdm_client.errors import NimRequestError, StructuredDataError @@ -620,7 +620,7 @@ def model_call(body: bytes) -> bytes: except NimRequestError as refusal: refused = refusal except StructuredDataError as failed: - if failed.code == 'TRANSPORT_ERROR' and 'timeout' in failed.details: + if failed.code == 'TRANSPORT_ERROR' and read_timed_out(failed): # Told to the core as a 408, so it asks the rows again in # halves; the timeout is what is raised if no half answers. timed_out = StructuredDataError( diff --git a/packages/structured-data-models-client/src/sdm_client/core/transport.py b/packages/structured-data-models-client/src/sdm_client/core/transport.py index 888c23b7d..95b401ec9 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/transport.py +++ b/packages/structured-data-models-client/src/sdm_client/core/transport.py @@ -315,6 +315,21 @@ def _build_retry( return Retry(**options) +def read_timed_out(error: BaseException | None) -> bool: + r"""Whether a request failure is the answer taking longer than the + timeout, which a smaller call can fix, rather than a connection that + could not be made (``NewConnectionError`` is a ``TimeoutError`` too).""" + seen: set[int] = set() + while error is not None and id(error) not in seen: + seen.add(id(error)) + if isinstance( + error, (requests.ReadTimeout, urllib3.exceptions.ReadTimeoutError) + ): + return True + error = error.__cause__ or error.__context__ + return False + + def _timed_out(error: BaseException | None) -> bool: r"""Whether a request failure is a timeout, however it reached us. diff --git a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py index b18afcfe9..615c51cf6 100644 --- a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py @@ -19,11 +19,6 @@ from sdm_client.sanitization import SanitizationReport -# Host memory limits passed to Rust for request-size enforcement. -MOST_ENTITIES = 100_000 -MOST_CALLS = 100 - - class RelationalMaterializer: """Retain host resources while Rust owns sampling and request construction.""" diff --git a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py index 85baf1abd..98d023a01 100644 --- a/packages/structured-data-models-client/tests/tabular_pql_count_cases.py +++ b/packages/structured-data-models-client/tests/tabular_pql_count_cases.py @@ -34,10 +34,9 @@ class StopWarning(Exception): def observe_case(name: str) -> dict[str, Any]: from sdm_client import KumoTabular, PredictiveQuery, SDMClient - from sdm_client.models import TABULAR_LIMITS assert name in CASES - cap = TABULAR_LIMITS['context_rows_max'] + cap = 10_000 events: list[Any] = [] recorded: list[dict[str, str]] = [] outcome: dict[str, Any] = {} diff --git a/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py b/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py index 360cfa4ef..418f006fc 100644 --- a/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py +++ b/packages/structured-data-models-client/tests/test_a_remedy_is_followed_to_the_wire.py @@ -256,7 +256,7 @@ def test_the_limits_page_records_the_default_the_code_has() -> None: Path(__file__).resolve().parents[3] / 'docs' / 'reference' / 'limits.md' ).read_text() assert '1,000 when explaining, otherwise 10,000' not in page - assert '1,000 by default, up to 10,000 with `num_context_samples`' in page + assert 'The context is sampled only to a `num_context_samples`' in page def test_a_concrete_timestamp_materializes() -> None: diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index 641cc7926..508bb2056 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -97,9 +97,7 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( ): """A part is one call, which is what makes the fan-out countable. The refusal reads the count it was given, so it costs no gathering.""" - from sdm_client.models import RELATIONAL_LIMITS - - count = RELATIONAL_LIMITS['predict_rows_max'] + 1 + count = 10_000 + 1 # one past the default batch with pytest.raises(StructuredDataError) as refused: built(query, indices=[1] * count, **how) assert refused.value.code == 'UNSUPPORTED_REQUEST' @@ -117,22 +115,6 @@ def test_a_smaller_batch_lowers_what_a_part_may_ask_for(monkeypatch): assert built(FORECAST, indices=[1, 2], rows_per_call=2).parts -def test_the_whole_decomposed_question_is_held_to_the_memory_ceiling( - monkeypatch, -): - """Each part gathers the whole population with a context of its own, so a - ranking over 200 candidates holds 200 times what one of them does. Bounding - each part on its own left the question itself unbounded.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_ENTITIES', 5) - built(RANK, indices=[1, 2], candidates=[0, 1]) - with pytest.raises(StructuredDataError) as refused: - built(RANK, indices=[1, 2, 3], candidates=[0, 1]) - assert refused.value.details['total'] == 6 - assert refused.value.details['gathered_times'] == 2 - - def test_a_step_that_fails_keeps_the_steps_already_paid_for(): """A question asked in parts pays per part, and the steps answered before a failure are as bought as the batches of one are. diff --git a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py index 60ee44e3a..a15cf85ca 100644 --- a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py +++ b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py @@ -78,7 +78,7 @@ def do_POST(self): elif outer.mode == 'malformed' and outer.scored == 2: answer = {'predictions': 'invalid'} elif outer.mode == 'fail' and outer.scored == 2: - status = 500 + status = 422 elif outer.mode == 'retry' and outer.scored == 1: status = 408 else: @@ -165,9 +165,9 @@ def test_failed_batch_preserves_paid_answers_and_cleanup(): with SDMClient(url=endpoint.url, max_retries=1) as client: with pytest.raises(StructuredDataError) as caught: client.predict(request(), model=KumoRelational()) - assert endpoint.scored == 2 # HTTP500 is not retried. + assert endpoint.scored == 2 # a 422 is not retried assert endpoint.calls[-1][:2] == ('DELETE', '/v1/sessions/s1') - assert getattr(caught.value, 'status_code', None) == 500 + assert getattr(caught.value, 'status_code', None) == 422 assert caught.value.details['answered'] is not None finally: endpoint.close() diff --git a/packages/structured-data-models-client/tests/test_context_samples.py b/packages/structured-data-models-client/tests/test_context_samples.py index d41d87c62..0ff71744a 100644 --- a/packages/structured-data-models-client/tests/test_context_samples.py +++ b/packages/structured-data-models-client/tests/test_context_samples.py @@ -25,7 +25,7 @@ from sdm_client.models import RELATIONAL_LIMITS FAST = RELATIONAL_LIMITS['context_rows_fast'] -MAX = RELATIONAL_LIMITS['context_rows_max'] +MAX = 10_000 # --- which value goes on the wire ----------------------------------------- @@ -41,6 +41,7 @@ (FAST + 1, 'best'), (MAX - 1, 'best'), (MAX, 'best'), + (MAX + 1, 'best'), ], ) def test_the_wire_value_is_the_one_that_admits_the_count(asked, expected): @@ -49,18 +50,10 @@ def test_the_wire_value_is_the_one_that_admits_the_count(asked, expected): assert request._payloads[0]['inference']['run_mode'] == expected -def test_asking_for_more_than_the_endpoint_allows_names_the_ceiling(): - with pytest.raises(StructuredDataError) as error: - _materialize(num_context_samples=MAX + 1) - message = str(error.value) - assert '10,000' in message - assert error.value.code == 'UNSUPPORTED_REQUEST' - - -def test_the_boundary_is_at_the_documented_row_counts(): - """A pin: these are the two numbers the endpoint actually distinguishes.""" +def test_the_boundary_is_at_the_documented_row_count(): + """A pin: the count past which a request asks for the larger setting. + The client sets no most on the count.""" assert FAST == 1_000 - assert MAX == 10_000 # --- how many examples are actually gathered ------------------------------ diff --git a/packages/structured-data-models-client/tests/test_custom_examples.py b/packages/structured-data-models-client/tests/test_custom_examples.py index 9968f8f0d..0abc2d5fc 100644 --- a/packages/structured-data-models-client/tests/test_custom_examples.py +++ b/packages/structured-data-models-client/tests/test_custom_examples.py @@ -385,6 +385,7 @@ def test_supplied_link_batches_obey_model_limit(): predicted, rows_per_call=500, ) + # A batch size asked is taken as asked; 200 is only the default. assert [ len(p['predict']['instance_table']['rows']) for p in request.payloads - ] == [200, 1] + ] == [201] diff --git a/packages/structured-data-models-client/tests/test_databricks_from_source.py b/packages/structured-data-models-client/tests/test_databricks_from_source.py index 5979eba8d..1e8231721 100644 --- a/packages/structured-data-models-client/tests/test_databricks_from_source.py +++ b/packages/structured-data-models-client/tests/test_databricks_from_source.py @@ -1576,25 +1576,6 @@ def test_refusals_preserve_source_read_order( assert _where_read(schema, connection) == {where} -def test_too_many_entities_are_refused_before_the_source_is_read( - monkeypatch: pytest.MonkeyPatch, -) -> None: - r"""More entities than one ``materialize`` holds are refused from the - count alone, before the old route's engine reads anything - (``relational.py:339``): the tracker's entity-count row.""" - from sdm_client.errors import StructuredDataError - from sdm_client.materializers import relational - - monkeypatch.setattr(relational, 'MOST_ENTITIES', 1) - connection = _CompletingConnection() - schema = _rebuilt(connection) - mark = len(connection.calls) - with pytest.raises(StructuredDataError) as refused: - _asked(schema) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert len(connection.calls) == mark - - class _EmptyingWarehouse(_CompletingWarehouse): r"""The completing warehouse, whose ``shop.users`` holds no row while ``empty`` says so: every read of its rows answers none, and its count diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 33e837fe8..6f8b65c61 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -4,10 +4,9 @@ """What the endpoint will not take, and what the answer did not contain. -Two silences. The tabular endpoint caps a context at 10,000 rows and the client -did not know it, so a caller could encode and transmit a table of any size to -be refused at the far end, while the identical overage on the other model was -refused locally with a message naming the limit. +The context the client sends: whole, or sampled only to a count the caller +names. The client holds it to no size of its own; a context larger than the +endpoint takes is the endpoint's refusal. And a field a caller names is validated against the model and never against what comes back. The endpoint answers 200, leaves the field out, and the frame @@ -27,7 +26,6 @@ from sdm_client import PredictiveQuery, Schema from sdm_client.client import _report_absent from sdm_client.errors import StructuredDataError -from sdm_client.models import KUMO_TABULAR from sdm_client.result import PredictionResult @@ -59,16 +57,12 @@ def query(schema: Schema) -> PredictiveQuery: # --- the context the endpoint will take ------------------------------------ -def test_the_declared_cap_is_the_one_the_endpoint_enforces(): - """Measured at the boundary: 10,000 answer, 10,001 refuse.""" - assert KUMO_TABULAR.runtime.limits['context_rows_max'] == 10_000 - - -def test_a_context_above_the_cap_is_refused_before_it_is_encoded(): +def test_a_count_past_the_rows_there_takes_them_all(): schema = table(200) - with pytest.raises(StructuredDataError) as raised: - query(schema).materialize(indices=[0, 1], num_context_samples=50_000) - assert 'accepts at most 10,000 context rows' in str(raised.value) + request = query(schema).materialize( + indices=[0, 1], num_context_samples=50_000 + ) + assert len(request.context.instance_table) == 198 def test_the_cap_itself_is_allowed(): @@ -79,14 +73,13 @@ def test_the_cap_itself_is_allowed(): assert len(request.context.instance_table) == 198 -def test_an_unnamed_context_is_capped_and_says_so(): +def test_an_unnamed_context_is_sent_whole(): schema = table(12_000) request, said = warnings_from( partial(query(schema).materialize, indices=[0, 1]) ) - assert len(request.context.instance_table) == 10_000 - assert any('were sampled' in s for s in said), said - assert any('num_context_samples' in s for s in said) + assert len(request.context.instance_table) == 11_998 + assert not any('were sampled' in s for s in said), said def test_a_context_under_the_cap_is_left_whole_and_unremarked(): @@ -98,27 +91,26 @@ def test_a_context_under_the_cap_is_left_whole_and_unremarked(): assert not any('were sampled' in s for s in said) -def test_the_capped_context_is_a_sample_and_not_the_first_rows(): +def test_a_counted_context_is_a_sample_and_not_the_first_rows(): """Taking the head would bias the context toward whatever the table is ordered by, which for a table ordered by time is the oldest rows.""" schema = table(12_000) - request = query(schema).materialize(indices=[0]) + request = query(schema).materialize(indices=[0], num_context_samples=10_000) taken = request.context.instance_table['rid'] assert taken.max() > 11_000 assert len(taken) == 10_000 -def test_the_predict_frame_route_is_capped_too(): - """The route with no knob at all. The schema's table became the context - whole, so a large table was encoded and sent in full every time, to be - refused.""" +def test_the_predict_frame_route_sends_the_table_whole(): + """The route with no knob at all: the schema's table, less the rows + scored, is the context.""" schema = table(12_000) rows = schema['t'].data.head(3).drop(columns=['amount']) request, said = warnings_from( partial(query(schema).materialize, predict=rows) ) - assert len(request.context.instance_table) == 10_000 - assert any('were sampled' in s for s in said) + assert len(request.context.instance_table) == 11_997 + assert not any('were sampled' in s for s in said) # --- what the answer did not carry ----------------------------------------- @@ -220,28 +212,18 @@ def test_the_result_is_returned_unchanged(): assert _report_absent(answer, ['quantiles'], 'kumo-x') is answer -def test_a_materialized_context_above_the_cap_is_refused_not_merely_the_number(): - """A small table makes this a check on the argument. The rows have to be - there for it to be a check on the context.""" +def test_a_count_past_the_old_cap_is_gathered(): + """The rows are there, so the count is the context's size.""" schema = table(12_000) - with pytest.raises(StructuredDataError) as raised: - query(schema).materialize(indices=[0, 1], num_context_samples=10_001) - assert 'accepts at most 10,000 context rows' in str(raised.value) + request = query(schema).materialize( + indices=[0, 1], num_context_samples=10_001 + ) + assert len(request.context.instance_table) == 10_001 -def test_the_predict_route_advises_something_that_route_allows(): - """Its own refusal says `num_context_samples` cannot be used here, so the - warning must not tell the caller to pass it.""" +def test_the_predict_route_takes_no_count(): schema = table(12_000) rows = schema['t'].data.head(3).drop(columns=['amount']) - _, said = warnings_from(partial(query(schema).materialize, predict=rows)) - capped = [s for s in said if 'were sampled' in s] - assert capped - remedy = capped[0].split('sampled.')[1] - assert 'Pass num_context_samples' not in remedy - assert 'takes no num_context_samples' in remedy - assert 'building the schema' in remedy - with pytest.raises(StructuredDataError) as raised: query(schema).materialize(predict=rows, num_context_samples=100) assert 'cannot be used with predict=' in str(raised.value) @@ -397,15 +379,14 @@ def test_a_target_that_varies_is_left_alone(): assert not any('every context row' in s for s in said) -def test_a_context_past_the_models_cap_is_refused_before_it_is_sent(): - """The client's own preamble says it refuses first where it can tell, and - the model is known by the time this check runs, so sending 12,500 rows to - be told the limit at the far end was work nobody needed to do.""" +def test_a_context_past_the_old_cap_is_sent(): + """The client holds a context to no size of its own: 12,500 rows are + sent, for the endpoint to answer or refuse.""" import numpy as np import pandas as pd from test_adapter_and_result import envelope - from sdm_client import Request, UnsupportedRequestError + from sdm_client import Request rng = np.random.default_rng(0) rows = 12_500 @@ -418,8 +399,7 @@ def test_a_context_past_the_models_cap_is_refused_before_it_is_sent(): task='classification', ) - with pytest.raises(UnsupportedRequestError, match='12,500'): - envelope(request) + envelope(request) def test_a_context_within_the_cap_is_not_refused(): diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index a3085a0f8..61984a506 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -238,20 +238,10 @@ def gathered_in_calls(*, calls: int, context_rows: int): ) -@pytest.mark.parametrize('over', [False, True], ids=['under', 'over']) -def test_the_context_cap_is_held_to_each_call_and_not_their_sum(over): - """Both sides of the cap, because only the allowed one was covered: a - per-call count that always read nought would have passed that alone.""" - - cap = 10_000 - each = cap + 1 if over else cap // 2 + 1 - request = gathered_in_calls(calls=2, context_rows=each) - if not over: - sent(request) - return - with pytest.raises(StructuredDataError) as refused: - sent(request) - assert f'{each:,} context rows' in str(refused.value) +def test_each_call_s_context_is_sent_at_any_size(): + """The client holds a context to no size of its own, in each call or + across them.""" + sent(gathered_in_calls(calls=2, context_rows=10_001)) def test_a_batched_request_reports_the_context_one_call_carries(): diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index deccb4296..4fd99bd8b 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -193,9 +193,7 @@ def test_the_endpoint_scores_what_this_client_says_it_does(client, query): """ import copy - from sdm_client.models import RELATIONAL_LIMITS - - cap = RELATIONAL_LIMITS['predict_rows_max'] + cap = 10_000 # the default batch payload = copy.deepcopy(query.materialize(indices=[0, 1]).payloads[0]) rows = payload['predict']['instance_table']['rows'] diff --git a/packages/structured-data-models-client/tests/test_materialize_options.py b/packages/structured-data-models-client/tests/test_materialize_options.py index 72698572f..10f638a55 100644 --- a/packages/structured-data-models-client/tests/test_materialize_options.py +++ b/packages/structured-data-models-client/tests/test_materialize_options.py @@ -372,21 +372,8 @@ def _refusal_of(**options: Any) -> StructuredDataError: def test_a_limit_is_raised_as_the_subclass_on_the_core_register_p1b_20() -> ( None ): - for options in ({'num_context_samples': 10001},): - core = _refusal_of(**options) - assert core.code == 'UNSUPPORTED_REQUEST', options - assert type(core).__name__ == 'UnsupportedRequestError', options - assert 'rule' not in core.details, options - for options, rule in ( - ({'indices': list(range(100001))}, 'most_entities'), - ({'indices': list(range(101)), 'rows_per_call': 1}, 'most_calls'), - ): - core = _refusal_of(**options) - assert (type(core), core.code) == ( - StructuredDataError, - 'UNSUPPORTED_REQUEST', - ), options - assert core.details.get('rule') == rule, options + # The client sets no most on the entities, the calls or the context: + # entities the source does not hold are refused for that alone. core = _refusal_of(indices=list(range(1001))) assert (type(core), core.code) == (StructuredDataError, 'INVALID_REQUEST') diff --git a/packages/structured-data-models-client/tests/test_payload_rules.py b/packages/structured-data-models-client/tests/test_payload_rules.py index 3b8d97d34..dc27a4765 100644 --- a/packages/structured-data-models-client/tests/test_payload_rules.py +++ b/packages/structured-data-models-client/tests/test_payload_rules.py @@ -198,6 +198,8 @@ def test_vectors_cover_required_success_and_failure_cases() -> None: # The client holds a payload to no size of its own: a call the endpoint # refuses as too large is asked again in halves when it is predicted. 'size.one-past-the-limit': 'no byte limit', + # Nor on the context: a count past 10,000 is gathered. + 'base.regression.num-context-10001': 'no context limit', } diff --git a/packages/structured-data-models-client/tests/test_pql_for_tabular.py b/packages/structured-data-models-client/tests/test_pql_for_tabular.py index 4bf351533..c7438da9b 100644 --- a/packages/structured-data-models-client/tests/test_pql_for_tabular.py +++ b/packages/structured-data-models-client/tests/test_pql_for_tabular.py @@ -154,10 +154,10 @@ def test_tabular_context_budget_limits_actual_rows(count): assert request.predict_rows == 1 -def test_tabular_context_budget_above_model_limit_is_refused(): - with pytest.raises(StructuredDataError) as error: - query().materialize(indices=[1], num_context_samples=10_001) - assert error.value.code == 'UNSUPPORTED_REQUEST' +def test_tabular_context_budget_past_the_old_limit_is_taken(): + # The client sets no most: more than the rows there takes them all. + request = query().materialize(indices=[1], num_context_samples=10_001) + assert request.context_rows > 0 @pytest.mark.parametrize('count', [500, 10_001]) diff --git a/packages/structured-data-models-client/tests/test_predict_result.py b/packages/structured-data-models-client/tests/test_predict_result.py index 3cbdfc196..9a9cfa739 100644 --- a/packages/structured-data-models-client/tests/test_predict_result.py +++ b/packages/structured-data-models-client/tests/test_predict_result.py @@ -285,14 +285,10 @@ def test_the_core_s_predict_is_the_reference_s( ) return if spec['covers'] == R.SIZE: - # The reference's code, and a message that explains the limit as - # the old route's does (it names 30MB). - assert got['outcome'] == 'refused', 'past the limit, taken' - assert got['code'] == expected['code'], got['message'] + # The client holds a call to no size of its own, where the + # reference refused one past 30 MiB: it is sent. assert '30MB' in expected['message'] - assert re.search(r'30 ?MB|30 ?MiB|31,?457,?280', got['message']), ( - f'the size refusal does not explain the limit: {got["message"]!r}' - ) + assert got['outcome'] != 'refused', got.get('message') return if spec['name'] in { 'prepare.regression.outputs-explanation-two-rows', diff --git a/packages/structured-data-models-client/tests/test_query_engine_boundary.py b/packages/structured-data-models-client/tests/test_query_engine_boundary.py index 365e2bb0f..699b76a61 100644 --- a/packages/structured-data-models-client/tests/test_query_engine_boundary.py +++ b/packages/structured-data-models-client/tests/test_query_engine_boundary.py @@ -300,59 +300,41 @@ def test_a_rows_per_call_that_is_not_a_size_is_refused(given, text): assert 'a call scores a whole number of rows' in str(refused.value) -def test_a_batch_size_of_one_is_refused_for_the_contexts_it_would_hold( - monkeypatch, -): - """A context is gathered per call, so a small enough batch holds one per - entity however few entities there are.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_CALLS', 3) +def test_a_batch_size_of_one_gathers_a_call_per_entity(): + """A context is gathered per call, and the client sets no most on the + calls: a batch of one is a call for each entity.""" with warnings.catch_warnings(): warnings.simplefilter('ignore') - with pytest.raises(StructuredDataError) as refused: - query().materialize( - indices=[5, 6, 7, 8, 9], - num_context_samples=20, - rows_per_call=1, - ) - assert refused.value.details['calls'] == 5 - assert '3 calls' in str(refused.value) - - -def test_a_population_past_the_ceiling_is_refused_before_the_engine_is_reached( - monkeypatch, -): - """Reaching the engine means building the graph and labelling every index, - which for a population far past the ceiling is minutes of work and the - memory with it before anything says no.""" - from sdm_client.materializers import relational as materializer - - monkeypatch.setattr(materializer, 'MOST_ENTITIES', 4) - with pytest.raises(StructuredDataError) as refused: - query().materialize(indices=list(range(5)), num_context_samples=20) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert refused.value.details['entities'] == 5 + request = query().materialize( + indices=[5, 6, 7, 8, 9], + num_context_samples=20, + rows_per_call=1, + ) + assert len(request.payloads) == 5 -def test_more_calls_than_fit_in_memory_are_refused_before_they_are_built( - monkeypatch, -): - """The calls a population comes to is known only once the engine has said - how many entities it really holds, so this one is checked there, and still - before the batches themselves are built. - """ - from sdm_client.materializers import relational as materializer +def test_more_than_100_000_entities_are_gathered(): + """The client sets no most on the entities one materialize gathers.""" + from sdm_client import KumoRelational - monkeypatch.setattr(materializer, 'MOST_CALLS', 2) + count = 100_001 + frame = pd.DataFrame( + {'uid': range(count + 10), 'label': [*range(count), *[None] * 10]} + ) + users = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + users['users']['label'].stype = 'numerical' with warnings.catch_warnings(): warnings.simplefilter('ignore') - with pytest.raises(StructuredDataError) as refused: - query().materialize( - indices=[5, 6, 7, 8, 9], num_context_samples=20, rows_per_call=2 - ) - assert refused.value.code == 'UNSUPPORTED_REQUEST' - assert refused.value.details['calls'] == 3 + request = PredictiveQuery( + 'PREDICT users.label FOR EACH users.uid', users + ).materialize( + indices=list(range(count)), + model=KumoRelational(), + rows_per_call=50_001, + ) + assert [ + len(p['predict']['instance_table']['rows']) for p in request.payloads + ] == [50_001, 50_000] def test_a_call_that_fails_keeps_the_answers_already_paid_for(monkeypatch): diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py new file mode 100644 index 000000000..e9e5e7219 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The client holds a call to no size of its own and recovers from what the +endpoint says: a call refused as too large, by a gateway, or timed out is +asked again in halves, each with the whole context; a 422 is not.""" + +from __future__ import annotations + +import json + +import pytest +import requests +from test_limits_and_absent_outputs import gathered +from test_tabular_direct_core import client + +from sdm_client.core.transport import _to_nim_error +from sdm_client.errors import StructuredDataError + + +class Scripted: + r"""An endpoint that fails the first call as ``failure`` says, then + answers every call with a prediction for each row it scores.""" + + url = 'http://localhost' + + def __init__(self, failure: Exception) -> None: + self.failure: Exception | None = failure + self.calls: list[dict] = [] + + def predict_body(self, body: bytes) -> bytes: + payload = json.loads(body) + self.calls.append(payload) + if self.failure is not None: + failure, self.failure = self.failure, None + raise failure + table = payload['predict']['instance_table'] + key = table['columns'].index('instance_id') + return json.dumps( + { + 'id': 'answer', + 'model': 'kumo-relational', + 'metadata': {}, + 'predictions': [ + {'id': str(row[key]), 'row_index': at, 'prediction': 1.0} + for at, row in enumerate(table['rows']) + ], + } + ).encode() + + def predict(self, payload: dict) -> dict: + return json.loads(self.predict_body(json.dumps(payload).encode())) + + def close(self) -> None: + pass + + +def _timed_out() -> StructuredDataError: + r"""The transport's account of an answer slower than its timeout.""" + error = StructuredDataError( + 'Request to http://localhost/v1/predictions timed out after 1 seconds', + code='TRANSPORT_ERROR', + details={'timeout': 1.0}, + ) + error.__cause__ = requests.ReadTimeout() + return error + + +@pytest.mark.parametrize( + 'failure', + [ + _to_nim_error(413, b'{"detail": "too large"}'), + _to_nim_error(502, b'{}'), + _to_nim_error(504, b'{}'), + _timed_out(), + ], + ids=['413', '502', '504', 'timeout'], +) +def test_a_call_too_large_or_too_slow_is_asked_again_in_halves(failure): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(failure) + result = client(endpoint).predict(request, model='kumo-relational') + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + whole, *halves = endpoint.calls + assert [len(c['predict']['instance_table']['rows']) for c in halves] == [ + 2, + 2, + ] + assert all(c['context'] == whole['context'] for c in halves) + + +def test_a_422_is_not_asked_again_and_says_what_the_endpoint_said(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_to_nim_error(422, b'{"detail": "N_test 4 exceeds"}')) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert len(endpoint.calls) == 1 + assert 'N_test 4 exceeds' in str(refused.value) + + +def test_a_call_of_one_row_refused_as_too_large_is_the_endpoint_s_refusal(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=1) + endpoint = Scripted(_to_nim_error(413, b'{"detail": "context too large"}')) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert len(endpoint.calls) == 1 + assert 'context too large' in str(refused.value) diff --git a/packages/structured-data-models-client/tests/test_serving_route.py b/packages/structured-data-models-client/tests/test_serving_route.py index a8bc2287f..12e2e191d 100644 --- a/packages/structured-data-models-client/tests/test_serving_route.py +++ b/packages/structured-data-models-client/tests/test_serving_route.py @@ -395,17 +395,21 @@ def test_malformed_serving_response_is_a_transport_error( assert len(received) == 1 -def test_a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() -> None: +def test_a_call_past_the_caller_s_limit_is_split_before_it_is_sent() -> None: request = _materialized() _, received = _predict(request) [(_, body, _)] = received - # The limit applies to the actual UTF-8 envelope posted to the SDK. + # The limit applies to the actual UTF-8 envelope posted to the SDK: past + # it, the call's rows are sent in calls that fit, the context whole. core, on_core = _predict(request, max_request_bytes=len(body) - 1) - assert on_core == [] - assert isinstance(core, StructuredDataError), core - assert core.details['limit'] == len(body) - 1 - assert core.details['bytes'] == len(body) - # The size itself is admitted. + assert not isinstance(core, Exception), core + assert len(on_core) == 2 + assert all(len(sent) <= len(body) - 1 for (_, sent, _) in on_core) + assert all( + _call(sent)['context'] == _call(body)['context'] + for (_, sent, _) in on_core + ) + # The size itself is admitted as one call. core, received = _predict(request, max_request_bytes=len(body)) assert not isinstance(core, Exception), core [(_, sent, _)] = received @@ -426,11 +430,13 @@ def test_utf8_serving_preserves_public_payload_and_measures_the_sent_body() -> ( assert call[field] == request.payloads[0][field], field assert result.to_pandas()['id'].tolist() == [0, 1] assert result.to_pandas()['prediction'].tolist() == [1.0, 1.0] - refused, sent = _predict(request, max_request_bytes=len(body) - 1) - assert isinstance(refused, StructuredDataError) - assert sent == [] - assert refused.details['bytes'] == len(body) - assert refused.details['limit'] == len(body) - 1 + # Past the endpoint's limit, the call is split before it is sent, each + # part within it, and the answer is the same. + split, sent = _predict(request, max_request_bytes=len(body) - 1) + assert not isinstance(split, Exception), split + assert len(sent) == 2 + assert all(len(part) <= len(body) - 1 for (_, part, _) in sent) + assert split.to_pandas()['id'].tolist() == [0, 1] @pytest.mark.parametrize( diff --git a/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py b/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py index 65d4d02c7..35f5bd4d0 100644 --- a/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py +++ b/packages/structured-data-models-client/tests/test_tabular_arrow_indexes.py @@ -11,9 +11,14 @@ def expected_case(name): """Same-runtime pandas ordering, independent of client implementation.""" family, entry, shape = name.split('/') - n, wanted = (81, 40) if entry == 'indices' else (10002, 10000) + # The predict route sends the frame's labelled rows whole: no sampling. + n, wanted = (81, 40) if entry == 'indices' else (10002, 10001) frame = frame_for(family, n, shape) - selected = frame.iloc[:-1].sample(n=wanted, random_state=42).sort_index() + selected = ( + frame.iloc[:-1] + if entry == 'predict' + else frame.iloc[:-1].sample(n=wanted, random_state=42).sort_index() + ) ids = selected['id'].tolist() return { 'context_count': wanted, diff --git a/packages/structured-data-models-client/tests/test_tabular_direct_core.py b/packages/structured-data-models-client/tests/test_tabular_direct_core.py index 0070d69a7..802f6fedf 100644 --- a/packages/structured-data-models-client/tests/test_tabular_direct_core.py +++ b/packages/structured-data-models-client/tests/test_tabular_direct_core.py @@ -340,9 +340,10 @@ def test_direct_context_limit_never_samples() -> None: scoring = pd.DataFrame({'feature': [30, 31]}) request = Request(context, scoring, target='target', task='classification') endpoint = Endpoint() - with pytest.raises(StructuredDataError, match='reads at most 10,000'): - client(endpoint).predict(request, model='kumo-tabular') - assert endpoint.calls == [] + # A direct context is sent as it was given, held to no size. + client(endpoint).predict(request, model='kumo-tabular') + (sent,) = endpoint.calls + assert len(sent['context']['instance_table']['rows']) == 10001 def test_target_container_guard_is_immediate() -> None: diff --git a/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py b/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py index 5c2b021fb..553fdde3b 100644 --- a/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py +++ b/packages/structured-data-models-client/tests/test_tabular_ordinary_indexes.py @@ -14,7 +14,8 @@ def expected_case(name): """Pandas ordering and Decimal effects without invoking any client code.""" family, entry, shape, mode = name.split('/') - n, wanted = (81, 40) if entry == 'indices' else (10002, 10000) + # The predict route sends the frame's labelled rows whole: no sampling. + n, wanted = (81, 40) if entry == 'indices' else (10002, 10001) with localcontext() as context: context.prec = 2 context.traps[FloatOperation] = False @@ -25,6 +26,8 @@ def expected_case(name): try: selected = ( frame.iloc[:-1] + if entry == 'predict' + else frame.iloc[:-1] .sample(n=wanted, random_state=42) .sort_index(kind='stable') ) @@ -38,17 +41,20 @@ def expected_case(name): ), } ids = selected['id'].tolist() + flags = { + signal.__name__ for signal, set_ in context.flags.items() if set_ + } + if entry == 'predict' and family == 'decimal_float': + # The whole frame is encoded unsorted, which compares its + # decimals with floats where a sample's sort would have first. + flags.add('FloatOperation') return { 'context_count': wanted, 'context_ids_sha256': digest(ids), 'encoded_ids_sha256': digest(ids), 'index_dtype': str(selected.index.dtype), 'predictions': [{'row_index': 0, 'id': n - 1, 'prediction': 0.0}], - 'flags': sorted( - signal.__name__ - for signal, set_ in context.flags.items() - if set_ - ), + 'flags': sorted(flags), } diff --git a/packages/structured-data-models-client/tests/test_tabular_pql_counts.py b/packages/structured-data-models-client/tests/test_tabular_pql_counts.py index 49ae3f5b3..8536d6aba 100644 --- a/packages/structured-data-models-client/tests/test_tabular_pql_counts.py +++ b/packages/structured-data-models-client/tests/test_tabular_pql_counts.py @@ -36,23 +36,17 @@ def test_public_count_task_journey(monkeypatch, route, name): } } assert not events and not warnings - elif name == 'warning_raise': - assert outcome == { - 'error': { - 'type': 'StopWarning', - 'code': None, - 'message': 'warning handler stopped', - } - } - assert len(events) == 1 and events[0][0] == 'warning' - assert len(warnings) == 1 and 'were sampled' in warnings[0]['message'] else: assert 'error' not in outcome + # The context is sampled only to a count the caller names: past the + # old cap, with none named, every labelled row is sent, unremarked. expected_count = ( 12 if name.startswith('task_') else 4 if name == 'count_owner' + else actual['cap'] + 1 + if name in ('over_default', 'warning_record', 'warning_raise') else actual['cap'] ) assert len(outcome['context_ids']) == expected_count @@ -67,12 +61,7 @@ def test_public_count_task_journey(monkeypatch, route, name): ['endpoint', expected_count], ['predicted'], ] - if name == 'count_owner': - assert events[:-3] == [['count_gt', actual['cap']]] - elif name.startswith('task_') or name in ( - 'over_default', - 'warning_record', - ): + if name.startswith('task_'): assert len(events) == 4 and events[0][0] == 'warning' else: assert len(events) == 3 @@ -81,9 +70,5 @@ def test_public_count_task_journey(monkeypatch, route, name): len(warnings) == 1 and 'two whole numbers' in warnings[0]['message'] ) - elif name in ('over_default', 'warning_record'): - assert ( - len(warnings) == 1 and 'were sampled' in warnings[0]['message'] - ) else: assert not warnings diff --git a/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py b/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py index 07e3ff93e..684e7631b 100644 --- a/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py +++ b/packages/structured-data-models-client/tests/test_tabular_sorting_scope.py @@ -48,7 +48,9 @@ def materialize(frame, entry, wanted): return query(frame).materialize(**arguments, task='regression') -@pytest.mark.parametrize('entry', ['indices', 'predict']) +# Only a count the caller names reduces the context; the predict route takes +# no count, and sends its context whole. +@pytest.mark.parametrize('entry', ['indices']) @pytest.mark.parametrize('owner', ['frame_sort', 'sampled_frame_sort', 'label']) def test_reduced_custom_sorting_is_refused_without_execution( monkeypatch, entry, owner diff --git a/packages/structured-data-models-client/tests/test_temporal_link_batching.py b/packages/structured-data-models-client/tests/test_temporal_link_batching.py index 02c6f1bf1..c1e0d2e6c 100644 --- a/packages/structured-data-models-client/tests/test_temporal_link_batching.py +++ b/packages/structured-data-models-client/tests/test_temporal_link_batching.py @@ -106,7 +106,8 @@ def materialize(schema, count=201, rows_per_call=None, query=QUERY): (200, None, [200]), (201, None, [200, 1]), (201, 100, [100, 100, 1]), - (201, 1000, [200, 1]), + # A size asked is taken as asked; 200 is only the default. + (201, 1000, [201]), ], ) def test_pql_link_batches_preserve_every_entity( @@ -135,15 +136,12 @@ def test_node_predictions_keep_the_ordinary_limit(link_schema): ] == [201] -def test_link_batches_respect_total_call_limit(link_schema, monkeypatch): - from sdm_client.materializers import relational - - monkeypatch.setattr(relational, 'MOST_CALLS', 1) - with pytest.raises(StructuredDataError) as caught: - materialize(link_schema) - assert caught.value.code == 'UNSUPPORTED_REQUEST' - assert caught.value.details['calls'] == 2 - assert caught.value.details['rows_per_call'] == 200 +def test_link_batches_are_as_many_calls_as_the_entities_need(link_schema): + request = materialize(link_schema) + assert [ + len(p['predict']['instance_table']['rows']) for p in request.payloads + ][:1] == [200] + assert len(request.payloads) == 2 def test_link_batch_result_order_and_partial_failure( diff --git a/packages/structured-data-models-client/tests/test_the_predict_route.py b/packages/structured-data-models-client/tests/test_the_predict_route.py index ad1bb939b..f7f94737b 100644 --- a/packages/structured-data-models-client/tests/test_the_predict_route.py +++ b/packages/structured-data-models-client/tests/test_the_predict_route.py @@ -761,21 +761,13 @@ def recording(self: Any, adapter: Any, body: bytes) -> Any: return written, received, refused.value -def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( +def test_the_bytes_sent_are_the_core_s_at_any_size( monkeypatch: pytest.MonkeyPatch, ) -> None: - # Register PD-2: the HTTP transport sends the - # core's call as the core wrote it, so the bytes the endpoint receives - # are the bytes the core counted against the limit, characters past - # ASCII as their own bytes, never json.dumps' escapes. At the limit's - # edge the call is sent, at most the limit; one character past it, - # nothing is sent. Since the reference's 26ecdd28 the call - # leaves out the engine's inference config, which the payload holds, - # so the payload is the larger and meets the limit first: the edge is - # the payload's, which materialize names in its refusal, and one past - # it materialize refuses (the call's own refusal, past the limit with - # the payload within it, is sdm-core's - # a_call_past_the_limit_is_refused_before_it_is_sent). + # Register PD-2: the HTTP transport sends the core's call as the core + # wrote it, characters past ASCII as their own bytes, never json.dumps' + # escapes; and the client holds a call to no size of its own, so one + # past the 30 MiB the reference refused is sent. limit = 30 * 1024 * 1024 lengths = [] for emoji, ascii in ((0, 0), (1, 0), (0, 1)): @@ -784,39 +776,17 @@ def test_the_bytes_sent_are_the_core_s_and_held_to_the_limit( ) assert written == received lengths.append(len(received[0])) - base, wide, narrow = ( - lengths[0], - lengths[1] - lengths[0], - lengths[2] - lengths[0], - ) - assert wide % 4 == 0 and narrow == 1 - emoji = (limit - base) // wide - ascii = (limit - base - emoji * wide) // narrow - # The call's edge is past the payload's: materialize refuses it, naming - # the payload's size, and the payload's edge is that much nearer. - with pytest.raises(StructuredDataError) as over: - _emoji_request(monkeypatch, emoji, ascii) - details = over.value.details - assert over.value.code == 'INVALID_REQUEST' - assert details['field'] == 'payload' and details['limit'] == limit - excess = details['bytes'] - limit - assert 0 < excess < wide * 64 - room = emoji * wide + ascii * narrow - excess - emoji = room // wide - ascii = (room - emoji * wide) // narrow + base, wide = lengths[0], lengths[1] - lengths[0] + assert wide % 4 == 0 and lengths[2] - lengths[0] == 1 + emoji = (limit - base) // wide + 1 written, received, refused = _sent( - monkeypatch, _emoji_request(monkeypatch, emoji, ascii) + monkeypatch, _emoji_request(monkeypatch, emoji, 0) ) assert refused.code == 'RECORDED' assert written == received - assert len(received[0]) == base + emoji * wide + ascii * narrow <= limit + assert len(received[0]) == base + emoji * wide > limit assert b'\\ud83d' not in received[0] assert '\U0001f600'.encode() in received[0] - with pytest.raises(StructuredDataError) as past: - _emoji_request(monkeypatch, emoji, ascii + 1) - assert past.value.code == 'INVALID_REQUEST' - assert past.value.details['field'] == 'payload' - assert past.value.details['bytes'] == limit + 1 def test_embeddings_are_the_old_route_s_on_the_core( diff --git a/packages/structured-data-models-client/tests/test_transport_socket.py b/packages/structured-data-models-client/tests/test_transport_socket.py index 1e4ae9faa..578b634aa 100644 --- a/packages/structured-data-models-client/tests/test_transport_socket.py +++ b/packages/structured-data-models-client/tests/test_transport_socket.py @@ -377,13 +377,11 @@ def test_a_detail_that_is_not_text_names_no_diagnosis(): def test_a_request_is_sent_again_only_for_what_sending_it_again_can_fix(): - """A 500 is not retried. The payload and the model are the same on every - attempt, so a server fault reproduces. Caller mistakes this endpoint - reports as a 500 are reclassified from the body, but only after three more - uploads of the whole gathered context. Overload, - gateway and timeout are the transient ones and are still retried.""" + """A connection error or a 5xx is retried with backoff, up to + ``max_retries``; overload is too. A 422 is the caller's, and sending the + same request again cannot fix it.""" for status, sends in ( - (500, 1), + (500, 4), (429, 4), (502, 4), (503, 4), diff --git a/packages/structured-data-models-client/tests/test_wording.py b/packages/structured-data-models-client/tests/test_wording.py index 12ac5eef7..2ca7c8b3e 100644 --- a/packages/structured-data-models-client/tests/test_wording.py +++ b/packages/structured-data-models-client/tests/test_wording.py @@ -515,16 +515,10 @@ def _users() -> Schema: def test_the_context_size_refused_is_named_as_the_caller_gave_it() -> None: # The old client's words, with the value as given, however large - # (review r1, P1-2 and P1-3). + # (review r1, P1-2 and P1-3). No size past one is refused. query = PredictiveQuery( 'PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', _users() ) - with pytest.raises(StructuredDataError) as past: - query.materialize(indices=[1], num_context_samples=10_001) - assert str(past.value).split('] ', 1)[1] == ( - 'num_context_samples is 10,001, and the endpoint accepts at most ' - '10,000 context rows' - ) with pytest.raises(StructuredDataError) as below: query.materialize(indices=[1], num_context_samples=-(10**100)) assert str(below.value).split('] ', 1)[1] == ( diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 44f475790..0b8998b74 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:d6ca0faa0176c2901562e9c734ebefee43d8d874c4b22276ea8e7ab260e507d6 sdm-contract-72.json +72 sha256:ae5e8122e4582b2bcbd186b1e26d77fd665b8ab7c085b39b65d1f46313025a41 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index e59e371ab..328f6a56c 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -7897,22 +7897,6 @@ ], "format": "uint32", "minimum": 0 - }, - "most_entities": { - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 0 - }, - "most_calls": { - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0 } }, "additionalProperties": false, @@ -14430,24 +14414,6 @@ "format": "uint32", "minimum": 0 }, - "most_entities": { - "description": "The most entities one `materialize` gathers, across the calls it is\nasked in, which the host lets its caller set (`main`'s\n`MOST_ENTITIES`); 100,000 where it names none (design choice 1,\nmulti-call `predict`, read which asks a request of more\nrows than one call scores in several calls; until then the core\nrefuses more than one call's rows, a lower bound than either).", - "type": [ - "integer", - "null" - ], - "format": "uint64", - "minimum": 0 - }, - "most_calls": { - "description": "The most model calls one request is asked in (`main`'s\n`MOST_CALLS`); 100 where the host names none (read as\n`most_entities`).", - "type": [ - "integer", - "null" - ], - "format": "uint32", - "minimum": 0 - }, "sampler_state": { "description": "The sampler state the materializer's last run left, which the host\nhands back as it took it; none for a materializer's\nfirst run.", "anyOf": [ @@ -17559,7 +17525,6 @@ "undeclared", "seed_past_range", "big_integer_past_type", - "lag_past_payload", "date_literal_packed", "local_zone_unread", "clock_past_years", @@ -17725,9 +17690,6 @@ "no_entity_passed_several", "anchor_too_late_several", "forecast_top_k", - "most_entities", - "most_entities_parts", - "most_calls", "not_yet_materialize", "missing_index_table", "missing_index_tables", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index bfba5b9e8..dc9b0d74b 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -941,7 +941,6 @@ sentences! { Undeclared = "undeclared", E::UnsupportedRequest; SeedPastRange = "seed_past_range", E::InvalidRequest; BigIntegerPastType = "big_integer_past_type", E::ValueOutOfRange; - LagPastPayload = "lag_past_payload", E::InvalidRequest; DateLiteralPacked = "date_literal_packed", E::InvalidRequest, rule "date_literal_packed"; LocalZoneUnread = "local_zone_unread", E::InvalidRequest; ClockPastYears = "clock_past_years", E::InvalidRequest; @@ -1107,9 +1106,6 @@ sentences! { NoEntityPassedSeveral = "no_entity_passed_several", E::InvalidRequest; AnchorTooLateSeveral = "anchor_too_late_several", E::InvalidRequest; ForecastTopK = "forecast_top_k", E::InvalidRequest, rule "forecast_top_k"; - MostEntities = "most_entities", E::UnsupportedRequest, rule "most_entities"; - MostEntitiesParts = "most_entities_parts", E::UnsupportedRequest, rule "most_entities"; - MostCalls = "most_calls", E::UnsupportedRequest, rule "most_calls"; NotYetMaterialize = "not_yet_materialize", E::UnsupportedRequest; MissingIndexTable = "missing_index_table", W::MissingIndices; MissingIndexTables = "missing_index_tables", W::MissingIndices; diff --git a/rust/crates/sdm-contract/src/frames.rs b/rust/crates/sdm-contract/src/frames.rs index a84df1c27..9e2c4c48f 100644 --- a/rust/crates/sdm-contract/src/frames.rs +++ b/rust/crates/sdm-contract/src/frames.rs @@ -83,8 +83,6 @@ pub struct FrameCall { #[serde(default)] pub options: Vec, pub most_parts: Option, - pub most_entities: Option, - pub most_calls: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 99cc93197..07b8954e2 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -1319,19 +1319,6 @@ pub struct Materialize { /// `too_many_parts`. #[serde(default, skip_serializing_if = "Option::is_none")] pub most_parts: Option, - /// The most entities one `materialize` gathers, across the calls it is - /// asked in, which the host lets its caller set (`main`'s - /// `MOST_ENTITIES`); 100,000 where it names none (design choice 1, - /// multi-call `predict`, read which asks a request of more - /// rows than one call scores in several calls; until then the core - /// refuses more than one call's rows, a lower bound than either). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub most_entities: Option, - /// The most model calls one request is asked in (`main`'s - /// `MOST_CALLS`); 100 where the host names none (read as - /// `most_entities`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub most_calls: Option, /// The sampler state the materializer's last run left, which the host /// hands back as it took it; none for a materializer's /// first run. diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index a3b2000af..2b5a12f57 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -201,7 +201,6 @@ advice! { EmptyRequest = "empty_request", []; GiveAHop = "give_a_hop", []; KeepOneLink = "keep_one_link", ["source_table", "target_table", "others"]; - MaterializeCall = "materialize_call", []; MissingTime = "missing_time", []; NameCandidates = "name_candidates", ["column"]; NamePrimaryKey = "name_primary_key", ["table"]; @@ -740,9 +739,6 @@ pub fn template(sentence: Sentence) -> Template { S::BigIntegerPastType => said( r#"column {column!r} holds a value that {admitted_type}, the type it was described as, cannot hold"#, ), - S::LagPastPayload => said( - r#"the {lag_timesteps:,} past steps lag_timesteps asks for make the request larger than the {limit:,} bytes the endpoint takes; ask for fewer steps"#, - ), S::DateLiteralPacked => said( r#"{query!r} is not a query a relational model can answer: {literal} is not a valid instance of time"#, ), @@ -1187,15 +1183,6 @@ pub fn template(sentence: Sentence) -> Template { S::ForecastTopK => said( r#"top_k is how many candidates a ranking keeps, and a forecast ranks nothing; it gives back one value per step"#, ), - S::MostEntities => said( - r#"{entities:,} entities is past the {limit:,} one {advice:materialize_call()} will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time"#, - ), - S::MostEntitiesParts => said( - r#"{entities:,} entities gathered {gathered_times:,} times over, {total:,} in all, is past the {limit:,} one {advice:materialize_call()} will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time"#, - ), - S::MostCalls => said( - r#"{entities:,} entities are gathered as {calls:,} calls of {rows_per_call:,}, and every one is built, with its own context, before any is sent. That is past the {limit:,} calls this will hold at once; ask about fewer entities at a time"#, - ), S::NotYetMaterialize => said( r#"materializing this request needs the {rule!r} rule, which is not one this client's core supports yet"#, ), diff --git a/rust/crates/sdm-contract/tests/corpus/mod.rs b/rust/crates/sdm-contract/tests/corpus/mod.rs index 79e7f9793..ee94a7a76 100644 --- a/rust/crates/sdm-contract/tests/corpus/mod.rs +++ b/rust/crates/sdm-contract/tests/corpus/mod.rs @@ -151,8 +151,6 @@ pub fn messages() -> Vec { values: None, options: vec!["num_neighbors".into()], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: Some(SamplerState { version: 1, diff --git a/rust/crates/sdm-contract/tests/wording.rs b/rust/crates/sdm-contract/tests/wording.rs index 0baf8070f..bc0113cec 100644 --- a/rust/crates/sdm-contract/tests/wording.rs +++ b/rust/crates/sdm-contract/tests/wording.rs @@ -377,7 +377,6 @@ fn the_sentences_the_hosts_choose_between_today_are_the_core_s() { Sentence::TimeColumnsRefusedEmpty, ), (Sentence::KeyNotNames, Sentence::KeyNotNamesGiven), - (Sentence::MostEntities, Sentence::MostEntitiesParts), (Sentence::BatchedPartForecast, Sentence::BatchedPartRanking), ( Sentence::CandidatesUnscoredOne, diff --git a/rust/crates/sdm-core/src/dispatch.rs b/rust/crates/sdm-core/src/dispatch.rs index 6f5f52a00..dbce2b3b4 100644 --- a/rust/crates/sdm-core/src/dispatch.rs +++ b/rust/crates/sdm-core/src/dispatch.rs @@ -779,8 +779,6 @@ pub(crate) fn materialize( values, options: request.options, most_parts: request.most_parts, - most_entities: request.most_entities, - most_calls: request.most_calls, facts, zones: Arc::clone(&zones), #[cfg(feature = "test-ops")] diff --git a/rust/crates/sdm-core/src/lag.rs b/rust/crates/sdm-core/src/lag.rs index 55812a9e5..715e7534c 100644 --- a/rust/crates/sdm-core/src/lag.rs +++ b/rust/crates/sdm-core/src/lag.rs @@ -34,23 +34,15 @@ //! than their own; the core puts each on its own row ( fix 1, a //! registered difference decided under ). -use sdm_contract::{CoreError, Detail, Scalar, Sentence}; +use sdm_contract::{CoreError, Sentence}; use sdm_engine::labels::{self, Labels, Target}; use sdm_engine::sample::out_of_range; use sdm_engine::time::offset::Offset; -use sdm_request::payload::{LagColumn, LagValues, PAYLOAD_BYTES}; +use sdm_request::payload::{LagColumn, LagValues}; /// The field a lag's refusal names. const FIELD: &str = "lag_timesteps"; -/// The fewest bytes a lagged column adds to a payload: its declaration in -/// the instance table's schema (`"__kumo_arl0__": {"dtype": "bool"}`). -const COLUMN_BYTES: usize = 30; - -/// The fewest bytes one lagged cell adds: the shortest value (`0.0`, `true` -/// or `null`) and its separator. -const CELL_BYTES: usize = 4; - /// `offset` taken `times` times, as pandas multiplies a `DateOffset`: a /// count of its unit `times` as large; past what the count holds, refused /// as a time out of range naming `lag_timesteps`, as pandas overflows. @@ -143,33 +135,6 @@ pub(crate) struct Lag { } impl Lag { - /// Refuses lags that would make a payload of `rows` instance rows - /// larger than the endpoint takes, before any column is computed, as - /// the payload's limit refuses it. - pub fn fits(&self, rows: usize) -> Result<(), CoreError> { - let least = usize::try_from(self.steps) - .ok() - .and_then(|steps| { - rows.checked_mul(CELL_BYTES)? - .checked_add(COLUMN_BYTES)? - .checked_mul(steps) - }) - .unwrap_or(usize::MAX); - if least > PAYLOAD_BYTES { - return Err(CoreError::new( - Sentence::LagPastPayload, - "the request is larger than the endpoint takes", - ) - .with_field("field", Detail::text("payload")) - .with_field("lag_timesteps", Detail::Scalar(Scalar::UInt64(self.steps))) - .with_field( - "limit", - Detail::Scalar(Scalar::UInt64(PAYLOAD_BYTES as u64)), - )); - } - Ok(()) - } - /// The lagged columns of the examples at `kept` (positions among the /// examples the lags were read for: the whole request's, the gathered /// context before narrowing and every entity predicted for), @@ -239,6 +204,7 @@ impl Lag { #[cfg(test)] mod tests { use super::*; + use sdm_contract::Detail; use sdm_contract::ErrorCode; use sdm_contract::pql::TimeUnit; use sdm_engine::labels::{Aggregation, Constant, RelOp}; @@ -520,8 +486,7 @@ mod tests { } /// Steps that reach outside what nanoseconds hold, or past what a count - /// holds, are refused naming `lag_timesteps`; a payload they would make - /// larger than the endpoint takes, before anything is computed. + /// holds, are refused naming `lag_timesteps`. #[test] fn too_many_steps_are_refused_by_name() { for steps in [100_000, u64::MAX] { @@ -529,21 +494,5 @@ mod tests { assert_eq!(e.code, ErrorCode::ValueOutOfRange); assert_eq!(e.fields["field"], Detail::text(FIELD)); } - let lag = single( - 10_000_000, - Aggregation::Count, - Offset::of(0, TimeUnit::Seconds), - Offset::of(1, TimeUnit::Seconds), - None, - vec![0, 0], - Rows { - batches: vec![], - values: labels::Values::Ones(0), - times: vec![], - }, - ); - let e = lag.fits(2).err().unwrap(); - assert_eq!(e.code, ErrorCode::InvalidRequest); - assert_eq!(e.fields["field"], Detail::text("payload")); } } diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index 7e1912f94..def8a1652 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -139,14 +139,6 @@ const DEFAULT_SEED: u128 = 42; /// How many times as many candidates as examples a run tries /// (`max_pq_iterations`' default). const ITERATIONS: usize = 10; -/// The most entities one `materialize` gathers where the host names no -/// limit (`main`'s `MOST_ENTITIES`): every call's payload is built, with -/// its own context, before any is sent, so the whole population is held at -/// once. -pub const MOST_ENTITIES: u64 = 100_000; -/// The most model calls one request is gathered as where the host names no -/// limit (`main`'s `MOST_CALLS`): a context is gathered for every call. -pub const MOST_CALLS: u32 = 100; /// How many examples the reference labels at once, stopping once it has /// the run mode's (`_sample_target_set`'s `batch_size`): the first batch /// holds every example a context takes whenever all of it is labelled. @@ -179,12 +171,6 @@ pub(crate) struct Inputs { pub options: Vec, /// The most steps a forecast is asked as (`Materialize.most_parts`). pub most_parts: Option, - /// The most entities one `materialize` gathers - /// (`Materialize.most_entities`). - pub most_entities: Option, - /// The most calls one request is gathered as - /// (`Materialize.most_calls`). - pub most_calls: Option, pub facts: HostFacts, /// The zones the run resolves names with: the host's local zone, which /// places a date literal's two-digit year (`pql::local_year`). @@ -2895,7 +2881,6 @@ pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result, -) -> Result<(), CoreError> { - let most = most.unwrap_or(MOST_ENTITIES); - let total = entities.saturating_mul(times); - if total <= most { - return Ok(()); - } - Err(CoreError::new( - if times == 1 { - Sentence::MostEntities - } else { - Sentence::MostEntitiesParts - }, - "more entities were asked for than one materialize holds at once", - ) - .with_field("field", Detail::text("indices")) - .with_field("entities", Detail::Scalar(Scalar::UInt64(entities))) - .with_field("gathered_times", Detail::Scalar(Scalar::UInt64(times))) - .with_field("total", Detail::Scalar(Scalar::UInt64(total))) - .with_field("limit", Detail::Scalar(Scalar::UInt64(most)))) -} - /// A request's payload and query, attached. pub(crate) fn attach_request( asker: &Asker, @@ -3101,8 +3056,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result most as usize { - return Err(CoreError::new( - Sentence::MostCalls, - "the entities asked for are gathered as more calls than one materialize holds at once", - ) - .with_field("field", Detail::text("indices")) - .with_field("entities", count(num_test)) - .with_field("calls", count(calls)) - .with_field("rows_per_call", count(per_call)) - .with_field("limit", Detail::Scalar(Scalar::UInt32(most)))); - } // How many candidates the call asked be kept (`_candidates_to_rank`, // ), which the engine writes into the task; then the inference // options, read for the task's config class, as the engine reads them @@ -3913,9 +3847,6 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result) -> Operation { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }) diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 75a4e644a..09a74fd46 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -240,8 +240,6 @@ fn materialize(held: &HeldSchema) -> Run { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }); diff --git a/rust/crates/sdm-core/tests/materialize.rs b/rust/crates/sdm-core/tests/materialize.rs index 4cb18f0a4..b897b2fa0 100644 --- a/rust/crates/sdm-core/tests/materialize.rs +++ b/rust/crates/sdm-core/tests/materialize.rs @@ -228,8 +228,6 @@ fn materialize_start( values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }); @@ -1110,11 +1108,6 @@ fn the_options_the_host_passes_through_are_refused_by_name() { for (asked, code) in [ ("0", ErrorCode::InvalidRequest), ("-3", ErrorCode::InvalidRequest), - ("10001", ErrorCode::UnsupportedRequest), - ( - "1267650600228229401496703205376", - ErrorCode::UnsupportedRequest, - ), ("2.5", ErrorCode::InvalidRequest), ] { let err = valued( diff --git a/rust/crates/sdm-core/tests/predict.rs b/rust/crates/sdm-core/tests/predict.rs index 3a37675bd..6d2ec01c5 100644 --- a/rust/crates/sdm-core/tests/predict.rs +++ b/rust/crates/sdm-core/tests/predict.rs @@ -1260,40 +1260,85 @@ fn an_output_named_and_left_out_is_noted() { } /// The call's size: the core writes the call's bytes, which the host sends -/// unchanged, and refuses a call past the endpoint's limit before the model -/// is called (register PD-2, fix round 1); counted to the byte, -/// with characters past ASCII counted as the bytes written. +/// unchanged (register PD-2), and holds a call to no size of its own. #[test] -fn a_call_past_the_limit_is_refused_before_it_is_sent() { - // Pad a non-truncated metadata field; diagnostic query clipping is independent. - let padded = |padding: &str| { - payload_text().replace( - "\"metadata\":{", - &format!("\"metadata\":{{\"caller_note\":\"{padding}\","), +fn a_call_of_any_size_is_sent() { + let padding = "a".repeat(31 * 1024 * 1024); + let payload = payload_text().replace( + "\"metadata\":{", + &format!("\"metadata\":{{\"caller_note\":\"{padding}\","), + ); + let mut run = predict_run(REQUEST_STATE_VERSION, payload, "kumo-relational").unwrap(); + let (_, body) = call(&mut run); + assert!(body.len() > 31 * 1024 * 1024); +} + +/// The model's answer to a call of the instance ids `ids`, each its row. +fn rows_answered(ids: &[i64]) -> ModelResult { + let predictions: Vec = ids + .iter() + .enumerate() + .map(|(at, id)| format!(r#"{{"id":"{id}","row_index":{at},"prediction":1.0}}"#)) + .collect(); + ModelResult::Response { + status: 200, + content_type: "application/json".into(), + body: format!( + r#"{{"id":"r1","model":"m","metadata":{{}},"predictions":[{}]}}"#, + predictions.join(",") ) - }; - let called = |payload: String| -> Result { - let mut run = predict_run(REQUEST_STATE_VERSION, payload, "kumo-relational").unwrap(); - match run.next()? { - Step::CallModel(call) => match run.take_attachment(call.body) { - Some(Attachment::Bytes(body)) => Ok(body.len()), - _ => panic!("the body is attached"), - }, - other => panic!("not a call: {other:?}"), + .into_bytes(), + } +} + +fn refused_with(status: u16) -> ModelResult { + ModelResult::Response { + status, + content_type: "application/json".into(), + body: b"{}".to_vec(), + } +} + +/// The rows a call scores: its predict half's instance ids. +fn scored(body: &[u8]) -> Vec { + let read: serde_json::Value = serde_json::from_slice(body).unwrap(); + read["predict"]["instance_table"]["rows"] + .as_array() + .unwrap() + .iter() + .map(|row| row[0].as_i64().unwrap_or(-1)) + .collect() +} + +/// A call refused as too large (413), or by a gateway (502, 504), or timed +/// out (the host's 408), is asked again in halves, each with the whole +/// context, and the answers are numbered as one. +#[test] +fn a_call_too_large_or_too_slow_is_asked_again_in_halves() { + for status in [408, 413, 502, 504] { + let mut run = + predict_run(REQUEST_STATE_VERSION, payload_text(), "kumo-relational").unwrap(); + let (id, whole) = call(&mut run); + assert_eq!(scored(&whole).len(), 2); + run.submit_model(id, refused_with(status)).unwrap(); + for _ in 0..2 { + let (id, half) = call(&mut run); + let read: serde_json::Value = serde_json::from_slice(&half).unwrap(); + let sent: serde_json::Value = serde_json::from_slice(&whole).unwrap(); + assert_eq!(read["context"], sent["context"], "{status}"); + let ids = scored(&half); + assert_eq!(ids.len(), 1, "{status}"); + run.submit_model(id, rows_answered(&ids)).unwrap(); } - }; - let limit = sdm_request::payload::PAYLOAD_BYTES; - let base = called(padded("")).unwrap(); - // ASCII to the limit, exactly: taken; one byte more: refused. - assert_eq!(called(padded(&"a".repeat(limit - base))).unwrap(), limit); - let past = called(padded(&"a".repeat(limit - base + 1))).unwrap_err(); - assert_eq!(past.code, ErrorCode::InvalidRequest); - assert_eq!(past.fields["field"], text("call")); - // Three-byte characters, each counted as its three bytes written. - let euros = (limit - base) / 3; - let rest = limit - base - 3 * euros; - let at = format!("{}{}", "\u{20ac}".repeat(euros), "a".repeat(rest)); - assert_eq!(called(padded(&at)).unwrap(), limit); - let over = format!("{}{}", "\u{20ac}".repeat(euros + 1), "a".repeat(rest)); - assert!(called(padded(&over)).is_err()); + assert!(matches!(run.next().unwrap(), Step::Done { .. }), "{status}"); + } +} + +/// A `422`, or any refusal not of a size, is not asked again: the run ends +/// with the endpoint's refusal. +#[test] +fn a_422_is_not_asked_again() { + let (_, end) = answered(422, b"{}"); + let error = end.unwrap_err(); + assert_eq!(error.fields["status"], Detail::Scalar(Scalar::UInt16(422))); } diff --git a/rust/crates/sdm-core/tests/serving.rs b/rust/crates/sdm-core/tests/serving.rs index 92f611d81..6fefd590c 100644 --- a/rust/crates/sdm-core/tests/serving.rs +++ b/rust/crates/sdm-core/tests/serving.rs @@ -349,10 +349,10 @@ fn each_call_is_wrapped_as_the_endpoint_reads_it_and_answered_from_its_row() { } #[test] -fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { - // The first call fits; the second, larger, does not: it is refused - // when it would be sent, carrying the first call's answers, as the old - // transport weighs each call as it sends it. +fn a_call_past_the_endpoint_s_limit_is_split_up_front_and_one_row_past_it_refused() { + // A serving endpoint drops a call too large as a connection, so a call + // past its limit is split before anything is sent, each part with the + // whole context; a call of one row past it is refused, nothing sent. let mut probe = predict(&[&[10]], serving(16_000_000)); let Ok(Step::CallModel(call)) = probe.next() else { panic!("a call"); @@ -367,18 +367,25 @@ fn a_call_past_the_caller_s_limit_is_refused_before_it_is_sent() { retryable: false, }; let (end, bodies) = drive(&mut run, None, &failure); - assert_eq!(bodies.len(), 1); - assert_eq!(bodies[0], one); - let error = end.unwrap_err(); - assert_eq!(error.sentence, Sentence::ServingPastLimit); - assert_eq!(error.code, ErrorCode::InvalidRequest); - assert_eq!(error.fields["limit"], Detail::Scalar(Scalar::UInt64(limit))); - assert_eq!(count(&error, "calls_answered"), Some(1)); - // A limit the first call misses: nothing is sent. + assert_eq!(bodies.len(), 4); + assert!(bodies.iter().all(|body| body.len() as u64 <= limit)); + let Ok(Step::Done { + result: OperationResult::Predict(predicted), + }) = end + else { + panic!("the run predicts: {end:?}"); + }; + let result = arrow(&mut run, predicted.result); + assert_eq!(int64s(&result, "row_index"), (0..4).collect::>()); let mut run = predict(&[&[10]], serving(limit - 1)); let (end, bodies) = drive(&mut run, None, &failure); assert!(bodies.is_empty()); - assert_eq!(end.unwrap_err().sentence, Sentence::ServingPastLimit); + let error = end.unwrap_err(); + assert_eq!(error.sentence, Sentence::ServingPastLimit); + assert_eq!( + error.fields["limit"], + Detail::Scalar(Scalar::UInt64(limit - 1)) + ); } #[test] diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index 23fa7ebe7..8dcb873ca 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -908,8 +908,6 @@ fn materialize_query( values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }), diff --git a/rust/crates/sdm-host/tests/runs.rs b/rust/crates/sdm-host/tests/runs.rs index 94393bf4e..01ea45f51 100644 --- a/rust/crates/sdm-host/tests/runs.rs +++ b/rust/crates/sdm-host/tests/runs.rs @@ -720,8 +720,6 @@ fn request_state_leaves_one_run_as_bytes_and_returns_under_the_host_s_id() { values: None, options: vec![], most_parts: None, - most_entities: None, - most_calls: None, candidates: None, sampler_state: None, }), diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index 80afdb7b0..716373a7c 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -79,11 +79,6 @@ pub const RESERVED: [&str; 2] = [INSTANCE_ID, ANCHOR_TIME]; const NO_ANCHOR: i64 = -9_223_372_036_854_775_807; /// The largest integer a JSON number carries exactly in every reader. const JSON_SAFE: i128 = 9_007_199_254_740_991; -/// The bytes a payload's lagged columns are held to before any is computed -/// (`_MAX_SIZE`, 30 MiB), a bound on the work `lag_timesteps` asks for: a -/// call itself is held to no size, and is asked in halves where the -/// endpoint refuses it as too large. -pub const PAYLOAD_BYTES: usize = 30 * 1024 * 1024; /// One column of a sampled table: as sampled, or a text feature as the /// reference's sampler sends it, each value its words diff --git a/rust/crates/sdm-testkit/src/host.rs b/rust/crates/sdm-testkit/src/host.rs index 722c21124..72afe4f7d 100644 --- a/rust/crates/sdm-testkit/src/host.rs +++ b/rust/crates/sdm-testkit/src/host.rs @@ -1794,8 +1794,6 @@ fn start_materialize( values: Some(AttachmentId(6)), options: vec![], most_parts: None, - most_entities: None, - most_calls: None, sampler_state: sampler .as_ref() .and_then(|held| held.as_ref()) diff --git a/rust/crates/sdm-testkit/tests/payload_rules.rs b/rust/crates/sdm-testkit/tests/payload_rules.rs index cc90a3b3f..3dfa5b642 100644 --- a/rust/crates/sdm-testkit/tests/payload_rules.rs +++ b/rust/crates/sdm-testkit/tests/payload_rules.rs @@ -190,12 +190,12 @@ fn base_multiclass_s_classification_payload_and_leakage_are_the_reference_s() { } #[test] -fn more_entities_than_one_materialize_holds_are_refused_by_its_limit() { - // The entity count, on base.regression. the core takes up to - // 100,000 entities, in calls of at most 10,000, as the reference at - // 6298913c (main's 1fc5a686): 1,001 entities are refused INVALID_REQUEST - // for the reference's reason (the base holds 8 users), never by a limit - // (B-3, retired). +fn no_entity_count_is_refused_by_a_limit() { + // The entity count, on base.regression: the core holds a materialize + // to no count, gathering as many calls as the entities need. 1,001 + // entities are refused INVALID_REQUEST for the reference's reason (the + // base holds 8 users), never by a limit (B-3, retired), and so are + // 100,001. let want = expected("base.regression.entities-1001"); assert_eq!(want["outcome"], "refused"); assert_eq!(want["code"], "INVALID_REQUEST"); @@ -211,26 +211,12 @@ fn more_entities_than_one_materialize_holds_are_refused_by_its_limit() { .collect(); host::run(&case, &bases).unwrap() }; - let observed = with(1001); - let Answer::Error { code, fields, .. } = &observed.answer else { - panic!("not refused: {:?}", observed.answer); - }; - assert_eq!(code, "INVALID_REQUEST"); - assert_eq!(fields.get("field").map(String::as_str), Some("indices")); - // Past 100,000 the core's own limit refuses them before anything is - // read, naming it (`most_entities`), not a rule it does not hold yet. - let observed = with(100_001); - assert!( - observed.materialized.is_none(), - "100,001 entities were taken" - ); - let Answer::Error { code, fields, .. } = &observed.answer else { - panic!("not refused: {:?}", observed.answer); - }; - assert_eq!(code, "UNSUPPORTED_REQUEST"); - assert_eq!( - fields.get("rule").map(String::as_str), - Some("most_entities") - ); - assert_eq!(fields.get("field").map(String::as_str), Some("indices")); + for n in [1001, 100_001] { + let observed = with(n); + let Answer::Error { code, fields, .. } = &observed.answer else { + panic!("not refused: {:?}", observed.answer); + }; + assert_eq!(code, "INVALID_REQUEST", "{n}"); + assert_eq!(fields.get("field").map(String::as_str), Some("indices")); + } } diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index eb2d67eb4..ef34daace 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -169,10 +169,9 @@ one result, whose rows are numbered across the whole request, so `to_pandas()` gives the indices back in the order they were asked for. Pass `rows_per_call=` to ask in batches of another size. -Every batch is built before the first is sent, so one `materialize()` gathers -at most 100,000 entities, in at most 100 calls. Both are this client's own: -the endpoint is never asked for more than one call holds. A question asked -in parts is not batched at all: a ranking is already a call per candidate and +Every batch is built before the first is sent, each with its own context, so +the whole population is held in memory at once; the client sets no most on +it. A question asked in parts is not batched at all: a ranking is already a call per candidate and a forecast one per step, and batching a part would send that many times as many calls. @@ -264,8 +263,6 @@ change. The ones worth knowing: |---|---| | every candidate's context holds one class | each gathered entity reached it, or none did, so there is nothing to tell apart and nothing is sent; widen with `num_context_samples` or ask over a table of pairs. One such candidate among several is dropped with a warning instead, and named on `result.unscored_candidates` | | no entity existed at the anchor | the entity table's time column starts after the anchor, so nothing existed to ask about | -| more than 100,000 entities at once | every batch is built before any is sent; ask about fewer at a time | -| more than 100 calls at once | a context is gathered per call; ask about fewer entities, or raise `rows_per_call` if you lowered it | | more entities than one call scores, on a ranking or a forecast | each candidate or step is already a call, and batching one would send that many times as many; ask about fewer at a time | | an output this question cannot give | `rankings` on a regression, `explanation` on a client-assembled ranking | | `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation` | From 9c8a58dc95e062cdae39f4472480e37ee24d5c26 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Mon, 9 Nov 2026 07:47:54 +0000 Subject: [PATCH 5/8] Split single-table calls too, and read Databricks failures for splitting - A single-table request built directly (Python tabular materialize and Request, Java ArrowTabular) is asked through the same splitting as a relational one: in calls of 10,000 rows, each with the whole context, split up front at a Databricks endpoint's byte limit and in halves on a 408, 413, 502 or 504; the answers merged as one. The split logic is one generic `answered`, shared by both. - Databricks: 413 and 502 are kept so they split; a deadline spent waiting on a 429, 503 or connection is a 503, 504 only for a call that timed out (Python reads the SDK's TimeoutError cause). Both transports try a 500 once more. Python for_databricks_serving takes max_request_bytes. - An answer past 64 MiB is told as a 413, so the call is halved; a Java connect timeout is a connection failure, never split. - Each split is logged at INFO (sdm_core.predict): the call's rows, the status, and the new size or number of calls. - Stale retry comments fixed; tests for each of the above. Signed-off-by: Adi Wadaskar --- .../java/com/nvidia/sdm/ClientOptions.java | 2 +- .../com/nvidia/sdm/DatabricksServing.java | 19 +- .../nvidia/sdm/internal/HttpTransport.java | 14 +- .../com/nvidia/sdm/DatabricksServingTest.java | 28 ++- .../nvidia/sdm/internal/TransportTest.java | 15 +- .../src/sdm_client/_rust/predict.py | 14 +- .../src/sdm_client/client.py | 21 +- .../src/sdm_client/core/databricks_serving.py | 62 +++-- .../src/sdm_client/core/transport.py | 9 +- .../tests/test_databricks_splitting.py | 147 ++++++++++++ .../tests/test_resilient_calls.py | 107 +++++++++ rust/crates/sdm-core/src/direct.rs | 3 +- rust/crates/sdm-core/src/predict.rs | 220 +++++++++++++----- rust/crates/sdm-core/src/ranking.rs | 9 +- rust/crates/sdm-core/src/tabular_arrow.rs | 3 +- rust/crates/sdm-request/src/serving.rs | 9 +- rust/crates/sdm-request/tests/serving.rs | 4 +- tests/compatibility/manifest.json | 32 ++- tests/golden/public_api.json | 2 +- 19 files changed, 605 insertions(+), 115 deletions(-) create mode 100644 packages/structured-data-models-client/tests/test_databricks_splitting.py diff --git a/java/src/main/java/com/nvidia/sdm/ClientOptions.java b/java/src/main/java/com/nvidia/sdm/ClientOptions.java index 292fee843..732c6a3cc 100644 --- a/java/src/main/java/com/nvidia/sdm/ClientOptions.java +++ b/java/src/main/java/com/nvidia/sdm/ClientOptions.java @@ -64,7 +64,7 @@ public Builder timeout(Duration timeout) { return this; } - /** How many times a transient failure (408, 429, 502, 503, 504, a dropped connection) is retried. */ + /** How many times a transient failure (408, 429, 500, 502, 503, 504, a dropped connection) is retried. */ public Builder maxRetries(int maxRetries) { this.maxRetries = maxRetries; return this; diff --git a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java index edfa4e823..605774da2 100644 --- a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java +++ b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java @@ -202,11 +202,13 @@ boolean healthReady() throws Host.EndpointFailed, Host.TransportFailed { private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.TransportFailed { long deadline = System.nanoTime() + timeout.toNanos(); + boolean retried500 = false; for (int attempt = 1; ; attempt++) { Authentication granted = authenticate(deadline); long remaining = deadline - System.nanoTime(); if (remaining <= 0) { - throw new Host.EndpointFailed(504, new byte[0]); + // The deadline spent before the call was made: the endpoint unavailable. + throw new Host.EndpointFailed(503, new byte[0]); } // Native SDK authentication can discover the workspace host. Validate it afterwards, // before any credential is sent, and on every attempt if the caller edits its config. @@ -236,11 +238,19 @@ private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.T if (status >= 200 && status < 300) { return response.body(); } - if (status != 429 && status != 503) { + // A 429 or a 503 is tried again until the deadline, as the SDK tries it; a + // 500 once more, as Python's transport tries it, then reported as it is. + if (status == 500 && !retried500) { + retried500 = true; + } else if (status != 429 && status != 503) { throw new Host.EndpointFailed(status, response.body()); } pause = retryAfter(response.headers().firstValue("Retry-After").orElse(null)); + } catch (java.net.http.HttpConnectTimeoutException e) { + // No connection made: tried again as any connection is, never split. + pause = Math.min(10, attempt) * 1000L; } catch (HttpTimeoutException e) { + // The call itself took too long: a 504, which the core splits on. throw new Host.EndpointFailed(504, new byte[0]); } catch (IOException e) { pause = Math.min(10, attempt) * 1000L; @@ -250,8 +260,9 @@ private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.T } pause += ThreadLocalRandom.current().nextLong(1000); if (System.nanoTime() + pause * 1_000_000L >= deadline) { - // The SDK's retries end as its own timeout, which Python reports as 504. - throw new Host.EndpointFailed(504, new byte[0]); + // The deadline spent waiting on a 429, a 503, a 500 or a connection: the + // endpoint is unavailable, a 503, which no smaller call fixes. + throw new Host.EndpointFailed(503, new byte[0]); } try { Thread.sleep(pause); diff --git a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java index 6e79d1950..234010241 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java +++ b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java @@ -211,6 +211,13 @@ else if (endpoint instanceof java.util.Map map) { HttpResponse response; try { response = attempt(built); + } catch (java.net.http.HttpConnectTimeoutException e) { + // No connection made: a failed transport, which no smaller call fixes. + if (attempt < retries) { + backoff(attempt, null); + continue; + } + throw new Failed("the request failed: " + e.getClass().getSimpleName()); } catch (HttpTimeoutException e) { if (attempt < retries) { backoff(attempt, null); @@ -249,13 +256,16 @@ else if (endpoint instanceof java.util.Map map) { * * @throws HttpTimeoutException the attempt did not end within the timeout * @throws IOException the endpoint could not be reached or its answer read - * @throws Failed the answer is past {@link #MAX_RESPONSE_BYTES}, or the call was interrupted + * @throws ModelEndpointException the answer is past {@link #MAX_RESPONSE_BYTES}: told as the + * endpoint's 413, the call too large, so the core asks its rows in smaller calls + * @throws Failed the call was interrupted */ private HttpResponse attempt(HttpRequest built) throws IOException, Failed { try { return exchange(client, built, info -> new Capped(), timeout.toNanos()); } catch (TooLarge e) { - throw new Failed("the response body exceeds the " + MAX_RESPONSE_BYTES + " byte limit"); + throw ModelEndpointException.of(413, ("the response body exceeds the " + MAX_RESPONSE_BYTES + + " byte limit").getBytes(java.nio.charset.StandardCharsets.UTF_8)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new Failed("the request was interrupted"); diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index f4de61970..e8c63c70f 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -461,7 +461,8 @@ void theFourBaseJ2ShapesUseTheSameCoreWithDirectAndServingEndpoints() throws Exc private static ModelEndpointException refused(Answer answer) throws Exception { try (Connection connection = Shop.open("duckdb"); - Workspace workspace = new Workspace(body -> answer); + // Answered twice: a 500 is tried once more, and a 502 asked in halves. + Workspace workspace = new Workspace(body -> answer, body -> answer); SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { Request request = asked(connection); StructuredDataException error = @@ -575,6 +576,28 @@ void aCallTheEndpointThrottlesIsTriedAgainWithTheSameBytes() throws Exception { } } + @Test + void a500IsTriedAgainAsA503Is() throws Exception { + try (Connection connection = Shop.open("duckdb"); + Workspace workspace = new Workspace(body -> Answer.of(500, "")); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); + assertEquals(2, workspace.bodies.size()); + } + } + + @Test + void aCallTheGatewayRefusesAsTooLargeIsAskedInHalves() throws Exception { + for (int status : new int[] {413, 502}) { + try (Connection connection = Shop.open("duckdb"); + Workspace workspace = new Workspace(body -> Answer.of(status, "")); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); + assertEquals(3, workspace.bodies.size(), "status " + status); + } + } + } + @Test void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { int port; @@ -591,7 +614,8 @@ void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { assertThrows( ModelEndpointException.class, () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); - assertEquals(504, error.status()); + // The deadline spent on connections never made: the endpoint unavailable, never split. + assertEquals(503, error.status()); assertNull(error.code()); } } diff --git a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java index 46fc651d3..4e3a44a38 100644 --- a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java +++ b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java @@ -29,7 +29,7 @@ * The model transport over {@code java.net.http}: the call is sent as the bytes given, * one past 16 MB included; a transient status or a 500 is tried again; a refusal is the endpoint's; * a timeout is told as a 408; an - * answer that is not JSON, or one past 64 MiB, is a failed transport. + * answer that is not JSON is a failed transport, and one past 64 MiB is told as a 413. */ class TransportTest { /** An endpoint answering {@code statuses} in turn, then 200, keeping every body. */ @@ -114,6 +114,19 @@ void anAnswerThatIsNoJsonObjectIsAFailedTransport() throws Exception { } } + @Test + void anAnswerPast64MiBIsToldAsA413() throws Exception { + byte[] answer = new byte[HttpTransport.MAX_RESPONSE_BYTES + 1]; + Arrays.fill(answer, (byte) ' '); + try (Endpoint endpoint = new Endpoint(answer)) { + ModelEndpointException refused = + assertThrows( + ModelEndpointException.class, + () -> endpoint.transport(0).predict("{}".getBytes(StandardCharsets.UTF_8))); + assertEquals(413, refused.status()); + } + } + @Test void anEmptyAnswerIsAnEmptyObject() throws Exception { try (Endpoint endpoint = new Endpoint(new byte[0])) { diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index 460dcad58..99cbcaf30 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -607,6 +607,7 @@ def model_call(body: bytes) -> bytes: unexpected: Exception | None = None transport_failed = False timed_out: StructuredDataError | None = None + too_large = 408 refused: NimRequestError | None = None try: # The core's call, its bytes as written (register PD-2), @@ -620,9 +621,14 @@ def model_call(body: bytes) -> bytes: except NimRequestError as refusal: refused = refusal except StructuredDataError as failed: - if failed.code == 'TRANSPORT_ERROR' and read_timed_out(failed): - # Told to the core as a 408, so it asks the rows again in - # halves; the timeout is what is raised if no half answers. + if failed.code == 'TRANSPORT_ERROR' and ( + read_timed_out(failed) + or failed.details.get('response_too_large') + ): + # Told to the core as a 408 or a 413, so it asks the rows + # again in halves; this is what is raised if no half answers. + if failed.details.get('response_too_large'): + too_large = 413 timed_out = StructuredDataError( failed.message, code=failed.code, details=failed.details ) @@ -649,7 +655,7 @@ def model_call(body: bytes) -> bytes: if transport_failed: raise TransportFailed if timed_out is not None: - raise Refused(timed_out, 408) + raise Refused(timed_out, too_large) raised: StructuredDataError | None = failure if refused is not None: # Built outside the handler, so nothing failing in the copy is diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 64e2d41e9..4660e7ab8 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -286,9 +286,9 @@ def __init__( timeout: Seconds to wait for each individual attempt, not for the call as a whole. max_retries: Transport-level retries of a transient failure (408, 429, - 502, 503, 504, or a dropped connection) with exponential - backoff. A 500 is not retried: the payload and the model are - the same on every attempt. ``POST /v1/sessions`` is excluded: + 500, 502, 503, 504, or a dropped connection) with exponential + backoff, a ``Retry-After`` honoured. ``POST /v1/sessions`` is + excluded: a re-sent create would orphan a pinned context on the endpoint. """ @@ -336,6 +336,7 @@ def for_databricks_serving( *, workspace_client: Any | None = None, timeout: float | None = None, + max_request_bytes: int | None = None, ) -> SDMClient: r"""Open a client against a Databricks Model Serving endpoint. @@ -367,6 +368,11 @@ def for_databricks_serving( timeouts. Given with a ``WorkspaceClient``, it is set on the copy, never on the caller's client; with a subclass or test double, which cannot be copied, it is refused. + max_request_bytes: The most bytes one call's wrapped body may + take, the gateway's 16 MB when omitted. The endpoint drops a + call past it as a connection, so a call past it is split + into smaller ones before it is sent, each with the whole + context. Raises: MissingExtraError: If ``databricks-sdk`` is not installed; install @@ -384,7 +390,14 @@ def for_databricks_serving( client = cls.__new__(cls) client._start( DatabricksServingTransport( - endpoint, workspace_client, timeout=timeout + endpoint, + workspace_client, + timeout=timeout, + **( + {} + if max_request_bytes is None + else {'max_request_bytes': max_request_bytes} + ), ) ) return client diff --git a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py index 66c4f302c..3c5bb33b6 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py +++ b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py @@ -31,6 +31,7 @@ _check_encodable, _diagnosis_in, _validate_limits, + read_timed_out, ) from sdm_client.errors import ( MissingExtraError, @@ -222,23 +223,29 @@ def predict_body(self, body: bytes) -> bytes: says, for the core to name; anything but an ``OSError`` escapes, as from:meth:`predict`.""" self._require_open() - try: - answered = self._workspace.api_client.do( - 'POST', - f'/serving-endpoints/{self._endpoint}/invocations', - headers={'Content-Type': 'application/json'}, - data=_Body(body), - raw=True, - ) - contents = answered['contents'] + # The SDK retries a 429 or a 503, not a 500, which this client tries + # once more as it does on its own endpoint. + for attempt in (0, 1): try: - return bytes(contents.read()) - finally: - contents.close() - # The SDK's errors and its retry deadline are all OSError, as are - # requests' own, as in predict. - except OSError as error: - raise _failed(error) from None + answered = self._workspace.api_client.do( + 'POST', + f'/serving-endpoints/{self._endpoint}/invocations', + headers={'Content-Type': 'application/json'}, + data=_Body(body), + raw=True, + ) + contents = answered['contents'] + try: + return bytes(contents.read()) + finally: + contents.close() + # The SDK's errors and its retry deadline are all OSError, as are + # requests' own, as in predict. + except OSError as error: + if attempt == 0 and _status_of(error) == 500: + continue + raise _failed(error) from None + raise AssertionError('unreachable') def _answered(self, payload: dict[str, Any]) -> str: r"""The served model's answer to ``payload``, as the text it @@ -566,6 +573,12 @@ def _response_body(reply: Any, endpoint: str) -> str: return body +#: The SDK's and ``requests``' names for a deadline reached. +_TIMEOUTS = frozenset( + {'DeadlineExceeded', 'OperationTimeout', 'TimeoutError', 'Timeout'} +) + + def _status_of(error: BaseException) -> int: r"""Map a Databricks SDK error onto an HTTP-equivalent status. @@ -583,12 +596,17 @@ def _status_of(error: BaseException) -> int: return 409 if names & {'TooManyRequests', 'RequestLimitExceeded'}: return 429 - if names & { - 'DeadlineExceeded', - 'OperationTimeout', - 'TimeoutError', - 'Timeout', - }: + if names & _TIMEOUTS: + # The SDK's deadline spent retrying a 429, a 503 or a connection is + # the endpoint being unavailable, not a call too slow to answer: + # 504 only where the call itself timed out, which a smaller call + # can fix. + cause = error.__cause__ + if cause is not None and not ( + {c.__name__ for c in type(cause).__mro__} & _TIMEOUTS + or read_timed_out(cause) + ): + return 503 return 504 if 'BadRequest' in names: return 400 diff --git a/packages/structured-data-models-client/src/sdm_client/core/transport.py b/packages/structured-data-models-client/src/sdm_client/core/transport.py index 95b401ec9..802dd6f5a 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/transport.py +++ b/packages/structured-data-models-client/src/sdm_client/core/transport.py @@ -31,10 +31,8 @@ _DEFAULT_TIMEOUT_SECONDS = 60.0 _DEFAULT_MAX_RETRIES = 3 _DEFAULT_BACKOFF_FACTOR = 0.5 -#: The transient ones. A 500 is not among them: the payload and the model are -#: the same on every attempt, so a server fault reproduces, and the caller -#: mistakes this endpoint reports as a 500 are reclassified from the body as -#: the caller's after three more uploads of the whole gathered context. +#: The statuses tried again with backoff, up to ``max_retries``: a timeout, +#: overload, and a server or gateway fault, a 500 included. _RETRY_STATUS = (408, 429, 500, 502, 503, 504) _RETRY_AFTER_MAX_SECONDS = 60 _MAX_BODY_SNIPPET = 512 @@ -419,10 +417,13 @@ def _read_capped(response: requests.Response, url: str) -> bytes: body = bytearray() for chunk in response.iter_content(_RESPONSE_CHUNK_BYTES): if len(body) + len(chunk) > _MAX_RESPONSE_BYTES: + # An answer too large to read is the call too large: a call of + # fewer rows is asked instead (``response_too_large``). raise StructuredDataError( f'Response body from {redact_url(url)} exceeds the ' f'{_MAX_RESPONSE_BYTES} byte limit', code='TRANSPORT_ERROR', + details={'response_too_large': True}, ) body.extend(chunk) return bytes(body) diff --git a/packages/structured-data-models-client/tests/test_databricks_splitting.py b/packages/structured-data-models-client/tests/test_databricks_splitting.py new file mode 100644 index 000000000..7ac1e3602 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_databricks_splitting.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A Databricks serving endpoint drops a call too large as a connection, so a +call past its byte limit is split before it is sent; a 413, a 502 or a call +that timed out is asked again in halves; a deadline spent waiting on an +unavailable endpoint is a 503, never split; and a 500 is tried once more.""" + +from __future__ import annotations + +import io +import json +from typing import Any + +import pytest +import requests +from test_databricks_serving import ( + ENDPOINT, + ApiClient, + Endpoints, + Workspace, + asked, +) + +from sdm_client import SDMClient +from sdm_client.core.databricks_serving import _status_of + + +def answer(sent: dict[str, Any]) -> dict[str, Any]: + r"""The served model's answer to ``sent``: one prediction per row.""" + table = sent['predict']['instance_table'] + keys = sent['schema']['instance_table'].get('primary_key') or [] + at = table['columns'].index(keys[0]) if keys else None + return { + 'object': 'prediction.response', + 'id': 'pred', + 'model': 'kumo-tabular', + 'metadata': {}, + 'predictions': [ + { + 'id': f'[{row[at]}]' if at is not None else str(index), + 'row_index': index, + 'prediction': 'yes', + } + for index, row in enumerate(table['rows']) + ], + } + + +class Failing(Endpoints): + r"""Raises each of ``failures`` on the calls in turn, then answers.""" + + def __init__(self, failures: list[BaseException]) -> None: + super().__init__(answer=answer) + self.failures = list(failures) + + def query(self, *, name: str, **kwargs: Any) -> Any: + if self.failures: + self.calls.append({'name': name, **kwargs}) + raise self.failures.pop(0) + return super().query(name=name, **kwargs) + + +class Sized(ApiClient): + r"""Records the bytes of each envelope posted.""" + + sizes: list[int] + + def do(self, method: str, path: str, *, headers, data, raw) -> Any: + body = data.read() + self.sizes.append(len(body)) + return super().do( + method, path, headers=headers, data=io.BytesIO(body), raw=raw + ) + + +def opened(failures=(), **kwargs: Any) -> tuple[SDMClient, Workspace]: + workspace = Workspace() + workspace.serving_endpoints = Failing(list(failures)) + workspace.api_client = Sized(workspace) + workspace.api_client.sizes = [] + client = SDMClient.for_databricks_serving( + ENDPOINT, workspace_client=workspace, **kwargs + ) + return client, workspace + + +class Status(OSError): + def __init__(self, status: int) -> None: + super().__init__(f'status {status}') + self.status_code = status + + +def test_max_request_bytes_is_passed_through(): + client, _ = opened(max_request_bytes=1234) + assert client._transport._max_request_bytes == 1234 + + +def test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent(caplog): + whole, workspace = opened() + whole.predict(asked(), model='kumo-tabular') + (size,) = workspace.api_client.sizes + caplog.set_level('INFO', logger='sdm_core') + split, workspace = opened(max_request_bytes=size - 1) + result = split.predict(asked(), model='kumo-tabular') + assert len(workspace.api_client.sizes) == 2 + assert all(sent <= size - 1 for sent in workspace.api_client.sizes) + assert result.to_pandas()['row_index'].tolist() == [0, 1] + assert any('smaller calls' in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize('status', [413, 502, 504]) +def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): + client, workspace = opened([Status(status)]) + result = client.predict(asked(), model='kumo-tabular') + calls = workspace.serving_endpoints.calls + assert len(calls) == 3 + sent = [ + json.loads(c['dataframe_records'][0]['request_json']) for c in calls + ] + assert [len(s['predict']['instance_table']['rows']) for s in sent] == [ + 2, + 1, + 1, + ] + assert result.to_pandas()['row_index'].tolist() == [0, 1] + + +def test_a_500_is_tried_once_more(): + client, workspace = opened([Status(500)]) + client.predict(asked(), model='kumo-tabular') + assert len(workspace.serving_endpoints.calls) == 2 + + +def test_a_deadline_spent_waiting_is_a_503_and_a_call_timed_out_a_504(): + def timed_out(cause: BaseException | None) -> TimeoutError: + error = TimeoutError('Timed out after 0:05:00') + error.__cause__ = cause + return error + + assert _status_of(timed_out(Status(429))) == 503 + assert _status_of(timed_out(Status(503))) == 503 + assert _status_of(timed_out(requests.ConnectionError('refused'))) == 503 + assert _status_of(timed_out(requests.ReadTimeout('slow'))) == 504 + assert _status_of(timed_out(None)) == 504 + assert _status_of(Status(413)) == 413 diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py index e9e5e7219..72a9a7555 100644 --- a/packages/structured-data-models-client/tests/test_resilient_calls.py +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -9,12 +9,16 @@ from __future__ import annotations import json +import threading +from collections.abc import Iterator +from contextlib import contextmanager import pytest import requests from test_limits_and_absent_outputs import gathered from test_tabular_direct_core import client +from sdm_client import SDMClient from sdm_client.core.transport import _to_nim_error from sdm_client.errors import StructuredDataError @@ -106,3 +110,106 @@ def test_a_call_of_one_row_refused_as_too_large_is_the_endpoint_s_refusal(): client(endpoint).predict(request, model='kumo-relational') assert len(endpoint.calls) == 1 assert 'context too large' in str(refused.value) + + +@contextmanager +def _nim(*failures: int) -> Iterator[tuple[SDMClient, list[dict]]]: + r"""A NIM on a real socket, reached over the client's own HTTP transport: + each call answered row by row, the first ones failed with ``failures`` + in turn, a 0 among them an answer too large to read.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + from test_databricks_splitting import answer + + calls: list[dict] = [] + script = list(failures) + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: + pass + + def _send(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + self._send(200, b'{"data": [{"id": "kumo-tabular"}]}') + + def do_POST(self) -> None: + length = int(self.headers.get('Content-Length') or 0) + payload = json.loads(self.rfile.read(length)) + calls.append(payload) + if script: + status = script.pop(0) + if status == 0: + self._send(200, b' ' * (64 * 1024 * 1024 + 1)) + else: + self._send(status, b'{"detail": "refused"}') + return + self._send(200, json.dumps(answer(payload)).encode()) + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread( + target=lambda: server.serve_forever(poll_interval=0.01), daemon=True + ) + thread.start() + try: + url = f'http://127.0.0.1:{server.server_address[1]}' + with SDMClient(url, max_retries=0) as nim: + yield nim, calls + finally: + server.shutdown() + server.server_close() + + +def _tabular(rows: int): + import numpy as np + import pandas as pd + + from sdm_client import Request + + context = pd.DataFrame( + { + 'row_id': np.arange(40), + 'feature': np.arange(40) % 7, + 'target': np.arange(40) % 2, + } + ) + scoring = pd.DataFrame( + {'row_id': np.arange(100, 100 + rows), 'feature': np.arange(rows) % 7} + ) + return Request( + context, + scoring, + target='target', + task='classification', + ) + + +def test_a_tabular_request_is_scored_in_calls_of_10_000_rows(): + with _nim() as (nim, calls): + result = nim.predict(_tabular(10_001), model='kumo-tabular') + assert [len(c['predict']['instance_table']['rows']) for c in calls] == [ + 10_000, + 1, + ] + assert all(c['context'] == calls[0]['context'] for c in calls) + assert list(result.to_pandas()['row_index']) == list(range(10_001)) + + +@pytest.mark.parametrize( + 'failure', [413, 502, 504, 0], ids=['413', '502', '504', 'answer-too-large'] +) +def test_a_tabular_call_too_large_is_asked_again_in_halves(failure, caplog): + caplog.set_level('INFO', logger='sdm_core') + with _nim(failure) as (nim, calls): + result = nim.predict(_tabular(4), model='kumo-tabular') + assert [len(c['predict']['instance_table']['rows']) for c in calls] == [ + 4, + 2, + 2, + ] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + assert any('smaller calls' in r.getMessage() for r in caplog.records) diff --git a/rust/crates/sdm-core/src/direct.rs b/rust/crates/sdm-core/src/direct.rs index e7ab051b0..ee2584f16 100644 --- a/rust/crates/sdm-core/src/direct.rs +++ b/rust/crates/sdm-core/src/direct.rs @@ -844,8 +844,7 @@ pub(crate) async fn predict( })); } let answer = - crate::predict::call_answer(&asker, payload.to_bytes(), request.model_endpoint.as_ref()) - .await?; + crate::predict::direct_answer(&asker, &payload, request.model_endpoint.as_ref()).await?; let mut payload = payload; if let Some(analysis) = analysis { payload.set("client_column_analysis", Value::Text(analysis)); diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 11bf19e11..8c9786104 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -436,7 +436,16 @@ pub(crate) async fn one( let mut calls = Vec::with_capacity(payloads.len()); let mut bodies = Vec::with_capacity(payloads.len()); for (payload, request_id) in payloads.iter().zip(&ids) { - for (call, body) in fitted(inputs, &outputs, payload, request_id)? { + let write = |call: &Value| { + body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + ) + }; + for (call, body) in fitted(inputs.endpoint.as_ref(), payload, &write)? { calls.push((call, request_id.as_str())); bodies.push(body); } @@ -475,10 +484,25 @@ pub(crate) async fn one( let response = PredictionResponse::read(&answer).map_err(refused)?; return layout(payload, &response, crate::pql::float_text); } - answered( - asker, inputs, &outputs, request_id, payload, bytes, &mut most, - ) - .await + let write = |call: &Value| { + sdm_request::predict::body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + ) + }; + let endpoint = inputs.endpoint.as_ref(); + let send = |call: Value, bytes: Vec| -> Sent<'_, Part> { + Box::pin(async move { self::call(asker, &call, bytes, endpoint).await }) + }; + let how = Asking { + write: &write, + send: &send, + merge: numbered_parts, + }; + answered(&how, payload.clone(), bytes, &mut most).await } .await; let answered = match response { @@ -513,30 +537,21 @@ pub(crate) async fn one( Ok((merged, outputs, scored)) } -/// `payload` as the calls it is sent in, each with its body: one call, or, -/// to a serving endpoint, as many as it is split into so that each one's -/// envelope fits the endpoint's `max_request_bytes` +/// `payload` as the calls it is sent in, each with its body (`write`): one +/// call, or, to a serving endpoint, whose refusal of a call too large is a +/// dropped connection, as many as it is split into up front so that each +/// one's envelope fits the endpoint's `max_request_bytes` /// ([`sdm_request::batching::fitted`]), each scoring a run of its rows with /// the whole context. A call of one row past it is refused. fn fitted( - inputs: &Inputs, - outputs: &[String], + endpoint: Option<&ModelEndpoint>, payload: &Value, - request_id: &str, + write: &Write<'_>, ) -> Result, CoreError> { - let write = |call: &Value| { - body( - call, - &inputs.model, - request_id, - outputs, - &inputs.inference_asked, - ) - }; let Some(ModelEndpoint::ServingEndpoint { name, max_request_bytes, - }) = &inputs.endpoint + }) = endpoint else { return Ok(vec![(payload.clone(), write(payload)?)]); }; @@ -549,7 +564,17 @@ fn fitted( Ok(body) }) }; - sdm_request::batching::fitted(payload, &mut fit) + let calls = sdm_request::batching::fitted(payload, &mut fit)?; + if calls.len() > 1 { + tracing::info!( + target: "sdm_core::predict", + rows = predict_rows(payload).unwrap_or(0), + calls = calls.len(), + limit = *max_request_bytes, + "a call past the serving endpoint's limit is sent as smaller calls" + ); + } + Ok(calls) } /// The statuses that say a call was too large or too slow to answer: the @@ -557,51 +582,69 @@ fn fitted( /// 504, which a call it gave up on comes back as. const TOO_LARGE: [u16; 4] = [408, 413, 502, 504]; -/// `payload`'s answer, its rows numbered from zero. Where the endpoint -/// refuses the call as too large or too slow ([`TOO_LARGE`]), its rows are -/// asked again in calls of half as many, each with the whole context, and -/// that size is kept in `most` for the calls after it. A call of one row is -/// not split: its refusal, the endpoint's own, stands, as does any other. -async fn answered( - asker: &Asker, - inputs: &Inputs, - outputs: &[String], - request_id: &str, - payload: &Value, +/// A call sent, its answer to come. +pub(crate) type Sent<'a, T> = + std::pin::Pin> + Send + 'a>>; + +/// How a call is asked and, split, put back together: the body a run of its +/// rows is sent as, the sending, and the answers merged, each with the row +/// of the call it starts at. +pub(crate) struct Asking<'a, T> { + pub write: &'a Write<'a>, + pub send: &'a Sender<'a, T>, + pub merge: Merge, +} + +/// The answers to runs of a call's rows, each with the row it starts at, +/// as the one answer to the call. +pub(crate) type Merge = fn(Vec<(usize, T)>) -> Result; + +/// The sending of a run of a call's rows, its body's bytes given. +pub(crate) type Sender<'a, T> = dyn Fn(Value, Vec) -> Sent<'a, T> + Sync + 'a; + +/// The body a run of a call's rows is sent as. +pub(crate) type Write<'a> = dyn Fn(&Value) -> Result + Sync + 'a; + +/// `payload`'s answer, its rows numbered from zero, sent as `bytes`. Where +/// the endpoint refuses the call as too large or too slow ([`TOO_LARGE`]), +/// its rows are asked again in calls of half as many, each with the whole +/// context, and that size is kept in `most` for the calls after it; a call +/// of more rows than `most` is asked in calls of that many. A call of one +/// row is not split: its refusal, the endpoint's own, stands, as does any +/// other. +pub(crate) async fn answered( + how: &Asking<'_, T>, + payload: Value, bytes: Vec, most: &mut Option, -) -> Result { - let rows = predict_rows(payload).unwrap_or(0); +) -> Result { + let rows = predict_rows(&payload).unwrap_or(0); let size = match *most { Some(most) if rows > most => most, - _ => match call(asker, payload, bytes, inputs.endpoint.as_ref()).await { + _ => match (how.send)(payload.clone(), bytes).await { Err(error) if rows > 1 && too_large(&error) => { let half = rows.div_ceil(2); + tracing::info!( + target: "sdm_core::predict", + rows, + status = ?error.fields.get(sdm_contract::error::REFUSED_STATUS), + size = half, + "a call refused as too large or too slow is asked again in smaller calls" + ); *most = Some(half); half } answer => return answer, }, }; - let mut parts = Vec::new(); + let mut answers = Vec::new(); for start in (0..rows).step_by(size) { let range = start..(start + size).min(rows); - let call = sdm_request::batching::predict_slice(payload, range)?; - let bytes = body( - &call, - &inputs.model, - request_id, - outputs, - &inputs.inference_asked, - )? - .to_bytes(); - let part = Box::pin(answered( - asker, inputs, outputs, request_id, &call, bytes, most, - )) - .await?; - parts.push(numbered_from(part, start)?); + let call = sdm_request::batching::predict_slice(&payload, range)?; + let bytes = (how.write)(&call)?.to_bytes(); + answers.push((start, Box::pin(answered(how, call, bytes, most)).await?)); } - merge(parts) + (how.merge)(answers) } /// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. @@ -612,6 +655,79 @@ fn too_large(error: &CoreError) -> bool { ) } +/// Parts of one call's rows, each numbered from the row it starts at. +fn numbered_parts(parts: Vec<(usize, Part)>) -> Result { + merge( + parts + .into_iter() + .map(|(start, part)| numbered_from(part, start)) + .collect::, _>>()?, + ) +} + +/// The answers to runs of one call's rows as the one answer to it: the +/// first's, with every answer's predictions in order, each numbered from +/// the row its run starts at (`row_index`). +fn merged_answers(answers: Vec<(usize, Value)>) -> Result { + let mut answers = answers.into_iter(); + let (first, mut whole) = answers + .next() + .ok_or_else(|| internal("a call answered by none"))?; + let mut predictions = Vec::new(); + for (start, answer) in std::iter::once((first, whole.clone())).chain(answers) { + let Some(Value::Array(items)) = answer.get("predictions") else { + // An answer that is not a list of predictions is left to be read + // and refused as it is. + return Ok(answer); + }; + for item in items { + let mut item = item.clone(); + if start > 0 + && let Some(Value::Number(at)) = item.get("row_index") + && let Ok(at) = at.parse::() + { + item.set("row_index", Value::Number((at + start as u64).to_string())); + } + predictions.push(item); + } + } + whole.set("predictions", Value::Array(predictions)); + Ok(whole) +} + +/// The answer to a call sent as it was built, a single-table request's: +/// asked in calls of at most [`sdm_request::options::PREDICT_ROWS`] rows, +/// each with the whole context, split up front to fit a serving endpoint +/// and in halves where the endpoint refuses one as too large or too slow +/// ([`answered`]), the answers merged as one ([`merged_answers`]). +pub(crate) async fn direct_answer( + asker: &Asker, + payload: &Value, + endpoint: Option<&ModelEndpoint>, +) -> Result { + let write = |call: &Value| Ok(call.clone()); + let send = |_: Value, bytes: Vec| -> Sent<'_, Value> { + Box::pin(call_answer(asker, bytes, endpoint)) + }; + let how = Asking { + write: &write, + send: &send, + merge: merged_answers, + }; + let mut most = usize::try_from(sdm_request::options::PREDICT_ROWS).ok(); + let mut answers = Vec::new(); + let mut start = 0; + for (call, body) in fitted(endpoint, payload, &write)? { + let rows = predict_rows(&call).unwrap_or(0); + answers.push(( + start, + answered(&how, call, body.to_bytes(), &mut most).await?, + )); + start += rows; + } + merged_answers(answers) +} + /// One payload's call, its body as written, sent and its answer laid out; /// to a serving endpoint, in its envelope, its answer read out of its row. async fn call( diff --git a/rust/crates/sdm-core/src/ranking.rs b/rust/crates/sdm-core/src/ranking.rs index 13b0b5e8f..2919078d4 100644 --- a/rust/crates/sdm-core/src/ranking.rs +++ b/rust/crates/sdm-core/src/ranking.rs @@ -168,14 +168,11 @@ pub(crate) async fn materialize( .map(|digits| digits.parse::().unwrap_or(u32::MAX)), }; // `_in_parts`, before anything is gathered, as the client refuses them - // (`main`'s 1fc5a686): the batch size asked for; a question over more - // entities than one call scores, since a part is one call and a + // (`main`'s 1fc5a686): the batch size asked for; and a question over + // more entities than one call scores, since a part is one call and a // candidate is reported unscored for the whole ranking, so one cold // batch would drop it from every row - // (`refuse_a_part_that_would_take_several_calls`); and the population - // each question gathers whole, held as many times over as there are - // questions (`refuse_more_entities_than_fit_in_memory`), past the - // host's limit, 100,000 where it names none. + // (`refuse_a_part_that_would_take_several_calls`). let per_call = sdm_request::options::rows_one_call_scores(asked_rows_per_call(inputs.values.as_ref())?); let entities = inputs.indices.as_ref().map_or(0, |data| { diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 4c092180e..6d00a3219 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -396,8 +396,7 @@ async fn predict( part.columns = analysis; return crate::predict::finish(asker, part, prepared.bound.outputs); } - let answer = - crate::predict::call_answer(asker, payload.to_bytes(), input.endpoint.as_ref()).await?; + let answer = crate::predict::direct_answer(asker, &payload, input.endpoint.as_ref()).await?; finish(asker, payload, answer, prepared.bound.outputs, analysis) } fn finish( diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index cb1f1c17c..b12486550 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -57,8 +57,9 @@ const STATUS_BY_CODE: &[(&str, u16)] = &[ ]; /// The HTTP statuses whose SDK error class the old client reports as the -/// status itself; any other it reports as 503, which callers retry. -const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 429, 504]; +/// status itself; any other it reports as 503, which callers retry. A 413, +/// a 502 or a 504 is kept, so the core asks the call's rows in halves. +const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 413, 429, 502, 504]; /// The call `call` (the core's own JSON) as the endpoint's envelope, the /// bytes a host posts, or `ServingPastLimit` when it is more than @@ -583,7 +584,9 @@ mod tests { #[test] fn a_status_is_reported_as_the_sdk_s_class_gives_it() { assert_eq!(reported_status(500, None), 503); - assert_eq!(reported_status(502, None), 503); + // Kept, so a call too large for the gateway is asked in halves. + assert_eq!(reported_status(502, None), 502); + assert_eq!(reported_status(413, None), 413); assert_eq!(reported_status(429, None), 429); assert_eq!(reported_status(400, Some("RESOURCE_DOES_NOT_EXIST")), 404); assert_eq!(reported_status(400, Some("BAD_REQUEST")), 400); diff --git a/rust/crates/sdm-request/tests/serving.rs b/rust/crates/sdm-request/tests/serving.rs index 6f86a0013..67c4af02d 100644 --- a/rust/crates/sdm-request/tests/serving.rs +++ b/rust/crates/sdm-request/tests/serving.rs @@ -696,7 +696,7 @@ fn an_http_failure_reads_as_the_sdk_reads_the_same_response() { ( 502, "Bad Gateway".to_owned(), - (Sentence::ServingFailed, 503, None), + (Sentence::ServingFailed, 502, None), ), ( 500, @@ -735,7 +735,7 @@ fn an_http_failure_reads_as_the_sdk_reads_the_same_response() { ]; for (status, body, reference) in cases { let expected = if status == 502 && body.starts_with("") { - (Sentence::ServingFailedCoded, 503, Some("BAD_GATEWAY")) + (Sentence::ServingFailedCoded, 502, Some("BAD_GATEWAY")) } else { reference }; diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index b795ebb9a..fdc3e15b9 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -3338,7 +3338,8 @@ "packages/structured-data-models-client/tests/test_databricks_serving.py::test_the_ambient_identity_is_used_when_no_client_is_given", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_a_workspace_that_cannot_authenticate_says_how_to_fix_it", "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_unusable_timeout_is_refused_at_construction", - "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not" + "packages/structured-data-models-client/tests/test_databricks_serving.py::test_an_answer_too_deep_to_read_is_said_so_and_not_json_is_not", + "packages/structured-data-models-client/tests/test_databricks_splitting.py" ], "java": [ "java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java#officialSdkAuthenticationRefreshesEveryHeaderWithoutRewritingTheCoreBytes", @@ -4748,7 +4749,7 @@ } } ], - "public_api_snapshot_sha256": "652bd2757ca1d8ed12360b795a5c7f80ce63978210306cee1319532503f9d031", + "public_api_snapshot_sha256": "5f1f001f45522d17f2a3527668d3313d5cd0d439ba7d21acedfdddf5b073a6d0", "api_inventory": { "python": { "sdm_api.Endpoint": { @@ -5262,7 +5263,17 @@ "capability": "sessions" }, "sdm_client.SDMClient.for_databricks_serving": { - "capability": "public-serving-refusals" + "capability": "public-serving-refusals", + "assertions": [ + { + "test": "packages/structured-data-models-client/tests/test_databricks_splitting.py::test_max_request_bytes_is_passed_through", + "behavior": "max_request_bytes reaches the Databricks transport." + }, + { + "test": "packages/structured-data-models-client/tests/test_databricks_splitting.py::test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent", + "behavior": "A call past max_request_bytes is split before it is sent, each part within it." + } + ] }, "sdm_client.SDMClient.health_ready": { "capability": "readiness-discovery" @@ -8233,6 +8244,21 @@ "base_commit": "ebc5746391d977fd8d9d779423db33ae547d64ee", "record": "type_contracts/arrow-admission-and-inference", "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "public_api_snapshot_sha256", + "reason": "SDMClient.for_databricks_serving takes an optional max_request_bytes, passed to the Databricks transport, whose calls past it are split before they are sent." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_inventory", + "reason": "SDMClient.for_databricks_serving gains max_request_bytes, asserted by the Databricks splitting tests." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/public-serving-refusals", + "reason": "Adds the Databricks splitting tests, which assert max_request_bytes and the split before a call is sent." } ], "capabilities": [ diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 5c81e999e..8908ca3ba 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -476,7 +476,7 @@ "capabilities": "(self, model: 'str | Model') -> 'ModelCapabilities'", "close": "(self) -> 'None'", "fit": "(self, request: 'Request', *, model: 'str | Model', outputs: 'Sequence[str] | None' = None, holdout: 'Iterable[Any] | None' = None) -> 'FittedModel'", - "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None) -> 'SDMClient'", + "for_databricks_serving": "(endpoint: 'str', *, workspace_client: 'Any | None' = None, timeout: 'float | None' = None, max_request_bytes: 'int | None' = None) -> 'SDMClient'", "health_ready": "(self) -> 'bool'", "in_process": "(*models: 'Any', device: 'Any' = None) -> 'SDMClient'", "models": "(self) -> 'list[str]'", From a8c23e1856ffb9a4cde477b04ca72053c1ac661e Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Mon, 9 Nov 2026 20:18:09 +0000 Subject: [PATCH 6/8] Address review: real SDK statuses, transport failures, contract gates - Python Databricks: map the real SDK's errors to statuses (InternalError as 500; a 413 or 502 by the error code the SDK gives it), so they split and a 500 is tried once more; tests raise the SDK's own errors, read by its parser. - Java: a timeout or an answer past 64 MiB stays a failed transport, told to the core by its status so it can split, and raised as a transport error when no smaller call answers; no public exception is made up. - Transports no longer try a 408, 413 or 504, or a read timeout, again at full size: the core splits at once. 429, 500, 502, 503 and connection errors are still retried. - The halves of a split call answered before a later one fails are kept in details['answered']. - Contract: the departures (no entity, call or ranking-entity ceiling, a 500 tried once more) recorded in contract/expected from the runner's observations on DuckDB and SQLite, Postgres taking the same outcomes; case why texts say what the reference did; test_ranking and test_prediction_batching skip departed cases and check them on their own; check_observations counts a body the transport sent again as one call. - Tests for nested and odd splits, the size kept for later calls, row numbering across batches, a Java single-table batching test; docs and CHANGELOG updated; leftover 15-table and 6-hop mentions removed. Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 6 ++ contract/cases/s1/multicall.calls.101.json | 2 +- .../cases/s1/multicall.entities.100000.json | 2 +- .../cases/s1/multicall.entities.100001.json | 2 +- .../s1/rank.entities-by-candidates.over.json | 2 +- contract/cases/s1/sample.num-hops.seven.json | 2 +- contract/expected/README.md | 4 + contract/expected/duckdb-python.json | 12 +-- contract/expected/postgres-python.json | 12 +-- contract/expected/sqlite-python.json | 12 +-- contract/runners/check_observations.py | 11 ++- contract/tests/test_contract_loader.py | 2 + contract/tests/test_prediction_batching.py | 4 +- contract/tests/test_ranking.py | 28 ++++++- docs/reference/limits.md | 12 +-- .../java/com/nvidia/sdm/ClientOptions.java | 2 +- .../main/java/com/nvidia/sdm/SdmClient.java | 18 +++- .../nvidia/sdm/internal/HttpTransport.java | 40 +++++---- .../nvidia/sdm/AutomaticBatchSessionTest.java | 2 +- .../test/java/com/nvidia/sdm/TabularTest.java | 20 +++++ .../nvidia/sdm/internal/TransportTest.java | 18 ++-- .../src/sdm_client/_rust/predict.py | 6 +- .../src/sdm_client/client.py | 8 +- .../src/sdm_client/core/databricks_serving.py | 17 ++++ .../src/sdm_client/core/transport.py | 12 ++- .../src/sdm_client/query.py | 10 +-- .../test_asked_in_the_questions_it_serves.py | 4 +- .../tests/test_automatic_batch_sessions.py | 2 +- .../tests/test_databricks_splitting.py | 70 ++++++++++++++-- .../tests/test_payload_rules.py | 11 ++- .../tests/test_query_engine_boundary.py | 4 +- .../tests/test_resilient_calls.py | 77 +++++++++++++++-- .../tests/test_serving_route.py | 9 +- .../tests/test_transport_socket.py | 9 +- rust/crates/sdm-core/src/materialize.rs | 3 +- rust/crates/sdm-core/src/predict.rs | 84 +++++++++++++++---- 36 files changed, 412 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5a4eaa6..197d0350a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The public API snapshot records this version update; publication remains pending ### Added +- Python `SDMClient.for_databricks_serving(max_request_bytes=...)` sets the most bytes one wrapped call may take, the gateway's 16 MB by default; a call past it is split before it is sent. - Python `Schema.from_relbench` and Java `Schema.fromRelbench` load the same pinned RelBench datasets with verified caches, offline mode and declared schema metadata. - Python and Java requests and fitted handles expose immutable sanitization reports for in-memory relational preparation, with per-table row exclusions and explicit unavailable/not-applicable states. @@ -50,6 +51,11 @@ The public API snapshot records this version update; publication remains pending ### Changed +- A single-table request is scored in calls of 10,000 rows, each with the whole context, split further as a relational request is. +- A `500` is retried with backoff as a `502` or `503` is; through Databricks model serving, once more. +- `rows_per_call` is taken as asked rather than lowered to 10,000 (200 for a link prediction), which is now only its default. +- `num_context_samples` past 10,000 and more than 6 hops are accepted. + - Sampled Tabular context uses stable Rust index ordering, including duplicate labels and mixed numeric labels. Equal labels retain sampled order; mixed numbers compare by their represented values. Context order and predictions can differ from platform-specific NumPy ordering; sampled membership and warehouse sampling are unchanged. diff --git a/contract/cases/s1/multicall.calls.101.json b/contract/cases/s1/multicall.calls.101.json index 804d9d27b..2d2ab3d32 100644 --- a/contract/cases/s1/multicall.calls.101.json +++ b/contract/cases/s1/multicall.calls.101.json @@ -579,6 +579,6 @@ "code": "UNSUPPORTED_REQUEST", "message": "101 entities are gathered as 101 calls of 1, and every one is built, with its own context, before any is sent. That is past the 100 calls this will hold at once; ask about fewer entities at a time" }, - "why": "rows_per_call=1 over 101 entities: 101 calls, one past the 100 one request is asked in; refused UNSUPPORTED_REQUEST at materialize", + "why": "rows_per_call=1 over 101 entities: 101 calls, one past the 100 the reference asked one request in; the reference refused it UNSUPPORTED_REQUEST at materialize; this client sets no most on calls and asks all 101 (contract/expected)", "source": "Correctness case multicall.calls.101 using the shop fixture." } diff --git a/contract/cases/s1/multicall.entities.100000.json b/contract/cases/s1/multicall.entities.100000.json index 6e732e562..ad74de405 100644 --- a/contract/cases/s1/multicall.entities.100000.json +++ b/contract/cases/s1/multicall.entities.100000.json @@ -69,7 +69,7 @@ "outcome": "success", "stage": "materialize" }, - "why": "100,000 entities, the most one materialize() gathers: gathered as ten calls of 10,000 (until materialize, so no payload of 100,000 rows is recorded); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", + "why": "100,000 entities, the most the reference's materialize() gathered (this client sets no most): gathered as ten calls of 10,000 (until materialize, so no payload of 100,000 rows is recorded); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", "source": "Correctness case multicall.entities.100000 using the shop fixture.", "input_factory": "entity_limit_100000" } diff --git a/contract/cases/s1/multicall.entities.100001.json b/contract/cases/s1/multicall.entities.100001.json index f78f7c05b..07b29b776 100644 --- a/contract/cases/s1/multicall.entities.100001.json +++ b/contract/cases/s1/multicall.entities.100001.json @@ -72,7 +72,7 @@ "code": "UNSUPPORTED_REQUEST", "message": "100,001 entities is past the 100,000 one materialize() will hold at once, since every batch is built, with its own context, before any is sent; ask about fewer entities at a time" }, - "why": "100,001 entities, one past the 100,000 one materialize() gathers: refused UNSUPPORTED_REQUEST from the count, before anything is read; user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", + "why": "100,001 entities, one past the 100,000 the reference's materialize() gathered: the reference refused it UNSUPPORTED_REQUEST from the count; this client sets no most and gathers it (contract/expected); user_id is declared the key, as the core and the reference read it either way, so the loader indexes it on SQLite and Postgres and a lookup there is not a scan of 100,101 rows", "source": "Correctness case multicall.entities.100001 using the shop fixture.", "input_factory": "entity_limit_100001" } diff --git a/contract/cases/s1/rank.entities-by-candidates.over.json b/contract/cases/s1/rank.entities-by-candidates.over.json index c8bf3cdfd..41c9341bc 100644 --- a/contract/cases/s1/rank.entities-by-candidates.over.json +++ b/contract/cases/s1/rank.entities-by-candidates.over.json @@ -76,6 +76,6 @@ "error": "StructuredDataError", "code": "UNSUPPORTED_REQUEST" }, - "why": "The entity ceiling over a ranking (main's MOST_ENTITIES, 100,000): the entities gathered once per candidate, 501 by 200, one past it by 200, refused before anything is read", + "why": "The reference's entity ceiling over a ranking (main's MOST_ENTITIES, 100,000): the entities gathered once per candidate, 501 by 200, one past it by 200, refused before anything is read; this client sets no such ceiling, and refuses the call only for the entities that name no user (contract/expected)", "source": "Correctness case rank.entities-by-candidates.over using the shop fixture." } diff --git a/contract/cases/s1/sample.num-hops.seven.json b/contract/cases/s1/sample.num-hops.seven.json index bcd5a8467..3b20bbc5e 100644 --- a/contract/cases/s1/sample.num-hops.seven.json +++ b/contract/cases/s1/sample.num-hops.seven.json @@ -87,6 +87,6 @@ "code": "UNSUPPORTED_REQUEST", "message": "this request samples 7 hops and the endpoint reads at most 6" }, - "why": "num_hops=7 is past the six hops the endpoint reads: refused UNSUPPORTED_REQUEST at materialize", + "why": "num_hops=7 is past the six hops the reference's endpoint read: the reference refused it UNSUPPORTED_REQUEST at materialize; this client sets no most on hops and gathers it (contract/expected)", "source": "Correctness case sample.num-hops.seven using the shop fixture." } diff --git a/contract/expected/README.md b/contract/expected/README.md index af4795e20..9bf9135b3 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -31,6 +31,10 @@ Their differences from DuckDB follow the fixture loader's explicit mappings in ` `sample.num-hops.seven` and `sample.num-neighbors.seven-hops` now reach predict on every backend: the client sets no most on hops, since the model reads as many as a request carries, where the reference refused past six. The `multicall.body-30mib` cases now reach predict on every backend: the client holds a call to no size of its own, and a call the endpoint refuses as too large is asked again in halves. +`multicall.entities.100001` and `multicall.calls.101` are now gathered: the client sets no most on the entities or the calls of one `materialize()`. +`rank.entities-by-candidates.over` is now refused at materialize only for the entities that name no user, as `rank.entities-by-candidates.at` is, since the client sets no entity ceiling over a ranking. +`multicall.fail-call-0`, `multicall.fail-call-2` and `multicall.forecast.fail-call-1` now succeed: their scripted endpoint fails one call with a `500`, which the client tries once more. +These outcomes were recorded from the runner's observations (`run_python.py`, then `check_observations.py --expect`) on DuckDB and SQLite; the Postgres entries take the same outcomes, as no Postgres server was available. These expectations describe the declared fixture mappings; they do not claim that every backend supports identical physical types. `explain.two-entities` now succeeds on all three backends: The client supports explanations for multiple prediction rows. diff --git a/contract/expected/duckdb-python.json b/contract/expected/duckdb-python.json index 44f56c90d..1ef2af283 100644 --- a/contract/expected/duckdb-python.json +++ b/contract/expected/duckdb-python.json @@ -111,14 +111,14 @@ "multicall.body-30mib.one-row": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.body-30mib.several-rows": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/postgres-python.json b/contract/expected/postgres-python.json index bbbb4baed..f4d7f3ff6 100644 --- a/contract/expected/postgres-python.json +++ b/contract/expected/postgres-python.json @@ -111,14 +111,14 @@ "multicall.body-30mib.one-row@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.body-30mib.several-rows@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000@postgres": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001@postgres": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates@postgres": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/sqlite-python.json b/contract/expected/sqlite-python.json index 8365c4b16..f8d2d832b 100644 --- a/contract/expected/sqlite-python.json +++ b/contract/expected/sqlite-python.json @@ -111,14 +111,14 @@ "multicall.body-30mib.one-row@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.body-30mib.several-rows@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.calls.100@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "multicall.calls.101@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "multicall.calls.101@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities-per-call.10001@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.entities.100000@sqlite": {"code": null, "error": null, "result": "success", "stage": "materialize"}, - "multicall.entities.100001@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.fail-call-0@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, - "multicall.fail-call-2@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.entities.100001@sqlite": {"code": null, "error": null, "result": "success", "stage": "materialize"}, + "multicall.fail-call-0@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, + "multicall.fail-call-2@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.batched@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "multicall.forecast.fail-call-1@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "multicall.forecast.fail-call-1@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.forecast.one-call@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call-3@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "multicall.rows-per-call.above-the-cap@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, @@ -160,7 +160,7 @@ "rank.candidates-not-a-ranking@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.decimal-key@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "rank.entities-by-candidates.at@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, - "rank.entities-by-candidates.over@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, + "rank.entities-by-candidates.over@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.most-parts@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.no-candidates@sqlite": {"code": "INVALID_REQUEST", "error": "StructuredDataError", "result": "refusal", "stage": "materialize"}, "rank.odd-column@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/runners/check_observations.py b/contract/runners/check_observations.py index 0d6fdeaf8..7e653f7c2 100644 --- a/contract/runners/check_observations.py +++ b/contract/runners/check_observations.py @@ -215,9 +215,14 @@ def check( for entry in (item.get('public') or {}).get('payloads', []) ] paths = (('metadata', 'request_id'),) - if [_payload(value, paths) for value in calls] != [ - _payload(value, paths) for value in received - ]: + # A body the transport sent again, a 500 tried once more, is the + # one call the core made. + sent = [] + for value in received: + value = _payload(value, paths) + if not sent or sent[-1] != value: + sent.append(value) + if [_payload(value, paths) for value in calls] != sent: failures.append( 'endpoint payload differs from native prediction' ) diff --git a/contract/tests/test_contract_loader.py b/contract/tests/test_contract_loader.py index b8658b923..c82849c7a 100644 --- a/contract/tests/test_contract_loader.py +++ b/contract/tests/test_contract_loader.py @@ -521,6 +521,8 @@ def test_entity_limit_factory_expands_exact_keys_and_preserves_source_digest( assert case['core']['materialize']['indices'] == [ {'t': 'i64', 'v': str(i)} for i in range(count) ] + # The case states the reference's outcome, which refused 100,001; this + # client gathers it, a departure recorded in contract/expected. assert case['expect']['outcome'] == ( 'success' if count == 100_000 else 'refusal' ) diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index dc7706477..8374f8ad5 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -621,7 +621,7 @@ def test_python_core_answers_each_case_as_the_reference_does( if database == 'postgres' and not _server(): return if case_key.split('@')[0] in DEPARTED: - return + pytest.skip('a departure from the reference, checked on its own') def check() -> None: run = core_run(database) @@ -670,7 +670,7 @@ def test_python_core_carries_what_the_reference_carries(case_key: str) -> None: if database == 'postgres' and not _server(): return if case_id in DEPARTED: - return + pytest.skip('a departure from the reference, checked on its own') def check() -> None: want = vectors()['vectors'][case_key] diff --git a/contract/tests/test_ranking.py b/contract/tests/test_ranking.py index 59f86c088..29fb80fc3 100644 --- a/contract/tests/test_ranking.py +++ b/contract/tests/test_ranking.py @@ -51,8 +51,8 @@ 'holds one class left out, unscored, before any call', 'rank.all-unscored': 'no candidate left to rank, refused before any call', 'rank.entities-by-candidates.over': 'the entities gathered once ' - 'per candidate past the 100,000 one materialize holds (501 by 200), ' - 'refused before anything is read', + "per candidate past the reference's 100,000 (501 by 200), which it " + 'refused; this client sets no such ceiling (DEPARTED)', 'rank.entities-by-candidates.at': 'the same at exactly 100,000 ' '(500 by 200), not refused by that ceiling', 'forecast.top-k': "the forecast's top_k refusal in the core", @@ -373,6 +373,12 @@ def test_the_vectors_are_the_reference_s_and_not_vacuous() -> None: } +#: The cases this client departs from the reference on, on purpose: it sets +#: no ceiling on the entities a ranking gathers across its candidates, so the +#: reference's refusal past 100,000 is gathered, and refused only for the +#: entities that name no user, as at the ceiling. +DEPARTED = ('rank.entities-by-candidates.over',) + #: What the reference's refusal of each case refused before any call says, #: so it is refused for the argument the case names. REFUSED_FOR = { @@ -531,6 +537,8 @@ def test_python_core_answers_each_case_as_the_reference_does( database = case_key.partition('@')[2] or 'duckdb' if database == 'postgres' and not _server(): return + if case_key.split('@')[0] in DEPARTED: + pytest.skip('a departure from the reference, checked on its own') def check() -> None: run = core_run(database) @@ -658,3 +666,19 @@ def check() -> None: held_to(recorder().observe(inputs), spec['expected'], name) check() + + +@pytest.mark.parametrize('case_id', DEPARTED) +def test_python_core_gathers_past_the_reference_s_entity_ceiling( + case_id: str, +) -> None: + r"""Past the reference's 100,000, the ranking is gathered and refused + only for the entities 8 to 500, which name no user, as at the ceiling: + the caller's ``INVALID_REQUEST``, not the ceiling's + ``UNSUPPORTED_REQUEST``.""" + line = core_run('duckdb').get(case_id) + assert line is not None, f'{case_id}: not observed' + assert (line['outcome']['result'], line['outcome']['code']) == ( + 'refusal', + 'INVALID_REQUEST', + ) diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 1d824aebb..0a1359aa4 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -5,9 +5,9 @@ It sends its best attempt and recovers from what the endpoint says: | The endpoint answers | The client | |---|---| -| `413`, a gateway's `502` or `504`, or the call times out | asks the call's rows again in two halves, each with the whole context, and keeps that smaller size for the request's later calls | -| `503` | waits as `Retry-After` says and tries again | -| a connection error or another `5xx` | tries again with backoff, up to `max_retries` (3 by default) | +| `413`, a gateway's `502` or `504`, the call times out, or its answer is past 64 MiB | asks the call's rows again in two halves at once, each with the whole context, and keeps that smaller size for the request's later calls; a `502` is first tried again at its size | +| `503` or `429` | waits as `Retry-After` says and tries again | +| a connection error, a `500` or a `502` | tries again with backoff, up to `max_retries` (3 by default); through Databricks model serving, a `500` is tried once more | | `422`, or a refusal of a call of one row | does not try again, and shows the endpoint's message | The context is never split, so a context too large on its own is the endpoint's refusal, shown as it said it. @@ -16,11 +16,13 @@ The context is sampled only to a `num_context_samples` you pass. Through Databricks model serving, an oversized call is dropped as an opaque connection error, so `predict()` splits a call whose wrapped envelope is past the endpoint's `max_request_bytes` (16 MB, the gateway's own decimal limit) before sending it. A call of one row past it is refused, since only a smaller context helps. +There, a deadline spent waiting on a `429`, a `503` or a connection is reported as a `503` and not split; a `504` is a call that timed out. ## Batches A relational request is gathered in batches of 10,000 rows to score, or 200 for a link prediction: a default, not a cap. -Pass `rows_per_call` to `materialize()` to ask in batches of another size. +Pass `rows_per_call` to `materialize()` to ask in batches of another size; it is taken as asked. +A single-table request is scored in calls of 10,000 rows too, each with the whole context, and the answers are put back together as one. Each batch has a context of its own, sampled around that batch's rows rather than copied from the first, and `predict()` sends one call per batch and returns one result. Its `row_index` counts across the whole request rather than from zero per call, so `to_pandas()`, which orders by it, gives the indices back in the order they were asked for. A session scores those batches against the context it pinned, so the context is uploaded once rather than once per call. @@ -31,7 +33,7 @@ A question asked in parts is different: a ranking is one call per candidate and A question asked in parts is not batched by rows, since a batched part would send the parts times the batches. A call that fails ends the request. -The answers already paid for are carried on the error as `details['answered']`, and the rows they cover as `details['rows_answered']`, so the rest can be asked for again without buying the whole population twice. +The answers already paid for, the halves of a split call included, are carried on the error as `details['answered']`, and the rows they cover as `details['rows_answered']`, so the rest can be asked for again without buying the whole population twice. A forecast keeps its answered steps the same way, under `details['answered']`, numbered by step, with how many of them under `details['parts_answered']`; a ranking keeps only the count, because an order over some of its candidates is a different answer rather than part of the one that was asked for. The relational model reads as many tables and hops as a request carries. diff --git a/java/src/main/java/com/nvidia/sdm/ClientOptions.java b/java/src/main/java/com/nvidia/sdm/ClientOptions.java index 732c6a3cc..7a412162b 100644 --- a/java/src/main/java/com/nvidia/sdm/ClientOptions.java +++ b/java/src/main/java/com/nvidia/sdm/ClientOptions.java @@ -64,7 +64,7 @@ public Builder timeout(Duration timeout) { return this; } - /** How many times a transient failure (408, 429, 500, 502, 503, 504, a dropped connection) is retried. */ + /** How many times a transient failure (429, 500, 502, 503, a failed connection) is retried. */ public Builder maxRetries(int maxRetries) { this.maxRetries = maxRetries; return this; diff --git a/java/src/main/java/com/nvidia/sdm/SdmClient.java b/java/src/main/java/com/nvidia/sdm/SdmClient.java index 6606e187d..a59395e72 100644 --- a/java/src/main/java/com/nvidia/sdm/SdmClient.java +++ b/java/src/main/java/com/nvidia/sdm/SdmClient.java @@ -206,7 +206,10 @@ public byte[] call(Object endpoint, byte[] body) throws Host.TransportFailed, Ho if (serving != null) throw new StructuredDataException("this transport does not support sessions", "SESSIONS_UNSUPPORTED", Map.of(), null); try { return transport.call(endpoint, body); } - catch (HttpTransport.Failed e) { throw new Host.TransportFailed(); } + catch (HttpTransport.Failed e) { + if (e.status() != null) throw smaller(e); + throw new Host.TransportFailed(); + } catch (ModelEndpointException e) { throw new Host.Refused(e::with, e.status()); } } }; @@ -268,12 +271,25 @@ private byte[] call(byte[] body) try { return transport.predict(body); } catch (HttpTransport.Failed failed) { + if (failed.status() != null) throw smaller(failed); throw new Host.TransportFailed(); } catch (ModelEndpointException refused) { throw new Host.Refused(refused::with, refused.status()); } } + /** + * A failed transport a smaller call can fix (a timeout, an answer too large), told to the core by + * its status, so the core asks the rows in halves; still a transport error if no smaller call + * answers. + */ + private static Host.Refused smaller(HttpTransport.Failed failed) { + return new Host.Refused( + answered -> new StructuredDataException( + failed.getMessage(), "TRANSPORT_ERROR", answered, "model_endpoint", false, null), + failed.status()); + } + /** A call's id, as the adapter draws one: {@code sdm-} and twelve hexadecimal digits. */ static String requestId() { byte[] random = new byte[6]; diff --git a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java index 234010241..d5b5db292 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java +++ b/java/src/main/java/com/nvidia/sdm/internal/HttpTransport.java @@ -31,7 +31,7 @@ * Transport} ({@code sdm_client/core/transport.py}): it posts the core's call to {@code * /v1/predictions} as the bytes the core wrote, never parsing or re-encoding them (design choice * 10, "Model calls"), and gives back the endpoint's answer as the bytes it sent, which the core - * reads. A transient failure (a status of 429, 500, 502, 503 or 504, or a connection that failed) is + * reads. A transient failure (a status of 429, 500, 502 or 503, or a connection that failed) is * tried again with exponential backoff, a {@code Retry-After} the endpoint sends honoured up to a * minute; the timeout bounds each attempt whole, its answer's body included; a status of 400 or more is the endpoint's refusal ({@link ModelEndpointException}); an * endpoint that cannot be reached, or an answer past 64 MiB or that is not a JSON object, is a @@ -42,7 +42,11 @@ public final class HttpTransport { public static final String PREDICTIONS = "/v1/predictions"; /** The statuses tried again. */ - static final Set RETRY_STATUS = Set.of(408, 429, 500, 502, 503, 504); + /** + * Tried again at the same size: overload, and a server or gateway fault. A 408, 413 or 504, or a + * call that timed out, is not: the core asks its rows in smaller calls at once. + */ + static final Set RETRY_STATUS = Set.of(429, 500, 502, 503); /** The largest answer read. */ static final int MAX_RESPONSE_BYTES = 64 * 1024 * 1024; @@ -61,9 +65,23 @@ public final class HttpTransport { /** The transport could not reach the endpoint or read its answer. */ public static final class Failed extends Exception { private static final long serialVersionUID = 1L; + private final Integer status; Failed(String message) { + this(message, null); + } + + Failed(String message, Integer status) { super(message); + this.status = status; + } + + /** + * The status the core is told, so it asks the call's rows in smaller calls: 408 for an answer + * slower than the timeout, 413 for one past {@link #MAX_RESPONSE_BYTES}; none otherwise. + */ + public Integer status() { + return status; } } @@ -219,13 +237,8 @@ else if (endpoint instanceof java.util.Map map) { } throw new Failed("the request failed: " + e.getClass().getSimpleName()); } catch (HttpTimeoutException e) { - if (attempt < retries) { - backoff(attempt, null); - continue; - } - // As the endpoint's 408, so the core asks the rows again in halves. - throw ModelEndpointException.of(408, ("the request timed out after " + timeout - + ", the timeout given").getBytes(java.nio.charset.StandardCharsets.UTF_8)); + // Not tried again at its size: told as a 408, so the core asks the rows in halves. + throw new Failed("the request timed out after " + timeout + ", the timeout given", 408); } catch (IOException e) { if (attempt < retries) { backoff(attempt, null); @@ -256,16 +269,15 @@ else if (endpoint instanceof java.util.Map map) { * * @throws HttpTimeoutException the attempt did not end within the timeout * @throws IOException the endpoint could not be reached or its answer read - * @throws ModelEndpointException the answer is past {@link #MAX_RESPONSE_BYTES}: told as the - * endpoint's 413, the call too large, so the core asks its rows in smaller calls - * @throws Failed the call was interrupted + * @throws Failed the answer is past {@link #MAX_RESPONSE_BYTES} (its {@link Failed#status} a + * 413), or the call was interrupted */ private HttpResponse attempt(HttpRequest built) throws IOException, Failed { try { return exchange(client, built, info -> new Capped(), timeout.toNanos()); } catch (TooLarge e) { - throw ModelEndpointException.of(413, ("the response body exceeds the " + MAX_RESPONSE_BYTES - + " byte limit").getBytes(java.nio.charset.StandardCharsets.UTF_8)); + // Told as a 413, the call too large, so the core asks its rows in smaller calls. + throw new Failed("the response body exceeds the " + MAX_RESPONSE_BYTES + " byte limit", 413); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new Failed("the request was interrupted"); diff --git a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java index 5af2861fa..5ee820cc8 100644 --- a/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java +++ b/java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java @@ -48,7 +48,7 @@ static final class Endpoint implements AutoCloseable { scores++; scoredBodies.add(payload); if(mode.equals("transport") && scores>=2) { exchange.close();return; } if(mode.equals("expired") && scores==2) status=404; - else if(mode.equals("retry") && scores==1) status=408; + else if(mode.equals("retry") && scores==1) status=503; else if(mode.equals("fail") && scores==2) status=422; else if(mode.equals("malformed") && scores==2) answer=Map.of("predictions","invalid"); else { diff --git a/java/src/test/java/com/nvidia/sdm/TabularTest.java b/java/src/test/java/com/nvidia/sdm/TabularTest.java index 2564301ad..d5bba0cdb 100644 --- a/java/src/test/java/com/nvidia/sdm/TabularTest.java +++ b/java/src/test/java/com/nvidia/sdm/TabularTest.java @@ -105,6 +105,26 @@ static final class Endpoint implements AutoCloseable { @Override public void close() { server.stop(0); } } + @Test void aSingleTableRequestIsScoredInCallsOf10000RowsEachWithTheWholeContext() throws Exception { + long[] scoring = java.util.stream.LongStream.range(100, 10_101).toArray(); + try (RootAllocator allocator = new RootAllocator(); + Table context = table(allocator, true, 1, 2, 3); + Table predict = table(allocator, false, scoring); + Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + Request request = new Request(context, predict, "y", "regression"); + assertEquals(10_001, client.predict(request, SdmClient.KUMO_TABULAR).rows()); + List sizes = new ArrayList<>(); + for (Map call : endpoint.calls) { + @SuppressWarnings("unchecked") Map half = (Map) call.get("predict"); + @SuppressWarnings("unchecked") Map rows = (Map) half.get("instance_table"); + sizes.add(((List) rows.get("rows")).size()); + assertEquals(endpoint.calls.get(0).get("context"), call.get("context")); + } + assertEquals(List.of(10_000, 1), sizes); + } + } + @Test void directRequestSurvivesCallerClosureAndRestoresRowOrder() throws Exception { Request request; try (RootAllocator allocator = new RootAllocator(); diff --git a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java index 4e3a44a38..a13791b8c 100644 --- a/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java +++ b/java/src/test/java/com/nvidia/sdm/internal/TransportTest.java @@ -28,8 +28,8 @@ /** * The model transport over {@code java.net.http}: the call is sent as the bytes given, * one past 16 MB included; a transient status or a 500 is tried again; a refusal is the endpoint's; - * a timeout is told as a 408; an - * answer that is not JSON is a failed transport, and one past 64 MiB is told as a 413. + * a timeout is a failed transport told as a 408, not tried again; an + * answer that is not JSON is a failed transport, and one past 64 MiB too, told as a 413. */ class TransportTest { /** An endpoint answering {@code statuses} in turn, then 200, keeping every body. */ @@ -115,15 +115,15 @@ void anAnswerThatIsNoJsonObjectIsAFailedTransport() throws Exception { } @Test - void anAnswerPast64MiBIsToldAsA413() throws Exception { + void anAnswerPast64MiBIsAFailedTransportToldAsA413() throws Exception { byte[] answer = new byte[HttpTransport.MAX_RESPONSE_BYTES + 1]; Arrays.fill(answer, (byte) ' '); try (Endpoint endpoint = new Endpoint(answer)) { - ModelEndpointException refused = + HttpTransport.Failed failed = assertThrows( - ModelEndpointException.class, + HttpTransport.Failed.class, () -> endpoint.transport(0).predict("{}".getBytes(StandardCharsets.UTF_8))); - assertEquals(413, refused.status()); + assertEquals(413, failed.status()); } } @@ -175,10 +175,10 @@ void anAnswerWhoseBodyIsHeldBackTimesOut() throws Exception { new HttpTransport( "http://127.0.0.1:" + server.getAddress().getPort(), null, Duration.ofMillis(300), 0); long started = System.nanoTime(); - // Told as the endpoint's 408, so the core asks the rows again in halves. - ModelEndpointException failed = + // A failed transport, told as a 408 so the core asks the rows again in halves. + HttpTransport.Failed failed = assertThrows( - ModelEndpointException.class, + HttpTransport.Failed.class, () -> transport.predict("{}".getBytes(StandardCharsets.UTF_8))); assertEquals(408, failed.status()); long millis = (System.nanoTime() - started) / 1_000_000; diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index 99cbcaf30..44891e076 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -607,7 +607,7 @@ def model_call(body: bytes) -> bytes: unexpected: Exception | None = None transport_failed = False timed_out: StructuredDataError | None = None - too_large = 408 + split_status = 408 refused: NimRequestError | None = None try: # The core's call, its bytes as written (register PD-2), @@ -628,7 +628,7 @@ def model_call(body: bytes) -> bytes: # Told to the core as a 408 or a 413, so it asks the rows # again in halves; this is what is raised if no half answers. if failed.details.get('response_too_large'): - too_large = 413 + split_status = 413 timed_out = StructuredDataError( failed.message, code=failed.code, details=failed.details ) @@ -655,7 +655,7 @@ def model_call(body: bytes) -> bytes: if transport_failed: raise TransportFailed if timed_out is not None: - raise Refused(timed_out, too_large) + raise Refused(timed_out, split_status) raised: StructuredDataError | None = failure if refused is not None: # Built outside the handler, so nothing failing in the copy is diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 4660e7ab8..c655cb754 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -285,9 +285,11 @@ def __init__( verify_ssl: Whether to verify the server's TLS certificate. timeout: Seconds to wait for each individual attempt, not for the call as a whole. - max_retries: Transport-level retries of a transient failure (408, 429, - 500, 502, 503, 504, or a dropped connection) with exponential - backoff, a ``Retry-After`` honoured. ``POST /v1/sessions`` is + max_retries: Transport-level retries of a transient failure (429, + 500, 502, 503, or a connection that failed) with exponential + backoff, a ``Retry-After`` honoured. A 408, 413 or 504, or a + call slower than ``timeout``, is not tried again at its size: + its rows are asked in halves. ``POST /v1/sessions`` is excluded: a re-sent create would orphan a pinned context on the endpoint. diff --git a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py index 3c5bb33b6..ea014c790 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py +++ b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py @@ -573,6 +573,16 @@ def _response_body(reply: Any, endpoint: str) -> str: return body +#: The statuses the SDK names only by an ``error_code``: a call too large, or +#: a gateway's failure, which the core asks again in halves. +_STATUS_OF_CODE = { + 'REQUEST_ENTITY_TOO_LARGE': 413, + 'PAYLOAD_TOO_LARGE': 413, + 'CONTENT_TOO_LARGE': 413, + 'BAD_GATEWAY': 502, + 'GATEWAY_TIMEOUT': 504, +} + #: The SDK's and ``requests``' names for a deadline reached. _TIMEOUTS = frozenset( {'DeadlineExceeded', 'OperationTimeout', 'TimeoutError', 'Timeout'} @@ -610,6 +620,13 @@ def _status_of(error: BaseException) -> int: return 504 if 'BadRequest' in names: return 400 + if 'InternalError' in names: + return 500 + # The SDK has no class for a 413 or a 502; it names one by the HTTP + # reason phrase, as its ``error_code``, where the body is no JSON. + code = getattr(error, 'error_code', None) + if code in _STATUS_OF_CODE: + return _STATUS_OF_CODE[code] status = getattr(error, 'status_code', None) return status if isinstance(status, int) else 503 diff --git a/packages/structured-data-models-client/src/sdm_client/core/transport.py b/packages/structured-data-models-client/src/sdm_client/core/transport.py index 802dd6f5a..77a8d119c 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/transport.py +++ b/packages/structured-data-models-client/src/sdm_client/core/transport.py @@ -31,9 +31,11 @@ _DEFAULT_TIMEOUT_SECONDS = 60.0 _DEFAULT_MAX_RETRIES = 3 _DEFAULT_BACKOFF_FACTOR = 0.5 -#: The statuses tried again with backoff, up to ``max_retries``: a timeout, -#: overload, and a server or gateway fault, a 500 included. -_RETRY_STATUS = (408, 429, 500, 502, 503, 504) +#: The statuses tried again with backoff, up to ``max_retries``: overload, and +#: a server or gateway fault, a 500 included. A 408, 413 or 504, or an answer +#: slower than the timeout, is not tried again at its size: the core asks the +#: call's rows in halves at once. +_RETRY_STATUS = (429, 500, 502, 503) _RETRY_AFTER_MAX_SECONDS = 60 _MAX_BODY_SNIPPET = 512 _LOCAL_HOSTS = frozenset({'localhost', '127.0.0.1', '::1'}) @@ -299,7 +301,9 @@ def _build_retry( options: dict[str, Any] = { 'total': max_retries, 'connect': max_retries, - 'read': max_retries, + # A read error, an answer slower than the timeout among them, is not + # tried again at its size: the core asks the call's rows in halves. + 'read': 0, 'status': max_retries, 'backoff_factor': backoff_factor, 'status_forcelist': _RETRY_STATUS, diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index d3e97a2c8..1f326433c 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -193,12 +193,10 @@ def materialize( and ``anchor_time``; ``prediction_statistic``: what a regression's prediction reports of its predicted distribution, ``'median'`` (the default) or ``'mean'``; and - ``rows_per_call``: how many entities one call scores, which - defaults to what the endpoint takes, 10,000 or 200 for a - temporal link prediction, and may only be lowered. Lower it - for a graph wide enough that a full batch outruns the 30 MiB - body, or for a deployment holding to less than this client - knows of. + ``rows_per_call``: how many entities one call scores, 10,000 + by default, or 200 for a temporal link prediction, taken as + asked. A call the endpoint refuses as too large, or times out + on, is asked again in halves, each with the whole context. Returns: A:class:`~sdm_client.Request` carrying the data. diff --git a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py index 508bb2056..b34edd12f 100644 --- a/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py +++ b/packages/structured-data-models-client/tests/test_asked_in_the_questions_it_serves.py @@ -106,8 +106,8 @@ def test_more_entities_than_one_call_scores_is_refused_before_gathering( def test_a_smaller_batch_lowers_what_a_part_may_ask_for(monkeypatch): - """`rows_per_call` only goes down, and a part is one call, so asking for a - smaller one asks for fewer entities rather than for more calls.""" + """A part is one call, so asking for a smaller ``rows_per_call`` asks for + fewer entities rather than for more calls.""" with pytest.raises(StructuredDataError) as refused: built(FORECAST, indices=[1, 2, 3], rows_per_call=2) diff --git a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py index a15cf85ca..5e9673f04 100644 --- a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py +++ b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py @@ -80,7 +80,7 @@ def do_POST(self): elif outer.mode == 'fail' and outer.scored == 2: status = 422 elif outer.mode == 'retry' and outer.scored == 1: - status = 408 + status = 503 else: table = payload['predict']['instance_table'] index = table['columns'].index('instance_id') diff --git a/packages/structured-data-models-client/tests/test_databricks_splitting.py b/packages/structured-data-models-client/tests/test_databricks_splitting.py index 7ac1e3602..ab0041bce 100644 --- a/packages/structured-data-models-client/tests/test_databricks_splitting.py +++ b/packages/structured-data-models-client/tests/test_databricks_splitting.py @@ -86,10 +86,58 @@ def opened(failures=(), **kwargs: Any) -> tuple[SDMClient, Workspace]: return client, workspace -class Status(OSError): - def __init__(self, status: int) -> None: - super().__init__(f'status {status}') - self.status_code = status +def sdk_error(status: int, reason: str, body: bytes, kind: str) -> OSError: + r"""The error the installed Databricks SDK raises for an HTTP answer of + ``status``, read by its own parser, as ``api_client.do`` raises it.""" + parser = pytest.importorskip('databricks.sdk.errors.parser') + response = requests.Response() + response.status_code = status + response.reason = reason + response._content = body + response.headers['Content-Type'] = kind + response.request = requests.Request( + 'POST', + f'https://example.invalid/serving-endpoints/{ENDPOINT}/invocations', + ).prepare() + response.url = response.request.url + error = parser._Parser().get_api_error(response) + assert error is not None + return error + + +HTML = 'text/html' +JSON = 'application/json' +#: The SDK's errors for each status, as the gateway answers them. +STATUS = { + 413: lambda: sdk_error( + 413, + 'Request Entity Too Large', + b'

too large
', + HTML, + ), + 502: lambda: sdk_error( + 502, 'Bad Gateway', b'Bad Gateway', HTML + ), + 504: lambda: sdk_error(504, 'Gateway Timeout', b'', 'text/plain'), + 500: lambda: sdk_error( + 500, + 'Internal Server Error', + b'{"error_code": "INTERNAL_ERROR", "message": "boom"}', + JSON, + ), + 429: lambda: sdk_error( + 429, + 'Too Many Requests', + b'{"error_code": "REQUEST_LIMIT_EXCEEDED", "message": "slow"}', + JSON, + ), + 503: lambda: sdk_error( + 503, + 'Service Unavailable', + b'{"error_code": "TEMPORARILY_UNAVAILABLE", "message": "later"}', + JSON, + ), +} def test_max_request_bytes_is_passed_through(): @@ -112,7 +160,9 @@ def test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent(caplog): @pytest.mark.parametrize('status', [413, 502, 504]) def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): - client, workspace = opened([Status(status)]) + error = STATUS[status]() + assert _status_of(error) == status, type(error).__mro__ + client, workspace = opened([error]) result = client.predict(asked(), model='kumo-tabular') calls = workspace.serving_endpoints.calls assert len(calls) == 3 @@ -128,7 +178,9 @@ def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): def test_a_500_is_tried_once_more(): - client, workspace = opened([Status(500)]) + error = STATUS[500]() + assert type(error).__name__ == 'InternalError' + client, workspace = opened([error]) client.predict(asked(), model='kumo-tabular') assert len(workspace.serving_endpoints.calls) == 2 @@ -139,9 +191,9 @@ def timed_out(cause: BaseException | None) -> TimeoutError: error.__cause__ = cause return error - assert _status_of(timed_out(Status(429))) == 503 - assert _status_of(timed_out(Status(503))) == 503 + assert _status_of(timed_out(STATUS[429]())) == 503 + assert _status_of(timed_out(STATUS[503]())) == 503 assert _status_of(timed_out(requests.ConnectionError('refused'))) == 503 assert _status_of(timed_out(requests.ReadTimeout('slow'))) == 504 assert _status_of(timed_out(None)) == 504 - assert _status_of(Status(413)) == 413 + assert _status_of(STATUS[504]()) == 504 diff --git a/packages/structured-data-models-client/tests/test_payload_rules.py b/packages/structured-data-models-client/tests/test_payload_rules.py index dc27a4765..1a2cb693d 100644 --- a/packages/structured-data-models-client/tests/test_payload_rules.py +++ b/packages/structured-data-models-client/tests/test_payload_rules.py @@ -24,12 +24,11 @@ the reference's refusals of an unknown statistic or one on a classification; - exit "payload limits and num_context_samples' bound": the reference's - refusals at each limit (num_context_samples 0 and 10,001, 1,001 entities, - a 16-table graph) and its acceptance at the limit (10,000 context - samples, 1,000 entities, 15 tables); and PD-2: at the byte limit, the - longest payload the reference takes is taken and the bytes the core - produced are the count the reference made, and one byte-step past it is - refused as the reference refuses it; + refusals (num_context_samples 0, 1,001 entities the base does not hold), + and the cases this client departs on (``DEPARTED``): it sets no most on + the context, the tables or the bytes, so a payload the reference refused + past one of them is gathered; and PD-2: the bytes the core produced are + the count the reference made; - exit "empty and reserved-name refusals": the class and code; - exit "inference_config written into the call": the payload's inference section, and the call's model, output and inference as the endpoint diff --git a/packages/structured-data-models-client/tests/test_query_engine_boundary.py b/packages/structured-data-models-client/tests/test_query_engine_boundary.py index 699b76a61..5fea16f30 100644 --- a/packages/structured-data-models-client/tests/test_query_engine_boundary.py +++ b/packages/structured-data-models-client/tests/test_query_engine_boundary.py @@ -1073,8 +1073,8 @@ def test_a_lone_table_is_not_told_which_was_taken(): ids=['an empty num_neighbors', 'num_hops of zero'], ) def test_both_spellings_of_no_hop_are_refused_alike(option, names): - """They are one request written two ways, and the endpoint reads between 1 - and 6 hops, so one slipping through to be taken literally was the defect.""" + """They are one request written two ways, and a request samples at least + one hop, so one slipping through to be taken literally was the defect.""" built = catalogue_schema() with pytest.raises(StructuredDataError, match='no hop at all') as raised: diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py index 72a9a7555..e52f4939e 100644 --- a/packages/structured-data-models-client/tests/test_resilient_calls.py +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -10,6 +10,7 @@ import json import threading +import warnings from collections.abc import Iterator from contextlib import contextmanager @@ -24,20 +25,21 @@ class Scripted: - r"""An endpoint that fails the first call as ``failure`` says, then - answers every call with a prediction for each row it scores.""" + r"""An endpoint that fails its calls in turn as ``failures`` say (``None`` + answering), then answers every call with a prediction for each row it + scores.""" url = 'http://localhost' - def __init__(self, failure: Exception) -> None: - self.failure: Exception | None = failure + def __init__(self, *failures: Exception | None) -> None: + self.failures = list(failures) self.calls: list[dict] = [] def predict_body(self, body: bytes) -> bytes: payload = json.loads(body) self.calls.append(payload) - if self.failure is not None: - failure, self.failure = self.failure, None + failure = self.failures.pop(0) if self.failures else None + if failure is not None: raise failure table = payload['predict']['instance_table'] key = table['columns'].index('instance_id') @@ -213,3 +215,66 @@ def test_a_tabular_call_too_large_is_asked_again_in_halves(failure, caplog): ] assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] assert any('smaller calls' in r.getMessage() for r in caplog.records) + + +def _sizes(endpoint: Scripted) -> list[int]: + return [len(c['predict']['instance_table']['rows']) for c in endpoint.calls] + + +def _too_large(): + return _to_nim_error(413, b'{"detail": "too large"}') + + +def test_a_half_still_too_large_is_split_again(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_too_large(), _too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + # 4 refused, then 2, then each run of one; the size kept, the second + # half is asked one row at a time without trying two. + assert _sizes(endpoint) == [4, 2, 1, 1, 1, 1] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] + + +def test_an_odd_count_is_split_into_the_larger_half_first(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=3) + endpoint = Scripted(_too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [3, 2, 1] + assert list(result.to_pandas()['row_index']) == [0, 1, 2] + + +def test_the_size_that_worked_is_kept_for_the_later_calls(): + import pandas as pd + + from sdm_client import KumoRelational, PredictiveQuery, Schema + + frame = pd.DataFrame({'uid': range(38), 'label': [*range(30), *[None] * 8]}) + schema = Schema.from_data({'users': frame}, primary_keys={'users': 'uid'}) + schema['users']['label'].stype = 'numerical' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + request = PredictiveQuery( + 'PREDICT users.label FOR EACH users.uid', schema + ).materialize( + indices=list(range(30, 38)), + model=KumoRelational(), + rows_per_call=4, + ) + endpoint = Scripted(_too_large()) + result = client(endpoint).predict(request, model='kumo-relational') + # The first batch's 4 refused and asked as 2 and 2; the second batch is + # asked as 2 and 2 at once. Rows are numbered across the whole request. + assert _sizes(endpoint) == [4, 2, 2, 2, 2] + assert list(result.to_pandas()['row_index']) == list(range(8)) + + +def test_the_halves_answered_before_one_fails_are_kept(): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted( + _too_large(), None, _to_nim_error(422, b'{"detail": "N_test"}') + ) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [4, 2, 2] + assert refused.value.details['rows_answered'] == 2 + assert len(refused.value.details['answered']) == 2 diff --git a/packages/structured-data-models-client/tests/test_serving_route.py b/packages/structured-data-models-client/tests/test_serving_route.py index 12e2e191d..11e8b7f19 100644 --- a/packages/structured-data-models-client/tests/test_serving_route.py +++ b/packages/structured-data-models-client/tests/test_serving_route.py @@ -308,8 +308,10 @@ def test_a_call_the_sdk_retries_sends_the_same_bytes_again() -> None: ], ) def test_endpoint_failure_is_sanitized_and_classified(reply: Reply) -> None: - error, received = _predict(_materialized(), reply) status, body, _ = reply + # A 500 is tried once more, so it is answered twice. + replies = (reply, reply) if status == 500 else (reply,) + error, received = _predict(_materialized(), *replies) if isinstance(body, dict): provider_code = body['error_code'] code = { @@ -330,7 +332,7 @@ def test_endpoint_failure_is_sanitized_and_classified(reply: Reply) -> None: 403 if status == 403 else 503, 'PERMISSION_DENIED' if status == 403 else None, ) - assert len(received) == 1 + assert len(received) == len(replies) @pytest.mark.parametrize( @@ -450,7 +452,8 @@ def test_the_sdk_s_debug_log_carries_no_rows_on_the_core_route( ) -> None: request = _materialized() caplog.set_level(logging.DEBUG) - replies = () if reply is None else (reply,) + # A 500 is tried once more, so it is answered twice. + replies = () if reply is None else (reply, reply) outcome, received = _predict(request, *replies) assert received if reply is not None: diff --git a/packages/structured-data-models-client/tests/test_transport_socket.py b/packages/structured-data-models-client/tests/test_transport_socket.py index 578b634aa..0954ef974 100644 --- a/packages/structured-data-models-client/tests/test_transport_socket.py +++ b/packages/structured-data-models-client/tests/test_transport_socket.py @@ -377,15 +377,16 @@ def test_a_detail_that_is_not_text_names_no_diagnosis(): def test_a_request_is_sent_again_only_for_what_sending_it_again_can_fix(): - """A connection error or a 5xx is retried with backoff, up to - ``max_retries``; overload is too. A 422 is the caller's, and sending the - same request again cannot fix it.""" + """A connection error, a 500, a 502 or a 503 is retried with backoff, up + to ``max_retries``; overload is too. A 504 is not tried again at its + size, as the core asks its rows in halves; a 422 is the caller's, and + sending the same request again cannot fix it.""" for status, sends in ( (500, 4), (429, 4), (502, 4), (503, 4), - (504, 4), + (504, 1), (422, 1), ): with _serve(_Reply(status=status, body=_EMPTY_CONTEXT)) as ( diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index def8a1652..89624220b 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -49,8 +49,7 @@ //! narrowed to the size asked, the hops the options ask bounded, the //! entity count, the reserved names, the statistic, the inference //! options, a static target's columns kept out of what is read and sent, -//! the subgraph's 15 tables, the payload's layout and byte limit -//! ([`sdm_request::payload`]), and the notices of what the request leaves +//! the payload's layout ([`sdm_request::payload`]), and the notices of what the request leaves //! out, each a code and fields the host words; and it refuses with //! `UNSUPPORTED_REQUEST` naming the rule ([`sdm_engine::not_yet`]) every //! input whose request takes a rule it does not port yet (register entry diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 8c9786104..437bb04db 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -482,7 +482,8 @@ pub(crate) async fn one( && let Some(answer) = session.score(asker, &body).await? { let response = PredictionResponse::read(&answer).map_err(refused)?; - return layout(payload, &response, crate::pql::float_text); + return layout(payload, &response, crate::pql::float_text) + .map_err(Unanswered::from); } let write = |call: &Value| { sdm_request::predict::body( @@ -506,19 +507,29 @@ pub(crate) async fn one( } .await; let answered = match response { - Ok(part) => numbered_from(part, start), - Err(error) => Err(error), + Ok(part) => numbered_from(part, start).map_err(Unanswered::from), + Err(unanswered) => Err(unanswered), }; match answered { Ok(mut part) => { part.columns = analysis.clone(); parts.push(part) } - Err(error) => { + Err(Unanswered { error, answered }) => { if let Some(session) = session.take() { session.close(asker).await; } - return Err(carry_calls(asker, error, parts, &outputs, start)); + // The runs of a split call answered before one failed + // are carried as the calls before it are. + let mut rows = start; + if let Some((partial, more)) = answered + && let Ok(mut partial) = numbered_from(partial, start) + { + partial.columns = analysis.clone(); + parts.push(partial); + rows += more; + } + return Err(carry_calls(asker, error, parts, &outputs, rows)); } } } @@ -611,13 +622,14 @@ pub(crate) type Write<'a> = dyn Fn(&Value) -> Result + Sync + /// context, and that size is kept in `most` for the calls after it; a call /// of more rows than `most` is asked in calls of that many. A call of one /// row is not split: its refusal, the endpoint's own, stands, as does any -/// other. +/// other. Where a later run of a split call fails, the runs answered before +/// it are kept, with how many rows they answer ([`Unanswered`]). pub(crate) async fn answered( how: &Asking<'_, T>, payload: Value, bytes: Vec, most: &mut Option, -) -> Result { +) -> Result> { let rows = predict_rows(&payload).unwrap_or(0); let size = match *most { Some(most) if rows > most => most, @@ -634,17 +646,56 @@ pub(crate) async fn answered( *most = Some(half); half } - answer => return answer, + answer => return answer.map_err(Unanswered::from), }, }; let mut answers = Vec::new(); for start in (0..rows).step_by(size) { let range = start..(start + size).min(rows); - let call = sdm_request::batching::predict_slice(&payload, range)?; - let bytes = (how.write)(&call)?.to_bytes(); - answers.push((start, Box::pin(answered(how, call, bytes, most)).await?)); + let written = sdm_request::batching::predict_slice(&payload, range).and_then(|call| { + let bytes = (how.write)(&call)?.to_bytes(); + Ok((call, bytes)) + }); + let outcome = match written { + Ok((call, bytes)) => Box::pin(answered(how, call, bytes, most)).await, + Err(error) => Err(Unanswered::from(error)), + }; + match outcome { + Ok(answer) => answers.push((start, answer)), + Err(Unanswered { error, answered }) => { + // The runs before this one answered whole, and this one's own + // answered runs, a prefix of its rows. + let mut rows = start; + if let Some((partial, more)) = answered { + answers.push((start, partial)); + rows += more; + } + let answered = match answers.is_empty() { + true => None, + false => (how.merge)(answers).ok().map(|merged| (merged, rows)), + }; + return Err(Unanswered { error, answered }); + } + } + } + (how.merge)(answers).map_err(Unanswered::from) +} + +/// A call that failed, with what of it was answered before it did: the +/// runs of its rows answered, merged, and how many rows they answer, a +/// prefix of the call's. +pub(crate) struct Unanswered { + pub error: CoreError, + pub answered: Option<(T, usize)>, +} + +impl From for Unanswered { + fn from(error: CoreError) -> Self { + Unanswered { + error, + answered: None, + } } - (how.merge)(answers) } /// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. @@ -666,8 +717,9 @@ fn numbered_parts(parts: Vec<(usize, Part)>) -> Result { } /// The answers to runs of one call's rows as the one answer to it: the -/// first's, with every answer's predictions in order, each numbered from -/// the row its run starts at (`row_index`). +/// first's own members (its `id`, `model` and `metadata`, which the runs +/// share but for their ids), with every answer's predictions in order, each +/// numbered from the row its run starts at (`row_index`). fn merged_answers(answers: Vec<(usize, Value)>) -> Result { let mut answers = answers.into_iter(); let (first, mut whole) = answers @@ -721,7 +773,9 @@ pub(crate) async fn direct_answer( let rows = predict_rows(&call).unwrap_or(0); answers.push(( start, - answered(&how, call, body.to_bytes(), &mut most).await?, + answered(&how, call, body.to_bytes(), &mut most) + .await + .map_err(|unanswered| unanswered.error)?, )); start += rows; } From adfb31a403f1bc474d7c256d0467702123623012 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Mon, 9 Nov 2026 22:32:26 +0000 Subject: [PATCH 7/8] Split by call shape: cap halvings at 3, never split a temporary failure - Split right away only on a clear "too large": 413, Databricks' 400 "Request size cannot exceed" (read as 413 in the core), an answer over 64 MiB, or a timeout / 504 / 408; at most 3 halvings per batch, then fail naming the rows, bytes, status and what to change. - 500 / 502 / 503 and connection errors are retried at the same size, never split; 502 leaves the split set. - A call whose context is at least half its bytes is not split; it fails saying the context is too large for the endpoint. - Session create (context only) fails at once with that message; session scoring (batch sessions and fit()-pinned) splits its predict rows without a context check. - Tests per call shape in Rust, Python and Java; limits.md and the CHANGELOG updated. Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 4 +- docs/reference/limits.md | 18 +- .../com/nvidia/sdm/DatabricksServing.java | 4 +- .../java/com/nvidia/sdm/internal/Host.java | 3 +- .../com/nvidia/sdm/DatabricksServingTest.java | 47 +++- java/src/test/java/com/nvidia/sdm/Shop.java | 4 +- .../test/java/com/nvidia/sdm/TabularTest.java | 42 +++- .../src/sdm_client/_rust/run.py | 20 +- .../src/sdm_client/_rust/sessions.py | 58 ++++- .../src/sdm_client/core/databricks_serving.py | 11 +- .../tests/test_databricks_splitting.py | 35 ++- .../tests/test_resilient_calls.py | 76 +++++- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 2 + rust/crates/sdm-contract/src/error.rs | 2 + rust/crates/sdm-contract/src/wording.rs | 6 + rust/crates/sdm-core/src/batch_session.rs | 5 + rust/crates/sdm-core/src/frame_session.rs | 60 ++++- rust/crates/sdm-core/src/predict.rs | 234 +++++++++++++++--- rust/crates/sdm-core/src/session.rs | 54 ++-- rust/crates/sdm-core/src/tabular_arrow.rs | 20 +- rust/crates/sdm-core/tests/predict.rs | 87 ++++++- rust/crates/sdm-request/src/serving.rs | 22 +- rust/crates/sdm-request/tests/serving.rs | 22 ++ 24 files changed, 708 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 197d0350a..0c2146757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ The public API snapshot records this version update; publication remains pending ### Changed - A single-table request is scored in calls of 10,000 rows, each with the whole context, split further as a relational request is. -- A `500` is retried with backoff as a `502` or `503` is; through Databricks model serving, once more. +- A `500` is retried with backoff as a `502` or `503` is, at the same size; through Databricks model serving, a `500` or `502` once more. - `rows_per_call` is taken as asked rather than lowered to 10,000 (200 for a link prediction), which is now only its default. - `num_context_samples` past 10,000 and more than 6 hops are accepted. @@ -73,7 +73,7 @@ The public API snapshot records this version update; publication remains pending ### Removed - The client's own size limits: the 10,000 context rows, the 30 MiB request body, the 100,000 entities and 100 calls of one `materialize()`, and the 15 tables and 6 hops of a relational request. - The client sends its best attempt: a call the endpoint refuses as too large (`413`, a gateway's `502` or `504`) or times out on is asked again in halves, each with the whole context, and a `500` is retried as a `503` is. + The client sends its best attempt: a call the endpoint refuses as too large (`413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB) or times out on (`504`, a timeout) is asked again in halves, each with the whole context, at most 3 times per batch; one whose context is half its bytes or more is refused at once, naming the sizes. A session's create call, which carries only the context, is refused at once when too large; a session's scoring call is halved as a prediction is. A `500`, `502` or `503` is retried at the same size and never split. A `422`, or a refusal of a call of one row, is shown as the endpoint said it. The context is sampled only to a `num_context_samples` the caller passes; through Databricks model serving, a call is still split up front to fit its 16 MB. diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 0a1359aa4..6d8da53f8 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -5,12 +5,22 @@ It sends its best attempt and recovers from what the endpoint says: | The endpoint answers | The client | |---|---| -| `413`, a gateway's `502` or `504`, the call times out, or its answer is past 64 MiB | asks the call's rows again in two halves at once, each with the whole context, and keeps that smaller size for the request's later calls; a `502` is first tried again at its size | -| `503` or `429` | waits as `Retry-After` says and tries again | -| a connection error, a `500` or a `502` | tries again with backoff, up to `max_retries` (3 by default); through Databricks model serving, a `500` is tried once more | +| `413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB, a `504`, or the call times out | asks the call's rows again in two halves at once, each with the whole context, and keeps that smaller size for the request's later calls; a batch is halved at most 3 times, to an eighth of its rows, and then refused naming the call's rows, bytes and status | +| the same, where the context is half the call's bytes or more | does not split, since fewer rows cannot help, and says the context is too large for the endpoint, with the sizes | +| `503` or `429` | waits as `Retry-After` says and tries again at the same size | +| a connection error, a `500` or a `502` | tries again at the same size with backoff, up to `max_retries` (3 by default), and never splits; through Databricks model serving, a `500` or `502` is tried once more | | `422`, or a refusal of a call of one row | does not try again, and shows the endpoint's message | -The context is never split, so a context too large on its own is the endpoint's refusal, shown as it said it. +What a call too large or too slow comes to depends on what it carries: + +| Call | Too large or too slow | +|---|---| +| a prediction, context and rows to score | its rows are halved, at most 3 times, unless the context is half the call's bytes or more: then it is refused at once, naming the sizes | +| a session's create (`fit()`, or an automatic batch session), the context alone | refused at once: the context is too large for the endpoint; send fewer context rows or columns | +| a session's scoring call, rows to score alone | its rows are halved, at most 3 times; no context to weigh | + +A temporary failure is tried again at the same size on every call; none is split. +The context is never split. The context is sampled only to a `num_context_samples` you pass. `timeout` and `max_retries` are optional settings of the client. diff --git a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java index 605774da2..c2144df7e 100644 --- a/java/src/main/java/com/nvidia/sdm/DatabricksServing.java +++ b/java/src/main/java/com/nvidia/sdm/DatabricksServing.java @@ -239,8 +239,8 @@ private byte[] send(String path, byte[] body) throws Host.EndpointFailed, Host.T return response.body(); } // A 429 or a 503 is tried again until the deadline, as the SDK tries it; a - // 500 once more, as Python's transport tries it, then reported as it is. - if (status == 500 && !retried500) { + // 500 or a 502 once more, as Python's transport tries it, then reported as it is. + if ((status == 500 || status == 502) && !retried500) { retried500 = true; } else if (status != 429 && status != 503) { throw new Host.EndpointFailed(status, response.body()); diff --git a/java/src/main/java/com/nvidia/sdm/internal/Host.java b/java/src/main/java/com/nvidia/sdm/internal/Host.java index 1807bc2db..9ec069a81 100644 --- a/java/src/main/java/com/nvidia/sdm/internal/Host.java +++ b/java/src/main/java/com/nvidia/sdm/internal/Host.java @@ -409,7 +409,8 @@ private static Finished steps(Run run, Call call) { } } StructuredDataException error = error(failed, call.held(), details); - if (refused != null && (!cleanupRefusal + // A call the core split and gave up on is the core's to name. + if (refused != null && !Boolean.TRUE.equals(error.details().get("after_split")) && (!cleanupRefusal || (refused.status != null && error.details().get("status") instanceof Number status && status.intValue() == refused.status) || (refused.status == null && "TRANSPORT_ERROR".equals(error.code())))) { throw carried(refused, error); diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index e8c63c70f..194d5ab66 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -46,8 +46,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.table.Table; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -577,27 +579,52 @@ void aCallTheEndpointThrottlesIsTriedAgainWithTheSameBytes() throws Exception { } @Test - void a500IsTriedAgainAsA503Is() throws Exception { - try (Connection connection = Shop.open("duckdb"); - Workspace workspace = new Workspace(body -> Answer.of(500, "")); - SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { - assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); - assertEquals(2, workspace.bodies.size()); + void a500OrA502IsTriedAgainAtItsSize() throws Exception { + for (int status : new int[] {500, 502}) { + try (Connection connection = Shop.open("duckdb"); + Workspace workspace = new Workspace(body -> Answer.of(status, "")); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); + assertEquals(2, workspace.bodies.size(), "status " + status); + assertArrayEquals(workspace.bodies.get(0), workspace.bodies.get(1), "never split"); + } } } @Test void aCallTheGatewayRefusesAsTooLargeIsAskedInHalves() throws Exception { - for (int status : new int[] {413, 502}) { - try (Connection connection = Shop.open("duckdb"); - Workspace workspace = new Workspace(body -> Answer.of(status, "")); + String oversized = "{\"error_code\":\"BAD_REQUEST\",\"message\":\"Request size cannot exceed 16MB\"}"; + for (int status : new int[] {413, 400}) { + String text = status == 400 ? oversized : ""; + Request request; + try (RootAllocator allocator = new RootAllocator(); + Table context = TabularTest.table(allocator, true, 1); + Table predict = TabularTest.table(allocator, false, 2, 3)) { + request = new Request(context, predict, "y", "regression"); + } + try (Workspace workspace = new Workspace(body -> Answer.of(status, text)); SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { - assertEquals(2, client.predict(asked(connection), SdmClient.KUMO_RELATIONAL).rows()); + assertEquals(2, client.predict(request, SdmClient.KUMO_TABULAR).rows()); assertEquals(3, workspace.bodies.size(), "status " + status); } } } + @Test + void aCallWhoseContextDominatesIsNotSplit() throws Exception { + try (Connection connection = Shop.open("duckdb"); + Workspace workspace = new Workspace(body -> Answer.of(413, "")); + SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { + Request request = asked(connection); + StructuredDataException error = + assertThrows( + StructuredDataException.class, + () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); + assertTrue(error.getMessage().contains("the context is too large"), error.getMessage()); + assertEquals(1, workspace.bodies.size()); + } + } + @Test void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { int port; diff --git a/java/src/test/java/com/nvidia/sdm/Shop.java b/java/src/test/java/com/nvidia/sdm/Shop.java index 738671908..25c29ef7d 100644 --- a/java/src/test/java/com/nvidia/sdm/Shop.java +++ b/java/src/test/java/com/nvidia/sdm/Shop.java @@ -216,7 +216,9 @@ static byte[] scripted(byte[] body) { for (int at = 0; at < rows.size(); at++) { Map item = new LinkedHashMap<>(); // The instance id as its text, as the scripted endpoint gives it. - item.put("id", String.valueOf(((List) rows.get(at)).get(id))); + if (id >= 0) { + item.put("id", String.valueOf(((List) rows.get(at)).get(id))); + } item.put("row_index", (long) at); item.put("prediction", prediction); if (call.get("output") instanceof Map output diff --git a/java/src/test/java/com/nvidia/sdm/TabularTest.java b/java/src/test/java/com/nvidia/sdm/TabularTest.java index d5bba0cdb..b491899ff 100644 --- a/java/src/test/java/com/nvidia/sdm/TabularTest.java +++ b/java/src/test/java/com/nvidia/sdm/TabularTest.java @@ -58,6 +58,8 @@ static final class Endpoint implements AutoCloseable { int deleteFailuresRemaining; boolean wrongId; boolean gradients; + int createStatus = 200; + int failScores; Endpoint() throws Exception { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); server.createContext("/", exchange -> { @@ -72,7 +74,11 @@ static final class Endpoint implements AutoCloseable { @SuppressWarnings("unchecked") Map body = (Map)Json.read(exchange.getRequestBody().readAllBytes()); calls.add(body); byte[] answer; - if (path.equals("/v1/sessions")) { + if (path.equals("/v1/sessions") && createStatus != 200) { + creates++; exchange.sendResponseHeaders(createStatus, -1); return; + } else if (path.endsWith("/predictions") && path.startsWith("/v1/sessions/") && failScores > 0) { + failScores--; exchange.sendResponseHeaders(413, -1); return; + } else if (path.equals("/v1/sessions")) { creates++; answer = Json.write(Map.of("session_id", "s" + creates)); } else { @SuppressWarnings("unchecked") Map predict = (Map)body.get("predict"); @@ -125,6 +131,40 @@ static final class Endpoint implements AutoCloseable { } } + @Test void aSessionWhoseContextIsTooLargeIsRefusedNamingIt() throws Exception { + try (RootAllocator allocator = new RootAllocator(); Table context = table(allocator, true, 1, 2); + Table predict = table(allocator, false, 3); Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + endpoint.createStatus = 413; + Request request = new Request(context, predict, "y", "regression"); + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> client.fit(request, SdmClient.KUMO_TABULAR)); + assertTrue(refused.getMessage().contains("the context is too large for this endpoint"), + refused.getMessage()); + assertEquals(1, endpoint.creates); + } + } + + @Test void aPinnedSessionsScoringCallIsAskedInHalves() throws Exception { + try (RootAllocator allocator = new RootAllocator(); Table context = table(allocator, true, 1, 2); + Table predict = table(allocator, false, 3, 4, 5, 6); Endpoint endpoint = new Endpoint(); + SdmClient client = SdmClient.connect(endpoint.url())) { + Request request = new Request(context, predict, "y", "regression"); + FittedModel fitted = client.fit(request, SdmClient.KUMO_TABULAR); + endpoint.failScores = 1; + assertEquals(4, fitted.predict(request).rows()); + List sizes = new ArrayList<>(); + for (Map call : endpoint.calls.subList(1, endpoint.calls.size())) { + @SuppressWarnings("unchecked") Map half = (Map) call.get("predict"); + @SuppressWarnings("unchecked") Map rows = (Map) half.get("instance_table"); + sizes.add(((List) rows.get("rows")).size()); + assertFalse(call.containsKey("context")); + } + assertEquals(List.of(4, 2, 2), sizes); + fitted.close(); + } + } + @Test void directRequestSurvivesCallerClosureAndRestoresRowOrder() throws Exception { Request request; try (RootAllocator allocator = new RootAllocator(); diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index 570bcb51e..5e8b266e3 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -301,13 +301,21 @@ def _steps( _answered(handle, failed, error) if error is not None: # Raised outside the handler, so nothing is its context. - if refused is not None and ( - not cleanup_refusal - or ( - refused.status is not None - and error.details.get('status') == refused.status + # A call the core split and gave up on is the core's to name. + if ( + refused is not None + and not error.details.get('after_split') + and ( + not cleanup_refusal + or ( + refused.status is not None + and error.details.get('status') == refused.status + ) + or ( + refused.status is None + and error.code == 'TRANSPORT_ERROR' + ) ) - or (refused.status is None and error.code == 'TRANSPORT_ERROR') ): # The call's own failure, as the caller sees it, carrying # what the core carried on its end of the run. diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py index e619de18b..d2042e1e4 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py @@ -132,21 +132,47 @@ def send( primitive = inference is None or direct_options.can_encode(inference) def transmit(target: str | dict[str, Any], body: bytes) -> bytes: + from sdm_client._rust.run import Refused + native = json.loads(body) if 'inference' in native and not primitive: native['inference'] = inference - observed['payload'] = native - if target == 'create_session': - observed['reply'] = client._transport.create_session(native) - elif isinstance(target, dict) and set(target) == {'session_predict'}: - observed['reply'] = client._transport.session_predict( - target['session_predict']['session_id'], native - ) - else: - raise NativeBoundaryError('unexpected session transport endpoint') - return b'{}' + # The call as asked; the core may ask a scoring call's rows in + # smaller calls, whose answers it puts back together. + observed.setdefault('payload', native) + try: + if target == 'create_session': + reply = client._transport.create_session(native) + elif isinstance(target, dict) and set(target) == { + 'session_predict' + }: + reply = client._transport.session_predict( + target['session_predict']['session_id'], native + ) + else: + raise NativeBoundaryError( + 'unexpected session transport endpoint' + ) + except NativeBoundaryError: + raise + except Exception as error: + # The endpoint's status, for the core to split a call too large + # or refuse a context too large; the error raised as it is. A + # timeout is told as a 408, an answer too large to read a 413. + from sdm_client.core.transport import read_timed_out + + status = getattr(error, 'status_code', None) + if status is None and read_timed_out(error): + status = 408 + if status is None and getattr(error, 'details', {}).get( + 'response_too_large' + ): + status = 413 + raise Refused(error, status) from None + observed.setdefault('replies', []).append(reply) + return json.dumps(reply).encode() - run( + finished = run( {'op': 'send_tabular_session', 'input': 1, 'payload': 2}, { 1: codec.attachment( @@ -162,7 +188,15 @@ def transmit(target: str | dict[str, Any], body: bytes) -> bytes: }, model_endpoints=transmit, ) - return observed['reply'], observed['payload'] + answer = results.expect(finished.result, results.DirectValue) + sent = json.loads(finished.take(answer.value)) + merged = finished.take(sent['response']) + replies = observed['replies'] + # One call is answered by the transport's own reply, as it gave it; a + # call asked in smaller calls, by the answers the core put together. + if len(replies) == 1: + return replies[0], observed['payload'] + return json.loads(merged), observed['payload'] def fit( diff --git a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py index ea014c790..a3732234f 100644 --- a/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py +++ b/packages/structured-data-models-client/src/sdm_client/core/databricks_serving.py @@ -22,6 +22,7 @@ import math import re import threading +import time from collections.abc import Mapping from typing import Any @@ -223,8 +224,8 @@ def predict_body(self, body: bytes) -> bytes: says, for the core to name; anything but an ``OSError`` escapes, as from:meth:`predict`.""" self._require_open() - # The SDK retries a 429 or a 503, not a 500, which this client tries - # once more as it does on its own endpoint. + # The SDK retries a 429 or a 503, not a 500 or a 502, which this + # client tries once more, after a pause, as temporary. for attempt in (0, 1): try: answered = self._workspace.api_client.do( @@ -242,7 +243,8 @@ def predict_body(self, body: bytes) -> bytes: # The SDK's errors and its retry deadline are all OSError, as are # requests' own, as in predict. except OSError as error: - if attempt == 0 and _status_of(error) == 500: + if attempt == 0 and _status_of(error) in (500, 502): + time.sleep(_RETRY_PAUSE_SECONDS) continue raise _failed(error) from None raise AssertionError('unreachable') @@ -573,6 +575,9 @@ def _response_body(reply: Any, endpoint: str) -> str: return body +#: The pause before a 500 or a 502 is tried once more. +_RETRY_PAUSE_SECONDS = 1.0 + #: The statuses the SDK names only by an ``error_code``: a call too large, or #: a gateway's failure, which the core asks again in halves. _STATUS_OF_CODE = { diff --git a/packages/structured-data-models-client/tests/test_databricks_splitting.py b/packages/structured-data-models-client/tests/test_databricks_splitting.py index ab0041bce..875e37611 100644 --- a/packages/structured-data-models-client/tests/test_databricks_splitting.py +++ b/packages/structured-data-models-client/tests/test_databricks_splitting.py @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 """A Databricks serving endpoint drops a call too large as a connection, so a -call past its byte limit is split before it is sent; a 413, a 502 or a call +call past its byte limit is split before it is sent; a 413, the gateway's +400 saying a request is too large, a 504 or a call that timed out is asked again in halves; a deadline spent waiting on an unavailable endpoint is a 503, never split; and a 500 is tried once more.""" @@ -30,7 +31,10 @@ def answer(sent: dict[str, Any]) -> dict[str, Any]: r"""The served model's answer to ``sent``: one prediction per row.""" table = sent['predict']['instance_table'] - keys = sent['schema']['instance_table'].get('primary_key') or [] + keys = ( + sent.get('schema', {}).get('instance_table', {}).get('primary_key') + or [] + ) at = table['columns'].index(keys[0]) if keys else None return { 'object': 'prediction.response', @@ -131,6 +135,13 @@ def sdk_error(status: int, reason: str, body: bytes, kind: str) -> OSError: b'{"error_code": "REQUEST_LIMIT_EXCEEDED", "message": "slow"}', JSON, ), + # The gateway's refusal of a call too large. + 400: lambda: sdk_error( + 400, + 'Bad Request', + b'{"error_code": "BAD_REQUEST", "message": "Request size cannot exceed 16MB"}', + JSON, + ), 503: lambda: sdk_error( 503, 'Service Unavailable', @@ -158,7 +169,7 @@ def test_a_tabular_call_past_the_byte_limit_is_split_before_it_is_sent(caplog): assert any('smaller calls' in r.getMessage() for r in caplog.records) -@pytest.mark.parametrize('status', [413, 502, 504]) +@pytest.mark.parametrize('status', [413, 400, 504]) def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): error = STATUS[status]() assert _status_of(error) == status, type(error).__mro__ @@ -177,12 +188,20 @@ def test_a_tabular_call_refused_as_too_large_is_asked_in_halves(status): assert result.to_pandas()['row_index'].tolist() == [0, 1] -def test_a_500_is_tried_once_more(): - error = STATUS[500]() - assert type(error).__name__ == 'InternalError' - client, workspace = opened([error]) +@pytest.mark.parametrize('status', [500, 502]) +def test_a_500_or_a_502_is_tried_once_more_at_its_size(status, monkeypatch): + from sdm_client.core import databricks_serving + + monkeypatch.setattr(databricks_serving, '_RETRY_PAUSE_SECONDS', 0) + client, workspace = opened([STATUS[status]()]) client.predict(asked(), model='kumo-tabular') - assert len(workspace.serving_endpoints.calls) == 2 + calls = workspace.serving_endpoints.calls + assert len(calls) == 2 + # Never split: both calls carry every row. + sent = [ + json.loads(c['dataframe_records'][0]['request_json']) for c in calls + ] + assert [len(s['predict']['instance_table']['rows']) for s in sent] == [2, 2] def test_a_deadline_spent_waiting_is_a_503_and_a_call_timed_out_a_504(): diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py index e52f4939e..e3e9ebcf3 100644 --- a/packages/structured-data-models-client/tests/test_resilient_calls.py +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -77,11 +77,10 @@ def _timed_out() -> StructuredDataError: 'failure', [ _to_nim_error(413, b'{"detail": "too large"}'), - _to_nim_error(502, b'{}'), _to_nim_error(504, b'{}'), _timed_out(), ], - ids=['413', '502', '504', 'timeout'], + ids=['413', '504', 'timeout'], ) def test_a_call_too_large_or_too_slow_is_asked_again_in_halves(failure): request = gathered(context_rows=30, run_mode='fast', predict_rows=4) @@ -115,7 +114,9 @@ def test_a_call_of_one_row_refused_as_too_large_is_the_endpoint_s_refusal(): @contextmanager -def _nim(*failures: int) -> Iterator[tuple[SDMClient, list[dict]]]: +def _nim( + *failures: int, create: int = 200 +) -> Iterator[tuple[SDMClient, list[dict]]]: r"""A NIM on a real socket, reached over the client's own HTTP transport: each call answered row by row, the first ones failed with ``failures`` in turn, a 0 among them an answer too large to read.""" @@ -139,10 +140,17 @@ def _send(self, status: int, body: bytes) -> None: def do_GET(self) -> None: self._send(200, b'{"data": [{"id": "kumo-tabular"}]}') + def do_DELETE(self) -> None: + self._send(204, b'') + def do_POST(self) -> None: length = int(self.headers.get('Content-Length') or 0) payload = json.loads(self.rfile.read(length)) calls.append(payload) + if self.path == '/v1/sessions': + body = b'{"detail": "too large"}' if create != 200 else b'' + self._send(create, body or b'{"session_id": "s1"}') + return if script: status = script.pop(0) if status == 0: @@ -202,7 +210,7 @@ def test_a_tabular_request_is_scored_in_calls_of_10_000_rows(): @pytest.mark.parametrize( - 'failure', [413, 502, 504, 0], ids=['413', '502', '504', 'answer-too-large'] + 'failure', [413, 504, 0], ids=['413', '504', 'answer-too-large'] ) def test_a_tabular_call_too_large_is_asked_again_in_halves(failure, caplog): caplog.set_level('INFO', logger='sdm_core') @@ -278,3 +286,63 @@ def test_the_halves_answered_before_one_fails_are_kept(): assert _sizes(endpoint) == [4, 2, 2] assert refused.value.details['rows_answered'] == 2 assert len(refused.value.details['answered']) == 2 + + +@pytest.mark.parametrize('status', [502, 503, 500]) +def test_a_temporary_failure_is_never_split(status): + request = gathered(context_rows=30, run_mode='fast', predict_rows=4) + endpoint = Scripted(_to_nim_error(status, b'{"detail": "later"}')) + with pytest.raises(StructuredDataError): + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [4] + + +def test_a_batch_is_halved_at_most_three_times(): + request = gathered(context_rows=5, run_mode='fast', predict_rows=16) + endpoint = Scripted(*[_too_large() for _ in range(20)]) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [16, 8, 4, 2] + assert 'halved 3 times' in str(refused.value) + assert refused.value.details['halvings'] == 3 + + +def test_a_call_whose_context_dominates_is_not_split(): + request = gathered(context_rows=400, run_mode='best', predict_rows=2) + endpoint = Scripted(_too_large()) + with pytest.raises(StructuredDataError) as refused: + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [2] + assert 'the context is too large for this endpoint' in str(refused.value) + + +def test_a_timeout_counts_toward_the_three_halvings(): + request = gathered(context_rows=5, run_mode='fast', predict_rows=16) + endpoint = Scripted(_timed_out(), *[_too_large() for _ in range(20)]) + with pytest.raises(StructuredDataError): + client(endpoint).predict(request, model='kumo-relational') + assert _sizes(endpoint) == [16, 8, 4, 2] + + +def test_a_session_whose_context_is_too_large_is_refused_naming_it(): + """A create call carries only the context, which cannot be split.""" + with _nim(create=413) as (nim, calls): + with pytest.raises(StructuredDataError) as refused: + nim.fit(_tabular(4), model='kumo-tabular') + assert len(calls) == 1 + assert 'the context is too large for this endpoint' in str(refused.value) + + +def test_a_pinned_session_s_scoring_call_is_asked_in_halves(): + """A scoring call against a pinned context carries no context: its rows + are halved, whatever the context's size.""" + with _nim(413) as (nim, calls): + fitted = nim.fit(_tabular(4), model='kumo-tabular') + result = fitted.predict(_tabular(4)) + scoring = [c for c in calls if 'context' not in c] + assert [len(c['predict']['instance_table']['rows']) for c in scoring] == [ + 4, + 2, + 2, + ] + assert list(result.to_pandas()['row_index']) == [0, 1, 2, 3] diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 0b8998b74..91ec993ea 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:ae5e8122e4582b2bcbd186b1e26d77fd665b8ab7c085b39b65d1f46313025a41 sdm-contract-72.json +72 sha256:0aed25018f2ee08388374658ca173d9d8aa960af18e6223d65d24a90a1896dd3 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index 328f6a56c..723b6467a 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -17799,6 +17799,8 @@ "seed_ignored", "serving_value_not_json", "serving_past_limit", + "call_too_large_after_halving", + "context_too_large_for_endpoint", "serving_not_json", "serving_not_an_object", "serving_refused", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index dc9b0d74b..af9b0bbe0 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -1215,6 +1215,8 @@ sentences! { SeedIgnored = "seed_ignored", W::SeedIgnored; ServingValueNotJson = "serving_value_not_json", E::InvalidRequest; ServingPastLimit = "serving_past_limit", E::InvalidRequest; + CallTooLargeAfterHalving = "call_too_large_after_halving", E::InvalidRequest; + ContextTooLargeForEndpoint = "context_too_large_for_endpoint", E::InvalidRequest; ServingNotJson = "serving_not_json", E::TransportError; ServingNotAnObject = "serving_not_an_object", E::TransportError; ServingRefused = "serving_refused", E::EndpointRefused; diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 2b5a12f57..3d886ea22 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -1513,6 +1513,12 @@ pub fn template(sentence: Sentence) -> Template { S::ServingValueNotJson => { said(r#"this request carries a value that cannot be sent as JSON: {reason}"#) } + S::CallTooLargeAfterHalving => said( + r#"the endpoint refused a call of {rows:,} rows to score, {bytes:,} bytes, with status {status}, after it was halved {halvings} times; ask in calls of fewer rows (rows_per_call), or send fewer context rows or columns"#, + ), + S::ContextTooLargeForEndpoint => said( + r#"the context is too large for this endpoint: it is {context_bytes:,} of the call's {bytes:,} bytes, refused with status {status}, so a call of fewer rows cannot help; send fewer context rows or columns"#, + ), S::ServingPastLimit => said( r#"this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller num_context_samples, or fewer entities to score"#, ), diff --git a/rust/crates/sdm-core/src/batch_session.rs b/rust/crates/sdm-core/src/batch_session.rs index 84d81106a..254699362 100644 --- a/rust/crates/sdm-core/src/batch_session.rs +++ b/rust/crates/sdm-core/src/batch_session.rs @@ -90,6 +90,11 @@ impl Session { Ok(true) } + /// The session the batches are scored against, once it is open. + pub(crate) fn current(&self) -> Option<&str> { + self.current.as_deref() + } + /// None requests ordinary stateless fallback, never another retry layer. pub(crate) async fn score( &mut self, diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 744261d3d..98d487912 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -63,7 +63,9 @@ pub(crate) async fn call( endpoint: Endpoint, body: Value, ) -> Result { - match asker + // A create call carries only the context, which cannot be split. + let create = matches!(endpoint, Endpoint::CreateSession); + let result = match asker .call_model(endpoint, "application/json".into(), body.to_bytes()) .await { @@ -84,6 +86,10 @@ pub(crate) async fn call( .with_field("status", Detail::Scalar(Scalar::UInt16(status)))) } _ => Err(internal("session transport returned another reply kind")), + }; + match create { + true => result.map_err(|error| crate::predict::created(error, &body)), + false => result, } } pub(crate) async fn run( @@ -291,22 +297,54 @@ pub(crate) async fn run( crate::columns::expose(&asker, state.column_analysis.as_deref(), &state.outputs)?; let mut parts = Vec::new(); let mut start = 0; + // A scoring call carries no context: its rows are asked in halves + // where the endpoint refuses it as too large or too slow, the + // size kept for the calls after it. + let mut most = None; + let write = |call: &Value| { + Ok(subset( + call, + &["predict", "output", "inference", "metadata"], + )) + }; + let session_id = state.session_id.clone(); + let send = |call: Value, + bytes: Vec| + -> crate::predict::Sent<'_, sdm_request::predict::Part> { + let session_id = session_id.clone(); + let asker = &asker; + Box::pin(async move { + let answer = self::call( + asker, + Endpoint::SessionPredict { session_id }, + crate::predict::payload(&bytes)?, + ) + .await?; + let response = sdm_wire::response::PredictionResponse::read(&answer) + .map_err(predict::refused)?; + predict::layout(&call, &response, crate::pql::float_text) + }) + }; + let how = crate::predict::Asking { + write: &write, + send: &send, + merge: crate::predict::numbered_parts, + context: false, + }; for (body, layout) in plans { let answered: Result<_, CoreError> = async { let part = if predict::nothing_to_score(&layout) { predict::empty(&state.model, &layout) } else { - let answer = call( - &asker, - Endpoint::SessionPredict { - session_id: state.session_id.clone(), - }, - body, + crate::predict::answered( + &how, + layout.clone(), + body.to_bytes(), + &mut most, + 0, ) - .await?; - let response = sdm_wire::response::PredictionResponse::read(&answer) - .map_err(predict::refused)?; - predict::layout(&layout, &response, crate::pql::float_text)? + .await + .map_err(|unanswered| unanswered.error)? }; predict::numbered_from(part, start) } diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 437bb04db..701239603 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -478,12 +478,58 @@ pub(crate) async fn one( let mut most = None; for (((payload, request_id), (body, bytes)), start) in calls.iter().zip(plans).zip(starts) { let response = async { - if let Some(session) = session.as_mut() - && let Some(answer) = session.score(asker, &body).await? - { - let response = PredictionResponse::read(&answer).map_err(refused)?; - return layout(payload, &response, crate::pql::float_text) - .map_err(Unanswered::from); + if let Some(session) = session.as_mut() { + match session.score(asker, &body).await { + Ok(Some(answer)) => { + let response = PredictionResponse::read(&answer).map_err(refused)?; + return layout(payload, &response, crate::pql::float_text) + .map_err(Unanswered::from); + } + Ok(None) => {} + // The session's scoring call carries no context: its + // rows are asked in halves as a call's are. + Err(error) if too_large(&error) && session.current().is_some() => { + let session_id = session.current().unwrap_or_default().to_owned(); + let write = |call: &Value| { + let written = sdm_request::predict::body( + call, + &inputs.model, + request_id, + &outputs, + &inputs.inference_asked, + )?; + Ok(sdm_request::frame_session::subset( + &written, + &["predict", "output", "inference", "metadata"], + )) + }; + let send = |call: Value, bytes: Vec| -> Sent<'_, Part> { + let session_id = session_id.clone(); + Box::pin(async move { + let answer = crate::frame_session::call( + asker, + Endpoint::SessionPredict { session_id }, + self::payload(&bytes)?, + ) + .await?; + let response = + PredictionResponse::read(&answer).map_err(refused)?; + layout(&call, &response, crate::pql::float_text) + }) + }; + let how = Asking { + write: &write, + send: &send, + merge: numbered_parts, + context: false, + }; + let sent = body.to_bytes().len(); + let (size, halvings) = + split(&how, payload, &error, sent, &mut most, 0)?; + return in_runs(&how, payload.clone(), size, &mut most, halvings).await; + } + Err(error) => return Err(Unanswered::from(error)), + } } let write = |call: &Value| { sdm_request::predict::body( @@ -502,8 +548,9 @@ pub(crate) async fn one( write: &write, send: &send, merge: numbered_parts, + context: true, }; - answered(&how, payload.clone(), bytes, &mut most).await + answered(&how, payload.clone(), bytes, &mut most, 0).await } .await; let answered = match response { @@ -589,9 +636,15 @@ fn fitted( } /// The statuses that say a call was too large or too slow to answer: the -/// host's timeout (told as 408), the endpoint's 413, and a gateway's 502 or -/// 504, which a call it gave up on comes back as. -const TOO_LARGE: [u16; 4] = [408, 413, 502, 504]; +/// host's timeout (told as 408) or an answer too large to read (told as +/// 413), the endpoint's 413, and a gateway's 504, which a call it gave up +/// on comes back as. A 502, a 503 or a 500 is temporary: the host tries it +/// again at its size, and it is never split. +const TOO_LARGE: [u16; 3] = [408, 413, 504]; + +/// The most times one batch is halved (to an eighth of its rows) before the +/// call is refused as too large. +const MOST_HALVINGS: u32 = 3; /// A call sent, its answer to come. pub(crate) type Sent<'a, T> = @@ -604,6 +657,9 @@ pub(crate) struct Asking<'a, T> { pub write: &'a Write<'a>, pub send: &'a Sender<'a, T>, pub merge: Merge, + /// Whether the calls carry a context, which a call too large is weighed + /// against before it is split: none on a session's scoring calls. + pub context: bool, } /// The answers to runs of a call's rows, each with the row it starts at, @@ -629,35 +685,82 @@ pub(crate) async fn answered( payload: Value, bytes: Vec, most: &mut Option, + halvings: u32, ) -> Result> { let rows = predict_rows(&payload).unwrap_or(0); - let size = match *most { - Some(most) if rows > most => most, - _ => match (how.send)(payload.clone(), bytes).await { - Err(error) if rows > 1 && too_large(&error) => { - let half = rows.div_ceil(2); - tracing::info!( - target: "sdm_core::predict", - rows, - status = ?error.fields.get(sdm_contract::error::REFUSED_STATUS), - size = half, - "a call refused as too large or too slow is asked again in smaller calls" - ); - *most = Some(half); - half + let (size, halvings) = match *most { + Some(most) if rows > most => (most, halvings), + _ => { + let sent = bytes.len(); + match (how.send)(payload.clone(), bytes).await { + Err(error) if rows > 1 && too_large(&error) => { + split(how, &payload, &error, sent, most, halvings)? + } + answer => return answer.map_err(Unanswered::from), } - answer => return answer.map_err(Unanswered::from), - }, + } }; + in_runs(how, payload, size, most, halvings).await +} + +/// The size a call refused as too large (`error`, `sent` bytes) is asked +/// again in, and the halvings that make: half its rows, at most +/// [`MOST_HALVINGS`] times a batch; refused where its context is half its +/// bytes or more ([`Asking::context`]), which fewer rows cannot help. +pub(crate) fn split( + how: &Asking<'_, T>, + payload: &Value, + error: &CoreError, + sent: usize, + most: &mut Option, + halvings: u32, +) -> Result<(usize, u32), Unanswered> { + let rows = predict_rows(payload).unwrap_or(0); + if how.context { + // A context that is half the call or more is sent whole with every + // smaller call: fewer rows cannot help. + let context = payload.get("context").map_or(0, |c| c.to_bytes().len()); + if context * 2 >= sent { + return Err(Unanswered::from(context_too_large(error, context, sent))); + } + } + if halvings >= MOST_HALVINGS { + return Err(Unanswered::from(still_too_large( + error, rows, sent, halvings, + ))); + } + let half = rows.div_ceil(2); + tracing::info!( + target: "sdm_core::predict", + rows, + status = ?error.fields.get(sdm_contract::error::REFUSED_STATUS), + size = half, + halving = halvings + 1, + "a call refused as too large or too slow is asked again in smaller calls" + ); + *most = Some(half); + Ok((half, halvings + 1)) +} + +/// `payload`'s answer asked in runs of `size` rows, each as [`answered`] +/// asks it, merged; where a run fails, the runs before it are kept. +pub(crate) async fn in_runs( + how: &Asking<'_, T>, + payload: Value, + size: usize, + most: &mut Option, + halvings: u32, +) -> Result> { + let rows = predict_rows(&payload).unwrap_or(0); let mut answers = Vec::new(); - for start in (0..rows).step_by(size) { + for start in (0..rows).step_by(size.max(1)) { let range = start..(start + size).min(rows); let written = sdm_request::batching::predict_slice(&payload, range).and_then(|call| { let bytes = (how.write)(&call)?.to_bytes(); Ok((call, bytes)) }); let outcome = match written { - Ok((call, bytes)) => Box::pin(answered(how, call, bytes, most)).await, + Ok((call, bytes)) => Box::pin(answered(how, call, bytes, most, halvings)).await, Err(error) => Err(Unanswered::from(error)), }; match outcome { @@ -698,6 +801,76 @@ impl From for Unanswered { } } +/// The refusal of a call the endpoint still refused as too large after +/// [`MOST_HALVINGS`] halvings: its rows, bytes and status, and what to +/// change. `after_split` tells a host to raise it in place of its own. +fn still_too_large(error: &CoreError, rows: usize, bytes: usize, halvings: u32) -> CoreError { + CoreError::new( + Sentence::CallTooLargeAfterHalving, + "a call was still too large after it was halved as often as it is", + ) + .with_field("rows", count(rows)) + .with_field("bytes", count(bytes)) + .with_field("halvings", Detail::Scalar(Scalar::UInt32(halvings))) + .with_field("status", refused_status(error)) + .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) +} + +/// The refusal of a call too large whose context is half its bytes or more, +/// which no call of fewer rows makes smaller. +fn context_too_large(error: &CoreError, context: usize, bytes: usize) -> CoreError { + CoreError::new( + Sentence::ContextTooLargeForEndpoint, + "the context alone is half a call too large or more", + ) + .with_field("context_bytes", count(context)) + .with_field("bytes", count(bytes)) + .with_field("status", refused_status(error)) + .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) +} + +/// The status an endpoint refused a call with. +fn refused_status(error: &CoreError) -> Detail { + error + .fields + .get(sdm_contract::error::REFUSED_STATUS) + .cloned() + .unwrap_or(Detail::Scalar(Scalar::Null)) +} + +/// The answer to a single-table session's scoring call, `payload`, sent by +/// `send`: it carries no context, so its rows are asked in halves where the +/// endpoint refuses it as too large or too slow ([`answered`]), the answers +/// merged as one ([`merged_answers`]). +pub(crate) async fn session_answer<'a>( + payload: &Value, + send: &'a (dyn Fn(Value) -> Sent<'a, Value> + Sync), +) -> Result { + let write = |call: &Value| Ok(call.clone()); + let sender = |call: Value, _: Vec| send(call); + let how = Asking { + write: &write, + send: &sender, + merge: merged_answers, + context: false, + }; + answered(&how, payload.clone(), payload.to_bytes(), &mut None, 0) + .await + .map_err(|unanswered| unanswered.error) +} + +/// A session's create call's refusal: the call carries only the context, +/// which cannot be split, so a call refused as too large or too slow is +/// refused naming the context's bytes; any other refusal stands. +pub(crate) fn created(error: CoreError, body: &Value) -> CoreError { + if !too_large(&error) { + return error; + } + let bytes = body.to_bytes().len(); + let context = body.get("context").map_or(bytes, |c| c.to_bytes().len()); + context_too_large(&error, context, bytes) +} + /// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. fn too_large(error: &CoreError) -> bool { matches!( @@ -707,7 +880,7 @@ fn too_large(error: &CoreError) -> bool { } /// Parts of one call's rows, each numbered from the row it starts at. -fn numbered_parts(parts: Vec<(usize, Part)>) -> Result { +pub(crate) fn numbered_parts(parts: Vec<(usize, Part)>) -> Result { merge( parts .into_iter() @@ -765,6 +938,7 @@ pub(crate) async fn direct_answer( write: &write, send: &send, merge: merged_answers, + context: true, }; let mut most = usize::try_from(sdm_request::options::PREDICT_ROWS).ok(); let mut answers = Vec::new(); @@ -773,7 +947,7 @@ pub(crate) async fn direct_answer( let rows = predict_rows(&call).unwrap_or(0); answers.push(( start, - answered(&how, call, body.to_bytes(), &mut most) + answered(&how, call, body.to_bytes(), &mut most, 0) .await .map_err(|unanswered| unanswered.error)?, )); diff --git a/rust/crates/sdm-core/src/session.rs b/rust/crates/sdm-core/src/session.rs index 4da783244..2120da0f5 100644 --- a/rust/crates/sdm-core/src/session.rs +++ b/rust/crates/sdm-core/src/session.rs @@ -281,32 +281,54 @@ pub(crate) async fn send( } } sdm_request::inference::envelope(&mut payload)?; - let reply = asker - .call_model( - request.endpoint, - "application/json".into(), - payload.to_bytes(), - ) - .await; - let (status, body) = match reply { + // A create call carries only the context, which cannot be split; a + // scoring call's rows are asked in halves where the endpoint refuses it + // as too large or too slow. + let (status, body) = match request.endpoint { + Endpoint::CreateSession => { + let created = sent(asker, Endpoint::CreateSession, &payload) + .await + .map_err(|error| crate::predict::created(error, &payload))?; + (200, created) + } + endpoint => { + let send = |call: Value| -> crate::predict::Sent<'_, Value> { + let endpoint = endpoint.clone(); + Box::pin( + async move { crate::predict::payload(&sent(asker, endpoint, &call).await?) }, + ) + }; + let answer = crate::predict::session_answer(&payload, &send).await?; + (200, answer.to_bytes()) + } + }; + let response = asker.attach(Attachment::Bytes(body))?; + crate::direct::value(asker, &SessionTransportReply { status, response }) +} + +/// The answer to `payload` sent to `endpoint`, its bytes; a status that is no +/// success, or a failed transport, refused. +async fn sent(asker: &Asker, endpoint: Endpoint, payload: &Value) -> Result, CoreError> { + match asker + .call_model(endpoint, "application/json".into(), payload.to_bytes()) + .await + { Reply::Model(ModelResult::Response { status, body, .. }) if (200..300).contains(&status) => { - (status, body) + Ok(body) } Reply::Model(ModelResult::Response { status, .. } | ModelResult::Failed { status, .. }) => { - return Err( + Err( sdm_request::predict::bad_response("status", "not_a_success") .with_field("status", Detail::Scalar(Scalar::UInt16(status))), - ); + ) } Reply::Model(ModelResult::TransportError { kind, retryable }) => { - return Err(CoreError::transport(kind, retryable)); + Err(CoreError::transport(kind, retryable)) } - Reply::Fetch(_) => return Err(internal("session call was answered as a fetch")), - }; - let response = asker.attach(Attachment::Bytes(body))?; - crate::direct::value(asker, &SessionTransportReply { status, response }) + Reply::Fetch(_) => Err(internal("session call was answered as a fetch")), + } } /// A deletion has no response document. Only a successful transport outcome diff --git a/rust/crates/sdm-core/src/tabular_arrow.rs b/rust/crates/sdm-core/src/tabular_arrow.rs index 6d00a3219..7b8a62c29 100644 --- a/rust/crates/sdm-core/src/tabular_arrow.rs +++ b/rust/crates/sdm-core/src/tabular_arrow.rs @@ -548,7 +548,9 @@ async fn fit( tabular::typed_inference(input.prediction.request.inference.clone())?; let column_analysis = Some(crate::columns::compute(std::slice::from_ref(&payload))?); crate::columns::expose(asker, column_analysis.as_deref(), &prepared.bound.outputs)?; - let answer = session_call(asker, Endpoint::CreateSession, payload).await?; + let answer = session_call(asker, Endpoint::CreateSession, payload.clone()) + .await + .map_err(|error| crate::predict::created(error, &payload))?; let session_id = answer .get("session_id") .or_else(|| answer.get("id")) @@ -719,13 +721,15 @@ async fn score( part.columns = analysis; return crate::predict::finish(asker, part, state.bound.outputs.clone()); } - let answer = session_call( - asker, - sdm_contract::run::Endpoint::SessionPredict { - session_id: state.session_id.clone(), - }, - payload, - ) + let answer = crate::predict::session_answer(&payload, &|body| { + Box::pin(session_call( + asker, + sdm_contract::run::Endpoint::SessionPredict { + session_id: state.session_id.clone(), + }, + body, + )) + }) .await?; finish(asker, layout, answer, state.bound.outputs.clone(), analysis) } diff --git a/rust/crates/sdm-core/tests/predict.rs b/rust/crates/sdm-core/tests/predict.rs index 6d2ec01c5..baa728e24 100644 --- a/rust/crates/sdm-core/tests/predict.rs +++ b/rust/crates/sdm-core/tests/predict.rs @@ -34,7 +34,8 @@ use sdm_contract::ops::{ }; use sdm_contract::run::{Endpoint, StartRun}; use sdm_contract::{ - CONTRACT_VERSION, CoreError, Detail, ErrorCode, ErrorKind, HostFacts, Scalar, Step, Unit, + CONTRACT_VERSION, CoreError, Detail, ErrorCode, ErrorKind, HostFacts, Scalar, Sentence, Step, + Unit, }; use sdm_core::{ Attachment, Attachments, CompositionRoot, ModelResult, Run, UnknownZone, Zones, start, @@ -1315,7 +1316,7 @@ fn scored(body: &[u8]) -> Vec { /// context, and the answers are numbered as one. #[test] fn a_call_too_large_or_too_slow_is_asked_again_in_halves() { - for status in [408, 413, 502, 504] { + for status in [408, 413, 504] { let mut run = predict_run(REQUEST_STATE_VERSION, payload_text(), "kumo-relational").unwrap(); let (id, whole) = call(&mut run); @@ -1334,11 +1335,83 @@ fn a_call_too_large_or_too_slow_is_asked_again_in_halves() { } } -/// A `422`, or any refusal not of a size, is not asked again: the run ends -/// with the endpoint's refusal. +/// A `422`, a `502`, a `503` or a `500`, or any refusal not of a size, is +/// not asked again by the core: the run ends with the endpoint's refusal +/// (the host has tried a temporary one again at its size). #[test] -fn a_422_is_not_asked_again() { - let (_, end) = answered(422, b"{}"); +fn a_refusal_not_of_a_size_is_not_split() { + for status in [422, 502, 503, 500] { + let (_, end) = answered(status, b"{}"); + let error = end.unwrap_err(); + assert_eq!( + error.fields["status"], + Detail::Scalar(Scalar::UInt16(status)), + "{status}" + ); + } +} + +/// `payload_text` scoring `rows` rows, its context of `context` rows. +fn sized_payload(rows: usize, context: usize) -> String { + let scored: Vec = (0..rows) + .map(|i| format!("[{},{}]", context + i, i)) + .collect(); + let known: Vec = (0..context).map(|i| format!("[{i},{i}.5]")).collect(); + payload_text() + .replace( + r#"[[2,7],[3,"9007199254740993"]]"#, + &format!("[{}]", scored.join(",")), + ) + .replace( + "[[0,1e-05],[1,12345678.900000001]]", + &format!("[{}]", known.join(",")), + ) +} + +/// A batch is halved at most three times, to an eighth of its rows; still +/// refused then, the call is refused naming its rows, bytes and status. +#[test] +fn a_batch_is_halved_at_most_three_times() { + let mut run = predict_run( + REQUEST_STATE_VERSION, + sized_payload(16, 2), + "kumo-relational", + ) + .unwrap(); + let mut sizes = Vec::new(); + let end = loop { + match run.next() { + Ok(Step::CallModel(call)) => { + let Some(Attachment::Bytes(body)) = run.take_attachment(call.body) else { + panic!("the body is attached"); + }; + sizes.push(scored(&body).len()); + run.submit_model(call.id, refused_with(413)).unwrap(); + } + other => break other, + } + }; + assert_eq!(sizes, [16, 8, 4, 2]); let error = end.unwrap_err(); - assert_eq!(error.fields["status"], Detail::Scalar(Scalar::UInt16(422))); + assert_eq!(error.sentence, Sentence::CallTooLargeAfterHalving); + assert_eq!(error.fields["rows"], Detail::Scalar(Scalar::UInt64(2))); + assert_eq!(error.fields["halvings"], Detail::Scalar(Scalar::UInt32(3))); + assert_eq!(error.fields["status"], Detail::Scalar(Scalar::UInt16(413))); +} + +/// A call whose context is half its bytes or more is not split: fewer rows +/// cannot make it small enough. +#[test] +fn a_call_whose_context_dominates_is_not_split() { + let mut run = predict_run( + REQUEST_STATE_VERSION, + sized_payload(2, 400), + "kumo-relational", + ) + .unwrap(); + let (id, _) = call(&mut run); + run.submit_model(id, refused_with(413)).unwrap(); + let error = run.next().unwrap_err(); + assert_eq!(error.sentence, Sentence::ContextTooLargeForEndpoint); + assert!(error.fields.contains_key("context_bytes")); } diff --git a/rust/crates/sdm-request/src/serving.rs b/rust/crates/sdm-request/src/serving.rs index b12486550..1c9e61e13 100644 --- a/rust/crates/sdm-request/src/serving.rs +++ b/rust/crates/sdm-request/src/serving.rs @@ -57,8 +57,9 @@ const STATUS_BY_CODE: &[(&str, u16)] = &[ ]; /// The HTTP statuses whose SDK error class the old client reports as the -/// status itself; any other it reports as 503, which callers retry. A 413, -/// a 502 or a 504 is kept, so the core asks the call's rows in halves. +/// status itself; any other it reports as 503, which callers retry. A 413 or +/// a 504 is kept, so the core asks the call's rows in halves; a 502, so it +/// is named as the gateway's. const STATUS_KEPT: &[u16] = &[400, 401, 403, 404, 409, 413, 429, 502, 504]; /// The call `call` (the core's own JSON) as the endpoint's envelope, the @@ -120,6 +121,15 @@ pub fn past_limit(endpoint: &str, max_request_bytes: u64, size: u64) -> CoreErro .with_field("limit", count(max_request_bytes)) } +/// What the gateway's refusal of a call too large says, a 400 whose message +/// is this (`Request size cannot exceed 16MB`), matched case-insensitively. +const OVERSIZED: &str = "request size cannot exceed"; + +/// Whether a failure's text is the gateway's refusal of a call too large. +pub fn oversized(text: &str) -> bool { + text.to_ascii_lowercase().contains(OVERSIZED) +} + /// The served model's answer in the endpoint's reply `body` (a 2xx), a JSON /// object, or the error the old client raises for it. pub fn answer(endpoint: &str, body: &[u8]) -> Result { @@ -216,7 +226,13 @@ pub fn failure(endpoint: &str, failed: &Failure<'_>) -> CoreError { rest_body(&detail, failed.status) }; let raw_code = raw_code.map(|code| canonical_code(&code)); - let status = reported_status(failed.status, raw_code.as_deref()); + // The gateway refuses a call too large as a 400 that says so, which the + // core asks again in halves as it does a 413. + let status = if failed.status == 400 && oversized(&text) { + 413 + } else { + reported_status(failed.status, raw_code.as_deref()) + }; let provider_code = raw_code.as_deref().filter(|c| code_shaped(c)); let code = model_diagnosis(&text, provider_code); if status >= 500 diff --git a/rust/crates/sdm-request/tests/serving.rs b/rust/crates/sdm-request/tests/serving.rs index 67c4af02d..c4795efad 100644 --- a/rust/crates/sdm-request/tests/serving.rs +++ b/rust/crates/sdm-request/tests/serving.rs @@ -758,3 +758,25 @@ fn an_invalid_status_is_a_malformed_response_not_a_clamped_status() { assert_eq!(error.code, ErrorCode::InvalidResponse); assert_eq!(error.fields["reason"], text("invalid_serving_status")); } + +/// The gateway refuses a call too large as a 400 saying so: read as a 413, +/// so the core asks the call's rows in halves, whichever host carried it. +#[test] +fn the_gateway_s_oversize_400_is_read_as_a_413() { + for body in [ + r#"{"error_code":"BAD_REQUEST","message":"Request size cannot exceed 16MB"}"#, + "BAD_REQUEST: Request size cannot exceed 16777216 bytes", + ] { + let error = rest(400, body); + assert_eq!( + error.fields[REFUSED_STATUS], + Detail::Scalar(Scalar::UInt16(413)), + "{body}" + ); + } + let other = rest(400, r#"{"error_code":"BAD_REQUEST","message":"bad row"}"#); + assert_eq!( + other.fields[REFUSED_STATUS], + Detail::Scalar(Scalar::UInt16(400)) + ); +} From cc494abc9adcad51bece6903df94cb0bbab49886 Mon Sep 17 00:00:00 2001 From: Adi Wadaskar Date: Tue, 10 Nov 2026 01:53:47 +0000 Subject: [PATCH 8/8] Drop the context-dominance rule: halve the rows on any call too large A prediction refused as too large or too slow (408, 413, 504) now just has its predict rows halved, at most 3 times, then fails naming its rows, bytes and status, whatever share of its bytes the context is. A session's create call, context only, still fails at once saying the context is too large; its message now gives the call's bytes and status. Signed-off-by: Adi Wadaskar --- CHANGELOG.md | 2 +- docs/reference/limits.md | 5 +-- .../com/nvidia/sdm/DatabricksServingTest.java | 15 -------- .../tests/test_resilient_calls.py | 9 ----- rust/crates/sdm-contract/src/wording.rs | 2 +- rust/crates/sdm-core/src/frame_session.rs | 1 - rust/crates/sdm-core/src/predict.rs | 37 ++++--------------- rust/crates/sdm-core/tests/predict.rs | 17 --------- 8 files changed, 12 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c2146757..ad98926ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,7 @@ The public API snapshot records this version update; publication remains pending ### Removed - The client's own size limits: the 10,000 context rows, the 30 MiB request body, the 100,000 entities and 100 calls of one `materialize()`, and the 15 tables and 6 hops of a relational request. - The client sends its best attempt: a call the endpoint refuses as too large (`413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB) or times out on (`504`, a timeout) is asked again in halves, each with the whole context, at most 3 times per batch; one whose context is half its bytes or more is refused at once, naming the sizes. A session's create call, which carries only the context, is refused at once when too large; a session's scoring call is halved as a prediction is. A `500`, `502` or `503` is retried at the same size and never split. + The client sends its best attempt: a call the endpoint refuses as too large (`413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB) or times out on (`504`, a timeout) is asked again in halves, each with the whole context, at most 3 times per batch, then refused naming its rows, bytes and status. A session's create call, which carries only the context, is refused at once when too large; a session's scoring call is halved as a prediction is. A `500`, `502` or `503` is retried at the same size and never split. A `422`, or a refusal of a call of one row, is shown as the endpoint said it. The context is sampled only to a `num_context_samples` the caller passes; through Databricks model serving, a call is still split up front to fit its 16 MB. diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 6d8da53f8..34286e76a 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -6,7 +6,6 @@ It sends its best attempt and recovers from what the endpoint says: | The endpoint answers | The client | |---|---| | `413`, Databricks' `400` saying "Request size cannot exceed", an answer past 64 MiB, a `504`, or the call times out | asks the call's rows again in two halves at once, each with the whole context, and keeps that smaller size for the request's later calls; a batch is halved at most 3 times, to an eighth of its rows, and then refused naming the call's rows, bytes and status | -| the same, where the context is half the call's bytes or more | does not split, since fewer rows cannot help, and says the context is too large for the endpoint, with the sizes | | `503` or `429` | waits as `Retry-After` says and tries again at the same size | | a connection error, a `500` or a `502` | tries again at the same size with backoff, up to `max_retries` (3 by default), and never splits; through Databricks model serving, a `500` or `502` is tried once more | | `422`, or a refusal of a call of one row | does not try again, and shows the endpoint's message | @@ -15,9 +14,9 @@ What a call too large or too slow comes to depends on what it carries: | Call | Too large or too slow | |---|---| -| a prediction, context and rows to score | its rows are halved, at most 3 times, unless the context is half the call's bytes or more: then it is refused at once, naming the sizes | +| a prediction, context and rows to score | its rows are halved, at most 3 times, each smaller call with the whole context; then refused, naming the rows, bytes and status | | a session's create (`fit()`, or an automatic batch session), the context alone | refused at once: the context is too large for the endpoint; send fewer context rows or columns | -| a session's scoring call, rows to score alone | its rows are halved, at most 3 times; no context to weigh | +| a session's scoring call, rows to score alone | its rows are halved, at most 3 times | A temporary failure is tried again at the same size on every call; none is split. The context is never split. diff --git a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java index 194d5ab66..adae7fd06 100644 --- a/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java +++ b/java/src/test/java/com/nvidia/sdm/DatabricksServingTest.java @@ -610,21 +610,6 @@ void aCallTheGatewayRefusesAsTooLargeIsAskedInHalves() throws Exception { } } - @Test - void aCallWhoseContextDominatesIsNotSplit() throws Exception { - try (Connection connection = Shop.open("duckdb"); - Workspace workspace = new Workspace(body -> Answer.of(413, "")); - SdmClient client = SdmClient.forDatabricksServing(workspace.serving(16_000_000))) { - Request request = asked(connection); - StructuredDataException error = - assertThrows( - StructuredDataException.class, - () -> client.predict(request, SdmClient.KUMO_RELATIONAL)); - assertTrue(error.getMessage().contains("the context is too large"), error.getMessage()); - assertEquals(1, workspace.bodies.size()); - } - } - @Test void anEndpointThatCannotBeReachedIsAFailureAtItsDeadline() throws Exception { int port; diff --git a/packages/structured-data-models-client/tests/test_resilient_calls.py b/packages/structured-data-models-client/tests/test_resilient_calls.py index e3e9ebcf3..673c037c4 100644 --- a/packages/structured-data-models-client/tests/test_resilient_calls.py +++ b/packages/structured-data-models-client/tests/test_resilient_calls.py @@ -307,15 +307,6 @@ def test_a_batch_is_halved_at_most_three_times(): assert refused.value.details['halvings'] == 3 -def test_a_call_whose_context_dominates_is_not_split(): - request = gathered(context_rows=400, run_mode='best', predict_rows=2) - endpoint = Scripted(_too_large()) - with pytest.raises(StructuredDataError) as refused: - client(endpoint).predict(request, model='kumo-relational') - assert _sizes(endpoint) == [2] - assert 'the context is too large for this endpoint' in str(refused.value) - - def test_a_timeout_counts_toward_the_three_halvings(): request = gathered(context_rows=5, run_mode='fast', predict_rows=16) endpoint = Scripted(_timed_out(), *[_too_large() for _ in range(20)]) diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 3d886ea22..ea72f0c42 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -1517,7 +1517,7 @@ pub fn template(sentence: Sentence) -> Template { r#"the endpoint refused a call of {rows:,} rows to score, {bytes:,} bytes, with status {status}, after it was halved {halvings} times; ask in calls of fewer rows (rows_per_call), or send fewer context rows or columns"#, ), S::ContextTooLargeForEndpoint => said( - r#"the context is too large for this endpoint: it is {context_bytes:,} of the call's {bytes:,} bytes, refused with status {status}, so a call of fewer rows cannot help; send fewer context rows or columns"#, + r#"the context is too large for this endpoint: the call that sends it, {bytes:,} bytes, was refused with status {status}; send fewer context rows or columns"#, ), S::ServingPastLimit => said( r#"this request is {bytes:,} bytes once wrapped for serving endpoint {endpoint!r}, above the {limit:,} byte limit it accepts. Send fewer rows per call: a smaller num_context_samples, or fewer entities to score"#, diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 98d487912..9120d747a 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -329,7 +329,6 @@ pub(crate) async fn run( write: &write, send: &send, merge: crate::predict::numbered_parts, - context: false, }; for (body, layout) in plans { let answered: Result<_, CoreError> = async { diff --git a/rust/crates/sdm-core/src/predict.rs b/rust/crates/sdm-core/src/predict.rs index 701239603..a2df3b6e7 100644 --- a/rust/crates/sdm-core/src/predict.rs +++ b/rust/crates/sdm-core/src/predict.rs @@ -521,11 +521,9 @@ pub(crate) async fn one( write: &write, send: &send, merge: numbered_parts, - context: false, }; let sent = body.to_bytes().len(); - let (size, halvings) = - split(&how, payload, &error, sent, &mut most, 0)?; + let (size, halvings) = split(payload, &error, sent, &mut most, 0)?; return in_runs(&how, payload.clone(), size, &mut most, halvings).await; } Err(error) => return Err(Unanswered::from(error)), @@ -548,7 +546,6 @@ pub(crate) async fn one( write: &write, send: &send, merge: numbered_parts, - context: true, }; answered(&how, payload.clone(), bytes, &mut most, 0).await } @@ -657,9 +654,6 @@ pub(crate) struct Asking<'a, T> { pub write: &'a Write<'a>, pub send: &'a Sender<'a, T>, pub merge: Merge, - /// Whether the calls carry a context, which a call too large is weighed - /// against before it is split: none on a session's scoring calls. - pub context: bool, } /// The answers to runs of a call's rows, each with the row it starts at, @@ -694,7 +688,7 @@ pub(crate) async fn answered( let sent = bytes.len(); match (how.send)(payload.clone(), bytes).await { Err(error) if rows > 1 && too_large(&error) => { - split(how, &payload, &error, sent, most, halvings)? + split(&payload, &error, sent, most, halvings)? } answer => return answer.map_err(Unanswered::from), } @@ -705,10 +699,8 @@ pub(crate) async fn answered( /// The size a call refused as too large (`error`, `sent` bytes) is asked /// again in, and the halvings that make: half its rows, at most -/// [`MOST_HALVINGS`] times a batch; refused where its context is half its -/// bytes or more ([`Asking::context`]), which fewer rows cannot help. +/// [`MOST_HALVINGS`] times a batch. pub(crate) fn split( - how: &Asking<'_, T>, payload: &Value, error: &CoreError, sent: usize, @@ -716,14 +708,6 @@ pub(crate) fn split( halvings: u32, ) -> Result<(usize, u32), Unanswered> { let rows = predict_rows(payload).unwrap_or(0); - if how.context { - // A context that is half the call or more is sent whole with every - // smaller call: fewer rows cannot help. - let context = payload.get("context").map_or(0, |c| c.to_bytes().len()); - if context * 2 >= sent { - return Err(Unanswered::from(context_too_large(error, context, sent))); - } - } if halvings >= MOST_HALVINGS { return Err(Unanswered::from(still_too_large( error, rows, sent, halvings, @@ -816,14 +800,13 @@ fn still_too_large(error: &CoreError, rows: usize, bytes: usize, halvings: u32) .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) } -/// The refusal of a call too large whose context is half its bytes or more, -/// which no call of fewer rows makes smaller. -fn context_too_large(error: &CoreError, context: usize, bytes: usize) -> CoreError { +/// The refusal of a session's create call, which carries only the context, +/// refused as too large or too slow: `bytes` cannot be split. +fn context_too_large(error: &CoreError, bytes: usize) -> CoreError { CoreError::new( Sentence::ContextTooLargeForEndpoint, - "the context alone is half a call too large or more", + "a call that sends only the context was refused as too large", ) - .with_field("context_bytes", count(context)) .with_field("bytes", count(bytes)) .with_field("status", refused_status(error)) .with_field("after_split", Detail::Scalar(Scalar::Bool(true))) @@ -852,7 +835,6 @@ pub(crate) async fn session_answer<'a>( write: &write, send: &sender, merge: merged_answers, - context: false, }; answered(&how, payload.clone(), payload.to_bytes(), &mut None, 0) .await @@ -866,9 +848,7 @@ pub(crate) fn created(error: CoreError, body: &Value) -> CoreError { if !too_large(&error) { return error; } - let bytes = body.to_bytes().len(); - let context = body.get("context").map_or(bytes, |c| c.to_bytes().len()); - context_too_large(&error, context, bytes) + context_too_large(&error, body.to_bytes().len()) } /// Whether `error` is a refusal by one of the [`TOO_LARGE`] statuses. @@ -938,7 +918,6 @@ pub(crate) async fn direct_answer( write: &write, send: &send, merge: merged_answers, - context: true, }; let mut most = usize::try_from(sdm_request::options::PREDICT_ROWS).ok(); let mut answers = Vec::new(); diff --git a/rust/crates/sdm-core/tests/predict.rs b/rust/crates/sdm-core/tests/predict.rs index baa728e24..cdb78ec4f 100644 --- a/rust/crates/sdm-core/tests/predict.rs +++ b/rust/crates/sdm-core/tests/predict.rs @@ -1398,20 +1398,3 @@ fn a_batch_is_halved_at_most_three_times() { assert_eq!(error.fields["halvings"], Detail::Scalar(Scalar::UInt32(3))); assert_eq!(error.fields["status"], Detail::Scalar(Scalar::UInt16(413))); } - -/// A call whose context is half its bytes or more is not split: fewer rows -/// cannot make it small enough. -#[test] -fn a_call_whose_context_dominates_is_not_split() { - let mut run = predict_run( - REQUEST_STATE_VERSION, - sized_payload(2, 400), - "kumo-relational", - ) - .unwrap(); - let (id, _) = call(&mut run); - run.submit_model(id, refused_with(413)).unwrap(); - let error = run.next().unwrap_err(); - assert_eq!(error.sentence, Sentence::ContextTooLargeForEndpoint); - assert!(error.fields.contains_key("context_bytes")); -}