From f52998a1247a365b6eff588699562c3e68e3dc34 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 15:29:46 -0800 Subject: [PATCH 01/24] Close the gaps before 2.0.0 Fixes the differences between client 1.0.0 and the Rust core found by the parity audits, so 2.0.0 keeps 1.0.0's behaviour or documents where it changes. - Release: CHANGELOG Removed section and upgrade note, native-wheel platforms, prerelease tags in publish.yml, every version location in RELEASING.md, client NOTICE and LICENSE, Java built and tested but not released. - Payloads: foreign keys outside the sampled hops are left out again, nullable large keys stay integers, client-only keys stay out of Request.payloads. - Results and errors: merge() accepts Decimal and keeps repeated candidates, unknown model ids and predict timeouts keep their details, empty predictions keep their columns, warnings and query refusals keep 1.0.0's wording, PROVIDER_ERROR is documented. - Column analysis follows #73. - Connectors: Snowflake semantic-view read failures are no longer reported as missing, an ignored random_seed warns, declared time columns keep their declaration. - Docs describe the Rust core, the retired QA runner is removed, and the tests lost in the rewrite are restored. Signed-off-by: Manush Murali Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish.yml | 47 +- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 86 ++- NOTICE | 2 +- README.md | 2 +- RELEASING.md | 45 +- contract/tests/test_exact_numeric_keys.py | 28 +- contract/tests/test_sqlite_source_edges.py | 15 - docs/about/overview.md | 7 +- docs/get-started/installation.md | 2 +- docs/get-started/quickstart.md | 5 +- docs/reference/prediction-output.md | 5 +- docs/reference/stability.md | 5 + examples/local/relational.ipynb | 2 +- java/README.md | 6 +- java/pom.xml | 2 +- .../com/nvidia/sdm/ColumnExplanations.java | 26 +- .../nvidia/sdm/ColumnExplanationsTest.java | 10 +- .../nvidia/sdm/GradientExplanationsTest.java | 1 - .../structured-data-models-client/LICENSE | 3 + packages/structured-data-models-client/NOTICE | 16 +- .../structured-data-models-client/README.md | 4 +- .../src/sdm_client/_rust/direct.py | 6 +- .../src/sdm_client/_rust/errors.py | 42 +- .../src/sdm_client/_rust/frame_schema.py | 9 + .../src/sdm_client/_rust/frame_sessions.py | 4 +- .../src/sdm_client/_rust/predict.py | 18 +- .../src/sdm_client/_rust/result_merge.py | 7 + .../src/sdm_client/_rust/run.py | 22 +- .../src/sdm_client/_rust/sessions.py | 2 +- .../assets/THIRD-PARTY-NOTICES-MERMAID.txt | 4 +- .../src/sdm_client/client.py | 4 +- .../src/sdm_client/explanations.py | 4 +- .../src/sdm_client/models.py | 3 +- .../src/sdm_client/query.py | 4 +- .../src/sdm_client/request.py | 105 +++- .../src/sdm_client/wire/keys.py | 4 +- .../tests/test_column_analysis_core.py | 155 ++++- .../tests/test_gradient_explanations.py | 26 +- .../tests/test_live_relational.py | 3 +- .../tests/test_materialize_payloads.py | 56 +- .../tests/test_pql_for_tabular.py | 20 + .../tests/test_result_merge_core.py | 32 +- .../tests/test_semantic_model_source.py | 77 +++ .../tests/test_snowflake_semantic_core.py | 97 ++++ .../tests/test_tabular_direct_core.py | 52 ++ .../tests/test_tabular_explanations.py | 44 +- .../tests/test_the_predict_route.py | 17 + .../test_the_time_column_is_inferred_alike.py | 28 + .../tests/test_transport_socket.py | 22 + qa/README.md | 7 +- qa/run_rfm_nim_live_tests.sh | 145 ----- rust/crates/sdm-contract/schema/contract.lock | 2 +- ...-contract-71.json => sdm-contract-72.json} | 37 +- rust/crates/sdm-contract/src/columns.rs | 1 - rust/crates/sdm-contract/src/lib.rs | 2 +- rust/crates/sdm-contract/src/result.rs | 4 + rust/crates/sdm-core/src/columns.rs | 21 +- rust/crates/sdm-core/src/materialize.rs | 32 +- rust/crates/sdm-core/src/predict.rs | 2 +- rust/crates/sdm-core/src/profiles.rs | 15 +- rust/crates/sdm-core/src/view.rs | 16 +- rust/crates/sdm-core/tests/predict.rs | 10 +- rust/crates/sdm-data/src/snowflake.rs | 2 +- rust/crates/sdm-request/src/columns.rs | 531 +++++++++++------- rust/crates/sdm-request/src/payload.rs | 96 +--- rust/crates/sdm-request/src/predict.rs | 7 +- rust/crates/sdm-request/src/result.rs | 41 +- rust/crates/sdm-request/tests/payload_lock.rs | 54 +- skills/README.md | 5 +- skills/relational/SKILL.md | 6 +- skills/relational/references/warehouses.md | 5 +- tests/compatibility/manifest.json | 29 +- tests/test_java_binding_ci.py | 53 -- tests/test_release_contract.py | 24 + tests/test_wheel_provenance.py | 50 ++ 76 files changed, 1576 insertions(+), 809 deletions(-) create mode 100644 packages/structured-data-models-client/tests/test_semantic_model_source.py delete mode 100755 qa/run_rfm_nim_live_tests.sh rename rust/crates/sdm-contract/schema/{sdm-contract-71.json => sdm-contract-72.json} (99%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 218a38057..e185d236b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,7 +40,12 @@ name: publish on: push: - tags: ['v[0-9]+.[0-9]+.[0-9]+'] + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + # PEP 440 pre-releases: v2.0.0a1, v2.0.0b1, v2.0.0rc1. + - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+' workflow_dispatch: # Least privilege at the top: a job that needs more says so itself. @@ -201,31 +206,6 @@ jobs: cp native-*/*.whl dist/ cp native-*/*.whl.json dist/ - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - github-token: ${{ github.token }} - run-id: ${{ steps.candidate-ci.outputs.run_id }} - name: java-jar - path: java-release - - name: Verify and assemble the tested Java artifacts - run: | - python - <<'PY' - import hashlib, json, os, pathlib, shutil, xml.etree.ElementTree as ET - root = pathlib.Path('java-release') - record = json.loads((root / 'java-artifacts.json').read_text()) - assert record['commit'] == os.environ['GITHUB_SHA'] - version = os.environ['GITHUB_REF_NAME'].removeprefix('v') - stem = f'structured-data-models-client-{version}' - assert set(record['files']) == {stem + '.jar', stem + '.pom'} - pom = ET.parse(root / (stem + '.pom')).getroot() - assert pom.findtext('{http://maven.apache.org/POM/4.0.0}version') == version - for name, digest in record['files'].items(): - path = root / name - assert hashlib.sha256(path.read_bytes()).hexdigest() == digest - shutil.copy2(path, pathlib.Path('dist') / name) - shutil.copy2(root / 'java-artifacts.json', 'dist/java-artifacts.json') - PY - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: built-distributions @@ -257,8 +237,8 @@ jobs: ref = os.environ['GITHUB_REF_NAME'] # `lstrip` would also have accepted `vv1.0.0`, and a dispatch can be # aimed at a tag called `1.0.0`. - if not re.fullmatch(r'v\d+\.\d+\.\d+', ref): - sys.exit(f'{ref} is not a release tag of the form v1.2.3') + if not re.fullmatch(r'v\d+\.\d+\.\d+((a|b|rc)\d+)?', ref): + sys.exit(f'{ref} is not a release tag of the form v1.2.3 or v1.2.3rc1') tag = ref.removeprefix('v') wheels = sorted(p.name for p in pathlib.Path('dist').glob('*.whl')) sdists = sorted(p.name for p in pathlib.Path('dist').glob('*.tar.gz')) @@ -371,11 +351,16 @@ jobs: # the Kitmaker publish has worked. This only withholds the # promotion: with no earlier release to stay Latest, GitHub still # reports the newest non-draft release as the latest one. + prerelease=() + if [[ "$GITHUB_REF_NAME" =~ (a|b|rc)[0-9]+$ ]]; then + prerelease=(--prerelease) + fi gh release create "$GITHUB_REF_NAME" \ --repo "$GITHUB_REPOSITORY" \ --title "$GITHUB_REF_NAME" \ --verify-tag \ --latest=false \ + ${prerelease[@]+"${prerelease[@]}"} \ --notes-file dist/notes.md fi # The SBOM is a release asset and was never meant for the index; the @@ -419,5 +404,9 @@ jobs: echo "NVIDIA's network. RELEASING.md says who runs it and what it" echo "has to do." echo - echo "Mark this release Latest once the index has all three distributions." + if [[ "$GITHUB_REF_NAME" =~ (a|b|rc)[0-9]+$ ]]; then + echo "This is a pre-release: leave it unmarked." + else + echo "Mark this release Latest once the index has all three distributions." + fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df2c52977..44b63eea2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -exclude: ^(packages/structured-data-models-pql/src/predictive_query/|packages/structured-data-models-api/src/sdm_api/generated/tfm_api\.py$|packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/assets/mermaid\.min\.js$) +exclude: ^(packages/structured-data-models-api/src/sdm_api/generated/tfm_api\.py$|packages/structured-data-models-client/src/sdm_client/assets/mermaid\.min\.js$) repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/CHANGELOG.md b/CHANGELOG.md index f651b43ea..b51951c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,12 @@ Kept in the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) shape, and versioned by [semantic versioning](https://semver.org) as [`docs/reference/stability.md`](docs/reference/stability.md) defines it for -these packages. The three Python distributions and Java library share one version. +these packages. The three Python distributions share one version. ## Unreleased -Release candidate versions are `2.0.0rc1` for Python and Java and `2.0.0-rc.1` for Rust. +The release candidate version is `2.0.0rc1`, spelled `2.0.0-rc.1` in Rust's `Cargo.toml`. +The Java library is built and tested in CI but is not released. The public API snapshot records this version update; publication remains pending. ### Added @@ -39,8 +40,8 @@ The public API snapshot records this version update; publication remains pending yourself is used as it is. - The client now contains the Rust core instead of depending on separate relational-engine and Python PQL packages. - Python wheels and the Java JAR target Linux x86-64/ARM64, macOS - x86-64/ARM64 and Windows x86-64. + Python wheels target Linux x86-64/ARM64, macOS x86-64/ARM64 and Windows + x86-64; the Java library, built from source, supports the same platforms. Native Windows ARM64 is deferred pending upstream Arrow dependencies. - Both models support gradient explanation collections when the model runtime supports them. @@ -48,8 +49,58 @@ The public API snapshot records this version update; publication remains pending `EXPLAIN PREDICT` requests explanations, including each forecast step; client-assembled rankings refuse them. Python also accepts manually supplied `PredictionResult(explanations=...)` mappings. +- `Schema` gains `infer_metadata`, `infer_links`, `link_report`, + `metadata_report` and `print_metadata`/`print_links`, `link_origins` + (as `RelationshipOrigin` pairs), `with_relationship`, `without_relationship`, + `without_table`, `with_source` and `with_sql_column`, and `to_mermaid`, + `to_html` and `visualize` for an offline diagram. +- `RelationalTask` and `Schema.materialize_examples` ask a relational + question from labelled examples without PQL; `build_timeseries` builds a + forecast's input from a series. +- `PredictiveQuery.labels` computes a query's historical labels without + calling a model. +- `InferenceConfig` holds relational inference settings. +- `ExplanationSummarizer` writes a short summary of an explanation with an + LLM endpoint. +- `Table.end_time_column` and `SourceTable.end_time_column` name a row's end + time. +- `PredictionResult.merge(parts, columns=...)` takes the column analysis to + keep on the merged result. + ### Changed +- `structured-data-models-client` is now a native wheel, `cp310-abi3`, built + for Linux x86-64 and aarch64 (manylinux), macOS arm64 and x86-64, and Windows + AMD64. 1.0.0 was pure Python. It no longer installs on PyPy, musl Linux, + 32-bit platforms or Windows ARM64. +- A failure in the data source a schema reads, such as a warehouse query that + errors, raises `StructuredDataError` with code `PROVIDER_ERROR`, and + `details['driver_class']` names the driver's exception class. 1.0.0 raised + these from the relational engine as `INTERNAL_ERROR`. A semantic view or + metric view whose expression names a column its base table lacks is one of + them: the warehouse refuses the read, where 1.0.0 refused the view as + `INVALID_REQUEST` before reading it. + +- A related table sampled from frames sends its own primary key, where + 1.0.0 sent a generated `__node_id`. +- Each `materialize()` on a schema from `Schema.from_source` reads its + tables' catalog and preview again, and a semantic model's definition, so a + column dropped or retyped since is found. 1.0.0 read them once and kept + them on the schema, so a warehouse sees more queries per call than it did. +- An HTTP 408 from `POST /v1/predictions` is retried, as 429, 502, 503 and + 504 are. +- A relational request reached over three or more hops fills a foreign key + from every path that reaches the row, where 1.0.0 took the first path and + left the rest null. The model sees more links, so predictions can differ. +- Related rows with the same timestamp keep their source order. 1.0.0 used + NumPy's unstable sort, whose order depends on the platform, so sampled + rows and predictions can differ where timestamps tie. +- `SUM` and `AVG` labels, and the lagged target columns built from them, are + summed in float32 with Kahan compensation, as + pandas' Cython grouped sum computes them. 1.0.0 left this to the installed + pandas, whose answer depends on its version: pandas 3.1 sums sorted groups + with NumPy and averages in float64. A label can differ in its last digit, + and a threshold on it can flip. - Sampled Tabular context uses stable Rust index ordering, including duplicate labels and mixed numeric labels. Equal labels retain sampled order; mixed numbers compare by their represented values. Context order and predictions can differ from platform-specific NumPy ordering; sampled membership and warehouse sampling are unchanged. @@ -83,6 +134,33 @@ The public API snapshot records this version update; publication remains pending the float range on every pandas release. pandas 3.1 had started to build a frame around one, where earlier releases refused it. +### Removed + +- `structured-data-models-pql` (`predictive_query`) and + `structured-data-models-relational-engine` (`sdm_relational_engine`) are no + longer distributed, and importing either fails. Write predictive queries + through `sdm_client.PredictiveQuery`. +- The `[relational]` extra installs nothing extra. It is kept so existing + install commands still work. +- Features that lived only in the relational engine are gone: Snowflake SPCS + serving, the engine's telemetry and progress logging, and its extras + `snowflake-serving`, `explain` and `relbench`. RelBench datasets load with + `Schema.from_relbench`. The LLM summary of an explanation is now + `ExplanationSummarizer`, a different API: `summarize(view, query, prediction)` + takes an `ExplanationView` and raises on failure, where 1.0.0's + `generate_summary` returned fallback text. + +### Upgrading from 1.0.0 + +Uninstall the two retired packages before upgrading: + +```bash +pip uninstall structured-data-models-relational-engine structured-data-models-pql +``` + +The 1.0.0 engine pins `structured-data-models-api==1.0.0`, which conflicts +with 2.0. + ## 1.0.0 - 2026-10-15 First public release. The five distributions are versioned and published diff --git a/NOTICE b/NOTICE index b527599b1..3a40851ac 100644 --- a/NOTICE +++ b/NOTICE @@ -17,7 +17,7 @@ the MIT license, and its bundled dependencies. The upstream license, source and full dependency attribution are retained in packages/structured-data-models-client/src/sdm_client/assets/MERMAID_LICENSE, VENDORED.md and THIRD-PARTY-NOTICES-MERMAID.txt in that directory. -Both Python and Java distributions include these resources. +The Python client wheel includes these resources, as does the Java library built from source. -------------------------------------------------------------------------------- Test data diff --git a/README.md b/README.md index 8ed461a00..b05be2de3 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ get an answer without training a model for each dataset.** ```bash -pip install 'structured-data-models-client[relational]' +pip install structured-data-models-client ``` ## Overview diff --git a/RELEASING.md b/RELEASING.md index 6e8807f41..a18801ff4 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -47,36 +47,35 @@ SDM_TABULAR_URL=... SDM_RELATIONAL_URL=... \ python -m pytest packages/structured-data-models-client/tests -m live_nim -q ``` -## Java artifacts +## Java -The same successful CI run supplies the Java JAR and POM. -The release workflow checks their source commit, version and SHA-256 hashes before attaching them to the GitHub Release. -The JAR must contain Linux and macOS x86-64/ARM64 libraries plus Windows x86-64. -Windows ARM64 is deferred for both languages because required Arrow binaries are not published upstream. +The Java library is built and tested in CI, but it is not released yet. +The release workflow attaches no JAR or POM, and nothing is published to a Maven repository. +Keep `java/pom.xml` at the same version as the rest, so the library is ready when it is released. -Attaching a JAR is not Maven repository publication. -The Java repository destination and release credentials remain to be configured before the first Java release; install the published coordinates in a clean consumer before marking Java publication complete. -Keep the JAR, POM, Rust workspace and three Python distribution versions aligned. +## Pre-releases -## Current release-candidate preparation - -Prepare `2.0.0rc1` for Python and Java; Rust uses the equivalent SemVer spelling `2.0.0-rc.1`. -Publication and tagging are on hold until the owner explicitly authorizes them. -Java initially targets GitHub Release assets; Maven Central is a follow-up. -The existing publication workflow accepts final-release tags only; RC publication would need a separately reviewed prerelease path. +A tag such as `v2.0.0rc1`, `v2.0.0b1` or `v2.0.0a1` runs the same workflow with the same checks. +The version is the PEP 440 spelling, as in `_version.py`; Rust's `Cargo.toml` writes the same version in SemVer (`2.0.0-rc.1`). +The GitHub Release is marked as a pre-release. +`CHANGELOG.md` needs a `## 2.0.0rc1` heading, since the release notes are taken from it. +Tagging `v2.0.0rc1` is on hold until the owner approves it. ## Bumping the version -One literal per distribution, in `_version.py`. All three must match, and -`tests/test_release_contract.py` fails if they do not. +The version is written in several places, and all of them move together: -```bash -for f in packages/*/src/*/_version.py; do - sed -i '' "s/__version__ = '.*'/__version__ = '1.2.0'/" "$f" -done -``` - -Internal pins (`structured-data-models-api==1.0.0` in `structured-data-models-client`'s dependencies) move with them. +| Where | Spelling for `2.0.0rc1` | +| --- | --- | +| `packages/*/src/*/_version.py` (one per distribution) | `2.0.0rc1` | +| `rust/Cargo.toml`, `[workspace.package] version`, which maturin stamps on the client wheel | `2.0.0-rc.1` | +| `rust/Cargo.lock`, the workspace crates' entries (`cargo update --workspace` rewrites them) | `2.0.0-rc.1` | +| `java/pom.xml`, `` | `2.0.0rc1` | +| `packages/structured-data-models-client/pyproject.toml`, the pins on `structured-data-models-api` and the connectors extras | `2.0.0rc1` | +| `tests/golden/public_api.json`, rewritten by `scripts/snapshot_public_api.py --write` | `2.0.0rc1` | + +`tests/test_release_contract.py` fails if `_version.py`, the Rust workspace, `java/pom.xml` or a first-party pin disagree. +Search for the old version afterwards to catch anything else, such as the coordinates in `java/README.md`. Update `CHANGELOG.md`: the entries for the release go under its own heading with the date you are tagging on, not the date you wrote them, so the tag diff --git a/contract/tests/test_exact_numeric_keys.py b/contract/tests/test_exact_numeric_keys.py index 12a6cee2d..5d1c0fa11 100644 --- a/contract/tests/test_exact_numeric_keys.py +++ b/contract/tests/test_exact_numeric_keys.py @@ -6,7 +6,6 @@ from __future__ import annotations -import copy import functools import importlib.util import json @@ -92,25 +91,6 @@ def _projection(answer: dict[str, Any]) -> dict[str, Any]: return {k: answer[k] for k in ('outcome', 'schema', 'payloads')} -def _observe_with_generated_key( - vector: dict[str, Any], dsn: str -) -> dict[str, Any]: - """Assert the added wire metadata, then retain the historical body digest.""" - regressions = R._regressions() - original = regressions.payloads_projection - - def projection(payloads: Any) -> dict[str, Any] | None: - bodies = copy.deepcopy([dict(body) for body in payloads]) - assert bodies - for body in bodies: - assert body.pop('client_generated_keys') == {'orders': '__node_id'} - return original(bodies) - - with pytest.MonkeyPatch.context() as patch: - patch.setattr(regressions, 'payloads_projection', projection) - return R.observe(vector, dsn) - - def _differs(got: dict[str, Any], want: dict[str, Any]) -> list[str]: return sorted(k for k in want if got.get(k) != want[k]) @@ -127,9 +107,7 @@ def test_python_core_answers_a_catalog_key_as_the_reference( def check() -> None: want = BY_ID[vector_id]['reference'] - got = _observe_with_generated_key( - BY_ID[vector_id], os.environ['SDM_POSTGRES_TEST_DSN'] - ) + got = R.observe(BY_ID[vector_id], os.environ['SDM_POSTGRES_TEST_DSN']) found = _differs(got, want) assert not found, ( f'{vector_id}: {found} differ: got {got["outcome"]}, the ' @@ -157,9 +135,7 @@ def check() -> None: control = R.CONTROL[vector_id] want = _projection(BY_ID[control]['reference']) got = _projection( - _observe_with_generated_key( - BY_ID[vector_id], os.environ['SDM_POSTGRES_TEST_DSN'] - ) + R.observe(BY_ID[vector_id], os.environ['SDM_POSTGRES_TEST_DSN']) ) found = _differs(got, want) assert not found, ( diff --git a/contract/tests/test_sqlite_source_edges.py b/contract/tests/test_sqlite_source_edges.py index 8fd3da9e5..7bc6eb8cc 100644 --- a/contract/tests/test_sqlite_source_edges.py +++ b/contract/tests/test_sqlite_source_edges.py @@ -10,7 +10,6 @@ from __future__ import annotations -import copy import importlib.util import json import sys @@ -82,20 +81,6 @@ def call(self: Any, *args: Any, **kwargs: Any) -> Any: def _reference(name: str) -> dict[str, Any]: found: dict[str, Any] = RECORDED['scenarios'][name]['reference'] - if name in {'integer-fractions', 'partial-unique', 'materialize-twice'}: - found = copy.deepcopy(found) - payloads = [ - *found['payloads'], - *( - p - for repeat in found.get('repeated', []) - for p in repeat['payloads'] - ), - ] - for payload in payloads: - # These scenarios declare no orders key; the current wire contract - # explicitly identifies the key generated by materialization. - payload['client_generated_keys'] = {'orders': '__node_id'} return found diff --git a/docs/about/overview.md b/docs/about/overview.md index 498baf4ac..8a47420e3 100644 --- a/docs/about/overview.md +++ b/docs/about/overview.md @@ -30,8 +30,11 @@ declared as data instead of discovered from a 422. | Model | For | Needs | |---|---|---| -| `kumo-tabular` | one table | nothing (pure Python, runs anywhere) | -| `kumo-relational` | several related tables, joined as a graph | `structured-data-models-client[relational]`, a native build | +| `kumo-tabular` | one table | `structured-data-models-client` | +| `kumo-relational` | several related tables, joined as a graph | `structured-data-models-client` | + +Both models run on the Rust core inside the client's native wheel, so no extra +is needed for either. ## When not to use it diff --git a/docs/get-started/installation.md b/docs/get-started/installation.md index 57456ced7..1f9fada53 100644 --- a/docs/get-started/installation.md +++ b/docs/get-started/installation.md @@ -7,7 +7,7 @@ | `pip install 'structured-data-models-client[snowflake]'` | Reads tables from a warehouse. Also `[sqlite]`, `[duckdb]`, `[postgres]`, `[databricks]`, `[lakebase]`, `[s3]`. | | `pip install 'structured-data-models-client[all]'` | Both models and every warehouse backend. | | `pip install 'structured-data-models-client[databricks-serving]'` | Reaches a model served on Databricks Model Serving, through `SDMClient.for_databricks_serving`. Not in `[all]`: it is a way to reach a model, not a backend. | -| `pip install 'structured-data-models-client[in-process]'` | Runs the models in this Python process with structured-data-models, through `SDMClient.in_process`. Brings PyTorch. The relational model also needs `[relational]`, which gathers its requests. **Python 3.11 or later**: on 3.10 the extra installs nothing, and `in_process()` says why. Not in `[all]`, for the same reason as `[databricks-serving]`. | +| `pip install 'structured-data-models-client[in-process]'` | Runs the models in this Python process with structured-data-models, through `SDMClient.in_process`. Brings PyTorch. **Python 3.11 or later**: on 3.10 the extra installs nothing, and `in_process()` says why. Not in `[all]`, for the same reason as `[databricks-serving]`. | **Install distribution names; import module names.** The three maintained distributions and their imports are listed below. diff --git a/docs/get-started/quickstart.md b/docs/get-started/quickstart.md index 102af90d4..2a47d596c 100644 --- a/docs/get-started/quickstart.md +++ b/docs/get-started/quickstart.md @@ -215,9 +215,8 @@ when the model is on a GPU, until it or the client is closed, so open it in a `with` block, as above, or call its `close()`. This needs `pip install 'structured-data-models-client[in-process]'`, which -brings PyTorch, on Python 3.11 or later. A relational request is still -gathered by the relational engine, so for `KumoRelational` install both: -`pip install 'structured-data-models-client[relational,in-process]'`. +brings PyTorch, on Python 3.11 or later. It covers both `KumoTabular` and +`KumoRelational`. ## Which rows get scored diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index aed79a436..3c8fcef80 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -206,7 +206,10 @@ When explanation output is requested, `result.explanations.columns` describes ho These are associations, not causal effects or model gradients. Use `.to_pandas()` in Python, or `result.explanations().columns().toArrow(allocator)` in Java. Java callers close the Arrow root before its allocator. -The table contains table, hop, column, semantic type, cohort, class, share, mean and normalized score. +The table contains table, hop, column, semantic type, cohort, class, share and mean, one line per cohort and class, columns in context order. +It describes the labelled rows the request sends; rows the leak check removed are not in it, and a request sent as several calls counts each row once. +A numeric cohort's edges print with as many decimals, from two to six, as keep adjacent edges apart. +A categorical column with more than ten cohorts merges the rarest into `OTHER(*)` and keeps missing values as `N/A`. An empty labelled context produces an empty table; unsupported tasks or unavailable statistics produce no column analysis and a warning, while prediction remains usable. Fitted models analyze the context after holdout and reuse it, including when EXPLAIN is requested only on a later prediction. Merging results with different analyses requires an explicit `columns=` override (the corresponding Java overload), so an unrelated request's analysis is never silently chosen. diff --git a/docs/reference/stability.md b/docs/reference/stability.md index 665dd89a9..f6337b874 100644 --- a/docs/reference/stability.md +++ b/docs/reference/stability.md @@ -47,6 +47,11 @@ What *is* promised beyond the names: the exception hierarchy under and the columns `PredictionResult.to_pandas()` produces for a given set of requested output fields. +`PROVIDER_ERROR` is the code of a failure in the data source a schema reads, +such as a warehouse query that errors; where the client saw the driver's +exception, `details['driver_class']` names its class. 1.0.0 reported these as +`INTERNAL_ERROR`. + ## Java and the shared Rust implementation Java's public surface is the public classes in `com.nvidia.sdm`, including public nested builders and their public/protected members. diff --git a/examples/local/relational.ipynb b/examples/local/relational.ipynb index b6a8fe7e0..bd3fa28c9 100644 --- a/examples/local/relational.ipynb +++ b/examples/local/relational.ipynb @@ -10,7 +10,7 @@ "Several related tables, joined as a graph, and a question asked in PQL about\n", "what an entity reaches in a window.\n", "\n", - "This needs `pip install 'structured-data-models-client[relational]'` and a\n", + "This needs `pip install structured-data-models-client` and a\n", "`kumo-relational` NIM. One table on its own is a separate notebook,\n", "[`tabular.ipynb`](tabular.ipynb)." ] diff --git a/java/README.md b/java/README.md index 8fd5c1a69..82c233bd7 100644 --- a/java/README.md +++ b/java/README.md @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 The Java library uses the same Rust engine as Python, loaded from the JAR through JNI; see the [engine boundary diagrams](../docs/development/engine-boundaries.md). It describes the tables of a DuckDB, SQLite, Postgres or Databricks database through the caller's own JDBC connection, gathers a predictive query's request from them, and predicts with a model endpoint, each on the core, as the Python client does. Databricks metric views are read through `source.semanticModel(name, trustExpressions)`, and `schema.conversionNotes()` explains their conversion in the core's words. -Publication requires separate release approval and the maintained release gates. +The library is built and tested in CI, but it is not released: there is no Maven publication and no release JAR. Build it from source as below. ## Read semantic predictions @@ -148,7 +148,7 @@ If a later model call fails, `exception.answered(allocator)` returns any complet |---|---| | Group | `com.nvidia` | | Artifact | `structured-data-models-client` | -| Version | the client's (`2.0.0rc1`, unpublished release candidate) | +| Version | the client's (`2.0.0rc1`); not released | | Java package | `com.nvidia.sdm` | | Bytecode | Java 17, tested on JDK 17 and 21 | | Natives in the JAR | Linux x86_64 and aarch64 (glibc 2.17 or later), macOS arm64 and x86_64, Windows x86_64 | @@ -279,7 +279,7 @@ Warehouse-backed schemas retain their existing Relational workflow. The opt-in `ParityRunnerTest` reads the shared [parity manifest](../tests/compatibility/README.md) and launches a JVM using the built client JAR and its recorded native artifact. It records raw reader output, materialized requests, model results and Rust operation witnesses; local mode cannot certify live model parity. -The test-only `parity-connect-pom.xml` resolves Databricks Connect for `sdm.parity.connect.classpath`, without adding it to the published client. +The test-only `parity-connect-pom.xml` resolves Databricks Connect for `sdm.parity.connect.classpath`, without adding it to the client's dependencies. The caller supplies private connection/endpoint configuration and a build provenance sidecar; missing configuration stays incomplete. ### Snowflake diff --git a/java/pom.xml b/java/pom.xml index e10da7f2f..002c9ac61 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -17,7 +17,7 @@ com.nvidia structured-data-models-client - + 2.0.0rc1 jar diff --git a/java/src/main/java/com/nvidia/sdm/ColumnExplanations.java b/java/src/main/java/com/nvidia/sdm/ColumnExplanations.java index de6623985..bee42f93b 100644 --- a/java/src/main/java/com/nvidia/sdm/ColumnExplanations.java +++ b/java/src/main/java/com/nvidia/sdm/ColumnExplanations.java @@ -23,7 +23,7 @@ /** * Descriptive context-column associations computed by Rust, not causal effects. - * Formatting preserves engine order and never recomputes cohorts or scores. + * Formatting preserves engine order and never recomputes cohorts. */ public final class ColumnExplanations { /** Informational engine cohort count; not a tuning parameter. */ @@ -43,7 +43,18 @@ public final class ColumnExplanations { /** One column/cohort/class row; classLabel is null for regression and otherwise an exact scalar. */ public record Row(String table, long hop, String column, String stype, String cohort, - Object classLabel, double share, double mean, double score) {} + Object classLabel, double share, double mean) { + /** Kept for source compatibility; the analysis no longer scores a column. */ + @Deprecated + public Row(String table, long hop, String column, String stype, String cohort, + Object classLabel, double share, double mean, double score) { + this(table, hop, column, stype, cohort, classLabel, share, mean); + } + + /** Kept for source compatibility; the analysis no longer scores a column, so this is NaN. */ + @Deprecated + public double score() { return Double.NaN; } + } private final List rows; private final String encodedAnalysis; @@ -81,7 +92,7 @@ private ColumnExplanations(Map analysis, List rawRows) { if (hop < 0 || hop > 0xffff_ffffL) throw new IllegalArgumentException("invalid hop"); values.add(new Row(requiredText(row, "table"), hop, requiredText(row, "column"), (String) row.get("stype"), requiredText(row, "cohort"), label, - finite(row, "share"), finite(row, "mean"), finite(row, "score"))); + finite(row, "share"), finite(row, "mean"))); } rows = List.copyOf(values); encodedAnalysis = new String(Json.write(analysis), StandardCharsets.UTF_8); @@ -117,11 +128,11 @@ private static double finite(Map row, String name) { String encoded() { return encodedAnalysis; } - /** Detached immutable rows in engine score/cohort/class order. */ + /** Detached immutable rows in context column, cohort and class order. */ public List rows() { return rows; } /** - * Render the same nine columns as Python's dataframe view. + * Render the same eight columns as Python's dataframe view. * The caller closes the returned root before its allocator; this method never closes the allocator. * Mixed, decimal or wide class labels use lossless JSON text with metadata sdm.class_encoding=json. */ @@ -136,9 +147,8 @@ public VectorSchemaRoot toArrow(BufferAllocator allocator) { FieldVector classes = classVector(labels, allocator); Float8Vector share = new Float8Vector("share", allocator); Float8Vector mean = new Float8Vector("mean", allocator); - Float8Vector score = new Float8Vector("score", allocator); boolean json = encodedLabels(labels); - List vectors = List.of(table, hop, column, stype, cohort, classes, share, mean, score); + List vectors = List.of(table, hop, column, stype, cohort, classes, share, mean); VectorSchemaRoot root = new VectorSchemaRoot(new Schema( vectors.stream().map(FieldVector::getField).toList(), json ? Map.of("sdm.class_encoding", "json") : Map.of()), vectors, 0); @@ -149,7 +159,7 @@ public VectorSchemaRoot toArrow(BufferAllocator allocator) { putText(table, i, row.table()); hop.setSafe(i, row.hop()); putText(column, i, row.column()); putText(stype, i, row.stype()); putText(cohort, i, row.cohort()); putClass(classes, i, row.classLabel(), json); - share.setSafe(i, row.share()); mean.setSafe(i, row.mean()); score.setSafe(i, row.score()); + share.setSafe(i, row.share()); mean.setSafe(i, row.mean()); } root.setRowCount(rows.size()); return root; diff --git a/java/src/test/java/com/nvidia/sdm/ColumnExplanationsTest.java b/java/src/test/java/com/nvidia/sdm/ColumnExplanationsTest.java index bc88e31a6..7e04d3d9f 100644 --- a/java/src/test/java/com/nvidia/sdm/ColumnExplanationsTest.java +++ b/java/src/test/java/com/nvidia/sdm/ColumnExplanationsTest.java @@ -38,7 +38,7 @@ private static Map row(Object label) throws Exception { row.put("table", "instance_table"); row.put("hop", 0L); row.put("column", "x"); row.put("stype", "categorical"); row.put("cohort", "a"); row.put("class_json", new String(Json.write(label), StandardCharsets.UTF_8)); - row.put("share", 0.5); row.put("mean", 1.0); row.put("score", 1.0); + row.put("share", 0.5); row.put("mean", 1.0); return row; } private static ColumnExplanations columns(Object... labels) throws Exception { @@ -59,7 +59,7 @@ private static Map payload() { @Test void publicConstructorUsesEngineAndMatchesIndependentMeans() { ColumnExplanations columns = new ColumnExplanations(payload()); assertEquals(2, columns.rows().size()); - assertEquals(new ColumnExplanations.Row("instance_table", 0, "x", "categorical", "a", null, 0.5, 1, 1), columns.rows().get(0)); + assertEquals(new ColumnExplanations.Row("instance_table", 0, "x", "categorical", "a", null, 0.5, 1), columns.rows().get(0)); assertEquals(9.0, columns.rows().get(1).mean()); assertEquals("b", columns.rows().get(1).cohort()); assertThrows(UnsupportedOperationException.class, () -> columns.rows().clear()); @@ -83,7 +83,7 @@ private static Map payload() { assertEquals(1.0, columns.rows().get(0).mean()); try (RootAllocator allocator = new RootAllocator()) { try (VectorSchemaRoot view = columns.toArrow(allocator)) { - assertEquals(List.of("table", "hop", "column", "stype", "cohort", "class", "share", "mean", "score"), + assertEquals(List.of("table", "hop", "column", "stype", "cohort", "class", "share", "mean"), view.getSchema().getFields().stream().map(org.apache.arrow.vector.types.pojo.Field::getName).toList()); assertEquals(1, view.getRowCount()); } @@ -135,7 +135,7 @@ private static Map payload() { assertNotNull(empty); assertTrue(empty.rows().isEmpty()); try (RootAllocator allocator = new RootAllocator(); VectorSchemaRoot view = empty.toArrow(allocator); VectorSchemaRoot regression = columns((Object) null).toArrow(allocator)) { - assertEquals(0, view.getRowCount()); assertEquals(9, view.getFieldVectors().size()); + assertEquals(0, view.getRowCount()); assertEquals(8, view.getFieldVectors().size()); assertNull(regression.getVector("class").getObject(0)); } } @@ -152,7 +152,7 @@ private static Map payload() { assertEquals("INTERNAL_ERROR", assertThrows(StructuredDataException.class, () -> ColumnExplanations.fromCore(analysis)).code()); } - for (String field : List.of("table", "column", "cohort", "mean", "share", "score", "hop")) { + for (String field : List.of("table", "column", "cohort", "mean", "share", "hop")) { Map row = row("yes"); row.remove(field); assertEquals("INTERNAL_ERROR", assertThrows(StructuredDataException.class, () -> ColumnExplanations.fromCore(Map.of("status", "ready", "rows", List.of(row)))).code()); diff --git a/java/src/test/java/com/nvidia/sdm/GradientExplanationsTest.java b/java/src/test/java/com/nvidia/sdm/GradientExplanationsTest.java index 34c71429a..bcaf0009a 100644 --- a/java/src/test/java/com/nvidia/sdm/GradientExplanationsTest.java +++ b/java/src/test/java/com/nvidia/sdm/GradientExplanationsTest.java @@ -64,7 +64,6 @@ class GradientExplanationsTest { for(ColumnExplanations.Row row:columns.rows()) { assertTrue(Double.isFinite(row.mean())); assertTrue(row.share()>=0 && row.share()<=1); - assertTrue(row.score()>=0 && row.score()<=1); if(target.equals("value")) { assertNull(row.classLabel()); assertTrue(row.mean()>=0 && row.mean()<=23.75); diff --git a/packages/structured-data-models-client/LICENSE b/packages/structured-data-models-client/LICENSE index 503ad4375..3689c2e41 100644 --- a/packages/structured-data-models-client/LICENSE +++ b/packages/structured-data-models-client/LICENSE @@ -202,3 +202,6 @@ Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. See the License for the specific language governing permissions and limitations under the License. +Third-party components included in this distribution are attributed in +NOTICE, THIRD-PARTY-NOTICES-RUST.txt and +sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt. diff --git a/packages/structured-data-models-client/NOTICE b/packages/structured-data-models-client/NOTICE index 25b2a541f..33567c015 100644 --- a/packages/structured-data-models-client/NOTICE +++ b/packages/structured-data-models-client/NOTICE @@ -3,8 +3,16 @@ Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. Licensed under the Apache License, Version 2.0. See LICENSE. -This distribution is first-party NVIDIA source: the client, the schema and -the model handles. It vendors no third-party source. +This distribution includes third-party code: -Everything it depends on is resolved by the package manager at install time and -licensed by its own authors. +- The native module, sdm_client._native, statically links Rust crates. Their + licenses and notices are in THIRD-PARTY-NOTICES-RUST.txt, beside this file + in the wheel's .dist-info/licenses directory. +- The offline schema visualization bundles Mermaid 11.15.0 and its + dependencies, MIT licensed, as sdm_client/assets/mermaid.min.js. Its license + is in sdm_client/assets/MERMAID_LICENSE, its source in + sdm_client/assets/VENDORED.md, and the full attribution of what it bundles in + sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt. + +The Python packages it depends on are resolved by the package manager at +install time and licensed by their own authors. diff --git a/packages/structured-data-models-client/README.md b/packages/structured-data-models-client/README.md index 5e9c5b09c..ca758e8bc 100644 --- a/packages/structured-data-models-client/README.md +++ b/packages/structured-data-models-client/README.md @@ -32,8 +32,8 @@ with SDMClient(url='http://localhost:8000') as client: ``` The model is an argument, not a method name. For several related tables, ask in -the Predictive Query Language and pass `KumoRelational()` instead; that needs -`pip install 'structured-data-models-client[relational]'`. +the Predictive Query Language and pass `KumoRelational()` instead; the same +install covers it. Documentation, the API and connector packages and the examples are in the repository: diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/direct.py b/packages/structured-data-models-client/src/sdm_client/_rust/direct.py index 9ad294687..0cc6eff9b 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/direct.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/direct.py @@ -669,7 +669,7 @@ def predict( if finished.result['op'] == 'decoded_direct_response': finished = _response(finished) answer = results.expect(finished.result, results.Predicted) - return prediction._result( + result = prediction._result( prediction._table(finished.take(answer.result)), model=answer.model, request_id=answer.request_id, @@ -677,6 +677,10 @@ def predict( outputs=answer.outputs, by_step=False, ) + from sdm_client.client import _names_its_rows + + result._names_its_rows = _names_its_rows(request) + return result def _builtin(value: Any) -> dict[str, Any]: diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py index 9da5854b0..1ca231ec9 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/errors.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/errors.py @@ -764,20 +764,10 @@ def warn_from_core( + words['between'].join(said(w) for w in together) + words['closing'] ) - # Native direct advice is presented without a diagnostic code - # in the public constructor. Keep this local to its direct sentences; - # warehouse and other core warnings retain their existing presentation. - native_direct = first['sentence'] in { - 'direct_native_missing_key', - 'direct_dates_as_text', - 'direct_outputs_absent', - 'direct_key_repeated', - 'direct_key_composite', - 'direct_key_no_composite', - 'direct_categorical_population', - } warnings.warn( - text if native_direct else f'[{first["code"]}] {text}', + text + if first['sentence'] in _NATIVE_DIRECT + else f'[{first["code"]}] {text}', category=WARNINGS.get(first['code'], SDMWarning), stacklevel=caller_stacklevel(), ) @@ -793,13 +783,37 @@ def _opening(template: str, fields: dict[str, Any]) -> str: return DEFECT.format(sentence=template) +#: Native direct advice is presented without a diagnostic code in the public +#: constructor, as the old client wrote it. Kept to its direct sentences; +#: warehouse and other core warnings retain their existing presentation. +_NATIVE_DIRECT = frozenset( + { + 'direct_native_missing_key', + 'direct_dates_as_text', + 'direct_outputs_absent', + 'direct_key_repeated', + 'direct_key_composite', + 'direct_key_no_composite', + 'direct_categorical_population', + 'direct_constant_class', + 'direct_constant_regression', + 'direct_native_constant_class', + 'direct_native_constant_regression', + 'direct_overlap', + } +) + + def warn_held(held: list[dict[str, Any]]) -> None: r"""Shows warnings:func:`warn_from_core` held back, in order, each as it would have been shown (none of them of a group).""" for warning in held: assert table().sentences[warning['sentence']]['group'] is None + text = said(warning) warnings.warn( - f'[{warning["code"]}] {said(warning)}', + text + if warning['sentence'] in _NATIVE_DIRECT + else f'[{warning["code"]}] {text}', category=WARNINGS.get(warning['code'], SDMWarning), stacklevel=caller_stacklevel(), ) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/frame_schema.py b/packages/structured-data-models-client/src/sdm_client/_rust/frame_schema.py index 0d4ec9610..3fff56f3c 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/frame_schema.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/frame_schema.py @@ -124,8 +124,17 @@ def assemble( rebound[name] = single[name] # These are frame observations, not warehouse catalog facts. # Catalog provenance would incorrectly route edits through source-only checks. + # A time column, the one inference took and the one the schema holds, + # is a meaning assigned, as on the source route. + inferred = ( + (description.get('inference') or {}).get('times', {}).get(name) + or {} + ).get('column') + timed = {inferred, definition['time_column']} - {None} for column in rebound[name].columns.values(): object.__setattr__(column, '_source', None) + if column.name in timed: + object.__setattr__(column, 'stype_declared', True) object.__setattr__(rebound[name], '_core_origin', 'frames') object.__setattr__(rebound[name], '_schema', result) for name, value in ( diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/frame_sessions.py b/packages/structured-data-models-client/src/sdm_client/_rust/frame_sessions.py index 60e9b7236..dc1924145 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/frame_sessions.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/frame_sessions.py @@ -29,7 +29,7 @@ def _run( attachments: dict[int, Any] = {1: codec.attachment(value)} payloads = [] if request is not None: - for payload in request.payloads: + for payload in request._envelopes: number = len(attachments) + 1 # Project the protocol's transmitted half before JSON conversion. # The unsent graph can be large and must not be inspected at score. @@ -184,7 +184,7 @@ def predict(fitted: Any, request: Any) -> Any: 'action': 'predict', 'state': fitted._relational_session, 'question': sessions.question(request), - 'request_ids': [_request_id() for _ in request.payloads], + 'request_ids': [_request_id() for _ in request._envelopes], 'inference_asked': sorted( _keys_the_caller_set( request.requested_inference.get('inference_config', {}) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py index c4feccaa3..3fcc29f83 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/predict.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/predict.py @@ -190,7 +190,7 @@ def _state(request: Request) -> results.CoreRequest | None: return None calls = 1 + len(state.batches or ()) steps = request.parts if request.parts is not None else (request,) - if any(len(step.payloads or ()) != calls for step in steps): + if any(len(step._envelopes or ()) != calls for step in steps): return None return state @@ -217,7 +217,7 @@ def _from_state( steps = request.parts if request.parts is not None else (request,) [(payload, query), *later] = [ - ((step.payloads or ())[0], step.query) for step in steps + ((step._envelopes or ())[0], step.query) for step in steps ] named: dict[str, Any] = {'version': state.version, 'payload': 1, 'query': 2} attachments = {1: _payload(payload), 2: codec.attachment(query)} @@ -227,7 +227,7 @@ def _from_state( # call order. named['batches'] = [] for at, (more, rows) in enumerate( - zip((request.payloads or ())[1:], state.batches, strict=True), + zip((request._envelopes or ())[1:], state.batches, strict=True), start=3, ): attachments[at] = _payload(more) @@ -294,7 +294,7 @@ def _held(request: Request) -> tuple[dict[str, Any], dict[int, Any]]: rewritten = request._decomposition if request.parts is None or rewritten is None: - first, *calls = request.payloads or () + first, *calls = request._envelopes or () named: dict[str, Any] = {'payload': 1, 'query': 2} attachments = {1: _payload(first), 2: codec.attachment(request.query)} if calls: @@ -308,7 +308,7 @@ def _held(request: Request) -> tuple[dict[str, Any], dict[int, Any]]: ) return named, attachments [(payload, query), *later] = [ - ((part.payloads or ())[0], part.query) for part in request.parts + ((part._envelopes or ())[0], part.query) for part in request.parts ] named = {'payload': 1, 'query': 2} attachments = {1: _payload(payload), 2: codec.attachment(query)} @@ -605,7 +605,7 @@ def model_call(body: bytes) -> bytes: # the run (``run``'s ``_carried``). failure: StructuredDataError | None = None unexpected: Exception | None = None - transport_failed = False + transport_failed: StructuredDataError | None = None refused: NimRequestError | None = None try: # The core's call, its bytes as written (register PD-2), @@ -620,7 +620,7 @@ def model_call(body: bytes) -> bytes: refused = refusal except StructuredDataError as failed: if failed.code == 'TRANSPORT_ERROR': - transport_failed = True + transport_failed = failed else: from sdm_client.in_process.backend import InProcessBackend @@ -639,8 +639,8 @@ def model_call(body: bytes) -> bytes: # reference's client does. An interrupt is no # Exception, and cancels the run as before. unexpected = other - if transport_failed: - raise TransportFailed + if transport_failed is not None: + raise TransportFailed(transport_failed) raised: StructuredDataError | None = failure if refused is not None: # Built outside the handler, so nothing failing in the copy is diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/result_merge.py b/packages/structured-data-models-client/src/sdm_client/_rust/result_merge.py index 734446440..8f56c6ecd 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/result_merge.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/result_merge.py @@ -9,6 +9,7 @@ import dataclasses import json from datetime import date, datetime, timezone +from decimal import Decimal from typing import Any from zoneinfo import ZoneInfo @@ -37,6 +38,12 @@ def _candidate( return {'kind': 'json', 'value': json.dumps(value)} if owner is bytes: return {'kind': 'bytes', 'value': list(value)} + if owner is Decimal: + if value.is_nan(): + return {'kind': 'json', 'value': 'NaN'} + if value.is_infinite(): + return _candidate(float(value)) + return {'kind': 'decimal', 'value': format(value, 'f')} if isinstance(value, (np.bool_, np.integer, np.floating, np.str_)): if isinstance(value, np.floating) and value.dtype.itemsize > 8: return {'kind': 'unsupported', 'type_name': owner.__name__} diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index 570bcb51e..b320208bf 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -82,7 +82,12 @@ def __init__(self, data: Any, continuation: str) -> None: class TransportFailed(Exception): r"""The client's transport failed: the endpoint could not be reached, or its answer could not be read. Raised by a:data:`Model`, and told to - the core as a transport error. It carries no text.""" + the core as a transport error. ``account`` is the transport's own + error, whose message and details the core's error is given.""" + + def __init__(self, account: StructuredDataError | None = None) -> None: + super().__init__() + self.account = account class Refused(Exception): # noqa: N818 - says what happened to the call @@ -289,6 +294,7 @@ def _steps( driver_class: str | None = None refused: Refused | None = None cleanup_refusal = False + transport: StructuredDataError | None = None while True: error: StructuredDataError | None = None try: @@ -314,6 +320,9 @@ def _steps( _carried(error, refused.raised) _of_the_part(error, refused.raised) raise refused.raised + if transport is not None and error.code == 'TRANSPORT_ERROR': + error.message = f'{error.message}: {transport.message}' + error.details = {**transport.details, **error.details} raise error from None _warn(handle, quiet, given, kept) step = codec.read(control, sent) @@ -327,7 +336,10 @@ def _steps( if step['step'] == 'call_model' and ( model is not None or model_endpoints is not None ): - latest = _call(core, handle, step, model, model_endpoints) + try: + latest = _call(core, handle, step, model, model_endpoints) + except TransportFailed as failed: + transport, latest = failed.account, None # Cleanup must not replace the primary prediction refusal. deleting = ( isinstance(step['endpoint'], dict) @@ -404,6 +416,7 @@ def _call( body = handle.take_attachment(step['body']) attached: dict[int, Any] = {} refused: Refused | None = None + account: StructuredDataError | None = None transport_error: dict[str, Any] = { 'outcome': 'transport_error', 'kind': 'model_endpoint', @@ -419,7 +432,8 @@ def _call( raise NativeBoundaryError( 'this host has no sender for the model endpoint' ) - except TransportFailed: + except TransportFailed as failed: + account = failed.account outcome: dict[str, Any] = transport_error except EndpointFailed as failed: said: dict[str, Any] = {'status': failed.status} @@ -463,6 +477,8 @@ def _call( handle.reply(reply, attached) except core.CoreError as failed: errors.raise_from_core(failed.control, failed.attachments) + if account is not None: + raise TransportFailed(account) return refused diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py index e619de18b..2de515dce 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/sessions.py @@ -421,7 +421,7 @@ def _report_overlap(fitted: Any, request: Any) -> None: def _require_frames(request: Any) -> None: - if request.payloads is not None: + if request._envelopes is not None: from sdm_client.errors import UnsupportedRequestError raise UnsupportedRequestError( diff --git a/packages/structured-data-models-client/src/sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt b/packages/structured-data-models-client/src/sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt index 7c249705c..1ef3e27bd 100644 --- a/packages/structured-data-models-client/src/sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt +++ b/packages/structured-data-models-client/src/sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt @@ -211,8 +211,8 @@ The components below are vendored into this repository as source and ship with it. Everything else this project depends on is resolved by the package manager at install time and is not covered by this section. -sdm_relational_engine/rfm/assets/mermaid.min.js ------------------------------------------------ +sdm_client/assets/mermaid.min.js +-------------------------------- mermaid, used for offline graph visualization. diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 64e2d41e9..a1ea2e0ba 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -631,7 +631,7 @@ def __repr__(self) -> str: def _prediction_handler(request: Request) -> Any: """Choose the native adapter for direct arrays or materialized payloads.""" - if not request.payloads: + if not request._envelopes: from sdm_client._rust import direct return direct @@ -734,7 +734,7 @@ def _names_its_rows(request: Request) -> bool: key. Asking the same question there answered no about requests whose every answer names its entity. """ - if request.payloads: + if request._envelopes: # A gathered request is about entities by construction: the envelope # carries an entity reference for every row it scores, and the adapter # names each answer with the entity it asked about. diff --git a/packages/structured-data-models-client/src/sdm_client/explanations.py b/packages/structured-data-models-client/src/sdm_client/explanations.py index b71b86346..68c4c605c 100644 --- a/packages/structured-data-models-client/src/sdm_client/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/explanations.py @@ -216,7 +216,7 @@ def _from_core( return result def to_pandas(self) -> pd.DataFrame: - """One row per column, cohort and class, in the engine's score order.""" + """One row per column, cohort and class, columns in context order.""" rows = [ { **{ @@ -240,14 +240,12 @@ def to_pandas(self) -> pd.DataFrame: 'class', 'share', 'mean', - 'score', ], ).astype( { 'hop': 'int64', 'share': 'float64', 'mean': 'float64', - 'score': 'float64', } ) diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 5a8a36717..aa1209264 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -190,8 +190,7 @@ class KumoRelational(Model): Takes every task kind the contract carries except ``ranking``, and produces all seven output fields including ``explanation``. - Needs ``pip install structured-data-models-client[relational]``: building the graph and - sampling it is heavy client-side work that lives in a native package. + Building the graph and sampling it run in the client's Rust core. """ _spec = KUMO_RELATIONAL diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index d3e97a2c8..889b15685 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -514,7 +514,7 @@ def _kept_as_parts( version = whole._core.version gathered = [] for position, named, payload, query in [ - (parts.position, parts.name, whole.payloads[0], parts.query), + (parts.position, parts.name, whole._envelopes[0], parts.query), *( (part.position, part.name, part.payload, part.query) for part in parts.later @@ -572,7 +572,7 @@ def _kept_as_steps( version = whole._core.version gathered = [] for step, payload, query in [ - (parts.position, whole.payloads[0], parts.query), + (parts.position, whole._envelopes[0], parts.query), *( (part.position, part.payload, part.query) for part in parts.later diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index 4c742d963..091ca5ce4 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -34,6 +34,7 @@ import re import sys import warnings +import weakref from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any @@ -214,7 +215,7 @@ def warn_of_rows_scored_from_their_own_answers(request: Request) -> None: and warning about a context that never travels made a correct call fail under warnings-as-errors. """ - if request.payloads: + if request._envelopes: return schema, entity_table = request.schema, request.entity_table target = request.task.target.column_name @@ -538,7 +539,7 @@ def _half(self, half: str) -> Split: read from the envelopes again and the edit is not there. Assigning the half is what changes the request, and does so on both. """ - payloads = self.payloads + payloads = self._envelopes if payloads: from sdm_client._rust.splits import read @@ -560,7 +561,7 @@ def _refuse_if_gathered(self, half: str) -> None: Gathered is read the same way here as everywhere else on this object, so that the route a request is on is one answer rather than four. """ - if not self.payloads: + if not self._envelopes: return raise StructuredDataError( f'{half} cannot be replaced on a request gathered by a predictive ' @@ -612,6 +613,22 @@ def payloads(self) -> tuple[dict[str, Any], ...] | None: came from its frames while the adapters refused it for carrying envelopes. """ + envelopes = self._envelopes + if envelopes is None: + return None + shown = [] + for envelope in envelopes: + view = _VIEWS.get(id(envelope)) + if view is None or view._envelope is not envelope: + view = _VIEWS[id(envelope)] = _Envelope(envelope) + else: + view._sync() + shown.append(view) + return tuple(shown) + + @property + def _envelopes(self) -> tuple[dict[str, Any], ...] | None: + r"""The gathered envelopes with the client's own keys beside them.""" mine = getattr(self, '_payloads', None) or None parts = self.parts if parts is None: @@ -621,7 +638,7 @@ def payloads(self) -> tuple[dict[str, Any], ...] | None: # be sent. Reporting one part's would undercount by as many parts as # there are, which for a ranking is one per candidate. gathered = tuple( - envelope for part in parts for envelope in part.payloads or () + envelope for part in parts for envelope in part._envelopes or () ) return gathered or mine @@ -662,9 +679,10 @@ def predict_rows(self) -> int: return self._rows('predict') def _rows(self, half: str) -> int: - if self.payloads: + if self._envelopes: return sum( - _envelope_rows(payload, half) or 0 for payload in self.payloads + _envelope_rows(payload, half) or 0 + for payload in self._envelopes ) return len(getattr(self, half).instance_table) @@ -675,10 +693,10 @@ def _context_rows_per_call(self) -> int: several calls sends a context with each, so ``context_rows``, their sum, is not what any call carries. """ - if self.payloads: + if self._envelopes: return max( _envelope_rows(payload, 'context') or 0 - for payload in self.payloads + for payload in self._envelopes ) return len(self.context.instance_table) @@ -694,9 +712,10 @@ def nothing_to_score(self) -> bool: empty answer. """ - if self.payloads: + if self._envelopes: counts = [ - _envelope_rows(payload, 'predict') for payload in self.payloads + _envelope_rows(payload, 'predict') + for payload in self._envelopes ] return all(count is not None for count in counts) and not any( counts @@ -712,7 +731,7 @@ def __repr__(self) -> str: is the widest one call's, since that is what the cap is held to. """ shape = 'relational' if self.is_relational else 'single-table' - calls = len(self.payloads or ()) + calls = len(self._envelopes or ()) context = ( f'up to {self._context_rows_per_call()} context rows per call ' f'across {calls} calls' @@ -983,6 +1002,70 @@ def joined(frames: list[pd.DataFrame]) -> pd.DataFrame: ) +#: The view of each envelope while one is held, by the envelope's identity, +#: so every request sharing an envelope shows the one view of it. +_VIEWS: weakref.WeakValueDictionary[int, _Envelope] = ( + weakref.WeakValueDictionary() +) + + +class _Envelope(dict): # type: ignore[type-arg] + r"""An envelope as it is sent, without the keys the client keeps beside + it for itself; a change to it is a change to the envelope.""" + + def __init__(self, envelope: dict[str, Any]) -> None: + super().__init__() + self._envelope = envelope + self._sync() + + def _sync(self) -> None: + super().clear() + super().update( + (name, value) + for name, value in self._envelope.items() + if not name.startswith('client_') + ) + + def __setitem__(self, name: str, value: Any) -> None: + self._envelope[name] = value + super().__setitem__(name, value) + + def __delitem__(self, name: str) -> None: + del self._envelope[name] + super().__delitem__(name) + + def __ior__(self, other: Any) -> _Envelope: # type: ignore[override,misc] + self.update(other) + return self + + def update(self, *args: Any, **kwargs: Any) -> None: + for name, value in dict(*args, **kwargs).items(): + self[name] = value + + def setdefault(self, name: str, default: Any = None) -> Any: + if name not in self: + self[name] = default + return self[name] + + def pop(self, name: str, *default: Any) -> Any: + if name not in self: + return super().pop(name, *default) + value = self[name] + del self[name] + return value + + def popitem(self) -> tuple[str, Any]: + name = next(reversed(self)) + return name, self.pop(name) + + def clear(self) -> None: + for name in list(self): + del self[name] + + def __reduce__(self) -> Any: + return dict, (dict(self),) + + def _envelope_rows(payload: Mapping[str, Any], half: str) -> int | None: r"""How many instance rows one wire envelope carries, or `None`. diff --git a/packages/structured-data-models-client/src/sdm_client/wire/keys.py b/packages/structured-data-models-client/src/sdm_client/wire/keys.py index bbd4893d4..cd880440b 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/keys.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/keys.py @@ -263,10 +263,10 @@ def scored_rows( pinned schema. Settled here instead, the same entity has one name on each side and neither meets the other. """ - if request.payloads: + if request._envelopes: return [ named - for payload in request.payloads + for payload in request._envelopes for named in rows_in(payload, 'predict', with_anchor=with_anchor) ] frame = getattr(request.predict, 'instance_table', None) diff --git a/packages/structured-data-models-client/tests/test_column_analysis_core.py b/packages/structured-data-models-client/tests/test_column_analysis_core.py index b2e6c0a9a..2bb00de86 100644 --- a/packages/structured-data-models-client/tests/test_column_analysis_core.py +++ b/packages/structured-data-models-client/tests/test_column_analysis_core.py @@ -18,6 +18,7 @@ KumoRelational, PredictionResult, PredictiveQuery, + Request, Schema, SDMClient, ) @@ -47,7 +48,6 @@ def test_public_constructor_has_independent_cohort_oracle(): assert list(frame['cohort']) == ['a', 'b'] assert list(frame['mean']) == [1.0, 9.0] assert list(frame['share']) == [0.5, 0.5] - assert list(frame['score']) == [1.0, 1.0] frame.loc[0, 'mean'] = -1 assert result.to_pandas().loc[0, 'mean'] == 1 @@ -66,7 +66,6 @@ def test_empty_labels_and_exact_mixed_classes(): 'class', 'share', 'mean', - 'score', ] labels = [2**80 + 1, 0.5, '1'] frame = ColumnExplanations( @@ -212,13 +211,12 @@ def gathered(): ) -def test_materialize_pickle_predict_restores_pretrim_context_without_sending_delta(): +def test_materialize_pickle_predict_analyses_the_rows_it_sends(): with warnings.catch_warnings(): warnings.simplefilter('ignore') request = gathered() - assert request.payloads[0]['client_analysis_delta']['version'] == 1 restored = pickle.loads(pickle.dumps(request)) - assert restored.payloads == request.payloads + assert restored._envelopes == request._envelopes endpoint = Endpoint() client = SDMClient.__new__(SDMClient) client._start(endpoint) @@ -229,19 +227,36 @@ def test_materialize_pickle_predict_restores_pretrim_context_without_sending_del ) frame = result.explanations.columns.to_pandas() counts = frame[frame['column'] == 'COUNT(*)'] - assert list(counts['mean']) == [11.5] assert len(endpoint.sent) == 1 - assert len(endpoint.sent[0]['context']['instance_table']['rows']) == 22 + sent = endpoint.sent[0]['context']['instance_table'] + assert len(sent['rows']) == 22 + at = sent['columns'].index('TARGET') + labels = [row[at] for row in sent['rows']] + assert list(counts['mean']) == pytest.approx([sum(labels) / len(labels)]) -def test_delta_corruption_is_not_silently_an_omitted_analysis(): - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - request = gathered() - p = copy.deepcopy(request.payloads[0]) - p['client_analysis_delta']['instance_table']['positions'] = [99999] - with pytest.raises(StructuredDataError): - ColumnExplanations(p) +def test_public_payloads_hide_client_keys_and_write_through(): + import json + + envelope = { + 'model': 'kumo-relational', + 'metadata': {}, + 'client_generated_keys': {'orders': '__node_id'}, + } + request = Request.__new__(Request) + object.__setattr__(request, '_payloads', (envelope,)) + [payload] = request.payloads + assert payload == {'model': 'kumo-relational', 'metadata': {}} + json.dumps(payload) + payload['metadata'] = {'note': 'edited'} + assert envelope['metadata'] == {'note': 'edited'} + request.payloads[0]['metadata'] = {'note': 'again'} + payload['metadata']['seen'] = True + assert envelope['metadata'] == {'note': 'again', 'seen': True} + assert 'client_generated_keys' in envelope + shared = Request.__new__(Request) + object.__setattr__(shared, '_payloads', (envelope,)) + assert shared.payloads[0] is payload def test_unavailable_statistics_warn_without_losing_predictions(): @@ -268,6 +283,107 @@ def test_unavailable_statistics_warn_without_losing_predictions(): assert any('column analysis' in str(w.message).lower() for w in caught) +class _SessionEndpoint: + url = 'http://localhost' + serves_sessions = True + + def predict(self, payload): + rows = payload['predict']['instance_table']['rows'] + return { + 'id': 'r', + 'model': 'kumo-tabular', + 'metadata': {}, + 'predictions': [ + {'row_index': at, 'prediction': 0.5} for at in range(len(rows)) + ], + } + + def create_session(self, payload): + return {'session_id': 's1'} + + def session_predict(self, session_id, payload): + return self.predict(payload) + + def delete_session(self, session_id): + return None + + def close(self): + pass + + +@pytest.mark.parametrize( + ('outputs', 'rows', 'warned'), + [ + (None, 2, False), + (['prediction', 'explanation'], 2, True), + (['prediction', 'explanation'], 0, True), + ], + ids=['not explained', 'explained', 'explained, no rows'], +) +def test_a_failed_analysis_does_not_block_a_session(outputs, rows, warned): + from test_tabular_direct_core import client + + from sdm_client import KumoTabular, Request + + def request(rows): + return Request( + pd.DataFrame( + {'x': ['a', 'a', 'b', 'b'], 'y': [1e308, 1e308, -1e308, -1e308]} + ), + pd.DataFrame({'x': ['a', 'b'][:rows]}, index=[90, 91][:rows]), + target='y', + task='regression', + ) + + def analysis_warnings(caught): + return [w for w in caught if 'column analysis' in str(w.message)] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + fitted = client(_SessionEndpoint()).fit( + request(2), model=KumoTabular(), outputs=outputs + ) + assert bool(analysis_warnings(caught)) is warned + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + result = fitted.predict(request(rows)) + assert bool(analysis_warnings(caught)) is warned + assert fitted.session_id == 's1' + assert len(result) == rows + assert result.explanations.columns is None + + +@pytest.mark.parametrize( + ('values', 'labels'), + [ + ( + [float(at) for at in range(11)], + ['[0 - 1]', *(f'({at} - {at + 1}]' for at in range(1, 10))], + ), + ([0.0, 1660 / 7], ['[0 - 23.71]', '(23.71 - 237.14]']), + ( + [10000.0 + at for at in range(10)], + [ + '[10000 - 10000.90]', + *( + f'({10000 + 0.9 * at:.2f} - {10000 + 0.9 * (at + 1):.2f}]' + for at in range(1, 9) + ), + '(10008.10 - 10009]', + ], + ), + ], + ids=['whole numbers', 'fractions', 'large and close'], +) +def test_numeric_cohort_labels(values, labels): + p = payload([[value, at] for at, value in enumerate(values)]) + p['schema']['instance_table']['columns']['x']['stype'] = 'numerical' + + frame = ColumnExplanations(p).to_pandas() + + assert list(frame[frame['column'] == 'x']['cohort']) == labels + + def test_public_policy_constants_remain_available(): assert { name: getattr(ColumnExplanations, name) @@ -323,7 +439,7 @@ def test_sqlite_materialize_explain_uses_sampled_context_without_new_queries(): finally: connection.close() # A closed in-memory database makes any new SQL fail: analysis must use - # the materialized evidence, including the pre-trim row delta. + # the materialized evidence, the rows sent once 1 and 2 are scored. endpoint = Endpoint() client = SDMClient.__new__(SDMClient) client._start(endpoint) @@ -332,9 +448,10 @@ def test_sqlite_materialize_explain_uses_sampled_context_without_new_queries(): ) frame = result.explanations.columns.to_pandas() groups = frame[frame['column'] == 'cohort'].set_index('cohort') - assert groups.loc['a', 'mean'] == 5.5 - assert groups.loc['b', 'mean'] == 17.5 - assert groups.loc['a', 'share'] == groups.loc['b', 'share'] == 0.5 + assert groups.loc['a', 'mean'] == pytest.approx(6.3) + assert groups.loc['b', 'mean'] == pytest.approx(17.5) + assert groups.loc['a', 'share'] == pytest.approx(10 / 22) + assert groups.loc['b', 'share'] == pytest.approx(12 / 22) assert len(endpoint.sent) == 1 diff --git a/packages/structured-data-models-client/tests/test_gradient_explanations.py b/packages/structured-data-models-client/tests/test_gradient_explanations.py index caf96a3b3..ae3e279e8 100644 --- a/packages/structured-data-models-client/tests/test_gradient_explanations.py +++ b/packages/structured-data-models-client/tests/test_gradient_explanations.py @@ -257,11 +257,10 @@ def test_pretrained_local_model_gradients(model_name, task): assert all( math.isfinite(float(value)) - for name in ('share', 'mean', 'score') + for name in ('share', 'mean') for value in associations[name] ) assert associations['share'].between(0, 1).all() - assert associations['score'].between(0, 1).all() assert len(result) == len(result.explanations.gradients) == 2 frame = result.explanations.gradients.to_pandas() assert not frame.empty @@ -382,6 +381,29 @@ def test_explain_forecast_keeps_step_identity(endpoint): ) +def test_a_forecast_not_explained_is_not_analysed(endpoint): + from test_frame_materialize_core import link_schema + + from sdm_client import KumoRelational, PredictiveQuery + + url, calls = endpoint + req = PredictiveQuery( + 'PREDICT COUNT(orders.*, 0, 30, days) FORECAST 2 TIMEFRAMES FOR EACH users.user_id', + link_schema(), + ).materialize( + indices=[1, 2], + num_context_samples=12, + random_seed=7, + model=KumoRelational(), + ) + with SDMClient(url=url) as client: + result = client.predict( + req, model=KumoRelational(), outputs=['prediction'] + ) + assert len([call for call in calls if 'predict' in call]) == 2 + assert result.explanations.columns is None + + def test_native_reply_nonfinite_gradient_uses_shared_refusal(): from test_tabular_direct_core import Endpoint, client diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index deccb4296..52886a52e 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -5,8 +5,7 @@ """End-to-end against a running ``kumo-relational`` NIM. Excluded by default. Point ``SDM_RELATIONAL_URL`` at the endpoint and run with -``-m live_nim``. Needs the ``[relational]`` extra, because the gathering step -runs in the engine. +``-m live_nim``. SDM_RELATIONAL_URL=http://localhost:8001 pytest -m live_nim diff --git a/packages/structured-data-models-client/tests/test_materialize_payloads.py b/packages/structured-data-models-client/tests/test_materialize_payloads.py index be8999ddc..f42c59916 100644 --- a/packages/structured-data-models-client/tests/test_materialize_payloads.py +++ b/packages/structured-data-models-client/tests/test_materialize_payloads.py @@ -68,60 +68,8 @@ def test_materialized_payload_matches_recorded_request(recording): assert expected['output'].pop('prediction_statistic') == 'median' assert actual['inference'].pop('inference_config') == defaults assert actual.pop('client_generated_keys') == {'orders': '__node_id'} - # Analysis restores labelled rows withheld from the model to avoid - # target leakage. This client-only delta is stripped at the wire boundary. - analysis = actual.pop('client_analysis_delta', None) - if recording['case'] == 'lag.static-2': - assert set(analysis) == { - 'version', - 'schema_json', - 'instance_table', - 'related_tables', - } - assert analysis['version'] == 1 - assert json.loads(analysis['schema_json']) == expected['schema'] - anchor = '2023-03-01T00:00:00.000000Z' - removed = { - 'instance_table': ( - 8, - [4, 6], - [[4, 1, anchor, 21], [6, 0, anchor, 20]], - ), - 'users': (8, [4, 6], [[4, 1, 'DE'], [6, 0, 'US']]), - 'orders': ( - 9, - [4, 6, 7], - [ - [4, 4, 11.5, '2023-01-08T00:00:00.000000Z', 1], - [6, 6, 22.0, '2023-02-26T00:00:00.000000Z', 0], - [6, 7, 10.0, '2023-01-01T00:00:00.000000Z', 0], - ], - ), - } - assert set(analysis['related_tables']) == {'users', 'orders'} - for name, (count, positions, rows) in removed.items(): - context = expected['context'] - table = ( - context[name] - if name == 'instance_table' - else context['related_tables'][name] - ) - delta = ( - analysis[name] - if name == 'instance_table' - else analysis['related_tables'][name] - ) - assert delta == { - 'original_rows': count, - 'positions': positions, - 'removed': { - 'format': 'arrays', - 'columns': table['columns'], - 'rows': rows, - }, - } - else: - assert analysis is None + # Column analysis reads the rows sent; no copy of withheld rows is kept. + assert 'client_analysis_delta' not in actual assert actual == expected assert request.entity_table == 'users' assert request.indices == [0, 1] diff --git a/packages/structured-data-models-client/tests/test_pql_for_tabular.py b/packages/structured-data-models-client/tests/test_pql_for_tabular.py index 4bf351533..5fbdd611b 100644 --- a/packages/structured-data-models-client/tests/test_pql_for_tabular.py +++ b/packages/structured-data-models-client/tests/test_pql_for_tabular.py @@ -129,6 +129,26 @@ def test_materialize_selects_the_model_from_the_call_shape( assert request.predict_rows == (2 if argument == 'predict' else 1) +@pytest.mark.parametrize( + ('text', 'advice'), + [ + ( + 'PREDICT users.churn FOR EACH users.user_id WHERE users.age > 30', + 'Filter the rows before constructing the query.', + ), + ( + "PREDICT users.churn = 'yes' FOR EACH users.user_id", + "Add the derived value as a column of 'users' and predict that " + 'column.', + ), + ], +) +def test_a_query_a_tabular_model_cannot_answer_says_what_to_do(text, advice): + with pytest.raises(StructuredDataError) as raised: + query(text).materialize(indices=[0, 1], model='kumo-tabular') + assert str(raised.value).endswith(advice) + + def test_a_query_predicting_a_column_that_is_not_there_is_refused(): with pytest.raises((StructuredDataError, ValueError)): PredictiveQuery( diff --git a/packages/structured-data-models-client/tests/test_result_merge_core.py b/packages/structured-data-models-client/tests/test_result_merge_core.py index 54fd80825..812b45b99 100644 --- a/packages/structured-data-models-client/tests/test_result_merge_core.py +++ b/packages/structured-data-models-client/tests/test_result_merge_core.py @@ -5,6 +5,7 @@ """Public merge retains native owners and delegates policy without encoding rows.""" from datetime import date, datetime, timedelta, timezone +from decimal import Decimal import numpy as np import pandas as pd @@ -65,18 +66,35 @@ def observe(operation, payload): assert not {'items', 'metadata', 'explanation'} & descriptor.keys() -def test_stable_union_deduplicates_within_parts_and_keeps_exact_numbers(): +def test_union_keeps_repeats_within_a_part_and_drops_earlier_values(): first = part([1, 1, np.int64(1), 2**53 + 1, 2**53, np.uint64(2**64 - 1)]) result = PredictionResult.merge( [first, part([True, 1.0, float(2**53), 2**64 - 1])] ) - assert result.unscored_candidates == ( - 1, - 2**53 + 1, - 2**53, - np.uint64(2**64 - 1), + assert result.unscored_candidates == first.unscored_candidates + assert all( + a is b + for a, b in zip( + result.unscored_candidates, first.unscored_candidates, strict=True + ) ) - assert result.unscored_candidates[-1] is first.unscored_candidates[-1] + + +@pytest.mark.parametrize( + ('earlier', 'later', 'kept'), + [ + (Decimal('1.50'), 1.5, 1), + (Decimal('2.0'), Decimal('2'), 1), + (Decimal(2**64), 2**64, 1), + (Decimal('-Infinity'), float('-inf'), 1), + (Decimal('0.1'), 0.1, 2), + (Decimal('NaN'), Decimal('NaN'), 2), + ], +) +def test_decimal_candidates_merge_by_value(earlier, later, kept): + result = PredictionResult.merge([part([earlier]), part([later])]) + assert len(result.unscored_candidates) == kept + assert result.unscored_candidates[0] is earlier def test_nan_values_remain_distinct_under_ieee_equality(): diff --git a/packages/structured-data-models-client/tests/test_semantic_model_source.py b/packages/structured-data-models-client/tests/test_semantic_model_source.py new file mode 100644 index 000000000..815c6ea7f --- /dev/null +++ b/packages/structured-data-models-client/tests/test_semantic_model_source.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""How a Databricks metric view read as a schema is refused. + +The warehouse is the fake in `test_a_metric_view_filter_holds`, which +answers the statements the Databricks provider sends. +""" + +from __future__ import annotations + +import warnings +from typing import Any + +import pytest +from sdm_connectors import DatabricksSource +from test_a_metric_view_filter_holds import _TYPES, _Cursor, _Warehouse + +from sdm_client import Schema, StructuredDataError + + +@pytest.fixture +def source() -> Any: + return DatabricksSource(_Warehouse(), catalog='cat', schema='sch') + + +def _schema(source: Any, tables: dict[str, str] | None = None) -> Schema: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + return Schema.from_source(source.semantic_model('sales_mv'), tables) + + +def test_a_table_the_model_does_not_have_is_named(source: Any) -> None: + with pytest.raises(StructuredDataError, match='not tables of') as caught: + _schema(source, {'orders': 'orders'}) + + assert caught.value.code == 'INVALID_REQUEST' + assert caught.value.details['absent'] == ['orders'] + + +def test_a_missing_source_table_is_an_invalid_request_naming_the_model( + source: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delitem(_TYPES, 'order_lines') + + with pytest.raises(StructuredDataError) as caught: + _schema(source) + + assert caught.value.code == 'INVALID_REQUEST' + assert "order_lines' does not exist" in caught.value.message + assert caught.value.details['semantic_model'] == 'sales_mv' + assert caught.value.details['source'] == ['cat', 'sch', 'order_lines'] + + +class _DriverError(Exception): + r"""Stands in for a warehouse driver's own exception.""" + + +def test_a_failure_that_is_not_a_refusal_is_not_an_invalid_request( + source: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + execute = _Cursor.execute + + def fails(self: _Cursor, sql: str, parameters: Any = None) -> None: + if 'LIMIT' in sql: + raise _DriverError('the warehouse went away') + execute(self, sql, parameters) + + monkeypatch.setattr(_Cursor, 'execute', fails) + + with pytest.raises(StructuredDataError) as caught: + _schema(source) + + assert caught.value.code == 'PROVIDER_ERROR' + assert caught.value.details['driver_class'].endswith('._DriverError') + assert 'the warehouse went away' not in str(caught.value) diff --git a/packages/structured-data-models-client/tests/test_snowflake_semantic_core.py b/packages/structured-data-models-client/tests/test_snowflake_semantic_core.py index 15d20aae9..a968c5faa 100644 --- a/packages/structured-data-models-client/tests/test_snowflake_semantic_core.py +++ b/packages/structured-data-models-client/tests/test_snowflake_semantic_core.py @@ -301,3 +301,100 @@ def test_missing_semantic_view_has_a_named_error_without_driver_text( assert 'row-canary' not in str(raised.value) assert 'driver-sql-canary' not in str(raised.value) assert len(connection.requests) == 1 + + +def test_a_definition_read_that_fails_otherwise_is_a_provider_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection, model = _source(monkeypatch) + connection.failure = RuntimeError('row-canary connection reset by peer') + with pytest.raises(StructuredDataError) as raised: + Schema.from_source(model) + assert raised.value.code == 'PROVIDER_ERROR' + assert 'row-canary' not in str(raised.value) + assert len(connection.requests) == 1 + + +@pytest.mark.parametrize( + ('options', 'warned'), + [({}, False), ({'random_seed': 7}, True)], + ids=['default seed', 'seed given'], +) +def test_a_seed_snowflake_cannot_draw_by_is_said_to_be_ignored( + monkeypatch: pytest.MonkeyPatch, options: dict[str, Any], warned: bool +) -> None: + import warnings + + _, model = _source(monkeypatch) + schema = Schema.from_source( + model, tables={'people': 'USERS'}, relationships=[] + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + PredictiveQuery( + 'PREDICT people.AGE FOR EACH people.USER_ID', schema + ).materialize( + indices=[0, 1], + num_context_samples=4, + model='kumo-relational', + **options, + ) + ignored = [ + str(w.message) for w in caught if 'random_seed' in str(w.message) + ] + assert bool(ignored) is warned + if warned: + assert "'snowflake' backend" in ignored[0] + + +class _ProgrammingError(Exception): + r"""Stands in for the Snowflake driver's error on an unknown column.""" + + +@pytest.mark.parametrize( + ('missing', 'code'), + [('table', 'INVALID_REQUEST'), ('column', 'PROVIDER_ERROR')], +) +def test_a_view_naming_what_is_not_there_is_refused( + monkeypatch: pytest.MonkeyPatch, missing: str, code: str +) -> None: + recorded = _RecordedSemanticProvider.__init__ + answer = _RecordedSemanticProvider.__call__ + + def without(self: _RecordedSemanticProvider, connection: Any) -> None: + recorded(self, connection) + users = self.catalogs['USERS'] + if missing == 'table': + self.catalogs['ORDERS'] = self.catalogs['ORDERS'].slice(0, 0) + else: + names = users.column('name').to_pylist() + self.catalogs['USERS'] = users.filter([n != 'AGE' for n in names]) + + def unknown_column( + self: _RecordedSemanticProvider, step: dict[str, Any], keys: Any = None + ) -> pa.Table: + request = step['request'] + if ( + missing == 'column' + and request['request'] == 'preview' + and request['table']['source'][-1] == 'USERS' + ): + raise _ProgrammingError("invalid identifier 'AGE'") + return answer(self, step, keys) + + monkeypatch.setattr(_RecordedSemanticProvider, '__init__', without) + monkeypatch.setattr(_RecordedSemanticProvider, '__call__', unknown_column) + _, model = _source(monkeypatch) + + with pytest.raises(StructuredDataError) as raised: + Schema.from_source(model) + + assert raised.value.code == code + if missing == 'table': + assert "ORDERS' does not exist" in raised.value.message + assert ( + raised.value.details['semantic_model'] + == 'view_semantic_from_source' + ) + else: + assert 'AGE' not in str(raised.value) diff --git a/packages/structured-data-models-client/tests/test_tabular_direct_core.py b/packages/structured-data-models-client/tests/test_tabular_direct_core.py index 0070d69a7..aa02aef5b 100644 --- a/packages/structured-data-models-client/tests/test_tabular_direct_core.py +++ b/packages/structured-data-models-client/tests/test_tabular_direct_core.py @@ -235,6 +235,58 @@ def test_empty_call_skips_late_inference_conversion() -> None: assert endpoint.calls == [] +def test_an_empty_answer_without_a_key_names_no_id() -> None: + context, scoring = frames() + request = Request( + context.drop(columns='row_id'), + scoring.drop(columns='row_id').iloc[:0], + target='target', + task='classification', + ) + result = client(Endpoint()).predict(request, model='kumo-tabular') + assert list(result.to_pandas().columns) == ['row_index', 'prediction'] + + +@pytest.mark.parametrize( + ('change', 'task', 'said'), + [ + ('constant', 'classification', 'every context row'), + ('constant', 'regression', 'every context row'), + ('overlap', 'classification', 'also in context'), + ], +) +def test_a_frame_warning_reads_as_the_old_client_wrote_it( + change: str, task: str, said: str +) -> None: + from sdm_client.core.warnings import SDMWarning + + context, scoring = frames() + if change == 'constant': + context['target'] = 1 + else: + scoring['row_id'] = [0, 1] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + request = Request(context, scoring, target='target', task=task) + if change == 'overlap': + reply = { + 'model': 'kumo-tabular', + 'id': 'answer', + 'metadata': {}, + 'predictions': [ + {'row_index': at, 'id': f'[{at}]', 'prediction': 0} + for at in range(2) + ], + } + client(Endpoint(reply)).predict(request, model=KumoTabular()) + [message] = [ + str(w.message) + for w in caught + if issubclass(w.category, SDMWarning) and said in str(w.message) + ] + assert not message.startswith('[') + + def test_constructor_guard_order() -> None: context, scoring = frames() with pytest.raises(StructuredDataError, match='task must'): diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py index 386a5c9af..472e964de 100644 --- a/packages/structured-data-models-client/tests/test_tabular_explanations.py +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -47,12 +47,15 @@ def response(**record) -> dict: } -def answered(request, gradient_explanation, *, keyed=True, outputs=None): +def answered( + request, gradient_explanation, *, keyed=True, outputs=None, unkeyed=() +): """Send ``request`` through the adapter and read back a mocked answer. Each scored row ``i`` comes back as a NIM answers it: under ``'[]'`` - when the request has a key, with probabilities, and with its own - explanation, whose gradients are ``-(i + 1)`` and ``i + 1`` per class. + when the request has a key, except the rows in ``unkeyed``, with + probabilities, and with its own explanation, whose gradients are + ``-(i + 1)`` and ``i + 1`` per class. """ import json @@ -76,7 +79,7 @@ def predict_body(self, body): gradients=[[-(index + 1.0)], [index + 1.0]], ), } - if keyed: + if keyed and index not in unkeyed: record['id'] = f'[{row[table["columns"].index("row_id")]}]' records.append(record) return json.dumps( @@ -181,6 +184,39 @@ def test_rows_whose_ids_repeat_are_explained_under_their_row_index( ) +def test_a_missing_id_keys_all_rows_by_row_index( + tabular_request, gradient_explanation +): + """If any explained row lacks an id, every explanation is under its + row_index, as with repeated ids, so the keying is all-or-nothing. + + Assumptions: + - The request's first row has no ``row_id`` and the second 91, so the + NIM answers the first with no id. + """ + built = tabular_request(rows=2) + request = Request( + context=built.context.instance_table, + predict=built.predict.instance_table.assign( + row_id=pd.array([None, 91], dtype='Int64') + ), + target='churn', + task='classification', + ) + + result = answered(request, gradient_explanation, unkeyed={0}) + + assert sorted(result.explanations.gradients) == [0, 1] + assert result.explanations.gradients[1]['instance']['gradients'] == [ + [-2.0], + [2.0], + ] + assert ( + list(result.explanations.gradients.to_pandas().columns)[0] + == 'row_index' + ) + + def test_a_tabular_entry_has_no_related_tables( tabular_request, gradient_explanation ): diff --git a/packages/structured-data-models-client/tests/test_the_predict_route.py b/packages/structured-data-models-client/tests/test_the_predict_route.py index ad1bb939b..6392f9476 100644 --- a/packages/structured-data-models-client/tests/test_the_predict_route.py +++ b/packages/structured-data-models-client/tests/test_the_predict_route.py @@ -871,3 +871,20 @@ def test_embeddings_are_the_old_route_s_on_the_core( core = _outcome(request, url, outputs=outputs) assert isinstance(core, StructuredDataError) assert core.code == 'INVALID_RESPONSE' + + +def test_an_unknown_model_names_itself_and_the_known_ones() -> None: + from sdm_client import UnknownModelError + + request = _materialized() + with SDMClient('http://127.0.0.1:9', max_retries=0) as client: + with pytest.raises(UnknownModelError) as raised: + client.predict(request, model='kumo-forcast') + + assert raised.value.model == 'kumo-forcast' + assert raised.value.details['model'] == 'kumo-forcast' + assert raised.value.details['known_models'] == [ + 'kumo-relational', + 'kumo-tabular', + ] + assert "Unknown model 'kumo-forcast'" in str(raised.value) diff --git a/packages/structured-data-models-client/tests/test_the_time_column_is_inferred_alike.py b/packages/structured-data-models-client/tests/test_the_time_column_is_inferred_alike.py index c65658090..40aa920f2 100644 --- a/packages/structured-data-models-client/tests/test_the_time_column_is_inferred_alike.py +++ b/packages/structured-data-models-client/tests/test_the_time_column_is_inferred_alike.py @@ -450,3 +450,31 @@ def test_a_customer_can_be_counted_once_its_edit_stamp_is_refused( schema, ).materialize(indices=[1, 2, 3]) assert request is not None + + +@pytest.mark.parametrize( + ('time_columns', 'declared'), + [ + (None, {'ts'}), + ({'t': 'sdate'}, {'ts', 'sdate'}), + ({'t': None}, {'ts'}), + ], + ids=['inferred', 'named', 'none'], +) +def test_a_frames_time_column_is_a_declared_meaning( + time_columns: Any, declared: set[str] +) -> None: + frame = pd.DataFrame( + { + 'id': [1, 2, 3], + 'ts': pd.to_datetime(['2024-01-01', '2024-01-02', '2024-01-03']), + 'sdate': ['2024-01-01', '2024-01-02', '2024-01-03'], + } + ) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + table = Schema.from_data({'t': frame}, time_columns=time_columns)['t'] + + assert { + name for name, column in table.columns.items() if column.stype_declared + } == declared diff --git a/packages/structured-data-models-client/tests/test_transport_socket.py b/packages/structured-data-models-client/tests/test_transport_socket.py index 1e4ae9faa..83d138094 100644 --- a/packages/structured-data-models-client/tests/test_transport_socket.py +++ b/packages/structured-data-models-client/tests/test_transport_socket.py @@ -459,6 +459,28 @@ def test_a_stalled_prediction_is_reported_as_a_timeout_too(): assert 'timed out after 0.5 seconds' in str(raised.value) +def test_a_stalled_one_shot_predict_keeps_the_transport_s_account(): + import pandas as pd + + from sdm_client import KumoTabular, Request, SDMClient + + request = Request( + pd.DataFrame({'x': [1.0, 2.0, 3.0, 4.0], 'y': [0, 1, 0, 1]}), + pd.DataFrame({'x': [5.0]}), + target='y', + task='classification', + ) + with _serve(_Reply(delay=5.0)) as (_, url): + with SDMClient(url, timeout=0.5, max_retries=0) as client: + with pytest.raises(StructuredDataError) as raised: + client.predict(request, model=KumoTabular()) + + assert raised.value.code == 'TRANSPORT_ERROR' + assert 'timed out after 0.5 seconds' in str(raised.value) + assert raised.value.details['timeout'] == 0.5 + assert raised.value.details['url'].startswith('http://127.0.0.1:') + + @pytest.mark.parametrize('session_id', ['..', '.', '']) def test_a_session_id_that_is_not_a_segment_is_refused(session_id): """`..` turned a scoring call against a pinned context into a one-shot diff --git a/qa/README.md b/qa/README.md index 26869645d..3f3342c30 100644 --- a/qa/README.md +++ b/qa/README.md @@ -48,12 +48,11 @@ python probe_cloud.py # S3 (needs AWS credentials and /tmp/qa_bucket) python probe_warehouse.py # Snowflake and Databricks (needs SNOWFLAKE_*, DATABRICKS_*) ``` -The relational live-NIM contract tests have a runner that creates its own -environment and defaults to the smoke suite: +The live-NIM tests of the client run from its own suite: ```bash -qa/run_rfm_nim_live_tests.sh --url http://localhost:8001 -qa/run_rfm_nim_live_tests.sh --url http://localhost:8001 --full +SDM_TABULAR_URL=http://localhost:8000 SDM_RELATIONAL_URL=http://localhost:8001 \ + python -m pytest packages/structured-data-models-client/tests -m live_nim -q ``` Each writes a JSON record to `~/sdm-qa/findings/` and prints a summary. A probe diff --git a/qa/run_rfm_nim_live_tests.sh b/qa/run_rfm_nim_live_tests.sh deleted file mode 100755 index dee6fa29b..000000000 --- a/qa/run_rfm_nim_live_tests.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ENGINE_ROOT="$WORKSPACE_ROOT/packages/structured-data-models-relational-engine" -VENV_DIR="${RFM_NIM_LIVE_VENV:-$ENGINE_ROOT/.tmp/rfm-nim-live-venv}" -BASE_URL="${KUMO_RELATIONAL_NIM_BASE_URL:-${RFM_NIM_BASE_URL:-}}" -MODE=smoke -REBUILD_ENV=0 - -usage() { - cat <<'USAGE' -Run URL-driven KumoRelational NIM live validation. - -Usage: - qa/run_rfm_nim_live_tests.sh --url [--full] [pytest args...] - -Options: - --url URL KumoRelational NIM service root. KUMO_RELATIONAL_NIM_BASE_URL - (or the older RFM_NIM_BASE_URL) is also accepted. - --full Run smoke and extended live validation. Default: smoke only. - --rebuild-env Recreate the script-owned virtualenv before running. - -h, --help Show this help. - -Optional environment: - KUMO_RELATIONAL_NIM_API_KEY X-API-Key value; never printed by this script. - RFM_NIM_API_KEY is still honoured. - RFM_NIM_VERIFY_SSL Set 0/false/no to disable TLS verification. - RFM_NIM_TIMEOUT_SECONDS Per-request timeout. Default: 30. - RFM_NIM_PYTHON Existing Python interpreter to use instead of the - script-owned virtualenv. -USAGE -} - -pytest_args=() -while (($#)); do - case "$1" in - --url) - [[ $# -ge 2 ]] || { echo 'error: --url requires a value' >&2; exit 2; } - BASE_URL="$2" - shift 2 - ;; - --full) - MODE=full - shift - ;; - --rebuild-env) - REBUILD_ENV=1 - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - pytest_args+=("$1") - shift - ;; - esac -done - -if [[ -z "$BASE_URL" ]]; then - echo 'error: pass --url or set KUMO_RELATIONAL_NIM_BASE_URL' >&2 - exit 2 -fi - -if [[ -n "${RFM_NIM_PYTHON:-}" ]]; then - PYTHON="$RFM_NIM_PYTHON" - [[ -x "$PYTHON" ]] || { echo "error: Python is not executable: $PYTHON" >&2; exit 2; } - "$PYTHON" -c 'import pytest, requests, sdm_relational_engine' >/dev/null || { - echo 'error: RFM_NIM_PYTHON is missing the client live-test dependencies' >&2 - exit 2 - } -else - PYTHON="$VENV_DIR/bin/python" - STAMP="$VENV_DIR/.rfm-sdk-live-dependencies" - fingerprint="$( - cksum "$ENGINE_ROOT/pyproject.toml" "$ENGINE_ROOT/setup.py" | - cksum | - awk '{print $1 ":" $2}' - )" - installed_fingerprint='' - [[ -f "$STAMP" ]] && installed_fingerprint="$(<"$STAMP")" - - needs_bootstrap="$REBUILD_ENV" - if [[ ! -x "$PYTHON" || "$installed_fingerprint" != "$fingerprint" ]]; then - needs_bootstrap=1 - elif ! "$PYTHON" -c 'import pytest, requests, sdm_relational_engine' >/dev/null 2>&1; then - needs_bootstrap=1 - fi - - if [[ "$needs_bootstrap" == 1 ]]; then - command -v python3 >/dev/null || { echo 'error: python3 is required' >&2; exit 2; } - mkdir -p "$(dirname "$VENV_DIR")" - python3 -m venv --clear "$VENV_DIR" - "$PYTHON" -m pip install --upgrade pip - WITH_RELATIONALLIB=0 \ - "$PYTHON" -m pip install --editable "${ENGINE_ROOT}[test]" - printf '%s\n' "$fingerprint" >"$STAMP" - fi -fi - -if ! PYTHONPATH="$ENGINE_ROOT/tests/client${PYTHONPATH:+:$PYTHONPATH}" \ - RFM_NIM_BASE_URL="$BASE_URL" \ - "$PYTHON" -c ' -import os - -from rfm_nim_live_harness import normalize_base_url - -try: - normalize_base_url(os.environ["RFM_NIM_BASE_URL"]) -except ValueError as exc: - raise SystemExit(f"error: {exc}") from None -'; then - exit 2 -fi - -report_dir="$ENGINE_ROOT/.tmp/rfm-nim-live-results" -mkdir -p "$report_dir" -if [[ "$MODE" == full ]]; then - marker='live_nim_smoke or live_nim_full' -else - marker='live_nim_smoke' -fi - -echo "Running RFM NIM $MODE validation" -pytest_command=( - "$PYTHON" -m pytest - "$ENGINE_ROOT/tests/client/test_rfm_nim_live.py" - --strict-markers - -m "$marker" - --junitxml="$report_dir/$MODE.xml" -) -if ((${#pytest_args[@]})); then - pytest_command+=("${pytest_args[@]}") -fi -RFM_NIM_BASE_URL="$BASE_URL" \ - KUMO_RELATIONAL_NIM_API_KEY="${KUMO_RELATIONAL_NIM_API_KEY:-${RFM_NIM_API_KEY:-}}" \ - exec "${pytest_command[@]}" diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 73a67e69f..a4fd0cb0a 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -71 sha256:6041dbc10ec757561b56eb6cc0decdedf1c9e7f7e1268e641e03401b492ea8be sdm-contract-71.json +72 sha256:d2cc138d8281084aed2457d2d603d93372c06ab459b0f978a6065665ec4dd35e sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-71.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json similarity index 99% rename from rust/crates/sdm-contract/schema/sdm-contract-71.json rename to rust/crates/sdm-contract/schema/sdm-contract-72.json index dcd05e49d..fb7cc7be4 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-71.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -1,5 +1,5 @@ { - "$id": "sdm-contract/71", + "$id": "sdm-contract/72", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ControlMessage", "description": "Every control message that crosses between a host and the core.", @@ -1114,6 +1114,24 @@ "value" ] }, + { + "description": "A finite decimal in fixed point, equal to every number of its value.", + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "kind": { + "type": "string", + "const": "decimal" + } + }, + "additionalProperties": false, + "required": [ + "kind", + "value" + ] + }, { "type": "object", "properties": { @@ -1518,10 +1536,6 @@ "mean": { "type": "number", "format": "double" - }, - "score": { - "type": "number", - "format": "double" } }, "additionalProperties": false, @@ -1532,8 +1546,7 @@ "cohort", "class_json", "share", - "mean", - "score" + "mean" ] }, "MergeInput": { @@ -10915,7 +10928,7 @@ ] }, "NamedTable": { - "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads (design choice 12), which a host naming a table\nby a string gives as one part.", + "description": "A table named for the schema: the name a query uses, and the relation\nof the source it reads, which a host naming a table\nby a string gives as one part.", "type": "object", "properties": { "name": { @@ -10993,7 +11006,7 @@ "type": "string" }, "source": { - "description": "The relation of the source this table reads (design choice 12): its\nown name, one part, unless the caller named it otherwise (decision\nE59).", + "description": "The relation of the source this table reads: its\nown name, one part, unless the caller named it otherwise (decision\nE59).", "$ref": "#/$defs/Relation" }, "columns": { @@ -14740,7 +14753,7 @@ "$ref": "#/$defs/Determinism" }, "max_seed": { - "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n(design choice 1). The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", + "description": "The largest seed the provider samples with (register entry RS-7),\nwhich every provider declares: there is no \"any seed\" default\n. The core refuses a sample with a larger seed,\n`INVALID_REQUEST` naming the seed, before the sample is planned;\nwhich seeds a provider takes is the provider's to declare, never a\nname the core matches.", "type": "integer", "format": "uint64", "minimum": 0 @@ -14779,7 +14792,7 @@ ] }, "fallback": { - "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads (design choice 1, the in-memory fallback); left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", + "description": "The limits within which the core answers an optional request the\nprovider does not declare itself, in memory, from the rows a `scan`\nreads; left out,\nthe provider declares none, and such a request is refused,\n`UNSUPPORTED_REQUEST`, as before. A declared request that fails is\nthe provider's error either way, never answered by the fallback.", "anyOf": [ { "$ref": "#/$defs/FallbackLimits" @@ -18010,7 +18023,7 @@ } }, "answered": { - "description": "What a request asked in several model calls had been answered when\na later call failed (design choice 1, `main`'s answered parts,\n): the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", + "description": "What a request asked in several model calls had been answered when\na later call failed: the error keeps what was paid for, so the rest can be asked\nagain. Absent for any other error.", "anyOf": [ { "$ref": "#/$defs/AnsweredParts" diff --git a/rust/crates/sdm-contract/src/columns.rs b/rust/crates/sdm-contract/src/columns.rs index 865f6c11d..6af8203c3 100644 --- a/rust/crates/sdm-contract/src/columns.rs +++ b/rust/crates/sdm-contract/src/columns.rs @@ -26,7 +26,6 @@ pub struct ColumnRow { pub class_json: String, pub share: f64, pub mean: f64, - pub score: f64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] diff --git a/rust/crates/sdm-contract/src/lib.rs b/rust/crates/sdm-contract/src/lib.rs index 024697026..47893f2cb 100644 --- a/rust/crates/sdm-contract/src/lib.rs +++ b/rust/crates/sdm-contract/src/lib.rs @@ -56,7 +56,7 @@ pub use scalar::Scalar; pub use types::{ColumnType, Field, Unit}; /// The contract version this build speaks. -pub const CONTRACT_VERSION: u32 = 71; +pub const CONTRACT_VERSION: u32 = 72; /// The JSON Schema of [`ControlMessage`], as committed: pretty-printed, with /// a trailing newline. Its `$defs` hold [`ops::Attachments`] too, the JSON diff --git a/rust/crates/sdm-contract/src/result.rs b/rust/crates/sdm-contract/src/result.rs index 232ba1306..db438e9c1 100644 --- a/rust/crates/sdm-contract/src/result.rs +++ b/rust/crates/sdm-contract/src/result.rs @@ -78,6 +78,10 @@ pub enum MergeCandidate { Bytes { value: Vec, }, + /// A finite decimal in fixed point, equal to every number of its value. + Decimal { + value: String, + }, Date { ordinal: i32, }, diff --git a/rust/crates/sdm-core/src/columns.rs b/rust/crates/sdm-core/src/columns.rs index e257aadf5..6609eb908 100644 --- a/rust/crates/sdm-core/src/columns.rs +++ b/rust/crates/sdm-core/src/columns.rs @@ -18,21 +18,24 @@ pub(crate) fn run( return Err(internal("column analysis payload is not bytes")); }; let payload = crate::predict::payload(&bytes)?; - let result = sdm_request::columns::analyze(&sdm_request::columns::restored(&payload)?)?; + let result = sdm_request::columns::analyze(&payload)?; crate::direct::value(asker, &result) } pub(crate) fn compute(payloads: &[sdm_wire::json::Value]) -> Result { - let restored = payloads - .iter() - .map(sdm_request::columns::restored) - .collect::, _>>()?; - let payload = if restored.len() == 1 { - restored[0].clone() + let payload = if payloads.len() == 1 { + payloads[0].clone() } else { - sdm_request::columns::forecast(&restored)? + sdm_request::columns::forecast(payloads)? }; - serde_json::to_string(&sdm_request::columns::analyze(&payload)?) + encode(&payload) +} +/// The analysis of the labelled rows a request's calls send, each once. +pub(crate) fn compute_sent(payloads: &[sdm_wire::json::Value]) -> Result { + encode(&sdm_request::columns::sent(payloads)?) +} +fn encode(payload: &sdm_wire::json::Value) -> Result { + serde_json::to_string(&sdm_request::columns::analyze(payload)?) .map_err(|_| internal("analysis result cannot be encoded")) } pub(crate) fn requested(outputs: &[String]) -> bool { diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index d171e3ebd..67a9af653 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -3086,6 +3086,24 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result Result Result Vec Vec Vec Vec Vec Result, view: &ViewChoice) -> Result reading.asker.read(&request, &data)?, Err(failed) => { - if reading.data.view_kind() == Some(ViewKind::Semantic) { + let facts = reading.data.failure(&failed); + let missing = + facts.unusable == Some(sdm_contract::data::Unusable::Missing) || facts.not_found; + let denied = facts.unusable == Some(sdm_contract::data::Unusable::Forbidden) + || failed.sqlstate.as_deref() == Some("42501"); + if (missing || denied) && reading.data.view_kind() == Some(ViewKind::Semantic) { let name = relation.dotted(); return Err(CoreError::new( Sentence::ViewSemanticNotFound, @@ -211,8 +222,7 @@ async fn read_definition(reading: &Reading<'_>, view: &ViewChoice) -> Result CoreError { CoreError::new( Sentence::InvalidMessage, @@ -160,34 +161,24 @@ struct Cohorts { labels: Vec, counts: Vec, means: Vec>, - score: f64, } -fn score(means: &[Vec], counts: &[f64]) -> f64 { - let total: f64 = counts.iter().sum(); - if total <= 1.0 || means.is_empty() { - return 0.0; +fn short(v: f64, places: usize) -> String { + let s = format!("{v:.places$}"); + match s.split_once('.') { + Some((whole, fraction)) if fraction.bytes().all(|b| b == b'0') => whole.to_owned(), + _ => s, } - let width = means[0].len(); - if width == 0 { - return 0.0; - } - (0..width) - .map(|j| { - let center = means.iter().zip(counts).map(|(m, c)| m[j] * c).sum::() / total; - (means - .iter() - .zip(counts) - .map(|(m, c)| c * (m[j] - center).powi(2)) - .sum::() - / (total - 1.0)) - .sqrt() - }) - .sum::() - / width as f64 } -fn short(v: f64) -> String { - let s = format!("{v:.2}"); - s.strip_suffix(".00").unwrap_or(&s).to_owned() +/// The fewest decimals, from 2 to 6, at which adjacent distinct edges print +/// apart. +fn places(edges: &[f64]) -> usize { + (2..=6) + .find(|&places| { + edges + .windows(2) + .all(|w| w[0] == w[1] || short(w[0], places) != short(w[1], places)) + }) + .unwrap_or(6) } fn ranges(values: &[Option], count: bool, outcome: &[Vec]) -> Option { let mut sorted = values.iter().flatten().copied().collect::>(); @@ -227,6 +218,7 @@ fn ranges(values: &[Option], count: bool, outcome: &[Vec]) -> Option], count: bool, outcome: &[Vec]) -> Option], count: bool, outcome: &[Vec]) -> Option], - labels: &[String], - other: Option, - outcome: &[Vec], -) -> Option { +fn categories(shares: &[Vec], labels: &[String], outcome: &[Vec]) -> Option { let mut result = Cohorts::default(); - let mut scoring = Vec::new(); - let mut scoring_counts = Vec::new(); for (i, label) in labels.iter().enumerate() { let count = shares.iter().map(|r| r[i]).sum::(); if count == 0.0 { @@ -274,10 +258,6 @@ fn categories( / count }) .collect::>(); - if other != Some(i) { - scoring.push(mean.clone()); - scoring_counts.push(count); - } result.labels.push(label.clone()); result.counts.push(count); result.means.push(mean); @@ -285,26 +265,25 @@ fn categories( if result.labels.is_empty() { return None; } - result.score = score(&scoring, &scoring_counts); if result.labels.len() > COHORTS { - let split = COHORTS - 1; - let total = result.counts[split..].iter().sum::(); + // Missing values keep a cohort of their own, outside the merge. + let na = usize::from(result.labels.last().is_some_and(|l| l == NA)); + let split = COHORTS - 1 - na; + let end = result.labels.len() - na; + let total = result.counts[split..end].iter().sum::(); let means = (0..outcome[0].len()) .map(|j| { - result.counts[split..] + result.counts[split..end] .iter() - .zip(&result.means[split..]) + .zip(&result.means[split..end]) .map(|(c, m)| c * m[j]) .sum::() / total }) .collect(); - result.labels.truncate(split); - result.labels.push(OTHER.into()); - result.counts.truncate(split); - result.counts.push(total); - result.means.truncate(split); - result.means.push(means); + result.labels.splice(split..end, [OTHER.to_owned()]); + result.counts.splice(split..end, [total]); + result.means.splice(split..end, [means]); } Some(result) } @@ -337,24 +316,15 @@ impl Analysis { )); } } - fn finish(mut self) -> ColumnAnalysis { + fn finish(self) -> ColumnAnalysis { if self.columns.iter().any(|c| { - !c.4.score.is_finite() - || c.4.counts.iter().any(|v| !v.is_finite()) + c.4.counts.iter().any(|v| !v.is_finite()) || c.4.means.iter().flatten().any(|v| !v.is_finite()) }) { return ColumnAnalysis::Failed { reason: "nonfinite_statistics".into(), }; } - let scale = self.columns.iter().map(|c| c.4.score).fold(0.0, f64::max); - if scale > 0.0 { - let total = self.columns.iter().map(|c| c.4.score / scale).sum::(); - for c in &mut self.columns { - c.4.score = (c.4.score / scale) / total; - } - } - self.columns.sort_by(|a, b| b.4.score.total_cmp(&a.4.score)); let mut rows = Vec::new(); for (table, hop, column, stype, c) in self.columns { let total = c.counts.iter().sum::(); @@ -369,7 +339,6 @@ impl Analysis { class_json: encoded(class), share: c.counts[i] / total, mean: c.means[i][j], - score: c.score, }); } } @@ -593,7 +562,7 @@ impl Analysis { hop, column, Some(stype), - categories(&shares, &labels, other, &self.outcome), + categories(&shares, &labels, &self.outcome), ); Ok(()) } @@ -654,7 +623,7 @@ impl Analysis { hop, column, Some("timestamp"), - categories(&shares, &labels, None, &self.outcome), + categories(&shares, &labels, &self.outcome), ); Ok(()) } @@ -856,10 +825,45 @@ mod tests { assert_eq!(rows[0].mean, 1.0); assert_eq!(rows[1].mean, 9.0); assert_eq!(rows[0].share, 0.5); - assert_eq!(rows[0].score, 1.0); - assert!( - (score(&[vec![1.0], vec![9.0]], &[2.0, 2.0]) - (64.0_f64 / 3.0).sqrt()).abs() < 1e-12 - ); + } + #[test] + fn adjacent_numeric_edges_get_distinct_labels() { + for (values, first, last) in [ + ( + (1..=11).map(|i| f64::from(i) / 1000.0).collect::>(), + "[0.001 - 0.002]", + "(0.010 - 0.011]", + ), + ( + (1..=11).map(|i| 10000.0 + f64::from(i) / 1000.0).collect(), + "[10000.001 - 10000.002]", + "(10000.010 - 10000.011]", + ), + ] { + let values = values.into_iter().map(Some).collect::>(); + let outcome = vec![vec![1.0]; values.len()]; + let labels = ranges(&values, false, &outcome).unwrap().labels; + assert_eq!(labels.first().unwrap(), first); + assert_eq!(labels.last().unwrap(), last); + assert_eq!(labels.iter().collect::>().len(), labels.len()); + } + } + #[test] + fn missing_values_keep_their_own_cohort_beside_the_merged_tail() { + let rows = (0..12) + .map(|i| format!(r#"["{}",{i}]"#, char::from(b'a' + i))) + .chain([r#"[null,20]"#.to_owned()]) + .collect::>() + .join(","); + let ColumnAnalysis::Ready { rows } = + analyze(&payload(&format!("[{rows}]"), "regression")).unwrap() + else { + panic!() + }; + let cohorts = rows.iter().map(|r| r.cohort.as_str()).collect::>(); + assert_eq!(cohorts.len(), COHORTS); + assert_eq!(cohorts[COHORTS - 2..], [OTHER, NA]); + assert_eq!(rows[COHORTS - 1].mean, 20.0); } #[test] fn empty_classification_is_ready_empty() { @@ -887,15 +891,6 @@ mod tests { ); } #[test] - fn constant_score_is_zero() { - let ColumnAnalysis::Ready { rows } = - analyze(&payload(r#"[["a",2],["b",2]]"#, "regression")).unwrap() - else { - panic!() - }; - assert!(rows.iter().all(|r| r.score == 0.0)); - } - #[test] fn exact_wide_negative_frequency_ties() { let a = Value::Number("-10000000000000000000000000000000000000001".into()); let b = Value::Number("-10000000000000000000000000000000000000002".into()); @@ -912,7 +907,11 @@ mod tests { } #[test] fn finite_extreme_targets_never_emit_nonfinite_statistics() { - let result = analyze(&payload(r#"[["a",1e308],["b",-1e308]]"#, "regression")).unwrap(); + let result = analyze(&payload( + r#"[["a",1e308],["a",1e308],["b",-1e308],["b",-1e308]]"#, + "regression", + )) + .unwrap(); let raw = serde_json::to_string(&result).unwrap(); validate(&raw).unwrap(); assert!(matches!(result, ColumnAnalysis::Failed { .. })); @@ -936,7 +935,7 @@ mod tests { #[test] fn malformed_analysis_is_not_absence() { assert!(validate(r#"{"status":"mystery"}"#).is_err()); - assert!(validate(r#"{"status":"ready","rows":[{"table":"t","hop":0,"column":"x","stype":null,"cohort":"a","class_json":"[]","share":0.5,"mean":1,"score":1}]}"#).is_err()); + assert!(validate(r#"{"status":"ready","rows":[{"table":"t","hop":0,"column":"x","stype":null,"cohort":"a","class_json":"[]","share":0.5,"mean":1}]}"#).is_err()); } } @@ -952,8 +951,6 @@ pub fn validate(raw: &str) -> Result { || !row.share.is_finite() || !(0.0..=1.0).contains(&row.share) || !row.mean.is_finite() - || !row.score.is_finite() - || !(0.0..=1.0).contains(&row.score) { return Err(invalid("invalid cohort statistics")); } @@ -962,104 +959,147 @@ pub fn validate(raw: &str) -> Result { Ok(analysis) } -/// Restore only excluded rows, validating the delta before any analysis or model call. -pub fn restored(payload: &Value) -> Result { - let Some(delta) = payload.get("client_analysis_delta") else { - return Ok(payload.clone()); +/// The labelled rows several calls of one request send, as one context: +/// every call carries the same labelled draw, less its own scored entities, +/// under the same instance ids, so each instance, keyed by its entity and +/// anchor, is read once, from the first call that sends it. +pub fn sent(payloads: &[Value]) -> Result { + let Some(first) = payloads.first() else { + return Err(invalid("empty analysis request")); }; - if delta.get("version") != Some(&Value::int(1)) { - return Err(CoreError::protocol( - Sentence::StateUnread, - "unsupported column context delta", - )); - } - if text(field(delta, "schema_json")?)? != encoded(field(payload, "schema")?) { - return Err(CoreError::protocol( - Sentence::StateUnread, - "column context delta schema differs", - )); + let entity_of = |table: &Table<'_>| table.names.iter().position(|c| c.starts_with(ENTITY_REF)); + let first_context = field(first, "context")?; + if payloads.len() == 1 + || entity_of(&Table::read(field(first_context, "instance_table")?)?).is_none() + { + return Ok(first.clone()); } - fn restore(table: &Value, delta: &Value) -> Result { - let removed = field(delta, "removed")?; - if field(table, "columns")? != field(removed, "columns")? { - return Err(CoreError::protocol( - Sentence::StateUnread, - "column context delta columns differ", - )); + let mut seen = BTreeSet::new(); + let mut instances = Vec::new(); + let mut related: Vec<(String, Value)> = Vec::new(); + let mut schema = field(first, "schema")?.clone(); + for payload in payloads { + let context = field(payload, "context")?; + let table = Table::read(field(context, "instance_table")?)?; + let id = table.at("instance_id")?; + let entity = entity_of(&table).ok_or_else(|| invalid("a call names no entity"))?; + let anchor = field(payload, "task")? + .get("anchor_time_column") + .and_then(Value::as_text) + .and_then(|name| table.at(name).ok()); + let mut keys = Vec::new(); + let mut kept = BTreeSet::new(); + for row in &table.rows { + let at = ( + key(&row[entity])?, + anchor.map(|a| key(&row[a])).transpose()?, + ); + if !seen.contains(&at) { + kept.insert(key(&row[id])?); + instances.push(Value::Array(row.to_vec())); + } + keys.push(at); } - let total = match field(delta, "original_rows")? { - Value::Number(n) => n.parse::().map_err(|_| invalid("delta row count"))?, - _ => return Err(invalid("delta row count")), - }; - let positions = array(field(delta, "positions")?)? - .iter() - .map(|v| match v { - Value::Number(n) => n.parse::().map_err(|_| invalid("delta position")), - _ => Err(invalid("delta position")), - }) - .collect::, _>>()?; - let before = Table::read(table)?; - let dropped = Table::read(removed)?; - if positions.len() != dropped.rows.len() - || before.rows.len().checked_add(positions.len()) != Some(total) - || positions.windows(2).any(|p| p[0] >= p[1]) - || positions.last().is_some_and(|p| *p >= total) - { - return Err(CoreError::protocol( - Sentence::StateUnread, - "invalid column context delta positions", - )); + seen.extend(keys); + if let Some(tables) = context.get("related_tables") { + for (name, rows) in members(tables)? { + let read = Table::read(rows)?; + let at = read.at("instance_id")?; + let held = match related.iter_mut().find(|(n, _)| n == name) { + Some((_, held)) => held, + None => { + let mut empty = rows.clone(); + empty.set("rows", Value::Array(vec![])); + related.push((name.clone(), empty)); + &mut related.last_mut().expect("just pushed").1 + } + }; + // A call may send a column another did not, such as a key + // to a table only it reached: rows take every column, a + // cell their call did not send left null. + let mut names = array(field(held, "columns")?)?.to_vec(); + for column in array(field(rows, "columns")?)? { + if !names.contains(column) { + names.push(column.clone()); + } + } + let widen = |row: &[Value], from: &[Value]| { + names + .iter() + .map(|name| { + from.iter() + .position(|f| f == name) + .map_or(Value::Null, |i| row[i].clone()) + }) + .collect::>() + }; + let before = array(field(held, "columns")?)?.to_vec(); + let mut all = array(field(held, "rows")?)? + .iter() + .map(|row| Ok(Value::Array(widen(array(row)?, &before)))) + .collect::, CoreError>>()?; + let from = array(field(rows, "columns")?)?; + for row in &read.rows { + if kept.contains(&key(&row[at])?) { + all.push(Value::Array(widen(row, from))); + } + } + held.set("columns", Value::Array(names.clone())); + held.set("rows", Value::Array(all)); + } } - let mut kept = before.rows.into_iter(); - let mut dropped = dropped.rows.into_iter(); - let mut at = positions.into_iter().peekable(); - let mut rows = Vec::with_capacity(total); - for i in 0..total { - let row = if at.peek() == Some(&i) { - at.next(); - dropped.next() - } else { - kept.next() + let step = field(payload, "schema")?; + if let Some(value) = step.get("related_tables") { + let mut combined = schema + .get("related_tables") + .cloned() + .unwrap_or(Value::Object(vec![])); + for (name, described) in members(value)? { + match combined.get(name).cloned() { + None => combined.set(name, described.clone()), + Some(mut held) => { + let mut columns = field(&held, "columns")?.clone(); + for (column, spec) in members(field(described, "columns")?)? { + if columns.get(column).is_none() { + columns.set(column, spec.clone()); + } + } + held.set("columns", columns); + combined.set(name, held); + } + } + } + schema.set("related_tables", combined); + } + if let Some(value) = step.get("relationships") { + let mut links = schema + .get("relationships") + .map(array) + .transpose()? + .unwrap_or(&[]) + .to_vec(); + for link in array(value)? { + if !links.contains(link) { + links.push(link.clone()); + } } - .ok_or_else(|| invalid("delta rows exhausted"))?; - rows.push(Value::Array(row.to_vec())); + schema.set("relationships", Value::Array(links)); } - let mut result = table.clone(); - result.set("rows", Value::Array(rows)); - Ok(result) } - let context = field(payload, "context")?; - let mut combined = context.clone(); - combined.set( - "instance_table", - restore( - field(context, "instance_table")?, - field(delta, "instance_table")?, - )?, - ); - let original = members(field(context, "related_tables")?)?; - let deltas = members(field(delta, "related_tables")?)?; - if original.len() != deltas.len() || original.iter().zip(deltas).any(|((a, _), (b, _))| a != b) - { - return Err(CoreError::protocol( - Sentence::StateUnread, - "column context delta tables differ", - )); + let mut instance_table = field(first_context, "instance_table")?.clone(); + instance_table.set("rows", Value::Array(instances)); + let mut context = Value::Object(vec![]); + context.set("instance_table", instance_table); + if payloads.iter().any(|p| { + p.get("context") + .is_some_and(|c| c.get("related_tables").is_some()) + }) { + context.set("related_tables", Value::Object(related)); } - combined.set( - "related_tables", - Value::Object( - original - .iter() - .zip(deltas) - .map(|((name, t), (_, d))| Ok((name.clone(), restore(t, d)?))) - .collect::>()?, - ), - ); - let mut result = payload.clone(); - result.set("context", combined); - result.remove("client_analysis_delta"); - Ok(result) + let mut out = first.clone(); + out.set("schema", schema); + out.set("context", context); + Ok(out) } /// Union forecast evidence before cohort selection. Steps namespace instance identity. @@ -1205,31 +1245,134 @@ mod lifecycle_tests { sdm_wire::json::read(text.as_bytes()).unwrap() } + fn step(labels: [f64; 3], table: &str, column: &str, values: [f64; 3]) -> Value { + let instances = (0..3) + .map(|i| format!("[{i},{},{}]", i + 1, labels[i])) + .collect::>() + .join(","); + let related = (0..3) + .map(|i| format!("[{i},{},{}]", i + 1, values[i])) + .collect::>() + .join(","); + parse(&format!( + r#"{{"task":{{"kind":"regression","target":{{"column_name":"y"}}}}, + "schema":{{"instance_table":{{"columns":{{"instance_id":{{"stype":"ID"}},"ref":{{"stype":"ID"}},"y":{{"stype":"numerical"}}}}}}, + "related_tables":{{"{table}":{{"primary_key":["instance_id","id"],"columns":{{"instance_id":{{"stype":"ID"}},"id":{{"stype":"ID"}},"{column}":{{"stype":"numerical"}}}}}}}}, + "relationships":[{{"source_columns":["ref"],"target_table":"{table}","target_columns":["id"]}}]}}, + "context":{{"instance_table":{{"columns":["instance_id","ref","y"],"rows":[{instances}]}}, + "related_tables":{{"{table}":{{"columns":["instance_id","id","{column}"],"rows":[{related}]}}}}}}}}"# + )) + } + #[test] - fn delta_restores_exact_positions_with_duplicate_values_and_unicode_schema() { - let mut p = parse( - r#"{"schema":{"instance_table":{"columns":{"é\n":{"stype":"categorical"}}}},"context":{"instance_table":{"columns":["é\n"],"rows":[["kept"],["kept"]]},"related_tables":{}}}"#, - ); - let mut delta = parse( - r#"{"version":1,"instance_table":{"original_rows":5,"positions":[0,2,4],"removed":{"columns":["é\n"],"rows":[["first"],["middle"],["last"]]}},"related_tables":{}}"#, - ); - delta.set( - "schema_json", - Value::Text(encoded(p.get("schema").unwrap())), + fn forecast_analysis_covers_every_step_and_tables_only_a_later_step_reached() { + let combined = forecast(&[ + step([0.0, 10.0, 0.0], "orders", "amount", [1.0, 2.0, 3.0]), + step( + [100.0, 200.0, 300.0], + "events", + "seconds", + [10.0, 20.0, 30.0], + ), + ]) + .unwrap(); + let ColumnAnalysis::Ready { rows } = analyze(&combined).unwrap() else { + panic!() + }; + let cohorts = |column: &str| { + rows.iter() + .filter(|r| r.column == column) + .map(|r| (r.cohort.as_str(), r.mean)) + .collect::>() + }; + let counted = rows + .iter() + .filter(|r| r.table == "orders" && r.column == "COUNT(*)") + .map(|r| (r.cohort.as_str(), r.mean)) + .collect::>(); + assert_eq!(counted, [("[0 - 0]", 200.0), ("(0 - 1+]", 10.0 / 3.0)]); + assert_eq!( + cohorts("amount"), + [("[1 - 1.20]", 0.0), ("(1.20 - 2]", 10.0), ("(2 - 3]", 0.0)] ); - p.set("client_analysis_delta", delta); - let restored = restored(&p).unwrap(); assert_eq!( - restored - .get("context") - .unwrap() - .get("instance_table") - .unwrap() - .get("rows") - .unwrap(), - &parse(r#"[["first"],["kept"],["middle"],["kept"],["last"]]"#) + cohorts("seconds"), + [ + ("[10 - 12]", 100.0), + ("(12 - 20]", 200.0), + ("(20 - 30]", 300.0) + ] ); - assert!(restored.get("client_analysis_delta").is_none()); + } + + #[test] + fn calls_of_one_request_are_analysed_over_each_sent_instance_once() { + let call = |rows: &str| { + parse(&format!( + r#"{{"task":{{"kind":"regression","target":{{"column_name":"y"}}}}, + "schema":{{"instance_table":{{"columns":{{"instance_id":{{"stype":"ID"}},"__kumo_entity_ref_0":{{"stype":"ID"}},"x":{{"stype":"categorical"}},"y":{{"stype":"numerical"}}}}}}}}, + "context":{{"instance_table":{{"columns":["instance_id","__kumo_entity_ref_0","x","y"],"rows":[{rows}]}}}}}}"# + )) + }; + let first = call(r#"[0,1,"p",0],[2,7,"p",0],[3,2,"q",1],[4,3,"q",0]"#); + let second = call(r#"[0,1,"p",0],[1,5,"p",1],[3,2,"q",1],[4,3,"q",0]"#); + let ColumnAnalysis::Ready { rows } = analyze(&sent(&[first, second]).unwrap()).unwrap() + else { + panic!() + }; + let cohorts = rows + .iter() + .filter(|r| r.column == "x") + .map(|r| (r.cohort.as_str(), r.share, r.mean)) + .collect::>(); + assert_eq!(cohorts, [("p", 0.6, 1.0 / 3.0), ("q", 0.4, 0.5)]); + } + + #[test] + fn calls_that_reached_different_tables_are_analysed_together() { + let users = r#""users":{"primary_key":["instance_id","uid"],"columns":{"instance_id":{"stype":"ID"},"uid":{"stype":"ID"},"age":{"stype":"numerical"}}}"#; + let orders = |key: &str| { + format!( + r#""orders":{{"primary_key":["instance_id","oid"],"columns":{{"instance_id":{{"stype":"ID"}},"oid":{{"stype":"ID"}},"uid":{{"stype":"ID"}}{key}}}}}"# + ) + }; + let link = |table: &str, column: &str, target: &str| { + format!( + r#"{{"source_table":"{table}","source_columns":["{column}"],"target_table":"{target}","target_columns":["{column}"]}}"# + ) + }; + let call = |instances: &str, extra: &str, product: bool| { + let entity = r#"{"source_columns":["__kumo_entity_ref_0"],"target_table":"users","target_columns":["uid"]}"#; + let (key, products, product_link, columns, rows) = if product { + ( + r#","pid":{"stype":"ID"}"#, + r#","products":{"primary_key":["instance_id","pid"],"columns":{"instance_id":{"stype":"ID"},"pid":{"stype":"ID"},"price":{"stype":"numerical"}}}"#, + format!(",{}", link("orders", "pid", "products")), + r#"["instance_id","oid","uid","pid"]"#, + r#","products":{"columns":["instance_id","pid","price"],"rows":[[1,9,5.0]]}"#, + ) + } else { + ("", "", String::new(), r#"["instance_id","oid","uid"]"#, "") + }; + let order_rows = if product { "[[1,8,2,9]]" } else { "[[0,7,1]]" }; + parse(&format!( + r#"{{"task":{{"kind":"regression","target":{{"column_name":"y"}}}}, + "schema":{{"instance_table":{{"columns":{{"instance_id":{{"stype":"ID"}},"__kumo_entity_ref_0":{{"stype":"ID"}},"y":{{"stype":"numerical"}}}}}}, + "related_tables":{{{users},{}{products}}}, + "relationships":[{entity},{}{product_link}]}}, + "context":{{"instance_table":{{"columns":["instance_id","__kumo_entity_ref_0","y"],"rows":[{instances}]}}, + "related_tables":{{"users":{{"columns":["instance_id","uid","age"],"rows":[{extra}]}}, + "orders":{{"columns":{columns},"rows":{order_rows}}}{rows}}}}}}}"#, + orders(key), + link("orders", "uid", "users"), + )) + }; + let first = call("[0,1,0],[1,2,1]", "[0,1,30],[1,2,40]", false); + let second = call("[0,1,0],[1,2,1]", "[0,1,30],[1,2,40]", true); + assert!(matches!( + analyze(&sent(&[first, second]).unwrap()).unwrap(), + ColumnAnalysis::Ready { .. } + )); } #[test] diff --git a/rust/crates/sdm-request/src/payload.rs b/rust/crates/sdm-request/src/payload.rs index 31df574d7..dff2002d4 100644 --- a/rust/crates/sdm-request/src/payload.rs +++ b/rust/crates/sdm-request/src/payload.rs @@ -515,24 +515,6 @@ fn cell(array: &ArrayRef, row: usize, column: &str, dtype: Dtype) -> Result Result { - let value = match cell { - Value::Number(token) => token - .parse::() - .map_err(|_| internal("an integer key is not an integer"))? - as f64, - Value::Text(digits) => digits - .parse::() - .map_err(|_| internal("an integer key's digits do not read"))? - as f64, - _ => return Err(internal("an integer key is not an integer")), - }; - Value::float(value).ok_or_else(|| internal("an integer key is not finite")) -} - /// `prefix`, or `prefix_1`, `prefix_2` and on, whichever `occupied` does not /// hold first. fn unique_name(occupied: &[String], prefix: &str) -> String { @@ -1242,8 +1224,8 @@ fn build_mode( // Each foreign key once, merging every partial link row by row. The // first partial map supplying a sampled source row gives its key, // preserving last-valid-pair precedence within each map. Only a row - // no link supplies is null. An integer key column holding a - // null, or a float32 key's column, is written as float64, as pandas + // no link supplies is null. An integer key keeps its exact value beside + // the nulls; a float32 key's column is written as float64, as pandas // types such a column. let mut written: Vec<(&str, &str, &str)> = Vec::new(); for link in &sampled.links { @@ -1282,21 +1264,16 @@ fn build_mode( if values.iter().all(Option::is_none) { continue; } - let as_float = key_dtype == Dtype::Int64 && values.iter().any(Option::is_none); let cells = values .into_iter() - .map(|v| match v { - Some(cell) if as_float => float_key(&cell), - Some(cell) => Ok(cell), - None => Ok(Value::Null), - }) - .collect::>()?; + .map(|v| v.unwrap_or(Value::Null)) + .collect(); // A float32 key's foreign key is a float64, as pandas types the // column the reference fills with the keys as Python floats ( // each value is written widened either way). let column = FrameColumn { name: link.fkey.clone(), - dtype: if as_float || key_dtype == Dtype::Float32 { + dtype: if key_dtype == Dtype::Float32 { Dtype::Float64 } else { key_dtype @@ -1317,6 +1294,15 @@ fn build_mode( }); written.push(edge); } + // A foreign key whose link was not written points at a table outside + // the sample. It is no feature, and is not sent. + for (_, frame, key, stypes) in &mut frames { + frame.columns.retain(|column| { + column.name == INSTANCE_ID + || column.name == *key + || stypes.iter().any(|(name, _)| *name == column.name) + }); + } let num_train = sampled.num_train; let mut instance_stypes = vec![ @@ -1397,41 +1383,6 @@ fn build_mode( } }; - // Retain only leakage-excluded rows. Surviving rows already live in context. - // Positions restore original order; identity deduplication must not reorder rows. - let removed = |frame: &Frame, batch: &[usize]| -> sdm_wire::payload::RemovedRows { - let original: Vec = batch - .iter() - .enumerate() - .filter_map(|(r, &instance)| (instance < num_train).then_some(r)) - .collect(); - let positions = original - .iter() - .enumerate() - .filter_map(|(at, &r)| answered.contains(&batch[r]).then_some(at)) - .collect(); - sdm_wire::payload::RemovedRows { - original_rows: original.len(), - positions, - removed: frame.table( - |r| batch[r] < num_train && answered.contains(&batch[r]), - None, - ), - } - }; - let client_analysis_delta = (!answered.is_empty()).then(|| sdm_wire::payload::AnalysisDelta { - version: 1, - schema_json: String::new(), // Filled after schema assembly below. - instance_table: removed(&instance, &(0..instance.rows).collect::>()), - related_tables: Named( - frames - .iter() - .zip(&sampled.tables) - .map(|((name, frame, ..), table)| (name.clone(), removed(frame, &table.batch))) - .collect(), - ), - }); - let mut fields: Vec = if link.is_some() { vec!["rankings".into()] } else { @@ -1483,10 +1434,6 @@ fn build_mode( fields, prediction_statistic: sampled.statistic.clone(), }; - let client_analysis_delta = client_analysis_delta.map(|mut delta| { - delta.schema_json = serde_json::to_string(&schema).expect("typed schema serializes"); - delta - }); let metadata = Metadata { source_query: (!sampled.source_query.is_empty()).then(|| sampled.source_query.clone()), num_context_examples: num_train as u64, @@ -1506,7 +1453,7 @@ fn build_mode( }; let bytes = if link.is_some() { StatePayload { - client_analysis_delta: client_analysis_delta.clone(), + client_analysis_delta: None, client_generated_keys, model, task: sdm_wire::payload::TemporalLinkTask { @@ -1529,7 +1476,7 @@ fn build_mode( .to_bytes() } else if kind.is_classification() { StatePayload { - client_analysis_delta: client_analysis_delta.clone(), + client_analysis_delta: None, client_generated_keys, model, task: task!(ClassTarget { @@ -1548,7 +1495,7 @@ fn build_mode( .to_bytes() } else { StatePayload { - client_analysis_delta: client_analysis_delta.clone(), + client_analysis_delta: None, client_generated_keys, model, task: task!(Target { @@ -1571,14 +1518,7 @@ fn build_mode( // most rows a call of the request scores (`smaller_calls`); a call of // one row cannot, and only a narrower context helps // (`no_smaller_call`). - let analysis_bytes = client_analysis_delta.as_ref().map_or(0, |delta| { - serde_json::to_vec(delta) - .expect("typed delta serializes") - .len() - + br#""client_analysis_delta":"#.len() - + 1 - }); - let model_bytes = bytes.len() - client_only_bytes - analysis_bytes; + let model_bytes = bytes.len() - client_only_bytes; if model_bytes > PAYLOAD_BYTES { let count = |n: usize| Detail::Scalar(sdm_contract::Scalar::UInt64(n as u64)); return Err(CoreError::new( diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index a4477af21..be968b948 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -399,11 +399,14 @@ pub fn prepare( RELATIONAL => &KUMO_RELATIONAL, TABULAR => &KUMO_TABULAR, _ => { + let mut known: Vec<_> = MODELS.iter().map(|spec| spec.id).collect(); + known.sort_unstable(); return Err(CoreError::new( - Sentence::UnknownModel, + Sentence::DirectUnknownModel, "the model named is not one the contract carries", ) - .with_field("field", Detail::text("model"))); + .with_field("model", Detail::text(model)) + .with_field("known_models", texts(known))); } }; let kind = kind_of(payload)?; diff --git a/rust/crates/sdm-request/src/result.rs b/rust/crates/sdm-request/src/result.rs index 9985dc9ee..80e53d089 100644 --- a/rust/crates/sdm-request/src/result.rs +++ b/rust/crates/sdm-request/src/result.rs @@ -82,6 +82,20 @@ pub(crate) fn candidate_key(value: &C) -> Result, CoreError> { } } C::Bytes { value } => framed("b", value.iter().map(|v| v.to_string())), + C::Decimal { value } => { + let digits = value.strip_prefix('-').unwrap_or(value); + let (whole, fraction) = digits.split_once('.').unwrap_or((digits, "")); + if whole.is_empty() + || !whole + .bytes() + .chain(fraction.bytes()) + .all(|v| v.is_ascii_digit()) + || (digits.contains('.') && fraction.is_empty()) + { + return Err(invalid("invalid decimal candidate")); + } + key::exact(value) + } C::Date { ordinal } => format!("d{ordinal}"), C::DateTime { ticks, unit, aware } => { // Decimal placement is exact and never casts through nanoseconds. @@ -231,6 +245,7 @@ pub fn plan(parts: &[MergeDescriptor]) -> Result { } let mut seen = BTreeSet::new(); let mut offset = 0u32; + let mut keys = Vec::new(); for (part_index, part) in parts.iter().enumerate() { if plan.explanation_part.is_none() && part.has_explanation { plan.explanation_part = Some(part_index); @@ -238,13 +253,16 @@ pub fn plan(parts: &[MergeDescriptor]) -> Result { plan.names_rows &= part.names_rows; plan.names_step |= part.names_step; for candidate in &part.candidates { - if candidate_key(candidate)?.is_none_or(|key| seen.insert(key)) { + let key = candidate_key(candidate)?; + if key.as_ref().is_none_or(|key| !seen.contains(key)) { plan.candidate_positions.push(offset); } + keys.extend(key); offset = offset .checked_add(1) .ok_or_else(|| invalid("too many result candidates"))?; } + seen.extend(keys.drain(..)); } Ok(plan) } @@ -282,12 +300,31 @@ mod tests { descriptor(vec![json("1"), json("NaN"), json("NaN")]), ]) .unwrap(); - assert_eq!(plan.candidate_positions, [0, 3, 4, 6, 7]); + assert_eq!(plan.candidate_positions, [0, 1, 2, 3, 4, 6, 7]); let a = json("{\"a\":[1,null],\"b\":2}"); let b = json("{\"b\":2.0,\"a\":[true,null]}"); assert_eq!(candidate_key(&a).unwrap(), candidate_key(&b).unwrap()); } #[test] + fn decimals_key_by_exact_value() { + let decimal = |v: &str| C::Decimal { value: v.into() }; + assert_eq!( + candidate_key(&decimal("-1.50")).unwrap(), + candidate_key(&json("-1.5")).unwrap() + ); + assert_eq!( + candidate_key(&decimal("1.000")).unwrap(), + candidate_key(&json("true")).unwrap() + ); + assert_ne!( + candidate_key(&decimal("0.1")).unwrap(), + candidate_key(&json("0.1")).unwrap() + ); + for invalid in ["", "-", "1.", ".5", "1e3", "1.2.3"] { + assert!(candidate_key(&decimal(invalid)).is_err(), "{invalid}"); + } + } + #[test] fn mixed_resolution_times_do_not_overflow_or_conflate_dates() { let a = C::DateTime { ticks: "253402300799999999".into(), diff --git a/rust/crates/sdm-request/tests/payload_lock.rs b/rust/crates/sdm-request/tests/payload_lock.rs index 9be1743fa..72da4d2ed 100644 --- a/rust/crates/sdm-request/tests/payload_lock.rs +++ b/rust/crates/sdm-request/tests/payload_lock.rs @@ -641,14 +641,62 @@ fn empty_and_invalid_partial_maps_do_not_hide_later_valid_rows() { }, ]; let payload = built(&sampled); - assert_eq!(foreign_keys(&payload, "context"), json!([null, 0.0, null])); - assert_eq!(foreign_keys(&payload, "predict"), json!([3.0])); + assert_eq!(foreign_keys(&payload, "context"), json!([null, 0, null])); + assert_eq!(foreign_keys(&payload, "predict"), json!([3])); + assert_eq!( + payload["schema"]["related_tables"]["orders"]["columns"]["user_id"]["dtype"], + json!("int64") + ); +} + +#[test] +fn a_foreign_key_beside_nulls_keeps_every_digit_of_a_large_key() { + let big = 1_i64 << 60; + let mut sampled = shop(); + sampled.tables[0].columns[0].1 = array(Int64Array::from( + (0..5).map(|i| big + i).collect::>(), + )); + let edge = sampled.links[0].clone(); + sampled.links = vec![SampledLink { + pairs: vec![(1, 1), (2, 2), (3, 3)], + ..edge + }]; + let payload = built(&sampled); + assert_eq!( + foreign_keys(&payload, "context"), + json!([null, (big + 1).to_string(), (big + 2).to_string()]) + ); + assert_eq!( + foreign_keys(&payload, "predict"), + json!([(big + 3).to_string()]) + ); assert_eq!( payload["schema"]["related_tables"]["orders"]["columns"]["user_id"]["dtype"], - json!("float64") + json!("int64") ); } +#[test] +fn a_foreign_key_to_a_table_outside_the_sample_is_not_sent() { + let mut sampled = shop(); + sampled.tables[1] + .columns + .push(("item_id".into(), array(Int64Array::from(vec![5, 6, 5, 6])))); + let payload = built(&sampled); + for half in ["context", "predict"] { + let columns = payload[half]["related_tables"]["orders"]["columns"] + .as_array() + .unwrap(); + assert!(!columns.contains(&json!("item_id")), "{half}"); + } + assert!( + payload["schema"]["related_tables"]["orders"]["columns"] + .get("item_id") + .is_none() + ); + assert_eq!(check("StatePayload", &payload), Ok(())); +} + fn foreign_keys(payload: &Value, half: &str) -> Value { let table = &payload[half]["related_tables"]["orders"]; let at = table["columns"] diff --git a/skills/README.md b/skills/README.md index 349e8df4d..55772f68e 100644 --- a/skills/README.md +++ b/skills/README.md @@ -70,7 +70,6 @@ code is right and the skill is stale: say so rather than working around it. ## What is not here - Training, fine-tuning, checkpoints, model registries. The models do not train. -- `predictive_query`, the parser package behind `PredictiveQuery`. It is - internal, as [`docs/reference/stability.md`](../docs/reference/stability.md) - says. Write the query as a string. +- The parser behind `PredictiveQuery`. It is part of the client's Rust core + and has no Python API of its own. Write the query as a string. - Deployment, authentication and endpoint operation. A skill assumes a URL. diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 1ab1a346e..0f05a2672 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -62,14 +62,12 @@ The task is not an argument. It follows from the query: see ## The whole interface -`KumoRelational` needs the engine, which is an extra: +`KumoRelational` needs nothing beyond the client: ``` -pip install 'structured-data-models-client[relational]' +pip install structured-data-models-client ``` -Without it the first call raises `MissingExtraError` naming that command. - ```python import pandas as pd from sdm_client import ( diff --git a/skills/relational/references/warehouses.md b/skills/relational/references/warehouses.md index cb154cb51..42648b22a 100644 --- a/skills/relational/references/warehouses.md +++ b/skills/relational/references/warehouses.md @@ -24,9 +24,8 @@ connection its own driver makes, and says so if given another. | `DuckDBSource` | `duckdb.connect()` | — | `duckdb` | | `SQLiteSource` | `adbc_driver_sqlite.dbapi.connect()` | — | `sqlite` | -Install the connector **and** the engine, since `Schema.from_source` needs it: -`pip install 'structured-data-models-client[relational,databricks]'`. With the connector alone the -first call raises `MissingExtraError`. +Install the connector's extra, for instance +`pip install 'structured-data-models-client[databricks]'`. `Source(connection)` picks the right one for you when you would rather not name it. Lakebase is `PostgresSource`, with the `lakebase` extra and diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index b795ebb9a..4eeaf3aa6 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -2978,7 +2978,7 @@ "tests": { "python": [ "packages/structured-data-models-client/tests/test_result_merge_core.py::test_merge_preserves_owners_without_row_or_metadata_serialization", - "packages/structured-data-models-client/tests/test_result_merge_core.py::test_stable_union_deduplicates_within_parts_and_keeps_exact_numbers", + "packages/structured-data-models-client/tests/test_result_merge_core.py::test_union_keeps_repeats_within_a_part_and_drops_earlier_values", "packages/structured-data-models-client/tests/test_result_merge_core.py::test_nan_values_remain_distinct_under_ieee_equality", "packages/structured-data-models-client/tests/test_result_merge_core.py::test_temporal_resolution_and_original_objects_survive", "packages/structured-data-models-client/tests/test_result_merge_core.py::test_container_candidates_keep_list_tuple_and_bytes_domains", @@ -4694,8 +4694,7 @@ "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_empty_labels_and_exact_mixed_classes", "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_unlabelled_related_rows_do_not_break_analysis", "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_public_merge_retains_equal_analysis_and_refuses_different_analysis", - "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_materialize_pickle_predict_restores_pretrim_context_without_sending_delta", - "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_delta_corruption_is_not_silently_an_omitted_analysis", + "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_materialize_pickle_predict_analyses_the_rows_it_sends", "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_unavailable_statistics_warn_without_losing_predictions", "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_public_policy_constants_remain_available", "packages/structured-data-models-client/tests/test_column_analysis_core.py::test_sqlite_materialize_explain_uses_sampled_context_without_new_queries", @@ -7819,6 +7818,15 @@ } ] }, + "com.nvidia.sdm.ColumnExplanations$Row#public com.nvidia.sdm.ColumnExplanations$Row(java.lang.String, long, java.lang.String, java.lang.String, java.lang.String, java.lang.Object, double, double);": { + "capability": "column-explanations", + "assertions": [ + { + "test": "java/src/test/java/com/nvidia/sdm/ColumnExplanationsTest.java#viewsAreDetachedAndClosingArrowDoesNotCloseCallerAllocator", + "behavior": "Typed analysis rows and detached Arrow views retain their values and caller-owned allocator lifetime." + } + ] + }, "com.nvidia.sdm.ColumnExplanations$Row#public double mean();": { "capability": "column-explanations", "assertions": [ @@ -8233,6 +8241,21 @@ "base_commit": "ebc5746391d977fd8d9d779423db33ae547d64ee", "record": "type_contracts/arrow-admission-and-inference", "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." + }, + { + "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "record": "api_contracts/results-merge", + "reason": "PredictionResult.merge keeps a candidate repeated within one part and compares Decimal candidates by value, as 1.0.0 did; the merge test is renamed for the restored behaviour." + }, + { + "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "record": "api_inventory", + "reason": "ColumnExplanations.Row gains a constructor without a score, as the column analysis on main no longer computes one; the scored constructor and score() remain, deprecated." + }, + { + "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "record": "api_contracts/column-explanations", + "reason": "Column analysis reads the labelled rows a request sends, as on main, so the request keeps no pre-trim delta and its tests are replaced by one over the sent rows." } ], "capabilities": [ diff --git a/tests/test_java_binding_ci.py b/tests/test_java_binding_ci.py index 1c80fe92a..750e4a5f4 100644 --- a/tests/test_java_binding_ci.py +++ b/tests/test_java_binding_ci.py @@ -6,14 +6,11 @@ from __future__ import annotations -import hashlib import itertools -import json import re from pathlib import Path from typing import Any -import pytest import yaml ROOT = Path(__file__).resolve().parents[1] @@ -218,56 +215,6 @@ def test_windows_java_builds_are_packaged_and_loaded(): assert "separator=';'" in commands -@pytest.mark.parametrize( - 'corruption', [None, 'commit', 'digest', 'missing', 'version'] -) -def test_java_release_rejects_mismatched_artifacts( - tmp_path, monkeypatch, corruption -): - document = yaml.safe_load( - (ROOT / '.github/workflows/publish.yml').read_text() - ) - step = next( - step - for step in document['jobs']['build']['steps'] - if step.get('name') == 'Verify and assemble the tested Java artifacts' - ) - code = step['run'].split("python - <<'PY'\n", 1)[1].rsplit('\nPY', 1)[0] - monkeypatch.chdir(tmp_path) - monkeypatch.setenv('GITHUB_SHA', 'candidate') - monkeypatch.setenv('GITHUB_REF_NAME', 'v2.3.4') - root = tmp_path / 'java-release' - root.mkdir() - (tmp_path / 'dist').mkdir() - stem = 'structured-data-models-client-2.3.4' - jar = root / (stem + '.jar') - jar.write_bytes(b'tested artifact fixture') - pom = root / (stem + '.pom') - version = '2.3.5' if corruption == 'version' else '2.3.4' - pom.write_text( - f'{version}' - ) - files = { - p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in [jar, pom] - } - if corruption == 'digest': - jar.write_bytes(b'changed after testing') - if corruption == 'missing': - del files[pom.name] - record = { - 'commit': 'other' if corruption == 'commit' else 'candidate', - 'files': files, - } - (root / 'java-artifacts.json').write_text(json.dumps(record)) - if corruption: - with pytest.raises(AssertionError): - exec(compile(code, '', 'exec'), {}) - else: - exec(compile(code, '', 'exec'), {}) - assert (tmp_path / 'dist' / jar.name).read_bytes() == jar.read_bytes() - assert (tmp_path / 'dist' / pom.name).read_bytes() == pom.read_bytes() - - def test_java_release_paths_follow_the_pom_version(): ci = CI.read_text() live = (ROOT / '.github/workflows/parity-live.yml').read_text() diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index ff446a1dd..945f48d8f 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -996,6 +996,20 @@ def test_the_root_license_points_to_native_attribution(self): assert path in (REPO / 'LICENSE').read_text() assert (REPO / path).is_file() + @pytest.mark.parametrize('name', ['NOTICE', 'LICENSE']) + def test_the_client_wheel_points_to_what_it_bundles(self, name): + client = PACKAGES / 'structured-data-models-client' + text = (client / name).read_text() + for path in ( + 'THIRD-PARTY-NOTICES-RUST.txt', + 'sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt', + ): + assert path in text + assert (client / 'THIRD-PARTY-NOTICES-RUST.txt').is_file() + assert ( + client / 'src/sdm_client/assets/THIRD-PARTY-NOTICES-MERMAID.txt' + ).is_file() + def test_the_readme_links_every_package_license(self): """A reader looking for the terms of one package should not have to guess that each directory holds its own copy. @@ -1141,6 +1155,16 @@ def test_the_native_module_is_versioned_with_the_five(self): assert client['build-system']['build-backend'] == 'maturin' assert client['project']['dynamic'] == ['version'] + def test_the_java_library_is_versioned_with_the_five(self): + import xml.etree.ElementTree as ET + + pom = ET.parse(REPO / 'java/pom.xml').getroot() + version = pom.findtext('{http://maven.apache.org/POM/4.0.0}version') + assert version == self._declared_version(), ( + f'java/pom.xml is at {version}, the five at ' + f'{self._declared_version()}' + ) + def test_every_floor_on_frames_and_arrow_is_the_supported_one(self): """pandas, NumPy and pyarrow carry one floor wherever they are named. diff --git a/tests/test_wheel_provenance.py b/tests/test_wheel_provenance.py index 9457c24ca..dfde3379d 100644 --- a/tests/test_wheel_provenance.py +++ b/tests/test_wheel_provenance.py @@ -193,3 +193,53 @@ def test_workflows_check_after_the_final_dependency_install(): 'check_wheel_provenance.py --write-version dist/VERSION dist/*.whl' in build ) + + +@pytest.mark.parametrize( + ('tag', 'accepted'), + [ + ('v2.0.0', True), + ('v2.0.0rc1', True), + ('v2.0.0a1', True), + ('v2.0.0b12', True), + ('v2.0.0-rc.1', False), + ('v2.0.0rc', False), + ('vv2.0.0', False), + ], +) +def test_the_release_accepts_final_and_pre_release_tags( + tmp_path, monkeypatch, tag, accepted +): + import re + + import yaml + + publish = yaml.safe_load( + (ROOT / '.github/workflows/publish.yml').read_text() + ) + filters = publish[True]['push']['tags'] + pushed = any(re.fullmatch(f.replace('.', r'\.'), tag) for f in filters) + step = next( + step + for step in publish['jobs']['assemble']['steps'] + if step.get('name') + == 'The tag must match what the distributions declare' + ) + code = step['run'].split("python - <<'EOF'\n", 1)[1].rsplit('\nEOF', 1)[0] + version = tag.removeprefix('v') + for package in PACKAGES: + (tmp_path / 'packages' / package).mkdir(parents=True) + (tmp_path / 'packages' / package / 'pyproject.toml').touch() + (tmp_path / 'dist').mkdir() + for package in PACKAGES: + name = package.replace('-', '_') + (tmp_path / 'dist' / f'{name}-{version}-py3-none-any.whl').touch() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv('GITHUB_REF_NAME', tag) + try: + exec(compile(code, '', 'exec'), {}) + except SystemExit: + checked = False + else: + checked = True + assert (pushed, checked) == (accepted, accepted) From 2f59ae2485385d78bf85645584247a2a78af23e6 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 16:32:49 -0800 Subject: [PATCH 02/24] Remove the empty env block from the rust-lint job The squash left the rust-lint job with an env key and no value, so GitHub rejects the workflow and no CI job runs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2aaad6ea..3d56368cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,6 @@ jobs: defaults: run: working-directory: rust - env: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: From f241e1bec74589ae46968510198fcc22a8878e8e Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 16:21:20 -0800 Subject: [PATCH 03/24] Take the thrift fix the connector now allows databricks-sql-connector pinned thrift<0.25.0, which held thrift at 0.24.0 and left its advisory open past the 2026-11-01 remediation date. 4.8.0 relaxed that bound to <0.26.0, so the fix is available by resolving rather than by waiting. pyproject.toml already allows it: the constraint is >=4.1.3 and nothing capped the connector. Only uv.lock was stale. Targeted upgrade of the two packages, nothing else. 149 packages before and after; the only versions that move are databricks-sql-connector 4.6.0 to 4.9.0 and thrift 0.24.0 to 0.25.0. Signed-off-by: Manush Murali Co-Authored-By: Claude Opus 5 --- uv.lock | 235 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 120 insertions(+), 115 deletions(-) diff --git a/uv.lock b/uv.lock index b49cdaad5..35aa54053 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -335,8 +335,8 @@ name = "anyio" version = "4.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.15'" }, + { name = "idna", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11' and python_full_version < '3.15' and sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/98/7e457a3b014976887a050561955c0b76e1e7e2d6e27b78ecef2ef80518f2/anyio-4.16.0.tar.gz", hash = "sha256:59bb03251907983d1603d27c3087ddb1e93299ea2163adcdba37899a4fb12ffb", size = 310971, upload-time = "2026-11-01T11:20:07.282Z" } wheels = [ @@ -814,7 +814,7 @@ name = "cuda-bindings" version = "13.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9b/1e/d90a8af3d589776c70e92d43d7a0234260709a1d552d105f4dd3bdf48fa3/cuda_bindings-13.4.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90f85a00fc89dd4a8f7c4beb0a2f3f788b845c8496fe84adc1217830206db9e0", size = 6512106, upload-time = "2026-10-14T12:35:04.07Z" }, @@ -853,40 +853,40 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(python_full_version >= '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -906,7 +906,7 @@ wheels = [ [[package]] name = "databricks-sql-connector" -version = "4.6.0" +version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lz4" }, @@ -921,9 +921,9 @@ dependencies = [ { name = "thrift" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/48/13691ccdf268aaddea0d981563616ad5f0811d8626a45a3d061de50a9174/databricks_sql_connector-4.6.0.tar.gz", hash = "sha256:2c3fdb28a08b7e9d4c10775f88aa7d6cf8212a1f88f5baacb05f294ee148c1e5", size = 244176, upload-time = "2026-09-24T18:30:55.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/0b/300b03d183c458384ef890a86347626acc8621140d760848dd915f37c950/databricks_sql_connector-4.9.0.tar.gz", hash = "sha256:e440c671a9f27a72e13d2e1ad88d0c6a25d176aa4528b1b422753b065266a81c", size = 247223, upload-time = "2026-11-04T23:32:24.719Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/4f/198110284fca731db1b7876b206138ee81b92bbf8d10f4c024cfd00e8b7d/databricks_sql_connector-4.6.0-py3-none-any.whl", hash = "sha256:9f2128592052d3aadb7a3c46df0a8ef414402ec6ea7a0fdb8490da737fe43cdc", size = 270341, upload-time = "2026-09-24T18:30:53.306Z" }, + { url = "https://files.pythonhosted.org/packages/0a/87/3df7d5b18600adfd331d308e69de8f9676039e618e2dfc8da715eaf08e24/databricks_sql_connector-4.9.0-py3-none-any.whl", hash = "sha256:3b96fe42a392150ef62e1c5b27c678a1f4c200a061f2cf7a13380b411a457425", size = 273228, upload-time = "2026-11-04T23:32:23.285Z" }, ] [[package]] @@ -1035,7 +1035,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1281,8 +1281,8 @@ name = "httpcore2" version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, + { name = "truststore", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/61/afa9d6b7452f9eca73f73d45324eb0bb52f098fd5e6049bf6b8263be9b4e/httpcore2-2.16.0.tar.gz", hash = "sha256:c72cd0413563b17e1cd8354677dff887466f8ec2551679759e93126534428643", size = 69886, upload-time = "2026-10-21T07:09:45.31Z" } wheels = [ @@ -1294,12 +1294,12 @@ name = "httpx2" version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform != 'emscripten'" }, - { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "anyio", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, - { name = "idna" }, - { name = "truststore", marker = "sys_platform != 'emscripten'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "idna", marker = "python_full_version >= '3.11'" }, + { name = "truststore", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/73/5ca2cedf5e4e50423d1c258877865a6b62793633b561c58499f8aeb33b7e/httpx2-2.16.0.tar.gz", hash = "sha256:0d54c98128e8f2ff7fbbde0fca8c359e5f032edeaaaac443cdb7f31f685bcd7f", size = 103116, upload-time = "2026-10-21T07:09:46.348Z" } wheels = [ @@ -1320,15 +1320,15 @@ name = "huggingface-hub" version = "2.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'ARM64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx2" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "click", marker = "python_full_version >= '3.11'" }, + { name = "filelock", marker = "python_full_version >= '3.11'" }, + { name = "fsspec", marker = "python_full_version >= '3.11'" }, + { name = "hf-xet", marker = "(python_full_version >= '3.11' and platform_machine == 'AMD64') or (python_full_version >= '3.11' and platform_machine == 'ARM64') or (python_full_version >= '3.11' and platform_machine == 'aarch64') or (python_full_version >= '3.11' and platform_machine == 'amd64') or (python_full_version >= '3.11' and platform_machine == 'arm64') or (python_full_version >= '3.11' and platform_machine == 'x86_64')" }, + { name = "httpx2", marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pyyaml", marker = "python_full_version >= '3.11'" }, + { name = "tqdm", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/3b/2fcef1eabe52aec90fc8cd253cc154da1d02a446d7bbcdec59b4a51d4be7/huggingface_hub-2.4.0.tar.gz", hash = "sha256:13a6158b9e3393d9d3f248ce99074d50c2a0a1449fb9d8e2f710751e2084c188", size = 1066194, upload-time = "2026-10-29T15:30:12.321Z" } wheels = [ @@ -2057,7 +2057,7 @@ name = "nvidia-cublas" version = "13.4.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d0/a0/5cd9d4b08a8320f7ca150ee0d66c97e98092922e338336cb4b62da7500c3/nvidia_cublas-13.4.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4eb1472bf92b741e7cf24da7978dde251b58ddec2092d9f6a04f3623eb4e9a15", size = 513048232, upload-time = "2026-07-16T09:45:07.751Z" }, @@ -2105,7 +2105,7 @@ name = "nvidia-cudnn-cu13" version = "9.26.0.51" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c5/1c/01e1dd3dec6ffe0baaff7e722b8946d867353d05a01301f128f1730fde0b/nvidia_cudnn_cu13-9.26.0.51-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8e5b4f5eccff721416f70ecf4353172fc4d5cdd8ca869fe6f983a4a5b0147b20", size = 595329497, upload-time = "2026-09-10T16:45:39.945Z" }, @@ -2117,7 +2117,7 @@ name = "nvidia-cufft" version = "12.2.0.57" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/07/8d/f650e6d527f9d22b5ded906c000eaebb037721c06d4ecd906df62f300979/nvidia_cufft-12.2.0.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ed438568ae1088ba2c25789d10205d4104b8b140b7bd942e39a787947a0fb74", size = 218244742, upload-time = "2026-07-16T09:47:08.76Z" }, @@ -2138,9 +2138,9 @@ name = "nvidia-cusolver" version = "12.2.0.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/df/9e6b603f1d64040ce3e076e9bce1b78e2d588d0aabc9783f351cecd861ab/nvidia_cusolver-12.2.0.11-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8be097fca28de6be0990ab216ee9b804dd51077e1d1d83acf6c9850650bdc64d", size = 250383075, upload-time = "2026-07-16T09:50:16.355Z" }, @@ -2152,7 +2152,7 @@ name = "nvidia-cusparse" version = "12.7.10.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/35/abbd278fb6d039fcfb5175efa10a21007a11878bec59920d84854b47d766/nvidia_cusparse-12.7.10.12-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:597eff2cd83e3ab1d1e3a0dce9c9c879d77cf64ec2659bfdb5af6a5d02e7aaa9", size = 169043093, upload-time = "2026-07-16T09:51:37.058Z" }, @@ -2191,7 +2191,7 @@ name = "nvidia-nvshmem-cu13" version = "3.7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-cccl" }, + { name = "nvidia-cuda-cccl", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/81/11/1767ef120299972822a43aceffbc81ac813aad2089d817e36e4e12aa4c9c/nvidia_nvshmem_cu13-3.7.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0e748308b9e0a8a8d81859353f3e6c98fbe85a85f652a915a2ba60149a0bd124", size = 135174619, upload-time = "2026-07-17T19:24:15.404Z" }, @@ -2282,10 +2282,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "pytz", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "tzdata", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2354,9 +2354,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } wheels = [ @@ -3796,12 +3796,12 @@ name = "structured-data-models" version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "pyarrow" }, - { name = "safetensors" }, - { name = "torch" }, - { name = "typing-extensions" }, + { name = "huggingface-hub", marker = "python_full_version >= '3.11'" }, + { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "pyarrow", marker = "python_full_version >= '3.11'" }, + { name = "safetensors", marker = "python_full_version >= '3.11'" }, + { name = "torch", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/97/33/cb15b007fc67fe53c9bff242e1ca9b7d57229ac39f82e3081db828405d1b/structured_data_models-0.2.1.tar.gz", hash = "sha256:035ead2627dff755987287fd13f7864b3a7be2c4b0bdc060a94add8d93c29a26", size = 2220106, upload-time = "2026-10-14T17:12:56.009Z" } wheels = [ @@ -3999,45 +3999,50 @@ wheels = [ [[package]] name = "thrift" -version = "0.24.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/bd/8f90501b11206e545da3343ed0e5740fc694be24a5f637d7dd7e4e3af927/thrift-0.24.0.tar.gz", hash = "sha256:9ef601c49e988475ff0e741d8e1b45feec23b48514e524341efc274191f1789c", size = 69228, upload-time = "2026-07-11T16:53:50.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/0b/5a7e760e711a4103557855613ae929c47a3949852cbcb403b7c21e0848fc/thrift-0.24.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efe85c4508adaf6c9e6f7fac2b1c3c9beb4b39b19c375519966335a70b28e64a", size = 184200, upload-time = "2026-07-11T16:53:05.385Z" }, - { url = "https://files.pythonhosted.org/packages/ec/57/1af719a59c1e75ff25a8f917c4111e7f9c17f2a58840d44f697e11a8066d/thrift-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2baaec0c5cd7ba3eace54b26d81fdf0f5a85010468687cf4a131871d65abfed", size = 183616, upload-time = "2026-07-11T16:53:06.578Z" }, - { url = "https://files.pythonhosted.org/packages/9a/51/5ec88cb4e7f0818e5048bb50c36a342675a40f381d8f53a62f588fc71de5/thrift-0.24.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cad27a826e86739a79a327e6afae5a9bea36aeed2989c4318fc2943b6cfad095", size = 488814, upload-time = "2026-07-11T16:53:07.711Z" }, - { url = "https://files.pythonhosted.org/packages/18/e7/3eb3b653f3eff1d14fe3c3ed17cab8d8ae1a0bd44f486fe7c667afe88f5d/thrift-0.24.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e26ca0f346e5ec2b6be9778573fb2e9d8b3eb162dbce94e61906176158b448b", size = 494575, upload-time = "2026-07-11T16:53:09.013Z" }, - { url = "https://files.pythonhosted.org/packages/2e/cc/d55245458f990b4f440b71012f355d0cdcac294d7d949d1340c42973826e/thrift-0.24.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:498ae392090d7ae17ab59ebd8dfabda76528eeb22d3890a57c9f226003d7ed6e", size = 1434147, upload-time = "2026-07-11T16:53:10.195Z" }, - { url = "https://files.pythonhosted.org/packages/54/35/505418d1797ba7940b5ea65d96e88c7e702ab2f3cbc64007c3f14f868aa5/thrift-0.24.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:acbd02baacfa0d2017f85b189ed69cd6baa522785d1ad86476eb7ce8cd16d063", size = 1492591, upload-time = "2026-07-11T16:53:11.519Z" }, - { url = "https://files.pythonhosted.org/packages/e7/a1/882c647a219ff036fb73bf51cec9e1f432e34a687cb92f747ff51ab7d7c4/thrift-0.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:887a10d718d85275da70fe4e9d4740268ae2854f205fb5869909d24cc5576b79", size = 369174, upload-time = "2026-07-11T16:53:13.287Z" }, - { url = "https://files.pythonhosted.org/packages/2a/3b/7ba9f07a7ac7b98dd7f29f2d9e6f6e98e332942dc8dbd82b71abbac1233d/thrift-0.24.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:873c93d8496cf71589efd3128ddc5af0be350e6c1a0e615475d840eaa54f6124", size = 228606, upload-time = "2026-07-11T16:53:14.788Z" }, - { url = "https://files.pythonhosted.org/packages/a9/8c/4d0d03b40063504e75c9a74456dbca097ee78af92ddfae6c0216da9d2717/thrift-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:048d768e5822c436e8920b987d89a6a26e4d438ddf2de4b9d3f81444cd03cd04", size = 227999, upload-time = "2026-07-11T16:53:16.178Z" }, - { url = "https://files.pythonhosted.org/packages/12/43/74758d030d8d1f103f32026b92e0c9c8b14a6d190197a5e8055b9697294d/thrift-0.24.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2592bb0bf232d0808583626a7a8d71e265851e974c182967d67dde1f16902aee", size = 535383, upload-time = "2026-07-11T16:53:17.375Z" }, - { url = "https://files.pythonhosted.org/packages/89/d5/2f1e521dd0e8525c460e6e2ed3938f56b0d8208d794e08e0309f6e2ae299/thrift-0.24.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1b1ba00151565dc4e6673c924debd557aaf25b2790f1570b3430cb35503671", size = 540968, upload-time = "2026-07-11T16:53:18.61Z" }, - { url = "https://files.pythonhosted.org/packages/9b/02/44abc539a6356d8ffc47fd36724b376d4d4d904924a720b4416911cd8009/thrift-0.24.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:afb7977ccb8ecd7b1520a5141de64b58e94712528a26c10ac8f5b4ef220112a5", size = 1480581, upload-time = "2026-07-11T16:53:19.772Z" }, - { url = "https://files.pythonhosted.org/packages/34/bc/77a6c2f8f4983473a22905d7c420eb6d6ea46ee590bc287f3443533d0525/thrift-0.24.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8d37a4311a87bda4bd2b8e4137eec638c18e3a57d11a1011bd98a8867523860", size = 1538742, upload-time = "2026-07-11T16:53:21.216Z" }, - { url = "https://files.pythonhosted.org/packages/06/b1/4c5e8ac71390292a70c4d75d695a562b6d6c081316d37f6121e18e1878d9/thrift-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbf461351940ddaa85bf8c2ee1754c9cfdd33bb78322e635e1ea5cd3947ae49a", size = 413572, upload-time = "2026-07-11T16:53:22.503Z" }, - { url = "https://files.pythonhosted.org/packages/0d/39/f3e6ec283fa6dbd490eab25a33c46edb6b3114ff9dc46b31bd88af3fd4ea/thrift-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:306e0fe3c96b300471c19cec60bd6816064fd93d04713488ef07120aff84653b", size = 223061, upload-time = "2026-07-11T16:53:23.804Z" }, - { url = "https://files.pythonhosted.org/packages/45/86/efd51714e5ce62c4d3c7f998eb1b1f5526cd1ee9fb4080545e7fc25fe5cf/thrift-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ac42199ea2a6fc6275b0aeec32ae09e7f9edb43890ff9a1af5e34191fb422cc", size = 222518, upload-time = "2026-07-11T16:53:24.885Z" }, - { url = "https://files.pythonhosted.org/packages/70/b0/4ea971750d449180363aa2273f1d80e66ce068643c5993589978d7a3951f/thrift-0.24.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cdd5f927eb98d0e8f00ab148b0cb66ae2598a396e907bd2b6febcbdcc46d80c", size = 531068, upload-time = "2026-07-11T16:53:26.044Z" }, - { url = "https://files.pythonhosted.org/packages/97/d8/cbd6352723ab4a6668c40c0d10a680b0f05178df2de9ea919ea8e8d2b834/thrift-0.24.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:134cd1a0d80f928377808348d1f9f647284ada093573beaa47040bed5507e219", size = 539609, upload-time = "2026-07-11T16:53:27.238Z" }, - { url = "https://files.pythonhosted.org/packages/46/b1/af2e258f9c135dc359acda9390e557ddc501a9185b65892fc253f94ca160/thrift-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:518199062feabfc0982767a0f7bceccb4fd1b485654f75e02913837444c3c130", size = 1474702, upload-time = "2026-07-11T16:53:28.669Z" }, - { url = "https://files.pythonhosted.org/packages/38/6f/93424dc6ceb2a561d055df4fee825ff7116e342e126c6aab357e0a94185e/thrift-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8c5de86af2a44641d423624c71c554a691d9f80c8b87709f15c7f98ccc6aeac", size = 1535581, upload-time = "2026-07-11T16:53:29.967Z" }, - { url = "https://files.pythonhosted.org/packages/c5/35/88debea28426b31ac417a83cda5c029f67cf72665fd2327562adb5c151b1/thrift-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f82bb16c4f2009dbc4fc9374604f05e998fb33be6a7787e095cea9842ecfa1d", size = 407669, upload-time = "2026-07-11T16:53:31.282Z" }, - { url = "https://files.pythonhosted.org/packages/93/5a/3b0d7b47ae73c28891997433447f87ba26de0fc11ca9fd4ea8c6b497bf23/thrift-0.24.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebb8b389142d67554d0de814dd6c1d62b962751f68721537efca429c57b09327", size = 224851, upload-time = "2026-07-11T16:53:32.528Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/4e17cce911ebbf76c59472b488dbe25c32d51cef2500b6e255a9fb59ff02/thrift-0.24.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18e7693f1bcef45937ad31a02564add55dec754710d42afc8ee3fc26ecf13028", size = 224313, upload-time = "2026-07-11T16:53:33.797Z" }, - { url = "https://files.pythonhosted.org/packages/fd/9a/eb8cb5a75fdff60b21b78c712a455adac01e1768792c3232184fa7202961/thrift-0.24.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937b398c311799fd5eddaeaffbd275510c68ead597eb1897e5e8113542fb2b50", size = 532517, upload-time = "2026-07-11T16:53:35.458Z" }, - { url = "https://files.pythonhosted.org/packages/30/b7/d0ecdf3573eb4026343d04efa64cf465a8453e699bd41a2d68156c3caf83/thrift-0.24.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b89a83d4e9ae6029e14f6f7c73305044bf02739d729c6aa8025ef3b6faef0ec0", size = 541124, upload-time = "2026-07-11T16:53:36.767Z" }, - { url = "https://files.pythonhosted.org/packages/df/ce/37daa851995ad441e38f83b2a8e67cdf68df10148779fb5bbafd4fb24f6b/thrift-0.24.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6381093f2c54e0f108554004a8cac3f1ef7ba99d6c5a9909c0eb64f450142685", size = 1476528, upload-time = "2026-07-11T16:53:38.056Z" }, - { url = "https://files.pythonhosted.org/packages/d6/67/80f3aeced7a38d17a281d1b9904ab9e93ab493f5c75e144b3180c278041c/thrift-0.24.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28e034658d724aaa66007babb258ef12f0294036812fc449d7ce6fd15b07100f", size = 1537515, upload-time = "2026-07-11T16:53:39.316Z" }, - { url = "https://files.pythonhosted.org/packages/37/1f/169f93d531a0d9546b6f18dacf37752c27cccfa719bf231162d1fcf8d0e0/thrift-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:f4a44391ae1e32817553639b2991b0753d84487259fe87f8111c956c6bdebf43", size = 409450, upload-time = "2026-07-11T16:53:40.711Z" }, - { url = "https://files.pythonhosted.org/packages/01/46/e229eec0531070a76f2d98335f57ec29df5fb991d7a00e8b5aac2064e893/thrift-0.24.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7beb05268c76895f7c3130ce747a8a8cf35558fbc7f464f994e20a0c92181cbd", size = 227137, upload-time = "2026-07-11T16:53:41.893Z" }, - { url = "https://files.pythonhosted.org/packages/6e/30/27b0702a183de1fc884622a584093756eba6eb33a2a21760b279ff124573/thrift-0.24.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f40bdfbaf8d2795b1593bb75b4d9c77831bbe96a1ef59da4634ccc125a17dca5", size = 226617, upload-time = "2026-07-11T16:53:43.135Z" }, - { url = "https://files.pythonhosted.org/packages/cd/b2/aa45a1d88c3692c14420095a3d3631c6add18e3c33527bdf5adfcaed5f1e/thrift-0.24.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:272624d36faa7bee01b94e5da5efc7305d9d3da57d005382c93fcbde8c3edf6a", size = 534129, upload-time = "2026-07-11T16:53:44.214Z" }, - { url = "https://files.pythonhosted.org/packages/7b/4f/b500ce0d9cf29a83cab88ab460ce7a7bab955f2a5d30bb1e7a9d4d4c2242/thrift-0.24.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bc91ae93005bc16c7362edcb58818a547850b9b52fe9b8075cc6b8922bcde2d", size = 543379, upload-time = "2026-07-11T16:53:45.394Z" }, - { url = "https://files.pythonhosted.org/packages/32/77/0ecb68cf0ea0af2c51ba88cf7f85ce59823efc7a7cb7573abaac0d60d49b/thrift-0.24.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c05bb4eac921836c12cdf30c237283df8a42aff6540f7970cd72c5959b139970", size = 1478854, upload-time = "2026-07-11T16:53:46.654Z" }, - { url = "https://files.pythonhosted.org/packages/fa/89/fd2a0b7ef3a54bac545fdaf9e34b3d3012471058a9eddbe5a2697bf69773/thrift-0.24.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e5da6b391828d46df9471f22181374760a91773b7e07ad7eb44f351c90580662", size = 1539741, upload-time = "2026-07-11T16:53:48.012Z" }, - { url = "https://files.pythonhosted.org/packages/a5/10/7b3368202c2333d448fa9b13288ecb8f68e9361531c8757a7b1eb27ab750/thrift-0.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:829db909a053d4064cb9fe574cc1f5994c24d727f61cdf6446ca774cd3ba5268", size = 419738, upload-time = "2026-07-11T16:53:49.773Z" }, +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/be/b3f5ef7af6c1209085f45d7004e148de1d157ac5a142a672021b44ef7195/thrift-0.25.0.tar.gz", hash = "sha256:e85615ae9245f5622a5e36a9d2d867e1e7b17a8ca467b257dece5dc1642ccd73", size = 95996, upload-time = "2026-09-30T21:55:51.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/5f/c18759346a6abf8c6ecd1f618214b20aeccde23e4d3c818e619cf61df044/thrift-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fd15b187a30609ace159f036f7af4469e1a0dacfeb590d283300ff9cdfb3b1f5", size = 194154, upload-time = "2026-09-30T21:54:54.906Z" }, + { url = "https://files.pythonhosted.org/packages/dd/be/b007f176498076f7bc164fb8e74e1df89b18f229c060a1cc9b08d3db609e/thrift-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2b0c492d975703ce006108c570b7adb61f573a6b667d676918d0122bee9392f2", size = 193650, upload-time = "2026-09-30T21:54:56.354Z" }, + { url = "https://files.pythonhosted.org/packages/48/d4/b3464d5d1a76c7453bd1f3b6c36e04fda2e4007a0365cf33ba3c70c3f8a1/thrift-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:579d891d2af0065cec44a3ed68e7a821f58aff3250af71109e13fee3775c7aaf", size = 506011, upload-time = "2026-09-30T21:54:57.602Z" }, + { url = "https://files.pythonhosted.org/packages/85/cc/0646c1dee93f9df48ff9cafb644683b6c425043b24dba70e138570347006/thrift-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51597cdb871998acea9f5b24b7b1eefc3a03d71f5d2190f897052fc9ed2bf7a8", size = 512260, upload-time = "2026-09-30T21:54:58.903Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ab/0d88eae269dce6ad23d6fd5b17ebdeaf1426552e4b2fe1e66d0cc8bbdb8d/thrift-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:21cfb2d32539f43efd728db6a30cdd692c2559a9f47c19ef8a864df27c06b9d1", size = 1446905, upload-time = "2026-09-30T21:55:00.298Z" }, + { url = "https://files.pythonhosted.org/packages/4b/17/ec2adb82dbac5db0fa9dff4355a61e5001e3fe8b395777640bb18c7bbd46/thrift-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0c3a63dd22977ecf96f412be78a563225ac2d81d9ca224ea2c52248884f36a20", size = 1506866, upload-time = "2026-09-30T21:55:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/aa/87/0fe926fa58a33c3633636772f88c833e06406d57ff0bd654f70c1bdf94e3/thrift-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:9726fc48f5487c4c2706cfb54444e3e518188ac6a43bef533151c6bd6450cad6", size = 379223, upload-time = "2026-09-30T21:55:03.4Z" }, + { url = "https://files.pythonhosted.org/packages/3a/f9/a38631593c80f2a699e1a4cb3e246e2f78bcb62e78c718fede0e491ecf19/thrift-0.25.0-cp310-cp310-win_arm64.whl", hash = "sha256:2b0ef8aa9773c8009550559d27a39ba01b999627f37cfcb3cccb9579349140ae", size = 548000, upload-time = "2026-09-30T21:55:04.853Z" }, + { url = "https://files.pythonhosted.org/packages/d3/fc/ef85e64ee58fd45d669732d23cd4df749f77202317f9d76c03a36f3dabc5/thrift-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc1ca37e289c7056f12c0acef8449ab036daa74fa396cdedb10eabbf79fd6a15", size = 239914, upload-time = "2026-09-30T21:55:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/15/2a/451ab3a03bfb2e27ccfc1060697c9120080929d61df0ec2c5974154158b9/thrift-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:894763113e191dd38424c3ec73a156d89c7d3fece5436af3e51e219ec15698d6", size = 239407, upload-time = "2026-09-30T21:55:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/ab/54/ad3a4ef9fe156b5d6ce4bbfcec30f4982d0f8ab614e97d41151884313dae/thrift-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a7b9081f378a39982e22f0ea7fa74ce9fb0179bb28e2ebc45707c3b66718de4f", size = 554025, upload-time = "2026-09-30T21:55:08.714Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0b/62d40ae403d1e9bfc884a112555d5bb6debe15ea61160194231e6436ff6c/thrift-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b968801fcf0974e3702130e372fd63c1cb96b67948d2dc4237cdee3d157face0", size = 560487, upload-time = "2026-09-30T21:55:10.04Z" }, + { url = "https://files.pythonhosted.org/packages/6e/88/3d25527ab3f05998df8486bba1e88c610d757df33f5ca9ab3aeaf06a4960/thrift-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91f5207e83c2129e1a03a75a9b01ce9c4fb0fb9ff18e53cb78ffc618d594085d", size = 1494571, upload-time = "2026-09-30T21:55:11.447Z" }, + { url = "https://files.pythonhosted.org/packages/49/7f/fe20790128743efbae9cc081d257fc09564eef2f75d9e787a27c731da870/thrift-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:91e667bd8ff58de4505fe463105655892b5b46e3ac00d1987d4258834e94ab1e", size = 1554365, upload-time = "2026-09-30T21:55:13.227Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/6c9d267d1686fcab593f1a2daa3df991fcf8acef91369cb82b68a4bdd4e9/thrift-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:49b3efa6388c76fde87d542428fa1aebc86d6b855bfde4bb006c932974c1c5bc", size = 424999, upload-time = "2026-09-30T21:55:14.65Z" }, + { url = "https://files.pythonhosted.org/packages/69/72/7bd75801fad80be10c39ead0a24d4ece25f167d488dc199835417d1ed04a/thrift-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:8a781b043d628bc5ffd4f9a14589d7bd5086121f52f674c0007e31a74cd0958d", size = 593786, upload-time = "2026-09-30T21:55:16.052Z" }, + { url = "https://files.pythonhosted.org/packages/a1/0d/6ef59ee4b97b86ad372bba2f18fe79543d2fdf56a64137e89559c82715a7/thrift-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:23d8a6f8ae12836e86b22cb24f2d9533a6f6b8bee76aee5bb8b9d9579abd507d", size = 234318, upload-time = "2026-09-30T21:55:17.411Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3e/0b9ed83d09110778d9790a8c9556f110c26dd95b17343e47e8e113330821/thrift-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47527c984a449c584b550f57a6e4d623473fcdcbf1fa5405b546be239bfc607a", size = 233658, upload-time = "2026-09-30T21:55:18.73Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0a/073073b091f8d9e70152f2031c89fa9fbebc8719491d35c4569f3584f08d/thrift-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ba6f27475c453df37cdf5f57b5937cf175d166d2a10dc5273d2a47d22c4329d", size = 549948, upload-time = "2026-09-30T21:55:20.01Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0e/108ef1a9e4e7e24979196ecb53b40ed3bd44034da0eccf69677984508d09/thrift-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:436b8d1ad069a90b18ca1d745f2a2fc9b70329317ba77eb67ff28e82de186930", size = 560387, upload-time = "2026-09-30T21:55:21.543Z" }, + { url = "https://files.pythonhosted.org/packages/b4/85/bb2d873b095f3ebc22ad6c445889e04c35dd261e1c909aaa552982ba5668/thrift-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f0c827fe55ad69eb10950628887e05720cb4f08a754835e27370bf50ca62b760", size = 1488854, upload-time = "2026-09-30T21:55:23.121Z" }, + { url = "https://files.pythonhosted.org/packages/57/5b/8276cbe5531be3e7b5ed1eaf981d9f26b965f450591328aba137cdbd1668/thrift-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41a12e5f3bcc0465ee4489ed0d74766933a7d16e5881f7245f225a4f04dc6be1", size = 1551020, upload-time = "2026-09-30T21:55:24.48Z" }, + { url = "https://files.pythonhosted.org/packages/67/56/d39ee13a67c66c0fbcfddf132d7cc02e1c4a34d5860c347f20f5e5a940bb/thrift-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6fb3fe5670d97e7cc8035504b82980d40efb978bde0e97b85e7cd08ba229404", size = 418913, upload-time = "2026-09-30T21:55:25.839Z" }, + { url = "https://files.pythonhosted.org/packages/89/aa/b82c19918ae57384c67d1f79eceae0272b1e527f0d2b99e8b396299bc3c7/thrift-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:bb12066efe3b19c44c05f97dbdd10c32317e1bd64e1aa98ec7d11f669a11f874", size = 588544, upload-time = "2026-09-30T21:55:27.091Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f1/ea98bea8331efd78d3d7638060ab8140e69331ebdc1959dd21d7ea0bdf4f/thrift-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5e92406068afacb25ea04f2734ba6aaefedae86deb6700f28d6c9f73660021b2", size = 236119, upload-time = "2026-09-30T21:55:28.392Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b9/4d3333a0ac7fa003691e8902befa2580c6bfeb5d9457f044e8148b53b1fd/thrift-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:68f685a14655a8546e9f733dfee948008749b69aa861b5c2ae5e87bb69cb8515", size = 235446, upload-time = "2026-09-30T21:55:29.841Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/e5ef1ae72c5810439479d7bf9142bdbcfa341ee070487acd3c35ff59a687/thrift-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ca4e36325b508976dcf2e98831297844e11151e5acbcc51c569b92a95d746a", size = 551378, upload-time = "2026-09-30T21:55:31.072Z" }, + { url = "https://files.pythonhosted.org/packages/47/44/af43d6e4251536df27704f75a7d222443d40c55a8a8c15a216ebc44bfff8/thrift-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aa0cfa1db4d2129114d9548c4fc7e85680013aa442692377606b6305d236a1c0", size = 561987, upload-time = "2026-09-30T21:55:32.411Z" }, + { url = "https://files.pythonhosted.org/packages/78/d5/7cc3e0d19695edda34bbdfc269039a2b1f272c208cede1e5a591d5fa88d0/thrift-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d0cac032e150dbde529456ac1f91af96e5eededcf097683ffcb2e875e6da4ef3", size = 1490698, upload-time = "2026-09-30T21:55:33.713Z" }, + { url = "https://files.pythonhosted.org/packages/05/f0/21152fbb7f9151b856725ef135162cfdeacb8487b50a231fc73b84b065db/thrift-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:60c0961232ec9049635138991631ac063220325c2537482d5b26979d2bf17f5d", size = 1552913, upload-time = "2026-09-30T21:55:35.174Z" }, + { url = "https://files.pythonhosted.org/packages/1d/be/a4f8c54ec939b6da36ed90edd96f11266a1bba4167eb662e68b62e510a6d/thrift-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:2dfc192cfebdca42cbdaaec7b2f6fea0d3cd4c0f9053bf1a3fdd7a65fa30c535", size = 420715, upload-time = "2026-09-30T21:55:36.711Z" }, + { url = "https://files.pythonhosted.org/packages/48/27/46b75d0db7beee175d97ba07c49dc4047229f35af3ebcc1bb3ef9acef7ba/thrift-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:9d48ef7804a8e7f5e28dc2c495d8b61da14314c9d615bdafdf28dd40e596b666", size = 590330, upload-time = "2026-09-30T21:55:38.321Z" }, + { url = "https://files.pythonhosted.org/packages/07/be/1dd50b7534dbe0310a7403239e531118986c3652d14bd7e7206f4c9ef959/thrift-0.25.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8eca5abeeb38d7948175b1b4254a8799c9c8fd9cd2c5ce5c9b9aa64709c2ae86", size = 238384, upload-time = "2026-09-30T21:55:40.566Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d5/04cfbedfceba5dad56214df29131b900444bb2627fd527caf39ba617de00/thrift-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c38e9dd1d79e77acccea314cbfac27a13cab7ca7d1c96e81d7bfb3d2cc3e2d20", size = 237793, upload-time = "2026-09-30T21:55:41.741Z" }, + { url = "https://files.pythonhosted.org/packages/67/32/b8df8fb6bc1f52eaf4634c746e046e4cf27fe66953b08a89612bf58af042/thrift-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b78d0274daa64f34257f7ccf4e1cc83567595ff90be09e267292ee633277a497", size = 553187, upload-time = "2026-09-30T21:55:43.105Z" }, + { url = "https://files.pythonhosted.org/packages/db/f0/84aeecf88c26af6247da91634b6074a991f2ef332b0bf59e144df4b1a2ca/thrift-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2a10a8d58cd34267bc06ee9a5274ded53b5be5da5765544be7734f5a096d4dfc", size = 564253, upload-time = "2026-09-30T21:55:44.615Z" }, + { url = "https://files.pythonhosted.org/packages/e9/63/38137333cdcfde8d91a73534b3f30d827c4bdfe3478a2d26988ce06216c6/thrift-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:674238ae8ea086c7c48f08562d18bb842f9f8789507ab0441f60eed676d3e0da", size = 1493226, upload-time = "2026-09-30T21:55:46.02Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b4/f1a89ddd900b901f25cd784f2ad696038920a0f97ab74136c78ebbd3a272/thrift-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6b42e755f38a536abfb3691ecc2850cdb61fe60848dd1ef434da26ac6dc31547", size = 1555187, upload-time = "2026-09-30T21:55:47.405Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b1/70e0b40452ba223500f725a7df649ecfe89c21ababfc7678791a8bbcbd5f/thrift-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:779da544bd3ce68ebb17a21b01db638a8e6c5a65f878e54374c90278e64d98ae", size = 431093, upload-time = "2026-09-30T21:55:48.891Z" }, + { url = "https://files.pythonhosted.org/packages/a0/19/4afcc4f4bd7fd81a6dc10fb796f69709266e7db0b8bcaa64a840585486a1/thrift-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:07aa3ccd64304f77b315e4087b7ebf8e537935c070612fc5ad12c8076085ef3a", size = 605619, upload-time = "2026-09-30T21:55:50.263Z" }, ] [[package]] @@ -4126,21 +4131,21 @@ name = "torch" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "cuda-bindings", marker = "python_full_version >= '3.11' and python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version >= '3.11' and sys_platform == 'linux'" }, + { name = "filelock", marker = "python_full_version >= '3.11'" }, + { name = "fsspec", marker = "python_full_version >= '3.11'" }, + { name = "jinja2", marker = "python_full_version >= '3.11'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "networkx", version = "3.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "python_full_version < '3.16' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "nvidia-cudnn-cu13", marker = "python_full_version >= '3.11' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "python_full_version >= '3.11' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "python_full_version >= '3.11' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "python_full_version >= '3.11' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.11'" }, + { name = "sympy", marker = "python_full_version >= '3.11'" }, + { name = "triton", marker = "python_full_version >= '3.11' and python_full_version < '3.16' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/2d/a5/fa947e2bbb072933f8c0ad5e02bd11126bcfc2b624fe234ac29a7b63600c/torch-2.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e808b63d616a43b326e1a5677d45a2e5babe7a8ce27edb03ddb48e7dd9b6d663", size = 138527954, upload-time = "2026-10-28T15:16:30.29Z" }, @@ -4178,7 +4183,7 @@ name = "tqdm" version = "4.70.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } wheels = [ From 7116dbfc5a4794e211ab5a92123bbfd306084a71 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 16:30:06 -0800 Subject: [PATCH 04/24] Take the oauthlib fix the connector now allows oauthlib 3.3.1 carries two High findings in the source scan: an authentication bypass in the device-code flow (BDSA-2026-38625) and a client-identity confusion in the client_credentials and password grant handlers (CVE-2026-118669). Both are fixed in 4.x. It reaches us transitively through databricks-sql-connector, which does not cap it, so this resolves the same way the thrift bump does: lock only, no constraint change. Signed-off-by: Manush Murali Co-Authored-By: Claude Opus 5 --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 35aa54053..4d22fce64 100644 --- a/uv.lock +++ b/uv.lock @@ -2209,11 +2209,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "3.3.1" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/17/2bb072de0b990303c985f26869fd47bb179024a051f47e7f9d208eb92f8e/oauthlib-4.0.1.tar.gz", hash = "sha256:3d73febdb6faa00f277c81fb04be7b7586fb96fc3b7f0b9c772dada69a25ede7", size = 191219, upload-time = "2026-10-31T09:52:58.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, + { url = "https://files.pythonhosted.org/packages/df/ef/3b8adde1b32a6f540758bff734d3442c621734c83708e5f02b0ba2f43b84/oauthlib-4.0.1-py3-none-any.whl", hash = "sha256:498a298876e996301950a5b053d12c31628bb7b895db03666430c69ce182eb61", size = 161934, upload-time = "2026-10-31T09:52:56.206Z" }, ] [[package]] From 12c22dadf9c93398b69db9bb8c7184e89e28b28f Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 16:32:14 -0800 Subject: [PATCH 05/24] Clear the open dependency alerts the lock can reach Nine of the nineteen alerts open on this repository are fixed by resolving newer versions already allowed by our constraints. None needed a constraint change; only uv.lock was stale. certifi 2026.7.22 -> 2026.10.23 one critical pyasn1 0.6.4 -> 0.6.5 four high joserfc 1.7.5 -> 1.7.7, 1.8.0 one medium setuptools 84.0.0 -> 84.2.1 one low joserfc resolves to two versions because the lock splits it on a Python version marker, which is how it was already resolved for other packages. The remaining ten are all filelock, and they cannot be fixed here: snowflake-connector-python caps it below 4.2.1, so asking for the fixed version makes the workspace unsatisfiable. That one waits on upstream. Signed-off-by: Manush Murali Co-Authored-By: Claude Opus 5 --- uv.lock | 58 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/uv.lock b/uv.lock index 4d22fce64..2cfd95042 100644 --- a/uv.lock +++ b/uv.lock @@ -422,11 +422,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -1394,14 +1394,43 @@ wheels = [ [[package]] name = "joserfc" -version = "1.7.5" +version = "1.7.7" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] dependencies = [ - { name = "cryptography" }, + { name = "cryptography", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/15/48ff246732232d35ee234228def22b52455fcc55577f06146602399a29b0/joserfc-1.7.7.tar.gz", hash = "sha256:471d0fd12a5a0d1c0942de280f39efe6f597878315033270b504d4d006d25510", size = 237446, upload-time = "2026-10-19T00:30:29.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/9f/934db284aa22f37af59708e382903860d4b81329c137b1b0ed2745daeb65/joserfc-1.7.7-py3-none-any.whl", hash = "sha256:06979cfbab78bc802f14c0358f32969826cc26720e13cb38926ffdc86bc0cb2c", size = 71758, upload-time = "2026-10-19T00:30:28.498Z" }, +] + +[[package]] +name = "joserfc" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "cryptography", marker = "python_full_version >= '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/94/80fea1514b7c6d7d37804d3fe9ca81455f633347fc98731bd71ffe1faa17/joserfc-1.7.5.tar.gz", hash = "sha256:d5ff536e658e17664f8c1b1ab60dc4aa62aa973fcef1edd33cc44bda45d6f5ea", size = 234990, upload-time = "2026-08-29T13:05:42.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/31/5c3f27e82c92a7bbe1f028033ff19e0251540395c9be244c01a50de42bdb/joserfc-1.8.0.tar.gz", hash = "sha256:cbf03cb2c6db8cc5289f1ef8718842e53dba244b4c0de5fefa0c0eaee4bde8db", size = 241023, upload-time = "2026-11-03T10:45:57.948Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/c5/82addfd375e5ee6520644e0553e4aadde92d668c4fc99cc716d337fe7bb3/joserfc-1.7.5-py3-none-any.whl", hash = "sha256:add2c2c84e8373b084d526a8b53daba5d7a513a118cd2dcd9fc9f979d0922159", size = 71269, upload-time = "2026-08-29T13:05:40.718Z" }, + { url = "https://files.pythonhosted.org/packages/5e/7e/7ffaae13415e8f51c8e8d5d1845f2de7a27ebf2717e97a38d7e7446d6030/joserfc-1.8.0-py3-none-any.whl", hash = "sha256:176b836ac7ed9a46a88629e24076314b475f33a75d0bad7f26146e572eed7c79", size = 73147, upload-time = "2026-11-03T10:45:56.531Z" }, ] [[package]] @@ -1774,7 +1803,8 @@ server = [ { name = "flask" }, { name = "flask-cors" }, { name = "graphql-core" }, - { name = "joserfc" }, + { name = "joserfc", version = "1.7.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "joserfc", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "jsonpath-ng", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "jsonpath-ng", version = "1.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "openapi-spec-validator" }, @@ -2772,11 +2802,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]] @@ -3709,11 +3739,11 @@ wheels = [ [[package]] name = "setuptools" -version = "84.0.0" +version = "84.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/b3/c19426f783614d625699a4b247faf628ea9ab1db228674491767b417f9e1/setuptools-84.2.1.tar.gz", hash = "sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc", size = 1230289, upload-time = "2026-10-20T19:01:37.742Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, + { url = "https://files.pythonhosted.org/packages/ef/55/6dc6d6bb953bdfc52519a0059592c52b6e3b8ad51cf8256dffceb55b0610/setuptools-84.2.1-py3-none-any.whl", hash = "sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2", size = 876263, upload-time = "2026-10-20T19:01:35.678Z" }, ] [[package]] From 89b9162b39fc4adce0c95fe02ebdbb6c017f3c0c Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 16:39:46 -0800 Subject: [PATCH 06/24] Tie the contract decisions to the v2-candidate merge base The compatibility ledger only accepts decisions recorded against the actual merge base. The three decisions still named the old rust-core commit, so the contract job rejected them after the retarget. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- tests/compatibility/manifest.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index 4eeaf3aa6..d9e4675f8 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -8243,17 +8243,17 @@ "reason": "Rename test files and remove historical fixture prose; all type vectors and their expected outcomes remain identical. Update the whole-file digest for metadata changes." }, { - "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "api_contracts/results-merge", "reason": "PredictionResult.merge keeps a candidate repeated within one part and compares Decimal candidates by value, as 1.0.0 did; the merge test is renamed for the restored behaviour." }, { - "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "api_inventory", "reason": "ColumnExplanations.Row gains a constructor without a score, as the column analysis on main no longer computes one; the scored constructor and score() remain, deprecated." }, { - "base_commit": "9488cfeb3fa8b78c35ebaf0ae12f969c6695af9f", + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "api_contracts/column-explanations", "reason": "Column analysis reads the labelled rows a request sends, as on main, so the request keeps no pre-trim delta and its tests are replaced by one over the sent rows." } From e986f46855070621ec2086cb9da3f3c614305163 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 17:04:05 -0800 Subject: [PATCH 07/24] Indent doc list continuations so clippy passes The pinned toolchain's clippy rejects a doc list item whose wrapped lines are not indented, and the cleanup that re-wrapped these comments left rust-lint and rust-test red. Only comment whitespace changes; the one contract description it reaches is regenerated, and its shape hash is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/bindings/sdm-jni/src/lib.rs | 28 +++++----- rust/bindings/sdm-py/src/lib.rs | 24 ++++----- .../sdm-contract/schema/sdm-contract-72.json | 2 +- rust/crates/sdm-contract/src/ops.rs | 32 +++++------ rust/crates/sdm-contract/src/run.rs | 24 ++++----- rust/crates/sdm-contract/src/wording.rs | 32 +++++------ .../sdm-contract/tests/data_vocabulary.rs | 24 ++++----- rust/crates/sdm-core/src/materialize.rs | 10 ++-- rust/crates/sdm-data/src/databricks.rs | 24 ++++----- rust/crates/sdm-data/src/pg_values.rs | 32 +++++------ rust/crates/sdm-data/src/snowflake.rs | 38 ++++++------- rust/crates/sdm-data/src/view_expr/pattern.rs | 24 ++++----- .../sdm-data/tests/databricks_goldens.rs | 10 ++-- rust/crates/sdm-data/tests/goldens/mod.rs | 20 +++---- rust/crates/sdm-data/tests/postgres_schema.rs | 16 +++--- rust/crates/sdm-engine/src/labels.rs | 54 +++++++++---------- rust/crates/sdm-engine/src/time/mod.rs | 28 +++++----- rust/crates/sdm-engine/src/time/text.rs | 16 +++--- .../crates/sdm-engine/src/time/text_shapes.rs | 12 ++--- rust/crates/sdm-engine/tests/rs1.rs | 16 +++--- rust/crates/sdm-host/src/codec.rs | 12 ++--- rust/crates/sdm-host/src/runs.rs | 14 ++--- .../tests/native_admission_vectors.rs | 10 ++-- rust/crates/sdm-pql/src/dates.rs | 10 ++-- rust/crates/sdm-request/src/predict.rs | 40 +++++++------- rust/crates/sdm-testkit/tests/carried_rows.rs | 16 +++--- .../sdm-testkit/tests/memory_provider.rs | 8 +-- .../sdm-testkit/tests/numeric_spelling.rs | 24 ++++----- .../crates/sdm-testkit/tests/payload_rules.rs | 26 ++++----- .../tests/payload_shape_validation.rs | 26 ++++----- .../sdm-testkit/tests/postgres_dialect.rs | 28 +++++----- .../tests/replay_past_recording.rs | 4 +- rust/crates/sdm-testkit/tests/snowflake.rs | 30 +++++------ .../sdm-testkit/tests/warehouse_sampling.rs | 8 +-- .../tests/payload_schema_from_types.rs | 14 ++--- 35 files changed, 368 insertions(+), 368 deletions(-) diff --git a/rust/bindings/sdm-jni/src/lib.rs b/rust/bindings/sdm-jni/src/lib.rs index cdda3cde4..6f62b1346 100644 --- a/rust/bindings/sdm-jni/src/lib.rs +++ b/rust/bindings/sdm-jni/src/lib.rs @@ -12,24 +12,24 @@ //! one `NativeError`. It makes no semantic call into Java. //! //! - Arrow crosses through the Arrow C Data interface only, with no copy: -//! an Arrow attachment in is an `ArrowArrayStream` struct Java exported -//! its data into and hands over by address, which this layer moves out -//! of; one out is a [`NativeArrow`] handle, which Java exports into a -//! struct of its own when it reads it ([`export_arrow`]). +//! an Arrow attachment in is an `ArrowArrayStream` struct Java exported +//! its data into and hands over by address, which this layer moves out +//! of; one out is a [`NativeArrow`] handle, which Java exports into a +//! struct of its own when it reads it ([`export_arrow`]). //! - Every native resource has one owner and an explicit release: a run is -//! a `Box` Java's `Run` owns and releases once -//! ([`Java_com_nvidia_sdm_internal_NativeCore_release`]); an Arrow -//! attachment out is a `Box` Java's `NativeArrow` owns and -//! releases once. Java guards each handle so no call reaches a released -//! one. +//! a `Box` Java's `Run` owns and releases once +//! ([`Java_com_nvidia_sdm_internal_NativeCore_release`]); an Arrow +//! attachment out is a `Box` Java's `NativeArrow` owns and +//! releases once. Java guards each handle so no call reaches a released +//! one. //! - Every exported entry contains a panic: the core's own are contained by -//! `sdm-host` (`contain`) and come back as `INTERNAL_ERROR` with -//! `fields.panic`, and one in this layer is caught at the entry and -//! thrown the same way, so no unwind crosses into the JVM. +//! `sdm-host` (`contain`) and come back as `INTERNAL_ERROR` with +//! `fields.panic`, and one in this layer is caught at the entry and +//! thrown the same way, so no unwind crosses into the JVM. //! - The core's `tracing` events reach SLF4J through the `log` facade -//! ([`logging`]), installed when Java first loads the library. +//! ([`logging`]), installed when Java first loads the library. //! - Loading the library starts no thread; it reads no environment -//! variable. +//! variable. mod logging; diff --git a/rust/bindings/sdm-py/src/lib.rs b/rust/bindings/sdm-py/src/lib.rs index b1bfabb91..d9d38c318 100644 --- a/rust/bindings/sdm-py/src/lib.rs +++ b/rust/bindings/sdm-py/src/lib.rs @@ -10,14 +10,14 @@ //! [`CoreError`]. It makes no semantic call into Python. //! //! - Arrow crosses through the PyCapsule interface only: an -//! Arrow attachment in is any object with `__arrow_c_stream__`, and one -//! out is an [`ArrowStream`] that has it. pyarrow's private -//! `_export_to_c` and `_import_from_c` are never reached. +//! Arrow attachment in is any object with `__arrow_c_stream__`, and one +//! out is an [`ArrowStream`] that has it. pyarrow's private +//! `_export_to_c` and `_import_from_c` are never reached. //! - The GIL is released for every core call and held only while touching -//! Python objects; the core's `tracing` events reach Python's `logging` -//! through `pyo3-log`, installed at the first core call. +//! Python objects; the core's `tracing` events reach Python's `logging` +//! through `pyo3-log`, installed at the first core call. //! - Importing the module starts no thread and installs no logger, signal -//! handler or global pool; it reads no environment variable. +//! handler or global pool; it reads no environment variable. use std::collections::BTreeMap; use std::ffi::CString; @@ -70,13 +70,13 @@ fn logging(py: Python<'_>) -> Option<&pyo3_log::ResetHandle> { /// returning it, so after the call: /// /// - an `Exception` is reported as `logging` reports its own handler -/// failures, through `sys.unraisablehook`, and the call's answer is -/// delivered; the exception is the handler's and holds nothing the core -/// wrote but the event; +/// failures, through `sys.unraisablehook`, and the call's answer is +/// delivered; the exception is the handler's and holds nothing the core +/// wrote but the event; /// - anything else, such as `KeyboardInterrupt` or `SystemExit`, is raised -/// in place of the answer, which is dropped: the run stays where the -/// call left it, so a step `next` gave is gone and the next call meets -/// the run's following state. +/// in place of the answer, which is dropped: the run stays where the +/// call left it, so a step `next` gave is gone and the next call meets +/// the run's following state. fn core_call(py: Python<'_>, call: impl FnOnce() -> T + Send) -> PyResult { let answer = py.detach(call); match PyErr::take(py) { diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index fb7cc7be4..6bdc238e5 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -7967,7 +7967,7 @@ ] }, "GivenCandidate": { - "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\nits own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\nholds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\nfixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\ntext the reference writes into its question (in Python, `str` of it);\nit is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\nno scalar holds, written into its question by its text as `spelled`\nis, with the name of the kind of value it is (`date`, `datetime` for a\ntimestamp with no zone, `datetime_zoned` for one with one, `time`).\nTwo of one kind and one text are one candidate; a host whose language\nholds others equal (two zones of one instant) says so with `repeat`\n.\n- `repeat`: a value the host's own language holds equal to one given\nbefore it, left out as the reference's `dict.fromkeys` leaves it out.\nA host whose candidates are all scalars sends none, the core keeping\neach distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\none it cannot hash), which names no row and which the core refuses by\nname, as the reference fails on it.", + "description": "One candidate of a ranking as a host gives it (`Materialize.candidates`,\n), each in one of these forms:\n\n- `{\"value\": }`: the host's value as the contract's scalar of\n its own type, a missing value `null`.\n- `{\"integer\": \"\"}`: an integer wider than an integer scalar\n holds, by its digits (`-` before a negative one).\n- `{\"decimal\": \"\"}`: an exact decimal of any precision, by its\n fixed-point digits (`-12.50`), its scale the digits after the point.\n- `{\"spelled\": \"\"}`: a value of a type no scalar holds, by the\n text the reference writes into its question (in Python, `str` of it);\n it is no text candidate, and the core holds it equal to no other.\n- `{\"typed\": {\"text\": \"\", \"type\": \"\"}}`: a value of a type\n no scalar holds, written into its question by its text as `spelled`\n is, with the name of the kind of value it is (`date`, `datetime` for a\n timestamp with no zone, `datetime_zoned` for one with one, `time`).\n Two of one kind and one text are one candidate; a host whose language\n holds others equal (two zones of one instant) says so with `repeat`\n .\n- `repeat`: a value the host's own language holds equal to one given\n before it, left out as the reference's `dict.fromkeys` leaves it out.\n A host whose candidates are all scalars sends none, the core keeping\n each distinct value once by its own equality.\n- `unreadable`: a value the host cannot look up by value (in Python,\n one it cannot hash), which names no row and which the core refuses by\n name, as the reference fails on it.", "oneOf": [ { "type": "string", diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 6c9daf634..508c61250 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -1184,28 +1184,28 @@ impl Given { /// ), each in one of these forms: /// /// - `{"value": }`: the host's value as the contract's scalar of -/// its own type, a missing value `null`. +/// its own type, a missing value `null`. /// - `{"integer": ""}`: an integer wider than an integer scalar -/// holds, by its digits (`-` before a negative one). +/// holds, by its digits (`-` before a negative one). /// - `{"decimal": ""}`: an exact decimal of any precision, by its -/// fixed-point digits (`-12.50`), its scale the digits after the point. +/// fixed-point digits (`-12.50`), its scale the digits after the point. /// - `{"spelled": ""}`: a value of a type no scalar holds, by the -/// text the reference writes into its question (in Python, `str` of it); -/// it is no text candidate, and the core holds it equal to no other. +/// text the reference writes into its question (in Python, `str` of it); +/// it is no text candidate, and the core holds it equal to no other. /// - `{"typed": {"text": "", "type": ""}}`: a value of a type -/// no scalar holds, written into its question by its text as `spelled` -/// is, with the name of the kind of value it is (`date`, `datetime` for a -/// timestamp with no zone, `datetime_zoned` for one with one, `time`). -/// Two of one kind and one text are one candidate; a host whose language -/// holds others equal (two zones of one instant) says so with `repeat` -///. +/// no scalar holds, written into its question by its text as `spelled` +/// is, with the name of the kind of value it is (`date`, `datetime` for a +/// timestamp with no zone, `datetime_zoned` for one with one, `time`). +/// Two of one kind and one text are one candidate; a host whose language +/// holds others equal (two zones of one instant) says so with `repeat` +/// . /// - `repeat`: a value the host's own language holds equal to one given -/// before it, left out as the reference's `dict.fromkeys` leaves it out. -/// A host whose candidates are all scalars sends none, the core keeping -/// each distinct value once by its own equality. +/// before it, left out as the reference's `dict.fromkeys` leaves it out. +/// A host whose candidates are all scalars sends none, the core keeping +/// each distinct value once by its own equality. /// - `unreadable`: a value the host cannot look up by value (in Python, -/// one it cannot hash), which names no row and which the core refuses by -/// name, as the reference fails on it. +/// one it cannot hash), which names no row and which the core refuses by +/// name, as the reference fails on it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case", deny_unknown_fields)] pub enum GivenCandidate { diff --git a/rust/crates/sdm-contract/src/run.rs b/rust/crates/sdm-contract/src/run.rs index 1ca392fca..97c394e84 100644 --- a/rust/crates/sdm-contract/src/run.rs +++ b/rust/crates/sdm-contract/src/run.rs @@ -16,21 +16,21 @@ //! The sender of a message numbers the attachments it sends. //! //! - Core to host: a run numbers its attachments from 1 in the order it -//! makes them (key sets, model bodies, result payloads), unique within -//! that run and never reused. An id stays valid until the host takes the -//! attachment, the run is cancelled, or the run is dropped; a run that -//! would pass `u32::MAX` ends with `INTERNAL_ERROR`. +//! makes them (key sets, model bodies, result payloads), unique within +//! that run and never reused. An id stays valid until the host takes the +//! attachment, the run is cancelled, or the run is dropped; a run that +//! would pass `u32::MAX` ends with `INTERNAL_ERROR`. //! - Host to core: the host names the attachments of each message it sends -//! (`start`, `fetch_reply`, `model_reply`) with ids of its choosing, -//! scoped to that one message; the core takes them when it reads the -//! message, and the ids mean nothing afterwards. +//! (`start`, `fetch_reply`, `model_reply`) with ids of its choosing, +//! scoped to that one message; the core takes them when it reads the +//! message, and the ids mean nothing afterwards. //! - A message the core writes whose encoding would pass the control cap -//! travels whole as an attachment, named by an `attached` message; its id -//! comes from the run's numbering, and it travels with that message. +//! travels whole as an attachment, named by an `attached` message; its id +//! comes from the run's numbering, and it travels with that message. //! - Across runs: no id carries over. State that outlives a run, such as -//! `RequestState.payload` from `materialize`, leaves as bytes the host -//! takes when that run finishes, and comes back attached to the `start` -//! of the `predict` run under an id the host picks for that message. +//! `RequestState.payload` from `materialize`, leaves as bytes the host +//! takes when that run finishes, and comes back attached to the `start` +//! of the `predict` run under an id the host picks for that message. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 7970f1dc6..c943507f3 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -12,27 +12,27 @@ //! for a brace: //! //! - `{name}`: the field `name` as text; `{name[0]}` the first of a list; -//! `{name:,}` a whole number with its thousands grouped; +//! `{name:,}` a whole number with its thousands grouped; //! - `{name!c}`: the field with a conversion, the host's one value -//! formatter: `r` the value as the host writes it as a literal, `l` a -//! list of them as an and-list (`'a', 'b' and 'c'`), `a` a list of text -//! as an and-list as it is, `j` joined by commas, `c` a list of text -//! joined by commas as it is, `R` joined by semicolons, `d` joined by -//! dots, `p` a row of values (a list as the host writes a tuple, else as -//! `r`), `t` a date or time as the host's time type writes it, `T` a list -//! of times of day or text as the host writes a list of them, and for a -//! value the caller gave, held as the JSON text the host wrote it in, `G` -//! as a literal, `S` as text, `N` the name of its type and `I` the whole -//! number it is (so `{name!I:,}` groups its thousands); +//! formatter: `r` the value as the host writes it as a literal, `l` a +//! list of them as an and-list (`'a', 'b' and 'c'`), `a` a list of text +//! as an and-list as it is, `j` joined by commas, `c` a list of text +//! joined by commas as it is, `R` joined by semicolons, `d` joined by +//! dots, `p` a row of values (a list as the host writes a tuple, else as +//! `r`), `t` a date or time as the host's time type writes it, `T` a list +//! of times of day or text as the host writes a list of them, and for a +//! value the caller gave, held as the JSON text the host wrote it in, `G` +//! as a literal, `S` as text, `N` the name of its type and `I` the whole +//! number it is (so `{name!I:,}` groups its thousands); //! - `{name:to(other)}`: a list paired with the list `other`, each pair as -//! the two literals joined by ` to `, the pairs by commas; +//! the two literals joined by ` to `, the pairs by commas; //! - `{name:place}`, `{name:backend}`: a database the field names, by the -//! names the data layer gives it ([`DatabaseNames`]); +//! names the data layer gives it ([`DatabaseNames`]); //! - `{host.name}`: a value only the host holds, such as a candidate as the -//! caller gave it, which the host supplies by name (conversions as -//! above); +//! caller gave it, which the host supplies by name (conversions as +//! above); //! - `{advice:id(field,...)}`: what to write in the host's language, an -//! [`Advice`] each host spells once, given the fields named. +//! [`Advice`] each host spells once, given the fields named. //! //! The sentence of the part of a question asked in parts an error happened //! in (a forecast's step, a ranking's candidate, [`crate::Code::Part`]) is diff --git a/rust/crates/sdm-contract/tests/data_vocabulary.rs b/rust/crates/sdm-contract/tests/data_vocabulary.rs index 0bbb66dfa..ca54fd7cd 100644 --- a/rust/crates/sdm-contract/tests/data_vocabulary.rs +++ b/rust/crates/sdm-contract/tests/data_vocabulary.rs @@ -17,22 +17,22 @@ //! holds: //! //! - outside the exact reviewed seams, no property is named as SQL or an -//! expression (`DENIED`), anywhere, -//! the typed values included, matched against the whole name and against -//! each of its `_`-separated parts, so `predicate_expr` is refused; +//! expression (`DENIED`), anywhere, +//! the typed values included, matched against the whole name and against +//! each of its `_`-separated parts, so `predicate_expr` is refused; //! - a property typed as a reviewed typed value is refused also when its -//! name, or a part of it, is `filter` or `order` (`DENIED_FOR_VALUES`): -//! a scalar or a field there would be a condition or an ordering in -//! disguise, where the vocabulary has typed structures for both; +//! name, or a part of it, is `filter` or `order` (`DENIED_FOR_VALUES`): +//! a scalar or a field there would be a condition or an ordering in +//! disguise, where the vocabulary has typed structures for both; //! - text (a string, at any depth of lists, maps, options or newtypes) is -//! held only by a property named as an identifier the request refers to -//! (`IDENTIFIERS`), the nearest property naming it, or by the exact -//! checked numeric-token value or scoped opaque-expression seam below; +//! held only by a property named as an identifier the request refers to +//! (`IDENTIFIERS`), the nearest property naming it, or by the exact +//! checked numeric-token value or scoped opaque-expression seam below; //! - an any-typed schema (`true` or `{}`, as `serde_json::Value` gives, or a -//! map of anything) is refused, since it can hold text of any shape; +//! map of anything) is refused, since it can hold text of any shape; //! - the typed values (`TYPED_VALUES`: a scalar, an Arrow field and its -//! type) are reviewed: a reference to one of them is accepted at that -//! reference, and only the name check goes inside. +//! type) are reviewed: a reference to one of them is accepted at that +//! reference, and only the name check goes inside. //! //! A new text field fails until it is added to `IDENTIFIERS` in review, so //! it is a name the request refers to, never text a provider interprets. diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index 67a9af653..5545faa67 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -4571,13 +4571,13 @@ fn null_compared(target: &Shape, mask: &[bool]) -> Result<(), CoreError> { /// ends of its aggregations' windows; `condition` the entity filter's. /// /// - `no_entity_at_anchor`: the entity table's earliest time is after the -/// anchor (`entity_table`, `column`, `anchor`, `earliest`, and `later` -/// where an anchor after it leaves the target's window room in the data); +/// anchor (`entity_table`, `column`, `anchor`, `earliest`, and `later` +/// where an anchor after it leaves the target's window room in the data); /// - `no_entity_passed`: the data leaves room after the anchor for the -/// target's whole window, and an entity filter chose no example -/// (`anchor`, `tables`, `min_time`, `max_time`, `condition`); +/// target's whole window, and an entity filter chose no example +/// (`anchor`, `tables`, `min_time`, `max_time`, `condition`); /// - `anchor_too_late`: the data leaves no such room (the same facts but -/// the condition); +/// the condition); /// - `too_restrictive`: anything else, as the reference's generic reason. /// /// A time bound that cannot be read gives the reason without it, as the diff --git a/rust/crates/sdm-data/src/databricks.rs b/rust/crates/sdm-data/src/databricks.rs index e183c22a1..eb6fc8cc5 100644 --- a/rust/crates/sdm-data/src/databricks.rs +++ b/rust/crates/sdm-data/src/databricks.rs @@ -14,24 +14,24 @@ //! //! - names quoted with backticks, a backtick in one doubled (`quote_ident`); //! - a relation of one to three parts (catalog, schema, table), a source's -//! relations read under its namespace of a catalog and a schema (design -//! fix C), Unity Catalog's names compared without regard to case; +//! relations read under its namespace of a catalog and a schema (design +//! fix C), Unity Catalog's names compared without regard to case; //! - the catalog read through `.information_schema`, the schema -//! and table bound as parameters and folded with `lower(?)`, as Unity -//! Catalog stores them (`DatabricksTable._get_source_columns`); the -//! table keeps the name its caller gives it, which the host takes back; +//! and table bound as parameters and folded with `lower(?)`, as Unity +//! Catalog stores them (`DatabricksTable._get_source_columns`); the +//! table keeps the name its caller gives it, which the host takes back; //! - a column typed by its declaration first, as the reference types it -//! (`_to_dtype`, [`declared_dtype`]); +//! (`_to_dtype`, [`declared_dtype`]); //! - a key set bound as one parameter of JSON text, unpacked by -//! `posexplode(from_json(?, ''))`, the position the seed's batch -//! (`_by_pkey`, `_by_fkey`, `_by_time`); +//! `posexplode(from_json(?, ''))`, the position the seed's batch +//! (`_by_pkey`, `_by_fkey`, `_by_time`); //! - an entity sample as `TABLESAMPLE (

PERCENT) REPEATABLE ()`, -//! the percentage the reference's arithmetic over the table's rows, -//! written as the reference's language writes a float -//! ([`percent_text`]); +//! the percentage the reference's arithmetic over the table's rows, +//! written as the reference's language writes a float +//! ([`percent_text`]); //! - a neighbor read limited per seed with `QUALIFY ROW_NUMBER`; //! - a derived relation read as the reference reads a metric view's table -//! (`_quoted_source_name`). +//! (`_quoted_source_name`). //! //! Where the reference fetches rows and finishes in its own process (a //! key lookup's duplicates removed after the fetch) the statement does the diff --git a/rust/crates/sdm-data/src/pg_values.rs b/rust/crates/sdm-data/src/pg_values.rs index 9a0e9632b..da34411af 100644 --- a/rust/crates/sdm-data/src/pg_values.rs +++ b/rust/crates/sdm-data/src/pg_values.rs @@ -8,24 +8,24 @@ //! Arrow and binds the text it is given. //! //! - A `json` or `jsonb` value, which the provider hands over as JSON text -//! (Arrow's `arrow.json`), is read as the reference's -//! `_sanitize_postgres_frame` writes it: a JSON null a null, an object or -//! an array as compact JSON, ASCII only (its strings' other characters -//! escaped, as `\u` and four hexadecimal digits), and any other value as -//! the reference's language writes it (`str`): `true` as `True`, a -//! number as its digits or the text of its double (`float_text`), a -//! string as itself. A JSON column's elements in an array alike. +//! (Arrow's `arrow.json`), is read as the reference's +//! `_sanitize_postgres_frame` writes it: a JSON null a null, an object or +//! an array as compact JSON, ASCII only (its strings' other characters +//! escaped, as `\u` and four hexadecimal digits), and any other value as +//! the reference's language writes it (`str`): `true` as `True`, a +//! number as its digits or the text of its double (`float_text`), a +//! string as itself. A JSON column's elements in an array alike. //! - A `numeric` declared past the 76 digits an Arrow decimal holds, which -//! the provider hands over as each value's exact text, its declared scale -//! marked ([`DECLARED_SCALE_MARKER`]), is read by its declaration, as the -//! reference's `_to_dtype` reads it: of a scale above zero as doubles, -//! of a scale of zero as integers, a column holding a whole number past -//! `int64` as its digits (where the reference overflows), a NaN missing. +//! the provider hands over as each value's exact text, its declared scale +//! marked ([`DECLARED_SCALE_MARKER`]), is read by its declaration, as the +//! reference's `_to_dtype` reads it: of a scale above zero as doubles, +//! of a scale of zero as integers, a column holding a whole number past +//! `int64` as its digits (where the reference overflows), a NaN missing. //! - A key set is written as the JSON text its statement reads, as the -//! reference writes it ([`key_rows`]): a time without a zone as -//! pandas' `Timestamp.isoformat` writes it (the reference compares a -//! text time column with it as text), binary as `\x` and its -//! hexadecimal digits, so the provider binds the text it is handed. +//! reference writes it ([`key_rows`]): a time without a zone as +//! pandas' `Timestamp.isoformat` writes it (the reference compares a +//! text time column with it as text), binary as `\x` and its +//! hexadecimal digits, so the provider binds the text it is handed. use std::collections::BTreeMap; use std::sync::Arc; diff --git a/rust/crates/sdm-data/src/snowflake.rs b/rust/crates/sdm-data/src/snowflake.rs index c1d41d063..da101b6b4 100644 --- a/rust/crates/sdm-data/src/snowflake.rs +++ b/rust/crates/sdm-data/src/snowflake.rs @@ -11,35 +11,35 @@ //! client (recordings hold them): //! //! - every name in double quotes, a quote in one doubled, its case never -//! changed (`quote_ident`); a relation of a database, a schema and a -//! table, each part quoted alone, so `a.b` is one name (`SNOW/table.py: +//! changed (`quote_ident`); a relation of a database, a schema and a +//! table, each part quoted alone, so `a.b` is one name (`SNOW/table.py: //! 115-117`); the source's database and schema each given or read once a -//! run from the session (`SELECT CURRENT_DATABASE, CURRENT_SCHEMA`), -//! and a relation the session leaves without a schema refused as the -//! reference refuses it; +//! run from the session (`SELECT CURRENT_DATABASE, CURRENT_SCHEMA`), +//! and a relation the session leaves without a schema refused as the +//! reference refuses it; //! - the catalog: `INFORMATION_SCHEMA.TABLES` for the listing, the schema -//! bound; `DESCRIBE TABLE` for a table's columns and its keys; -//! `SHOW IMPORTED KEYS IN TABLE` for its foreign keys; a semantic view's -//! YAML through `SYSTEM$READ_YAML_FROM_SEMANTIC_VIEW(?)`, its name bound; +//! bound; `DESCRIBE TABLE` for a table's columns and its keys; +//! `SHOW IMPORTED KEYS IN TABLE` for its foreign keys; a semantic view's +//! YAML through `SYSTEM$READ_YAML_FROM_SEMANTIC_VIEW(?)`, its name bound; //! - the answers of `SHOW` and `DESCRIBE` come as text, which the provider's -//! row adapter builds (the connector gives them as rows only), each read -//! by its position in the result, as the reference reads the rows; +//! row adapter builds (the connector gives them as rows only), each read +//! by its position in the result, as the reference reads the rows; //! - an exact aggregate row count, also supporting views (approved P3D-2/3); //! - an entity sample `SAMPLE ROW ( ROWS)`, at most 1,000,000 rows, -//! which takes no seed (owner option C); +//! which takes no seed (owner option C); //! - a declared type read by the reference's type map ([`declared_dtype`]); //! - the connector's Arrow conformed as the reference reads its frames -//! ([`conform`]); +//! ([`conform`]); //! - a failure with errno 2003 or a diagnostic containing "does not exist" is -//! "not found", which the core's spelling retry reads; a name's -//! spellings are as given, then all its parts upper-cased, once. +//! "not found", which the core's spelling retry reads; a name's +//! spellings are as given, then all its parts upper-cased, once. //! //! - a key set bound as the reference binds it, one JSON text the core -//! writes ([`crate::key_text`]), unpacked by `FLATTEN(PARSE_JSON(?))`, -//! its seeds separated by `, ` as `json.dumps` writes them, a window's -//! bounds over every seed bound after it; a lookup's first row of each -//! seed and the rows holding its key kept in the statement, where the -//! reference keeps them after its fetch. +//! writes ([`crate::key_text`]), unpacked by `FLATTEN(PARSE_JSON(?))`, +//! its seeds separated by `, ` as `json.dumps` writes them, a window's +//! bounds over every seed bound after it; a lookup's first row of each +//! seed and the rows holding its key kept in the statement, where the +//! reference keeps them after its fetch. use std::sync::Arc; diff --git a/rust/crates/sdm-data/src/view_expr/pattern.rs b/rust/crates/sdm-data/src/view_expr/pattern.rs index 7fb08ce09..72519206b 100644 --- a/rust/crates/sdm-data/src/view_expr/pattern.rs +++ b/rust/crates/sdm-data/src/view_expr/pattern.rs @@ -7,19 +7,19 @@ //! as its `re` matched: //! //! - `\s` is the old client's white space, Python's (`str.isspace`), which -//! holds the four separators U+001C to U+001F that Rust's `\s` does not: -//! read either way, `WITH\x1cx AS (x)` would open a -//! sub-query for the warehouse and none for the check. The rule is -//! `sdm-pql`'s, handed in by the core as [`Spaces`]; a pattern's `\s` is -//! the class derived from it (every code point the rule holds), and `\S` -//! its complement, so no pattern holds the engine's own `\s`. Each fixed -//! pattern is compiled on first use with that class, and kept -//! ([`Lazy`]); +//! holds the four separators U+001C to U+001F that Rust's `\s` does not: +//! read either way, `WITH\x1cx AS (x)` would open a +//! sub-query for the warehouse and none for the check. The rule is +//! `sdm-pql`'s, handed in by the core as [`Spaces`]; a pattern's `\s` is +//! the class derived from it (every code point the rule holds), and `\S` +//! its complement, so no pattern holds the engine's own `\s`. Each fixed +//! pattern is compiled on first use with that class, and kept +//! ([`Lazy`]); //! - a match is total: a pattern that runs out of its backtracking budget -//! answers [`Unreadable::TooDeep`], never "no match", so no check can -//! accept an expression because a pattern failed on it. Each caller -//! refuses on it. So does a pattern made of a definition's text that is -//! past the compiler's bounds ([`Pattern::build`]). +//! answers [`Unreadable::TooDeep`], never "no match", so no check can +//! accept an expression because a pattern failed on it. Each caller +//! refuses on it. So does a pattern made of a definition's text that is +//! past the compiler's bounds ([`Pattern::build`]). use std::borrow::Cow; use std::ops::Range; diff --git a/rust/crates/sdm-data/tests/databricks_goldens.rs b/rust/crates/sdm-data/tests/databricks_goldens.rs index dabad542c..cc78b8ab3 100644 --- a/rust/crates/sdm-data/tests/databricks_goldens.rs +++ b/rust/crates/sdm-data/tests/databricks_goldens.rs @@ -20,15 +20,15 @@ //! the old client's sampling SQL; Scope item 1, key sets bound as the old //! client binds them): //! - each golden request renders exactly its statements ([`goldens::render`]), -//! and every request kind has one; +//! and every request kind has one; //! - the catalog statements are the old client's, text and parameters, after -//! whitespace is normalised, so the stand-in replays C-1's catalog answers -//! and the catalog reader reads the warehouse's real rows; +//! whitespace is normalised, so the stand-in replays C-1's catalog answers +//! and the catalog reader reads the warehouse's real rows; //! - every entity draw the old client writes (the percentage as Python's -//! `str(float)`, `REPEATABLE (seed)`, `LIMIT n`) is a golden's statement; +//! `str(float)`, `REPEATABLE (seed)`, `LIMIT n`) is a golden's statement; //! - every key set is bound through `posexplode(from_json(?, 'array<...>'))`; //! - each part of a relation is quoted alone with backticks, a backtick in a -//! name doubled, and a relation of more than three parts refused. +//! name doubled, and a relation of more than three parts refused. //! //! The other statements' text is the dialect's, pinned here and reviewed //! against the old client's: a golden seeded with the old client's text may diff --git a/rust/crates/sdm-data/tests/goldens/mod.rs b/rust/crates/sdm-data/tests/goldens/mod.rs index 150210417..ceb2e49f8 100644 --- a/rust/crates/sdm-data/tests/goldens/mod.rs +++ b/rust/crates/sdm-data/tests/goldens/mod.rs @@ -8,17 +8,17 @@ //! renders for the contract cases, beside the request it answers. //! //! - [`render`]: each golden request, planned here, renders exactly its -//! statements, so a change of the SQL without a golden update fails. +//! statements, so a change of the SQL without a golden update fails. //! - [`run_is_pinned`]: every fetch a case run made on the database (the -//! observations `SDM_L4_OBSERVATIONS` names, python-core's run in -//! `contract-python (ref-pandas3, )`, which executed each -//! statement on that database and whose results the comparator held to -//! the reference's) is a golden, its statements the golden's, so the -//! goldens are the SQL the run executed and a request the dialect newly -//! answers is never left unreviewed. CI's `l4-goldens ()` names -//! the run; with no run named, as in `cargo test --workspace`, there is -//! nothing to hold and the check says so. A run named but missing, -//! unreadable, empty, or holding no fetch on the database fails. +//! observations `SDM_L4_OBSERVATIONS` names, python-core's run in +//! `contract-python (ref-pandas3, )`, which executed each +//! statement on that database and whose results the comparator held to +//! the reference's) is a golden, its statements the golden's, so the +//! goldens are the SQL the run executed and a request the dialect newly +//! answers is never left unreviewed. CI's `l4-goldens ()` names +//! the run; with no run named, as in `cargo test --workspace`, there is +//! nothing to hold and the check says so. A run named but missing, +//! unreadable, empty, or holding no fetch on the database fails. //! - [`seeded_faults`]: both checks shown to fail on their seeded faults. //! //! Coverage ignores only resolved identity projections and non-rendering type diff --git a/rust/crates/sdm-data/tests/postgres_schema.rs b/rust/crates/sdm-data/tests/postgres_schema.rs index af6ebc952..dafb6c291 100644 --- a/rust/crates/sdm-data/tests/postgres_schema.rs +++ b/rust/crates/sdm-data/tests/postgres_schema.rs @@ -9,17 +9,17 @@ //! (`old-behaviour/-lakebase.md`, rows 16, 21 and 38). //! //! - A source given no schema is settled first: the one statement that -//! asks the session is the namespace read, the old client's own -//! `SELECT current_schema`. +//! asks the session is the namespace read, the old client's own +//! `SELECT current_schema`. //! - Every request kind, under a schema, names it in every statement, and -//! none asks the session for its schema or finds a relation by the -//! search path (`to_regclass`): a temporary or `pg_catalog` relation of -//! a table's name is never read in its place, nor refused. +//! none asks the session for its schema or finds a relation by the +//! search path (`to_regclass`): a temporary or `pg_catalog` relation of +//! a table's name is never read in its place, nor refused. //! - The catalog's column read is the old client's statement, but for the -//! schema and table it binds and the core writes in place, as recorded -//! on Lakebase (C-1, `contract/captures/lakebase/base.binary`). +//! schema and table it binds and the core writes in place, as recorded +//! on Lakebase (C-1, `contract/captures/lakebase/base.binary`). //! - A dotted name is one table under the schema, as the old client never -//! splits a name (row 20). +//! splits a name (row 20). use sdm_contract::catalog::CatalogAnswer; use sdm_contract::data::{ diff --git a/rust/crates/sdm-engine/src/labels.rs b/rust/crates/sdm-engine/src/labels.rs index 2bb62d4af..f7fe8e98c 100644 --- a/rust/crates/sdm-engine/src/labels.rs +++ b/rust/crates/sdm-engine/src/labels.rs @@ -7,34 +7,34 @@ //! compared exactly. //! //! - A column is read as the executor reads it (`execute_column`): a float -//! at float32, an integer at int64, a decimal exactly; a null, and a -//! float's NaN, is no value. +//! at float32, an integer at int64, a decimal exactly; a null, and a +//! float's NaN, is no value. //! - A windowed aggregation ([`aggregate`], `execute_aggregation`) takes, of -//! each example, the rows whose value is present and whose time is after -//! its window's start, when it has one, and not after its end, in row -//! order: `SUM` of a float at float32 with Kahan's compensation (register -//! PK-2), of an integer in int64, of a decimal exactly at any size; -//! `COUNT` the rows; `AVG` a float's -//! Kahan sum at float32 over the count, an integer's at float64, a -//! decimal's exact sum over the count, at any size; `MIN` and `MAX` -//! exactly; then each read at float32, a decimal's exact value rounded -//! once to the nearest float32. -//! An example with no row is 0 for `SUM` and `COUNT` and has no label -//! for the others. +//! each example, the rows whose value is present and whose time is after +//! its window's start, when it has one, and not after its end, in row +//! order: `SUM` of a float at float32 with Kahan's compensation (register +//! PK-2), of an integer in int64, of a decimal exactly at any size; +//! `COUNT` the rows; `AVG` a float's +//! Kahan sum at float32 over the count, an integer's at float64, a +//! decimal's exact sum over the count, at any size; `MIN` and `MAX` +//! exactly; then each read at float32, a decimal's exact value rounded +//! once to the nearest float32. +//! An example with no row is 0 for `SUM` and `COUNT` and has no label +//! for the others. //! - A condition (`column_compare`, `execute_rel_op`) compares the values its -//! target has with a constant read at the target's type: at float32 for a -//! float32 target, at float64 for an integer target and a float constant, -//! at int64 for an integer target and an integer constant; a decimal -//! target exactly, at no scale but their own; a boolean, -//! text or time with its own kind. `IN` (`execute_member_op`) reads its -//! list at the target's type the same way, then tests membership. +//! target has with a constant read at the target's type: at float32 for a +//! float32 target, at float64 for an integer target and a float constant, +//! at int64 for an integer target and an integer constant; a decimal +//! target exactly, at no scale but their own; a boolean, +//! text or time with its own kind. `IN` (`execute_member_op`) reads its +//! list at the target's type the same way, then tests membership. //! - `NOT`, `AND` and `OR` ([`truth`], `execute_logical_operation`) join -//! their conditions' answers row by row, and label only the rows every -//! condition has a value for; a target's filter (`execute_filter`) keeps -//! the rows its condition holds for, which [`aggregate_kept`] aggregates. +//! their conditions' answers row by row, and label only the rows every +//! condition has a value for; a target's filter (`execute_filter`) keeps +//! the rows its condition holds for, which [`aggregate_kept`] aggregates. //! //! - [`label`] puts them together as `execute` does: the examples whose -//! target has a value keep their label, in order; the rest have none. +//! target has a value keep their label, in order; the rest have none. //! //! What the executor does by a path this does not port is refused by name //! ([`crate::not_yet`], B-3). @@ -695,11 +695,11 @@ impl Column { /// A decimal column as a target. At scale 0, where every value fits, an /// int64, as the reference reads it since `main`'s eb6c68eb. Otherwise: /// - a regression's labels are numbers, each value rounded once to the -/// nearest float32 (round 1: a regression has no classes to keep apart); +/// nearest float32 (round 1: a regression has no classes to keep apart); /// - a classification's are spelled as the reference spells its classes -/// (float64, then float32, as [`decimal_to_f64`] reads it), and where two -/// distinct values meet at one float32, each value's own digits instead, -/// so no two values become one class. +/// (float64, then float32, as [`decimal_to_f64`] reads it), and where two +/// distinct values meet at one float32, each value's own digits instead, +/// so no two values become one class. fn decimal_target(values: &[Option], scale: i8, classes: bool) -> (Vec, Labels) { let mask: Vec = values.iter().map(Option::is_some).collect(); let present: Vec = values.iter().flatten().copied().collect(); diff --git a/rust/crates/sdm-engine/src/time/mod.rs b/rust/crates/sdm-engine/src/time/mod.rs index b2e847576..880fbeff5 100644 --- a/rust/crates/sdm-engine/src/time/mod.rs +++ b/rust/crates/sdm-engine/src/time/mod.rs @@ -7,24 +7,24 @@ //! of times a source stores as text. //! //! - [`Instant`] is nanoseconds since 1970-01-01T00:00Z and [`Span`] a -//! fixed duration in nanoseconds, both `i128`, so every count an int64 -//! of any unit holds converts in and out exactly; a conversion to a -//! named unit is checked and refuses with `VALUE_OUT_OF_RANGE`. +//! fixed duration in nanoseconds, both `i128`, so every count an int64 +//! of any unit holds converts in and out exactly; a conversion to a +//! named unit is checked and refuses with `VALUE_OUT_OF_RANGE`. //! - [`Civil`] is a proleptic Gregorian date and time of day for any year -//! such a count reaches, by civil arithmetic, not `jiff`'s calendar, -//! whose timestamps stop at 9999-12-30T22:00Z. +//! such a count reaches, by civil arithmetic, not `jiff`'s calendar, +//! whose timestamps stop at 9999-12-30T22:00Z. //! - A zone's offset comes from `jiff` inside its range; outside it the -//! instant is moved by whole 400-year Gregorian cycles into a window -//! `jiff` holds on the same side of the zone's history and the offset -//! read there ([`Instant::to_local`], [`localize`]): before a zone's -//! first transition its offset is the one it starts with, after its last -//! the rule its database ends with repeats every cycle. +//! instant is moved by whole 400-year Gregorian cycles into a window +//! `jiff` holds on the same side of the zone's history and the offset +//! read there ([`Instant::to_local`], [`localize`]): before a zone's +//! first transition its offset is the one it starts with, after its last +//! the rule its database ends with repeats every cycle. //! - [`text_shapes`] is the client's two shapes `date-rules.md` names a -//! date in text by, and [`text`] reads text by the core's fixed list of -//! formats within them (`date-rules.md`, "The core's text formats"; -//! ), refusing by name any other text it is asked to read. +//! date in text by, and [`text`] reads text by the core's fixed list of +//! formats within them (`date-rules.md`, "The core's text formats"; +//! ), refusing by name any other text it is asked to read. //! - [`offset`] is a query window's `pd.DateOffset`, months by `jiff`'s -//! calendar arithmetic, clamped to the month's end. +//! calendar arithmetic, clamped to the month's end. pub mod offset; pub mod text; diff --git a/rust/crates/sdm-engine/src/time/text.rs b/rust/crates/sdm-engine/src/time/text.rs index 41da5f652..749034f05 100644 --- a/rust/crates/sdm-engine/src/time/text.rs +++ b/rust/crates/sdm-engine/src/time/text.rs @@ -7,17 +7,17 @@ //! spec this module follows. In short: //! //! - a value is read with its surrounding white space taken off, and an -//! empty value holds no time; +//! empty value holds no time; //! - its format is the one of the list it is written in, found from the -//! value alone ([`super::text_shapes`], the client's two shapes), and it -//! is parsed by that format with `jiff`'s `strtime`; +//! value alone ([`super::text_shapes`], the client's two shapes), and it +//! is parsed by that format with `jiff`'s `strtime`; //! - a day or month first value is read in the order the values read -//! together decide by the client's rule (`day_comes_first`), and a -//! two-digit year in the hundred years around the clock's; +//! together decide by the client's rule (`day_comes_first`), and a +//! two-digit year in the hundred years around the clock's; //! - a value either reads as an instant or is missing (empty); a value in -//! a format whose fields name no date or time of day, and any other -//! value, is outside the list, and the reading that meets one refuses it -//! by name rather than guess ([`Unread::Outside`]). +//! a format whose fields name no date or time of day, and any other +//! value, is outside the list, and the reading that meets one refuses it +//! by name rather than guess ([`Unread::Outside`]). //! //! No reading here follows pandas or `dateutil`: where the reference reads //! text otherwise, the difference is registered. diff --git a/rust/crates/sdm-engine/src/time/text_shapes.rs b/rust/crates/sdm-engine/src/time/text_shapes.rs index e39dfd28f..c753ef3d2 100644 --- a/rust/crates/sdm-engine/src/time/text_shapes.rs +++ b/rust/crates/sdm-engine/src/time/text_shapes.rs @@ -8,14 +8,14 @@ //! (design, row; `dependencies.md`): //! //! - year first: `YYYY-MM-DD` or `YYYY/MM/DD` (either separator in either -//! place), two-digit month and day, optionally followed by a space or -//! `T`, `HH:MM`, optional `:SS` with an optional fraction of any length, -//! and an optional zone with or without one space before it; +//! place), two-digit month and day, optionally followed by a space or +//! `T`, `HH:MM`, optional `:SS` with an optional fraction of any length, +//! and an optional zone with or without one space before it; //! - day or month first: `D/M/YY` to `DD/MM/YYYY` with `/` or `-`, a year -//! of two to four digits, optionally followed by a space or `T`, `H:MM` or -//! `HH:MM`, optional seconds and fraction, and an optional zone; +//! of two to four digits, optionally followed by a space or `T`, `H:MM` or +//! `HH:MM`, optional seconds and fraction, and an optional zone; //! - a zone is `Z`, `+HH`, `+HHMM`, `+HH:MM` (or with `-`), or a name -//! starting with a letter (`[A-Za-z][A-Za-z0-9_+/-]*`). +//! starting with a letter (`[A-Za-z][A-Za-z0-9_+/-]*`). //! //! A value matches whole, as given; whether surrounding white space is //! taken off first is the caller's rule. Digits are ASCII digits: the diff --git a/rust/crates/sdm-engine/tests/rs1.rs b/rust/crates/sdm-engine/tests/rs1.rs index b09c26110..47f6a7008 100644 --- a/rust/crates/sdm-engine/tests/rs1.rs +++ b/rust/crates/sdm-engine/tests/rs1.rs @@ -7,15 +7,15 @@ //! bump, F4): //! //! - `rs1.json`: the seed sequence's state, the raw 64-bit draws, -//! and a run of shuffles on one generator, which carries the 32-bit -//! draw's kept half from one shuffle to the next, for seeds from 0 to -//! past 2**200 (more entropy words than the pool holds); +//! and a run of shuffles on one generator, which carries the 32-bit +//! draw's kept half from one shuffle to the next, for seeds from 0 to +//! past 2**200 (more entropy words than the pool holds); //! - `rs1-draws.json` (`rs1_record.py`): model of the example -//! shuffle for each of its seeds, 0, 42, 123456789 and 2**70 (a fresh -//! generator's state, raw draws and `shuffle(arange(50))`), and numpy's -//! bounded draw itself, `random_interval`, over maxima that cross the -//! 32-bit path's edge and reach the 64-bit path's end, called from -//! numpy's compiled library. +//! shuffle for each of its seeds, 0, 42, 123456789 and 2**70 (a fresh +//! generator's state, raw draws and `shuffle(arange(50))`), and numpy's +//! bounded draw itself, `random_interval`, over maxima that cross the +//! 32-bit path's edge and reach the 64-bit path's end, called from +//! numpy's compiled library. //! //! [`check`] compares a document with the port and counts what it //! compared; it fails closed: a perturbed draw, and evidence missing, diff --git a/rust/crates/sdm-host/src/codec.rs b/rust/crates/sdm-host/src/codec.rs index 855700eec..d1cba8220 100644 --- a/rust/crates/sdm-host/src/codec.rs +++ b/rust/crates/sdm-host/src/codec.rs @@ -8,13 +8,13 @@ //! envelope only: input-sized fields travel as attachments. //! //! - From the host, a message past [`CONTROL_CAP`] bytes, or one the -//! contract cannot read, is refused with `INVALID_MESSAGE` before anything -//! reads it ([`decode`]). +//! contract cannot read, is refused with `INVALID_MESSAGE` before anything +//! reads it ([`decode`]). //! - From the core, nothing is truncated, refused or dropped for size: a -//! message whose encoding would pass the cap travels whole as a bytes -//! attachment of its JSON, named by a minimal `attached` message -//! ([`write`], [`write_error`]). A message the core's own -//! encoder refuses is a defect in the core, `INTERNAL_ERROR`. +//! message whose encoding would pass the cap travels whole as a bytes +//! attachment of its JSON, named by a minimal `attached` message +//! ([`write`], [`write_error`]). A message the core's own +//! encoder refuses is a defect in the core, `INTERNAL_ERROR`. use sdm_contract::data::AttachmentId; use sdm_contract::{ControlMessage, CoreError, Detail, Scalar, Sentence}; diff --git a/rust/crates/sdm-host/src/runs.rs b/rust/crates/sdm-host/src/runs.rs index 2180bcc86..fc42de483 100644 --- a/rust/crates/sdm-host/src/runs.rs +++ b/rust/crates/sdm-host/src/runs.rs @@ -11,15 +11,15 @@ //! threads. Every call: //! //! - is contained: a panic anywhere below it poisons the run and comes back -//! as `INTERNAL_ERROR` with `fields.panic = true` and the panic's -//! location, never its message, and never as an unwind through the -//! binding; [`contain`] reports the panic out of band, so no error a host -//! hands in can poison a run; +//! as `INTERNAL_ERROR` with `fields.panic = true` and the panic's +//! location, never its message, and never as an unwind through the +//! binding; [`contain`] reports the panic out of band, so no error a host +//! hands in can poison a run; //! - is refused with `RUN_BUSY` (kind `protocol`) while another call is in -//! the run, rather than waiting; +//! the run, rather than waiting; //! - reads control messages through the codecs, under the control cap, and -//! writes them whole: one whose encoding would pass the cap travels as an -//! attachment numbered in the run's sequence. +//! writes them whole: one whose encoding would pass the cap travels as an +//! attachment numbered in the run's sequence. //! //! Attachments follow. The core numbers its own from 1 in each //! run, and the host takes them by id ([`HostRun::take_attachment`]); an diff --git a/rust/crates/sdm-host/tests/native_admission_vectors.rs b/rust/crates/sdm-host/tests/native_admission_vectors.rs index 51ec4b421..d5a37bca2 100644 --- a/rust/crates/sdm-host/tests/native_admission_vectors.rs +++ b/rust/crates/sdm-host/tests/native_admission_vectors.rs @@ -6,13 +6,13 @@ //! `jiff`. //! //! - Zone answers go through the host's resolver, once over the vendored -//! 2026c directory and once over the bundled database; every answer must -//! equal the reference's, and each run first checks that its database is -//! the version the expectations are tied to. +//! 2026c directory and once over the bundled database; every answer must +//! equal the reference's, and each run first checks that its database is +//! the version the expectations are tied to. //! - `DateOffset` answers must equal the reference's. //! - Unit conversions, single values and PQL literals repeat the -//! probe's own reading with `jiff` in place of `chrono`, and must equal the -//! probe's `chrono` answers, with explicit exceptions named here. +//! probe's own reading with `jiff` in place of `chrono`, and must equal the +//! probe's `chrono` answers, with explicit exceptions named here. #![allow( clippy::disallowed_methods, diff --git a/rust/crates/sdm-pql/src/dates.rs b/rust/crates/sdm-pql/src/dates.rs index 859cebc67..faaacd824 100644 --- a/rust/crates/sdm-pql/src/dates.rs +++ b/rust/crates/sdm-pql/src/dates.rs @@ -13,12 +13,12 @@ //! digits joined by `:` (D1). //! //! - A literal whose every run has four digits or fewer is read by D2 to -//! D7 and D9 ([`Reader::plain`]); a day-first reading warns by D4. +//! D7 and D9 ([`Reader::plain`]); a day-first reading warns by D4. //! - A literal with a run of five digits or more (D8) is not read: it is -//! [`Reading::Packed`], refused by name before any field is read, by the -//! owner's decision of 2026-10-12 (register DT-5). The -//! reference reads such a literal through `dateutil` (a packed date, a -//! long zero-padded field) or refuses it; the core reads neither. +//! [`Reading::Packed`], refused by name before any field is read, by the +//! owner's decision of 2026-10-12 (register DT-5). The +//! reference reads such a literal through `dateutil` (a packed date, a +//! long zero-padded field) or refuses it; the core reads neither. //! //! What the reader takes from the host: the current year, which places a //! two-digit year (D5), and whether the platform's `strftime` pads a year diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index be968b948..aa61ebbe8 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -8,32 +8,32 @@ //! design's "Host-held semantics" rows it moved): //! //! - [`prepare`] is `SDMClient._prepare`: the model the request is sent to -//! (`UNKNOWN_MODEL` for one the contract does not carry), the outputs -//! asked for (the task's default ones when the caller named none, an -//! empty list refused `INVALID_REQUEST`) and `check_request_against_model` -//! (the request's shape, its task's kind, each output and the context's -//! size against what the model serves); +//! (`UNKNOWN_MODEL` for one the contract does not carry), the outputs +//! asked for (the task's default ones when the caller named none, an +//! empty list refused `INVALID_REQUEST`) and `check_request_against_model` +//! (the request's shape, its task's kind, each output and the context's +//! size against what the model serves); //! - [`nothing_to_score`] is `Request.nothing_to_score`: a request with no -//! row to score is answered without a call (`_empty_answer`, [`empty`]); +//! row to score is answered without a call (`_empty_answer`, [`empty`]); //! - [`preflight`] is the relational adapter's `_preflight`: an empty -//! context, more classes than the model takes and a classification whose -//! context has one answer refused, a regression's constant context given -//! as a notice, and an explanation's row count and run mode; +//! context, more classes than the model takes and a classification whose +//! context has one answer refused, a regression's constant context given +//! as a notice, and an explanation's row count and run mode; //! - [`body`] is `prediction_plan`'s call of one payload: the payload with -//! the model named, the outputs asked for (a regression's with its -//! statistic), the request's id in its metadata, and only the inference -//! options the model reads (`_inference`); +//! the model named, the outputs asked for (a regression's with its +//! statistic), the request's id in its metadata, and only the inference +//! options the model reads (`_inference`); //! - [`layout`] is `normalize` and `_named_entities`, with the answer's -//! classes given back as the target's own type (`as_declared_target`): -//! each answer checked against the row it claims and given back under the -//! entity it is about, read back as the key's dtype (`as_declared`), or, -//! where the payload names no entity, given back as written, as a -//! [`Part`]; +//! classes given back as the target's own type (`as_declared_target`): +//! each answer checked against the row it claims and given back under the +//! entity it is about, read back as the key's dtype (`as_declared`), or, +//! where the payload names no entity, given back as written, as a +//! [`Part`]; //! - [`merge`] is `PredictionResult.merge`, the parts of one request put -//! together, and [`absent`] is `_report_absent`'s finding, the outputs the -//! caller named that no answer carries; +//! together, and [`absent`] is `_report_absent`'s finding, the outputs the +//! caller named that no answer carries; //! - [`table`] lays a part out as the result: one row per answer, ordered -//! by `row_index` as `PredictionResult.to_pandas` orders them. +//! by `row_index` as `PredictionResult.to_pandas` orders them. //! //! What the result does not carry yet is refused by name, never dropped //! (`UNSUPPORTED_REQUEST`, B-4): the tabular model on this relational diff --git a/rust/crates/sdm-testkit/tests/carried_rows.rs b/rust/crates/sdm-testkit/tests/carried_rows.rs index 741158fa9..6591c8c59 100644 --- a/rust/crates/sdm-testkit/tests/carried_rows.rs +++ b/rust/crates/sdm-testkit/tests/carried_rows.rs @@ -7,15 +7,15 @@ //! package's test author before the implementation: //! //! - P2--1: "The second data layer drops a whole key list when one name -//! in it is null, where `sdm-data` drops only the null name -//! (`sdm-testkit/src/layer.rs`)": a key's and a foreign key's column list -//! holding a null name is read with the null name dropped and every other -//! name kept, as `sdm-data`'s reader (`read.rs`, `names`) reads it; +//! in it is null, where `sdm-data` drops only the null name +//! (`sdm-testkit/src/layer.rs`)": a key's and a foreign key's column list +//! holding a null name is read with the null name dropped and every other +//! name kept, as `sdm-data`'s reader (`read.rs`, `names`) reads it; //! - P2--2: "The memory test provider... does not answer -//! `column_ranges`, so on a table bigger than the preview it picks the -//! time column from the preview (`sdm-testkit/src/memory.rs`)": the -//! provider declares `column_ranges` and answers it over every row of a -//! table whose times all lie past its first rows. +//! `column_ranges`, so on a table bigger than the preview it picks the +//! time column from the preview (`sdm-testkit/src/memory.rs`)": the +//! provider declares `column_ranges` and answers it over every row of a +//! table whose times all lie past its first rows. //! use std::collections::BTreeMap; use std::sync::Arc; diff --git a/rust/crates/sdm-testkit/tests/memory_provider.rs b/rust/crates/sdm-testkit/tests/memory_provider.rs index c4dc2f286..2e8dc3323 100644 --- a/rust/crates/sdm-testkit/tests/memory_provider.rs +++ b/rust/crates/sdm-testkit/tests/memory_provider.rs @@ -8,11 +8,11 @@ //! (`sdm_testkit::layer`): //! //! - a prediction whose two schema tables read one source is the one the -//! same provider gives over two equal sources, whole; +//! same provider gives over two equal sources, whole; //! - against DuckDB's SQL, a prediction's description, its result and its -//! request but the in-context examples are DuckDB's; the examples are -//! drawn by each provider's own seeded sample (DuckDB's reservoir -//! shuffles even a table it keeps whole), so they are not compared; +//! request but the in-context examples are DuckDB's; the examples are +//! drawn by each provider's own seeded sample (DuckDB's reservoir +//! shuffles even a table it keeps whole), so they are not compared; //! - the swapped layer refuses a SQL dialect by name. use std::path::PathBuf; diff --git a/rust/crates/sdm-testkit/tests/numeric_spelling.rs b/rust/crates/sdm-testkit/tests/numeric_spelling.rs index 6f5096828..f0d3a0a65 100644 --- a/rust/crates/sdm-testkit/tests/numeric_spelling.rs +++ b/rust/crates/sdm-testkit/tests/numeric_spelling.rs @@ -25,20 +25,20 @@ //! number of hops (two: a user, its orders, and the orders' user again). //! //! - `spelling.same`: each order's link written `.0`, as its user's -//! key is: answered (the control); +//! key is: answered (the control); //! - `spelling.equal`: each link written `.00`: one number, so one -//! key: answered, and the call the core wrote is `spelling.same`'s (the -//! `payload` rule: but its request id). When written, the non-SQL -//! provider, which looks keys up by the core's key identity, found no -//! order for any user (the text `.0` is not `.00`), and -//! the call carried no order; on Postgres, whose SQL joins the two by -//! value, the second hop took `.00`'s user for another node and -//! failed (`INTERNAL_ERROR`, "a row looked up was already reached"), which -//! `test_postgres_source_contracts.py` holds; +//! key: answered, and the call the core wrote is `spelling.same`'s (the +//! `payload` rule: but its request id). When written, the non-SQL +//! provider, which looks keys up by the core's key identity, found no +//! order for any user (the text `.0` is not `.00`), and +//! the call carried no order; on Postgres, whose SQL joins the two by +//! value, the second hop took `.00`'s user for another node and +//! failed (`INTERNAL_ERROR`, "a row looked up was already reached"), which +//! `test_postgres_source_contracts.py` holds; //! - `spelling.different`: each link written `.01`, another number, -//! which no user's key is, though a double rounds every one of them to a -//! user's (ids past 2**53 are a double's every other integer): answered, -//! with no order linked to any user, so the call is not `spelling.same`'s. +//! which no user's key is, though a double rounds every one of them to a +//! user's (ids past 2**53 are a double's every other integer): answered, +//! with no order linked to any user, so the call is not `spelling.same`'s. //! use std::collections::BTreeSet; diff --git a/rust/crates/sdm-testkit/tests/payload_rules.rs b/rust/crates/sdm-testkit/tests/payload_rules.rs index cc90a3b3f..bfdae4419 100644 --- a/rust/crates/sdm-testkit/tests/payload_rules.rs +++ b/rust/crates/sdm-testkit/tests/payload_rules.rs @@ -9,20 +9,20 @@ //! holds `python-core` to as well, so the two hosts agree through them. //! //! - Exit "on both core hosts, over `base.binary` and `base.multiclass`, -//! the classes, their order, the positive class and the context-leakage -//! payload fields match the reference", from the real payload (not the -//! test-only labels operation): the task, the context's and predicted -//! rows, and each related table's columns and the instance ids it keeps -//! (the context that answers the question taken out; a static target's -//! column left out). +//! the classes, their order, the positive class and the context-leakage +//! payload fields match the reference", from the real payload (not the +//! test-only labels operation): the task, the context's and predicted +//! rows, and each related table's columns and the instance ids it keeps +//! (the context that answers the question taken out; a static target's +//! column left out). //! - Exit "python-core and rust-host public outcomes match for context -//! leakage and limits": the leakage fields above, and the entity count's -//! limit. The reference at b413e9ad refused 1,001 entities -//! `UNSUPPORTED_REQUEST`; since main's 1fc5a686 (6298913c) it takes up to -//! 10,000, so on `base.regression` (8 users) it refuses the 1,001 indices -//! `INVALID_REQUEST` for another reason, while the core's 1,000-entity -//! limit, field `indices`, still refuses them first, `UNSUPPORTED_REQUEST` -//! (B-3, until ); `python-core` is held to the same. +//! leakage and limits": the leakage fields above, and the entity count's +//! limit. The reference at b413e9ad refused 1,001 entities +//! `UNSUPPORTED_REQUEST`; since main's 1fc5a686 (6298913c) it takes up to +//! 10,000, so on `base.regression` (8 users) it refuses the 1,001 indices +//! `INVALID_REQUEST` for another reason, while the core's 1,000-entity +//! limit, field `indices`, still refuses them first, `UNSUPPORTED_REQUEST` +//! (B-3, until ); `python-core` is held to the same. //! //! Before the core refuses both classification cases by its rule //! `classification_task`, and these tests say so. diff --git a/rust/crates/sdm-testkit/tests/payload_shape_validation.rs b/rust/crates/sdm-testkit/tests/payload_shape_validation.rs index bdd25db42..3238ffd55 100644 --- a/rust/crates/sdm-testkit/tests/payload_shape_validation.rs +++ b/rust/crates/sdm-testkit/tests/payload_shape_validation.rs @@ -6,20 +6,20 @@ //! independent test author), at `rust-host`'s public route: //! //! - Exit 1, the lock refuses a payload whose shape changed: the payload -//! `rust-host` materializes for each case that reaches materialize holds -//! to the contract's locked `$defs/StatePayload`, and every one-step -//! change of its shape (a key the format requires dropped, a key it does -//! not name added, a value retyped, a dtype or row format the wire does -//! not name) is refused, at every path outside the open data (a row's -//! cells). A lock that took any of them would not be the typed payload's. +//! `rust-host` materializes for each case that reaches materialize holds +//! to the contract's locked `$defs/StatePayload`, and every one-step +//! change of its shape (a key the format requires dropped, a key it does +//! not name added, a value retyped, a dtype or row format the wire does +//! not name) is refused, at every path outside the open data (a row's +//! cells). A lock that took any of them would not be the typed payload's. //! - Exit 2, rust-host follows the core's word on a held sampler state: a -//! materializer handed a state the core refuses (another version, a memo -//! that does not read, a memo no run left) answers that call with the -//! refusal, and its next call starts afresh, exactly as a materializer -//! that never held one (the same fetches and answer); the one after it is -//! handed the state that fresh run left, as a materializer's second call. -//! A host that ignored `drop_held` would hand the refused state back and -//! be refused again. +//! materializer handed a state the core refuses (another version, a memo +//! that does not read, a memo no run left) answers that call with the +//! refusal, and its next call starts afresh, exactly as a materializer +//! that never held one (the same fetches and answer); the one after it is +//! handed the state that fresh run left, as a materializer's second call. +//! A host that ignored `drop_held` would hand the refused state back and +//! be refused again. use std::path::PathBuf; diff --git a/rust/crates/sdm-testkit/tests/postgres_dialect.rs b/rust/crates/sdm-testkit/tests/postgres_dialect.rs index 9596edc86..0a6d16ce4 100644 --- a/rust/crates/sdm-testkit/tests/postgres_dialect.rs +++ b/rust/crates/sdm-testkit/tests/postgres_dialect.rs @@ -6,22 +6,22 @@ //! independent test author before the implementation). //! //! - The staging rule's markers: the data layer renders Postgres -//! (`sdm_host::providers`, which the binding reports as `PROVIDERS`), -//! and rust-host has a Postgres provider (`sdm_testkit::PROVIDERS`, the -//! replay of recorded fetch results), so the core legs run on -//! Postgres. +//! (`sdm_host::providers`, which the binding reports as `PROVIDERS`), +//! and rust-host has a Postgres provider (`sdm_testkit::PROVIDERS`, the +//! replay of recorded fetch results), so the core legs run on +//! Postgres. //! - Every request kind the core asks, on every base of -//! `contract/request-kinds.json`, is planned for Postgres through the -//! root's data access (so each optional request is one the dialect -//! declares) as SQL statements of the Postgres dialect; a keyed request -//! binds its key set as a parameter of a statement (`SqlParam::KeySet`, -//! `P1 d:131`: the binding is the dialect's). +//! `contract/request-kinds.json`, is planned for Postgres through the +//! root's data access (so each optional request is one the dialect +//! declares) as SQL statements of the Postgres dialect; a keyed request +//! binds its key set as a parameter of a statement (`SqlParam::KeySet`, +//! `P1 d:131`: the binding is the dialect's). //! - Choice 12's relation forms on Postgres: one part or two (schema and -//! table) plan, each part quoted alone (a dotted name one identifier, a -//! double quote doubled); none, or three, is refused by name (rule -//! `relation`), as DuckDB refuses none or four. The same forms are -//! executed on a real Postgres by the client's -//! `tests_test_ops/test_the_postgres_provider.py`. +//! table) plan, each part quoted alone (a dotted name one identifier, a +//! double quote doubled); none, or three, is refused by name (rule +//! `relation`), as DuckDB refuses none or four. The same forms are +//! executed on a real Postgres by the client's +//! `tests_test_ops/test_the_postgres_provider.py`. use std::path::PathBuf; diff --git a/rust/crates/sdm-testkit/tests/replay_past_recording.rs b/rust/crates/sdm-testkit/tests/replay_past_recording.rs index c9142a055..c79d2d4ee 100644 --- a/rust/crates/sdm-testkit/tests/replay_past_recording.rs +++ b/rust/crates/sdm-testkit/tests/replay_past_recording.rs @@ -23,9 +23,9 @@ //! sum-window`'s, its id and status its own. //! //! - The diagnostic case before the answered one: the run succeeds and -//! writes both lines, the diagnostic case's a refusal, `PROVIDER_ERROR`. +//! writes both lines, the diagnostic case's a refusal, `PROVIDER_ERROR`. //! - The acceptance case before the answered one: the run fails, naming -//! the case and the fetch past the recording, and writes no file. +//! the case and the fetch past the recording, and writes no file. #![allow( clippy::disallowed_methods, diff --git a/rust/crates/sdm-testkit/tests/snowflake.rs b/rust/crates/sdm-testkit/tests/snowflake.rs index f5724de1d..304096ae7 100644 --- a/rust/crates/sdm-testkit/tests/snowflake.rs +++ b/rust/crates/sdm-testkit/tests/snowflake.rs @@ -8,25 +8,25 @@ //! 4.8.0, database and schema recorded as `{catalog}` and `{schema}`). //! //! - Goldens: each request kind the `snowflake` dialect renders without a -//! key set is planned under the source's database and schema, and each -//! statement is the one the old client sent for the same step, text and -//! parameters exactly (the catalog reads, the preview, the key counts, the -//! time bounds, the entity sample, the column ranges, the row count's -//! listing, a semantic view's YAML read). +//! key set is planned under the source's database and schema, and each +//! statement is the one the old client sent for the same step, text and +//! parameters exactly (the catalog reads, the preview, the key counts, the +//! time bounds, the entity sample, the column ranges, the row count's +//! listing, a semantic view's YAML read). //! - The catalog reader: each recorded `DESCRIBE TABLE`, `SHOW IMPORTED //! KEYS`, table listing and session read, rebuilt in the Python row -//! adapter's text form (every column nullable text, in the result's -//! order, each non-null cell its `str`), read into the contract's catalog -//! answer as the old client reads the rows (`SNOW/table.py:160-189`, -//! `RE/graph.py:1024-1051`), the type map included (`:330-383`). +//! adapter's text form (every column nullable text, in the result's +//! order, each non-null cell its `str`), read into the contract's catalog +//! answer as the old client reads the rows (`SNOW/table.py:160-189`, +//! `RE/graph.py:1024-1051`), the type map included (`:330-383`). //! - `conform` over the connector's own Arrow as recorded: narrow integers -//! and a `NUMBER(38,0)` that fits read as `int64`, one past it left a -//! decimal, a `NUMBER(p,s)` with a fraction read as a double, a zoned time -//! at its UTC instant with no zone (`RE/base/table.py:1093-1186`). +//! and a `NUMBER(38,0)` that fits read as `int64`, one past it left a +//! decimal, a `NUMBER(p,s)` with a fraction read as a double, a zoned time +//! at its UTC instant with no zone (`RE/base/table.py:1093-1186`). //! - The data layer's answers for Snowflake: a failure with errno 2003 is -//! "not found", nothing else is; a name's spellings are as given, then -//! all three parts upper-cased, once; the session's parts fill only those -//! the source leaves out; the capabilities. +//! "not found", nothing else is; a name's spellings are as given, then +//! all three parts upper-cased, once; the session's parts fill only those +//! the source leaves out; the capabilities. use std::fs::File; use std::io::BufReader; diff --git a/rust/crates/sdm-testkit/tests/warehouse_sampling.rs b/rust/crates/sdm-testkit/tests/warehouse_sampling.rs index bdd940d92..5bdc30c53 100644 --- a/rust/crates/sdm-testkit/tests/warehouse_sampling.rs +++ b/rust/crates/sdm-testkit/tests/warehouse_sampling.rs @@ -8,11 +8,11 @@ //! `neighbor_lookback` (contract v28), each request it was asked logged. //! //! - every table's rows counted once, in the graph's order, before the -//! entity sample, which names its table's rows (`num_rows_dict`, the -//! `TABLESAMPLE` percentage's denominator); +//! entity sample, which names its table's rows (`num_rows_dict`, the +//! `TABLESAMPLE` percentage's denominator); //! - each neighbor read by a foreign key over a table with a time column -//! bounded below by a `start` seed column, exclusive, beside the anchor -//! (`_by_fkey`'s look-back window); +//! bounded below by a `start` seed column, exclusive, beside the anchor +//! (`_by_fkey`'s look-back window); //! - a provider declaring neither is asked no row count and no look-back. use std::path::PathBuf; diff --git a/rust/crates/sdm-wire/tests/payload_schema_from_types.rs b/rust/crates/sdm-wire/tests/payload_schema_from_types.rs index daa9b5450..b5b35e634 100644 --- a/rust/crates/sdm-wire/tests/payload_schema_from_types.rs +++ b/rust/crates/sdm-wire/tests/payload_schema_from_types.rs @@ -7,15 +7,15 @@ //! typed `sdm-wire` structs. //! //! - The committed `state-payload.json` is, byte for byte, what the types -//! generate. +//! generate. //! - The definition the contract lock covers (`$defs/StatePayload` of the -//! schema file `contract.lock` names for this version) is that same -//! schema, with the state's version alone added; so is the one -//! `sdm_contract::schema_json` generates. Neither holds a hand-kept copy. +//! schema file `contract.lock` names for this version) is that same +//! schema, with the state's version alone added; so is the one +//! `sdm_contract::schema_json` generates. Neither holds a hand-kept copy. //! - The schema names every member the types write: a payload with every -//! optional member present serializes to keys each of which the schema's -//! node at that path names, and each required key is one the types -//! always write. +//! optional member present serializes to keys each of which the schema's +//! node at that path names, and each required key is one the types +//! always write. #![allow( clippy::disallowed_methods, From 74eb425336aa98ec2c991f199c2e926a11f74b75 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Thu, 5 Nov 2026 17:04:05 -0800 Subject: [PATCH 08/24] Take arrow 58.4.1 for RUSTSEC-2026-0442 arrow-data 58.4.0 can read past a RunEndEncoded child array, and the wheel and the Java library link it, so cargo deny fails. Only the arrow crates move; the Rust notices, the type vectors' arrow-schema version and their digest follow. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- contract/type-set-vectors.json | 2 +- .../THIRD-PARTY-NOTICES-RUST.txt | 60 +++++++++---------- rust/Cargo.lock | 48 +++++++-------- tests/compatibility/manifest.json | 7 ++- 4 files changed, 61 insertions(+), 56 deletions(-) diff --git a/contract/type-set-vectors.json b/contract/type-set-vectors.json index c1318da3d..8aa9ee79b 100644 --- a/contract/type-set-vectors.json +++ b/contract/type-set-vectors.json @@ -1,7 +1,7 @@ { "format": "sdm-type-set-vectors/1", "about": "Arrow type admission, reachability and schema-inference expectations.", - "arrow_schema": "58.4.0", + "arrow_schema": "58.4.1", "derived_from": [ "rust/crates/sdm-host/src/admission.rs:9-22 (the admission table) and :101-149 (convert_type)", "rust/crates/sdm-contract/src/types.rs:291-379 (ColumnType::from_arrow_unchecked, the closed set) and :250-256 (a decimal's precision and scale)", diff --git a/packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt b/packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt index 29b39b6b6..26324c8a5 100644 --- a/packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt +++ b/packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt @@ -15,14 +15,14 @@ ahash 0.8.12 MIT OR Apache-2.0 https://github.com/tkaitchuck/ahash aho-corasick 1.1.5 Unlicense OR MIT https://github.com/BurntSushi/aho-corasick arc-swap 1.9.2 MIT OR Apache-2.0 https://github.com/vorner/arc-swap arraydeque 0.5.1 MIT/Apache-2.0 https://github.com/andylokandy/arraydeque -arrow-array 58.4.0 Apache-2.0 AND MIT https://github.com/apache/arrow-rs -arrow-buffer 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-cast 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-data 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-ord 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-row 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-schema 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs -arrow-select 58.4.0 Apache-2.0 https://github.com/apache/arrow-rs +arrow-array 58.4.1 Apache-2.0 AND MIT https://github.com/apache/arrow-rs +arrow-buffer 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-cast 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-data 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-ord 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-row 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-schema 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs +arrow-select 58.4.1 Apache-2.0 https://github.com/apache/arrow-rs atoi 2.0.0 MIT https://github.com/pacman82/atoi-rs base64 0.22.1 MIT OR Apache-2.0 https://github.com/marshallpierce/rust-base64 bit-set 0.8.0 Apache-2.0 OR MIT https://github.com/contain-rs/bit-set @@ -449,7 +449,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -arrow-array 58.4.0 +arrow-array 58.4.1 ================== Licence: Apache-2.0 AND MIT @@ -695,81 +695,81 @@ This product includes software from the compact-thrift project (Apache 2.0) * https://github.com/jhorstmann/compact-thrift -arrow-buffer 58.4.0 +arrow-buffer 58.4.1 =================== Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-cast 58.4.0 +arrow-cast 58.4.1 ================= Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-data 58.4.0 +arrow-data 58.4.1 ================= Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-ord 58.4.0 +arrow-ord 58.4.1 ================ Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-row 58.4.0 +arrow-row 58.4.1 ================ Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-schema 58.4.0 +arrow-schema 58.4.1 =================== Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) -arrow-select 58.4.0 +arrow-select 58.4.1 =================== Licence: Apache-2.0 --- LICENSE.txt --- -(the same text as LICENSE.txt of arrow-array 58.4.0 above) +(the same text as LICENSE.txt of arrow-array 58.4.1 above) --- NOTICE.txt --- -(the same text as NOTICE.txt of arrow-array 58.4.0 above) +(the same text as NOTICE.txt of arrow-array 58.4.1 above) atoi 2.0.0 diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7ef46744d..5a7370831 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -81,9 +81,9 @@ checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" [[package]] name = "arrow" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "c943b9bec582e47f8394e44713c3cbfbe645fd049e3a7771585f54849c710fe6" dependencies = [ "arrow-arith", "arrow-array", @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "8b13b85a6a547a1669114e418f6ab3c3e39c2edb1b8be5ab2889e5739a433ae9" dependencies = [ "arrow-array", "arrow-buffer", @@ -113,9 +113,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "7b11eb61b95868592ad6edbd537465a6940eb3c81a875addecec24283301b267" dependencies = [ "ahash", "arrow-buffer", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "53ee609a07eb721c2e4a7669d7bb88e2106c560e7dec1219096fecf5eae356d1" dependencies = [ "bytes", "half", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "9575323ffeb4bc6955f39bf8716f734671a32780f1fac9ba6c2891048fc0ea1f" dependencies = [ "arrow-array", "arrow-buffer", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "10791765e6927cdd8ace5bf0e51a69f38a1c3aded2db93561b003c332d02ee2a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "165051f1f996f19b50c4d2cdb7ed02ddd400d8d984b422bf6848507eee099c3d" dependencies = [ "arrow-array", "arrow-buffer", @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "908a1b5aeefce5c6eb2a0a4e98f4ccacbd441fe0ed23fb799e634cd0135821e5" dependencies = [ "arrow-array", "arrow-buffer", @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "5383b1676492a509032f06fa4ca9522c886036aec4ebe8261bebfbce904af667" dependencies = [ "arrow-array", "arrow-buffer", @@ -218,18 +218,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "8506ad465ebb9483bb6efa1e2e78a313549122c6dd67dc2f7087297b762eaede" dependencies = [ "bitflags 2.13.2", ] [[package]] name = "arrow-select" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "fd0f0a0d641356ae2df829af5d635ed9fb93fbeb0d39f3239487cb5fdb37a6ef" dependencies = [ "ahash", "arrow-array", @@ -241,9 +241,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "58.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "bd245a358d52e77bfcb99dc3e21f04b947383271b7a0b9ca8feec43e9039234d" dependencies = [ "arrow-array", "arrow-buffer", diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index d9e4675f8..e5d241a65 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -8256,6 +8256,11 @@ "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "api_contracts/column-explanations", "reason": "Column analysis reads the labelled rows a request sends, as on main, so the request keeps no pre-trim delta and its tests are replaced by one over the sent rows." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "type_contracts/arrow-admission-and-inference", + "reason": "Arrow moves from 58.4.0 to 58.4.1 for RUSTSEC-2026-0442, so the vectors name the locked arrow-schema version; every type vector and expected outcome is unchanged. Update the whole-file digest." } ], "capabilities": [ @@ -9795,7 +9800,7 @@ ] }, "evidence_level": "local-boundary", - "vectors_sha256": "7affcc95bf8f09ab93695adf80a42d05746722ae28e87e9d7a406c7e5d4bd904" + "vectors_sha256": "b1ae6bd5fd9eebe14020b88ce02326696f70757a7a6535cd79da5e491995adef" }, { "id": "semantic-types", From 73f7d35f036d190b6ffb2e40658fdde1a3986a9c Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Fri, 6 Nov 2026 14:25:02 -0800 Subject: [PATCH 09/24] Close the gaps the Databricks parity runs found - A column of a type a described table does not take is refused naming the column, its table and its Arrow type, under the same code and rule. - A batch session the endpoint loses again after one recreation ends reuse, and the remaining calls are sent stateless. - Java Request.payloads() leaves out the client-only keys, as Python does. - CHANGELOG: the column type refusal, batch sessions, and the Spark time zone and decimal fixes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 16 +++++++ docs/about/architecture.md | 2 +- docs/development/engine-boundaries.md | 2 +- .../src/main/java/com/nvidia/sdm/Request.java | 26 ++++++++++- .../com/nvidia/sdm/MainMergeContractTest.java | 3 +- .../tests/test_automatic_batch_sessions.py | 14 +++--- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 1 + rust/crates/sdm-contract/src/error.rs | 1 + rust/crates/sdm-contract/src/wording.rs | 3 ++ rust/crates/sdm-core/src/batch_session.rs | 28 +++++++---- rust/crates/sdm-schema/src/infer.rs | 46 ++++++++++++++----- .../crates/sdm-schema/tests/declared_types.rs | 34 +++++++++++--- 13 files changed, 139 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b51951c0f..adc6c86af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,16 @@ The public API snapshot records this version update; publication remains pending metric view whose expression names a column its base table lacks is one of them: the warehouse refuses the read, where 1.0.0 refused the view as `INVALID_REQUEST` before reading it. +- A column of a type a described table does not take, such as a list of + booleans, a struct or a map, raises `UnsupportedRequestError` with code + `UNSUPPORTED_REQUEST` and `details['rule']` `column_type`. The message and + details name the column, its table and its Arrow type. 1.0.0 raised a + `ValueError` naming the column and table. +- `predict()` on a request gathered in several calls that share one context + scores them through a temporary endpoint session, so the context is + uploaded once. 1.0.0 sent every call stateless. An expired session is + recreated once; if the endpoint loses it again, as replicas that do not + share sessions do, the remaining calls are sent stateless. - A related table sampled from frames sends its own primary key, where 1.0.0 sent a generated `__node_id`. @@ -133,6 +143,12 @@ The public API snapshot records this version update; publication remains pending - A request's description leaves out a wire table carrying an integer past the float range on every pandas release. pandas 3.1 had started to build a frame around one, where earlier releases refused it. +- On Databricks through Spark, a `TIMESTAMP` column is read in UTC, so the + anchor time and the rows a window takes no longer move with the client + machine's time zone. 1.0.0 read its range as local times. +- On Databricks through Spark, a `DECIMAL` value with a fraction is sent + exactly, such as `997.67`. 1.0.0 sent it through a float, such as + `997.6700000000001`. ### Removed diff --git a/docs/about/architecture.md b/docs/about/architecture.md index 8fb170829..42fff36c7 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -45,7 +45,7 @@ The internal protocol lets Rust request host I/O without embedding Python or Jav | `Schema.from_source` / Java `Schema.fromSource` | Describe selected tables in a connected source | | `PredictiveQuery.materialize` | Validate a question and gather context plus rows to score for the selected model | | Direct `Request` | Supply context and prediction data directly for Tabular | -| `client.predict` | Send a stateless request and return interpreted predictions | +| `client.predict` | Send a request and return interpreted predictions; batches that share one context reuse a temporary session, and are sent stateless where the endpoint does not keep it | | `client.fit` | Pin context in a model endpoint session; this does not train model weights | | Fitted handle `predict` / `close` | Score against pinned context, then release the session | diff --git a/docs/development/engine-boundaries.md b/docs/development/engine-boundaries.md index 8468d07ef..2cc71aefa 100644 --- a/docs/development/engine-boundaries.md +++ b/docs/development/engine-boundaries.md @@ -36,7 +36,7 @@ The [compatibility manifest](../../tests/compatibility/manifest.json) maps exact | Visualization | Hosts render metadata and share renderer assets. Mermaid/HTML is offline; only explicit PNG/SVG image export sends schema text to the documented external renderer. Escaping and deterministic diagram meaning are tested in both hosts. | | RelBench loading | Hosts download/cache/verify pinned files, then use ordinary schema admission. Offline mode never connects; hash failures cannot replace a valid cache entry. Rust does not own HTTP or cache utilities. | | Batch sessions | Rust reuses a temporary session only for matching actual context/task/schema/output/inference semantics, never merely matching seeds. Reuse is request-local; completed batches survive later failures. | -| Retry/cleanup | Hosts implement one bounded transport retry policy; no outer retry multiplier or blind retry of uncertain session creation. Rust permits one expiry recreation; cleanup cannot mask the primary error. | +| Retry/cleanup | Hosts implement one bounded transport retry policy; no outer retry multiplier or blind retry of uncertain session creation. Rust permits one expiry recreation; a session missing again ends reuse, and the remaining calls are sent stateless. Cleanup cannot mask the primary error. | | Typed inference settings | Hosts collect explicitly supplied values; Rust owns defaults and task-specific validation. A mapping and its typed equivalent must behave alike. | | Optional text summaries | Explicit host HTTP operation with a caller-selected endpoint. Existing prose returns without a request; no ambient credentials, implicit prediction-time call or mutation of the explanation. | | Tabular context ordering | Rust orders supported labels, preserving sampled order between equal labels. Compare represented numeric values consistently; custom comparison functions are unsupported. This does not change sampled membership or warehouse SQL. | diff --git a/java/src/main/java/com/nvidia/sdm/Request.java b/java/src/main/java/com/nvidia/sdm/Request.java index 17360f85c..0010a4f6c 100644 --- a/java/src/main/java/com/nvidia/sdm/Request.java +++ b/java/src/main/java/com/nvidia/sdm/Request.java @@ -276,15 +276,37 @@ public String query() { return query; } - /** Each call's payload, as the JSON text the core wrote, in the order the calls are made. */ + /** + * Each call's payload as it is sent, as JSON text, in the order the calls are made: without the + * {@code client_} keys the client keeps beside a payload for itself, as the Python host shows it. + */ public List payloads() { List texts = new ArrayList<>(); for (byte[] payload : payloads) { - texts.add(new String(payload, StandardCharsets.UTF_8)); + texts.add(new String(sent(payload), StandardCharsets.UTF_8)); } return texts; } + /** {@code payload} without its client-only keys; the core's text itself where it has none. */ + private static byte[] sent(byte[] payload) { + try { + if (Json.read(payload) instanceof Map envelope + && envelope.keySet().stream().anyMatch(Request::clientOnly)) { + Map kept = new java.util.LinkedHashMap<>(envelope); + kept.keySet().removeIf(Request::clientOnly); + return Json.write(kept); + } + } catch (Json.JsonException e) { + return payload; + } + return payload; + } + + private static boolean clientOnly(Object key) { + return key instanceof String name && name.startsWith("client_"); + } + /** The table whose entities the query is about. */ public String entityTable() { return direct() ? (String) direct.get("entity_table") : facts.entityTable(); diff --git a/java/src/test/java/com/nvidia/sdm/MainMergeContractTest.java b/java/src/test/java/com/nvidia/sdm/MainMergeContractTest.java index 941c4a0c5..b8f0b4178 100644 --- a/java/src/test/java/com/nvidia/sdm/MainMergeContractTest.java +++ b/java/src/test/java/com/nvidia/sdm/MainMergeContractTest.java @@ -142,7 +142,8 @@ static Map gradient(Map payload, int index) { var saved = new java.io.ByteArrayOutputStream(); request.writeTo(saved); Request restored = Request.readFrom(new java.io.ByteArrayInputStream(saved.toByteArray())); - assertTrue(RelationalTest.payload(restored, 0).containsKey("client_generated_keys")); + assertTrue(RelationalTest.map(Json.read(restored.payloadBytes().get(0))).containsKey("client_generated_keys")); + assertFalse(RelationalTest.payload(restored, 0).containsKey("client_generated_keys")); try (RelationalTest.Endpoint endpoint = new RelationalTest.Endpoint(); SdmClient client = endpoint.client(); Operations operations = new Operations()) { endpoint.gradients = true; diff --git a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py index 60ee44e3a..a14e10bcf 100644 --- a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py +++ b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py @@ -72,7 +72,9 @@ def do_POST(self): self.close_connection = True return if (outer.mode == 'expired' and outer.scored == 2) or ( - outer.mode == 'expired_twice' and outer.scored >= 2 + outer.mode == 'expired_twice' + and outer.scored >= 2 + and self.path.startswith('/v1/sessions/') ): status = 404 elif outer.mode == 'malformed' and outer.scored == 2: @@ -277,16 +279,16 @@ def test_session_and_stateless_scoring_preserve_inference_and_outputs(): assert sent[0] == sent[1] -def test_expired_session_is_recreated_only_once(): +def test_a_session_missing_after_recreation_falls_back_to_stateless_calls(): endpoint = Endpoint('expired_twice') try: with SDMClient(url=endpoint.url, max_retries=0) as client: - with pytest.raises(StructuredDataError) as caught: - client.predict(request(), model=KumoRelational()) + result = client.predict(request(), model=KumoRelational()) + assert len(result.to_pandas()) == 3 assert endpoint.created == 2 - assert endpoint.scored == 3 + posts = [path for method, path, _ in endpoint.calls if method == 'POST'] + assert posts[-2:] == ['/v1/predictions', '/v1/predictions'] assert sum(method == 'DELETE' for method, _, _ in endpoint.calls) == 2 - assert caught.value.details['answered'] is not None finally: endpoint.close() diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index a4fd0cb0a..376e26a3f 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:d2cc138d8281084aed2457d2d603d93372c06ab459b0f978a6065665ec4dd35e sdm-contract-72.json +72 sha256:2cea8692cda9d3a3e4731f259c7fd72a3a66aee2f337d5cbe76600c5b5014f9e sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index 6bdc238e5..c0f2444f8 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -17541,6 +17541,7 @@ "data_request_refused", "unsupported_type", "not_yet_describe", + "column_type_unsupported", "empty_table", "table_name_empty", "table_name_characters", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index e2a198142..72f1571a9 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -859,6 +859,7 @@ sentences! { DataRequestRefused = "data_request_refused", E::InvalidRequest; UnsupportedType = "unsupported_type", E::UnsupportedType; NotYetDescribe = "not_yet_describe", E::UnsupportedRequest; + ColumnTypeUnsupported = "column_type_unsupported", E::UnsupportedRequest, rule "column_type"; EmptyTable = "empty_table", E::InvalidRequest; TableNameEmpty = "table_name_empty", E::InvalidRequest; TableNameCharacters = "table_name_characters", E::InvalidRequest; diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index c943507f3..cedd7c882 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -509,6 +509,9 @@ pub fn template(sentence: Sentence) -> Template { S::NotYetDescribe => said( r#"describing this source needs the {rule!r} rule, which is not one this client's core supports yet"#, ), + S::ColumnTypeUnsupported => said( + r#"column {column!r} of table '{source!d}' has the Arrow type {arrow_type}, which a described table does not take. Change the column's type, or remove the column from the table."#, + ), S::EmptyTable => said(r#"Table '{source!d}' is empty"#), S::TableNameEmpty => said(r#"The name {table!r} must be a non-empty string."#), S::TableNameCharacters => { diff --git a/rust/crates/sdm-core/src/batch_session.rs b/rust/crates/sdm-core/src/batch_session.rs index 84d81106a..a38a66766 100644 --- a/rust/crates/sdm-core/src/batch_session.rs +++ b/rust/crates/sdm-core/src/batch_session.rs @@ -91,6 +91,9 @@ impl Session { } /// None requests ordinary stateless fallback, never another retry layer. + /// A session that expired is recreated once; one missing again (an + /// endpoint whose replicas do not share sessions) ends reuse, and this + /// and every later call is sent stateless. pub(crate) async fn score( &mut self, asker: &Asker, @@ -111,20 +114,27 @@ impl Session { if !self.open(asker).await? { return Ok(None); } - crate::frame_session::call( - asker, - Endpoint::SessionPredict { - session_id: self.current.clone().unwrap(), - }, - scoring, - ) - .await - .map(Some) + let endpoint = Endpoint::SessionPredict { + session_id: self.current.clone().unwrap(), + }; + match crate::frame_session::call(asker, endpoint, scoring).await { + Ok(answer) => Ok(Some(answer)), + Err(error) if has_status(&error, 404) => Ok(self.stateless()), + Err(error) => Err(error), + } } + Err(error) if has_status(&error, 404) => Ok(self.stateless()), Err(error) => Err(error), } } + /// Ends reuse: every call from here is sent stateless. + fn stateless(&mut self) -> Option { + self.disabled = true; + self.current = None; + None + } + pub(crate) async fn close(self, asker: &Asker) { for session_id in self.known { if let Err(error) = crate::frame_session::call( diff --git a/rust/crates/sdm-schema/src/infer.rs b/rust/crates/sdm-schema/src/infer.rs index ff6db9147..42b439b14 100644 --- a/rust/crates/sdm-schema/src/infer.rs +++ b/rust/crates/sdm-schema/src/infer.rs @@ -300,6 +300,24 @@ impl InferredTable { } } +/// The refusal of column `column` of `source`, whose preview gave it an +/// Arrow type no described column takes (`column_type`), naming the +/// column, its table and the type, as the reference names the column and +/// its table (`_to_dtype`). +pub fn column_type(source: &Relation, column: &str, data_type: &DataType) -> CoreError { + CoreError::new( + Sentence::ColumnTypeUnsupported, + format!("column {column:?} has the Arrow type {data_type}, which a described table does not take"), + ) + .with_field("operation", Detail::text("describe_source")) + .with_field( + "source", + Detail::List(source.source.iter().cloned().map(Scalar::Utf8).collect()), + ) + .with_field("column", Detail::text(column)) + .with_field("arrow_type", Detail::text(data_type.to_string())) +} + /// The owner's switch of P1B-11 (2026-10-13): an empty table is refused as /// an invalid request, which the reference's unnamed error says it is, not /// as a rule the core lacks; the operation and rule stay named, and the @@ -611,9 +629,12 @@ impl Keyed { // A declaration the reference reads as unsupported (Databricks' // struct, map or interval, `_to_dtype`) is refused by name, as // a struct or map the values give is (B-10). - Some(Dtype::Unsupported) => return Err(not_yet("column_type")), + Some(Dtype::Unsupported) => { + return Err(column_type(relation, &declared.name, field.data_type())); + } Some(declared) => declared, - None => dtype_of(field.data_type())?, + None => dtype_of(field.data_type()) + .ok_or_else(|| column_type(relation, &declared.name, field.data_type()))?, }; let is_primary = declared.primary_key || self.primary.contains(&declared.name); Ok(SourceFacts { @@ -794,7 +815,9 @@ fn preview_with( }); continue; } - let sample = values::column(batches, index, floats)?; + // Its one refusal is of a type it does not read: named here. + let sample = values::column(batches, index, floats) + .map_err(|_| column_type(source, &declared.name, field.data_type()))?; if dtype == Dtype::Time { // The engine reads a time of day's values as times, which none // is, and says how many it dropped. @@ -937,15 +960,16 @@ fn preview_with( /// reference's reading of the catalog's types gives it for the plain types, /// binary and the lists of them; a duration is `timedelta`, which the /// reference's `_WIRE_DTYPE` has no wire dtype for, so it is left out (no -/// DuckDB type hands one over). Any other is refused by name (`column_type`). +/// DuckDB type hands one over). Any other is none, which the caller refuses +/// by name ([`column_type`]). /// A list is read as the reference reads a catalog type ending `[]`: of /// integers, floats or text (binary is text to it, so a list of binary is /// a list of text), refused otherwise where the reference fails unnamed /// (P1B-51). Admission hands every list over as a `list`, so a fixed-size /// list (P1B-9) and a list view (P1B-50) are read as lists too. A struct /// and a map are refused (P1B-10). -fn dtype_of(data_type: &DataType) -> Result { - Ok(match data_type { +fn dtype_of(data_type: &DataType) -> Option { + Some(match data_type { DataType::Boolean => Dtype::Bool, DataType::Int8 | DataType::Int16 @@ -964,13 +988,13 @@ fn dtype_of(data_type: &DataType) -> Result { DataType::Duration(_) => Dtype::Timedelta, DataType::List(element) | DataType::LargeList(element) => { match dtype_of(element.data_type()) { - Ok(Dtype::Int) => Dtype::Intlist, - Ok(Dtype::Float) => Dtype::Floatlist, - Ok(Dtype::String | Dtype::Binary) => Dtype::Stringlist, - _ => return Err(not_yet("column_type")), + Some(Dtype::Int) => Dtype::Intlist, + Some(Dtype::Float) => Dtype::Floatlist, + Some(Dtype::String | Dtype::Binary) => Dtype::Stringlist, + _ => return None, } } - _ => return Err(not_yet("column_type")), + _ => return None, }) } diff --git a/rust/crates/sdm-schema/tests/declared_types.rs b/rust/crates/sdm-schema/tests/declared_types.rs index faad90bfa..8039df8c9 100644 --- a/rust/crates/sdm-schema/tests/declared_types.rs +++ b/rust/crates/sdm-schema/tests/declared_types.rs @@ -10,11 +10,11 @@ use std::sync::Arc; -use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; +use arrow_array::{ArrayRef, Int64Array, NullArray, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; -use sdm_contract::Detail; use sdm_contract::catalog::CatalogColumn; use sdm_contract::ops::Dtype; +use sdm_contract::{Detail, Scalar, Sentence}; use sdm_schema::infer::{self, TextShare, TextTimes, TextUnread}; struct NoTimes; @@ -39,16 +39,20 @@ impl TextTimes for NoTimes { } fn previewed(declared: Option) -> Result { + previewed_as(declared, Arc::new(StringArray::from(vec!["a", "b"]))) +} + +fn previewed_as( + declared: Option, + profile: ArrayRef, +) -> Result { let schema = Arc::new(Schema::new(vec![ Field::new("user_id", DataType::Int64, true), - Field::new("profile", DataType::Utf8, true), + Field::new("profile", profile.data_type().clone(), true), ])); let batch = RecordBatch::try_new( Arc::clone(&schema), - vec![ - Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef, - Arc::new(StringArray::from(vec!["a", "b"])), - ], + vec![Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef, profile], ) .unwrap(); let column = |name: &str, dtype: Option| CatalogColumn { @@ -79,3 +83,19 @@ fn a_declared_unsupported_type_is_refused_by_name_whatever_the_values() { assert!(previewed(Some(Dtype::String)).is_ok()); assert!(previewed(None).is_ok()); } + +#[test] +fn a_refused_column_type_names_the_column_its_table_and_its_type() { + let declared = previewed(Some(Dtype::Unsupported)).unwrap_err(); + let read = previewed_as(None, Arc::new(NullArray::new(2))).unwrap_err(); + for (refused, arrow_type) in [(declared, "Utf8"), (read, "Null")] { + assert_eq!(refused.sentence, Sentence::ColumnTypeUnsupported); + assert_eq!(refused.fields["rule"], Detail::text("column_type")); + assert_eq!(refused.fields["column"], Detail::text("profile")); + assert_eq!( + refused.fields["source"], + Detail::List(vec![Scalar::Utf8("users".into())]) + ); + assert_eq!(refused.fields["arrow_type"], Detail::text(arrow_type)); + } +} From 5c7b87409106af6ea0e6a6fc27e029286b917aec Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Sun, 8 Nov 2026 23:49:35 -0800 Subject: [PATCH 10/24] Name a refused column by its declared type The Snowflake parity run found two gaps: - A Snowflake VARIANT, OBJECT or ARRAY column was refused as "the Arrow type Utf8", the text the driver hands it over as, which every string column also is. The refusal now names the type the source declares, and the Arrow type only where the source declares none. - The CHANGELOG now says that an empty table raises StructuredDataError INVALID_REQUEST, not RuntimeError, and that a column whose name holds a dot is kept. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 10 ++++-- rust/crates/sdm-contract/src/wording.rs | 2 +- rust/crates/sdm-schema/src/infer.rs | 32 ++++++++++++------- .../crates/sdm-schema/tests/declared_types.rs | 9 ++++-- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adc6c86af..6ef89f5f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,8 +83,14 @@ The public API snapshot records this version update; publication remains pending - A column of a type a described table does not take, such as a list of booleans, a struct or a map, raises `UnsupportedRequestError` with code `UNSUPPORTED_REQUEST` and `details['rule']` `column_type`. The message and - details name the column, its table and its Arrow type. 1.0.0 raised a - `ValueError` naming the column and table. + details name the column, its table and its type, as the source declares it + where it does, such as a Snowflake `VARIANT`. 1.0.0 raised a `ValueError` + naming the column and table. +- Describing an empty table raises `StructuredDataError` with code + `INVALID_REQUEST` and `details['rule']` `empty_table`. 1.0.0 raised a + `RuntimeError`, so `except RuntimeError` no longer catches it. +- A column whose name holds a dot is kept and sent. 1.0.0 left it out with a + warning. - `predict()` on a request gathered in several calls that share one context scores them through a temporary endpoint session, so the context is uploaded once. 1.0.0 sent every call stateless. An expired session is diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index cedd7c882..26bca640f 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -510,7 +510,7 @@ pub fn template(sentence: Sentence) -> Template { r#"describing this source needs the {rule!r} rule, which is not one this client's core supports yet"#, ), S::ColumnTypeUnsupported => said( - r#"column {column!r} of table '{source!d}' has the Arrow type {arrow_type}, which a described table does not take. Change the column's type, or remove the column from the table."#, + r#"column {column!r} of table '{source!d}' has the type {column_type}, which a described table does not take. Change the column's type, or remove the column from the table."#, ), S::EmptyTable => said(r#"Table '{source!d}' is empty"#), S::TableNameEmpty => said(r#"The name {table!r} must be a non-empty string."#), diff --git a/rust/crates/sdm-schema/src/infer.rs b/rust/crates/sdm-schema/src/infer.rs index 42b439b14..f907eb0b4 100644 --- a/rust/crates/sdm-schema/src/infer.rs +++ b/rust/crates/sdm-schema/src/infer.rs @@ -300,14 +300,16 @@ impl InferredTable { } } -/// The refusal of column `column` of `source`, whose preview gave it an -/// Arrow type no described column takes (`column_type`), naming the -/// column, its table and the type, as the reference names the column and -/// its table (`_to_dtype`). -pub fn column_type(source: &Relation, column: &str, data_type: &DataType) -> CoreError { +/// The refusal of column `column` of `source`, whose type `column_type` +/// no described column takes (`column_type`), naming the column, its table +/// and the type, as the reference names the column and its table +/// (`_to_dtype`). +pub fn column_type(source: &Relation, column: &str, column_type: &str) -> CoreError { CoreError::new( Sentence::ColumnTypeUnsupported, - format!("column {column:?} has the Arrow type {data_type}, which a described table does not take"), + format!( + "column {column:?} has the type {column_type}, which a described table does not take" + ), ) .with_field("operation", Detail::text("describe_source")) .with_field( @@ -315,7 +317,7 @@ pub fn column_type(source: &Relation, column: &str, data_type: &DataType) -> Cor Detail::List(source.source.iter().cloned().map(Scalar::Utf8).collect()), ) .with_field("column", Detail::text(column)) - .with_field("arrow_type", Detail::text(data_type.to_string())) + .with_field("column_type", Detail::text(column_type)) } /// The owner's switch of P1B-11 (2026-10-13): an empty table is refused as @@ -628,13 +630,19 @@ impl Keyed { let dtype = match declared.dtype { // A declaration the reference reads as unsupported (Databricks' // struct, map or interval, `_to_dtype`) is refused by name, as - // a struct or map the values give is (B-10). + // a struct or map the values give is (B-10), and by the type it + // declares: a Snowflake VARIANT arrives as text. Some(Dtype::Unsupported) => { - return Err(column_type(relation, &declared.name, field.data_type())); + let named = match declared.declared_type.trim() { + "" => field.data_type().to_string(), + spelled => spelled.to_owned(), + }; + return Err(column_type(relation, &declared.name, &named)); } Some(declared) => declared, - None => dtype_of(field.data_type()) - .ok_or_else(|| column_type(relation, &declared.name, field.data_type()))?, + None => dtype_of(field.data_type()).ok_or_else(|| { + column_type(relation, &declared.name, &field.data_type().to_string()) + })?, }; let is_primary = declared.primary_key || self.primary.contains(&declared.name); Ok(SourceFacts { @@ -817,7 +825,7 @@ fn preview_with( } // Its one refusal is of a type it does not read: named here. let sample = values::column(batches, index, floats) - .map_err(|_| column_type(source, &declared.name, field.data_type()))?; + .map_err(|_| column_type(source, &declared.name, &field.data_type().to_string()))?; if dtype == Dtype::Time { // The engine reads a time of day's values as times, which none // is, and says how many it dropped. diff --git a/rust/crates/sdm-schema/tests/declared_types.rs b/rust/crates/sdm-schema/tests/declared_types.rs index 8039df8c9..e800e0531 100644 --- a/rust/crates/sdm-schema/tests/declared_types.rs +++ b/rust/crates/sdm-schema/tests/declared_types.rs @@ -57,7 +57,10 @@ fn previewed_as( .unwrap(); let column = |name: &str, dtype: Option| CatalogColumn { name: name.to_owned(), - declared_type: String::new(), + declared_type: match dtype { + Some(Dtype::Unsupported) => "VARIANT".to_owned(), + _ => String::new(), + }, not_null: false, primary_key: false, key_nullable: false, @@ -88,7 +91,7 @@ fn a_declared_unsupported_type_is_refused_by_name_whatever_the_values() { fn a_refused_column_type_names_the_column_its_table_and_its_type() { let declared = previewed(Some(Dtype::Unsupported)).unwrap_err(); let read = previewed_as(None, Arc::new(NullArray::new(2))).unwrap_err(); - for (refused, arrow_type) in [(declared, "Utf8"), (read, "Null")] { + for (refused, column_type) in [(declared, "VARIANT"), (read, "Null")] { assert_eq!(refused.sentence, Sentence::ColumnTypeUnsupported); assert_eq!(refused.fields["rule"], Detail::text("column_type")); assert_eq!(refused.fields["column"], Detail::text("profile")); @@ -96,6 +99,6 @@ fn a_refused_column_type_names_the_column_its_table_and_its_type() { refused.fields["source"], Detail::List(vec![Scalar::Utf8("users".into())]) ); - assert_eq!(refused.fields["arrow_type"], Detail::text(arrow_type)); + assert_eq!(refused.fields["column_type"], Detail::text(column_type)); } } From 38a81eba65117ee41fa2248db17bd44b7fdbc460 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 09:01:55 -0800 Subject: [PATCH 11/24] Keep a payload view to the dictionary contract Request.payloads views raised StopIteration from popitem() once empty, where the plain dictionaries they replace raise KeyError. Deleting a hidden client_* key raised KeyError after removing it from the envelope, which dropped the generated-key record explanations decode with. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../structured-data-models-client/src/sdm_client/request.py | 4 +++- .../tests/test_column_analysis_core.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index 091ca5ce4..554428dc0 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -1031,8 +1031,8 @@ def __setitem__(self, name: str, value: Any) -> None: super().__setitem__(name, value) def __delitem__(self, name: str) -> None: - del self._envelope[name] super().__delitem__(name) + self._envelope.pop(name, None) def __ior__(self, other: Any) -> _Envelope: # type: ignore[override,misc] self.update(other) @@ -1055,6 +1055,8 @@ def pop(self, name: str, *default: Any) -> Any: return value def popitem(self) -> tuple[str, Any]: + if not self: + return super().popitem() name = next(reversed(self)) return name, self.pop(name) diff --git a/packages/structured-data-models-client/tests/test_column_analysis_core.py b/packages/structured-data-models-client/tests/test_column_analysis_core.py index 2bb00de86..d47d4de54 100644 --- a/packages/structured-data-models-client/tests/test_column_analysis_core.py +++ b/packages/structured-data-models-client/tests/test_column_analysis_core.py @@ -257,6 +257,12 @@ def test_public_payloads_hide_client_keys_and_write_through(): shared = Request.__new__(Request) object.__setattr__(shared, '_payloads', (envelope,)) assert shared.payloads[0] is payload + payload.clear() + with pytest.raises(KeyError): + payload.popitem() + with pytest.raises(KeyError): + del payload['client_generated_keys'] + assert envelope == {'client_generated_keys': {'orders': '__node_id'}} def test_unavailable_statistics_warn_without_losing_predictions(): From 11ec821d193578101978f441412d415c7f857a72 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 10:13:31 -0800 Subject: [PATCH 12/24] Give each analysed instance its own id across calls Column analysis over a request's several calls read every call's rows under the ids they were sent with. Calls gathered apart, such as envelopes passed to Request.materialized, can use one id for different entities, so asking for explanations failed with INVALID_MESSAGE while predictions alone worked. Each kept instance and its related rows now take a fresh id, as the forecast union already does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/crates/sdm-request/src/columns.rs | 57 ++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/rust/crates/sdm-request/src/columns.rs b/rust/crates/sdm-request/src/columns.rs index 0c212ac25..308641b7a 100644 --- a/rust/crates/sdm-request/src/columns.rs +++ b/rust/crates/sdm-request/src/columns.rs @@ -960,9 +960,9 @@ pub fn validate(raw: &str) -> Result { } /// The labelled rows several calls of one request send, as one context: -/// every call carries the same labelled draw, less its own scored entities, -/// under the same instance ids, so each instance, keyed by its entity and -/// anchor, is read once, from the first call that sends it. +/// each instance, keyed by its entity and anchor, is read once, from the +/// first call that sends it, under an id of its own, since calls gathered +/// apart can use one instance id for different entities. pub fn sent(payloads: &[Value]) -> Result { let Some(first) = payloads.first() else { return Err(invalid("empty analysis request")); @@ -975,6 +975,7 @@ pub fn sent(payloads: &[Value]) -> Result { return Ok(first.clone()); } let mut seen = BTreeSet::new(); + let mut next_id = 0i64; let mut instances = Vec::new(); let mut related: Vec<(String, Value)> = Vec::new(); let mut schema = field(first, "schema")?.clone(); @@ -988,15 +989,20 @@ pub fn sent(payloads: &[Value]) -> Result { .and_then(Value::as_text) .and_then(|name| table.at(name).ok()); let mut keys = Vec::new(); - let mut kept = BTreeSet::new(); + let mut kept = BTreeMap::new(); for row in &table.rows { let at = ( key(&row[entity])?, anchor.map(|a| key(&row[a])).transpose()?, ); if !seen.contains(&at) { - kept.insert(key(&row[id])?); - instances.push(Value::Array(row.to_vec())); + kept.insert(key(&row[id])?, next_id); + let mut row = row.to_vec(); + row[id] = Value::int(next_id); + instances.push(Value::Array(row)); + next_id = next_id + .checked_add(1) + .ok_or_else(|| invalid("analysis too large"))?; } keys.push(at); } @@ -1040,8 +1046,10 @@ pub fn sent(payloads: &[Value]) -> Result { .collect::, CoreError>>()?; let from = array(field(rows, "columns")?)?; for row in &read.rows { - if kept.contains(&key(&row[at])?) { - all.push(Value::Array(widen(row, from))); + if let Some(&new) = kept.get(&key(&row[at])?) { + let mut row = row.to_vec(); + row[at] = Value::int(new); + all.push(Value::Array(widen(&row, from))); } } held.set("columns", Value::Array(names.clone())); @@ -1328,6 +1336,39 @@ mod lifecycle_tests { assert_eq!(cohorts, [("p", 0.6, 1.0 / 3.0), ("q", 0.4, 0.5)]); } + #[test] + fn calls_gathered_apart_keep_each_instance_and_its_rows() { + let call = |instances: &str, users: &str| { + parse(&format!( + r#"{{"task":{{"kind":"regression","target":{{"column_name":"y"}}}}, + "schema":{{"instance_table":{{"columns":{{"instance_id":{{"stype":"ID"}},"__kumo_entity_ref_0":{{"stype":"ID"}},"y":{{"stype":"numerical"}}}}}}, + "related_tables":{{"users":{{"primary_key":["instance_id","uid"],"columns":{{"instance_id":{{"stype":"ID"}},"uid":{{"stype":"ID"}},"age":{{"stype":"numerical"}}}}}}}}, + "relationships":[{{"source_columns":["__kumo_entity_ref_0"],"target_table":"users","target_columns":["uid"]}}]}}, + "context":{{"instance_table":{{"columns":["instance_id","__kumo_entity_ref_0","y"],"rows":[{instances}]}}, + "related_tables":{{"users":{{"columns":["instance_id","uid","age"],"rows":[{users}]}}}}}}}}"# + )) + }; + let first = call("[0,1,0],[1,2,1]", "[0,1,30],[1,2,40]"); + let second = call("[0,3,1],[1,4,1]", "[0,3,50],[1,4,60]"); + let combined = sent(&[first, second]).unwrap(); + let context = field(&combined, "context").unwrap(); + let instances = Table::read(field(context, "instance_table").unwrap()).unwrap(); + let entity = instances + .rows + .iter() + .map(|r| (key(&r[0]).unwrap(), key(&r[1]).unwrap())) + .collect::>(); + assert_eq!(entity.len(), 4); + let users = field(field(context, "related_tables").unwrap(), "users").unwrap(); + for row in &Table::read(users).unwrap().rows { + assert_eq!(entity[&key(&row[0]).unwrap()], key(&row[1]).unwrap()); + } + assert!(matches!( + analyze(&combined).unwrap(), + ColumnAnalysis::Ready { .. } + )); + } + #[test] fn calls_that_reached_different_tables_are_analysed_together() { let users = r#""users":{"primary_key":["instance_id","uid"],"columns":{"instance_id":{"stype":"ID"},"uid":{"stype":"ID"},"age":{"stype":"numerical"}}}"#; From e1ce7795c7920f0d5e6c3ae5032969234ff4bb29 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 10:30:36 -0800 Subject: [PATCH 13/24] Say decimals are exact through the SQL warehouse too Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ef89f5f8..2e775ad87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -152,8 +152,8 @@ The public API snapshot records this version update; publication remains pending - On Databricks through Spark, a `TIMESTAMP` column is read in UTC, so the anchor time and the rows a window takes no longer move with the client machine's time zone. 1.0.0 read its range as local times. -- On Databricks through Spark, a `DECIMAL` value with a fraction is sent - exactly, such as `997.67`. 1.0.0 sent it through a float, such as +- On Databricks, through Spark or a SQL warehouse, a `DECIMAL` value with a + fraction is sent exactly, such as `997.67`. 1.0.0 sent it through a float, such as `997.6700000000001`. ### Removed From c38f8e5c1be047e175a59789a47c5ce0425d74b6 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 11:04:51 -0800 Subject: [PATCH 14/24] Take rusqlite 0.41 for RUSTSEC-2026-0459 libsqlite3-sys 0.38.2 bundles SQLite 3.53.2, which the advisory covers; 0.39.1 bundles 3.54.1. Only the test kit links it. rusqlite 0.41 hands column names back as an iterator. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/Cargo.lock | 8 ++++---- rust/Cargo.toml | 2 +- rust/crates/sdm-testkit/src/sqlite.rs | 6 +----- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5a7370831..266ef3914 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1024,9 +1024,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" dependencies = [ "cc", "pkg-config", @@ -1475,9 +1475,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "da9bba6f3506135e120a6586df05be18b48d18896600646a4bec8f993d91a594" dependencies = [ "bitflags 2.13.2", "fallible-iterator", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9fd3a469d..e47af089e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -59,7 +59,7 @@ serde_json = { version = "1", features = ["float_roundtrip", "preserve_order"] } serde_stacker = "0.1" regex = "1" # The SQLite engine of the dev-only test host and its tests. -rusqlite = { version = "0.40", features = ["bundled"] } +rusqlite = { version = "0.41", features = ["bundled"] } sha2 = "0.10" stacker = "0.1" thiserror = "2" diff --git a/rust/crates/sdm-testkit/src/sqlite.rs b/rust/crates/sdm-testkit/src/sqlite.rs index a00497427..dc962576f 100644 --- a/rust/crates/sdm-testkit/src/sqlite.rs +++ b/rust/crates/sdm-testkit/src/sqlite.rs @@ -436,11 +436,7 @@ fn number_text(connection: &rusqlite::Connection, value: &Stored) -> Result Result { let mut statement = connection.prepare(sql)?; - let names: Vec = statement - .column_names() - .iter() - .map(|n| (*n).to_owned()) - .collect(); + let names: Vec = statement.column_names().map(str::to_owned).collect(); let width = names.len(); let mut rows: Vec> = Vec::new(); let mut queried = statement.query([])?; From 4f7393b335341af3d4fc5280501f0cf3a7fef8c0 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 13:32:29 -0800 Subject: [PATCH 15/24] Name the field a relational session's refusal finds changed A frame session asked another question refused without saying which of the query, task, target or entity table differs, though the check found it and the relational skill promises the refusal names it. It now says what was pinned and what was asked, in the Tabular session's words. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../tests/test_frame_sessions_core.py | 3 +- rust/crates/sdm-core/src/frame_session.rs | 8 ++--- .../crates/sdm-request/src/tabular/session.rs | 33 +++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_frame_sessions_core.py b/packages/structured-data-models-client/tests/test_frame_sessions_core.py index 7d3c61c69..95bc16f08 100644 --- a/packages/structured-data-models-client/tests/test_frame_sessions_core.py +++ b/packages/structured-data-models-client/tests/test_frame_sessions_core.py @@ -140,8 +140,9 @@ def test_session_refuses_changed_question_and_retries_failed_close(client): changed = PredictiveQuery( 'PREDICT users.value > 3 FOR EACH users.id', asked.schema ) - with pytest.raises(StructuredDataError, match='different question'): + with pytest.raises(StructuredDataError, match='the query pinned is') as refused: fitted.predict(gather(changed, [2])) + assert 'users.value > 3' in str(refused.value) assert not endpoint.scored endpoint.fail_delete = True with pytest.raises(StructuredDataError, match='retry deletion'): diff --git a/rust/crates/sdm-core/src/frame_session.rs b/rust/crates/sdm-core/src/frame_session.rs index 7650900c7..2b0051767 100644 --- a/rust/crates/sdm-core/src/frame_session.rs +++ b/rust/crates/sdm-core/src/frame_session.rs @@ -222,10 +222,10 @@ pub(crate) async fn run( let mut asked_question = question.clone(); normalize_question(&mut pinned_question); normalize_question(&mut asked_question); - if keys::check_question(pinned_question.as_ref(), asked_question.as_ref()).is_some() { - return Err(invalid( - "this session pinned a different question; fit again to ask something else", - )); + if let Some(difference) = + keys::check_question(pinned_question.as_ref(), asked_question.as_ref()) + { + return Err(invalid(&keys::question_refusal(&difference))); } if payloads.is_empty() || payloads.len() != request_ids.len() { return Err(invalid("session scoring needs one request id per payload")); diff --git a/rust/crates/sdm-request/src/tabular/session.rs b/rust/crates/sdm-request/src/tabular/session.rs index efe7ede24..263866a09 100644 --- a/rust/crates/sdm-request/src/tabular/session.rs +++ b/rust/crates/sdm-request/src/tabular/session.rs @@ -441,6 +441,39 @@ pub fn check_question( }) } +/// The refusal of a scoring call that asks another question than the pinned +/// one, naming the field that differs, as the Python Tabular session does. +pub fn question_refusal(difference: &SessionQuestionDifference) -> String { + use SessionQuestionField as Field; + let shown = |value: &Option| { + value + .as_ref() + .map_or("None".to_owned(), |v| format!("'{v}'")) + }; + let detail = match difference.field { + Field::Context => "the context pinned is not the one this request carries, and the \ + context is what the answer is learnt from" + .to_owned(), + field => format!( + "{} pinned is {} and this asks {}", + match field { + Field::Query => "the query", + Field::Kind => "the task kind", + Field::Target => "the target column", + Field::Dtype => "the target column's dtype", + _ => "the entity table", + }, + shown(&difference.pinned), + shown(&difference.asked), + ), + }; + format!( + "this session pinned a different question: {detail}. Only the rows travel on a \ + scoring call, so the pinned question is the one that would be answered. Fit again \ + to ask something else" + ) +} + /// Commit closure only after the host confirms successful remote deletion. /// A failed deletion leaves the returned value retryable; already closed /// values remain closed. The caller skips sending when `closed` is already set. From b450c957fdb381023c7614c106b909e57d976db3 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 13:45:20 -0800 Subject: [PATCH 16/24] Format the session refusal test Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../tests/test_frame_sessions_core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/structured-data-models-client/tests/test_frame_sessions_core.py b/packages/structured-data-models-client/tests/test_frame_sessions_core.py index 95bc16f08..d654aa41a 100644 --- a/packages/structured-data-models-client/tests/test_frame_sessions_core.py +++ b/packages/structured-data-models-client/tests/test_frame_sessions_core.py @@ -140,7 +140,9 @@ def test_session_refuses_changed_question_and_retries_failed_close(client): changed = PredictiveQuery( 'PREDICT users.value > 3 FOR EACH users.id', asked.schema ) - with pytest.raises(StructuredDataError, match='the query pinned is') as refused: + with pytest.raises( + StructuredDataError, match='the query pinned is' + ) as refused: fitted.predict(gather(changed, [2])) assert 'users.value > 3' in str(refused.value) assert not endpoint.scored From 368555c80baec6f8bbbca1b5d660cfac2c81e467 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 15:46:38 -0800 Subject: [PATCH 17/24] Read a described table's catalog again, and the rest only if it changed Each materialize on a schema from Schema.from_source read every table's catalog, keys and a 1,000-row preview again, so a warehouse saw several more queries per call than 1.0.0. A described column now keeps the type its catalog declared (catalog_type), and a run reads the catalog alone, reusing what describe read where every column is declared as it was. A dropped or retyped column still triggers the whole read, so its refusal is unchanged. Python and Java carry the new fact with the column. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 9 ++- contract/tests/test_sqlite_source_edges.py | 4 +- java/src/main/java/com/nvidia/sdm/Schema.java | 3 + .../src/sdm_client/_rust/results.py | 11 ++- .../tests/test_schema_copy_provenance.py | 2 +- .../tests/test_snowflake_from_source.py | 9 ++- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 14 ++++ rust/crates/sdm-contract/src/ops.rs | 9 +++ rust/crates/sdm-core/src/big_integer.rs | 1 + rust/crates/sdm-core/src/describe.rs | 68 ++++++++++++++-- rust/crates/sdm-core/src/discover.rs | 28 +++++++ rust/crates/sdm-core/src/materialize.rs | 5 +- rust/crates/sdm-core/src/sql_column.rs | 1 + rust/crates/sdm-core/src/view.rs | 1 + rust/crates/sdm-core/tests/edit_schema.rs | 2 + rust/crates/sdm-core/tests/held_schema.rs | 2 + rust/crates/sdm-core/tests/materialize.rs | 1 + rust/crates/sdm-core/tests/view_captures.rs | 1 + .../sdm-engine/tests/keyless_tie_types.rs | 1 + .../sdm-engine/tests/neighbor_requests.rs | 1 + rust/crates/sdm-host/tests/codec.rs | 1 + rust/crates/sdm-schema/src/assemble.rs | 1 + rust/crates/sdm-schema/src/held.rs | 3 + rust/crates/sdm-schema/src/infer.rs | 2 + rust/crates/sdm-schema/tests/assemble.rs | 1 + rust/crates/sdm-schema/tests/edit.rs | 1 + rust/crates/sdm-schema/tests/exact_numbers.rs | 1 + rust/crates/sdm-schema/tests/held.rs | 1 + rust/crates/sdm-testkit/src/edits.rs | 2 + .../sdm-testkit/tests/duckdb_provider.rs | 77 ++++++++++++++++--- tests/golden/statements.json | 26 +------ 32 files changed, 237 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e775ad87..928aff9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,9 +100,12 @@ The public API snapshot records this version update; publication remains pending - A related table sampled from frames sends its own primary key, where 1.0.0 sent a generated `__node_id`. - Each `materialize()` on a schema from `Schema.from_source` reads its - tables' catalog and preview again, and a semantic model's definition, so a - column dropped or retyped since is found. 1.0.0 read them once and kept - them on the schema, so a warehouse sees more queries per call than it did. + tables' catalog again, one query per table, and a semantic model's + definition, so a column dropped or retyped since is found. A table whose + catalog changed is read again in full, its keys and first rows included. + 1.0.0 read them once and kept them on the schema. A table emptied since it + was described is refused as empty by a warehouse's row count; elsewhere it + reads as a table with no times, as 1.0.0 read it. - An HTTP 408 from `POST /v1/predictions` is retried, as 429, 502, 503 and 504 are. - A relational request reached over three or more hops fills a foreign key diff --git a/contract/tests/test_sqlite_source_edges.py b/contract/tests/test_sqlite_source_edges.py index 7bc6eb8cc..df1db2ddc 100644 --- a/contract/tests/test_sqlite_source_edges.py +++ b/contract/tests/test_sqlite_source_edges.py @@ -184,7 +184,7 @@ def test_a_partial_unique_index_keeps_the_key_unique( wanted = _reference('partial-unique') locate: list[int] = [] got = _on_the_core(monkeypatch, 'partial-unique', locate) - assert locate == [7] + assert locate == [2] assert got['schema'] == wanted['schema'] assert got['statements'] == { **wanted['statements'], @@ -203,7 +203,7 @@ def test_each_sql_materialize_refreshes_its_sampler_facts_once( wanted = _reference('materialize-twice') locate: list[int] = [] got = _on_the_core(monkeypatch, 'materialize-twice', locate) - assert locate == [6, 6] + assert locate == [2, 2] assert got['outcome'] == wanted['outcome'] assert got['statements'] == { **wanted['statements'], diff --git a/java/src/main/java/com/nvidia/sdm/Schema.java b/java/src/main/java/com/nvidia/sdm/Schema.java index dca3108aa..fea0653dc 100644 --- a/java/src/main/java/com/nvidia/sdm/Schema.java +++ b/java/src/main/java/com/nvidia/sdm/Schema.java @@ -658,6 +658,9 @@ private static Schema described(Object read, Source source) { if (column.get("declared_type") != null) { facts.put("declared_type", column.get("declared_type")); } + if (column.get("catalog_type") != null) { + facts.put("catalog_type", column.get("catalog_type")); + } if (column.get("exact_number") != null) { facts.put("exact_number", column.get("exact_number")); } diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/results.py b/packages/structured-data-models-client/src/sdm_client/_rust/results.py index 03ede4456..3d5994aff 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/results.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/results.py @@ -115,8 +115,8 @@ class SourceFacts: since), the data type the source's catalog and values give it, the type the provider handed over (the contract's ``ColumnType``, as JSON), whether it may hold no value, whether a key of it alone keeps it - unique, a big integer's or temporal column's declared type, and whether - it is a column of exact numbers.""" + unique, a big integer's or temporal column's declared type, the type + its catalog declares, and whether it is a column of exact numbers.""" relation: Any dtype: str @@ -124,6 +124,7 @@ class SourceFacts: nullable: bool unique: bool declared_type: str | None = None + catalog_type: str | None = None exact_number: str | None = None @classmethod @@ -133,7 +134,7 @@ def from_contract(cls, value: Any) -> SourceFacts: value, "a column's source facts", ('relation', 'dtype', 'received', 'nullable', 'unique'), - ('declared_type', 'exact_number'), + ('declared_type', 'catalog_type', 'exact_number'), ) return cls( relation=read['relation'], @@ -142,6 +143,7 @@ def from_contract(cls, value: Any) -> SourceFacts: nullable=_flag(read['nullable'], 'nullable'), unique=_flag(read['unique'], 'unique'), declared_type=_maybe(read['declared_type'], _text, 'declared_type'), + catalog_type=_maybe(read['catalog_type'], _text, 'catalog_type'), exact_number=_maybe(read['exact_number'], _text, 'exact_number'), ) @@ -162,6 +164,7 @@ def of_described( 'nullable', 'unique', 'declared_type', + 'catalog_type', 'exact_number', ) if key in column @@ -181,6 +184,8 @@ def to_contract(self) -> dict[str, Any]: } if self.declared_type is not None: written['declared_type'] = self.declared_type + if self.catalog_type is not None: + written['catalog_type'] = self.catalog_type if self.exact_number is not None: written['exact_number'] = self.exact_number return written diff --git a/packages/structured-data-models-client/tests/test_schema_copy_provenance.py b/packages/structured-data-models-client/tests/test_schema_copy_provenance.py index c88dd7f45..dc7dbb72a 100644 --- a/packages/structured-data-models-client/tests/test_schema_copy_provenance.py +++ b/packages/structured-data-models-client/tests/test_schema_copy_provenance.py @@ -155,7 +155,7 @@ def answering(request, keys): assert isinstance(once, list) assert again == once assert asked == first - assert 'describe' in asked and 'preview' in asked + assert 'describe' in asked and 'preview' not in asked def test_source_facts_are_immutable_and_excluded_from_column_equality(): diff --git a/packages/structured-data-models-client/tests/test_snowflake_from_source.py b/packages/structured-data-models-client/tests/test_snowflake_from_source.py index f7c6230cc..d26e770b9 100644 --- a/packages/structured-data-models-client/tests/test_snowflake_from_source.py +++ b/packages/structured-data-models-client/tests/test_snowflake_from_source.py @@ -666,10 +666,13 @@ def test_each_materialize_resolves_each_tables_spelling_again() -> None: describes = _describes(warehouse, mark) assert describes[:1] == [f'DESCRIBE TABLE "{DB}"."{SCHEMA}"."users"'] assert f'DESCRIBE TABLE {_UPPER}."USERS"' in describes - previews = [ - text for text in warehouse.asked(mark) if text.endswith('LIMIT 1000') + reads = [ + text + for text in warehouse.asked(mark) + if not text.startswith('DESCRIBE') ] - assert any('"USERS"' in text for text in previews), previews + assert not any(text.endswith('LIMIT 1000') for text in reads), reads + assert any('"USERS"' in text for text in reads), reads warehouse.upper = set() mark = len(warehouse.calls) _materialize_reads(schema) diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 376e26a3f..6e748461a 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:2cea8692cda9d3a3e4731f259c7fd72a3a66aee2f337d5cbe76600c5b5014f9e sdm-contract-72.json +72 sha256:3fef3439c3e7d82f871ef0f6d47f5536c64aeea9e62c0e594d3a3d91d350a520 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index c0f2444f8..10736fac4 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -2040,6 +2040,13 @@ "null" ] }, + "catalog_type": { + "description": "[`DescribedColumn::catalog_type`]; absent from the JSON when none.", + "type": [ + "string", + "null" + ] + }, "exact_number": { "description": "[`DescribedColumn::exact_number`]; absent from the JSON when none.", "anyOf": [ @@ -11104,6 +11111,13 @@ "null" ] }, + "catalog_type": { + "description": "The type the source's catalog declares the column, as it spells it,\nfor every column a catalog lists. A later `materialize` compares it\nwith the catalog then, and reads the table's preview again only\nwhere they differ. Absent where no catalog was read.", + "type": [ + "string", + "null" + ] + }, "exact_number": { "description": "Set for a column of numbers the host provider handed over as their\nexact decimal text (marked [`crate::data::EXACT_NUMBER_MARKER`]):\nwhether its catalog declares it a key. The schema reads such a column\nby its role in the schema as it stands: as its text\n(`received` `utf8`, `dtype` `string`) where the catalog declares it a\nkey, or where the schema keys it or joins it and every column it\njoins to is such a column too; as a double (`float64`) otherwise.\n`None` for every other column, and absent from the JSON.", "anyOf": [ diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index 508c61250..b5bb135fa 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -863,6 +863,12 @@ pub struct DescribedColumn { /// Other columns keep none. Absent when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub declared_type: Option, + /// The type the source's catalog declares the column, as it spells it, + /// for every column a catalog lists. A later `materialize` compares it + /// with the catalog then, and reads the table's preview again only + /// where they differ. Absent where no catalog was read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_type: Option, /// Set for a column of numbers the host provider handed over as their /// exact decimal text (marked [`crate::data::EXACT_NUMBER_MARKER`]): /// whether its catalog declares it a key. The schema reads such a column @@ -1067,6 +1073,9 @@ pub struct SourceFacts { /// [`DescribedColumn::declared_type`]; absent from the JSON when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub declared_type: Option, + /// [`DescribedColumn::catalog_type`]; absent from the JSON when none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_type: Option, /// [`DescribedColumn::exact_number`]; absent from the JSON when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub exact_number: Option, diff --git a/rust/crates/sdm-core/src/big_integer.rs b/rust/crates/sdm-core/src/big_integer.rs index 44263345e..0906491c5 100644 --- a/rust/crates/sdm-core/src/big_integer.rs +++ b/rust/crates/sdm-core/src/big_integer.rs @@ -215,6 +215,7 @@ mod tests { nullable: true, unique: false, declared_type: Some("HUGEINT".into()), + catalog_type: None, exact_number: None, }; let mut refreshed = old.clone(); diff --git a/rust/crates/sdm-core/src/describe.rs b/rust/crates/sdm-core/src/describe.rs index 44d878ca7..2c24c8e7f 100644 --- a/rust/crates/sdm-core/src/describe.rs +++ b/rust/crates/sdm-core/src/describe.rs @@ -17,8 +17,8 @@ use arrow_schema::TimeUnit; use sdm_contract::catalog::{CatalogAnswer, CatalogForeignKey, KeyKind}; use sdm_contract::data::{DataRequest, DescribePart, ProviderError, Relation, TableKey}; use sdm_contract::ops::{ - ColumnRef, DescribedSource, HeldSchema, NamedTable, OperationResult, SourceDescription, - SourceSpec, + ColumnRef, DescribedSource, HeldSchema, HeldTable, NamedTable, OperationResult, + SourceDescription, SourceFacts, SourceSpec, }; use sdm_contract::{CoreError, Detail, ErrorCode, HostFacts, Scalar, Sentence, Unit}; use sdm_engine::time::text::Reader; @@ -117,6 +117,9 @@ pub(crate) async fn run( let mut tables = Vec::with_capacity(names.len()); // Catalog types retained for big-integer key casts and temporal windows. let mut source_types = Vec::with_capacity(names.len()); + // Every column's catalog type, which a later run compares with the + // catalog then. + let mut catalog_types = Vec::with_capacity(names.len()); let mut primary_constraints = Vec::with_capacity(names.len()); for table in &names { // The table read as `materialize` reads a column it holds no facts @@ -130,6 +133,12 @@ pub(crate) async fn run( .. } = crate::discover::table(&asker, table, caller_named, None).await?; source_types.push(types); + catalog_types.push( + declared + .iter() + .map(|c| (c.name.clone(), c.declared_type.clone())) + .collect::>(), + ); primary_constraints.push( constraints .iter() @@ -298,12 +307,21 @@ pub(crate) async fn run( for warning in warnings { asker.warn(warning); } - for (table, types) in description.tables.iter_mut().zip(source_types) { + for ((table, types), catalog) in description + .tables + .iter_mut() + .zip(source_types) + .zip(catalog_types) + { for column in &mut table.columns { column.declared_type = types .iter() .find(|(name, _)| *name == column.name) .map(|(_, declared)| declared.clone()); + column.catalog_type = catalog + .iter() + .find(|(name, declared)| *name == column.name && !declared.is_empty()) + .map(|(_, declared)| declared.clone()); } } // Each table's repeated keys, as the reference's count sets them. @@ -761,8 +779,9 @@ pub(crate) async fn settled(asker: Asker) -> Result { /// and nothing of them is remembered between runs : each table's location (a part the provider leaves to the /// connection read from the session once), its catalog (a table the source /// does not hold refused by name, every later read of it naming the -/// columns the catalog lists, [`Asker::guard`]) and its preview (a table of -/// no rows refused, `Table '…' is empty`), every column of the schema read +/// columns the catalog lists, [`Asker::guard`]) and, where the catalog +/// declares a column otherwise than when it was read, its preview (a table +/// of no rows refused, `Table '…' is empty`), every column of the schema read /// by the type the source holds now, the caller's meanings kept. A column /// the schema names that the source does not hold is refused by name /// (`column_not_in_source`), and so is a type the caller gave a column @@ -870,6 +889,10 @@ async fn read_as_it_stands( .iter() .filter(|t| t.source == *relation) .collect(); + if let Some(kept) = unchanged(asker, relation, &tables).await? { + found.extend(kept); + continue; + } // The table's catalog whole, which says which columns the source // holds, and the preview and facts of the columns the schema uses // only, as the reference's gather copies of a table only the @@ -955,6 +978,41 @@ async fn read_as_it_stands( Ok(sdm_schema::held::read(held.clone(), &found, false)?.by_the_source()) } +/// The facts `tables` hold of their columns, where the catalog of +/// `relation` declares each of them now as it did when they were read: the +/// run then reads neither the table's keys nor its preview again. None +/// where a column is gone, is declared another type or holds no catalog +/// type, and where the table projects its source: the whole read then says +/// what changed. A catalog that cannot be read is refused as the whole +/// read refuses it. +async fn unchanged( + asker: &Asker, + relation: &Relation, + tables: &[&HeldTable], +) -> Result>, CoreError> { + if relation.is_derived() || tables.iter().any(|t| !t.calculated_columns.is_empty()) { + return Ok(None); + } + let mut kept = Vec::new(); + for table in tables { + for column in &table.columns { + let Some(facts) = sdm_schema::held::facts_held(table, column) + .filter(|facts| facts.catalog_type.is_some()) + else { + return Ok(None); + }; + kept.push(((table.name.clone(), column.name.clone()), facts.clone())); + } + } + let declared = crate::discover::catalog(asker, relation).await?; + let same = kept.iter().all(|((_, name), facts)| { + declared.iter().any(|c| { + c.name == *name && facts.catalog_type.as_deref() == Some(c.declared_type.as_str()) + }) + }); + Ok(same.then_some(kept)) +} + /// The refusal of `table.column`, a column of the schema that the source's /// `relation` does not hold (the owner's item 1). fn not_in_source(table: &str, column: &str, relation: &Relation) -> CoreError { diff --git a/rust/crates/sdm-core/src/discover.rs b/rust/crates/sdm-core/src/discover.rs index 06b7e9a49..02d5f4ad6 100644 --- a/rust/crates/sdm-core/src/discover.rs +++ b/rust/crates/sdm-core/src/discover.rs @@ -100,6 +100,34 @@ pub(crate) fn facts_read( .collect() } +/// The catalog's columns of `source`, a relation the caller named that +/// projects nothing, read and refused as [`table`] reads and refuses them +/// first, and nothing else of the table. +pub(crate) async fn catalog( + asker: &Asker, + source: &Relation, +) -> Result, CoreError> { + let columns = DataRequest::Describe { + tables: vec![source.clone()], + part: DescribePart::Columns, + }; + let [columns] = match spelled(asker, source, [columns]).await { + Ok(answers) => answers, + Err(failed) if failed.code == ErrorCode::ProviderError => { + return Err(match holds(asker, source).await { + Ok(false) => not_held(asker, source).await, + Ok(true) | Err(_) => failed, + }); + } + Err(failed) => return Err(failed), + }; + let declared = declared_columns(columns)?; + if declared.is_empty() { + return Err(not_held(asker, source).await); + } + Ok(declared) +} + /// One table read as `describe_source` reads it, under the name and source /// `table` gives: its catalog's columns and keys over the columns it keeps /// (`table.columns`; every column when none), and the preview of those diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index 5545faa67..32de4a3cc 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -3223,8 +3223,9 @@ pub(crate) async fn one(asker: &Asker, inputs: Inputs) -> Result DescribedColumn { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, } } @@ -131,6 +132,7 @@ fn held(description: &SourceDescription) -> HeldSchema { nullable: c.nullable, unique: c.unique, declared_type: c.declared_type.clone(), + catalog_type: None, exact_number: c.exact_number, }), }) diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 75a4e644a..4755e7ade 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -70,6 +70,7 @@ fn source(relation: &str, dtype: Dtype, received: ColumnType, unique: bool) -> O nullable: !unique, unique, declared_type: None, + catalog_type: None, exact_number: None, }) } @@ -552,6 +553,7 @@ fn numeric() -> HeldSchema { nullable: exact == ExactNumber::Other, unique: exact == ExactNumber::DeclaredKey, declared_type: None, + catalog_type: None, exact_number: Some(exact), }; for table in &mut held.tables { diff --git a/rust/crates/sdm-core/tests/materialize.rs b/rust/crates/sdm-core/tests/materialize.rs index 906629481..6ee754f90 100644 --- a/rust/crates/sdm-core/tests/materialize.rs +++ b/rust/crates/sdm-core/tests/materialize.rs @@ -80,6 +80,7 @@ fn column( nullable: !unique, unique, declared_type: None, + catalog_type: None, exact_number: None, } } diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index 23fa7ebe7..d1fd3b616 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -821,6 +821,7 @@ fn held(description: &SourceDescription) -> sdm_contract::ops::HeldSchema { nullable: c.nullable, unique: c.unique, declared_type: c.declared_type.clone(), + catalog_type: None, exact_number: c.exact_number, }), }) diff --git a/rust/crates/sdm-engine/tests/keyless_tie_types.rs b/rust/crates/sdm-engine/tests/keyless_tie_types.rs index d1262faf5..3e9b3ae3a 100644 --- a/rust/crates/sdm-engine/tests/keyless_tie_types.rs +++ b/rust/crates/sdm-engine/tests/keyless_tie_types.rs @@ -18,6 +18,7 @@ fn keyless_ties_preserve_types_without_reinterpreting_keyed_tables() { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, }; let mut table = Table { diff --git a/rust/crates/sdm-engine/tests/neighbor_requests.rs b/rust/crates/sdm-engine/tests/neighbor_requests.rs index da8d5fcba..f0ba2095b 100644 --- a/rust/crates/sdm-engine/tests/neighbor_requests.rs +++ b/rust/crates/sdm-engine/tests/neighbor_requests.rs @@ -27,6 +27,7 @@ fn column(name: &str, received: ColumnType, dtype: Dtype, stype: Stype) -> Descr nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, } } diff --git a/rust/crates/sdm-host/tests/codec.rs b/rust/crates/sdm-host/tests/codec.rs index 46369e4b8..696f426ce 100644 --- a/rust/crates/sdm-host/tests/codec.rs +++ b/rust/crates/sdm-host/tests/codec.rs @@ -186,6 +186,7 @@ fn description_of(size: usize) -> Vec { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, }], primary_key: vec![], diff --git a/rust/crates/sdm-schema/src/assemble.rs b/rust/crates/sdm-schema/src/assemble.rs index 88d63a815..90bdd8254 100644 --- a/rust/crates/sdm-schema/src/assemble.rs +++ b/rust/crates/sdm-schema/src/assemble.rs @@ -384,6 +384,7 @@ fn described_table(table: InferredTable, duplicates: Option) -> Describe nullable: c.nullable, unique: c.unique, declared_type: None, + catalog_type: None, exact_number: c.exact_number, }) .collect(), diff --git a/rust/crates/sdm-schema/src/held.rs b/rust/crates/sdm-schema/src/held.rs index 7544d73a3..2fa9e8d2f 100644 --- a/rust/crates/sdm-schema/src/held.rs +++ b/rust/crates/sdm-schema/src/held.rs @@ -127,6 +127,7 @@ pub fn by_data_type(relation: &Relation, dtype: Option<&str>) -> SourceFacts { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, } } @@ -356,6 +357,7 @@ fn read_with_frame_widths( nullable: facts.nullable, unique: facts.unique, declared_type: facts.declared_type, + catalog_type: facts.catalog_type, exact_number: facts.exact_number, }); } @@ -461,6 +463,7 @@ pub fn facts_of(relation: &Relation, column: &DescribedColumn) -> SourceFacts { nullable: column.nullable, unique: column.unique, declared_type: column.declared_type.clone(), + catalog_type: column.catalog_type.clone(), exact_number: column.exact_number, } } diff --git a/rust/crates/sdm-schema/src/infer.rs b/rust/crates/sdm-schema/src/infer.rs index f907eb0b4..05e38320d 100644 --- a/rust/crates/sdm-schema/src/infer.rs +++ b/rust/crates/sdm-schema/src/infer.rs @@ -652,6 +652,8 @@ impl Keyed { nullable: !declared.not_null && (!is_primary || declared.key_nullable), unique: self.unique.contains(&declared.name), declared_type: None, + catalog_type: (!declared.declared_type.is_empty()) + .then(|| declared.declared_type.clone()), exact_number: exact_number(field, constraints), }) } diff --git a/rust/crates/sdm-schema/tests/assemble.rs b/rust/crates/sdm-schema/tests/assemble.rs index 7a9e7cead..5e575d7c8 100644 --- a/rust/crates/sdm-schema/tests/assemble.rs +++ b/rust/crates/sdm-schema/tests/assemble.rs @@ -555,6 +555,7 @@ fn the_columns_joined_on_are_marked_again_from_what_the_host_holds() { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, }; assemble::mark_keys(&mut stamped); diff --git a/rust/crates/sdm-schema/tests/edit.rs b/rust/crates/sdm-schema/tests/edit.rs index feb7bdf0b..2688e6208 100644 --- a/rust/crates/sdm-schema/tests/edit.rs +++ b/rust/crates/sdm-schema/tests/edit.rs @@ -53,6 +53,7 @@ fn column(name: &str, dtype: Dtype, stype: Stype) -> DescribedColumn { nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, } } diff --git a/rust/crates/sdm-schema/tests/exact_numbers.rs b/rust/crates/sdm-schema/tests/exact_numbers.rs index a400d45e5..d246afba4 100644 --- a/rust/crates/sdm-schema/tests/exact_numbers.rs +++ b/rust/crates/sdm-schema/tests/exact_numbers.rs @@ -37,6 +37,7 @@ fn plain(name: &str, dtype: Dtype, received: ColumnType, stype: Stype) -> Descri nullable: true, unique: false, declared_type: None, + catalog_type: None, exact_number: None, } } diff --git a/rust/crates/sdm-schema/tests/held.rs b/rust/crates/sdm-schema/tests/held.rs index b98b5a150..c261d7214 100644 --- a/rust/crates/sdm-schema/tests/held.rs +++ b/rust/crates/sdm-schema/tests/held.rs @@ -22,6 +22,7 @@ fn facts(dtype: Dtype, received: ColumnType) -> SourceFacts { nullable: false, unique: true, declared_type: Some("HUGEINT".into()), + catalog_type: None, exact_number: None, } } diff --git a/rust/crates/sdm-testkit/src/edits.rs b/rust/crates/sdm-testkit/src/edits.rs index b30d913b2..93a859696 100644 --- a/rust/crates/sdm-testkit/src/edits.rs +++ b/rust/crates/sdm-testkit/src/edits.rs @@ -179,6 +179,7 @@ pub fn held(described: &SourceDescription) -> HeldSchema { nullable: column.nullable, unique: column.unique, declared_type: column.declared_type.clone(), + catalog_type: column.catalog_type.clone(), exact_number: column.exact_number, }), }) @@ -211,6 +212,7 @@ pub fn described(held: &HeldSchema) -> Option { nullable: facts.nullable, unique: facts.unique, declared_type: facts.declared_type, + catalog_type: facts.catalog_type, exact_number: facts.exact_number, }); } diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index ebe290fc2..c9121fa92 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -209,10 +209,9 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { ); assert!(tables.iter().all(|t| t["duplicate_keys"]["v"] == "0")); // One fetch per statement the reference runs: ten in Schema.from_source, - // six in materialize; and the six of the two tables' catalogs - // and previews each materialize reads again, as the source holds them - // at that run. - assert_eq!(observed.fetches.len(), 22); + // six in materialize; and the two tables' catalogs each materialize + // reads again, as the source holds them at that run. + assert_eq!(observed.fetches.len(), 18); // The whole description, the request state with its payload, the // inputs and the step trace are the ones python-core records // (contract/pinned, which contract/tests holds python-core to), so CI @@ -237,6 +236,29 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { "origin": "inferred" }] }); + // The snapshot also predates each column's catalog type, which a later + // materialize compares with the catalog then. + let catalog_types = [ + ("orders", "order_id", "BIGINT"), + ("orders", "user_id", "BIGINT"), + ("orders", "price", "DOUBLE"), + ("orders", "ts", "TIMESTAMP"), + ("users", "user_id", "BIGINT"), + ("users", "age", "INTEGER"), + ("users", "country", "VARCHAR"), + ]; + for (table, column, declared) in catalog_types { + let columns = expected_description["tables"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|t| t["name"] == table) + .unwrap()["columns"] + .as_array_mut() + .unwrap(); + let held = columns.iter_mut().find(|c| c["name"] == column).unwrap(); + held["catalog_type"] = serde_json::json!(declared); + } assert_eq!(described, expected_description); let mut expected_materialized = pinned["materialized"].clone(); expected_materialized["payload"]["client_generated_keys"] = @@ -278,8 +300,43 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { } } } + // A table whose catalog is unchanged since it was described is read + // again by its catalog only, neither its keys nor its preview. + expected_fetches + .as_array_mut() + .unwrap() + .retain(|fetch| fetch.get("purpose").is_none() || fetch["request"]["part"] == "columns"); assert_eq!(Value::from(observed.fetches), expected_fetches); let mut expected_inputs = pinned["inputs"].clone(); + // The schema each later run is handed keeps those catalog types with + // its columns' facts. + fn with_catalog_types(value: &mut Value, types: &[(&str, &str, &str)]) { + match value { + Value::Object(object) => { + if let (Some(Value::String(table)), Some(Value::Array(columns))) = + (object.get("name").cloned(), object.get_mut("columns")) + { + for column in columns.iter_mut() { + let name = column["name"].as_str().unwrap_or_default().to_owned(); + if let Some(facts) = column.get_mut("source").and_then(Value::as_object_mut) + && let Some((_, _, declared)) = + types.iter().find(|(t, c, _)| *t == table && *c == name) + { + facts.insert("catalog_type".into(), serde_json::json!(declared)); + } + } + } + for child in object.values_mut() { + with_catalog_types(child, types); + } + } + Value::Array(items) => items + .iter_mut() + .for_each(|item| with_catalog_types(item, types)), + _ => {} + } + } + with_catalog_types(&mut expected_inputs, &catalog_types); expected_inputs[2]["state"] = expected_materialized; assert_eq!(Value::from(observed.inputs.clone()), expected_inputs); // predict's call and result: the call's request id compared by @@ -879,10 +936,11 @@ fn base_regression_s_statements_return_the_reference_s_results() { ("predict".to_owned(), 0), ]; // The reads the core marks as plan B's taken out (A-1): two - // tables' catalog, keys and preview; every statement after them is the - // reference's. + // tables' catalogs, unchanged since they were described, so neither + // table's keys nor preview is read again; every statement after them is + // the reference's. let fresh = ours.len() - sdm_testkit::plan_b::as_the_old_client_ran(&ours).len(); - assert_eq!(fresh, 6); + assert_eq!(fresh, 2); let ours = sdm_testkit::plan_b::as_the_old_client_ran(&ours); assert_eq!(per_stage(&ours), counts); assert_eq!(per_stage(&theirs), counts); @@ -891,8 +949,9 @@ fn base_regression_s_statements_return_the_reference_s_results() { let logged = &driver["query_log"]; assert_eq!(logged["statements"], 16 + fresh as u64); assert_eq!(logged["bindings"], 3); - // Of the six read again, the four whose text DuckDB logs as written. - assert_eq!(logged["compared"], 14 + 4); + // The two read again are catalog pragmas, whose text DuckDB does not + // log as written. + assert_eq!(logged["compared"], 14); assert_eq!(logged["differing"], serde_json::json!([])); assert_eq!(driver["capture"]["bindings"].as_array().unwrap().len(), 3); for (position, (a, b)) in ours.iter().zip(&theirs).enumerate() { diff --git a/tests/golden/statements.json b/tests/golden/statements.json index 7ff7c845d..a022a5970 100644 --- a/tests/golden/statements.json +++ b/tests/golden/statements.json @@ -1,6 +1,6 @@ { "about": "Every SQL statement the core hands the Python host's DuckDB provider, per fetch in order, for Schema.from_source and PredictiveQuery.materialize over the shop base, as it is (\"plain\") and with orders held in a table named \"a.b\" beside a schema a holding a table b (\"dotted\"). Recorded by scripts/record_sql_statements.py at the contract version named; regenerated, never edited.", - "contract": 71, + "contract": 72, "query": "PREDICT SUM(orders.price, 0, 30, days) FOR EACH users.user_id", "anchor_time": "2024-03-01", "indices": [ @@ -45,21 +45,9 @@ [ "PRAGMA table_info(\"users\")" ], - [ - "SELECT constraint_type, constraint_column_names\nFROM duckdb_constraints()\nWHERE table_name = 'users'\n AND database_name = current_database() AND schema_name = current_schema()" - ], - [ - "SELECT \"user_id\", \"age\", \"country\" FROM \"users\" LIMIT 1000" - ], [ "PRAGMA table_info(\"orders\")" ], - [ - "SELECT constraint_type, constraint_column_names\nFROM duckdb_constraints()\nWHERE table_name = 'orders'\n AND database_name = current_database() AND schema_name = current_schema()" - ], - [ - "SELECT \"order_id\", \"user_id\", \"price\", \"ts\" FROM \"orders\" LIMIT 1000" - ], [ "SELECT table_name, expressions FROM duckdb_indexes()" ], @@ -110,21 +98,9 @@ [ "PRAGMA table_info(\"users\")" ], - [ - "SELECT constraint_type, constraint_column_names\nFROM duckdb_constraints()\nWHERE table_name = 'users'\n AND database_name = current_database() AND schema_name = current_schema()" - ], - [ - "SELECT \"user_id\", \"age\", \"country\" FROM \"users\" LIMIT 1000" - ], [ "SELECT CAST(c.column_index - 1 AS INTEGER) AS cid, c.column_name AS name,\n c.data_type AS type, NOT c.is_nullable AS notnull,\n c.column_default AS dflt_value,\n COALESCE(list_contains(k.columns, c.column_name), false) AS pk\nFROM duckdb_columns() c\nLEFT JOIN (\n SELECT constraint_column_names AS columns\n FROM duckdb_constraints()\n WHERE table_name = 'a.b'\n AND database_name = current_database() AND schema_name = current_schema()\n AND constraint_type = 'PRIMARY KEY'\n) k ON true\nWHERE c.table_name = 'a.b'\n AND c.database_name = current_database() AND c.schema_name = current_schema()\nORDER BY c.column_index" ], - [ - "SELECT constraint_type, constraint_column_names\nFROM duckdb_constraints()\nWHERE table_name = 'a.b'\n AND database_name = current_database() AND schema_name = current_schema()" - ], - [ - "SELECT \"order_id\", \"user_id\", \"price\", \"ts\" FROM \"a.b\" LIMIT 1000" - ], [ "SELECT table_name, expressions FROM duckdb_indexes()" ], From ff17f07edc64bba73d1be3e7091a0a55a4b99a9b Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 15:46:38 -0800 Subject: [PATCH 18/24] Bring the live tests to what the endpoints serve A related table sends its own primary key, so orders' explanations come back keyed; the live explanation tests expected them keyless. No endpoint serves temporal link prediction, so its two live tests are removed; the offline batching tests stay. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../nvidia/sdm/TemporalLinkBatchingTest.java | 47 ------------- .../tests/test_live_relational.py | 38 +++++------ .../tests/test_temporal_link_batching.py | 66 ------------------- tests/compatibility/manifest.json | 10 --- 4 files changed, 19 insertions(+), 142 deletions(-) diff --git a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java index 172317164..1559880bb 100644 --- a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java +++ b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java @@ -117,51 +117,4 @@ static List sizes(Request request) throws Exception { } } - @Test - void liveShopBatches() throws Exception { - String url = System.getenv("SDM_LINK_URL"); - if (url == null || url.isBlank()) url = System.getenv("SDM_RELATIONAL_URL"); - org.junit.jupiter.api.Assumptions.assumeTrue(url != null && !url.isBlank(), - "SDM_LINK_URL or SDM_RELATIONAL_URL is required"); - var fixture = ParityRunner.read(java.nio.file.Path.of("../tests/compatibility/fixtures/shop.json")); - var original = object(fixture.get("tables")); - List> users = new ArrayList<>(), events = new ArrayList<>(); - var sourceUsers = rows(object(original.get("users"))); - var sourceEvents = rows(object(original.get("events"))); - for (int i = 0; i < 201; i++) { - var source = sourceUsers.get(i % sourceUsers.size()); - var user = new ArrayList<>(source); user.set(0, i); users.add(user); - for (var sourceEvent : sourceEvents) { - if (((Number) sourceEvent.get(1)).longValue() != ((Number) source.get(0)).longValue()) continue; - var event = new ArrayList<>(sourceEvent); - event.set(0, events.size()); event.set(1, i); events.add(event); - } - } - var userTable = new LinkedHashMap<>(object(original.get("users"))); userTable.put("rows", users); - var eventTable = new LinkedHashMap<>(object(original.get("events"))); eventTable.put("rows", events); - // The same expanded shop fixture as Python, before public ingestion. - try (RootAllocator allocator = new RootAllocator(); - Table userData = ParityRunner.table(userTable, allocator); - Table eventData = ParityRunner.table(eventTable, allocator); - Table itemData = ParityRunner.table(object(original.get("items")), allocator)) { - Schema schema = Schema.fromData(Map.of("users", userData, "events", eventData, "items", itemData)) - .withPrimaryKey("users", "id").withPrimaryKey("events", "event").withPrimaryKey("items", "item") - .withTimeColumn("events", "event_time") - .withRelationships(List.of(new Relationship("events", "id", "users", "id"), - new Relationship("events", "item", "items", "item"))); - var ids = java.util.stream.IntStream.range(0, 201).boxed().toList(); - Request request = PredictiveQuery.parse("PREDICT LIST_DISTINCT(events.item, 0, 30, days) FOR EACH users.id") - .materialize(schema, MaterializeOptions.builder().model(SdmClient.KUMO_RELATIONAL) - .indices(new ArrayList<>(ids)).option("random_seed", 7).build()); - assertEquals(List.of(200, 1), sizes(request)); - try (SdmClient client = SdmClient.connect(url, - ClientOptions.builder().timeout(java.time.Duration.ofSeconds(180)).build())) { - var result = client.predict(request, SdmClient.KUMO_RELATIONAL); - assertEquals(201, result.rows()); - assertEquals(ids, result.items().stream().map(item -> ((Number) item.id()).intValue()).toList()); - assertTrue(result.absent(List.of("rankings")).isEmpty()); - } - } - } - } diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index 52886a52e..dbe6d44cd 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -348,15 +348,16 @@ def test_run_mode_is_refused_as_an_option(query): EXPLAIN = ['prediction', 'probabilities', 'explanation'] BINARY_QUERY = 'PREDICT COUNT(orders.*, 0, 30, days) > 2 FOR EACH users.user_id' -#: The table whose related blocks carry ``keys``, and its key. The engine -#: sends a related table's primary key only when a relationship joins on it: -#: ``user_id`` is the join column, so only ``users`` is keyed. -KEYS = {'users': 'user_id'} +#: The tables whose related blocks carry ``keys``, and their keys: a related +#: table sends its own primary key, which comes back as the row's key. +KEYS = {'users': 'user_id', 'orders': 'order_id'} +#: A column of each keyed table whose sent value the returned row is checked +#: against. +CHECKED = {'users': 'segment', 'orders': 'price'} #: The tables whose related blocks carry ``rows`` and no ``keys``, each with a -#: column whose sent values the returned rows are checked against: ``orders``, -#: whose ``order_id`` no relationship joins on, and ``events``, which has no -#: primary key. -KEYLESS = {'orders': 'price', 'events': 'value'} +#: column whose sent values the returned rows are checked against: ``events``, +#: which has no primary key. +KEYLESS = {'events': 'value'} @pytest.fixture(scope='module') @@ -371,9 +372,9 @@ def assert_explained_as_sent(result, asked): among the related ones. Each block's ``keys`` are the primary keys of its ``rows``, whose values are the caller's for that key and belong to that entity, and its gradients are ``[output][row][column]``, aligned with - ``rows``. Keys come back only for a join column, so only ``users`` has - them; every other block has ``rows`` and no ``keys``, each row the - caller's columns of one of that entity's rows. In ``to_pandas()`` the + ``rows``. Keys come back for a table's primary key, so ``users`` and + ``orders`` have them; ``events`` has ``rows`` and no ``keys``, each row + the caller's columns of one of that entity's rows. In ``to_pandas()`` the entity's own table is at hop 0, the others at 1 or more, and an unkeyed table's lines have no ``key``. """ @@ -420,7 +421,7 @@ def assert_explained_as_sent(result, asked): assert int(row[KEYS[name]]) == int(row_key) assert int(row['user_id']) == entity sent = graph[name].loc[int(row_key)] - assert row['segment'] == sent['segment'] + assert row[CHECKED[name]] == sent[CHECKED[name]] assert [int(one) for one in related['users']['keys']] == [entity] flat = result.explanations.gradients.to_pandas() @@ -458,10 +459,9 @@ def test_every_entity_is_explained_with_its_own_related_rows( Assumptions: - The NIM explains every predict row, both probabilities of a binary target. - - Keys come back only for a join column: ``users`` (joined on - ``user_id``) carries ``keys``; ``orders`` (``order_id`` joins nothing) - and ``events`` (no primary key) carry ``rows`` and no ``keys``, so - both kinds of related block are read back. + - Keys come back for a table's primary key: ``users`` and ``orders`` + carry ``keys``; ``events`` (no primary key) carries ``rows`` and no + ``keys``, so both kinds of related block are read back. """ asked = [0, 1, 2, 3] request = binary_query.materialize(indices=asked) @@ -481,9 +481,9 @@ def test_a_request_sent_in_several_calls_explains_every_entity( related rows, keyed only where the key is a join column. Assumptions: - - Keys come back only for a join column, so only ``users`` has them; - ``orders`` and ``events`` rows are checked by entity and value rather - than by key. + - Keys come back for a table's primary key, so ``users`` and ``orders`` + have them; ``events`` rows are checked by entity and value rather than + by key. - ``rows_per_call=2`` stands in for the cap, as in the batching test above, and splits eight entities into four calls. """ diff --git a/packages/structured-data-models-client/tests/test_temporal_link_batching.py b/packages/structured-data-models-client/tests/test_temporal_link_batching.py index 02c6f1bf1..dbc6450f5 100644 --- a/packages/structured-data-models-client/tests/test_temporal_link_batching.py +++ b/packages/structured-data-models-client/tests/test_temporal_link_batching.py @@ -192,69 +192,3 @@ def answer(call, response): assert answered.to_pandas()['id'].astype(int).tolist() == list( range(1, 201) ) - - -@pytest.mark.live_nim -def test_live_temporal_link_batches_preserve_entity_identity(): - import json - import os - from pathlib import Path - - from sdm_client import Relationship - - url = os.environ.get('SDM_LINK_URL') or os.environ.get('SDM_RELATIONAL_URL') - if not url: - pytest.skip('SDM_LINK_URL or SDM_RELATIONAL_URL is required') - fixture = json.loads( - ( - Path(__file__).resolve().parents[3] - / 'tests/compatibility/fixtures/shop.json' - ).read_text() - ) - frames = { - name: pd.DataFrame(table['rows'], columns=table['columns']) - for name, table in fixture['tables'].items() - } - # Expand the maintained shop population before public ingestion. No wire - # payload or sampled data is patched to accommodate the model endpoint. - users, events = [], [] - population = len(frames['users']) - for identity in range(201): - original = frames['users'].iloc[identity % population] - user = original.copy() - user['id'] = identity - users.append(user) - for _, event in frames['events'][ - frames['events']['id'] == original['id'] - ].iterrows(): - event = event.copy() - event['id'], event['event'] = identity, len(events) - events.append(event) - frames['users'] = pd.DataFrame(users).reset_index(drop=True) - frames['events'] = pd.DataFrame(events).reset_index(drop=True) - frames['events']['event_time'] = pd.to_datetime( - frames['events']['event_time'] - ) - schema = Schema.from_data( - frames, - primary_keys=fixture['primary_keys'], - time_columns={'events': 'event_time'}, - relationships=[ - Relationship('events', 'id', 'users', 'id'), - Relationship('events', 'item', 'items', 'item'), - ], - ) - request = PredictiveQuery( - 'PREDICT LIST_DISTINCT(events.item, 0, 30, days) FOR EACH users.id', - schema, - ).materialize( - indices=list(range(201)), model=KumoRelational(), random_seed=7 - ) - assert [ - len(p['predict']['instance_table']['rows']) for p in request.payloads - ] == [200, 1] - with SDMClient(url=url, timeout=180) as client: - result = client.predict(request, model=KumoRelational()) - assert result.to_pandas()['id'].astype(int).tolist() == list(range(201)) - assert len(result) == 201 - assert not result.absent(['rankings']) diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index e5d241a65..b98c76e32 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -9938,16 +9938,6 @@ "test": "java/src/test/java/com/nvidia/sdm/AutomaticBatchSessionTest.java#livePreparedBatchPrediction", "evidence_level": "live-required" }, - { - "api_contract": "automatic-batch-sessions", - "test": "packages/structured-data-models-client/tests/test_temporal_link_batching.py::test_live_temporal_link_batches_preserve_entity_identity", - "evidence_level": "live-required" - }, - { - "api_contract": "automatic-batch-sessions", - "test": "java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java#liveShopBatches", - "evidence_level": "live-required" - }, { "api_contract": "gradient-explanations", "test": "packages/structured-data-models-client/tests/test_gradient_explanations.py::test_pretrained_local_model_gradients", From 1e0e8ec06f4464725cd6b628516ac5edefac9282 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 16:14:40 -0800 Subject: [PATCH 19/24] Reuse a table's facts only while its catalog declares it unchanged A column's facts were reused whenever its declared type matched, so a dropped NOT NULL kept the old nullable=false and a key column holding a null could be sampled. A column now keeps its whole catalog declaration (type, nullability, key and uniqueness, as CatalogDeclaration), and the facts are reused only where all of it matches. SQLite, whose values decide a column's type, is always read in full. Removing the live link tests is recorded against the merge base, as the ledger requires. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 13 ++-- contract/tests/test_sqlite_source_edges.py | 4 +- java/src/main/java/com/nvidia/sdm/Schema.java | 4 +- .../src/sdm_client/_rust/results.py | 23 ++++--- .../tests/test_schema_copy_provenance.py | 39 +++++++++-- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 67 ++++++++++++++++--- rust/crates/sdm-contract/src/ops.rs | 39 +++++++++-- rust/crates/sdm-core/src/big_integer.rs | 2 +- rust/crates/sdm-core/src/describe.rs | 45 +++++++------ rust/crates/sdm-core/src/sql_column.rs | 2 +- rust/crates/sdm-core/src/view.rs | 2 +- rust/crates/sdm-core/tests/edit_schema.rs | 4 +- rust/crates/sdm-core/tests/held_schema.rs | 4 +- rust/crates/sdm-core/tests/materialize.rs | 2 +- rust/crates/sdm-core/tests/view_captures.rs | 2 +- .../sdm-engine/tests/keyless_tie_types.rs | 2 +- .../sdm-engine/tests/neighbor_requests.rs | 2 +- rust/crates/sdm-host/tests/codec.rs | 2 +- rust/crates/sdm-schema/src/assemble.rs | 2 +- rust/crates/sdm-schema/src/held.rs | 6 +- rust/crates/sdm-schema/src/infer.rs | 3 +- rust/crates/sdm-schema/tests/assemble.rs | 2 +- rust/crates/sdm-schema/tests/edit.rs | 2 +- rust/crates/sdm-schema/tests/exact_numbers.rs | 2 +- rust/crates/sdm-schema/tests/held.rs | 2 +- rust/crates/sdm-testkit/src/edits.rs | 4 +- .../sdm-testkit/tests/duckdb_provider.rs | 42 ++++++++---- tests/compatibility/manifest.json | 6 ++ 29 files changed, 236 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 928aff9fb..a16094791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,12 +100,13 @@ The public API snapshot records this version update; publication remains pending - A related table sampled from frames sends its own primary key, where 1.0.0 sent a generated `__node_id`. - Each `materialize()` on a schema from `Schema.from_source` reads its - tables' catalog again, one query per table, and a semantic model's - definition, so a column dropped or retyped since is found. A table whose - catalog changed is read again in full, its keys and first rows included. - 1.0.0 read them once and kept them on the schema. A table emptied since it - was described is refused as empty by a warehouse's row count; elsewhere it - reads as a table with no times, as 1.0.0 read it. + tables' catalog again, one query per table, so a column dropped or + retyped since is found. A table whose catalog declares a column otherwise + is read again in full, its keys and first rows included, as are SQLite's + tables and a semantic model's, with its definition. 1.0.0 read them once + and kept them on the schema. A table emptied since it was described is + refused as empty by a warehouse's row count; elsewhere it reads as a + table with no times, as 1.0.0 read it. - An HTTP 408 from `POST /v1/predictions` is retried, as 429, 502, 503 and 504 are. - A relational request reached over three or more hops fills a foreign key diff --git a/contract/tests/test_sqlite_source_edges.py b/contract/tests/test_sqlite_source_edges.py index df1db2ddc..7bc6eb8cc 100644 --- a/contract/tests/test_sqlite_source_edges.py +++ b/contract/tests/test_sqlite_source_edges.py @@ -184,7 +184,7 @@ def test_a_partial_unique_index_keeps_the_key_unique( wanted = _reference('partial-unique') locate: list[int] = [] got = _on_the_core(monkeypatch, 'partial-unique', locate) - assert locate == [2] + assert locate == [7] assert got['schema'] == wanted['schema'] assert got['statements'] == { **wanted['statements'], @@ -203,7 +203,7 @@ def test_each_sql_materialize_refreshes_its_sampler_facts_once( wanted = _reference('materialize-twice') locate: list[int] = [] got = _on_the_core(monkeypatch, 'materialize-twice', locate) - assert locate == [2, 2] + assert locate == [6, 6] assert got['outcome'] == wanted['outcome'] assert got['statements'] == { **wanted['statements'], diff --git a/java/src/main/java/com/nvidia/sdm/Schema.java b/java/src/main/java/com/nvidia/sdm/Schema.java index fea0653dc..1189ed3ff 100644 --- a/java/src/main/java/com/nvidia/sdm/Schema.java +++ b/java/src/main/java/com/nvidia/sdm/Schema.java @@ -658,8 +658,8 @@ private static Schema described(Object read, Source source) { if (column.get("declared_type") != null) { facts.put("declared_type", column.get("declared_type")); } - if (column.get("catalog_type") != null) { - facts.put("catalog_type", column.get("catalog_type")); + if (column.get("catalog") != null) { + facts.put("catalog", column.get("catalog")); } if (column.get("exact_number") != null) { facts.put("exact_number", column.get("exact_number")); diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/results.py b/packages/structured-data-models-client/src/sdm_client/_rust/results.py index 3d5994aff..d09113432 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/results.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/results.py @@ -103,6 +103,12 @@ def _list(value: Any, where: str) -> list[Any]: return value +def _object(value: Any, where: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise CodecError(f'{where} is not an object') + return value + + # --- What the source said of a column ------------------------------------------ @@ -115,8 +121,9 @@ class SourceFacts: since), the data type the source's catalog and values give it, the type the provider handed over (the contract's ``ColumnType``, as JSON), whether it may hold no value, whether a key of it alone keeps it - unique, a big integer's or temporal column's declared type, the type - its catalog declares, and whether it is a column of exact numbers.""" + unique, a big integer's or temporal column's declared type, how its + catalog declares it (the contract's ``CatalogDeclaration``, as JSON), + and whether it is a column of exact numbers.""" relation: Any dtype: str @@ -124,7 +131,7 @@ class SourceFacts: nullable: bool unique: bool declared_type: str | None = None - catalog_type: str | None = None + catalog: Any = None exact_number: str | None = None @classmethod @@ -134,7 +141,7 @@ def from_contract(cls, value: Any) -> SourceFacts: value, "a column's source facts", ('relation', 'dtype', 'received', 'nullable', 'unique'), - ('declared_type', 'catalog_type', 'exact_number'), + ('declared_type', 'catalog', 'exact_number'), ) return cls( relation=read['relation'], @@ -143,7 +150,7 @@ def from_contract(cls, value: Any) -> SourceFacts: nullable=_flag(read['nullable'], 'nullable'), unique=_flag(read['unique'], 'unique'), declared_type=_maybe(read['declared_type'], _text, 'declared_type'), - catalog_type=_maybe(read['catalog_type'], _text, 'catalog_type'), + catalog=_maybe(read['catalog'], _object, 'catalog'), exact_number=_maybe(read['exact_number'], _text, 'exact_number'), ) @@ -164,7 +171,7 @@ def of_described( 'nullable', 'unique', 'declared_type', - 'catalog_type', + 'catalog', 'exact_number', ) if key in column @@ -184,8 +191,8 @@ def to_contract(self) -> dict[str, Any]: } if self.declared_type is not None: written['declared_type'] = self.declared_type - if self.catalog_type is not None: - written['catalog_type'] = self.catalog_type + if self.catalog is not None: + written['catalog'] = self.catalog if self.exact_number is not None: written['exact_number'] = self.exact_number return written diff --git a/packages/structured-data-models-client/tests/test_schema_copy_provenance.py b/packages/structured-data-models-client/tests/test_schema_copy_provenance.py index dc7dbb72a..d488d78b9 100644 --- a/packages/structured-data-models-client/tests/test_schema_copy_provenance.py +++ b/packages/structured-data-models-client/tests/test_schema_copy_provenance.py @@ -126,12 +126,11 @@ def rebuilt(schema: Schema) -> Schema: } -@pytest.mark.parametrize('made', sorted(_copies())) -def test_copies_refresh_the_same_source_metadata(made, monkeypatch): +def _watched(monkeypatch: pytest.MonkeyPatch) -> list[str]: + r"""The kind of each request a source is asked from now on.""" from sdm_client._rust import providers - source = _shop() - asked = [] + asked: list[str] = [] real = providers.for_source def watched(given): @@ -144,6 +143,13 @@ def answering(request, keys): return choice, answering monkeypatch.setattr(providers, 'for_source', watched) + return asked + + +@pytest.mark.parametrize('made', sorted(_copies())) +def test_copies_refresh_the_same_source_metadata(made, monkeypatch): + source = _shop() + asked = _watched(monkeypatch) described = _described(source) asked.clear() copied = _copies()[made](described) @@ -158,6 +164,31 @@ def answering(request, keys): assert 'describe' in asked and 'preview' not in asked +@pytest.mark.parametrize( + 'change', + [ + 'ALTER TABLE users ALTER age SET NOT NULL', + 'ALTER TABLE users ALTER age TYPE DOUBLE', + ], +) +def test_a_column_declared_otherwise_since_is_read_again_in_full( + change, monkeypatch +): + r"""A run reads a table's catalog alone while it declares every column + as describe read it, and the table in full once one is declared + otherwise, its nullability as much as its type.""" + source = _shop() + asked = _watched(monkeypatch) + schema = _described(source) + asked.clear() + _payload(schema) + assert 'describe' in asked and 'preview' not in asked + source.connection.execute(change) + asked.clear() + _payload(schema) + assert 'preview' in asked + + def test_source_facts_are_immutable_and_excluded_from_column_equality(): from sdm_client._rust import results diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 6e748461a..486f4f5b0 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:3fef3439c3e7d82f871ef0f6d47f5536c64aeea9e62c0e594d3a3d91d350a520 sdm-contract-72.json +72 sha256:88a10d45842d7a2a66f0a65ad4b43ac7a7522b566f78cbb6b9d0de96b6f8fde0 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index 10736fac4..c95be4c1e 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -2040,11 +2040,15 @@ "null" ] }, - "catalog_type": { - "description": "[`DescribedColumn::catalog_type`]; absent from the JSON when none.", - "type": [ - "string", - "null" + "catalog": { + "description": "[`DescribedColumn::catalog`]; absent from the JSON when none.", + "anyOf": [ + { + "$ref": "#/$defs/CatalogDeclaration" + }, + { + "type": "null" + } ] }, "exact_number": { @@ -2563,6 +2567,45 @@ "uint64" ] }, + "CatalogDeclaration": { + "description": "One column as its source's catalog declared it\n([`crate::catalog::CatalogColumn`], less its name).", + "type": "object", + "properties": { + "declared_type": { + "type": "string" + }, + "not_null": { + "type": "boolean" + }, + "primary_key": { + "type": "boolean" + }, + "key_nullable": { + "type": "boolean" + }, + "unique": { + "type": "boolean" + }, + "dtype": { + "anyOf": [ + { + "$ref": "#/$defs/Dtype" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "declared_type", + "not_null", + "primary_key", + "key_nullable", + "unique" + ] + }, "ExactNumber": { "description": "A column of numbers handed over as their exact text\n([`DescribedColumn::exact_number`]), by what its catalog declares of it.", "oneOf": [ @@ -11111,11 +11154,15 @@ "null" ] }, - "catalog_type": { - "description": "The type the source's catalog declares the column, as it spells it,\nfor every column a catalog lists. A later `materialize` compares it\nwith the catalog then, and reads the table's preview again only\nwhere they differ. Absent where no catalog was read.", - "type": [ - "string", - "null" + "catalog": { + "description": "How the source's catalog declared the column when it was read, for\nevery column a catalog lists. A later `materialize` compares it with\nthe catalog then, and reads the table's keys and preview again only\nwhere they differ. Absent where no catalog was read.", + "anyOf": [ + { + "$ref": "#/$defs/CatalogDeclaration" + }, + { + "type": "null" + } ] }, "exact_number": { diff --git a/rust/crates/sdm-contract/src/ops.rs b/rust/crates/sdm-contract/src/ops.rs index b5bb135fa..edf8ff1ae 100644 --- a/rust/crates/sdm-contract/src/ops.rs +++ b/rust/crates/sdm-contract/src/ops.rs @@ -863,12 +863,12 @@ pub struct DescribedColumn { /// Other columns keep none. Absent when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub declared_type: Option, - /// The type the source's catalog declares the column, as it spells it, - /// for every column a catalog lists. A later `materialize` compares it - /// with the catalog then, and reads the table's preview again only + /// How the source's catalog declared the column when it was read, for + /// every column a catalog lists. A later `materialize` compares it with + /// the catalog then, and reads the table's keys and preview again only /// where they differ. Absent where no catalog was read. #[serde(default, skip_serializing_if = "Option::is_none")] - pub catalog_type: Option, + pub catalog: Option, /// Set for a column of numbers the host provider handed over as their /// exact decimal text (marked [`crate::data::EXACT_NUMBER_MARKER`]): /// whether its catalog declares it a key. The schema reads such a column @@ -1046,6 +1046,33 @@ pub struct HeldColumn { pub source: Option, } +/// One column as its source's catalog declared it +/// ([`crate::catalog::CatalogColumn`], less its name). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CatalogDeclaration { + pub declared_type: String, + pub not_null: bool, + pub primary_key: bool, + pub key_nullable: bool, + pub unique: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dtype: Option, +} + +impl From<&crate::catalog::CatalogColumn> for CatalogDeclaration { + fn from(column: &crate::catalog::CatalogColumn) -> Self { + Self { + declared_type: column.declared_type.clone(), + not_null: column.not_null, + primary_key: column.primary_key, + key_nullable: column.key_nullable, + unique: column.unique, + dtype: column.dtype, + } + } +} + /// What the source said of one column ([`DescribedColumn`]'s facts), which /// a host keeps on the column and hands back whole (contract v28). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -1073,9 +1100,9 @@ pub struct SourceFacts { /// [`DescribedColumn::declared_type`]; absent from the JSON when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub declared_type: Option, - /// [`DescribedColumn::catalog_type`]; absent from the JSON when none. + /// [`DescribedColumn::catalog`]; absent from the JSON when none. #[serde(default, skip_serializing_if = "Option::is_none")] - pub catalog_type: Option, + pub catalog: Option, /// [`DescribedColumn::exact_number`]; absent from the JSON when none. #[serde(default, skip_serializing_if = "Option::is_none")] pub exact_number: Option, diff --git a/rust/crates/sdm-core/src/big_integer.rs b/rust/crates/sdm-core/src/big_integer.rs index 0906491c5..f537a7631 100644 --- a/rust/crates/sdm-core/src/big_integer.rs +++ b/rust/crates/sdm-core/src/big_integer.rs @@ -215,7 +215,7 @@ mod tests { nullable: true, unique: false, declared_type: Some("HUGEINT".into()), - catalog_type: None, + catalog: None, exact_number: None, }; let mut refreshed = old.clone(); diff --git a/rust/crates/sdm-core/src/describe.rs b/rust/crates/sdm-core/src/describe.rs index 2c24c8e7f..924ed0647 100644 --- a/rust/crates/sdm-core/src/describe.rs +++ b/rust/crates/sdm-core/src/describe.rs @@ -17,8 +17,8 @@ use arrow_schema::TimeUnit; use sdm_contract::catalog::{CatalogAnswer, CatalogForeignKey, KeyKind}; use sdm_contract::data::{DataRequest, DescribePart, ProviderError, Relation, TableKey}; use sdm_contract::ops::{ - ColumnRef, DescribedSource, HeldSchema, HeldTable, NamedTable, OperationResult, - SourceDescription, SourceFacts, SourceSpec, + CatalogDeclaration, ColumnRef, DescribedSource, HeldSchema, HeldTable, NamedTable, + OperationResult, SourceDescription, SourceFacts, SourceSpec, }; use sdm_contract::{CoreError, Detail, ErrorCode, HostFacts, Scalar, Sentence, Unit}; use sdm_engine::time::text::Reader; @@ -117,9 +117,9 @@ pub(crate) async fn run( let mut tables = Vec::with_capacity(names.len()); // Catalog types retained for big-integer key casts and temporal windows. let mut source_types = Vec::with_capacity(names.len()); - // Every column's catalog type, which a later run compares with the - // catalog then. - let mut catalog_types = Vec::with_capacity(names.len()); + // Every column as its catalog declares it, which a later run compares + // with the catalog then. + let mut catalogs = Vec::with_capacity(names.len()); let mut primary_constraints = Vec::with_capacity(names.len()); for table in &names { // The table read as `materialize` reads a column it holds no facts @@ -133,10 +133,10 @@ pub(crate) async fn run( .. } = crate::discover::table(&asker, table, caller_named, None).await?; source_types.push(types); - catalog_types.push( + catalogs.push( declared .iter() - .map(|c| (c.name.clone(), c.declared_type.clone())) + .map(|c| (c.name.clone(), CatalogDeclaration::from(c))) .collect::>(), ); primary_constraints.push( @@ -311,16 +311,16 @@ pub(crate) async fn run( .tables .iter_mut() .zip(source_types) - .zip(catalog_types) + .zip(catalogs) { for column in &mut table.columns { column.declared_type = types .iter() .find(|(name, _)| *name == column.name) .map(|(_, declared)| declared.clone()); - column.catalog_type = catalog + column.catalog = catalog .iter() - .find(|(name, declared)| *name == column.name && !declared.is_empty()) + .find(|(name, _)| *name == column.name) .map(|(_, declared)| declared.clone()); } } @@ -979,25 +979,30 @@ async fn read_as_it_stands( } /// The facts `tables` hold of their columns, where the catalog of -/// `relation` declares each of them now as it did when they were read: the -/// run then reads neither the table's keys nor its preview again. None -/// where a column is gone, is declared another type or holds no catalog -/// type, and where the table projects its source: the whole read then says -/// what changed. A catalog that cannot be read is refused as the whole -/// read refuses it. +/// `relation` declares each of them now as it did when they were read (its +/// type, nullability and keys): the run then reads neither the table's keys +/// nor its preview again. None where a column is gone, is declared +/// otherwise or holds no declaration, where the table projects its source, +/// and where the source's columns hold affinities rather than types +/// (SQLite's), whose values decide a column's type: the whole read then +/// says what changed. A catalog that cannot be read is refused as the +/// whole read refuses it. async fn unchanged( asker: &Asker, relation: &Relation, tables: &[&HeldTable], ) -> Result>, CoreError> { - if relation.is_derived() || tables.iter().any(|t| !t.calculated_columns.is_empty()) { + let typed = asker + .data() + .is_none_or(crate::compose::DataAccess::holds_column_types); + if !typed || relation.is_derived() || tables.iter().any(|t| !t.calculated_columns.is_empty()) { return Ok(None); } let mut kept = Vec::new(); for table in tables { for column in &table.columns { - let Some(facts) = sdm_schema::held::facts_held(table, column) - .filter(|facts| facts.catalog_type.is_some()) + let Some(facts) = + sdm_schema::held::facts_held(table, column).filter(|facts| facts.catalog.is_some()) else { return Ok(None); }; @@ -1007,7 +1012,7 @@ async fn unchanged( let declared = crate::discover::catalog(asker, relation).await?; let same = kept.iter().all(|((_, name), facts)| { declared.iter().any(|c| { - c.name == *name && facts.catalog_type.as_deref() == Some(c.declared_type.as_str()) + c.name == *name && facts.catalog.as_ref() == Some(&CatalogDeclaration::from(c)) }) }); Ok(same.then_some(kept)) diff --git a/rust/crates/sdm-core/src/sql_column.rs b/rust/crates/sdm-core/src/sql_column.rs index 17830a33c..a5a5714f0 100644 --- a/rust/crates/sdm-core/src/sql_column.rs +++ b/rust/crates/sdm-core/src/sql_column.rs @@ -159,7 +159,7 @@ async fn run( nullable: column.nullable, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: column.exact_number, }); // The source identity stays logical; only the declarations are durable. diff --git a/rust/crates/sdm-core/src/view.rs b/rust/crates/sdm-core/src/view.rs index bbdb865f5..f29147335 100644 --- a/rust/crates/sdm-core/src/view.rs +++ b/rust/crates/sdm-core/src/view.rs @@ -596,7 +596,7 @@ pub(crate) fn describe( })?; crate::discover::retained_source_type(field, declared).map(str::to_owned) }), - catalog_type: None, + catalog: None, exact_number: None, }); } diff --git a/rust/crates/sdm-core/tests/edit_schema.rs b/rust/crates/sdm-core/tests/edit_schema.rs index 34e027086..87620472b 100644 --- a/rust/crates/sdm-core/tests/edit_schema.rs +++ b/rust/crates/sdm-core/tests/edit_schema.rs @@ -80,7 +80,7 @@ fn column(name: &str, dtype: Dtype, stype: Stype) -> DescribedColumn { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } @@ -132,7 +132,7 @@ fn held(description: &SourceDescription) -> HeldSchema { nullable: c.nullable, unique: c.unique, declared_type: c.declared_type.clone(), - catalog_type: None, + catalog: None, exact_number: c.exact_number, }), }) diff --git a/rust/crates/sdm-core/tests/held_schema.rs b/rust/crates/sdm-core/tests/held_schema.rs index 4755e7ade..21be4e556 100644 --- a/rust/crates/sdm-core/tests/held_schema.rs +++ b/rust/crates/sdm-core/tests/held_schema.rs @@ -70,7 +70,7 @@ fn source(relation: &str, dtype: Dtype, received: ColumnType, unique: bool) -> O nullable: !unique, unique, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, }) } @@ -553,7 +553,7 @@ fn numeric() -> HeldSchema { nullable: exact == ExactNumber::Other, unique: exact == ExactNumber::DeclaredKey, declared_type: None, - catalog_type: None, + catalog: None, exact_number: Some(exact), }; for table in &mut held.tables { diff --git a/rust/crates/sdm-core/tests/materialize.rs b/rust/crates/sdm-core/tests/materialize.rs index 6ee754f90..78825a779 100644 --- a/rust/crates/sdm-core/tests/materialize.rs +++ b/rust/crates/sdm-core/tests/materialize.rs @@ -80,7 +80,7 @@ fn column( nullable: !unique, unique, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } diff --git a/rust/crates/sdm-core/tests/view_captures.rs b/rust/crates/sdm-core/tests/view_captures.rs index d1fd3b616..26b193383 100644 --- a/rust/crates/sdm-core/tests/view_captures.rs +++ b/rust/crates/sdm-core/tests/view_captures.rs @@ -821,7 +821,7 @@ fn held(description: &SourceDescription) -> sdm_contract::ops::HeldSchema { nullable: c.nullable, unique: c.unique, declared_type: c.declared_type.clone(), - catalog_type: None, + catalog: None, exact_number: c.exact_number, }), }) diff --git a/rust/crates/sdm-engine/tests/keyless_tie_types.rs b/rust/crates/sdm-engine/tests/keyless_tie_types.rs index 3e9b3ae3a..94da01acd 100644 --- a/rust/crates/sdm-engine/tests/keyless_tie_types.rs +++ b/rust/crates/sdm-engine/tests/keyless_tie_types.rs @@ -18,7 +18,7 @@ fn keyless_ties_preserve_types_without_reinterpreting_keyed_tables() { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, }; let mut table = Table { diff --git a/rust/crates/sdm-engine/tests/neighbor_requests.rs b/rust/crates/sdm-engine/tests/neighbor_requests.rs index f0ba2095b..9245631d9 100644 --- a/rust/crates/sdm-engine/tests/neighbor_requests.rs +++ b/rust/crates/sdm-engine/tests/neighbor_requests.rs @@ -27,7 +27,7 @@ fn column(name: &str, received: ColumnType, dtype: Dtype, stype: Stype) -> Descr nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } diff --git a/rust/crates/sdm-host/tests/codec.rs b/rust/crates/sdm-host/tests/codec.rs index 696f426ce..6b51378d3 100644 --- a/rust/crates/sdm-host/tests/codec.rs +++ b/rust/crates/sdm-host/tests/codec.rs @@ -186,7 +186,7 @@ fn description_of(size: usize) -> Vec { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, }], primary_key: vec![], diff --git a/rust/crates/sdm-schema/src/assemble.rs b/rust/crates/sdm-schema/src/assemble.rs index 90bdd8254..b22def5db 100644 --- a/rust/crates/sdm-schema/src/assemble.rs +++ b/rust/crates/sdm-schema/src/assemble.rs @@ -384,7 +384,7 @@ fn described_table(table: InferredTable, duplicates: Option) -> Describe nullable: c.nullable, unique: c.unique, declared_type: None, - catalog_type: None, + catalog: None, exact_number: c.exact_number, }) .collect(), diff --git a/rust/crates/sdm-schema/src/held.rs b/rust/crates/sdm-schema/src/held.rs index 2fa9e8d2f..9c4c1b6ee 100644 --- a/rust/crates/sdm-schema/src/held.rs +++ b/rust/crates/sdm-schema/src/held.rs @@ -127,7 +127,7 @@ pub fn by_data_type(relation: &Relation, dtype: Option<&str>) -> SourceFacts { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } @@ -357,7 +357,7 @@ fn read_with_frame_widths( nullable: facts.nullable, unique: facts.unique, declared_type: facts.declared_type, - catalog_type: facts.catalog_type, + catalog: facts.catalog, exact_number: facts.exact_number, }); } @@ -463,7 +463,7 @@ pub fn facts_of(relation: &Relation, column: &DescribedColumn) -> SourceFacts { nullable: column.nullable, unique: column.unique, declared_type: column.declared_type.clone(), - catalog_type: column.catalog_type.clone(), + catalog: column.catalog.clone(), exact_number: column.exact_number, } } diff --git a/rust/crates/sdm-schema/src/infer.rs b/rust/crates/sdm-schema/src/infer.rs index 05e38320d..c9fabcab2 100644 --- a/rust/crates/sdm-schema/src/infer.rs +++ b/rust/crates/sdm-schema/src/infer.rs @@ -652,8 +652,7 @@ impl Keyed { nullable: !declared.not_null && (!is_primary || declared.key_nullable), unique: self.unique.contains(&declared.name), declared_type: None, - catalog_type: (!declared.declared_type.is_empty()) - .then(|| declared.declared_type.clone()), + catalog: Some(declared.into()), exact_number: exact_number(field, constraints), }) } diff --git a/rust/crates/sdm-schema/tests/assemble.rs b/rust/crates/sdm-schema/tests/assemble.rs index 5e575d7c8..315dfd3ca 100644 --- a/rust/crates/sdm-schema/tests/assemble.rs +++ b/rust/crates/sdm-schema/tests/assemble.rs @@ -555,7 +555,7 @@ fn the_columns_joined_on_are_marked_again_from_what_the_host_holds() { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, }; assemble::mark_keys(&mut stamped); diff --git a/rust/crates/sdm-schema/tests/edit.rs b/rust/crates/sdm-schema/tests/edit.rs index 2688e6208..a39b36e1c 100644 --- a/rust/crates/sdm-schema/tests/edit.rs +++ b/rust/crates/sdm-schema/tests/edit.rs @@ -53,7 +53,7 @@ fn column(name: &str, dtype: Dtype, stype: Stype) -> DescribedColumn { nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } diff --git a/rust/crates/sdm-schema/tests/exact_numbers.rs b/rust/crates/sdm-schema/tests/exact_numbers.rs index d246afba4..ac86aa3f2 100644 --- a/rust/crates/sdm-schema/tests/exact_numbers.rs +++ b/rust/crates/sdm-schema/tests/exact_numbers.rs @@ -37,7 +37,7 @@ fn plain(name: &str, dtype: Dtype, received: ColumnType, stype: Stype) -> Descri nullable: true, unique: false, declared_type: None, - catalog_type: None, + catalog: None, exact_number: None, } } diff --git a/rust/crates/sdm-schema/tests/held.rs b/rust/crates/sdm-schema/tests/held.rs index c261d7214..df65b84f7 100644 --- a/rust/crates/sdm-schema/tests/held.rs +++ b/rust/crates/sdm-schema/tests/held.rs @@ -22,7 +22,7 @@ fn facts(dtype: Dtype, received: ColumnType) -> SourceFacts { nullable: false, unique: true, declared_type: Some("HUGEINT".into()), - catalog_type: None, + catalog: None, exact_number: None, } } diff --git a/rust/crates/sdm-testkit/src/edits.rs b/rust/crates/sdm-testkit/src/edits.rs index 93a859696..b602fdc9c 100644 --- a/rust/crates/sdm-testkit/src/edits.rs +++ b/rust/crates/sdm-testkit/src/edits.rs @@ -179,7 +179,7 @@ pub fn held(described: &SourceDescription) -> HeldSchema { nullable: column.nullable, unique: column.unique, declared_type: column.declared_type.clone(), - catalog_type: column.catalog_type.clone(), + catalog: column.catalog.clone(), exact_number: column.exact_number, }), }) @@ -212,7 +212,7 @@ pub fn described(held: &HeldSchema) -> Option { nullable: facts.nullable, unique: facts.unique, declared_type: facts.declared_type, - catalog_type: facts.catalog_type, + catalog: facts.catalog, exact_number: facts.exact_number, }); } diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index c9121fa92..fde990779 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -236,8 +236,18 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { "origin": "inferred" }] }); - // The snapshot also predates each column's catalog type, which a later - // materialize compares with the catalog then. + // The snapshot also predates each column's catalog declaration, which a + // later materialize compares with the catalog then; none of this + // fixture's columns is declared NOT NULL, a key or unique. + let declared = |declared_type: &str| { + serde_json::json!({ + "declared_type": declared_type, + "not_null": false, + "primary_key": false, + "key_nullable": false, + "unique": false, + }) + }; let catalog_types = [ ("orders", "order_id", "BIGINT"), ("orders", "user_id", "BIGINT"), @@ -247,7 +257,8 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { ("users", "age", "INTEGER"), ("users", "country", "VARCHAR"), ]; - for (table, column, declared) in catalog_types { + for (table, column, declared_type) in catalog_types { + let declared = declared(declared_type); let columns = expected_description["tables"] .as_array_mut() .unwrap() @@ -257,7 +268,7 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { .as_array_mut() .unwrap(); let held = columns.iter_mut().find(|c| c["name"] == column).unwrap(); - held["catalog_type"] = serde_json::json!(declared); + held["catalog"] = serde_json::json!(declared); } assert_eq!(described, expected_description); let mut expected_materialized = pinned["materialized"].clone(); @@ -308,9 +319,9 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { .retain(|fetch| fetch.get("purpose").is_none() || fetch["request"]["part"] == "columns"); assert_eq!(Value::from(observed.fetches), expected_fetches); let mut expected_inputs = pinned["inputs"].clone(); - // The schema each later run is handed keeps those catalog types with + // The schema each later run is handed keeps those declarations with // its columns' facts. - fn with_catalog_types(value: &mut Value, types: &[(&str, &str, &str)]) { + fn with_catalogs(value: &mut Value, types: &[(&str, &str, &str)]) { match value { Value::Object(object) => { if let (Some(Value::String(table)), Some(Value::Array(columns))) = @@ -322,21 +333,28 @@ fn base_regression_s_schema_completes_on_the_pure_rust_host() { && let Some((_, _, declared)) = types.iter().find(|(t, c, _)| *t == table && *c == name) { - facts.insert("catalog_type".into(), serde_json::json!(declared)); + facts.insert( + "catalog".into(), + serde_json::json!({ + "declared_type": declared, + "not_null": false, + "primary_key": false, + "key_nullable": false, + "unique": false, + }), + ); } } } for child in object.values_mut() { - with_catalog_types(child, types); + with_catalogs(child, types); } } - Value::Array(items) => items - .iter_mut() - .for_each(|item| with_catalog_types(item, types)), + Value::Array(items) => items.iter_mut().for_each(|item| with_catalogs(item, types)), _ => {} } } - with_catalog_types(&mut expected_inputs, &catalog_types); + with_catalogs(&mut expected_inputs, &catalog_types); expected_inputs[2]["state"] = expected_materialized; assert_eq!(Value::from(observed.inputs.clone()), expected_inputs); // predict's call and result: the call's request id compared by diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index b98c76e32..d0a8bdb22 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -8261,6 +8261,12 @@ "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "type_contracts/arrow-admission-and-inference", "reason": "Arrow moves from 58.4.0 to 58.4.1 for RUSTSEC-2026-0442, so the vectors name the locked arrow-schema version; every type vector and expected outcome is unchanged. Update the whole-file digest." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "live_api_tests", + "reason": "No released Kumo Relational NIM serves temporal link prediction: 1.2.0 refuses it with UNSUPPORTED_TASK. The two live link-batching tests could never pass, so they are removed; the offline batching tests keep the 200-row call cap and entity order covered.", + "retained_test": "packages/structured-data-models-client/tests/test_temporal_link_batching.py::test_pql_link_batches_preserve_every_entity" } ], "capabilities": [ From f541b4cc205d8ed795136bad2ec0f7a2e4b29438 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 16:35:48 -0800 Subject: [PATCH 20/24] Read a table in full where a fact rests on its keys The catalog's columns do not list a table's keys, so a key added or dropped since describe went unseen: a column of exact numbers is read as text or as a number by whether a key declares it, and a column kept free of nulls by a key rather than NOT NULL could keep that fact after the key was dropped. Such columns now take the whole read. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/crates/sdm-core/src/describe.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rust/crates/sdm-core/src/describe.rs b/rust/crates/sdm-core/src/describe.rs index 924ed0647..28c0de2ed 100644 --- a/rust/crates/sdm-core/src/describe.rs +++ b/rust/crates/sdm-core/src/describe.rs @@ -985,8 +985,11 @@ async fn read_as_it_stands( /// otherwise or holds no declaration, where the table projects its source, /// and where the source's columns hold affinities rather than types /// (SQLite's), whose values decide a column's type: the whole read then -/// says what changed. A catalog that cannot be read is refused as the -/// whole read refuses it. +/// says what changed. So too where a fact rests on the table's keys, which +/// its catalog's columns do not list: a column of exact numbers, read as +/// text or as a number by whether a key declares it, and one kept free of +/// nulls by a key rather than by `NOT NULL`. A catalog that cannot be read +/// is refused as the whole read refuses it. async fn unchanged( asker: &Asker, relation: &Relation, @@ -1001,8 +1004,8 @@ async fn unchanged( let mut kept = Vec::new(); for table in tables { for column in &table.columns { - let Some(facts) = - sdm_schema::held::facts_held(table, column).filter(|facts| facts.catalog.is_some()) + let Some(facts) = sdm_schema::held::facts_held(table, column) + .filter(|facts| facts.catalog.is_some() && facts.exact_number.is_none()) else { return Ok(None); }; @@ -1012,7 +1015,9 @@ async fn unchanged( let declared = crate::discover::catalog(asker, relation).await?; let same = kept.iter().all(|((_, name), facts)| { declared.iter().any(|c| { - c.name == *name && facts.catalog.as_ref() == Some(&CatalogDeclaration::from(c)) + c.name == *name + && facts.catalog.as_ref() == Some(&CatalogDeclaration::from(c)) + && (facts.nullable || c.not_null) }) }); Ok(same.then_some(kept)) From f7a445fb8d2d316cbe1d389a5cddbe22009c2da9 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 22:20:03 -0800 Subject: [PATCH 21/24] Address review: cleanup errors, payload reads, local option checks, docs - A failed session cleanup no longer replaces the transport failure of the call that failed; the cleanup stays a SESSION_CLEANUP warning. - Reading Request.payloads writes only what changed into a held view, so another thread never sees it emptied. - The in-process client refuses an inference option before it loads a model, as it refuses an output. - Docs: merge keeps repeats within a part, Latest is for final releases only, the Java Arrow view has no score column, and nullable integer foreign keys are sent exact. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 3 ++ RELEASING.md | 3 +- java/README.md | 2 +- .../src/sdm_client/_rust/run.py | 19 ++++++---- .../src/sdm_client/in_process/backend.py | 6 +++- .../src/sdm_client/request.py | 14 +++++--- .../src/sdm_client/result.py | 5 +-- .../test_a_model_runs_in_this_process.py | 30 ++++++++++++++++ .../tests/test_automatic_batch_sessions.py | 30 +++++++++++++++- .../tests/test_column_analysis_core.py | 35 +++++++++++++++++++ 10 files changed, 131 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a16094791..bc2b797d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,9 @@ The public API snapshot records this version update; publication remains pending table with no times, as 1.0.0 read it. - An HTTP 408 from `POST /v1/predictions` is retried, as 429, 502, 503 and 504 are. +- A foreign key holding integers and nulls is sent as exact integers beside + nulls. 1.0.0 sent such a column as `float64`, so its keys arrived as floats + and any past 2**53 was rounded. - A relational request reached over three or more hops fills a foreign key from every path that reaches the row, where 1.0.0 took the first path and left the rest null. The model sees more links, so predictions can differ. diff --git a/RELEASING.md b/RELEASING.md index a18801ff4..12fe2e97a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -214,7 +214,8 @@ Check the three projects on pypi.org and confirm each shows NVIDIA organization ownership. Portal reporting that a release completed is not the same as the package being installable, so install it the way a user would. -Then mark the GitHub Release Latest. The workflow leaves it unmarked on +For a final release, then mark the GitHub Release Latest; leave an alpha, beta +or release candidate unmarked. The workflow leaves every release unmarked on purpose, because until the publish has worked it would advertise a version the index does not have. One exception is worth knowing: when there is no earlier release to hold the position, GitHub reports the newest non-draft release as diff --git a/java/README.md b/java/README.md index 82c233bd7..7e4db784d 100644 --- a/java/README.md +++ b/java/README.md @@ -28,7 +28,7 @@ The semantic view is decoded only when requested and is not cached alongside a s These describe correlations in the context, not causal effects. `ColumnExplanations.rows()` provides immutable typed rows in engine order. -`toArrow(allocator)` exposes `table, hop, column, stype, cohort, class, share, mean, score`; close the returned root before the caller-owned allocator. +`toArrow(allocator)` exposes `table, hop, column, stype, cohort, class, share, mean`; close the returned root before the caller-owned allocator. Mixed, decimal and wide class labels are lossless JSON strings identified by schema metadata `sdm.class_encoding=json`; ordinary strings, integers and booleans use their native Arrow types. Rust computes the statistics once; these Java views do not regroup or recompute them. The explicit `ColumnExplanations(payload)` constructor also runs that Rust analysis on a model request's labelled context. diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/run.py b/packages/structured-data-models-client/src/sdm_client/_rust/run.py index b320208bf..bfb56dfc1 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/run.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/run.py @@ -313,7 +313,11 @@ def _steps( refused.status is not None and error.details.get('status') == refused.status ) - or (refused.status is None and error.code == 'TRANSPORT_ERROR') + or ( + refused.status is None + and error.code == 'TRANSPORT_ERROR' + and transport is None + ) ): # The call's own failure, as the caller sees it, carrying # what the core carried on its end of the run. @@ -336,15 +340,18 @@ def _steps( if step['step'] == 'call_model' and ( model is not None or model_endpoints is not None ): - try: - latest = _call(core, handle, step, model, model_endpoints) - except TransportFailed as failed: - transport, latest = failed.account, None - # Cleanup must not replace the primary prediction refusal. + # Cleanup must not replace the primary prediction failure, a + # refusal or the transport's. deleting = ( isinstance(step['endpoint'], dict) and 'delete_session' in step['endpoint'] ) + try: + latest = _call(core, handle, step, model, model_endpoints) + except TransportFailed as failed: + latest = None + if transport is None or not deleting: + transport = failed.account if refused is None or not deleting: refused = latest cleanup_refusal = deleting diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index 06ea49306..f5b1645f2 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -223,10 +223,14 @@ def delete_session(self, session_id: str) -> None: self._sessions.pop(session_id, None) def _preflight(self, payload: Mapping[str, Any]) -> None: - """Reject unsupported local-runtime outputs before downloading weights.""" + """Reject unsupported local-runtime outputs and inference options + before downloading weights.""" served = _SERVED.get(payload['model']) if served is not None: _fields(served, None, payload['task']['kind'], payload) + served.check_inference( + payload.get('inference') or {}, payload['task'] + ) def _serving(self, model_id: str) -> tuple[Tabular | Relational, Any]: r"""What reads requests for ``model_id``, and the model it runs. diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index 554428dc0..1707518d7 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -1019,12 +1019,18 @@ def __init__(self, envelope: dict[str, Any]) -> None: self._sync() def _sync(self) -> None: - super().clear() - super().update( - (name, value) + # Only what changed is written, so another thread reading a view + # it holds never sees it emptied by a read of `payloads`. + shown = { + name: value for name, value in self._envelope.items() if not name.startswith('client_') - ) + } + for name in [name for name in self if name not in shown]: + super().__delitem__(name) + for name, value in shown.items(): + if name not in self or dict.__getitem__(self, name) is not value: + super().__setitem__(name, value) def __setitem__(self, name: str, value: Any) -> None: self._envelope[name] = value diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index 615e68cc7..fdb609ebb 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -279,8 +279,9 @@ def merge( last answer would silently drop every earlier batch, so the parts are concatenated here. - The native route keeps a stable union of unscored candidates, removing - duplicates within and across parts. NaNs remain distinct under IEEE + The native route keeps a stable union of unscored candidates: a value + an earlier part already reported is dropped, and a repeat within one + part is kept. NaNs remain distinct under IEEE equality. Aware datetimes compare by UTC instant, distinguishing DST folds; dates and naive datetimes remain separate. Result items, metadata values and explanation owners are retained. diff --git a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py index 4596b7540..43d77bf51 100644 --- a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py +++ b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py @@ -466,6 +466,36 @@ def test_an_explanation_is_answered_in_this_process(request, model): assert len(result.explanations.gradients) == len(result) > 0 +def test_an_inference_option_the_model_refuses_is_refused_before_it_loads( + sdm, monkeypatch +): + """An inference option this process does not act on is refused before + the model's weights are loaded, as an output it does not produce is. + + Assumptions: + - The client was opened with no model, so it loads one on first use. + - Constructing ``sdm.models.KumoTabular`` is replaced by a stub that + records the call; any call means the weights were loaded. + - Kumo Tabular takes ``use_prediction_time`` only as ``False``. + """ + loads = [] + monkeypatch.setattr( + sdm.models, 'KumoTabular', lambda **kwargs: loads.append(kwargs) + ) + frame = people()[['person_id', 'age', 'visits', 'churn']] + request = Request( + context=frame.iloc[:45], + predict=frame.iloc[45:].drop(columns=['churn']), + target='churn', + task='classification', + inference={'use_prediction_time': True}, + ) + with SDMClient.in_process() as client: + error = refused(lambda: client.predict(request, model=KumoTabular())) + assert error.code == 'INVALID_REQUEST' + assert loads == [] + + def test_an_output_the_model_does_not_produce_is_refused_before_it_loads( sdm, monkeypatch ): diff --git a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py index a14e10bcf..33c469abe 100644 --- a/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py +++ b/packages/structured-data-models-client/tests/test_automatic_batch_sessions.py @@ -71,8 +71,14 @@ def do_POST(self): if outer.mode == 'transport' and outer.scored == 2: self.close_connection = True return + if ( + outer.mode == 'lost_then_dropped' + and self.path == '/v1/predictions' + ): + self.close_connection = True + return if (outer.mode == 'expired' and outer.scored == 2) or ( - outer.mode == 'expired_twice' + outer.mode in ('expired_twice', 'lost_then_dropped') and outer.scored >= 2 and self.path.startswith('/v1/sessions/') ): @@ -106,6 +112,9 @@ def do_POST(self): def do_DELETE(self): outer.calls.append((self.command, self.path, None)) + if outer.mode == 'lost_then_dropped': + self.close_connection = True + return self.send_response( 400 if outer.mode in ('malformed', 'transport') else 204 ) @@ -219,6 +228,25 @@ def test_transport_failure_keeps_primary_error_when_cleanup_fails(): endpoint.close() +def test_a_failed_cleanup_does_not_hide_the_call_that_failed(): + """Sessions lost twice send the remaining calls stateless; when one of + those and the leftover sessions' cleanup both fail in transport, the + error names the call that failed, and the cleanup is a warning.""" + from sdm_client import SDMWarning + + endpoint = Endpoint('lost_then_dropped') + try: + with SDMClient(url=endpoint.url, max_retries=0) as client: + with pytest.warns(SDMWarning, match='SESSION_CLEANUP'): + with pytest.raises(StructuredDataError) as caught: + client.predict(request(), model=KumoRelational()) + assert caught.value.code == 'TRANSPORT_ERROR' + assert caught.value.details['url'].endswith('/v1/predictions') + assert endpoint.calls[-1][0] == 'DELETE' + finally: + endpoint.close() + + def test_distinct_materialized_contexts_remain_stateless(): from sdm_client import PredictiveQuery diff --git a/packages/structured-data-models-client/tests/test_column_analysis_core.py b/packages/structured-data-models-client/tests/test_column_analysis_core.py index d47d4de54..ed8b46a12 100644 --- a/packages/structured-data-models-client/tests/test_column_analysis_core.py +++ b/packages/structured-data-models-client/tests/test_column_analysis_core.py @@ -265,6 +265,41 @@ def test_public_payloads_hide_client_keys_and_write_through(): assert envelope == {'client_generated_keys': {'orders': '__node_id'}} +def test_reading_payloads_never_empties_a_view_another_thread_holds(): + import sys + import threading + + envelope = { + 'model': 'kumo-relational', + 'metadata': {}, + 'task': {'kind': 'regression'}, + 'client_generated_keys': {}, + } + request = Request.__new__(Request) + object.__setattr__(request, '_payloads', (envelope,)) + [held] = request.payloads + partial = [] + stop = threading.Event() + + def watch(): + while not stop.is_set(): + if len(held) != 3: + partial.append(len(held)) + + previous = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + watcher = threading.Thread(target=watch) + try: + watcher.start() + for _ in range(5000): + assert request.payloads is not None + finally: + stop.set() + watcher.join() + sys.setswitchinterval(previous) + assert partial == [] + + def test_unavailable_statistics_warn_without_losing_predictions(): from test_tabular_direct_core import Endpoint as TabularEndpoint from test_tabular_direct_core import client From 27d1cfca5999c6edccf20674f2a624e510c448d8 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 22:24:52 -0800 Subject: [PATCH 22/24] Widen forecast analysis to every column a step sent An explained forecast failed with INVALID_MESSAGE when its steps sent a related table under different columns, such as a key to a table only one step reached. Forecast steps now gather related rows as batched calls do, a missing cell null, and held rows are widened only when a call adds one. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/crates/sdm-request/src/columns.rs | 296 +++++++++++++++---------- 1 file changed, 175 insertions(+), 121 deletions(-) diff --git a/rust/crates/sdm-request/src/columns.rs b/rust/crates/sdm-request/src/columns.rs index 308641b7a..ced10c57f 100644 --- a/rust/crates/sdm-request/src/columns.rs +++ b/rust/crates/sdm-request/src/columns.rs @@ -977,7 +977,7 @@ pub fn sent(payloads: &[Value]) -> Result { let mut seen = BTreeSet::new(); let mut next_id = 0i64; let mut instances = Vec::new(); - let mut related: Vec<(String, Value)> = Vec::new(); + let mut related = Vec::new(); let mut schema = field(first, "schema")?.clone(); for payload in payloads { let context = field(payload, "context")?; @@ -1011,49 +1011,15 @@ pub fn sent(payloads: &[Value]) -> Result { for (name, rows) in members(tables)? { let read = Table::read(rows)?; let at = read.at("instance_id")?; - let held = match related.iter_mut().find(|(n, _)| n == name) { - Some((_, held)) => held, - None => { - let mut empty = rows.clone(); - empty.set("rows", Value::Array(vec![])); - related.push((name.clone(), empty)); - &mut related.last_mut().expect("just pushed").1 - } - }; - // A call may send a column another did not, such as a key - // to a table only it reached: rows take every column, a - // cell their call did not send left null. - let mut names = array(field(held, "columns")?)?.to_vec(); - for column in array(field(rows, "columns")?)? { - if !names.contains(column) { - names.push(column.clone()); - } - } - let widen = |row: &[Value], from: &[Value]| { - names - .iter() - .map(|name| { - from.iter() - .position(|f| f == name) - .map_or(Value::Null, |i| row[i].clone()) - }) - .collect::>() - }; - let before = array(field(held, "columns")?)?.to_vec(); - let mut all = array(field(held, "rows")?)? - .iter() - .map(|row| Ok(Value::Array(widen(array(row)?, &before)))) - .collect::, CoreError>>()?; - let from = array(field(rows, "columns")?)?; + let mut renamed = Vec::new(); for row in &read.rows { if let Some(&new) = kept.get(&key(&row[at])?) { let mut row = row.to_vec(); row[at] = Value::int(new); - all.push(Value::Array(widen(&row, from))); + renamed.push(row); } } - held.set("columns", Value::Array(names.clone())); - held.set("rows", Value::Array(all)); + gather(&mut related, name, rows, renamed)?; } } let step = field(payload, "schema")?; @@ -1102,7 +1068,7 @@ pub fn sent(payloads: &[Value]) -> Result { p.get("context") .is_some_and(|c| c.get("related_tables").is_some()) }) { - context.set("related_tables", Value::Object(related)); + context.set("related_tables", gathered(related)); } let mut out = first.clone(); out.set("schema", schema); @@ -1110,13 +1076,96 @@ pub fn sent(payloads: &[Value]) -> Result { Ok(out) } -/// Union forecast evidence before cohort selection. Steps namespace instance identity. +/// A related table's rows as several calls or forecast steps send them. +/// One may send a column another did not, such as a key to a table only it +/// reached: rows take every column, a cell their call did not send left +/// null. +struct Held { + name: String, + table: Value, + columns: Vec, + rows: Vec>, +} + +/// Adds the rows a call sent of `table`, under its own columns. Held rows +/// are widened only when the call adds a column; otherwise its rows are +/// appended. +fn gather( + held: &mut Vec, + name: &str, + table: &Value, + rows: Vec>, +) -> Result<(), CoreError> { + let from = array(field(table, "columns")?)?; + let at = match held.iter().position(|h| h.name == name) { + Some(at) => at, + None => { + let mut empty = table.clone(); + empty.set("rows", Value::Array(vec![])); + held.push(Held { + name: name.into(), + table: empty, + columns: from.to_vec(), + rows: vec![], + }); + held.len() - 1 + } + }; + let held = &mut held[at]; + let width = held.columns.len(); + for column in from { + if !held.columns.contains(column) { + held.columns.push(column.clone()); + } + } + if held.columns.len() > width { + for row in &mut held.rows { + row.resize(held.columns.len(), Value::Null); + } + } + let cells = held + .columns + .iter() + .map(|c| from.iter().position(|f| f == c)) + .collect::>(); + held.rows.extend(rows.into_iter().map(|mut row| { + cells + .iter() + .map(|i| i.map_or(Value::Null, |i| std::mem::replace(&mut row[i], Value::Null))) + .collect() + })); + Ok(()) +} + +/// The related tables held, in the order first sent, each under every +/// column sent. +fn gathered(held: Vec) -> Value { + Value::Object( + held.into_iter() + .map(|mut h| { + h.table.set("columns", Value::Array(h.columns)); + h.table.set( + "rows", + Value::Array(h.rows.into_iter().map(Value::Array).collect()), + ); + (h.name, h.table) + }) + .collect(), + ) +} + +/// Union forecast evidence before cohort selection. Steps namespace instance +/// identity, and a related table takes every column any step sent, as the +/// calls of one request do in [`sent`]: a step may reach a table another +/// did not, and so send a key to it. pub fn forecast(payloads: &[Value]) -> Result { let Some(first) = payloads.first() else { return Err(invalid("empty analysis request")); }; let mut out = first.clone(); - let mut contexts = Vec::new(); + let mut context = field(first, "context")?.clone(); + let mut instances = Vec::new(); + let mut related = Vec::new(); let mut next_id = 0i64; let mut schema = field(first, "schema")?.clone(); for payload in payloads { @@ -1125,10 +1174,16 @@ pub fn forecast(payloads: &[Value]) -> Result { { return Err(invalid("forecast tasks differ")); } - let mut context = field(payload, "context")?.clone(); - let instance = Table::read(field(&context, "instance_table")?)?; + let step = field(payload, "context")?; + if field(field(step, "instance_table")?, "columns")? + != field(field(&context, "instance_table")?, "columns")? + { + return Err(invalid("forecast columns differ")); + } + let instance = Table::read(field(step, "instance_table")?)?; + let mut ids = None; if let Ok(id) = instance.at("instance_id") { - let mut ids = BTreeMap::new(); + let ids = ids.insert(BTreeMap::new()); for row in &instance.rows { let k = key(&row[id])?; if ids.insert(k, next_id).is_some() { @@ -1138,37 +1193,29 @@ pub fn forecast(payloads: &[Value]) -> Result { .checked_add(1) .ok_or_else(|| invalid("forecast too large"))?; } - fn rename(table: &mut Value, ids: &BTreeMap) -> Result<(), CoreError> { - let read = Table::read(table)?; - let id = read.at("instance_id")?; - let rows = read - .rows - .iter() - .map(|r| { - let mut row = r.to_vec(); - row[id] = Value::int( - *ids.get(&key(&r[id])?) - .ok_or_else(|| invalid("unknown forecast instance"))?, - ); - Ok(Value::Array(row)) - }) - .collect::, CoreError>>()?; - table.set("rows", Value::Array(rows)); - Ok(()) - } - let mut table = field(&context, "instance_table")?.clone(); - rename(&mut table, &ids)?; - context.set("instance_table", table); - if let Some(related) = context.get("related_tables") { - let tables = members(related)? - .iter() - .map(|(name, t)| { - let mut t = t.clone(); - rename(&mut t, &ids)?; - Ok((name.clone(), t)) - }) - .collect::>()?; - context.set("related_tables", Value::Object(tables)); + } + let rename = |table: &Table<'_>| -> Result>, CoreError> { + let Some(ids) = &ids else { + return Ok(table.rows.iter().map(|r| r.to_vec()).collect()); + }; + let id = table.at("instance_id")?; + table + .rows + .iter() + .map(|r| { + let mut row = r.to_vec(); + row[id] = Value::int( + *ids.get(&key(&r[id])?) + .ok_or_else(|| invalid("unknown forecast instance"))?, + ); + Ok(row) + }) + .collect() + }; + instances.extend(rename(&instance)?); + if let Some(tables) = step.get("related_tables") { + for (name, table) in members(tables)? { + gather(&mut related, name, table, rename(&Table::read(table)?)?)?; } } let step_schema = field(payload, "schema")?; @@ -1178,20 +1225,39 @@ pub fn forecast(payloads: &[Value]) -> Result { { return Err(invalid("forecast instance schema or anchor differs")); } - for name in ["related_tables"] { - if let Some(value) = step_schema.get(name) { - let mut combined = schema.get(name).cloned().unwrap_or(Value::Object(vec![])); - for (table, desc) in members(value)? { - if let Some(existing) = combined.get(table) { - if existing != desc { + if let Some(value) = step_schema.get("related_tables") { + let mut combined = schema + .get("related_tables") + .cloned() + .unwrap_or(Value::Object(vec![])); + fn rest(described: &Value) -> Result, CoreError> { + Ok(members(described)? + .iter() + .filter(|(name, _)| name != "columns") + .collect()) + } + for (table, desc) in members(value)? { + let Some(mut held) = combined.get(table).cloned() else { + combined.set(table, desc.clone()); + continue; + }; + if rest(&held)? != rest(desc)? { + return Err(invalid("forecast table schemas differ")); + } + let mut columns = field(&held, "columns")?.clone(); + for (column, spec) in members(field(desc, "columns")?)? { + match columns.get(column) { + None => columns.set(column, spec.clone()), + Some(other) if other != spec => { return Err(invalid("forecast table schemas differ")); } - } else { - combined.set(table, desc.clone()); + Some(_) => {} } } - schema.set(name, combined); + held.set("columns", columns); + combined.set(table, held); } + schema.set("related_tables", combined); } if let Some(value) = step_schema.get("relationships") { let mut links = schema @@ -1207,40 +1273,20 @@ pub fn forecast(payloads: &[Value]) -> Result { } schema.set("relationships", Value::Array(links)); } - contexts.push(context); - } - fn append(target: &mut Value, source: &Value) -> Result<(), CoreError> { - if field(target, "columns")? != field(source, "columns")? { - return Err(invalid("forecast columns differ")); - } - let mut rows = array(field(target, "rows")?)?.to_vec(); - rows.extend_from_slice(array(field(source, "rows")?)?); - target.set("rows", Value::Array(rows)); - Ok(()) } - let mut combined = contexts.remove(0); - for context in contexts { - let mut table = field(&combined, "instance_table")?.clone(); - append(&mut table, field(&context, "instance_table")?)?; - combined.set("instance_table", table); - if let Some(related) = context.get("related_tables") { - let mut tables = combined - .get("related_tables") - .cloned() - .unwrap_or(Value::Object(vec![])); - for (name, table) in members(related)? { - let mut target = tables.get(name).cloned().unwrap_or_else(|| { - let mut t = table.clone(); - t.set("rows", Value::Array(vec![])); - t - }); - append(&mut target, table)?; - tables.set(name, target); - } - combined.set("related_tables", tables); - } + let mut instance_table = field(&context, "instance_table")?.clone(); + instance_table.set( + "rows", + Value::Array(instances.into_iter().map(Value::Array).collect()), + ); + context.set("instance_table", instance_table); + if payloads.iter().any(|p| { + p.get("context") + .is_some_and(|c| c.get("related_tables").is_some()) + }) { + context.set("related_tables", gathered(related)); } - out.set("context", combined); + out.set("context", context); out.set("schema", schema); Ok(out) } @@ -1370,7 +1416,7 @@ mod lifecycle_tests { } #[test] - fn calls_that_reached_different_tables_are_analysed_together() { + fn calls_or_forecast_steps_that_reached_different_tables_are_analysed_together() { let users = r#""users":{"primary_key":["instance_id","uid"],"columns":{"instance_id":{"stype":"ID"},"uid":{"stype":"ID"},"age":{"stype":"numerical"}}}"#; let orders = |key: &str| { format!( @@ -1410,10 +1456,18 @@ mod lifecycle_tests { }; let first = call("[0,1,0],[1,2,1]", "[0,1,30],[1,2,40]", false); let second = call("[0,1,0],[1,2,1]", "[0,1,30],[1,2,40]", true); - assert!(matches!( - analyze(&sent(&[first, second]).unwrap()).unwrap(), - ColumnAnalysis::Ready { .. } - )); + for combine in [sent, forecast] { + let combined = combine(&[first.clone(), second.clone()]).unwrap(); + let context = field(&combined, "context").unwrap(); + let orders = field(field(context, "related_tables").unwrap(), "orders").unwrap(); + let orders = Table::read(orders).unwrap(); + assert_eq!(orders.names, ["instance_id", "oid", "uid", "pid"]); + assert_eq!(orders.rows[0][3], Value::Null); + assert!(matches!( + analyze(&combined).unwrap(), + ColumnAnalysis::Ready { .. } + )); + } } #[test] From a8e4bd520a7a35f11da333f5b60c70f9996cdcb0 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 23:09:14 -0800 Subject: [PATCH 23/24] Refuse candidate-free temporal link prediction up front No released Kumo Relational model serves temporal_link_prediction: the 1.2.0 NIM answers 422 UNSUPPORTED_TASK and in-process mode does not serve it. materialize() read the source and built the request anyway. materialize() and materialize_examples() now refuse a LIST_DISTINCT target asked without RANK TOP k, before any source is read, with UNSUPPORTED_REQUEST, rule temporal_link_prediction, naming RANK TOP k with candidates= as the remedy. Prediction admission gives a saved or hand-built request of that kind the same refusal before anything is sent. The task leaves kumo-relational's capability tables in Rust and Python. Rankings with candidates are unchanged, and the wire types, payload builders and decoders stay so the task can be re-enabled. The new sentence is added to the v72 schema, which is new in this PR. Tests that built or predicted link requests now assert the refusal and that no source was read; the contract case, compatibility manifest, docs, skills and changelog follow. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 10 ++ .../s1/target.aggregation.list-distinct.json | 17 +-- contract/expected/README.md | 2 +- contract/expected/duckdb-python.json | 2 +- contract/expected/postgres-python.json | 2 +- contract/expected/sqlite-python.json | 2 +- docs/reference/limits.md | 5 +- docs/reference/prediction-output.md | 2 +- .../com/nvidia/sdm/CustomExamplesTest.java | 6 +- .../sdm/ModelConnectorCoverageTest.java | 24 +-- .../java/com/nvidia/sdm/RelationalTest.java | 7 +- .../nvidia/sdm/TemporalLinkBatchingTest.java | 39 ++--- .../src/sdm_client/models.py | 16 +- .../src/sdm_client/query.py | 14 +- .../src/sdm_client/relational_task.py | 4 + .../src/sdm_client/schema.py | 3 + .../tests/conftest.py | 32 ++++ .../tests/test_custom_examples.py | 109 ++++++------- .../tests/test_frame_materialize_core.py | 143 +++--------------- .../tests/test_limits_and_sessions.py | 10 +- .../tests/test_query_engine_boundary.py | 35 +---- .../test_refusals_that_lead_somewhere.py | 26 ++-- .../tests/test_temporal_link_batching.py | 114 +++++--------- .../tests/test_warehouse_links_core.py | 106 ++----------- rust/crates/sdm-contract/schema/contract.lock | 2 +- .../sdm-contract/schema/sdm-contract-72.json | 1 + rust/crates/sdm-contract/src/error.rs | 1 + rust/crates/sdm-contract/src/wording.rs | 3 + rust/crates/sdm-core/src/materialize.rs | 28 +++- .../sdm-core/src/materialize/examples.rs | 5 + rust/crates/sdm-core/tests/materialize.rs | 54 ++++++- rust/crates/sdm-request/src/predict.rs | 50 +++++- skills/README.md | 6 +- skills/relational/SKILL.md | 6 +- skills/relational/pql.md | 12 +- skills/tabular/SKILL.md | 2 +- tests/compatibility/manifest.json | 40 ++++- tests/test_skills.py | 3 - 38 files changed, 429 insertions(+), 514 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc2b797d7..45392a4b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,16 @@ The public API snapshot records this version update; publication remains pending - Gradient explanations have no separate one-row or 1,000-context-row client restriction. Endpoint capabilities and ordinary request limits still apply. The singular raw `result.explanation` view remains supported. +- A temporal link prediction asked without candidates, such as + `PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id`, + raises `UnsupportedRequestError` with code `UNSUPPORTED_REQUEST` and + `details['rule']` `temporal_link_prediction`. `materialize()` and + `materialize_examples()` refuse it before any source is read, and + `predict()` refuses a request that names the task before anything is sent. + No released Kumo Relational model serves it, so `kumo-relational` no longer + lists it among its tasks. Rank an explicit candidate set with `RANK TOP k` + and `candidates=` instead. 1.0.0 built the request, and the endpoint + refused it. ### Fixed diff --git a/contract/cases/s1/target.aggregation.list-distinct.json b/contract/cases/s1/target.aggregation.list-distinct.json index 0d95e8b6e..13c1d4b13 100644 --- a/contract/cases/s1/target.aggregation.list-distinct.json +++ b/contract/cases/s1/target.aggregation.list-distinct.json @@ -5,8 +5,7 @@ "slice": "S1", "paths": [ "schema.from-source/all-tables/duckdb", - "query.materialize/indices-link-prediction/duckdb", - "client.predict/link-prediction/duckdb" + "query.materialize/indices-link-prediction/duckdb" ], "routes": [ "schema.from-source/all-tables/duckdb", @@ -44,7 +43,7 @@ "anchor_time": "2023-03-01" } }, - "until": "predict", + "until": "materialize", "core": { "describe_source": { "provider": { @@ -80,18 +79,14 @@ "zone": null }, "seed": null - }, - "predict": { - "model": "kumo-relational" } }, "expect": { "outcome": "refusal", - "stage": "predict", - "error": "NimRequestError", - "code": null, - "message": "[400] {\"error\": \"no_classes\"}" + "stage": "materialize", + "error": "UnsupportedRequestError", + "code": "UNSUPPORTED_REQUEST" }, - "why": "An aggregation other than SUM, COUNT, AVG, MIN and MAX: LIST_DISTINCT of buys.item_id, a foreign key, for each user, a temporal link prediction whose context rows are each labelled with the distinct items bought in its window (the core's rule aggregation)", + "why": "LIST_DISTINCT of buys.item_id, a foreign key, for each user, with no RANK TOP k: a candidate-free temporal link prediction, which no released Kumo Relational model serves, refused by materialize before the source is read (rule temporal_link_prediction)", "source": "Correctness case target.aggregation.list-distinct using the shop fixture." } diff --git a/contract/expected/README.md b/contract/expected/README.md index c3422e18f..46dff694c 100644 --- a/contract/expected/README.md +++ b/contract/expected/README.md @@ -15,7 +15,7 @@ Do not update expectations just to make a run pass; establish whether the public | `s0.boundary.date-packed`, `date-long-field` | Reject at materialize with `INVALID_REQUEST` | Rust `sdm-testkit/tests/s0_cases.rs` explicitly rejects packed date literals | | `s0.boundary.depth-999` | Accept at materialize | Native parser accepts depth 999; the deleted Python shadow parser caused the old recursion failure | | `s0.boundary.escape-lone-surrogate`, `escape-surrogate-pair` | Accept at materialize | Native parser replaces lone surrogates and joins pairs; existing PQL-1 decision and Rust S0 checks | -| `target.aggregation.list-distinct` | Reach predict, then receive `NimRequestError` | The client supports temporal link materialization; this case's scripted endpoint deliberately has no link prediction response | +| `target.aggregation.list-distinct` | Reject at materialize with `UNSUPPORTED_REQUEST` before the source is read | No released Kumo Relational model serves candidate-free temporal link prediction, so the client refuses it up front, naming `RANK TOP k` with `candidates=` | All 315 DuckDB cases were exercised through the clean installed native client during effort 2. This is local scripted-endpoint coverage, not new live-model or performance acceptance. diff --git a/contract/expected/duckdb-python.json b/contract/expected/duckdb-python.json index 2a5376ff1..6f17223a2 100644 --- a/contract/expected/duckdb-python.json +++ b/contract/expected/duckdb-python.json @@ -279,7 +279,7 @@ "shape.star.stores": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyed": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyless": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "target.aggregation.list-distinct": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "target.aggregation.list-distinct": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, "target.condition.in": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.condition.text": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.filter.and": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/postgres-python.json b/contract/expected/postgres-python.json index de5fe0707..31f136d65 100644 --- a/contract/expected/postgres-python.json +++ b/contract/expected/postgres-python.json @@ -277,7 +277,7 @@ "shape.star.stores@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyed@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyless@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "target.aggregation.list-distinct@postgres": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "target.aggregation.list-distinct@postgres": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, "target.condition.in@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.condition.text@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.filter.and@postgres": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/contract/expected/sqlite-python.json b/contract/expected/sqlite-python.json index 209fbf7ab..a1efac620 100644 --- a/contract/expected/sqlite-python.json +++ b/contract/expected/sqlite-python.json @@ -261,7 +261,7 @@ "shape.star.stores@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyed@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "shape.two-paths.keyless@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, - "target.aggregation.list-distinct@sqlite": {"code": null, "error": "NimRequestError", "result": "refusal", "stage": "predict"}, + "target.aggregation.list-distinct@sqlite": {"code": "UNSUPPORTED_REQUEST", "error": "UnsupportedRequestError", "result": "refusal", "stage": "materialize"}, "target.condition.in@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.condition.text@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, "target.filter.and@sqlite": {"code": null, "error": null, "result": "success", "stage": "predict"}, diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 4424d8c2d..fc605e8c6 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -22,7 +22,7 @@ Both endpoints enforce sizes, and most of these are the relational one's. |---|---|---|---| | context rows | relational | 1,000 by default, up to 10,000 with `num_context_samples` | at `materialize()` | | context rows | tabular | 10,000 | sampled at `materialize()`, refused before sending on a direct `Request` | -| rows scored per call | relational | 10,000, or 200 for link prediction | more are gathered as several calls at `materialize()` | +| rows scored per call | relational | 10,000 | more are gathered as several calls at `materialize()` | | entities one question gathers | relational | 100,000, in at most 100 calls | at `materialize()`, and this client's own: the endpoint is never asked for more than one call's worth | | calls a question asked in parts makes | relational | 200, one per candidate or step | at `materialize()`, as `sdm_client.decompose.MOST_PARTS` | | request body | relational | 30 MiB per call | at `materialize()` | @@ -80,8 +80,7 @@ sent, each with its own context, so the whole population is held in memory at once, and 100,000 entities is about a gigabyte of it. One `materialize()` gathers at most that many, in at most 100 calls, and the refusal names whichever of the two it met. The second is what a small `rows_per_call` -reaches, and what a temporal link prediction reaches at 20,001 entities, since -it scores 200 a call. +reaches. Both govern the batches of one question, and neither counts the parts of a question asked in several. A ranking is one call per candidate and a forecast diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index 3c8fcef80..08a77929e 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -19,7 +19,7 @@ The shape depends on the **task**, not the model. | classification | one per entity | | multiclass classification | one per entity, with a probability per class **as columns** | | regression | one per entity | -| ranking / temporal link prediction | one per entity, the `k` candidates **in a list** | +| ranking | one per entity, the `k` candidates **in a list** | | forecasting | one per entity per step | Ranking is the exception to the row-per-answer rule: expanding the candidates diff --git a/java/src/test/java/com/nvidia/sdm/CustomExamplesTest.java b/java/src/test/java/com/nvidia/sdm/CustomExamplesTest.java index bdb18b380..e4e3b6801 100644 --- a/java/src/test/java/com/nvidia/sdm/CustomExamplesTest.java +++ b/java/src/test/java/com/nvidia/sdm/CustomExamplesTest.java @@ -108,12 +108,16 @@ static Object normalized(Object value) { for(Map test:cases(fixture)) { String name=test.get("id").toString(); Schema schema=memory(fixture,"timeless".equals(test.get("schema_variant"))); + if(name.equals("temporal_link_prediction")) { + StructuredDataException refused=assertThrows(StructuredDataException.class,()->prepare(schema,test,object(test.get("predict")))); + assertEquals("UNSUPPORTED_REQUEST",refused.code(),name); + continue; + } Request request=prepare(schema,test,object(test.get("predict"))); // Inputs and their allocator are already closed: the prepared request owns its data. Map payload=payload(request,0),expected=object(test.get("expected")); List labels=column(instance(payload,"context"),"TARGET"); List wanted=(List)expected.get("context_targets"); - if(name.equals("temporal_link_prediction")) wanted=wanted.stream().map(v->((List)v).stream().map(Object::toString).toList()).toList(); assertEquals(normalized(wanted),normalized(labels),name); assertEquals(rows(object(test.get("context"))).size(),rows(instance(payload,"context")).size(),name+" overlaps retained"); assertEquals(expected.get("use_prediction_time"),object(payload.get("inference")).get("use_prediction_time")); diff --git a/java/src/test/java/com/nvidia/sdm/ModelConnectorCoverageTest.java b/java/src/test/java/com/nvidia/sdm/ModelConnectorCoverageTest.java index 76cd2eb8e..ade966366 100644 --- a/java/src/test/java/com/nvidia/sdm/ModelConnectorCoverageTest.java +++ b/java/src/test/java/com/nvidia/sdm/ModelConnectorCoverageTest.java @@ -108,25 +108,13 @@ void failedLaterForecastKeepsCompletedRows(String provider) throws Exception { } @ParameterizedTest @ValueSource(strings = {"duckdb", "sqlite"}) - void jdbcTemporalLinksPredictAndReuseSessionsAfterConnectionCloses(String provider) throws Exception { - Request request; + void jdbcTemporalLinksAreRefusedBeforeMaterializing(String provider) throws Exception { try (Connection connection = WarehouseSessionTest.open(provider)) { - request = PredictiveQuery.parse(LINK).materialize(linkSchema(provider, connection), - WarehouseSessionTest.asked(1L, 2L).build()); - } - Map body = payload(request, 0); - assertEquals("temporal_link_prediction", map(body.get("task")).get("kind")); - assertEquals(List.of("users", "items"), map(body.get("task")).get("entity_table_names")); - try (Endpoint endpoint = new Endpoint(); SdmClient client = endpoint.client()) { - PredictionResult direct = client.predict(request, SdmClient.KUMO_RELATIONAL); - assertEquals(2, direct.rows()); assertTrue(direct.outputs().contains("rankings")); - try (FittedModel fitted = client.fit(request, SdmClient.KUMO_RELATIONAL)) { - assertEquals(2, fitted.predict(request).rows()); - assertEquals(2, fitted.predict(request).rows()); - assertEquals(1, endpoint.created.size()); assertEquals(2, endpoint.scored.size()); - assertTrue(endpoint.scored.stream().noneMatch(value -> value.containsKey("context"))); - } - assertEquals(1, endpoint.deletes.get()); + Schema schema = linkSchema(provider, connection); + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> PredictiveQuery.parse(LINK).materialize(schema, WarehouseSessionTest.asked(1L, 2L).build())); + assertEquals("UNSUPPORTED_REQUEST", refused.code()); + assertEquals("temporal_link_prediction", refused.details().get("rule")); } } diff --git a/java/src/test/java/com/nvidia/sdm/RelationalTest.java b/java/src/test/java/com/nvidia/sdm/RelationalTest.java index d8413b679..0bf4047b2 100644 --- a/java/src/test/java/com/nvidia/sdm/RelationalTest.java +++ b/java/src/test/java/com/nvidia/sdm/RelationalTest.java @@ -248,10 +248,9 @@ static Set instanceIds(Map table) { Request temporal = PredictiveQuery.parse(TEMPORAL).materialize(owner, options(1L, 2L).anchorTime(LocalDate.of(2024, 6, 1)).build()); assertTrue(temporal.targetMovesWithTime()); assertEquals(2, client.predict(temporal, SdmClient.KUMO_RELATIONAL).rows()); - Request link = PredictiveQuery.parse(LINK).materialize(owner, options(1L, 2L).build()); - assertEquals("temporal_link_prediction", map(payload(link, 0).get("task")).get("kind")); - PredictionResult linked = client.predict(link, SdmClient.KUMO_RELATIONAL); - assertEquals(2, linked.rows()); assertTrue(linked.outputs().contains("rankings")); + StructuredDataException link = assertThrows(StructuredDataException.class, + () -> PredictiveQuery.parse(LINK).materialize(owner, options(1L, 2L).build())); + assertEquals("UNSUPPORTED_REQUEST", link.code()); assertThrows(StructuredDataException.class, () -> PredictiveQuery.parse(RANK).materialize(owner, options(1L, 2L).build())); Request ranked = PredictiveQuery.parse(RANK).materialize(owner, diff --git a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java index 1559880bb..934f15b97 100644 --- a/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java +++ b/java/src/test/java/com/nvidia/sdm/TemporalLinkBatchingTest.java @@ -14,7 +14,7 @@ import org.apache.arrow.vector.table.Table; import org.junit.jupiter.api.Test; -/** Public Java/JNI PQL paths must share the model's temporal-link batch limit. */ +/** Public Java/JNI paths refuse candidate-free temporal link prediction before reading a source. */ class TemporalLinkBatchingTest { static final String QUERY = "PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id"; @@ -78,43 +78,32 @@ static List sizes(Request request) throws Exception { return result; } - @Test void allPublicInputRoutesUseOneLimitAndKeepEveryEntity() throws Exception { + @Test void everyPublicInputRouteRefusesCandidateFreeLinksUpFront() throws Exception { for (String kind : List.of("memory", "duckdb", "sqlite")) { try (Connection connection = kind.equals("memory") ? null : DriverManager.getConnection(kind.equals("duckdb") ? "jdbc:duckdb:" : "jdbc:sqlite::memory:")) { Schema schema = schema(kind, connection); - for (int count : List.of(199, 200, 201)) { - Request request = materialize(schema, QUERY, count, null); - assertEquals(count <= 200 ? List.of(count) : List.of(200, 1), sizes(request), kind); - List ids = new ArrayList<>(); - for (String text : request.payloads()) { - var payload = object(Json.read(text.getBytes(java.nio.charset.StandardCharsets.UTF_8))); - assertEquals("temporal_link_prediction", object(payload.get("task")).get("kind")); - var table = object(object(payload.get("predict")).get("instance_table")); - int column = ((List) table.get("columns")).indexOf("__kumo_entity_ref_0"); - for (var row : rows(table)) ids.add(Integer.parseInt(row.get(column).toString())); - } - assertEquals(java.util.stream.IntStream.rangeClosed(1, count).boxed().toList(), ids); - } - assertEquals(List.of(100, 100, 1), sizes(materialize(schema, QUERY, 201, 100))); - assertEquals(List.of(200, 1), sizes(materialize(schema, QUERY, 201, 1000))); + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> materialize(schema, QUERY, 201, null), kind); + assertEquals("UNSUPPORTED_REQUEST", refused.code(), kind); + assertEquals("temporal_link_prediction", refused.details().get("rule"), kind); + assertTrue(refused.getMessage().contains("RANK TOP k with candidates="), refused::getMessage); assertEquals(List.of(201), sizes(materialize(schema, "PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id", 201, null))); } } } - @Test void suppliedExamplesKeepTheSameCap() throws Exception { + @Test void suppliedLinkExamplesAreRefusedUpFront() throws Exception { var fixture = fixture(); var test = cases(fixture).stream().filter(c -> c.get("id").equals("temporal_link_prediction")).findFirst().orElseThrow(); - var predicted = new LinkedHashMap<>(object(test.get("predict"))); - predicted.put("rows", Collections.nCopies(201, rows(predicted).get(0))); + Schema schema = memory(fixture, false); try (RootAllocator allocator = new RootAllocator(); Table context = arrow(allocator, object(test.get("context")), "temporal_link_prediction"); - Table predict = arrow(allocator, predicted, "temporal_link_prediction")) { - Request request = memory(fixture, false).materializeExamples(task(test), context, predict, - ExamplesOptions.builder().rowsPerCall(1000).build()); - assertEquals(List.of(200, 1), sizes(request)); + Table predict = arrow(allocator, object(test.get("predict")), "temporal_link_prediction")) { + StructuredDataException refused = assertThrows(StructuredDataException.class, + () -> schema.materializeExamples(task(test), context, predict)); + assertEquals("UNSUPPORTED_REQUEST", refused.code()); + assertEquals("temporal_link_prediction", refused.details().get("rule")); } } - } diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index aa1209264..3cc4c9261 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -19,7 +19,7 @@ **A runtime contract describes a deployment, and one model id may have more than one.** The relational declaration below is the broader of those in -service: six task kinds and all seven output fields. A narrower build of the +service: five task kinds and all seven output fields. A narrower build of the same id serves only classification and regression, and produces only ``prediction``, ``probabilities`` and ``quantiles``. Asking that one for a forecast passes the checks here and is refused by the endpoint. @@ -28,6 +28,11 @@ contract has no capabilities route and ``/v1/models`` returns an id and nothing else. So this is the broader of the two on purpose: refusing locally what one deployment can answer would be worse than letting the other say no. + +Candidate-free temporal link prediction is the exception: no released +deployment serves it, so it is not declared, and the client refuses it before +anything is read or sent. A ranking with candidates is asked as one +classification per candidate, which every deployment serves. """ from __future__ import annotations @@ -56,7 +61,6 @@ 'context_rows_fast': 1_000, 'context_rows_max': 10_000, 'predict_rows_max': 10_000, - 'predict_rows_temporal_link_prediction': 200, 'hops_max': 6, 'tables_max': 15, 'classes_max': 10, @@ -107,7 +111,6 @@ 'multiclass_classification', 'regression', 'forecasting', - 'temporal_link_prediction', } ), outputs=frozenset( @@ -187,8 +190,11 @@ class KumoTabular(Model): class KumoRelational(Model): r"""Several related tables, joined as a graph. - Takes every task kind the contract carries except ``ranking``, and produces - all seven output fields including ``explanation``. + Takes every task kind the contract carries except ``ranking`` and + ``temporal_link_prediction``, and produces all seven output fields + including ``explanation``. A ranking is asked as one classification per + candidate; a temporal link prediction asked without candidates is refused + before anything is read, since no released model serves it. Building the graph and sampling it run in the client's Rust core. """ diff --git a/packages/structured-data-models-client/src/sdm_client/query.py b/packages/structured-data-models-client/src/sdm_client/query.py index 889b15685..ff5e048c6 100644 --- a/packages/structured-data-models-client/src/sdm_client/query.py +++ b/packages/structured-data-models-client/src/sdm_client/query.py @@ -194,11 +194,10 @@ def materialize( regression's prediction reports of its predicted distribution, ``'median'`` (the default) or ``'mean'``; and ``rows_per_call``: how many entities one call scores, which - defaults to what the endpoint takes, 10,000 or 200 for a - temporal link prediction, and may only be lowered. Lower it - for a graph wide enough that a full batch outruns the 30 MiB - body, or for a deployment holding to less than this client - knows of. + defaults to what the endpoint takes, 10,000, and may only be + lowered. Lower it for a graph wide enough that a full batch + outruns the 30 MiB body, or for a deployment holding to less + than this client knows of. Returns: A:class:`~sdm_client.Request` carrying the data. @@ -209,6 +208,11 @@ def materialize( StructuredDataError: If neither ``indices`` nor ``predict`` is given, if both are, if an option is not one this accepts, or if ``model`` cannot answer a schema of this shape. + UnsupportedRequestError: If the query is a temporal link + prediction asked without candidates, a ``LIST_DISTINCT`` + with no ``RANK TOP k``, which no released Kumo Relational + model serves. It is refused before any source is read; rank + with ``RANK TOP k`` and ``candidates=`` instead. """ spec = spec_for(model) if model is not None else None in_place = self.schema.source is not None diff --git a/packages/structured-data-models-client/src/sdm_client/relational_task.py b/packages/structured-data-models-client/src/sdm_client/relational_task.py index 7085d1310..525438d4a 100644 --- a/packages/structured-data-models-client/src/sdm_client/relational_task.py +++ b/packages/structured-data-models-client/src/sdm_client/relational_task.py @@ -15,6 +15,10 @@ class RelationalTask: Forecast steps are fixed nanoseconds. Link targets name another schema table. Context columns are ENTITY, ANCHOR_TIMESTAMP and TARGET; prediction columns omit TARGET. Static tasks may omit ANCHOR_TIMESTAMP. + + ``temporal_link_prediction`` is refused by ``materialize_examples`` before + any work, with ``UnsupportedRequestError``, since no released Kumo + Relational model serves it. """ kind: Literal[ diff --git a/packages/structured-data-models-client/src/sdm_client/schema.py b/packages/structured-data-models-client/src/sdm_client/schema.py index 8c5da7f77..be4e65439 100644 --- a/packages/structured-data-models-client/src/sdm_client/schema.py +++ b/packages/structured-data-models-client/src/sdm_client/schema.py @@ -984,6 +984,9 @@ def materialize_examples( Use RelationalTask for task meaning and DataFrame/Arrow example tables. Context has ENTITY, TARGET and optional ANCHOR_TIMESTAMP; predictions omit TARGET. Rust validates and snapshots every input before returning. + A ``temporal_link_prediction`` task is refused with + ``UnsupportedRequestError`` before any work, since no released Kumo + Relational model serves it. """ from sdm_client._rust.examples import materialize diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py index 923ffff00..31a76d4bd 100644 --- a/packages/structured-data-models-client/tests/conftest.py +++ b/packages/structured-data-models-client/tests/conftest.py @@ -179,3 +179,35 @@ def build( ) return build + + +@pytest.fixture +def source_reads(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + """Each fetch a core run asks of its source's provider, in order. + + Counted where the run loop hands a fetch to the provider, so every route + over a source is seen, and a run over frames, which has no provider, + records none. + """ + from sdm_client._rust import run as run_module + + reads: list[dict[str, Any]] = [] + steps = run_module._steps + + class Counted: + def __init__(self, provider: Any) -> None: + self._provider = provider + + def __call__(self, request: dict[str, Any], keys: Any) -> Any: + reads.append(request) + return self._provider(request, keys) + + def __getattr__(self, name: str) -> Any: + return getattr(self._provider, name) + + def counted(core: Any, handle: Any, provider: Any, *rest: Any) -> Any: + wrapped = None if provider is None else Counted(provider) + return steps(core, handle, wrapped, *rest) + + monkeypatch.setattr(run_module, '_steps', counted) + return reads diff --git a/packages/structured-data-models-client/tests/test_custom_examples.py b/packages/structured-data-models-client/tests/test_custom_examples.py index 9968f8f0d..7b5482f9c 100644 --- a/packages/structured-data-models-client/tests/test_custom_examples.py +++ b/packages/structured-data-models-client/tests/test_custom_examples.py @@ -11,7 +11,7 @@ import pytest from sdm_client import RelationalTask, Relationship, Schema -from sdm_client.errors import StructuredDataError +from sdm_client.errors import StructuredDataError, UnsupportedRequestError FIXTURE = json.loads( ( @@ -63,13 +63,18 @@ def test_shared_examples_labels_order_ownership_and_persistence(case): task = RelationalTask(**case['task']) for field in ('link_target_table', 'num_forecasts', 'step_size_ns'): assert getattr(task, field) == case['task'].get(field) + if task.kind == 'temporal_link_prediction': + with pytest.raises(UnsupportedRequestError) as refused: + schema.materialize_examples( + task, context, predict, num_neighbors=[10, 10] + ) + assert refused.value.details['rule'] == 'temporal_link_prediction' + return request = schema.materialize_examples( task, context, predict, num_neighbors=[10, 10] ) payload = request.payloads[0] expected = case['expected']['context_targets'] - if case['task']['kind'] == 'temporal_link_prediction': - expected = [[str(v) for v in row] for row in expected] assert column(payload['context']['instance_table'], 'TARGET') == expected assert len(payload['context']['instance_table']['rows']) == len(context) assert payload['inference']['use_prediction_time'] is False @@ -81,33 +86,6 @@ def test_shared_examples_labels_order_ownership_and_persistence(case): assert restored.payloads == request.payloads -@pytest.mark.parametrize('large', [False, True]) -def test_sliced_link_targets_ignore_hidden_backing_values(large): - case = next( - c for c in FIXTURE['cases'] if c['id'] == 'temporal_link_prediction' - ) - context = pa.Table.from_pandas(frame(case['context']), preserve_index=False) - dtype = pa.large_list(pa.int64()) if large else pa.list_(pa.int64()) - targets = pa.array( - [[999]] + context['TARGET'].to_pylist() + [[999]], type=dtype - ) - padded = pa.concat_tables( - [context.slice(0, 1), context, context.slice(0, 1)] - ) - padded = padded.set_column( - padded.schema.get_field_index('TARGET'), 'TARGET', targets - ) - request = schema_for(case).materialize_examples( - RelationalTask(**case['task']), - padded.slice(1, len(context)), - frame(case['predict']), - num_neighbors=[10, 10], - ) - assert column( - request.payloads[0]['context']['instance_table'], 'TARGET' - ) == [[str(x) for x in row] for row in case['expected']['context_targets']] - - @pytest.mark.parametrize('bad', [float('nan'), float('inf')]) def test_nonfinite_labels_rejected(bad): case = FIXTURE['cases'][0] @@ -230,6 +208,54 @@ def test_sql_supplied_labels_and_repeated_entities(provider): connection.close() +def test_supplied_link_examples_are_refused_before_any_read(source_reads): + import duckdb + from sdm_connectors import DuckDBSource + + connection = duckdb.connect() + try: + connection.execute( + 'CREATE TABLE accounts(id INTEGER PRIMARY KEY, feature DOUBLE)' + ) + connection.execute( + 'INSERT INTO accounts SELECT i, i FROM range(6) t(i)' + ) + schema = Schema.from_source( + DuckDBSource(connection), + {'accounts': 'accounts'}, + primary_keys={'accounts': 'id'}, + relationships=[], + ) + source_reads.clear() + anchor = pd.Timestamp('2024-01-01') + with pytest.raises(UnsupportedRequestError) as refused: + schema.materialize_examples( + RelationalTask( + 'temporal_link_prediction', + 'accounts', + link_target_table='accounts', + ), + pd.DataFrame( + { + 'ENTITY': [0, 1], + 'ANCHOR_TIMESTAMP': [anchor, anchor], + 'TARGET': [[2], [3]], + } + ), + pd.DataFrame({'ENTITY': [4], 'ANCHOR_TIMESTAMP': [anchor]}), + ) + assert refused.value.details['rule'] == 'temporal_link_prediction' + assert source_reads == [] + schema.materialize_examples( + RelationalTask('regression', 'accounts'), + pd.DataFrame({'ENTITY': [0, 1], 'TARGET': [2.5, 3.5]}), + pd.DataFrame({'ENTITY': [4]}), + ) + assert source_reads + finally: + connection.close() + + @pytest.mark.live_nim @pytest.mark.parametrize( 'kind,horizon', @@ -237,7 +263,6 @@ def test_sql_supplied_labels_and_repeated_entities(provider): ('regression', None), ('binary_classification', None), ('multiclass_classification', None), - ('temporal_link_prediction', None), ('forecasting', 3), ('forecasting', 1), ], @@ -264,10 +289,6 @@ def test_live_supplied_examples(kind, horizon): endpoint = ( config['endpoints_by_case']['relational.forecast']['relational'] if horizon is not None - else config['endpoints_by_case']['relational.temporal-link'][ - 'relational' - ] - if kind == 'temporal_link_prediction' else config['endpoints']['relational'] ) client = ( @@ -370,21 +391,3 @@ def test_live_warehouse_supplied_examples(provider): assert len(client.predict(request, model=KumoRelational())) == 2 finally: client.close() - - -def test_supplied_link_batches_obey_model_limit(): - case = next( - c for c in FIXTURE['cases'] if c['id'] == 'temporal_link_prediction' - ) - predicted = pd.concat( - [frame(case['predict']).iloc[:1]] * 201, ignore_index=True - ) - request = schema_for(case).materialize_examples( - RelationalTask(**case['task']), - frame(case['context']), - predicted, - rows_per_call=500, - ) - assert [ - len(p['predict']['instance_table']['rows']) for p in request.payloads - ] == [200, 1] diff --git a/packages/structured-data-models-client/tests/test_frame_materialize_core.py b/packages/structured-data-models-client/tests/test_frame_materialize_core.py index 811b1cf5e..3396e9428 100644 --- a/packages/structured-data-models-client/tests/test_frame_materialize_core.py +++ b/packages/structured-data-models-client/tests/test_frame_materialize_core.py @@ -11,7 +11,7 @@ import pytest from sdm_client import KumoRelational, PredictiveQuery, Schema -from sdm_client.errors import StructuredDataError +from sdm_client.errors import StructuredDataError, UnsupportedRequestError def materialize(query): @@ -139,33 +139,28 @@ def link_schema(): ) -@pytest.mark.parametrize('rank', [False, True]) -def test_link_prediction_frame_materialization(rank): +def test_link_prediction_frame_materialization(): data = link_schema() query = PredictiveQuery( 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id', data, ) - if rank: - ranked = PredictiveQuery( - 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) RANK TOP 2 ' - 'FOR EACH users.user_id', - data, - ) - with pytest.raises(StructuredDataError, match='candidates'): - materialize(ranked) - request = ranked.materialize( - indices=[1, 2], - random_seed=7, - model=KumoRelational(), - candidates=[1, 2, 3], - ) - assert len(request.parts) == 3 - return - first = materialize(query) - assert len(first) == 1 - assert first[0]['task']['kind'] == 'temporal_link_prediction' - assert first == materialize(query) + with pytest.raises(UnsupportedRequestError, match='RANK TOP k'): + materialize(query) + ranked = PredictiveQuery( + 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) RANK TOP 2 ' + 'FOR EACH users.user_id', + data, + ) + with pytest.raises(StructuredDataError, match='candidates'): + materialize(ranked) + request = ranked.materialize( + indices=[1, 2], + random_seed=7, + model=KumoRelational(), + candidates=[1, 2, 3], + ) + assert len(request.parts) == 3 def test_composite_neighbor_identity_is_materialized(): @@ -195,108 +190,6 @@ def test_composite_neighbor_identity_is_materialized(): assert 'orders' in first[0]['schema']['related_tables'] -def test_float_link_identifiers_keep_source_precision(): - data = Schema.from_data( - { - 'users': pd.DataFrame({'user_id': range(1, 21)}), - 'items': pd.DataFrame( - {'item_id': [16777216.0, 16777217.0], 'price': [1.0, 2.0]} - ), - 'orders': pd.DataFrame( - { - 'order_id': range(160), - 'user_id': [i % 20 + 1 for i in range(160)], - 'item_id': [16777216.0 + (i // 20) % 2 for i in range(160)], - 'time': pd.date_range('2024-01-01', periods=160, freq='D'), - } - ), - }, - primary_keys={ - 'users': 'user_id', - 'items': 'item_id', - 'orders': 'order_id', - }, - time_columns={'orders': 'time'}, - ) - query = PredictiveQuery( - 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id', - data, - ) - payload = materialize(query)[0] - context = payload['context']['instance_table'] - position = context['columns'].index('TARGET') - targets = [row[position] for row in context['rows']] - assert {value for group in targets for value in group} == { - '16777216.0', - '16777217.0', - } - assert any(set(group) == {'16777216.0', '16777217.0'} for group in targets) - - -def test_link_prediction_preserves_rankings_and_row_identity(monkeypatch): - from sdm_client import SDMClient - - class Transport: - url = 'http://127.0.0.1:9' - - def __init__(self): - self.sent = [] - - def predict(self, payload): - self.sent.append(payload) - assert payload['task']['kind'] == 'temporal_link_prediction' - assert payload['output']['fields'] == ['rankings'] - rows = payload['predict']['instance_table']['rows'] - columns = payload['predict']['instance_table']['columns'] - identity = columns.index('instance_id') - return { - 'id': 'link-response', - 'model': 'kumo-relational', - 'metadata': {}, - 'predictions': [ - { - 'row_index': 1, - 'id': str(rows[1][identity]), - 'rankings': [ - {'item': '3', 'score': 0.9}, - {'item': '1', 'score': 0.4}, - ], - }, - { - 'row_index': 0, - 'id': str(rows[0][identity]), - 'rankings': [{'item': '2', 'score': 0.7}], - }, - ], - } - - query = PredictiveQuery( - 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id', - link_schema(), - ) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - request = query.materialize( - indices=[1, 2], random_seed=7, model=KumoRelational() - ) - client = SDMClient('http://127.0.0.1:9', 'none') - transport = Transport() - monkeypatch.setattr(client, '_transport', transport) - result = client.predict(request, model=KumoRelational()) - assert [(item.row_index, item.id) for item in result.items] == [ - (1, 2), - (0, 1), - ] - frame = result.to_pandas() - assert frame['row_index'].tolist() == [0, 1] - assert frame['id'].tolist() == [1, 2] - assert frame['rankings'].tolist() == [ - [{'item': '2', 'score': 0.7}], - [{'item': '3', 'score': 0.9}, {'item': '1', 'score': 0.4}], - ] - assert len(transport.sent) == 1 - - @pytest.mark.parametrize('seed', range(8)) def test_generated_temporal_context_matches_independent_join_sum(seed): import random diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 9940e7c6f..97ac9f8e8 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -64,7 +64,6 @@ def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): assert RELATIONAL_LIMITS['context_rows_fast'] == 1_000 assert RELATIONAL_LIMITS['context_rows_max'] == 10_000 assert RELATIONAL_LIMITS['predict_rows_max'] == 10_000 - assert RELATIONAL_LIMITS['predict_rows_temporal_link_prediction'] == 200 assert RELATIONAL_LIMITS['hops_max'] == 6 assert RELATIONAL_LIMITS['tables_max'] == 15 @@ -154,9 +153,14 @@ def test_the_default_outputs_follow_the_relational_task_kind(): def test_the_relational_model_declares_the_deployment_it_was_read_from(): - """A second build of the same id serves less; this is the shipping one.""" + """A second build of the same id serves less; this is the shipping one. + + No released build serves candidate-free temporal link prediction, so it + is not declared. + """ runtime = KUMO_RELATIONAL.runtime - assert {'forecasting', 'temporal_link_prediction'} <= runtime.tasks + assert 'forecasting' in runtime.tasks + assert 'temporal_link_prediction' not in runtime.tasks assert 'explanation' in runtime.outputs # Only these inference options reach the wire; the engine defaults to more. assert 'output_type' not in runtime.inference_options diff --git a/packages/structured-data-models-client/tests/test_query_engine_boundary.py b/packages/structured-data-models-client/tests/test_query_engine_boundary.py index 46b3393df..5b4b94f27 100644 --- a/packages/structured-data-models-client/tests/test_query_engine_boundary.py +++ b/packages/structured-data-models-client/tests/test_query_engine_boundary.py @@ -141,43 +141,20 @@ def test_an_entity_that_is_not_in_the_table_is_a_structured_data_error(): query().materialize(indices=[-1, -2], num_context_samples=20) -def linked() -> Schema: - items = pd.DataFrame( - {'item_id': range(5), 'price': [1.0, 2.0, 3.0, 4.0, 5.0]} - ) - bought = orders().assign(item_id=[index % 5 for index in range(80)]) - return Schema.from_data( - {'users': users(), 'items': items, 'orders': bought} - ) - - -@pytest.mark.parametrize( - ('text', 'data', 'sizes'), - [ - ('PREDICT users.age FOR EACH users.user_id', schema, [2, 2, 1]), - ( - 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) ' - 'FOR EACH users.user_id', - linked, - [1, 1, 1, 1, 1], - ), - ], - ids=['regression', 'link_prediction'], -) -def test_more_entities_than_one_call_scores_are_gathered_in_batches( - monkeypatch, text, data, sizes -): +def test_more_entities_than_one_call_scores_are_gathered_in_batches(): with warnings.catch_warnings(): warnings.simplefilter('ignore') - request = PredictiveQuery(text, data()).materialize( + request = PredictiveQuery( + 'PREDICT users.age FOR EACH users.user_id', schema() + ).materialize( indices=[5, 6, 7, 8, 9], num_context_samples=20, - rows_per_call=sizes[0], + rows_per_call=2, ) assert [ len(payload['predict']['instance_table']['rows']) for payload in request.payloads - ] == sizes + ] == [2, 2, 1] class Echo: diff --git a/packages/structured-data-models-client/tests/test_refusals_that_lead_somewhere.py b/packages/structured-data-models-client/tests/test_refusals_that_lead_somewhere.py index ed17d5af2..500e32234 100644 --- a/packages/structured-data-models-client/tests/test_refusals_that_lead_somewhere.py +++ b/packages/structured-data-models-client/tests/test_refusals_that_lead_somewhere.py @@ -34,7 +34,7 @@ Relationship, Schema, ) -from sdm_client.errors import StructuredDataError +from sdm_client.errors import StructuredDataError, UnsupportedRequestError RNG = np.random.default_rng(4) @@ -200,22 +200,20 @@ def test_a_foreign_key_list_distinct_still_works(query): """Both forms resolve to link prediction, and the multilabel refusal must not widen onto them. - A ranking has no task on the endpoint, so the ranked form is asked as one - question per candidate and needs them named. That it refuses for the want - of candidates is the refusal working: before, it built a request the - endpoint answered with `422 UNSUPPORTED_TASK`. + Neither has a task on the endpoint. The ranked form is asked as one + question per candidate and needs them named; the plain form is refused + before anything is read, and names the ranked form. Either refusal is the + refusal working: before, each built a request the endpoint answered with + `422 UNSUPPORTED_TASK`. """ - + with pytest.raises(StructuredDataError) as refused: + answer(query, graph(timed=True), KumoRelational()) + assert 'multilabel' not in str(refused.value) if 'RANK' in query: - with pytest.raises(StructuredDataError) as refused: - answer(query, graph(timed=True), KumoRelational()) assert 'candidates' in str(refused.value) - assert 'multilabel' not in str(refused.value) - return - - assert answer(query, graph(timed=True), KumoRelational()) == ( - 'temporal_link_prediction' - ) + else: + assert isinstance(refused.value, UnsupportedRequestError) + assert 'RANK TOP k with candidates=' in str(refused.value) # --- advice on a graph with no time column --------------------------------- diff --git a/packages/structured-data-models-client/tests/test_temporal_link_batching.py b/packages/structured-data-models-client/tests/test_temporal_link_batching.py index dbc6450f5..d1462041d 100644 --- a/packages/structured-data-models-client/tests/test_temporal_link_batching.py +++ b/packages/structured-data-models-client/tests/test_temporal_link_batching.py @@ -2,7 +2,11 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Public PQL batching obeys the model limit before any prediction is sent.""" +"""Candidate-free temporal link prediction is refused before anything is read. + +No released Kumo Relational model serves it, so building its request would +only spend a source's reads on a certain refusal. +""" from datetime import datetime, timedelta @@ -13,7 +17,7 @@ from sdm_connectors import DuckDBSource, SQLiteSource from sdm_client import KumoRelational, PredictiveQuery, Schema, SDMClient -from sdm_client.errors import StructuredDataError +from sdm_client.errors import UnsupportedRequestError QUERY = ( 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id' @@ -99,33 +103,24 @@ def materialize(schema, count=201, rows_per_call=None, query=QUERY): ) -@pytest.mark.parametrize( - ('count', 'limit', 'sizes'), - [ - (199, None, [199]), - (200, None, [200]), - (201, None, [200, 1]), - (201, 100, [100, 100, 1]), - (201, 1000, [200, 1]), - ], -) -def test_pql_link_batches_preserve_every_entity( - link_schema, count, limit, sizes +def assert_refused(refused): + assert refused.value.code == 'UNSUPPORTED_REQUEST' + assert refused.value.details['rule'] == 'temporal_link_prediction' + assert 'RANK TOP k with candidates=' in str(refused.value) + + +def test_candidate_free_link_prediction_is_refused_before_any_read( + link_schema, source_reads ): - request = materialize(link_schema, count, limit) - assert [ - len(p['predict']['instance_table']['rows']) for p in request.payloads - ] == sizes - entities = [] - for payload in request.payloads: - assert payload['task']['kind'] == 'temporal_link_prediction' - table = payload['predict']['instance_table'] - column = table['columns'].index('__kumo_entity_ref_0') - entities.extend(int(row[column]) for row in table['rows']) - assert entities == list(range(1, count + 1)) + source_reads.clear() + with pytest.raises(UnsupportedRequestError) as refused: + materialize(link_schema) + assert_refused(refused) + assert source_reads == [] -def test_node_predictions_keep_the_ordinary_limit(link_schema): +def test_node_predictions_keep_the_ordinary_limit(link_schema, source_reads): + source_reads.clear() request = materialize( link_schema, query='PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', @@ -133,62 +128,23 @@ def test_node_predictions_keep_the_ordinary_limit(link_schema): assert [ len(p['predict']['instance_table']['rows']) for p in request.payloads ] == [201] + assert bool(source_reads) == (link_schema.source is not None) -def test_link_batches_respect_total_call_limit(link_schema, monkeypatch): - from sdm_client.materializers import relational - - monkeypatch.setattr(relational, 'MOST_CALLS', 1) - with pytest.raises(StructuredDataError) as caught: - materialize(link_schema) - assert caught.value.code == 'UNSUPPORTED_REQUEST' - assert caught.value.details['calls'] == 2 - assert caught.value.details['rows_per_call'] == 200 - - -def test_link_batch_result_order_and_partial_failure( +def test_a_request_naming_the_link_task_is_refused_before_it_is_sent( link_schema, requests_mock ): - request = materialize(link_schema) - calls = [] - fail = False - - def answer(call, response): - payload = call.json() - table = payload['predict']['instance_table'] - calls.append(len(table['rows'])) - if fail and len(calls) == 2: - response.status_code = 500 - return {'error': 'second batch failed'} - identity = table['columns'].index('instance_id') - return { - 'model': 'kumo-relational', - 'id': 'links', - 'metadata': {}, - 'predictions': [ - { - 'row_index': i, - 'id': str(row[identity]), - 'rankings': [{'item': '3', 'score': 0.9}], - } - for i, row in reversed(list(enumerate(table['rows']))) - ], - } - - requests_mock.post('http://localhost/v1/predictions', json=answer) - requests_mock.post('http://localhost/v1/sessions', status_code=405) + """As a request saved by 1.0.0, which built one, would be.""" + request = materialize( + link_schema, + query='PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', + ) + request._core = None + request._payloads[0]['task']['kind'] = 'temporal_link_prediction' + requests_mock.post('http://localhost/v1/predictions', json={}) + requests_mock.post('http://localhost/v1/sessions', json={}) with SDMClient('http://localhost', max_retries=0) as client: - result = client.predict(request, model=KumoRelational()) - assert calls == [200, 1] - assert result.to_pandas()['id'].astype(int).tolist() == list( - range(1, 202) - ) - calls.clear() - fail = True - with pytest.raises(StructuredDataError) as caught: + with pytest.raises(UnsupportedRequestError) as refused: client.predict(request, model=KumoRelational()) - assert calls == [200, 1] - answered = caught.value.details['answered'] - assert answered.to_pandas()['id'].astype(int).tolist() == list( - range(1, 201) - ) + assert_refused(refused) + assert requests_mock.call_count == 0 diff --git a/packages/structured-data-models-client/tests/test_warehouse_links_core.py b/packages/structured-data-models-client/tests/test_warehouse_links_core.py index 909f23761..e24085048 100644 --- a/packages/structured-data-models-client/tests/test_warehouse_links_core.py +++ b/packages/structured-data-models-client/tests/test_warehouse_links_core.py @@ -2,7 +2,8 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Direct warehouse link tasks preserve exact targets and historical context.""" +"""Warehouse link questions: one asked without candidates is refused before +any read, and a ranking is asked as one classification per candidate.""" from datetime import datetime, timedelta @@ -12,12 +13,8 @@ from adbc_driver_sqlite import dbapi as sqlite from sdm_connectors import DuckDBSource, SQLiteSource -from sdm_client import ( - KumoRelational, - PredictiveQuery, - Schema, - SDMClient, -) +from sdm_client import KumoRelational, PredictiveQuery, Schema +from sdm_client.errors import UnsupportedRequestError QUERY = ( 'PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id' @@ -104,96 +101,13 @@ def materialize(schema, query=QUERY, **options): ) -@pytest.mark.parametrize('key_kind', ['integer', 'large_integer', 'text']) -def test_link_labels_and_related_rows_obey_each_example_time(source, key_kind): - schema, orders = prepare(source, key_kind) - request = materialize(schema) - assert len(request.payloads) == 1 - payload = request.payloads[0] - assert payload['task']['kind'] == 'temporal_link_prediction' - assert payload['task']['entity_table_names'] == ['users', 'items'] - assert payload['task']['target']['dtype'] == 'stringlist' - anchor_name = payload['task']['anchor_time_column'] - for half in ('context', 'predict'): - instances = payload[half]['instance_table'] - columns = instances['columns'] - anchors = {} - for row in instances['rows']: - values = dict(zip(columns, row, strict=True)) - anchor = pd.Timestamp(values[anchor_name]) - anchors[values['instance_id']] = anchor - if half == 'context': - expected = { - str(item) - for _, user, item, when in orders - if user == values['__kumo_entity_ref_0'] - and anchor - < pd.Timestamp(when, tz='UTC') - <= anchor + pd.Timedelta(days=30) - } - assert set(values['TARGET']) == expected - assert anchors - items = payload[half]['related_tables']['items'] - item_rows = [ - dict(zip(items['columns'], row, strict=True)) - for row in items['rows'] - ] - # Large integers are encoded as exact decimal strings on the wire. - source_items = {str(item) for _, _, item, _ in orders} - assert all(str(row['item_id']) in source_items for row in item_rows) - item_keys = {(row['instance_id'], row['item_id']) for row in item_rows} - related = payload[half]['related_tables']['orders'] - for row in related['rows']: - values = dict(zip(related['columns'], row, strict=True)) - if values['item_id'] is not None: - assert (values['instance_id'], values['item_id']) in item_keys - assert ( - pd.Timestamp(values['time']) <= anchors[values['instance_id']] - ) - - -def test_link_prediction_and_session_results_use_rankings( - source, requests_mock -): +def test_candidate_free_links_are_refused_before_any_read(source, source_reads): schema, _ = prepare(source) - request = materialize(schema) - - def answer(call, context): - sent = call.json() - assert sent['output']['fields'] == ['rankings'] - rows = sent['predict']['instance_table'] - identity = rows['columns'].index('instance_id') - return { - 'model': 'kumo-relational', - 'id': 'links', - 'metadata': {}, - 'predictions': [ - { - 'row_index': i, - 'id': str(row[identity]), - 'rankings': [{'item': '3', 'score': 0.9}], - } - for i, row in reversed(list(enumerate(rows['rows']))) - ], - } - - requests_mock.post('http://localhost/v1/predictions', json=answer) - requests_mock.post( - 'http://localhost/v1/sessions', json={'session_id': 'links'} - ) - requests_mock.post( - 'http://localhost/v1/sessions/links/predictions', json=answer - ) - deleted = requests_mock.delete( - 'http://localhost/v1/sessions/links', status_code=204 - ) - with SDMClient('http://localhost', max_retries=0) as client: - direct = client.predict(request, model=KumoRelational()) - with client.fit(request, model=KumoRelational()) as fitted: - scored = fitted.predict(request) - pd.testing.assert_frame_equal(direct.to_pandas(), scored.to_pandas()) - assert direct.items == scored.items - assert deleted.call_count == 1 + source_reads.clear() + with pytest.raises(UnsupportedRequestError) as refused: + materialize(schema) + assert refused.value.details['rule'] == 'temporal_link_prediction' + assert source_reads == [] def test_rank_top_keeps_candidate_classification_path(source): diff --git a/rust/crates/sdm-contract/schema/contract.lock b/rust/crates/sdm-contract/schema/contract.lock index 486f4f5b0..7e5638785 100644 --- a/rust/crates/sdm-contract/schema/contract.lock +++ b/rust/crates/sdm-contract/schema/contract.lock @@ -1,2 +1,2 @@ # Current sdm-contract schema; regenerate with schema_lock.rs. -72 sha256:88a10d45842d7a2a66f0a65ad4b43ac7a7522b566f78cbb6b9d0de96b6f8fde0 sdm-contract-72.json +72 sha256:9bb2ddbb1b5eea8a8a7359b0c1aef58c510db5b9bbe6a7856c2860d06d8a44f2 sdm-contract-72.json diff --git a/rust/crates/sdm-contract/schema/sdm-contract-72.json b/rust/crates/sdm-contract/schema/sdm-contract-72.json index c95be4c1e..ced88f19e 100644 --- a/rust/crates/sdm-contract/schema/sdm-contract-72.json +++ b/rust/crates/sdm-contract/schema/sdm-contract-72.json @@ -17737,6 +17737,7 @@ "metadata_not_a_mapping", "model_lacks_shape", "model_lacks_task", + "temporal_link_unserved", "model_lacks_outputs", "context_past_model", "no_context_rows", diff --git a/rust/crates/sdm-contract/src/error.rs b/rust/crates/sdm-contract/src/error.rs index 72f1571a9..99a57cc40 100644 --- a/rust/crates/sdm-contract/src/error.rs +++ b/rust/crates/sdm-contract/src/error.rs @@ -994,6 +994,7 @@ sentences! { MetadataNotAMapping = "metadata_not_a_mapping", E::InvalidResponse; ModelLacksShape = "model_lacks_shape", E::UnsupportedRequest; ModelLacksTask = "model_lacks_task", E::UnsupportedRequest; + TemporalLinkUnserved = "temporal_link_unserved", E::UnsupportedRequest, rule "temporal_link_prediction"; ModelLacksOutputs = "model_lacks_outputs", E::UnsupportedRequest; ContextPastModel = "context_past_model", E::UnsupportedRequest; NoContextRows = "no_context_rows", E::UnsupportedRequest; diff --git a/rust/crates/sdm-contract/src/wording.rs b/rust/crates/sdm-contract/src/wording.rs index 26bca640f..63b6f93b8 100644 --- a/rust/crates/sdm-contract/src/wording.rs +++ b/rust/crates/sdm-contract/src/wording.rs @@ -882,6 +882,9 @@ pub fn template(sentence: Sentence) -> Template { S::ModelLacksTask => { said(r#"{model} does not serve the task kind {task!r}; it serves {supported!r}"#) } + S::TemporalLinkUnserved => said( + r#"candidate-free temporal link prediction ('temporal_link_prediction') is not served by any released Kumo Relational model. Use RANK TOP k with candidates= to rank an explicit candidate set."#, + ), S::ModelLacksOutputs => said( r#"{model} does not produce {unsupported_outputs!r}; it produces {supported_outputs!r}"#, ), diff --git a/rust/crates/sdm-core/src/materialize.rs b/rust/crates/sdm-core/src/materialize.rs index 32de4a3cc..9d2799e71 100644 --- a/rust/crates/sdm-core/src/materialize.rs +++ b/rust/crates/sdm-core/src/materialize.rs @@ -2785,7 +2785,7 @@ pub(crate) enum Gathered { } pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result { - let (steps, ranked, facts) = resolved(&inputs); + let (steps, ranked, facts, links) = resolved(&inputs); // A ranking, asked as one binary question per candidate. if let Some(plan) = ranked { if crate::pql::explained_query(&inputs.query).is_some() { @@ -2808,6 +2808,11 @@ pub(crate) async fn run(asker: Asker, inputs: Inputs) -> Result Ok(result), @@ -3006,11 +3011,12 @@ pub(crate) fn attach_request( /// A forecast's plan ([`crate::pql::resolve`]'s `steps`), when the query /// resolves to one, none otherwise, the query then gathered as itself, -/// which refuses it where it does not resolve as before; and the facts of -/// the query a host reads ([`QueryFacts`]), when it resolves. +/// which refuses it where it does not resolve as before; the facts of +/// the query a host reads ([`QueryFacts`]), when it resolves; and whether +/// its target is a link target ([`Shape::Links`]). #[allow( clippy::type_complexity, - reason = "three plain parts, named where they are taken" + reason = "four plain parts, named where they are taken" )] fn resolved( inputs: &Inputs, @@ -3018,10 +3024,11 @@ fn resolved( Option, Option, Option, + bool, ) { let mut description = inputs.description.clone(); if consistent(&mut description, &inputs.types).is_err() { - return (None, None, None); + return (None, None, None, false); } let profile = Profile::resolve(&ProfileSpec { name: ProfileName::Relational, @@ -3038,9 +3045,16 @@ fn resolved( ) { Ok(validation) => { let facts = validation.facts(&schema); - (validation.steps, validation.ranking, facts) + let links = match (&validation.query, &facts) { + (Some(query), Some(facts)) => matches!( + target_shape(&query.target, &facts.entity_table), + Ok(Shape::Links(_)) + ), + _ => false, + }; + (validation.steps, validation.ranking, facts, links) } - Err(_) => (None, None, None), + Err(_) => (None, None, None, false), } } diff --git a/rust/crates/sdm-core/src/materialize/examples.rs b/rust/crates/sdm-core/src/materialize/examples.rs index 820e46e63..4613a4fbd 100644 --- a/rust/crates/sdm-core/src/materialize/examples.rs +++ b/rust/crates/sdm-core/src/materialize/examples.rs @@ -113,6 +113,11 @@ async fn run( facts: HostFacts, ) -> Result { let call: ExampleCall = crate::attached::take_json(&mut attachments, request.input)?; + // Refused before anything is read, as `materialize` refuses a link + // target asked without candidates. + if call.task.kind == ExampleKind::TemporalLinkPrediction { + return Err(sdm_request::predict::temporal_link_unserved()); + } if call.rows_per_call == Some(0) { return Err(refuse("rows_per_call must be positive")); } diff --git a/rust/crates/sdm-core/tests/materialize.rs b/rust/crates/sdm-core/tests/materialize.rs index 78825a779..96b73ffec 100644 --- a/rust/crates/sdm-core/tests/materialize.rs +++ b/rust/crates/sdm-core/tests/materialize.rs @@ -31,8 +31,8 @@ use sdm_contract::ops::{ }; use sdm_contract::run::StartRun; use sdm_contract::{ - CONTRACT_VERSION, ColumnType, CoreError, Detail, ErrorCode, HeldState, HostFacts, Scalar, Step, - Unit, + CONTRACT_VERSION, ColumnType, CoreError, Detail, ErrorCode, HeldState, HostFacts, Scalar, + Sentence, Step, Unit, }; use sdm_core::{ ArrowData, Attachment, Attachments, CompositionRoot, FetchResult, Run, UnknownZone, Zones, @@ -748,6 +748,56 @@ fn what_the_core_does_not_materialize_yet_is_refused_by_name() { assert_eq!(err.fields["field"], Detail::text("relationships")); } +/// A link target asked without candidates is refused before anything is +/// read, since no released model serves the request it would build; the +/// same schema's scalar question reaches its first read. +#[test] +fn a_link_target_without_candidates_is_refused_before_anything_is_read() { + let mut linked = shop(); + linked.tables[0].columns.push(column( + "item_id", + ColumnType::Int64, + Dtype::Int, + Stype::Id, + false, + )); + linked.tables.push(DescribedTable { + calculated_columns: vec![], + name: "items".into(), + source: "items".into(), + columns: vec![column( + "item_id", + ColumnType::Int64, + Dtype::Int, + Stype::Id, + true, + )], + primary_key: vec!["item_id".into()], + time_column: None, + end_time_column: None, + time_column_inferred: None, + duplicate_keys: None, + }); + linked.links.push(vec![ + ColumnRef { + table: "orders".into(), + column: "item_id".into(), + }, + ColumnRef { + table: "items".into(), + column: "item_id".into(), + }, + ]); + let query = "PREDICT LIST_DISTINCT(orders.item_id, 0, 30, days) FOR EACH users.user_id"; + let mut run = materialize(host(), &linked, query, None).unwrap(); + let err = run.next().unwrap_err(); + assert_eq!(err.sentence, Sentence::TemporalLinkUnserved, "{err:?}"); + assert_eq!(err.code, ErrorCode::UnsupportedRequest); + assert_eq!(err.fields["rule"], Detail::text("temporal_link_prediction")); + let mut run = materialize(host(), &linked, QUERY, None).unwrap(); + assert!(matches!(run.next(), Ok(Step::Fetch { .. }))); +} + /// The schema `materialize` is handed is held as the reference holds a /// schema it makes a request of (`Schema.check_consistent`), /// before anything is read: a meaning the column's values cannot carry is diff --git a/rust/crates/sdm-request/src/predict.rs b/rust/crates/sdm-request/src/predict.rs index aa61ebbe8..7e74e5583 100644 --- a/rust/crates/sdm-request/src/predict.rs +++ b/rust/crates/sdm-request/src/predict.rs @@ -173,8 +173,10 @@ pub fn capabilities() -> Vec { .collect() } -/// `KUMO_RELATIONAL`: every shape (the relational profile), its six task -/// kinds and seven outputs, each list sorted. +/// `KUMO_RELATIONAL`: every shape (the relational profile), its five task +/// kinds and seven outputs, each list sorted. Candidate-free temporal link +/// prediction is left out: no released model serves it +/// ([`temporal_link_unserved`]). const KUMO_RELATIONAL: Spec = Spec { id: RELATIONAL, shapes: &SHAPES, @@ -184,7 +186,6 @@ const KUMO_RELATIONAL: Spec = Spec { "forecasting", "multiclass_classification", "regression", - "temporal_link_prediction", ], outputs: &[ "embeddings", @@ -220,6 +221,22 @@ pub fn not_yet(rule: &'static str) -> CoreError { .with_field("rule", Detail::text(rule)) } +/// The task kind of a candidate-free temporal link prediction. +pub const TEMPORAL_LINK_PREDICTION: &str = "temporal_link_prediction"; + +/// The refusal of a candidate-free temporal link prediction, which no +/// released Kumo Relational model serves: given before anything is read or +/// sent, by `materialize`, by `materialize_examples` and at prediction +/// admission, naming the ranking with candidates that is served instead. +pub fn temporal_link_unserved() -> CoreError { + CoreError::new( + Sentence::TemporalLinkUnserved, + "no released model serves candidate-free temporal link prediction", + ) + .with_field("model", Detail::text(RELATIONAL)) + .with_field("task", Detail::text(TEMPORAL_LINK_PREDICTION)) +} + /// The model cannot answer the request, as the client refuses it before /// anything is sent (`UnsupportedRequestError`): the model named, never a /// value of the request. @@ -2302,6 +2319,9 @@ fn check_task_outputs( outputs: &[String], deduplicate: bool, ) -> Result<(), CoreError> { + if kind == TEMPORAL_LINK_PREDICTION { + return Err(temporal_link_unserved()); + } if !spec.tasks.contains(&kind) { return Err(unsupported( Sentence::ModelLacksTask, @@ -2442,6 +2462,29 @@ mod tests { } use sdm_contract::ErrorCode; + #[test] + fn a_temporal_link_prediction_is_refused_before_it_is_sent() { + let payload = + sdm_wire::json::read(br#"{"task":{"kind":"temporal_link_prediction"}}"#).unwrap(); + let prepared = prepare(&[payload], RELATIONAL, &[], true).unwrap_err(); + let bound = bind_direct(sdm_contract::direct::DirectCallFacts { + query: None, + model: RELATIONAL.to_owned(), + outputs: None, + kind: TEMPORAL_LINK_PREDICTION.to_owned(), + anchored: true, + relationships: true, + tables: 2, + context_rows: 1, + }) + .unwrap_err(); + for error in [prepared, bound] { + assert_eq!(error.sentence, Sentence::TemporalLinkUnserved); + assert_eq!(error.code, ErrorCode::UnsupportedRequest); + assert_eq!(error.fields["rule"], Detail::text(TEMPORAL_LINK_PREDICTION)); + } + } + #[test] fn every_model_is_described_as_the_client_describes_it() { let all = capabilities(); @@ -2456,7 +2499,6 @@ mod tests { "forecasting", "multiclass_classification", "regression", - "temporal_link_prediction", ] ); assert!(relational.single_table && relational.multiple_tables && relational.relationships); diff --git a/skills/README.md b/skills/README.md index 55772f68e..3ad61fba4 100644 --- a/skills/README.md +++ b/skills/README.md @@ -11,9 +11,9 @@ Start at one of these two. Each is enough on its own to ask a first question. | [`relational/SKILL.md`](relational/SKILL.md) | `kumo-relational` | The question is about an entity and what it is linked to | An agent answering a question about a single table does not need the -relational material, and the four task kinds that make the relational model -what it is — the two shapes of classification, forecasting and link prediction -— are not on the tabular one at all. +relational material. What makes the relational model what it is, the two +shapes of classification, forecasting and ranking an entity's candidate links, +is not on the tabular one at all. Each skill names the references it needs. They stand on their own, so one can be loaded without the skill it belongs to: diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 0f05a2672..247f6c1dd 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -1,6 +1,6 @@ --- name: sdm-relational -description: "Ask kumo-relational a question about an entity and the records linked to it: write the question as a predictive query over a schema of related tables, covering classification, regression, forecasting and link prediction. Triggers: kumo-relational, KumoRelational, PREDICT, FOR EACH, predictive query, PQL, Schema.from_data, Schema.from_source, Relationship, ranking, forecast, link prediction, explanation, Snowflake, Databricks, warehouse, metric view, semantic view, SQL, catalog." +description: "Ask kumo-relational a question about an entity and the records linked to it: write the question as a predictive query over a schema of related tables, covering classification, regression, forecasting and ranking the candidates an entity may link to. Triggers: kumo-relational, KumoRelational, PREDICT, FOR EACH, predictive query, PQL, Schema.from_data, Schema.from_source, Relationship, ranking, forecast, link prediction, explanation, Snowflake, Databricks, warehouse, metric view, semantic view, SQL, catalog." --- # Asking kumo-relational @@ -32,7 +32,7 @@ client is willing to ask. What the client will send: | | `kumo-relational` | |---|---| -| Tasks | `binary_classification`, `classification`, `multiclass_classification`, `regression`, `forecasting`, `temporal_link_prediction` | +| Tasks | `binary_classification`, `classification`, `multiclass_classification`, `regression`, `forecasting` | | Outputs | `prediction`, `probabilities`, `quantiles`, `rankings`, `scores`, `embeddings`, `explanation` | | Several tables | yes | | Relationships | yes | @@ -159,7 +159,6 @@ would break rather than letting the endpoint return a 500. | context rows, by default | `1,000` | | context rows, with `num_context_samples` | `10,000` | | rows scored at once | `10,000` | -| rows scored for a link prediction | `200` | | tables in the gathered graph | `15` | | hops gathered | `6` | | classes | `10` | @@ -275,3 +274,4 @@ change. The ones worth knowing: | an output this question cannot give | `rankings` on a regression, `explanation` on a client-assembled ranking | | `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation` | | an aggregation the model does not answer | `COUNT_DISTINCT`, `FIRST`, `LAST`, or any aggregation of a category | +| a link prediction with no candidates | no released model serves `temporal_link_prediction`, so `materialize()` refuses it before reading anything; rank with `RANK TOP k` and `candidates=` | diff --git a/skills/relational/pql.md b/skills/relational/pql.md index 98b43f248..c2b36130d 100644 --- a/skills/relational/pql.md +++ b/skills/relational/pql.md @@ -164,15 +164,15 @@ The second line of that refusal is a rule of its own: **aggregate a number, not a category.** The one way to ask about a category is `LIST_DISTINCT` over a foreign key. -**Give that one `RANK TOP k`.** Without the clause it is a link prediction, -which the client builds natively and the deployment has no task for: +**Give that one `RANK TOP k`.** Without the clause it is a temporal link +prediction, which no released model serves, so the client refuses it before +reading anything: ``` PREDICT LIST_DISTINCT(orders.item, 0, 30, days) FOR EACH users.user_id -→ [422 UNSUPPORTED_TASK] task 'temporal_link_prediction' not supported by - model 'kumo-relational'; supported: ['classification', - 'binary_classification', - 'multiclass_classification', 'regression']. +→ candidate-free temporal link prediction ('temporal_link_prediction') is not + served by any released Kumo Relational model. Use RANK TOP k with + candidates= to rank an explicit candidate set. ``` That is what `RANK` is for: it asks the same question as one classification diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 93f5f2d5b..88a02f060 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -213,7 +213,7 @@ names what to change. | Refusal | Cause | |---|---| -| a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | +| a task the model does not serve | `forecasting` and the rest are relational; no released model serves `temporal_link_prediction` | | an output it does not produce | `rankings`, `embeddings`, `scores` | | a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | diff --git a/tests/compatibility/manifest.json b/tests/compatibility/manifest.json index d0a8bdb22..eb1e909c5 100644 --- a/tests/compatibility/manifest.json +++ b/tests/compatibility/manifest.json @@ -861,7 +861,10 @@ "random_seed": 7 }, "expected": { - "status": "supported" + "status": "refused", + "stage": "materialize", + "code": "UNSUPPORTED_REQUEST", + "rule": "temporal_link_prediction" }, "tolerance": { "absolute": 0, @@ -3796,9 +3799,7 @@ "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_materializer_snapshot_does_not_alias_input_buffers", "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_temporal_frame_materialization", "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_link_prediction_frame_materialization", - "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_composite_neighbor_identity_is_materialized", - "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_float_link_identifiers_keep_source_precision", - "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_link_prediction_preserves_rankings_and_row_identity" + "packages/structured-data-models-client/tests/test_frame_materialize_core.py::test_composite_neighbor_identity_is_materialized" ], "java": [ "java/src/test/java/com/nvidia/sdm/RelationalTest.java#candidateNumbersKeepTheirExactValuesAcrossTheBoundary", @@ -4568,7 +4569,7 @@ ] }, "gate": "local", - "note": "Exact-context reuse, bounded expiry recovery, unsupported-session fallback, centralized retries and primary-error/partial-result preservation; controlled HTTP tests plus separate live stateless comparison. PQL and supplied temporal-link batches share the 200-row model cap, with ordered entity identities on memory, DuckDB and SQLite routes.", + "note": "Exact-context reuse, bounded expiry recovery, unsupported-session fallback, centralized retries and primary-error/partial-result preservation; controlled HTTP tests plus separate live stateless comparison. PQL and supplied candidate-free temporal link prediction is refused before any source read on memory, DuckDB and SQLite routes, since no released model serves it.", "ownership": { "paths": [ "rust/crates/sdm-core/src/batch_session.rs" @@ -8265,8 +8266,28 @@ { "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", "record": "live_api_tests", - "reason": "No released Kumo Relational NIM serves temporal link prediction: 1.2.0 refuses it with UNSUPPORTED_TASK. The two live link-batching tests could never pass, so they are removed; the offline batching tests keep the 200-row call cap and entity order covered.", - "retained_test": "packages/structured-data-models-client/tests/test_temporal_link_batching.py::test_pql_link_batches_preserve_every_entity" + "reason": "No released Kumo Relational NIM serves temporal link prediction: 1.2.0 refuses it with UNSUPPORTED_TASK. The two live link-batching tests could never pass, so they are removed; the client now refuses the task before any source read, which the offline refusal tests cover.", + "retained_test": "packages/structured-data-models-client/tests/test_temporal_link_batching.py::test_candidate_free_link_prediction_is_refused_before_any_read" + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "scenarios/relational.temporal-link", + "reason": "No released Kumo Relational model serves candidate-free temporal link prediction (the NIM's 1.2.0 answers 422 UNSUPPORTED_TASK, and in-process mode does not serve it), so materialize() now refuses it with UNSUPPORTED_REQUEST, rule temporal_link_prediction, before any source is read, where 1.0.0 built the request and the endpoint refused it. The scenario expects that refusal at materialize." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "capabilities/prediction.temporal-link", + "reason": "No released Kumo Relational model serves candidate-free temporal link prediction (the NIM's 1.2.0 answers 422 UNSUPPORTED_TASK, and in-process mode does not serve it), so materialize() now refuses it with UNSUPPORTED_REQUEST, rule temporal_link_prediction, before any source is read, where 1.0.0 built the request and the endpoint refused it. The capability's requirement mirrors the scenario's refusal." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/relational-materialization", + "reason": "Candidate-free temporal link prediction is refused before materializing, so the two frame tests that built and predicted its payload are removed; test_link_prediction_frame_materialization keeps the refusal and the ranked path with candidates." + }, + { + "base_commit": "622a4f4b3bc8ef3ddc356805eb2cb868011a48c6", + "record": "api_contracts/automatic-batch-sessions", + "reason": "Candidate-free temporal link prediction is refused before any source read, so its 200-row batches are never built; the note describes the refusal the linked batching tests now cover." } ], "capabilities": [ @@ -9235,7 +9256,10 @@ "snowflake-semantic" ], "expected": { - "status": "supported" + "status": "refused", + "stage": "materialize", + "code": "UNSUPPORTED_REQUEST", + "rule": "temporal_link_prediction" }, "expected_by_model": {}, "expected_by_language": {}, diff --git a/tests/test_skills.py b/tests/test_skills.py index 4e9695f35..36336fbe8 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2266,9 +2266,6 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'context rows, by default': 'context_rows_fast', 'context rows, with `num_context_samples`': 'context_rows_max', 'rows scored at once': 'predict_rows_max', - 'rows scored for a link prediction': ( - 'predict_rows_temporal_link_prediction' - ), 'tables in the gathered graph': 'tables_max', 'hops gathered': 'hops_max', 'classes': 'classes_max', From 39e79068e556a0c40a1097262954f3f0b2879308 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 9 Nov 2026 23:44:43 -0800 Subject: [PATCH 24/24] Expect the link refusal in the in-process and Java result tests Every backend now refuses a candidate-free link prediction with UNSUPPORTED_REQUEST before anything is sent. The Java result test checks that refusal, and the immutable ranking dictionaries are checked on the candidate ranking instead. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../java/com/nvidia/sdm/ResultItemsTest.java | 20 +++++++++---------- .../tests/test_every_backend_answers_alike.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/java/src/test/java/com/nvidia/sdm/ResultItemsTest.java b/java/src/test/java/com/nvidia/sdm/ResultItemsTest.java index b0eba065e..c3f2bfbf5 100644 --- a/java/src/test/java/com/nvidia/sdm/ResultItemsTest.java +++ b/java/src/test/java/com/nvidia/sdm/ResultItemsTest.java @@ -63,6 +63,9 @@ void rankingOwnersSurviveDifferentCandidateListsAndRepeatedMixedTaskMerges() thr PredictionResult b = client.predict(second, SdmClient.KUMO_RELATIONAL); assertEquals(List.of(1L, 2L), a.items().get(0).prediction()); assertEquals(Map.of("candidate", 1L, "score", .9), a.items().get(0).rankings().get(0)); + assertThrows( + UnsupportedOperationException.class, + () -> a.items().get(0).rankings().get(0).put("score", 0)); assertEquals(List.of(999L), a.unscoredCandidates()); PredictionResult joined = PredictionResult.merge(List.of(a, b)); assertEquals(List.of(1L, 2L), joined.items().get(0).prediction()); @@ -89,16 +92,13 @@ void temporalLinkDictionariesAndForecastStepsNeedNoJsonParsingByCaller() throws Schema schema = RelationalTest.fixture(true, true); try (RelationalTest.Endpoint endpoint = new RelationalTest.Endpoint(); SdmClient client = endpoint.client()) { - Request link = - PredictiveQuery.parse(RelationalTest.LINK) - .materialize(schema, RelationalTest.options(1L, 2L).build()); - List linked = client.predict(link, SdmClient.KUMO_RELATIONAL).items(); - assertEquals(2, linked.size()); - assertNotNull(linked.get(0).rankings()); - assertTrue(linked.get(0).rankings().get(0).containsKey("score")); - assertThrows( - UnsupportedOperationException.class, - () -> linked.get(0).rankings().get(0).put("score", 0)); + StructuredDataException refused = + assertThrows( + StructuredDataException.class, + () -> + PredictiveQuery.parse(RelationalTest.LINK) + .materialize(schema, RelationalTest.options(1L, 2L).build())); + assertEquals("UNSUPPORTED_REQUEST", refused.code()); Request forecast = PredictiveQuery.parse(RelationalTest.FORECAST) .materialize(schema, RelationalTest.options(1L, 2L).build()); diff --git a/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py b/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py index 3e7fe043c..c1f285b0e 100644 --- a/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py +++ b/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py @@ -340,7 +340,7 @@ def _text_schema() -> Schema: ).materialize(indices=[0, 1]), model=KumoRelational(), ), - 'UNSUPPORTED_TASK', + 'UNSUPPORTED_REQUEST', ), ( 'kumo-tabular',