From c8dce98b6ba7890ae63720862b2ce69a54604894 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 14:47:31 -0800 Subject: [PATCH 1/6] Hold every native error code to its exact Java message The error table checked the message of 2 of its 23 codes, so the other 21 sentences, and the three with placeholders, could be reworded or filled wrong without failing. Give each row the message the sentence's template says, with valid fields for the endpoint refusal, the reserved column name and the unsupported Arrow type, and compare it whole. Seeded fault: Java's `!r` conversion rendered as plain text drops the quotes around the endpoint, the suggested name and the column; the three placeholder rows fail, which the old two-row checks did not see. Run with mvn -Dtest=BindingTest on JDK 21 over a test-ops sdm-jni build. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- .../test/java/com/nvidia/sdm/BindingTest.java | 98 ++++++++++++------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/java/src/test/java/com/nvidia/sdm/BindingTest.java b/java/src/test/java/com/nvidia/sdm/BindingTest.java index fda7b4a19..80bb62bbd 100644 --- a/java/src/test/java/com/nvidia/sdm/BindingTest.java +++ b/java/src/test/java/com/nvidia/sdm/BindingTest.java @@ -196,38 +196,77 @@ void largeWarningsUseAttachmentsAndPreserveOrderUntilDrained() throws Exception } finally { Names.close(run); } } - /** Authored code/kind/sentence inputs cover each supported native error mapping. */ + private static Map text(String value) { + return Map.of("t", "utf8", "v", value); + } + + /** Each native error code reaches Java with its code and its sentence's exact words. */ @TestFactory Stream everyErrorCodeReachesJavaWithItsCode() { String[][] errors = { - new String[]{"ENDPOINT_REFUSED", "model_endpoint", "serving_refused"}, - new String[]{"INTERNAL_ERROR", "internal", "result_merge_empty"}, - new String[]{"INVALID_CONFIGURATION", "invalid_request", "invalid_configuration"}, - new String[]{"INVALID_MESSAGE", "protocol", "invalid_message"}, - new String[]{"INVALID_REQUEST", "invalid_request", "direct_empty"}, - new String[]{"INVALID_RESPONSE", "model_endpoint", "tabular_missing_row_index"}, - new String[]{"INVALID_SCHEMA", "invalid_request", "wire_reserved_one"}, - new String[]{"MISSING_EXTRA", "unsupported", "missing_extra"}, - new String[]{"PROVIDER_ERROR", "not_found", "provider_failed"}, - new String[]{"PROVIDER_SCHEMA_MISMATCH", "data_source", "provider_schema_mismatch"}, - new String[]{"REPLY_MISSING", "protocol", "reply_missing"}, - new String[]{"RUN_BUSY", "protocol", "run_busy"}, - new String[]{"RUN_CANCELLED", "cancelled", "run_cancelled"}, - new String[]{"RUN_FINISHED", "protocol", "run_finished"}, - new String[]{"RUN_POISONED", "protocol", "run_poisoned"}, - new String[]{"SESSIONS_UNSUPPORTED", "unsupported", "sessions_unsupported"}, - new String[]{"TRANSPORT_ERROR", "model_endpoint", "transport_failed"}, - new String[]{"UNKNOWN_MODEL", "not_found", "unknown_model"}, - new String[]{"UNKNOWN_REPLY", "protocol", "unknown_reply"}, - new String[]{"UNSUPPORTED_REQUEST", "unsupported", "direct_relational_frames"}, - new String[]{"UNSUPPORTED_TYPE", "unsupported", "unsupported_type"}, - new String[]{"VALUE_OUT_OF_RANGE", "unsupported", "lag_past_times"}, - new String[]{"WRONG_REPLY", "protocol", "wrong_reply"} + new String[]{"ENDPOINT_REFUSED", "model_endpoint", "serving_refused", + "serving endpoint \"fraud-scorer\" refused the request (UNSUPPORTED_TASK)"}, + new String[]{"INTERNAL_ERROR", "internal", "result_merge_empty", "no parts to merge"}, + new String[]{"INVALID_CONFIGURATION", "invalid_request", "invalid_configuration", + "the configuration of this call is not one the core accepts"}, + new String[]{"INVALID_MESSAGE", "protocol", "invalid_message", + "the host sent a message the core cannot read"}, + new String[]{"INVALID_REQUEST", "invalid_request", "direct_empty", + "context has no rows, so there is nothing for the model to learn the answer from"}, + new String[]{"INVALID_RESPONSE", "model_endpoint", "tabular_missing_row_index", + "kumo-tabular returned a response that does not match the contract: " + + "PredictionItem response is missing required field 'row_index'"}, + new String[]{"INVALID_SCHEMA", "invalid_request", "wire_reserved_one", + "orders.instance_id uses a name this client needs for the request it builds, so it cannot " + + "also be a column of the data. Rename it, for instance to \"instance_id_from_the_data\"."}, + new String[]{"MISSING_EXTRA", "unsupported", "missing_extra", + "an optional dependency this call needs is not installed"}, + new String[]{"PROVIDER_ERROR", "not_found", "provider_failed", "the data source failed"}, + new String[]{"PROVIDER_SCHEMA_MISMATCH", "data_source", "provider_schema_mismatch", + "the data source answered with columns other than the ones asked for"}, + new String[]{"REPLY_MISSING", "protocol", "reply_missing", + "the host went on before answering every request"}, + new String[]{"RUN_BUSY", "protocol", "run_busy", + "the run was already in a call; a run takes one call at a time"}, + new String[]{"RUN_CANCELLED", "cancelled", "run_cancelled", "the run was cancelled"}, + new String[]{"RUN_FINISHED", "protocol", "run_finished", "the run had already finished"}, + new String[]{"RUN_POISONED", "protocol", "run_poisoned", + "the run failed earlier and can no longer be used"}, + new String[]{"SESSIONS_UNSUPPORTED", "unsupported", "sessions_unsupported", + "the model endpoint does not support sessions"}, + new String[]{"TRANSPORT_ERROR", "model_endpoint", "transport_failed", + "the transport to the model endpoint failed: the endpoint could not be reached, " + + "or its answer could not be read"}, + new String[]{"UNKNOWN_MODEL", "not_found", "unknown_model", "the model named does not exist"}, + new String[]{"UNKNOWN_REPLY", "protocol", "unknown_reply", + "the host answered a request the core did not make"}, + new String[]{"UNSUPPORTED_REQUEST", "unsupported", "direct_relational_frames", + "a relational request has to be gathered first, and this one was not. Build it with " + + "PredictiveQuery(...).materialize(indices=[...], model='kumo-relational'). A schema of " + + "one table is gathered for the tabular model unless the relational one is named, rather " + + "than with Request(context=..., predict=...), which is the single-table form"}, + new String[]{"UNSUPPORTED_TYPE", "unsupported", "unsupported_type", + "column \"tags\" has the Arrow type Float16, which the core does not take"}, + new String[]{"VALUE_OUT_OF_RANGE", "unsupported", "lag_past_times", + "lag_timesteps reaches further back than a time can be held (before 1677); ask for fewer steps"}, + new String[]{"WRONG_REPLY", "protocol", "wrong_reply", + "the host answered a request with the wrong kind of reply"} }; + Map> fields = Map.of( + "ENDPOINT_REFUSED", Map.of( + "endpoint", text("fraud-scorer"), "endpoint_code", text("UNSUPPORTED_TASK")), + "INVALID_SCHEMA", Map.of( + "reserved", List.of(text("orders.instance_id")), + "renamed", List.of(text("instance_id_from_the_data"))), + "UNSUPPORTED_TYPE", Map.of("column", text("tags"), "arrow_type", text("Float16"))); return Stream.of(errors).map(error -> DynamicTest.dynamicTest(error[0], () -> { - byte[] input = com.nvidia.sdm.internal.Json.write(Map.of( + Map given = new LinkedHashMap<>(Map.of( "kind", error[1], "code", error[0], "sentence", error[2], "message", "native-free-text-sentinel", "retryable", false)); + if (fields.containsKey(error[0])) { + given.put("fields", fields.get(error[0])); + } + byte[] input = com.nvidia.sdm.internal.Json.write(given); Object run = Names.startRun( "{\"op\":\"test\",\"test\":\"fail\",\"error\":1}", Map.of(1, input)); try { @@ -236,14 +275,7 @@ Stream everyErrorCodeReachesJavaWithItsCode() { } catch (Exception raised) { assertTrue(Names.isStructuredDataException(raised)); assertEquals(error[0], Names.code(raised)); - assertTrue(raised.getMessage() != null && !raised.getMessage().isBlank()); - assertTrue(!raised.getMessage().contains("native-free-text-sentinel")); - if (error[0].equals("INVALID_REQUEST")) { - assertEquals("context has no rows, so there is nothing for the model to learn the answer from", raised.getMessage()); - } - if (error[0].equals("RUN_BUSY")) { - assertEquals("the run was already in a call; a run takes one call at a time", raised.getMessage()); - } + assertEquals(error[3], raised.getMessage(), error[0]); } finally { Names.close(run); } From 21e7bce0f4cc07e57740d8947fb156b81db76588 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 14:49:45 -0800 Subject: [PATCH 2/6] Pin PQL error spans and the packed-date refusal The corpus rewrite kept each refusal's codes but not where it points, so an unknown name, an aggregation or type error, or a task error could move its span without failing; the fuzz test counted structured errors without naming one. Assert code and span for selected unknown names, aggregation and type errors and task errors, the spans the recorded corpus gave, and assert that the authored five-digit date seed is refused as `date_literal_packed` with its literal. The fuzz proportions stay unpinned. Seeded faults: collapsing each diagnostic span to its start column fails the span test on SUM(orders.status, ...), (1,8,1,38); answering the packed literal with an internal error fails the fuzz control while its counts still pass. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/crates/sdm-core/tests/pql_corpus.rs | 95 ++++++++++++++++++++++++ rust/crates/sdm-core/tests/pql_fuzz.rs | 24 ++++++ 2 files changed, 119 insertions(+) diff --git a/rust/crates/sdm-core/tests/pql_corpus.rs b/rust/crates/sdm-core/tests/pql_corpus.rs index e32fed7ba..bf3e5839b 100644 --- a/rust/crates/sdm-core/tests/pql_corpus.rs +++ b/rust/crates/sdm-core/tests/pql_corpus.rs @@ -444,6 +444,101 @@ fn diagnostic_rules_return_their_named_errors() { } } +#[test] +fn selected_errors_point_at_what_they_name() { + use sdm_contract::ops::Span; + let at = |start_col, end_col| { + Some(Span { + start_row: 1, + start_col, + end_row: 1, + end_col, + }) + }; + type Errors = Vec<(&'static str, Option)>; + let cases: Vec<(&str, &str, &str, Errors)> = vec![ + ( + "schema-1", + "RELATIONAL", + "PREDICT users.nope FOR EACH users.user_id", + vec![("PQL_SCHEMA_UNKNOWN_COLUMN", at(8, 8))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(order.*, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_SCHEMA_UNKNOWN_TABLE", at(14, 14))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.id", + vec![("PQL_SCHEMA_UNKNOWN_COLUMN", at(46, 46))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.status, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_AGGREGATION_NEEDS_NUMERIC", at(8, 38))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.product_id, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_AGGREGATION_ON_KEY_COLUMN", at(8, 42))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(orders.* WHERE orders.amount > 'ten', 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_CONSTANT_MISMATCH", at(29, 45))], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT COUNT(orders.* WHERE orders.quantity > true, 0, 30, days) FOR EACH users.user_id", + vec![ + ("PQL_TYPE_CONSTANT_MISMATCH", at(29, 47)), + ("PQL_TYPE_BOOL_COMPARISON", at(29, 47)), + ], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.quantity, 0, 30, days) FORECAST 0 TIMEFRAMES FOR EACH users.user_id", + vec![("PQL_TASK_FORECAST_NOT_POSITIVE", None)], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT LIST_DISTINCT(orders.product_id, 0, 30, days) RANK TOP 0 FOR EACH users.user_id", + vec![("PQL_TASK_TOP_K_NOT_POSITIVE", None)], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT LIST_DISTINCT(orders.product_id, 0, 30, days) RANK FOR EACH users.user_id", + vec![("PQL_TASK_RANK_REQUIRES_TOP_K", at(68, 68))], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT products.tags FOR EACH products.product_id", + vec![("PQL_TASK_PROBLEM_TYPE_REQUIRED", at(31, 31))], + ), + ]; + for (which, model, text, expected) in cases { + let got = resolve(text, &schema(which), &profile(model), &facts(), &NoZones).unwrap(); + let errors: Vec<_> = got + .diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .map(|d| (d.code.as_str(), d.span)) + .collect(); + assert_eq!(errors, expected, "{text}"); + } +} + fn nodes(node: &sdm_contract::pql::PqlNode) -> Vec<&sdm_contract::pql::NodeKind> { use sdm_contract::pql::NodeKind::*; let mut out = vec![&node.kind]; diff --git a/rust/crates/sdm-core/tests/pql_fuzz.rs b/rust/crates/sdm-core/tests/pql_fuzz.rs index cf5597e09..7d15543f8 100644 --- a/rust/crates/sdm-core/tests/pql_fuzz.rs +++ b/rust/crates/sdm-core/tests/pql_fuzz.rs @@ -21,6 +21,7 @@ mod pql_support; use std::time::{Duration, Instant}; use pql_support::{NoZones, facts, profile, schema}; +use sdm_contract::{Detail, ErrorCode, Sentence}; use sdm_core::pql::validate; /// The seed and the number of inputs of each kind. @@ -230,6 +231,29 @@ fn the_parser_answers_every_fuzzed_input_without_a_panic() { accepted ); } + // A run of five digits is a packed date (D8), refused by name as a + // structured error rather than read or given a diagnostic. + let (packed, schema_name, profile_name) = seeds[7]; + let error = validate( + packed, + &schema(schema_name), + &profile(profile_name), + &facts(), + &NoZones, + ) + .expect_err(packed); + assert_eq!( + (error.code, error.sentence), + (ErrorCode::InvalidRequest, Sentence::DateLiteralPacked) + ); + assert_eq!( + error.fields.get("rule"), + Some(&Detail::text("date_literal_packed")) + ); + assert_eq!( + error.fields.get("literal"), + Some(&Detail::text("12345-1-1")) + ); let mut rng = Rng(SEED); // Each input with the schema and profile it is validated with: a // mutated case keeps its own; the rest take the first schema, which From 36f45785f58fe17a8c0e560e2dbdcbf9e221e57f Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 14:53:28 -0800 Subject: [PATCH 3/6] Scan the Python-held DuckDB setups for their time zone Deleting the vector files left the zone guard reading 89 setups where it read 473, and the setups now held in Python (the decimal, empty-null, value, ranking and error-wording scenarios, the wording probes among them) were read by nothing. Enumerate them beside the vector files, hold each to the allowed zone forms, and require a global pin wherever a setup holds zoned timestamps. provider-vectors.json's global and session zones are read with its setup, so that intentional pair still passes. Seeded faults: the wording probes' pin turned into a session SET, and the empty-null null.time-timestamptz pin removed; each fails by name. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- tests/test_contract_cases.py | 109 ++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/tests/test_contract_cases.py b/tests/test_contract_cases.py index b342bc0a1..255689f9b 100644 --- a/tests/test_contract_cases.py +++ b/tests/test_contract_cases.py @@ -11,6 +11,7 @@ import importlib.util import json import re +import sys from decimal import Decimal from pathlib import Path from types import ModuleType @@ -133,6 +134,23 @@ def test_a_case_sets_its_time_zone_for_every_cursor() -> None: _VECTOR_SETUP_KEYS = ('setup', 'witness') +#: Zone statements a vector holds beside its setup, which its readers run +#: on the setup's database: provider-vectors.json's global zone and the +#: session zone it shows the source does not read in. +_VECTOR_ZONE_KEYS = ('zone', 'session_zone') + + +#: The DuckDB scenario setups held in Python, by module and the name it +#: holds them under; ranking's are those naming no other database. +_SCENARIO_SETUPS = ( + ('decimal_vectors', 'VECTORS'), + ('duckdb_value_scenarios', 'SOURCES'), + ('empty_null_vectors', 'VECTORS'), + ('error_wording_scenarios', 'PROBES'), + ('ranking_scenarios', 'VECTORS'), +) + + _VECTOR_ZONE_PENDING: set[str] = set() @@ -168,11 +186,22 @@ def _vector_setups(node: Any, where: str) -> list[tuple[str, list[str]]]: found: list[tuple[str, list[str]]] = [] if isinstance(node, dict): where = str(node.get('id', where)) + zones = [ + node[key] + for key in _VECTOR_ZONE_KEYS + if isinstance(node.get(key), str) + ] for key, value in node.items(): if key in _VECTOR_SETUP_KEYS and isinstance(value, (str, list)): statements = [value] if isinstance(value, str) else value found.append( - (where, [s for s in statements if isinstance(s, str)]) + ( + where, + [ + *zones, + *(s for s in statements if isinstance(s, str)), + ], + ) ) found += _vector_setups(value, where) elif isinstance(node, list): @@ -181,20 +210,78 @@ def _vector_setups(node: Any, where: str) -> list[tuple[str, list[str]]]: return found +def _support(stem: str) -> ModuleType: + if stem not in sys.modules: + path = ROOT / 'contract/support' / f'{stem}.py' + spec = importlib.util.spec_from_file_location(stem, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[stem] = module + spec.loader.exec_module(module) + return sys.modules[stem] + + +def _scenario_setups() -> list[tuple[str, list[str]]]: + r"""Each DuckDB scenario setup held in Python (:data:`_SCENARIO_SETUPS` + and the giant-value sources), named by module and scenario.""" + found: list[tuple[str, list[str]]] = [] + for stem, name in _SCENARIO_SETUPS: + held = getattr(_support(stem), name) + entries = held.items() if isinstance(held, dict) else enumerate(held) + for key, entry in entries: + if isinstance(entry, list): + found.append((f'{stem} {key}', entry)) + elif entry.get('database', 'duckdb') == 'duckdb': + where = str(entry.get('name', key)) + found += [ + (f'{stem} {each}', setup) + for each, setup in _vector_setups(entry, where) + ] + values = _support('duckdb_value_scenarios') + found += [ + ( + f'duckdb_value_scenarios big.{kind}.{mode}', + values.big_setup(kind, mode), + ) + for kind in values.BIG_KINDS + for mode in values.BIG_MODES + ] + return found + + def test_a_vector_sets_its_time_zone_for_every_cursor() -> None: # Setup and source reads use separate cursors. Set the database-wide # timezone so results do not depend on each cursor's process-local zone. + # The vector files and the DuckDB scenarios held in Python both name it + # only in the allowed forms, and a setup holding zoned timestamps pins + # it globally. + setups = [ + (f'{path.name} {vector}', setup) + for path in sorted((ROOT / 'contract').glob('*-vectors.json')) + for vector, setup in _vector_setups( + json.loads(path.read_text(encoding='utf-8')), '?' + ) + ] + scenarios = _scenario_setups() problems = set() named = 0 - for path in sorted((ROOT / 'contract').glob('*-vectors.json')): - document = json.loads(path.read_text(encoding='utf-8')) - for vector, setup in _vector_setups(document, '?'): - named += sum(bool(_ZONE.search(each)) for each in setup) - problems.update( - _vector_zone_problems(f'{path.name} {vector}', setup) - ) + for where, setup in [*setups, *scenarios]: + named += sum(bool(_ZONE.search(each)) for each in setup) + problems.update(_vector_zone_problems(where, setup)) + problems.update(_unzoned(where, setup)) assert problems == _VECTOR_ZONE_PENDING assert named > len(_VECTOR_ZONE_PENDING), 'no vector sets its zone' + # Not vacuous: the probes and the zoned empty-null scenarios are read. + zoned = { + where + for where, setup in scenarios + if any(_ZONED.search(each) for each in setup) + } + assert { + 'error_wording_scenarios infinity.timestamptz.positive', + 'empty_null_vectors null.time-timestamptz', + } <= zoned, sorted(zoned) + assert len(scenarios) > 300, len(scenarios) # Seeded: a session SET passes only beside a global one, in the same # setup; nothing else does. pinned = "SET GLOBAL TimeZone = 'UTC'" @@ -214,6 +301,12 @@ def test_a_vector_sets_its_time_zone_for_every_cursor() -> None: [pinned], ): assert not _vector_zone_problems('seed', setup), setup + # Seeded: zoned timestamps need the global pin; a session one alone, + # or none, fails. + zoned_table = "CREATE TABLE t AS SELECT TIMESTAMPTZ '2024-01-01' AS ts" + assert _unzoned('seed', [zoned_table]) + assert _unzoned('seed', ["SET TimeZone = 'UTC'", zoned_table]) + assert not _unzoned('seed', [pinned, zoned_table]) def test_every_committed_case_validates_with_a_unique_id() -> None: From 63f497b598ae8ccb414a9eea50c7f7bb7c4e9303 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 15:06:56 -0800 Subject: [PATCH 4/6] Hold PostgreSQL keys past int64 to their exact wire text key.uint64-above-int64-max@postgres now succeeds, reading NUMERIC(20,0) keys exactly and sending them as strings where 1.0.0 raised OverflowError in Schema.from_source, but only its outcome was checked. Assert the adjacent keys 2**63 and 2**63 + 1 reach the endpoint as two exact strings in the instance table and in the users and orders rows, and record the fix in the changelog. Seeded fault: key_dtype choosing float64 for a decimal key past int64, which can make adjacent keys one, still succeeds but fails here on the entity reference's dtype. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- CHANGELOG.md | 3 + contract/tests/test_exact_numeric_keys.py | 79 +++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4716abd3e..79adfd6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,6 +180,9 @@ The public API snapshot records this version update; publication remains pending - On Databricks, through Spark or a SQL warehouse, a `DECIMAL` value with a fraction is sent exactly, such as `997.67`. 1.0.0 sent it through a float, such as `997.6700000000001`. +- On PostgreSQL, a whole-number `NUMERIC` key above `2**63 - 1` is kept + exactly and sent as a string. 1.0.0 raised `OverflowError` in + `Schema.from_source`. ### Removed diff --git a/contract/tests/test_exact_numeric_keys.py b/contract/tests/test_exact_numeric_keys.py index a3d3242e6..5e587ff13 100644 --- a/contract/tests/test_exact_numeric_keys.py +++ b/contract/tests/test_exact_numeric_keys.py @@ -8,8 +8,12 @@ import functools import importlib.util +import json import os +import shutil +import subprocess import sys +import tempfile from decimal import Decimal from pathlib import Path from types import ModuleType @@ -38,6 +42,8 @@ CASE_IDS = (GIVEN_KEY, GIVEN_LINK, EDITED_LINK, CATALOG) +WIDE = 'key.uint64-above-int64-max' + LINK = { 'source_table': 'orders', 'source_column': 'user_id', @@ -386,3 +392,76 @@ def test_each_case_holds_an_exact_key_past_2_53(case_id: str) -> None: ), }[case_id] assert (given['from_source'], given.get('edits')) == expected + + +@functools.cache +def wide_on_postgres() -> dict[str, Any]: + r"""python-core's observation of the wide-key case on PostgreSQL.""" + with tempfile.TemporaryDirectory(prefix='exact_numeric_keys-') as tmp: + given = Path(tmp) / 'cases' + given.mkdir() + shutil.copy(CASES / f'{WIDE}.json', given) + out = Path(tmp) / 'python-core.jsonl' + done = subprocess.run( + [ + sys.executable, + '-I', + str(RUNNER), + '--mode', + 'python-core', + '--database', + 'postgres', + '--out', + str(out), + str(given), + ], + capture_output=True, + text=True, + cwd=ROOT, + stdin=subprocess.DEVNULL, + check=False, + timeout=600, + ) + assert done.returncode == 0, done.stdout + done.stderr + return json.loads(out.read_text(encoding='utf-8')) + + +def test_adjacent_keys_past_int64_reach_the_wire_as_exact_strings() -> None: + r"""PostgreSQL's whole-number ``NUMERIC`` keys past ``2**63 - 1``, where + 1.0.0 raised ``OverflowError``: the adjacent 2**63 and 2**63 + 1 the + case asks for are sent as their exact text, two keys, in the instance + table and in each related table, where a double would make them one.""" + _server() + line = wide_on_postgres() + assert line['outcome'] == { + 'result': 'success', + 'stage': 'predict', + 'error': None, + 'code': None, + } + asked = [str(2**63), str(2**63 + 1)] + case, _ = runner().load_cases([CASES / f'{WIDE}.json'])[0] + assert [str(i) for i in case['input']['materialize']['indices']] == asked + (payload,) = [each['parsed'] for each in line['public']['payloads']] + schema = payload['schema'] + assert schema['instance_table']['columns']['__kumo_entity_ref_0'] == { + 'dtype': 'string', + 'stype': 'ID', + } + for name in ('users', 'orders'): + column = schema['related_tables'][name]['columns']['user_id'] + assert column['dtype'] == 'string', name + predict = payload['predict'] + table = predict['instance_table'] + at = table['columns'].index('__kumo_entity_ref_0') + keys = {row[0]['number']: row[at] for row in table['rows']} + assert list(keys.values()) == asked + users = predict['related_tables']['users'] + user_at = users['columns'].index('user_id') + assert [row[user_at] for row in users['rows']] == asked + orders = predict['related_tables']['orders'] + order_at = orders['columns'].index('user_id') + assert orders['rows'] + assert {row[order_at] for row in orders['rows']} == set(asked) + for row in orders['rows']: + assert row[order_at] == keys[row[0]['number']] From 2c99d97028002f7e16a2ce498430872378f6b469 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 17:08:46 -0800 Subject: [PATCH 5/6] Derive each relational request's context, labels and related rows The rewrite dropped whole-request comparison, so a request whose labels were constant or swapped, whose related table went missing, or whose later batch carried the wrong context still passed while the scripted endpoint answered it. Add contract/support/shop_requests.py, which derives from a shop case's authored inputs each call's context entities and anchors (the entity filter and ASSUMING applied at each anchor), its labels (the target window, a forecast's step, a ranking candidate's membership, one-class candidates not asked), the entities it scores and every related table's rows per instance, and assert it for every query-option and ranking case that succeeds. Assert the three seeded calls of a tiny five-user fixture as whole requests, request ids aside. Seeded faults, on the real requests: every label set to 0.0, two labels swapped, the orders table dropped, the second batch given a short context, and a ranking part sent the first part's labels; each fails by name. The tiny fixture fails with its labels made constant. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- contract/support/shop_requests.py | 345 +++++++++++++++++++++ contract/tests/test_prediction_batching.py | 134 ++++++++ contract/tests/test_query_options.py | 9 + contract/tests/test_ranking.py | 8 + 4 files changed, 496 insertions(+) create mode 100644 contract/support/shop_requests.py diff --git a/contract/support/shop_requests.py b/contract/support/shop_requests.py new file mode 100644 index 000000000..40fa089ab --- /dev/null +++ b/contract/support/shop_requests.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The relational requests a shop case should send, derived from its authored inputs. + +Each call's context is every user the entity filter and ASSUMING admit at +each anchor, walking back one target window at a time from the anchor the +case gives while there is data, labelled by the target the call asks; its +related tables hold each instance's user and the facts up to its anchor. +Nothing here reads a recorded request. +""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta +from typing import Any + +#: A window over orders: an aggregation and its days. +_SUM = re.compile(r'^PREDICT SUM\(orders\.price, (-?\d+), (-?\d+), days\)') +_ANY = re.compile(r'^PREDICT COUNT\(orders\.\*, (-?\d+), (-?\d+), days\) > 0') +_RANK = re.compile( + r'^PREDICT LIST_DISTINCT\(buys\.item_id, (-?\d+), (-?\d+), days\) RANK ' +) +_FORECAST = re.compile(r' FORECAST (\d+) TIMEFRAMES ') +_COUNT_ORDERS = re.compile(r'^COUNT\(orders\.\*, (-?\d+), (-?\d+), days\) > 0$') +_STATIC = re.compile(r"^users\.(\w+) (>|<|=) ('?[\w ]+'?)$") +#: The rows sent for a call, at most. +_ROWS_PER_CALL = 10_000 + + +def _stamp(value: datetime) -> str: + return value.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +class Shop: + r"""A case's database read once: users, orders and, where its setup + makes them, buys and items.""" + + def __init__(self, case: dict[str, Any], bases: dict[str, list[str]]): + import duckdb + + connection = duckdb.connect() + try: + for statement in [ + *bases[case['input']['base']], + *case['input'].get('setup', []), + ]: + connection.execute(statement) + tables = { + row[0] + for row in connection.execute( + 'SELECT table_name FROM duckdb_tables()' + ).fetchall() + } + self.tables = tables + self.users = { + row[0]: row[1:] + for row in connection.execute( + 'SELECT user_id, age, country FROM users' + ).fetchall() + } + self.orders = connection.execute( + 'SELECT user_id, price, ts FROM orders' + ).fetchall() + self.buys = ( + connection.execute( + 'SELECT user_id, item_id, ts FROM buys' + ).fetchall() + if 'buys' in tables + else [] + ) + self.items = ( + dict( + connection.execute( + 'SELECT item_id, kind FROM items' + ).fetchall() + ) + if 'items' in tables + else {} + ) + self._static = connection + self.earliest = min(ts for *_, ts in [*self.orders, *self.buys]) + except BaseException: + connection.close() + raise + + def close(self) -> None: + self._static.close() + + def admitted(self, condition: str | None, at: datetime) -> set[int]: + r"""The users ``condition`` admits at ``at``: a column of the user + compared with a literal, or a count of orders in a window of it.""" + if condition is None: + return set(self.users) + window = _COUNT_ORDERS.match(condition) + if window: + start, end = (int(n) for n in window.groups()) + return { + user + for user, _, ts in self.orders + if at + timedelta(days=start) < ts <= at + timedelta(days=end) + } + static = _STATIC.match(condition) + assert static, f'no rule derives the users {condition!r} admits' + column, operator, literal = static.groups() + return { + row[0] + for row in self._static.execute( + f'SELECT user_id FROM users WHERE {column} {operator} {literal}' + ).fetchall() + } + + def orders_until(self, user: int, at: datetime) -> list[tuple[Any, ...]]: + return sorted( + (price, _stamp(ts), user) + for owner, price, ts in self.orders + if owner == user and ts <= at + ) + + def buys_until(self, user: int, at: datetime) -> list[tuple[Any, ...]]: + return sorted( + (_stamp(ts), user, self.items[item]) + for owner, item, ts in self.buys + if owner == user and ts <= at + ) + + +def _target(pql: str) -> tuple[str, int, int]: + body = re.sub(r'^(EVALUATE|EXPLAIN) ', '', pql) + for kind, pattern in (('sum', _SUM), ('any', _ANY), ('member', _RANK)): + found = pattern.match(body) + if found: + start, end = (int(n) for n in found.groups()) + return kind, start, end + raise AssertionError(f'no rule derives the labels of {pql!r}') + + +def _clauses(pql: str) -> tuple[str | None, str | None]: + r"""The entity filter and the ASSUMING condition ``pql`` gives.""" + assuming = None + if ' ASSUMING ' in pql: + pql, assuming = pql.split(' ASSUMING ', 1) + _, _, condition = pql.partition(' FOR EACH users.user_id WHERE ') + return condition or None, assuming + + +def _label( + shop: Shop, + kind: str, + user: int, + at: datetime, + window: tuple[int, int], + candidate: Any, +) -> Any: + low, high = (at + timedelta(days=d) for d in window) + if kind == 'sum': + return float( + sum( + price + for owner, price, ts in shop.orders + if owner == user and low < ts <= high + ) + ) + if kind == 'any': + return any( + owner == user and low < ts <= high for owner, _, ts in shop.orders + ) + return any( + owner == user and item == candidate and low < ts <= high + for owner, item, ts in shop.buys + ) + + +def expected_calls( + case: dict[str, Any], bases: dict[str, list[str]] +) -> list[dict[str, Any]]: + r"""Each call the case should make, in order: its context, by anchor + and entity, each with its label; the entities it scores; the anchor + they are scored at; and its related tables. A ranking is asked one + candidate at a time, each candidate once, and one whose context holds + one class is not asked; a forecast is asked a step at a time.""" + given = case['input'] + spec = given['materialize'] + anchor = datetime.fromisoformat(spec['anchor_time']) + kind, start, end = _target(given['pql']) + condition, assuming = _clauses(given['pql']) + steps = _FORECAST.search(given['pql']) + parts: list[tuple[Any, tuple[int, int]]] + if kind == 'member': + parts = [(c, (start, end)) for c in dict.fromkeys(spec['candidates'])] + elif steps: + width = end - start + parts = [ + (None, (start + width * k, end + width * k)) + for k in range(int(steps.group(1))) + ] + else: + parts = [(None, (start, end))] + shop = Shop(case, bases) + try: + calls = [] + for candidate, window in parts: + width = window[1] - window[0] + at = anchor - timedelta(days=window[1]) + context = {} + while at >= shop.earliest: + users = shop.admitted(condition, at) & shop.admitted( + assuming, at + ) + for user in users: + context[(_stamp(at), user)] = _label( + shop, kind, user, at, window, candidate + ) + at -= timedelta(days=width) + if kind == 'member' and len(set(context.values())) < 2: + continue + batch = spec.get('rows_per_call') or _ROWS_PER_CALL + indices = spec['indices'] + calls += [ + { + 'context': context, + 'predict': indices[first : first + batch], + 'anchor': _stamp(anchor), + 'related': related(shop), + } + for first in range(0, len(indices), batch) + ] + return calls + finally: + shop.close() + + +def related(shop: Shop) -> dict[str, Any]: + r"""What each related table holds for an instance of ``user`` at + ``at``, by table.""" + out: dict[str, Any] = { + 'users': lambda user, at: [(user, *shop.users[user])], + 'orders': shop.orders_until, + } + if 'buys' in shop.tables: + out['buys'] = shop.buys_until + out['items'] = lambda user, at: sorted( + {kind for *_, kind in shop.buys_until(user, at)} + ) + return out + + +def plain(value: Any) -> Any: + r"""A parsed request's value with each number as the number it + writes (the runner keeps a number as ``{"number": }``).""" + if isinstance(value, dict): + if set(value) == {'number'} and isinstance(value['number'], str): + text = value['number'] + return float(text) if any(c in text for c in '.eE') else int(text) + return {name: plain(item) for name, item in value.items()} + if isinstance(value, list): + return [plain(item) for item in value] + return value + + +def _rows(table: dict[str, Any]) -> list[dict[str, Any]]: + return [ + dict(zip(table['columns'], row, strict=True)) for row in table['rows'] + ] + + +def assert_sent( + case: dict[str, Any], bases: dict[str, list[str]], sent: list[Any] +) -> None: + r"""Each request in ``sent`` (parsed, numbers as numbers) is the call + :func:`expected_calls` derives for ``case``: its context's entities, + anchors and labels, the entities it scores, and every related table's + rows for each instance.""" + wanted = expected_calls(case, bases) + sent = [plain(body) for body in sent] + assert len(sent) == len(wanted), (case['id'], len(sent), len(wanted)) + for call, (body, want) in enumerate(zip(sent, wanted, strict=True)): + where = f'{case["id"]} call {call}' + target = body['task']['target']['column_name'] + context = _rows(body['context']['instance_table']) + predict = _rows(body['predict']['instance_table']) + got = { + (row['__kumo_anchor_time'], row['__kumo_entity_ref_0']): row[target] + for row in context + } + assert len(got) == len(context), f'{where}: an instance twice' + missing = sorted(set(want['context']) - set(got)) + extra = sorted(set(got) - set(want['context'])) + assert not missing and not extra, ( + f'{where}: context lacks {missing[:4]} and holds {extra[:4]}' + ) + wrong = [ + (each, got[each], label) + for each, label in sorted(want['context'].items()) + if got[each] != label or type(got[each]) is not type(label) + ] + assert not wrong, f'{where}: labels (sent, derived): {wrong[:4]}' + assert [row['__kumo_entity_ref_0'] for row in predict] == ( + want['predict'] + ), f'{where}: scores the wrong entities' + assert {row['__kumo_anchor_time'] for row in predict} == { + want['anchor'] + }, f'{where}: scored at the wrong anchor' + assert len({row['instance_id'] for row in [*context, *predict]}) == ( + len(context) + len(predict) + ), f'{where}: an instance id twice' + for half, instances in (('context', context), ('predict', predict)): + tables = body[half]['related_tables'] + assert set(tables) == set(want['related']), ( + f'{where}: {half} related tables {sorted(tables)}' + ) + held = {name: _rows(table) for name, table in tables.items()} + for instance in instances: + user = instance['__kumo_entity_ref_0'] + at = datetime.fromisoformat(instance['__kumo_anchor_time'][:-1]) + for name, rule in want['related'].items(): + rows = [ + row + for row in held[name] + if row['instance_id'] == instance['instance_id'] + ] + assert _related(name, rows, held) == rule(user, at), ( + f'{where}: {half} {name} of {instance}' + ) + + +def _related( + name: str, rows: list[dict[str, Any]], held: dict[str, Any] +) -> list[Any]: + if name == 'users': + return [(r['user_id'], r['age'], r['country']) for r in rows] + if name == 'orders': + return sorted((r['price'], r['ts'], r['user_id']) for r in rows) + if name == 'items': + return sorted(r['kind'] for r in rows) + kinds = { + (r['instance_id'], r['__node_id']): r['kind'] for r in held['items'] + } + return sorted( + (r['ts'], r['user_id'], kinds[(r['instance_id'], r['item_id'])]) + for r in rows + ) diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 9af000863..e7cede99e 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -19,6 +19,7 @@ import subprocess import sys import tempfile +import warnings from pathlib import Path from types import ModuleType from typing import Any @@ -590,3 +591,136 @@ def test_python_core_gathers_and_predicts_what_the_reference_refused( case is answered.""" got = core_carried(case_id) assert got['outcome']['result'] == 'success', got['outcome'] + + +#: A tiny seeded three-call fixture: five users scored ``user_id % 3``, +#: three of them asked one per call, sampled with a fixed seed. +TINY = { + 'base': 'shop', + 'setup': [ + 'DROP TABLE orders', + 'DROP TABLE users', + 'CREATE TABLE users AS SELECT i::BIGINT AS user_id, ' + '(i % 3)::DOUBLE AS score FROM range(5) t(i)', + ], + 'pql': 'PREDICT users.score FOR EACH users.user_id', + 'materialize': {'indices': [0, 1, 2], 'rows_per_call': 1, 'random_seed': 7}, +} +#: The users in the order the seed samples them, each at its instance id. +TINY_SAMPLE = [3, 1, 2, 4, 0] + + +def test_three_seeded_calls_send_whole_requests() -> None: + r"""Each of the three calls is the whole request: the shared task, + schema, options and query, and its own context, which is the seeded + sample less the user that call scores, each labelled with its score, + and that user scored at the instance id after the sample's.""" + from sdm_client import KumoRelational, PredictiveQuery, Schema, SDMClient + + run = runner() + connection, cleanup, _ = run._open( + {'input': TINY}, run.load_bases(), 'duckdb' + ) + try: + with ( + recorder().endpoint().serve() as (url, received), + warnings.catch_warnings(), + ): + warnings.simplefilter('ignore') + schema = Schema.from_source(run._source('duckdb', connection)) + request = PredictiveQuery(TINY['pql'], schema).materialize( + **run._materialize_arguments(TINY['materialize']) + ) + with SDMClient(url) as client: + client.predict(request, model=KumoRelational()) + finally: + cleanup() + sent = [] + for _, path, body in received.requests: + assert path == '/v1/predictions' + parsed = json.loads(body) + parsed['metadata'].pop('request_id') + sent.append(parsed) + + def table(columns: list[str], rows: list[list[Any]]) -> dict[str, Any]: + return {'columns': columns, 'format': 'arrays', 'rows': rows} + + def half(rows: list[list[Any]], columns: list[str]) -> dict[str, Any]: + return { + 'instance_table': table(columns, rows), + 'related_tables': { + 'users': table( + ['instance_id', 'user_id'], [row[:2] for row in rows] + ) + }, + } + + shared = { + 'model': 'kumo-relational', + 'task': { + 'kind': 'regression', + 'target': {'column_name': 'TARGET', 'dtype': 'float32'}, + 'entity_table_names': ['users'], + }, + 'schema': { + 'instance_table': { + 'columns': { + 'instance_id': { + 'dtype': 'int64', + 'nullable': False, + 'stype': 'ID', + }, + '__kumo_entity_ref_0': {'dtype': 'int64', 'stype': 'ID'}, + 'TARGET': {'dtype': 'float32', 'stype': 'numerical'}, + }, + 'primary_key': 'instance_id', + }, + 'related_tables': { + 'users': { + 'columns': { + 'instance_id': { + 'dtype': 'int64', + 'nullable': False, + 'stype': 'ID', + }, + 'user_id': {'dtype': 'int64', 'nullable': False}, + }, + 'primary_key': ['instance_id', 'user_id'], + } + }, + 'relationships': [ + { + 'source_columns': ['__kumo_entity_ref_0'], + 'target_table': 'users', + 'target_columns': ['user_id'], + } + ], + }, + 'output': {'fields': ['prediction'], 'prediction_statistic': 'median'}, + 'inference': {'run_mode': 'fast', 'use_prediction_time': False}, + 'metadata': { + 'num_context_examples': len(TINY_SAMPLE), + 'num_prediction_examples': 1, + 'source_query': TINY['pql'], + }, + } + expected = [] + for scored in TINY['materialize']['indices']: + context = [ + [instance, user, float(user % 3)] + for instance, user in enumerate(TINY_SAMPLE) + if user != scored + ] + expected.append( + { + **shared, + 'context': half( + context, ['instance_id', '__kumo_entity_ref_0', 'TARGET'] + ), + 'predict': half( + [[len(TINY_SAMPLE), scored]], + ['instance_id', '__kumo_entity_ref_0'], + ), + } + ) + assert sent == expected diff --git a/contract/tests/test_query_options.py b/contract/tests/test_query_options.py index 2a525d918..fc0309f43 100644 --- a/contract/tests/test_query_options.py +++ b/contract/tests/test_query_options.py @@ -29,6 +29,7 @@ RUNNER = ROOT / 'contract/runners/run_python.py' RECORDER = ROOT / 'contract/support/query_option_scenarios.py' ENDPOINT = ROOT / 'tests/support/scripted_endpoint.py' +SHOP = ROOT / 'contract/support/shop_requests.py' CASES = ROOT / 'contract/cases/s1' DATABASES = ('duckdb', 'sqlite', 'postgres') @@ -209,6 +210,10 @@ def endpoint() -> ModuleType: return _module('scripted_endpoint', ENDPOINT) +def shop() -> ModuleType: + return _module('shop_requests', SHOP) + + def key(case_id: str, database: str) -> str: return runner().case_key(case_id, database) @@ -620,6 +625,10 @@ def check() -> None: if case['input']['pql'].startswith('EXPLAIN '): assert 'explanation' in body['output']['fields'] assert_request_options(case, sent) + if expected[0] == 'success': + # Each call's context, anchors, labels and related rows, as + # the filter, ASSUMING and the target derive them. + shop().assert_sent(case, runner().load_bases(), sent) check() diff --git a/contract/tests/test_ranking.py b/contract/tests/test_ranking.py index 86754e6f5..3635f0f07 100644 --- a/contract/tests/test_ranking.py +++ b/contract/tests/test_ranking.py @@ -29,6 +29,7 @@ RUNNER = ROOT / 'contract/runners/run_python.py' RECORDER = ROOT / 'contract/support/ranking_scenarios.py' CASE_DIR = ROOT / 'contract/cases/s1' +SHOP = ROOT / 'contract/support/shop_requests.py' DATABASES = ('duckdb', 'sqlite', 'postgres') @@ -85,6 +86,10 @@ def recorder() -> ModuleType: return _module('ranking_scenarios', RECORDER) +def shop() -> ModuleType: + return _module('shop_requests', SHOP) + + def key(case_id: str, database: str) -> str: return runner().case_key(case_id, database) @@ -364,6 +369,9 @@ def check() -> None: not in body['predict']['instance_table']['columns'] ) if expect['outcome'] == 'success': + # One call per candidate, each candidate's membership in the + # window the label, over the context the anchors gather. + shop().assert_sent(case, runner().load_bases(), sent) result = line['public']['result'] assert result['rows'] == len( case['input']['materialize']['indices'] From cbf6ac8251c1ca26abc0fa4459b0d25f8290fa08 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 10 Nov 2026 23:42:24 -0800 Subject: [PATCH 6/6] Budget each SQL journey's reads and pin its keyed statements whole The golden-SQL guard went with the recorded statements, and the DuckDB journey's compared count only checks that the capture and the query log agree, so an extra scan or refresh, a missing read, or a lost cast or window bound passed. Pin base.regression's budget on DuckDB (10 statements describing, 8 materializing, 2 of them plan B's refreshes, 18 logged) and add the same journey on SQLite (24 fetches, 6 refreshes, 10 and 14 statements). Assert one whole entity lookup and one whole neighbor read with the key set each binds on DuckDB and SQLite, and as PostgreSQL renders them. In the empty-null, decimal, nested and time-feature families, assert the label window's two cast bounds, the neighbors' cast bound, order and per-seed cap, the summed and wide decimal projections, the wide-key TRY_CAST and the list column left out of the lookup. Seeded faults: unchanged tables' keys and previews read again (DuckDB materialize 14 statements, not 8); the stored-time scan not kept per table (SQLite 25 fetches, not 24); DuckDB's CAST(... AS TIMESTAMP) dropped; the window's lower bound dropped; the wide-key TRY_CAST dropped; SQLite's per-seed LIMIT dropped; PostgreSQL's neighbor bound left out; and, on the expectation, a list column added to the lookup. Each fails by name. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Manush Murali --- rust/crates/sdm-testkit/tests/decimal_sql.rs | 44 +++++++ .../sdm-testkit/tests/duckdb_provider.rs | 84 ++++++++++++ .../sdm-testkit/tests/empty_null_sql.rs | 49 +++++++ rust/crates/sdm-testkit/tests/nested_sql.rs | 40 ++++++ .../sdm-testkit/tests/postgres_dialect.rs | 122 +++++++++++++++++- rust/crates/sdm-testkit/tests/sqlite.rs | 108 +++++++++++++++- .../sdm-testkit/tests/time_feature_sql.rs | 55 ++++++++ 7 files changed, 500 insertions(+), 2 deletions(-) diff --git a/rust/crates/sdm-testkit/tests/decimal_sql.rs b/rust/crates/sdm-testkit/tests/decimal_sql.rs index 53d819cc2..5fc61a454 100644 --- a/rust/crates/sdm-testkit/tests/decimal_sql.rs +++ b/rust/crates/sdm-testkit/tests/decimal_sql.rs @@ -95,3 +95,47 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// A summed decimal is the label read's one value column; a wide decimal +/// feature is read by the entity lookup; a key past `BIGINT` joins each +/// keyed read through the seed cast to the key's type. +#[test] +fn decimal_reads_project_their_columns_and_cast_wide_keys() { + let seen: std::collections::BTreeMap = observed() + .into_iter() + .map(|(case, seen)| (case.id, seen)) + .collect(); + let summed = ran(&seen["decimal.sum-10-2"], "materialize"); + assert!( + summed.iter().any(|sql| sql.starts_with( + "SELECT s.\"batch\" AS \"__kumo_batch__\", t.\"ts\", t.\"amount\"\nFROM \"__sdm_seeds__\" s\n" + )), + "{summed:#?}" + ); + let wide = ran(&seen["decimal.feature-38-0-wide"], "materialize"); + assert!( + wide.iter().any(|sql| sql.contains( + "t.\"user_id\", t.\"age\", t.\"country\", t.\"f\"\nFROM \"__sdm_seeds__\" s\nJOIN \"users\" t" + )), + "{wide:#?}" + ); + let keyed = ran(&seen["decimal.key-wide"], "materialize"); + let cast = r#"ON t."user_id" = TRY_CAST(s."key" AS DECIMAL(38,0))"#; + assert_eq!( + keyed.iter().filter(|sql| sql.contains(cast)).count(), + 3, + "{keyed:#?}" + ); +} diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index a7f76ec8e..d23d9a226 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -1071,6 +1071,61 @@ fn materialization_records_complete_encodable_database_reads() { assert!(comparable > 0); assert_eq!(logged["compared"].as_u64(), Some(comparable as u64)); assert_eq!(logged["differing"], serde_json::json!([])); + // The journey's budget: ten reads describe the source; materialize + // re-reads the two catalogs (plan B's refreshes) and runs six more. + let per_stage = |stage: &str| statements.iter().filter(|s| s["stage"] == stage).count(); + assert_eq!( + ( + per_stage("from_source"), + per_stage("materialize"), + per_stage("predict") + ), + (10, 8, 0) + ); + let refreshes = statements + .iter() + .filter(|s| s["purpose"] == sdm_testkit::plan_b::LOCATE) + .count(); + assert_eq!(refreshes, 2); + assert_eq!(logged["statements"].as_u64(), Some(18)); + // One whole entity lookup and one whole neighbor read, each with the + // key set it binds. + let (lookup, neighbors) = keyed_reads(&observed.fetches); + assert_eq!( + lookup, + serde_json::json!([{ + "dialect": "duckdb", + "text": r#"SELECT s."batch" AS "__kumo_batch__", COUNT(*) OVER (PARTITION BY s."batch") AS "__kumo_matches__", t."user_id", t."age", t."country" +FROM "__sdm_seeds__" s +JOIN "users" t + ON t."user_id" = s."key" +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY s."batch" +) = 1"#, + "params": [key_set(2, &[])], + }]) + ); + assert_eq!( + neighbors, + serde_json::json!([{ + "dialect": "duckdb", + "text": r#"SELECT s."batch" AS "__kumo_batch__", t."order_id", t."user_id", t."price", t."ts" +FROM "__sdm_seeds__" s +JOIN "orders" t + ON t."user_id" = s."key" + AND CAST(t."ts" AS TIMESTAMP) <= s."time" +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY s."batch" + ORDER BY CAST(t."ts" AS TIMESTAMP) DESC NULLS LAST, t."order_id" ASC NULLS LAST +) <= 32 +ORDER BY s."batch", CAST(t."ts" AS TIMESTAMP) DESC NULLS LAST, t."order_id" ASC NULLS LAST"#, + "params": [key_set(3, &[serde_json::json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + })])], + }]) + ); for stage in ["from_source", "materialize"] { assert!( statements @@ -1097,6 +1152,35 @@ fn materialization_records_complete_encodable_database_reads() { } } +/// The planned statements of a run's entity lookup and of its neighbor +/// read with a per-seed cap. +fn keyed_reads(fetches: &[Value]) -> (Value, Value) { + let planned = |wanted: &dyn Fn(&Value) -> bool| { + let found: Vec<_> = fetches.iter().filter(|f| wanted(&f["request"])).collect(); + assert_eq!(found.len(), 1, "one such read"); + found[0]["plan"]["statements"].clone() + }; + ( + planned(&|r| r["request"] == "lookup"), + planned(&|r| r["request"] == "neighbors" && !r["per_seed"].is_null()), + ) +} + +/// A key-set parameter bound as `__sdm_seeds__`: its attachment, the key +/// and batch columns, then `more`. +fn key_set(attachment: u64, more: &[Value]) -> Value { + let mut columns = vec![ + serde_json::json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + serde_json::json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + serde_json::json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": "__sdm_seeds__", + }) +} + /// The answer to the one statement the shipped data layer plans for the /// columns of `table` on DuckDB, and that statement. fn columns_of(connection: &Connection, table: &str) -> (String, Vec) { diff --git a/rust/crates/sdm-testkit/tests/empty_null_sql.rs b/rust/crates/sdm-testkit/tests/empty_null_sql.rs index 7ad47b32b..6ff2c3a03 100644 --- a/rust/crates/sdm-testkit/tests/empty_null_sql.rs +++ b/rust/crates/sdm-testkit/tests/empty_null_sql.rs @@ -94,3 +94,52 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// Nulls and empty facts leave every windowed read whole: the label window's +/// two cast bounds, and the neighbors' cast bound, nulls-last newest-first +/// order and per-seed cap; an all-null feature is still read by the lookup. +#[test] +fn null_and_empty_reads_keep_their_window_order_and_projection() { + let cast = r#"CAST(t."ts" AS TIMESTAMP)"#; + let mut checked = 0; + for (case, seen) in observed() { + if seen.answer.outcome().0 != "success" { + continue; + } + let materialized = ran(&seen, "materialize"); + let reads = |text: &str| materialized.iter().any(|sql| sql.contains(text)); + assert!( + reads(&format!("{cast} <= s.\"end\"\n AND {cast} > s.\"start\"")), + "{}: the label window", + case.id + ); + assert!( + reads(&format!( + "{cast} <= s.\"time\"\nQUALIFY ROW_NUMBER() OVER (\n PARTITION BY s.\"batch\"\n ORDER BY {cast} DESC NULLS LAST, t.\"order_id\" ASC NULLS LAST\n) <= 32" + )), + "{}: the neighbors", + case.id + ); + if case.id == "null.feature-all-null" { + assert!( + reads("t.\"user_id\", t.\"age\", t.\"country\", t.\"f\"\nFROM"), + "{}: the all-null feature", + case.id + ); + } + checked += 1; + } + assert_eq!(checked, 5); +} diff --git a/rust/crates/sdm-testkit/tests/nested_sql.rs b/rust/crates/sdm-testkit/tests/nested_sql.rs index bd280b508..7ba5031c9 100644 --- a/rust/crates/sdm-testkit/tests/nested_sql.rs +++ b/rust/crates/sdm-testkit/tests/nested_sql.rs @@ -115,3 +115,43 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// A list column is read by the preview, which describes it, and left out +/// of the entity lookup, which sends no list. +#[test] +fn a_list_column_is_previewed_and_left_out_of_the_lookup() { + let mut checked = 0; + for (case, seen) in observed() { + if !matches!(case.id.as_str(), "nested.list-int" | "nested.fixed-array") { + continue; + } + assert!( + ran(&seen, "from_source").contains( + &r#"SELECT "user_id", "age", "country", "f" FROM "users" LIMIT 1000"#.to_owned() + ), + "{}: the preview", + case.id + ); + assert!( + ran(&seen, "materialize").iter().any(|sql| sql.contains( + "t.\"user_id\", t.\"age\", t.\"country\"\nFROM \"__sdm_seeds__\" s\nJOIN \"users\" t" + )), + "{}: the lookup", + case.id + ); + checked += 1; + } + assert_eq!(checked, 2); +} diff --git a/rust/crates/sdm-testkit/tests/postgres_dialect.rs b/rust/crates/sdm-testkit/tests/postgres_dialect.rs index 0a6d16ce4..9ad200063 100644 --- a/rust/crates/sdm-testkit/tests/postgres_dialect.rs +++ b/rust/crates/sdm-testkit/tests/postgres_dialect.rs @@ -29,7 +29,7 @@ use sdm_contract::data::{ DataRequest, DescribePart, Dialect, FetchPlan, ProviderChoice, Relation, SqlParam, }; use sdm_contract::{Detail, ErrorCode}; -use serde_json::Value; +use serde_json::{Value, json}; fn contract() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../contract") @@ -170,3 +170,123 @@ fn a_relation_of_no_part_or_of_three_is_refused_by_name() { } } } + +/// A key-set parameter bound as `__sdm_seeds__`: its attachment, the key +/// and batch columns, then `more`. +fn key_set(attachment: u64, more: &[Value]) -> Value { + let mut columns = vec![ + json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": "__sdm_seeds__", + }) +} + +/// The statements `request` plans on Postgres, as the contract writes them. +fn planned(request: Value) -> Value { + let request: DataRequest = serde_json::from_value(request).unwrap(); + let FetchPlan::Sql { statements, .. } = plan(&request).unwrap() else { + panic!("not SQL"); + }; + serde_json::to_value(statements).unwrap() +} + +/// base.regression's entity lookup and its neighbor read, as Postgres +/// renders them whole: the key set bound once as JSON, each column read, +/// the time bound and the per-seed cap, newest first. +#[test] +fn a_lookup_and_a_neighbor_read_render_whole_with_their_key_sets() { + let time = json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + }); + let column = |name: &str| json!({"name": name, "column": name}); + let lookup = planned(json!({ + "request": "lookup", + "table": { + "source": ["users"], + "columns": [column("user_id"), column("age"), column("country")], + "found": true, + }, + "key": ["user_id"], + "columns": ["user_id", "age", "country"], + "seeds": {"attachment": 2, "columns": key_set(2, &[])["keys"]["columns"]}, + })); + assert_eq!( + lookup, + json!([{ + "dialect": "postgres", + "text": r#"WITH "__sdm_seeds__" AS ( + SELECT (r.value #>> '{}')::bigint AS "key", (r.ordinality - 1)::bigint AS "batch" + FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS r(value, ordinality) +) +SELECT l.b AS "__kumo_batch__", l.c0 AS "user_id", l.c1 AS "age", l.c2 AS "country", l.r AS "__kumo_row__", l.m AS "__kumo_matches__" +FROM ( + SELECT s."batch" AS b, t."user_id" AS c0, t."age" AS c1, t."country" AS c2, + ROW_NUMBER() OVER (PARTITION BY s."batch" + ORDER BY t."user_id") AS r, + COUNT(*) OVER (PARTITION BY s."batch") AS m + FROM "__sdm_seeds__" s + JOIN "public"."users" t + ON t."user_id" = s."key" +) AS l +WHERE l.r = 1 +ORDER BY l.b"#, + "params": [key_set(2, &[])], + }]) + ); + let neighbors = planned(json!({ + "request": "neighbors", + "table": { + "source": ["orders"], + "columns": [column("order_id"), column("user_id"), column("price"), column("ts")], + "found": true, + }, + "key": ["user_id"], + "columns": ["order_id", "user_id", "price", "ts"], + "seeds": { + "attachment": 3, + "columns": key_set(3, std::slice::from_ref(&time))["keys"]["columns"], + }, + "window": { + "column": "ts", + "source_type": "timestamp without time zone", + "lower": null, + "upper": {"seed_column": "time", "inclusive": true}, + }, + "per_seed": 32, + "order": [ + {"column": "ts", "as_timestamp": true, "descending": true, "nulls_last": true}, + {"column": "order_id", "as_timestamp": false, "descending": false, "nulls_last": true}, + ], + })); + assert_eq!( + neighbors, + json!([{ + "dialect": "postgres", + "text": r#"WITH "__sdm_seeds__" AS ( + SELECT (r.value->>'id')::bigint AS "key", (r.ordinality - 1)::bigint AS "batch", w2."ts" AS "time" + FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS r(value, ordinality) + CROSS JOIN (SELECT ROW(t.*)::"public"."orders" AS base FROM "public"."orders" AS t LIMIT 1) AS b + CROSS JOIN LATERAL jsonb_populate_record(b.base, jsonb_build_object('ts', r.value->'e')) AS w2 +) +SELECT s."batch" AS "__kumo_batch__", n.c0 AS "order_id", n.c1 AS "user_id", n.c2 AS "price", n.c3 AS "ts", n.r AS "__kumo_row__" +FROM "__sdm_seeds__" s +CROSS JOIN LATERAL ( + SELECT t."order_id" AS c0, t."user_id" AS c1, t."price" AS c2, t."ts" AS c3, ROW_NUMBER() OVER (ORDER BY t."ts" DESC NULLS LAST, t."order_id" ASC NULLS LAST) AS r + FROM "public"."orders" t + WHERE t."user_id" = s."key" + AND t."ts" <= s."time" + ORDER BY t."ts" DESC NULLS LAST, t."order_id" ASC NULLS LAST + LIMIT 32 +) AS n +ORDER BY s."batch", n.r"#, + "params": [key_set(3, &[time])], + }]) + ); +} diff --git a/rust/crates/sdm-testkit/tests/sqlite.rs b/rust/crates/sdm-testkit/tests/sqlite.rs index d17852d3a..5d16f58f8 100644 --- a/rust/crates/sdm-testkit/tests/sqlite.rs +++ b/rust/crates/sdm-testkit/tests/sqlite.rs @@ -2,10 +2,16 @@ // All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! SQLite relation names and invalid qualification through the native provider. +//! SQLite relation names and invalid qualification through the native provider, +//! and one case's journey on SQLite: its read budget and its keyed reads whole. + +use std::path::PathBuf; use sdm_contract::data::{DataRequest, DescribePart, Dialect, FetchPlan, ProviderChoice, Relation}; use sdm_contract::{Detail, ErrorCode}; +use sdm_testkit::host::{self, Answer}; +use sdm_testkit::{bases, case}; +use serde_json::{Value, json}; const DATABASE: &str = "sqlite"; @@ -118,3 +124,103 @@ fn a_relation_of_no_part_or_of_three_is_refused_by_name_on_sqlite() { } } } + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// A key-set parameter: its attachment, its relation, the key and batch +/// columns, then `more`. +fn key_set(attachment: u64, relation: &str, more: &[Value]) -> Value { + let mut columns = vec![ + json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": relation, + }) +} + +/// base.regression on SQLite, as the DuckDB journey runs it: a fixed budget +/// of fetches, plan B's refreshes and statements, and the entity lookup and +/// the neighbor read whole, with the key sets they bind. +#[test] +fn base_regression_s_journey_reads_a_fixed_budget_on_sqlite() { + let contract = root().join("contract"); + let loaded = case::load(&[contract.join("cases/s1/base.regression.json")]).unwrap(); + let bases = bases::load(&contract.join("bases.yaml")).unwrap(); + let observed = host::run_sqlite(&loaded[0].case, &bases, &root()).unwrap(); + assert!(matches!(observed.answer, Answer::Predicted(_))); + // Each table's catalog, keys and preview are read again at materialize. + let refreshes = observed + .fetches + .iter() + .filter(|f| f["purpose"] == sdm_testkit::plan_b::LOCATE) + .count(); + assert_eq!((observed.fetches.len(), refreshes), (24, 6)); + let driver = observed.driver.as_ref().expect("a driver record"); + let statements = driver["capture"]["statements"].as_array().unwrap(); + let per_stage = |stage: &str| statements.iter().filter(|s| s["stage"] == stage).count(); + assert_eq!( + ( + per_stage("from_source"), + per_stage("materialize"), + per_stage("predict") + ), + (10, 14, 0) + ); + assert_eq!(driver["query_log"]["statements"].as_u64(), Some(24)); + assert_eq!(driver["capture"]["bindings"].as_array().unwrap().len(), 3); + let planned = |wanted: &dyn Fn(&Value) -> bool| { + let found: Vec<_> = observed + .fetches + .iter() + .filter(|f| wanted(&f["request"])) + .collect(); + assert_eq!(found.len(), 1, "one such read"); + found[0]["plan"]["statements"].clone() + }; + assert_eq!( + planned(&|r| r["request"] == "lookup"), + json!([{ + "dialect": "sqlite", + "text": r#"SELECT s."batch" AS "__kumo_batch__", (SELECT COUNT(*) FROM "users" WHERE "user_id" == s."key") AS "__kumo_matches__", t."user_id", t."age", t."country" +FROM temp."__sdm_seeds_19f857a1097d1e84__" s +JOIN "users" t + ON t.rowid = ( + SELECT rowid + FROM "users" + WHERE "user_id" == s."key" + LIMIT 1 +)"#, + "params": [key_set(2, "__sdm_seeds_19f857a1097d1e84__", &[])], + }]) + ); + assert_eq!( + planned(&|r| r["request"] == "neighbors" && !r["per_seed"].is_null()), + json!([{ + "dialect": "sqlite", + "text": r#"SELECT s."batch" AS "__kumo_batch__", t."order_id", t."user_id", t."price", t."ts" +FROM temp."__sdm_seeds_0086307c4757992b__" s +JOIN "orders" t +ON t.rowid IN ( + SELECT rowid + FROM "orders" + WHERE "user_id" = s."key" + AND "ts" <= substr(s."time", 1, 10) || ' ' || substr(s."time", 12, 8) + ORDER BY "ts" DESC, "order_id" + LIMIT 32 +) +ORDER BY "__kumo_batch__", t."ts" DESC, t."order_id""#, + "params": [key_set(3, "__sdm_seeds_0086307c4757992b__", &[json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + "metadata": {"sdm:stored_time": "2023-01-01 00:00:00"}, + })])], + }]) + ); +} diff --git a/rust/crates/sdm-testkit/tests/time_feature_sql.rs b/rust/crates/sdm-testkit/tests/time_feature_sql.rs index 50668e621..8c2560b13 100644 --- a/rust/crates/sdm-testkit/tests/time_feature_sql.rs +++ b/rust/crates/sdm-testkit/tests/time_feature_sql.rs @@ -103,3 +103,58 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// Each answered case reads its time column through a cast to `TIMESTAMP` +/// wherever it bounds or orders by it: the range of times, both bounds of +/// the label window, and the neighbors' upper bound and newest-first order. +#[test] +fn each_time_column_is_cast_in_every_bound_and_order() { + let mut checked = 0; + for (case, seen) in observed() { + if seen.answer.outcome().0 != "success" { + continue; + } + let time = if case.id == "time.text-declared" { + "happened" + } else { + "ts" + }; + let cast = format!(r#"CAST(t."{time}" AS TIMESTAMP)"#); + let materialized = ran(&seen, "materialize"); + let reads = |text: &str| materialized.iter().any(|sql| sql.contains(text)); + assert!( + reads(&format!(r#"MIN(CAST("{time}" AS TIMESTAMP)) as min_date"#)), + "{}: the range of times", + case.id + ); + assert!( + reads(&format!("{cast} <= s.\"end\"\n AND {cast} > s.\"start\"")), + "{}: the label window", + case.id + ); + assert!( + reads(&format!("{cast} <= s.\"time\"")), + "{}: the neighbors' bound", + case.id + ); + assert!( + reads(&format!("ORDER BY {cast} DESC NULLS LAST")), + "{}: newest first", + case.id + ); + checked += 1; + } + assert_eq!(checked, 9); +}