diff --git a/CHANGELOG.md b/CHANGELOG.md index 4716abd3e..79adfd6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,6 +180,9 @@ The public API snapshot records this version update; publication remains pending - On Databricks, through Spark or a SQL warehouse, a `DECIMAL` value with a fraction is sent exactly, such as `997.67`. 1.0.0 sent it through a float, such as `997.6700000000001`. +- On PostgreSQL, a whole-number `NUMERIC` key above `2**63 - 1` is kept + exactly and sent as a string. 1.0.0 raised `OverflowError` in + `Schema.from_source`. ### Removed diff --git a/contract/support/shop_requests.py b/contract/support/shop_requests.py new file mode 100644 index 000000000..40fa089ab --- /dev/null +++ b/contract/support/shop_requests.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The relational requests a shop case should send, derived from its authored inputs. + +Each call's context is every user the entity filter and ASSUMING admit at +each anchor, walking back one target window at a time from the anchor the +case gives while there is data, labelled by the target the call asks; its +related tables hold each instance's user and the facts up to its anchor. +Nothing here reads a recorded request. +""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta +from typing import Any + +#: A window over orders: an aggregation and its days. +_SUM = re.compile(r'^PREDICT SUM\(orders\.price, (-?\d+), (-?\d+), days\)') +_ANY = re.compile(r'^PREDICT COUNT\(orders\.\*, (-?\d+), (-?\d+), days\) > 0') +_RANK = re.compile( + r'^PREDICT LIST_DISTINCT\(buys\.item_id, (-?\d+), (-?\d+), days\) RANK ' +) +_FORECAST = re.compile(r' FORECAST (\d+) TIMEFRAMES ') +_COUNT_ORDERS = re.compile(r'^COUNT\(orders\.\*, (-?\d+), (-?\d+), days\) > 0$') +_STATIC = re.compile(r"^users\.(\w+) (>|<|=) ('?[\w ]+'?)$") +#: The rows sent for a call, at most. +_ROWS_PER_CALL = 10_000 + + +def _stamp(value: datetime) -> str: + return value.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +class Shop: + r"""A case's database read once: users, orders and, where its setup + makes them, buys and items.""" + + def __init__(self, case: dict[str, Any], bases: dict[str, list[str]]): + import duckdb + + connection = duckdb.connect() + try: + for statement in [ + *bases[case['input']['base']], + *case['input'].get('setup', []), + ]: + connection.execute(statement) + tables = { + row[0] + for row in connection.execute( + 'SELECT table_name FROM duckdb_tables()' + ).fetchall() + } + self.tables = tables + self.users = { + row[0]: row[1:] + for row in connection.execute( + 'SELECT user_id, age, country FROM users' + ).fetchall() + } + self.orders = connection.execute( + 'SELECT user_id, price, ts FROM orders' + ).fetchall() + self.buys = ( + connection.execute( + 'SELECT user_id, item_id, ts FROM buys' + ).fetchall() + if 'buys' in tables + else [] + ) + self.items = ( + dict( + connection.execute( + 'SELECT item_id, kind FROM items' + ).fetchall() + ) + if 'items' in tables + else {} + ) + self._static = connection + self.earliest = min(ts for *_, ts in [*self.orders, *self.buys]) + except BaseException: + connection.close() + raise + + def close(self) -> None: + self._static.close() + + def admitted(self, condition: str | None, at: datetime) -> set[int]: + r"""The users ``condition`` admits at ``at``: a column of the user + compared with a literal, or a count of orders in a window of it.""" + if condition is None: + return set(self.users) + window = _COUNT_ORDERS.match(condition) + if window: + start, end = (int(n) for n in window.groups()) + return { + user + for user, _, ts in self.orders + if at + timedelta(days=start) < ts <= at + timedelta(days=end) + } + static = _STATIC.match(condition) + assert static, f'no rule derives the users {condition!r} admits' + column, operator, literal = static.groups() + return { + row[0] + for row in self._static.execute( + f'SELECT user_id FROM users WHERE {column} {operator} {literal}' + ).fetchall() + } + + def orders_until(self, user: int, at: datetime) -> list[tuple[Any, ...]]: + return sorted( + (price, _stamp(ts), user) + for owner, price, ts in self.orders + if owner == user and ts <= at + ) + + def buys_until(self, user: int, at: datetime) -> list[tuple[Any, ...]]: + return sorted( + (_stamp(ts), user, self.items[item]) + for owner, item, ts in self.buys + if owner == user and ts <= at + ) + + +def _target(pql: str) -> tuple[str, int, int]: + body = re.sub(r'^(EVALUATE|EXPLAIN) ', '', pql) + for kind, pattern in (('sum', _SUM), ('any', _ANY), ('member', _RANK)): + found = pattern.match(body) + if found: + start, end = (int(n) for n in found.groups()) + return kind, start, end + raise AssertionError(f'no rule derives the labels of {pql!r}') + + +def _clauses(pql: str) -> tuple[str | None, str | None]: + r"""The entity filter and the ASSUMING condition ``pql`` gives.""" + assuming = None + if ' ASSUMING ' in pql: + pql, assuming = pql.split(' ASSUMING ', 1) + _, _, condition = pql.partition(' FOR EACH users.user_id WHERE ') + return condition or None, assuming + + +def _label( + shop: Shop, + kind: str, + user: int, + at: datetime, + window: tuple[int, int], + candidate: Any, +) -> Any: + low, high = (at + timedelta(days=d) for d in window) + if kind == 'sum': + return float( + sum( + price + for owner, price, ts in shop.orders + if owner == user and low < ts <= high + ) + ) + if kind == 'any': + return any( + owner == user and low < ts <= high for owner, _, ts in shop.orders + ) + return any( + owner == user and item == candidate and low < ts <= high + for owner, item, ts in shop.buys + ) + + +def expected_calls( + case: dict[str, Any], bases: dict[str, list[str]] +) -> list[dict[str, Any]]: + r"""Each call the case should make, in order: its context, by anchor + and entity, each with its label; the entities it scores; the anchor + they are scored at; and its related tables. A ranking is asked one + candidate at a time, each candidate once, and one whose context holds + one class is not asked; a forecast is asked a step at a time.""" + given = case['input'] + spec = given['materialize'] + anchor = datetime.fromisoformat(spec['anchor_time']) + kind, start, end = _target(given['pql']) + condition, assuming = _clauses(given['pql']) + steps = _FORECAST.search(given['pql']) + parts: list[tuple[Any, tuple[int, int]]] + if kind == 'member': + parts = [(c, (start, end)) for c in dict.fromkeys(spec['candidates'])] + elif steps: + width = end - start + parts = [ + (None, (start + width * k, end + width * k)) + for k in range(int(steps.group(1))) + ] + else: + parts = [(None, (start, end))] + shop = Shop(case, bases) + try: + calls = [] + for candidate, window in parts: + width = window[1] - window[0] + at = anchor - timedelta(days=window[1]) + context = {} + while at >= shop.earliest: + users = shop.admitted(condition, at) & shop.admitted( + assuming, at + ) + for user in users: + context[(_stamp(at), user)] = _label( + shop, kind, user, at, window, candidate + ) + at -= timedelta(days=width) + if kind == 'member' and len(set(context.values())) < 2: + continue + batch = spec.get('rows_per_call') or _ROWS_PER_CALL + indices = spec['indices'] + calls += [ + { + 'context': context, + 'predict': indices[first : first + batch], + 'anchor': _stamp(anchor), + 'related': related(shop), + } + for first in range(0, len(indices), batch) + ] + return calls + finally: + shop.close() + + +def related(shop: Shop) -> dict[str, Any]: + r"""What each related table holds for an instance of ``user`` at + ``at``, by table.""" + out: dict[str, Any] = { + 'users': lambda user, at: [(user, *shop.users[user])], + 'orders': shop.orders_until, + } + if 'buys' in shop.tables: + out['buys'] = shop.buys_until + out['items'] = lambda user, at: sorted( + {kind for *_, kind in shop.buys_until(user, at)} + ) + return out + + +def plain(value: Any) -> Any: + r"""A parsed request's value with each number as the number it + writes (the runner keeps a number as ``{"number": }``).""" + if isinstance(value, dict): + if set(value) == {'number'} and isinstance(value['number'], str): + text = value['number'] + return float(text) if any(c in text for c in '.eE') else int(text) + return {name: plain(item) for name, item in value.items()} + if isinstance(value, list): + return [plain(item) for item in value] + return value + + +def _rows(table: dict[str, Any]) -> list[dict[str, Any]]: + return [ + dict(zip(table['columns'], row, strict=True)) for row in table['rows'] + ] + + +def assert_sent( + case: dict[str, Any], bases: dict[str, list[str]], sent: list[Any] +) -> None: + r"""Each request in ``sent`` (parsed, numbers as numbers) is the call + :func:`expected_calls` derives for ``case``: its context's entities, + anchors and labels, the entities it scores, and every related table's + rows for each instance.""" + wanted = expected_calls(case, bases) + sent = [plain(body) for body in sent] + assert len(sent) == len(wanted), (case['id'], len(sent), len(wanted)) + for call, (body, want) in enumerate(zip(sent, wanted, strict=True)): + where = f'{case["id"]} call {call}' + target = body['task']['target']['column_name'] + context = _rows(body['context']['instance_table']) + predict = _rows(body['predict']['instance_table']) + got = { + (row['__kumo_anchor_time'], row['__kumo_entity_ref_0']): row[target] + for row in context + } + assert len(got) == len(context), f'{where}: an instance twice' + missing = sorted(set(want['context']) - set(got)) + extra = sorted(set(got) - set(want['context'])) + assert not missing and not extra, ( + f'{where}: context lacks {missing[:4]} and holds {extra[:4]}' + ) + wrong = [ + (each, got[each], label) + for each, label in sorted(want['context'].items()) + if got[each] != label or type(got[each]) is not type(label) + ] + assert not wrong, f'{where}: labels (sent, derived): {wrong[:4]}' + assert [row['__kumo_entity_ref_0'] for row in predict] == ( + want['predict'] + ), f'{where}: scores the wrong entities' + assert {row['__kumo_anchor_time'] for row in predict} == { + want['anchor'] + }, f'{where}: scored at the wrong anchor' + assert len({row['instance_id'] for row in [*context, *predict]}) == ( + len(context) + len(predict) + ), f'{where}: an instance id twice' + for half, instances in (('context', context), ('predict', predict)): + tables = body[half]['related_tables'] + assert set(tables) == set(want['related']), ( + f'{where}: {half} related tables {sorted(tables)}' + ) + held = {name: _rows(table) for name, table in tables.items()} + for instance in instances: + user = instance['__kumo_entity_ref_0'] + at = datetime.fromisoformat(instance['__kumo_anchor_time'][:-1]) + for name, rule in want['related'].items(): + rows = [ + row + for row in held[name] + if row['instance_id'] == instance['instance_id'] + ] + assert _related(name, rows, held) == rule(user, at), ( + f'{where}: {half} {name} of {instance}' + ) + + +def _related( + name: str, rows: list[dict[str, Any]], held: dict[str, Any] +) -> list[Any]: + if name == 'users': + return [(r['user_id'], r['age'], r['country']) for r in rows] + if name == 'orders': + return sorted((r['price'], r['ts'], r['user_id']) for r in rows) + if name == 'items': + return sorted(r['kind'] for r in rows) + kinds = { + (r['instance_id'], r['__node_id']): r['kind'] for r in held['items'] + } + return sorted( + (r['ts'], r['user_id'], kinds[(r['instance_id'], r['item_id'])]) + for r in rows + ) diff --git a/contract/tests/test_exact_numeric_keys.py b/contract/tests/test_exact_numeric_keys.py index a3d3242e6..5e587ff13 100644 --- a/contract/tests/test_exact_numeric_keys.py +++ b/contract/tests/test_exact_numeric_keys.py @@ -8,8 +8,12 @@ import functools import importlib.util +import json import os +import shutil +import subprocess import sys +import tempfile from decimal import Decimal from pathlib import Path from types import ModuleType @@ -38,6 +42,8 @@ CASE_IDS = (GIVEN_KEY, GIVEN_LINK, EDITED_LINK, CATALOG) +WIDE = 'key.uint64-above-int64-max' + LINK = { 'source_table': 'orders', 'source_column': 'user_id', @@ -386,3 +392,76 @@ def test_each_case_holds_an_exact_key_past_2_53(case_id: str) -> None: ), }[case_id] assert (given['from_source'], given.get('edits')) == expected + + +@functools.cache +def wide_on_postgres() -> dict[str, Any]: + r"""python-core's observation of the wide-key case on PostgreSQL.""" + with tempfile.TemporaryDirectory(prefix='exact_numeric_keys-') as tmp: + given = Path(tmp) / 'cases' + given.mkdir() + shutil.copy(CASES / f'{WIDE}.json', given) + out = Path(tmp) / 'python-core.jsonl' + done = subprocess.run( + [ + sys.executable, + '-I', + str(RUNNER), + '--mode', + 'python-core', + '--database', + 'postgres', + '--out', + str(out), + str(given), + ], + capture_output=True, + text=True, + cwd=ROOT, + stdin=subprocess.DEVNULL, + check=False, + timeout=600, + ) + assert done.returncode == 0, done.stdout + done.stderr + return json.loads(out.read_text(encoding='utf-8')) + + +def test_adjacent_keys_past_int64_reach_the_wire_as_exact_strings() -> None: + r"""PostgreSQL's whole-number ``NUMERIC`` keys past ``2**63 - 1``, where + 1.0.0 raised ``OverflowError``: the adjacent 2**63 and 2**63 + 1 the + case asks for are sent as their exact text, two keys, in the instance + table and in each related table, where a double would make them one.""" + _server() + line = wide_on_postgres() + assert line['outcome'] == { + 'result': 'success', + 'stage': 'predict', + 'error': None, + 'code': None, + } + asked = [str(2**63), str(2**63 + 1)] + case, _ = runner().load_cases([CASES / f'{WIDE}.json'])[0] + assert [str(i) for i in case['input']['materialize']['indices']] == asked + (payload,) = [each['parsed'] for each in line['public']['payloads']] + schema = payload['schema'] + assert schema['instance_table']['columns']['__kumo_entity_ref_0'] == { + 'dtype': 'string', + 'stype': 'ID', + } + for name in ('users', 'orders'): + column = schema['related_tables'][name]['columns']['user_id'] + assert column['dtype'] == 'string', name + predict = payload['predict'] + table = predict['instance_table'] + at = table['columns'].index('__kumo_entity_ref_0') + keys = {row[0]['number']: row[at] for row in table['rows']} + assert list(keys.values()) == asked + users = predict['related_tables']['users'] + user_at = users['columns'].index('user_id') + assert [row[user_at] for row in users['rows']] == asked + orders = predict['related_tables']['orders'] + order_at = orders['columns'].index('user_id') + assert orders['rows'] + assert {row[order_at] for row in orders['rows']} == set(asked) + for row in orders['rows']: + assert row[order_at] == keys[row[0]['number']] diff --git a/contract/tests/test_prediction_batching.py b/contract/tests/test_prediction_batching.py index 9af000863..e7cede99e 100644 --- a/contract/tests/test_prediction_batching.py +++ b/contract/tests/test_prediction_batching.py @@ -19,6 +19,7 @@ import subprocess import sys import tempfile +import warnings from pathlib import Path from types import ModuleType from typing import Any @@ -590,3 +591,136 @@ def test_python_core_gathers_and_predicts_what_the_reference_refused( case is answered.""" got = core_carried(case_id) assert got['outcome']['result'] == 'success', got['outcome'] + + +#: A tiny seeded three-call fixture: five users scored ``user_id % 3``, +#: three of them asked one per call, sampled with a fixed seed. +TINY = { + 'base': 'shop', + 'setup': [ + 'DROP TABLE orders', + 'DROP TABLE users', + 'CREATE TABLE users AS SELECT i::BIGINT AS user_id, ' + '(i % 3)::DOUBLE AS score FROM range(5) t(i)', + ], + 'pql': 'PREDICT users.score FOR EACH users.user_id', + 'materialize': {'indices': [0, 1, 2], 'rows_per_call': 1, 'random_seed': 7}, +} +#: The users in the order the seed samples them, each at its instance id. +TINY_SAMPLE = [3, 1, 2, 4, 0] + + +def test_three_seeded_calls_send_whole_requests() -> None: + r"""Each of the three calls is the whole request: the shared task, + schema, options and query, and its own context, which is the seeded + sample less the user that call scores, each labelled with its score, + and that user scored at the instance id after the sample's.""" + from sdm_client import KumoRelational, PredictiveQuery, Schema, SDMClient + + run = runner() + connection, cleanup, _ = run._open( + {'input': TINY}, run.load_bases(), 'duckdb' + ) + try: + with ( + recorder().endpoint().serve() as (url, received), + warnings.catch_warnings(), + ): + warnings.simplefilter('ignore') + schema = Schema.from_source(run._source('duckdb', connection)) + request = PredictiveQuery(TINY['pql'], schema).materialize( + **run._materialize_arguments(TINY['materialize']) + ) + with SDMClient(url) as client: + client.predict(request, model=KumoRelational()) + finally: + cleanup() + sent = [] + for _, path, body in received.requests: + assert path == '/v1/predictions' + parsed = json.loads(body) + parsed['metadata'].pop('request_id') + sent.append(parsed) + + def table(columns: list[str], rows: list[list[Any]]) -> dict[str, Any]: + return {'columns': columns, 'format': 'arrays', 'rows': rows} + + def half(rows: list[list[Any]], columns: list[str]) -> dict[str, Any]: + return { + 'instance_table': table(columns, rows), + 'related_tables': { + 'users': table( + ['instance_id', 'user_id'], [row[:2] for row in rows] + ) + }, + } + + shared = { + 'model': 'kumo-relational', + 'task': { + 'kind': 'regression', + 'target': {'column_name': 'TARGET', 'dtype': 'float32'}, + 'entity_table_names': ['users'], + }, + 'schema': { + 'instance_table': { + 'columns': { + 'instance_id': { + 'dtype': 'int64', + 'nullable': False, + 'stype': 'ID', + }, + '__kumo_entity_ref_0': {'dtype': 'int64', 'stype': 'ID'}, + 'TARGET': {'dtype': 'float32', 'stype': 'numerical'}, + }, + 'primary_key': 'instance_id', + }, + 'related_tables': { + 'users': { + 'columns': { + 'instance_id': { + 'dtype': 'int64', + 'nullable': False, + 'stype': 'ID', + }, + 'user_id': {'dtype': 'int64', 'nullable': False}, + }, + 'primary_key': ['instance_id', 'user_id'], + } + }, + 'relationships': [ + { + 'source_columns': ['__kumo_entity_ref_0'], + 'target_table': 'users', + 'target_columns': ['user_id'], + } + ], + }, + 'output': {'fields': ['prediction'], 'prediction_statistic': 'median'}, + 'inference': {'run_mode': 'fast', 'use_prediction_time': False}, + 'metadata': { + 'num_context_examples': len(TINY_SAMPLE), + 'num_prediction_examples': 1, + 'source_query': TINY['pql'], + }, + } + expected = [] + for scored in TINY['materialize']['indices']: + context = [ + [instance, user, float(user % 3)] + for instance, user in enumerate(TINY_SAMPLE) + if user != scored + ] + expected.append( + { + **shared, + 'context': half( + context, ['instance_id', '__kumo_entity_ref_0', 'TARGET'] + ), + 'predict': half( + [[len(TINY_SAMPLE), scored]], + ['instance_id', '__kumo_entity_ref_0'], + ), + } + ) + assert sent == expected diff --git a/contract/tests/test_query_options.py b/contract/tests/test_query_options.py index 2a525d918..fc0309f43 100644 --- a/contract/tests/test_query_options.py +++ b/contract/tests/test_query_options.py @@ -29,6 +29,7 @@ RUNNER = ROOT / 'contract/runners/run_python.py' RECORDER = ROOT / 'contract/support/query_option_scenarios.py' ENDPOINT = ROOT / 'tests/support/scripted_endpoint.py' +SHOP = ROOT / 'contract/support/shop_requests.py' CASES = ROOT / 'contract/cases/s1' DATABASES = ('duckdb', 'sqlite', 'postgres') @@ -209,6 +210,10 @@ def endpoint() -> ModuleType: return _module('scripted_endpoint', ENDPOINT) +def shop() -> ModuleType: + return _module('shop_requests', SHOP) + + def key(case_id: str, database: str) -> str: return runner().case_key(case_id, database) @@ -620,6 +625,10 @@ def check() -> None: if case['input']['pql'].startswith('EXPLAIN '): assert 'explanation' in body['output']['fields'] assert_request_options(case, sent) + if expected[0] == 'success': + # Each call's context, anchors, labels and related rows, as + # the filter, ASSUMING and the target derive them. + shop().assert_sent(case, runner().load_bases(), sent) check() diff --git a/contract/tests/test_ranking.py b/contract/tests/test_ranking.py index 86754e6f5..3635f0f07 100644 --- a/contract/tests/test_ranking.py +++ b/contract/tests/test_ranking.py @@ -29,6 +29,7 @@ RUNNER = ROOT / 'contract/runners/run_python.py' RECORDER = ROOT / 'contract/support/ranking_scenarios.py' CASE_DIR = ROOT / 'contract/cases/s1' +SHOP = ROOT / 'contract/support/shop_requests.py' DATABASES = ('duckdb', 'sqlite', 'postgres') @@ -85,6 +86,10 @@ def recorder() -> ModuleType: return _module('ranking_scenarios', RECORDER) +def shop() -> ModuleType: + return _module('shop_requests', SHOP) + + def key(case_id: str, database: str) -> str: return runner().case_key(case_id, database) @@ -364,6 +369,9 @@ def check() -> None: not in body['predict']['instance_table']['columns'] ) if expect['outcome'] == 'success': + # One call per candidate, each candidate's membership in the + # window the label, over the context the anchors gather. + shop().assert_sent(case, runner().load_bases(), sent) result = line['public']['result'] assert result['rows'] == len( case['input']['materialize']['indices'] diff --git a/java/src/test/java/com/nvidia/sdm/BindingTest.java b/java/src/test/java/com/nvidia/sdm/BindingTest.java index fda7b4a19..80bb62bbd 100644 --- a/java/src/test/java/com/nvidia/sdm/BindingTest.java +++ b/java/src/test/java/com/nvidia/sdm/BindingTest.java @@ -196,38 +196,77 @@ void largeWarningsUseAttachmentsAndPreserveOrderUntilDrained() throws Exception } finally { Names.close(run); } } - /** Authored code/kind/sentence inputs cover each supported native error mapping. */ + private static Map text(String value) { + return Map.of("t", "utf8", "v", value); + } + + /** Each native error code reaches Java with its code and its sentence's exact words. */ @TestFactory Stream everyErrorCodeReachesJavaWithItsCode() { String[][] errors = { - new String[]{"ENDPOINT_REFUSED", "model_endpoint", "serving_refused"}, - new String[]{"INTERNAL_ERROR", "internal", "result_merge_empty"}, - new String[]{"INVALID_CONFIGURATION", "invalid_request", "invalid_configuration"}, - new String[]{"INVALID_MESSAGE", "protocol", "invalid_message"}, - new String[]{"INVALID_REQUEST", "invalid_request", "direct_empty"}, - new String[]{"INVALID_RESPONSE", "model_endpoint", "tabular_missing_row_index"}, - new String[]{"INVALID_SCHEMA", "invalid_request", "wire_reserved_one"}, - new String[]{"MISSING_EXTRA", "unsupported", "missing_extra"}, - new String[]{"PROVIDER_ERROR", "not_found", "provider_failed"}, - new String[]{"PROVIDER_SCHEMA_MISMATCH", "data_source", "provider_schema_mismatch"}, - new String[]{"REPLY_MISSING", "protocol", "reply_missing"}, - new String[]{"RUN_BUSY", "protocol", "run_busy"}, - new String[]{"RUN_CANCELLED", "cancelled", "run_cancelled"}, - new String[]{"RUN_FINISHED", "protocol", "run_finished"}, - new String[]{"RUN_POISONED", "protocol", "run_poisoned"}, - new String[]{"SESSIONS_UNSUPPORTED", "unsupported", "sessions_unsupported"}, - new String[]{"TRANSPORT_ERROR", "model_endpoint", "transport_failed"}, - new String[]{"UNKNOWN_MODEL", "not_found", "unknown_model"}, - new String[]{"UNKNOWN_REPLY", "protocol", "unknown_reply"}, - new String[]{"UNSUPPORTED_REQUEST", "unsupported", "direct_relational_frames"}, - new String[]{"UNSUPPORTED_TYPE", "unsupported", "unsupported_type"}, - new String[]{"VALUE_OUT_OF_RANGE", "unsupported", "lag_past_times"}, - new String[]{"WRONG_REPLY", "protocol", "wrong_reply"} + new String[]{"ENDPOINT_REFUSED", "model_endpoint", "serving_refused", + "serving endpoint \"fraud-scorer\" refused the request (UNSUPPORTED_TASK)"}, + new String[]{"INTERNAL_ERROR", "internal", "result_merge_empty", "no parts to merge"}, + new String[]{"INVALID_CONFIGURATION", "invalid_request", "invalid_configuration", + "the configuration of this call is not one the core accepts"}, + new String[]{"INVALID_MESSAGE", "protocol", "invalid_message", + "the host sent a message the core cannot read"}, + new String[]{"INVALID_REQUEST", "invalid_request", "direct_empty", + "context has no rows, so there is nothing for the model to learn the answer from"}, + new String[]{"INVALID_RESPONSE", "model_endpoint", "tabular_missing_row_index", + "kumo-tabular returned a response that does not match the contract: " + + "PredictionItem response is missing required field 'row_index'"}, + new String[]{"INVALID_SCHEMA", "invalid_request", "wire_reserved_one", + "orders.instance_id uses a name this client needs for the request it builds, so it cannot " + + "also be a column of the data. Rename it, for instance to \"instance_id_from_the_data\"."}, + new String[]{"MISSING_EXTRA", "unsupported", "missing_extra", + "an optional dependency this call needs is not installed"}, + new String[]{"PROVIDER_ERROR", "not_found", "provider_failed", "the data source failed"}, + new String[]{"PROVIDER_SCHEMA_MISMATCH", "data_source", "provider_schema_mismatch", + "the data source answered with columns other than the ones asked for"}, + new String[]{"REPLY_MISSING", "protocol", "reply_missing", + "the host went on before answering every request"}, + new String[]{"RUN_BUSY", "protocol", "run_busy", + "the run was already in a call; a run takes one call at a time"}, + new String[]{"RUN_CANCELLED", "cancelled", "run_cancelled", "the run was cancelled"}, + new String[]{"RUN_FINISHED", "protocol", "run_finished", "the run had already finished"}, + new String[]{"RUN_POISONED", "protocol", "run_poisoned", + "the run failed earlier and can no longer be used"}, + new String[]{"SESSIONS_UNSUPPORTED", "unsupported", "sessions_unsupported", + "the model endpoint does not support sessions"}, + new String[]{"TRANSPORT_ERROR", "model_endpoint", "transport_failed", + "the transport to the model endpoint failed: the endpoint could not be reached, " + + "or its answer could not be read"}, + new String[]{"UNKNOWN_MODEL", "not_found", "unknown_model", "the model named does not exist"}, + new String[]{"UNKNOWN_REPLY", "protocol", "unknown_reply", + "the host answered a request the core did not make"}, + new String[]{"UNSUPPORTED_REQUEST", "unsupported", "direct_relational_frames", + "a relational request has to be gathered first, and this one was not. Build it with " + + "PredictiveQuery(...).materialize(indices=[...], model='kumo-relational'). A schema of " + + "one table is gathered for the tabular model unless the relational one is named, rather " + + "than with Request(context=..., predict=...), which is the single-table form"}, + new String[]{"UNSUPPORTED_TYPE", "unsupported", "unsupported_type", + "column \"tags\" has the Arrow type Float16, which the core does not take"}, + new String[]{"VALUE_OUT_OF_RANGE", "unsupported", "lag_past_times", + "lag_timesteps reaches further back than a time can be held (before 1677); ask for fewer steps"}, + new String[]{"WRONG_REPLY", "protocol", "wrong_reply", + "the host answered a request with the wrong kind of reply"} }; + Map> fields = Map.of( + "ENDPOINT_REFUSED", Map.of( + "endpoint", text("fraud-scorer"), "endpoint_code", text("UNSUPPORTED_TASK")), + "INVALID_SCHEMA", Map.of( + "reserved", List.of(text("orders.instance_id")), + "renamed", List.of(text("instance_id_from_the_data"))), + "UNSUPPORTED_TYPE", Map.of("column", text("tags"), "arrow_type", text("Float16"))); return Stream.of(errors).map(error -> DynamicTest.dynamicTest(error[0], () -> { - byte[] input = com.nvidia.sdm.internal.Json.write(Map.of( + Map given = new LinkedHashMap<>(Map.of( "kind", error[1], "code", error[0], "sentence", error[2], "message", "native-free-text-sentinel", "retryable", false)); + if (fields.containsKey(error[0])) { + given.put("fields", fields.get(error[0])); + } + byte[] input = com.nvidia.sdm.internal.Json.write(given); Object run = Names.startRun( "{\"op\":\"test\",\"test\":\"fail\",\"error\":1}", Map.of(1, input)); try { @@ -236,14 +275,7 @@ Stream everyErrorCodeReachesJavaWithItsCode() { } catch (Exception raised) { assertTrue(Names.isStructuredDataException(raised)); assertEquals(error[0], Names.code(raised)); - assertTrue(raised.getMessage() != null && !raised.getMessage().isBlank()); - assertTrue(!raised.getMessage().contains("native-free-text-sentinel")); - if (error[0].equals("INVALID_REQUEST")) { - assertEquals("context has no rows, so there is nothing for the model to learn the answer from", raised.getMessage()); - } - if (error[0].equals("RUN_BUSY")) { - assertEquals("the run was already in a call; a run takes one call at a time", raised.getMessage()); - } + assertEquals(error[3], raised.getMessage(), error[0]); } finally { Names.close(run); } diff --git a/rust/crates/sdm-core/tests/pql_corpus.rs b/rust/crates/sdm-core/tests/pql_corpus.rs index e32fed7ba..bf3e5839b 100644 --- a/rust/crates/sdm-core/tests/pql_corpus.rs +++ b/rust/crates/sdm-core/tests/pql_corpus.rs @@ -444,6 +444,101 @@ fn diagnostic_rules_return_their_named_errors() { } } +#[test] +fn selected_errors_point_at_what_they_name() { + use sdm_contract::ops::Span; + let at = |start_col, end_col| { + Some(Span { + start_row: 1, + start_col, + end_row: 1, + end_col, + }) + }; + type Errors = Vec<(&'static str, Option)>; + let cases: Vec<(&str, &str, &str, Errors)> = vec![ + ( + "schema-1", + "RELATIONAL", + "PREDICT users.nope FOR EACH users.user_id", + vec![("PQL_SCHEMA_UNKNOWN_COLUMN", at(8, 8))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(order.*, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_SCHEMA_UNKNOWN_TABLE", at(14, 14))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.id", + vec![("PQL_SCHEMA_UNKNOWN_COLUMN", at(46, 46))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.status, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_AGGREGATION_NEEDS_NUMERIC", at(8, 38))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.product_id, 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_AGGREGATION_ON_KEY_COLUMN", at(8, 42))], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT COUNT(orders.* WHERE orders.amount > 'ten', 0, 30, days) FOR EACH users.user_id", + vec![("PQL_TYPE_CONSTANT_MISMATCH", at(29, 45))], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT COUNT(orders.* WHERE orders.quantity > true, 0, 30, days) FOR EACH users.user_id", + vec![ + ("PQL_TYPE_CONSTANT_MISMATCH", at(29, 47)), + ("PQL_TYPE_BOOL_COMPARISON", at(29, 47)), + ], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT SUM(orders.quantity, 0, 30, days) FORECAST 0 TIMEFRAMES FOR EACH users.user_id", + vec![("PQL_TASK_FORECAST_NOT_POSITIVE", None)], + ), + ( + "schema-1", + "RELATIONAL", + "PREDICT LIST_DISTINCT(orders.product_id, 0, 30, days) RANK TOP 0 FOR EACH users.user_id", + vec![("PQL_TASK_TOP_K_NOT_POSITIVE", None)], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT LIST_DISTINCT(orders.product_id, 0, 30, days) RANK FOR EACH users.user_id", + vec![("PQL_TASK_RANK_REQUIRES_TOP_K", at(68, 68))], + ), + ( + "schema-1", + "FINE_TUNING", + "PREDICT products.tags FOR EACH products.product_id", + vec![("PQL_TASK_PROBLEM_TYPE_REQUIRED", at(31, 31))], + ), + ]; + for (which, model, text, expected) in cases { + let got = resolve(text, &schema(which), &profile(model), &facts(), &NoZones).unwrap(); + let errors: Vec<_> = got + .diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .map(|d| (d.code.as_str(), d.span)) + .collect(); + assert_eq!(errors, expected, "{text}"); + } +} + fn nodes(node: &sdm_contract::pql::PqlNode) -> Vec<&sdm_contract::pql::NodeKind> { use sdm_contract::pql::NodeKind::*; let mut out = vec![&node.kind]; diff --git a/rust/crates/sdm-core/tests/pql_fuzz.rs b/rust/crates/sdm-core/tests/pql_fuzz.rs index cf5597e09..7d15543f8 100644 --- a/rust/crates/sdm-core/tests/pql_fuzz.rs +++ b/rust/crates/sdm-core/tests/pql_fuzz.rs @@ -21,6 +21,7 @@ mod pql_support; use std::time::{Duration, Instant}; use pql_support::{NoZones, facts, profile, schema}; +use sdm_contract::{Detail, ErrorCode, Sentence}; use sdm_core::pql::validate; /// The seed and the number of inputs of each kind. @@ -230,6 +231,29 @@ fn the_parser_answers_every_fuzzed_input_without_a_panic() { accepted ); } + // A run of five digits is a packed date (D8), refused by name as a + // structured error rather than read or given a diagnostic. + let (packed, schema_name, profile_name) = seeds[7]; + let error = validate( + packed, + &schema(schema_name), + &profile(profile_name), + &facts(), + &NoZones, + ) + .expect_err(packed); + assert_eq!( + (error.code, error.sentence), + (ErrorCode::InvalidRequest, Sentence::DateLiteralPacked) + ); + assert_eq!( + error.fields.get("rule"), + Some(&Detail::text("date_literal_packed")) + ); + assert_eq!( + error.fields.get("literal"), + Some(&Detail::text("12345-1-1")) + ); let mut rng = Rng(SEED); // Each input with the schema and profile it is validated with: a // mutated case keeps its own; the rest take the first schema, which diff --git a/rust/crates/sdm-testkit/tests/decimal_sql.rs b/rust/crates/sdm-testkit/tests/decimal_sql.rs index 53d819cc2..5fc61a454 100644 --- a/rust/crates/sdm-testkit/tests/decimal_sql.rs +++ b/rust/crates/sdm-testkit/tests/decimal_sql.rs @@ -95,3 +95,47 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// A summed decimal is the label read's one value column; a wide decimal +/// feature is read by the entity lookup; a key past `BIGINT` joins each +/// keyed read through the seed cast to the key's type. +#[test] +fn decimal_reads_project_their_columns_and_cast_wide_keys() { + let seen: std::collections::BTreeMap = observed() + .into_iter() + .map(|(case, seen)| (case.id, seen)) + .collect(); + let summed = ran(&seen["decimal.sum-10-2"], "materialize"); + assert!( + summed.iter().any(|sql| sql.starts_with( + "SELECT s.\"batch\" AS \"__kumo_batch__\", t.\"ts\", t.\"amount\"\nFROM \"__sdm_seeds__\" s\n" + )), + "{summed:#?}" + ); + let wide = ran(&seen["decimal.feature-38-0-wide"], "materialize"); + assert!( + wide.iter().any(|sql| sql.contains( + "t.\"user_id\", t.\"age\", t.\"country\", t.\"f\"\nFROM \"__sdm_seeds__\" s\nJOIN \"users\" t" + )), + "{wide:#?}" + ); + let keyed = ran(&seen["decimal.key-wide"], "materialize"); + let cast = r#"ON t."user_id" = TRY_CAST(s."key" AS DECIMAL(38,0))"#; + assert_eq!( + keyed.iter().filter(|sql| sql.contains(cast)).count(), + 3, + "{keyed:#?}" + ); +} diff --git a/rust/crates/sdm-testkit/tests/duckdb_provider.rs b/rust/crates/sdm-testkit/tests/duckdb_provider.rs index a7f76ec8e..d23d9a226 100644 --- a/rust/crates/sdm-testkit/tests/duckdb_provider.rs +++ b/rust/crates/sdm-testkit/tests/duckdb_provider.rs @@ -1071,6 +1071,61 @@ fn materialization_records_complete_encodable_database_reads() { assert!(comparable > 0); assert_eq!(logged["compared"].as_u64(), Some(comparable as u64)); assert_eq!(logged["differing"], serde_json::json!([])); + // The journey's budget: ten reads describe the source; materialize + // re-reads the two catalogs (plan B's refreshes) and runs six more. + let per_stage = |stage: &str| statements.iter().filter(|s| s["stage"] == stage).count(); + assert_eq!( + ( + per_stage("from_source"), + per_stage("materialize"), + per_stage("predict") + ), + (10, 8, 0) + ); + let refreshes = statements + .iter() + .filter(|s| s["purpose"] == sdm_testkit::plan_b::LOCATE) + .count(); + assert_eq!(refreshes, 2); + assert_eq!(logged["statements"].as_u64(), Some(18)); + // One whole entity lookup and one whole neighbor read, each with the + // key set it binds. + let (lookup, neighbors) = keyed_reads(&observed.fetches); + assert_eq!( + lookup, + serde_json::json!([{ + "dialect": "duckdb", + "text": r#"SELECT s."batch" AS "__kumo_batch__", COUNT(*) OVER (PARTITION BY s."batch") AS "__kumo_matches__", t."user_id", t."age", t."country" +FROM "__sdm_seeds__" s +JOIN "users" t + ON t."user_id" = s."key" +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY s."batch" +) = 1"#, + "params": [key_set(2, &[])], + }]) + ); + assert_eq!( + neighbors, + serde_json::json!([{ + "dialect": "duckdb", + "text": r#"SELECT s."batch" AS "__kumo_batch__", t."order_id", t."user_id", t."price", t."ts" +FROM "__sdm_seeds__" s +JOIN "orders" t + ON t."user_id" = s."key" + AND CAST(t."ts" AS TIMESTAMP) <= s."time" +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY s."batch" + ORDER BY CAST(t."ts" AS TIMESTAMP) DESC NULLS LAST, t."order_id" ASC NULLS LAST +) <= 32 +ORDER BY s."batch", CAST(t."ts" AS TIMESTAMP) DESC NULLS LAST, t."order_id" ASC NULLS LAST"#, + "params": [key_set(3, &[serde_json::json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + })])], + }]) + ); for stage in ["from_source", "materialize"] { assert!( statements @@ -1097,6 +1152,35 @@ fn materialization_records_complete_encodable_database_reads() { } } +/// The planned statements of a run's entity lookup and of its neighbor +/// read with a per-seed cap. +fn keyed_reads(fetches: &[Value]) -> (Value, Value) { + let planned = |wanted: &dyn Fn(&Value) -> bool| { + let found: Vec<_> = fetches.iter().filter(|f| wanted(&f["request"])).collect(); + assert_eq!(found.len(), 1, "one such read"); + found[0]["plan"]["statements"].clone() + }; + ( + planned(&|r| r["request"] == "lookup"), + planned(&|r| r["request"] == "neighbors" && !r["per_seed"].is_null()), + ) +} + +/// A key-set parameter bound as `__sdm_seeds__`: its attachment, the key +/// and batch columns, then `more`. +fn key_set(attachment: u64, more: &[Value]) -> Value { + let mut columns = vec![ + serde_json::json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + serde_json::json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + serde_json::json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": "__sdm_seeds__", + }) +} + /// The answer to the one statement the shipped data layer plans for the /// columns of `table` on DuckDB, and that statement. fn columns_of(connection: &Connection, table: &str) -> (String, Vec) { diff --git a/rust/crates/sdm-testkit/tests/empty_null_sql.rs b/rust/crates/sdm-testkit/tests/empty_null_sql.rs index 7ad47b32b..6ff2c3a03 100644 --- a/rust/crates/sdm-testkit/tests/empty_null_sql.rs +++ b/rust/crates/sdm-testkit/tests/empty_null_sql.rs @@ -94,3 +94,52 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// Nulls and empty facts leave every windowed read whole: the label window's +/// two cast bounds, and the neighbors' cast bound, nulls-last newest-first +/// order and per-seed cap; an all-null feature is still read by the lookup. +#[test] +fn null_and_empty_reads_keep_their_window_order_and_projection() { + let cast = r#"CAST(t."ts" AS TIMESTAMP)"#; + let mut checked = 0; + for (case, seen) in observed() { + if seen.answer.outcome().0 != "success" { + continue; + } + let materialized = ran(&seen, "materialize"); + let reads = |text: &str| materialized.iter().any(|sql| sql.contains(text)); + assert!( + reads(&format!("{cast} <= s.\"end\"\n AND {cast} > s.\"start\"")), + "{}: the label window", + case.id + ); + assert!( + reads(&format!( + "{cast} <= s.\"time\"\nQUALIFY ROW_NUMBER() OVER (\n PARTITION BY s.\"batch\"\n ORDER BY {cast} DESC NULLS LAST, t.\"order_id\" ASC NULLS LAST\n) <= 32" + )), + "{}: the neighbors", + case.id + ); + if case.id == "null.feature-all-null" { + assert!( + reads("t.\"user_id\", t.\"age\", t.\"country\", t.\"f\"\nFROM"), + "{}: the all-null feature", + case.id + ); + } + checked += 1; + } + assert_eq!(checked, 5); +} diff --git a/rust/crates/sdm-testkit/tests/nested_sql.rs b/rust/crates/sdm-testkit/tests/nested_sql.rs index bd280b508..7ba5031c9 100644 --- a/rust/crates/sdm-testkit/tests/nested_sql.rs +++ b/rust/crates/sdm-testkit/tests/nested_sql.rs @@ -115,3 +115,43 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// A list column is read by the preview, which describes it, and left out +/// of the entity lookup, which sends no list. +#[test] +fn a_list_column_is_previewed_and_left_out_of_the_lookup() { + let mut checked = 0; + for (case, seen) in observed() { + if !matches!(case.id.as_str(), "nested.list-int" | "nested.fixed-array") { + continue; + } + assert!( + ran(&seen, "from_source").contains( + &r#"SELECT "user_id", "age", "country", "f" FROM "users" LIMIT 1000"#.to_owned() + ), + "{}: the preview", + case.id + ); + assert!( + ran(&seen, "materialize").iter().any(|sql| sql.contains( + "t.\"user_id\", t.\"age\", t.\"country\"\nFROM \"__sdm_seeds__\" s\nJOIN \"users\" t" + )), + "{}: the lookup", + case.id + ); + checked += 1; + } + assert_eq!(checked, 2); +} diff --git a/rust/crates/sdm-testkit/tests/postgres_dialect.rs b/rust/crates/sdm-testkit/tests/postgres_dialect.rs index 0a6d16ce4..9ad200063 100644 --- a/rust/crates/sdm-testkit/tests/postgres_dialect.rs +++ b/rust/crates/sdm-testkit/tests/postgres_dialect.rs @@ -29,7 +29,7 @@ use sdm_contract::data::{ DataRequest, DescribePart, Dialect, FetchPlan, ProviderChoice, Relation, SqlParam, }; use sdm_contract::{Detail, ErrorCode}; -use serde_json::Value; +use serde_json::{Value, json}; fn contract() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../contract") @@ -170,3 +170,123 @@ fn a_relation_of_no_part_or_of_three_is_refused_by_name() { } } } + +/// A key-set parameter bound as `__sdm_seeds__`: its attachment, the key +/// and batch columns, then `more`. +fn key_set(attachment: u64, more: &[Value]) -> Value { + let mut columns = vec![ + json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": "__sdm_seeds__", + }) +} + +/// The statements `request` plans on Postgres, as the contract writes them. +fn planned(request: Value) -> Value { + let request: DataRequest = serde_json::from_value(request).unwrap(); + let FetchPlan::Sql { statements, .. } = plan(&request).unwrap() else { + panic!("not SQL"); + }; + serde_json::to_value(statements).unwrap() +} + +/// base.regression's entity lookup and its neighbor read, as Postgres +/// renders them whole: the key set bound once as JSON, each column read, +/// the time bound and the per-seed cap, newest first. +#[test] +fn a_lookup_and_a_neighbor_read_render_whole_with_their_key_sets() { + let time = json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + }); + let column = |name: &str| json!({"name": name, "column": name}); + let lookup = planned(json!({ + "request": "lookup", + "table": { + "source": ["users"], + "columns": [column("user_id"), column("age"), column("country")], + "found": true, + }, + "key": ["user_id"], + "columns": ["user_id", "age", "country"], + "seeds": {"attachment": 2, "columns": key_set(2, &[])["keys"]["columns"]}, + })); + assert_eq!( + lookup, + json!([{ + "dialect": "postgres", + "text": r#"WITH "__sdm_seeds__" AS ( + SELECT (r.value #>> '{}')::bigint AS "key", (r.ordinality - 1)::bigint AS "batch" + FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS r(value, ordinality) +) +SELECT l.b AS "__kumo_batch__", l.c0 AS "user_id", l.c1 AS "age", l.c2 AS "country", l.r AS "__kumo_row__", l.m AS "__kumo_matches__" +FROM ( + SELECT s."batch" AS b, t."user_id" AS c0, t."age" AS c1, t."country" AS c2, + ROW_NUMBER() OVER (PARTITION BY s."batch" + ORDER BY t."user_id") AS r, + COUNT(*) OVER (PARTITION BY s."batch") AS m + FROM "__sdm_seeds__" s + JOIN "public"."users" t + ON t."user_id" = s."key" +) AS l +WHERE l.r = 1 +ORDER BY l.b"#, + "params": [key_set(2, &[])], + }]) + ); + let neighbors = planned(json!({ + "request": "neighbors", + "table": { + "source": ["orders"], + "columns": [column("order_id"), column("user_id"), column("price"), column("ts")], + "found": true, + }, + "key": ["user_id"], + "columns": ["order_id", "user_id", "price", "ts"], + "seeds": { + "attachment": 3, + "columns": key_set(3, std::slice::from_ref(&time))["keys"]["columns"], + }, + "window": { + "column": "ts", + "source_type": "timestamp without time zone", + "lower": null, + "upper": {"seed_column": "time", "inclusive": true}, + }, + "per_seed": 32, + "order": [ + {"column": "ts", "as_timestamp": true, "descending": true, "nulls_last": true}, + {"column": "order_id", "as_timestamp": false, "descending": false, "nulls_last": true}, + ], + })); + assert_eq!( + neighbors, + json!([{ + "dialect": "postgres", + "text": r#"WITH "__sdm_seeds__" AS ( + SELECT (r.value->>'id')::bigint AS "key", (r.ordinality - 1)::bigint AS "batch", w2."ts" AS "time" + FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS r(value, ordinality) + CROSS JOIN (SELECT ROW(t.*)::"public"."orders" AS base FROM "public"."orders" AS t LIMIT 1) AS b + CROSS JOIN LATERAL jsonb_populate_record(b.base, jsonb_build_object('ts', r.value->'e')) AS w2 +) +SELECT s."batch" AS "__kumo_batch__", n.c0 AS "order_id", n.c1 AS "user_id", n.c2 AS "price", n.c3 AS "ts", n.r AS "__kumo_row__" +FROM "__sdm_seeds__" s +CROSS JOIN LATERAL ( + SELECT t."order_id" AS c0, t."user_id" AS c1, t."price" AS c2, t."ts" AS c3, ROW_NUMBER() OVER (ORDER BY t."ts" DESC NULLS LAST, t."order_id" ASC NULLS LAST) AS r + FROM "public"."orders" t + WHERE t."user_id" = s."key" + AND t."ts" <= s."time" + ORDER BY t."ts" DESC NULLS LAST, t."order_id" ASC NULLS LAST + LIMIT 32 +) AS n +ORDER BY s."batch", n.r"#, + "params": [key_set(3, &[time])], + }]) + ); +} diff --git a/rust/crates/sdm-testkit/tests/sqlite.rs b/rust/crates/sdm-testkit/tests/sqlite.rs index d17852d3a..5d16f58f8 100644 --- a/rust/crates/sdm-testkit/tests/sqlite.rs +++ b/rust/crates/sdm-testkit/tests/sqlite.rs @@ -2,10 +2,16 @@ // All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! SQLite relation names and invalid qualification through the native provider. +//! SQLite relation names and invalid qualification through the native provider, +//! and one case's journey on SQLite: its read budget and its keyed reads whole. + +use std::path::PathBuf; use sdm_contract::data::{DataRequest, DescribePart, Dialect, FetchPlan, ProviderChoice, Relation}; use sdm_contract::{Detail, ErrorCode}; +use sdm_testkit::host::{self, Answer}; +use sdm_testkit::{bases, case}; +use serde_json::{Value, json}; const DATABASE: &str = "sqlite"; @@ -118,3 +124,103 @@ fn a_relation_of_no_part_or_of_three_is_refused_by_name_on_sqlite() { } } } + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// A key-set parameter: its attachment, its relation, the key and batch +/// columns, then `more`. +fn key_set(attachment: u64, relation: &str, more: &[Value]) -> Value { + let mut columns = vec![ + json!({"name": "key", "type": {"type": "int64"}, "nullable": true}), + json!({"name": "batch", "type": {"type": "int64"}, "nullable": false}), + ]; + columns.extend(more.iter().cloned()); + json!({ + "param": "key_set", + "keys": {"attachment": attachment, "columns": columns}, + "relation": relation, + }) +} + +/// base.regression on SQLite, as the DuckDB journey runs it: a fixed budget +/// of fetches, plan B's refreshes and statements, and the entity lookup and +/// the neighbor read whole, with the key sets they bind. +#[test] +fn base_regression_s_journey_reads_a_fixed_budget_on_sqlite() { + let contract = root().join("contract"); + let loaded = case::load(&[contract.join("cases/s1/base.regression.json")]).unwrap(); + let bases = bases::load(&contract.join("bases.yaml")).unwrap(); + let observed = host::run_sqlite(&loaded[0].case, &bases, &root()).unwrap(); + assert!(matches!(observed.answer, Answer::Predicted(_))); + // Each table's catalog, keys and preview are read again at materialize. + let refreshes = observed + .fetches + .iter() + .filter(|f| f["purpose"] == sdm_testkit::plan_b::LOCATE) + .count(); + assert_eq!((observed.fetches.len(), refreshes), (24, 6)); + let driver = observed.driver.as_ref().expect("a driver record"); + let statements = driver["capture"]["statements"].as_array().unwrap(); + let per_stage = |stage: &str| statements.iter().filter(|s| s["stage"] == stage).count(); + assert_eq!( + ( + per_stage("from_source"), + per_stage("materialize"), + per_stage("predict") + ), + (10, 14, 0) + ); + assert_eq!(driver["query_log"]["statements"].as_u64(), Some(24)); + assert_eq!(driver["capture"]["bindings"].as_array().unwrap().len(), 3); + let planned = |wanted: &dyn Fn(&Value) -> bool| { + let found: Vec<_> = observed + .fetches + .iter() + .filter(|f| wanted(&f["request"])) + .collect(); + assert_eq!(found.len(), 1, "one such read"); + found[0]["plan"]["statements"].clone() + }; + assert_eq!( + planned(&|r| r["request"] == "lookup"), + json!([{ + "dialect": "sqlite", + "text": r#"SELECT s."batch" AS "__kumo_batch__", (SELECT COUNT(*) FROM "users" WHERE "user_id" == s."key") AS "__kumo_matches__", t."user_id", t."age", t."country" +FROM temp."__sdm_seeds_19f857a1097d1e84__" s +JOIN "users" t + ON t.rowid = ( + SELECT rowid + FROM "users" + WHERE "user_id" == s."key" + LIMIT 1 +)"#, + "params": [key_set(2, "__sdm_seeds_19f857a1097d1e84__", &[])], + }]) + ); + assert_eq!( + planned(&|r| r["request"] == "neighbors" && !r["per_seed"].is_null()), + json!([{ + "dialect": "sqlite", + "text": r#"SELECT s."batch" AS "__kumo_batch__", t."order_id", t."user_id", t."price", t."ts" +FROM temp."__sdm_seeds_0086307c4757992b__" s +JOIN "orders" t +ON t.rowid IN ( + SELECT rowid + FROM "orders" + WHERE "user_id" = s."key" + AND "ts" <= substr(s."time", 1, 10) || ' ' || substr(s."time", 12, 8) + ORDER BY "ts" DESC, "order_id" + LIMIT 32 +) +ORDER BY "__kumo_batch__", t."ts" DESC, t."order_id""#, + "params": [key_set(3, "__sdm_seeds_0086307c4757992b__", &[json!({ + "name": "time", + "type": {"type": "timestamp", "unit": "ns", "zone": null}, + "nullable": false, + "metadata": {"sdm:stored_time": "2023-01-01 00:00:00"}, + })])], + }]) + ); +} diff --git a/rust/crates/sdm-testkit/tests/time_feature_sql.rs b/rust/crates/sdm-testkit/tests/time_feature_sql.rs index 50668e621..8c2560b13 100644 --- a/rust/crates/sdm-testkit/tests/time_feature_sql.rs +++ b/rust/crates/sdm-testkit/tests/time_feature_sql.rs @@ -103,3 +103,58 @@ fn family_cases_execute_with_expected_outcomes_and_encodable_results() { } } } + +/// The texts `seen`'s `stage` ran. +fn ran(seen: &Observed, stage: &str) -> Vec { + let driver = seen.driver.as_ref().expect("a driver record"); + driver["capture"]["statements"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["stage"] == stage) + .map(|s| s["sql"].as_str().unwrap().to_owned()) + .collect() +} + +/// Each answered case reads its time column through a cast to `TIMESTAMP` +/// wherever it bounds or orders by it: the range of times, both bounds of +/// the label window, and the neighbors' upper bound and newest-first order. +#[test] +fn each_time_column_is_cast_in_every_bound_and_order() { + let mut checked = 0; + for (case, seen) in observed() { + if seen.answer.outcome().0 != "success" { + continue; + } + let time = if case.id == "time.text-declared" { + "happened" + } else { + "ts" + }; + let cast = format!(r#"CAST(t."{time}" AS TIMESTAMP)"#); + let materialized = ran(&seen, "materialize"); + let reads = |text: &str| materialized.iter().any(|sql| sql.contains(text)); + assert!( + reads(&format!(r#"MIN(CAST("{time}" AS TIMESTAMP)) as min_date"#)), + "{}: the range of times", + case.id + ); + assert!( + reads(&format!("{cast} <= s.\"end\"\n AND {cast} > s.\"start\"")), + "{}: the label window", + case.id + ); + assert!( + reads(&format!("{cast} <= s.\"time\"")), + "{}: the neighbors' bound", + case.id + ); + assert!( + reads(&format!("ORDER BY {cast} DESC NULLS LAST")), + "{}: newest first", + case.id + ); + checked += 1; + } + assert_eq!(checked, 9); +} diff --git a/tests/test_contract_cases.py b/tests/test_contract_cases.py index b342bc0a1..255689f9b 100644 --- a/tests/test_contract_cases.py +++ b/tests/test_contract_cases.py @@ -11,6 +11,7 @@ import importlib.util import json import re +import sys from decimal import Decimal from pathlib import Path from types import ModuleType @@ -133,6 +134,23 @@ def test_a_case_sets_its_time_zone_for_every_cursor() -> None: _VECTOR_SETUP_KEYS = ('setup', 'witness') +#: Zone statements a vector holds beside its setup, which its readers run +#: on the setup's database: provider-vectors.json's global zone and the +#: session zone it shows the source does not read in. +_VECTOR_ZONE_KEYS = ('zone', 'session_zone') + + +#: The DuckDB scenario setups held in Python, by module and the name it +#: holds them under; ranking's are those naming no other database. +_SCENARIO_SETUPS = ( + ('decimal_vectors', 'VECTORS'), + ('duckdb_value_scenarios', 'SOURCES'), + ('empty_null_vectors', 'VECTORS'), + ('error_wording_scenarios', 'PROBES'), + ('ranking_scenarios', 'VECTORS'), +) + + _VECTOR_ZONE_PENDING: set[str] = set() @@ -168,11 +186,22 @@ def _vector_setups(node: Any, where: str) -> list[tuple[str, list[str]]]: found: list[tuple[str, list[str]]] = [] if isinstance(node, dict): where = str(node.get('id', where)) + zones = [ + node[key] + for key in _VECTOR_ZONE_KEYS + if isinstance(node.get(key), str) + ] for key, value in node.items(): if key in _VECTOR_SETUP_KEYS and isinstance(value, (str, list)): statements = [value] if isinstance(value, str) else value found.append( - (where, [s for s in statements if isinstance(s, str)]) + ( + where, + [ + *zones, + *(s for s in statements if isinstance(s, str)), + ], + ) ) found += _vector_setups(value, where) elif isinstance(node, list): @@ -181,20 +210,78 @@ def _vector_setups(node: Any, where: str) -> list[tuple[str, list[str]]]: return found +def _support(stem: str) -> ModuleType: + if stem not in sys.modules: + path = ROOT / 'contract/support' / f'{stem}.py' + spec = importlib.util.spec_from_file_location(stem, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[stem] = module + spec.loader.exec_module(module) + return sys.modules[stem] + + +def _scenario_setups() -> list[tuple[str, list[str]]]: + r"""Each DuckDB scenario setup held in Python (:data:`_SCENARIO_SETUPS` + and the giant-value sources), named by module and scenario.""" + found: list[tuple[str, list[str]]] = [] + for stem, name in _SCENARIO_SETUPS: + held = getattr(_support(stem), name) + entries = held.items() if isinstance(held, dict) else enumerate(held) + for key, entry in entries: + if isinstance(entry, list): + found.append((f'{stem} {key}', entry)) + elif entry.get('database', 'duckdb') == 'duckdb': + where = str(entry.get('name', key)) + found += [ + (f'{stem} {each}', setup) + for each, setup in _vector_setups(entry, where) + ] + values = _support('duckdb_value_scenarios') + found += [ + ( + f'duckdb_value_scenarios big.{kind}.{mode}', + values.big_setup(kind, mode), + ) + for kind in values.BIG_KINDS + for mode in values.BIG_MODES + ] + return found + + def test_a_vector_sets_its_time_zone_for_every_cursor() -> None: # Setup and source reads use separate cursors. Set the database-wide # timezone so results do not depend on each cursor's process-local zone. + # The vector files and the DuckDB scenarios held in Python both name it + # only in the allowed forms, and a setup holding zoned timestamps pins + # it globally. + setups = [ + (f'{path.name} {vector}', setup) + for path in sorted((ROOT / 'contract').glob('*-vectors.json')) + for vector, setup in _vector_setups( + json.loads(path.read_text(encoding='utf-8')), '?' + ) + ] + scenarios = _scenario_setups() problems = set() named = 0 - for path in sorted((ROOT / 'contract').glob('*-vectors.json')): - document = json.loads(path.read_text(encoding='utf-8')) - for vector, setup in _vector_setups(document, '?'): - named += sum(bool(_ZONE.search(each)) for each in setup) - problems.update( - _vector_zone_problems(f'{path.name} {vector}', setup) - ) + for where, setup in [*setups, *scenarios]: + named += sum(bool(_ZONE.search(each)) for each in setup) + problems.update(_vector_zone_problems(where, setup)) + problems.update(_unzoned(where, setup)) assert problems == _VECTOR_ZONE_PENDING assert named > len(_VECTOR_ZONE_PENDING), 'no vector sets its zone' + # Not vacuous: the probes and the zoned empty-null scenarios are read. + zoned = { + where + for where, setup in scenarios + if any(_ZONED.search(each) for each in setup) + } + assert { + 'error_wording_scenarios infinity.timestamptz.positive', + 'empty_null_vectors null.time-timestamptz', + } <= zoned, sorted(zoned) + assert len(scenarios) > 300, len(scenarios) # Seeded: a session SET passes only beside a global one, in the same # setup; nothing else does. pinned = "SET GLOBAL TimeZone = 'UTC'" @@ -214,6 +301,12 @@ def test_a_vector_sets_its_time_zone_for_every_cursor() -> None: [pinned], ): assert not _vector_zone_problems('seed', setup), setup + # Seeded: zoned timestamps need the global pin; a session one alone, + # or none, fails. + zoned_table = "CREATE TABLE t AS SELECT TIMESTAMPTZ '2024-01-01' AS ts" + assert _unzoned('seed', [zoned_table]) + assert _unzoned('seed', ["SET TimeZone = 'UTC'", zoned_table]) + assert not _unzoned('seed', [pinned, zoned_table]) def test_every_committed_case_validates_with_a_unique_id() -> None: