diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c7f5fd2b..3ad44bb38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,9 +166,23 @@ jobs: # index, since the default one resolves to a far larger CUDA build. - run: | pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install -e './packages/structured-data-models-client[test,in-process]' + pip install -e './packages/structured-data-models-client[test]' + pip install "structured-data-models @ git+https://github.com/NVIDIA/structured-data-models.git@${SDM_REF}" python -m pytest packages/structured-data-models-client/tests -q \ -m 'not live_nim and not live_warehouse' + env: + # The time-series models are on structured-data-models' main and in + # no release of it, so the `in-process` extra cannot reach them: its + # range names published versions, and main's own version string is + # below the floor of that range. This commit is installed instead, + # which is why the extra is not asked for here. + # + # It is pinned here rather than in the package's metadata on + # purpose. A direct URL in `optional-dependencies` is rejected at + # upload, so declaring it there would trade a test that runs for a + # release that cannot be published. Move the extra's range forward + # and delete this once a release carries the models. + SDM_REF: b5c36413dd30bcb93b39bfed2afdccf3e509781d # And the root suite, which holds the relational wire snapshots. They # only exercise anything with an engine installed, so running them in # the test job would prove nothing. diff --git a/packages/structured-data-models-client/pyproject.toml b/packages/structured-data-models-client/pyproject.toml index 9dfbb274b..920cd3f6e 100644 --- a/packages/structured-data-models-client/pyproject.toml +++ b/packages/structured-data-models-client/pyproject.toml @@ -78,7 +78,11 @@ s3 = ["structured-data-models-connectors[s3]==1.0.0"] # model rather than a backend to read, so `all` does not carry it; ask for it by # name. databricks-serving = ["databricks-sdk>=0.68.0,<1.0"] -# Running the models in this process rather than reaching a NIM. Like +# Running the models in this process rather than reaching a NIM. The range +# below names published releases, and none of them carries the time-series +# models, so `KumoAnomaly` needs that package installed from its own main +# until a release does. Widen this and drop the pin in `ci.yml` then. +# Like # `databricks-serving` it is a way to reach a model, and it brings PyTorch, so # `all` does not carry it. structured-data-models needs Python 3.11; on 3.10 # the extra installs nothing, and `SDMClient.in_process()` says why. The marker diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index e861ef39b..1b20c794c 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -35,7 +35,12 @@ Explanations, GradientExplanations, ) -from sdm_client.models import KumoRelational, KumoTabular, Model +from sdm_client.models import ( + KumoAnomaly, + KumoRelational, + KumoTabular, + Model, +) from sdm_client.query import PredictiveQuery from sdm_client.request import Request, Split, Target, Task from sdm_client.result import PredictionResult @@ -54,6 +59,7 @@ 'Explanations', 'FittedModel', 'GradientExplanations', + 'KumoAnomaly', 'KumoRelational', 'KumoTabular', 'MissingExtraError', diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py b/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py index 9519e2f24..de347bcb2 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py @@ -4,6 +4,7 @@ """Per-model adapters. One module per model.""" +from sdm_client.adapters.anomaly import AnomalyAdapter from sdm_client.adapters.tabular import TabularAdapter -__all__ = ['TabularAdapter'] +__all__ = ['AnomalyAdapter', 'TabularAdapter'] diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py new file mode 100644 index 000000000..7725e3a23 --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Shaping a request for ``kumo-anomaly``. + +Short, because this model is asked the least of any here. There is no target +column to type, no context to agree with the rows being scored, and no session +to pin, so what is left is the schema the readings are written against. + +The envelope still carries an empty ``context``. The contract requires that +section, and a model with nothing to learn from has a context with no rows +rather than no context, which is a thing the envelope can say. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator, Mapping, Sequence +from typing import Any + +from sdm_api.generated.tfm_api import ( + PredictionItem, + PredictionResponse, + ResponseContractError, +) + +from sdm_client.adapters.single_table import ( + MEANING_OF, + coherent_stype, + named_entities, +) +from sdm_client.base import ModelSpec +from sdm_client.errors import StructuredDataError, UnsupportedRequestError +from sdm_client.models import KUMO_ANOMALY +from sdm_client.request import Request +from sdm_client.result import PredictionResult +from sdm_client.wire import ( + UNTYPED, + encode_table, + infer_tfm_dtype, + json_ready, + widen_tfm_dtype, +) +from sdm_client.wire.inference import typed_inference + +_DEFAULT_OUTPUTS = ('prediction', 'probabilities') + + +def _request_id() -> str: + return f'sdm-{uuid.uuid4().hex[:12]}' + + +class AnomalyAdapter: + r"""Shapes and reads ``kumo-anomaly`` requests.""" + + spec: ModelSpec = KUMO_ANOMALY + + def default_output(self, request: Request) -> tuple[str, ...]: + return _DEFAULT_OUTPUTS + + def prediction_plan( + self, request: Request, outputs: tuple[str, ...] + ) -> Iterator[dict[str, Any]]: + r"""One payload. The score for a row depends on the rows around it, + so splitting a series into batches would change the answer.""" + yield self._envelope(request, outputs) + + def session_plan( + self, request: Request, outputs: tuple[str, ...] + ) -> dict[str, Any]: + raise UnsupportedRequestError( + f'{self.spec.model_id} holds no session: it is shown no labelled ' + f'rows, so there is no fitted context to pin and score against. ' + f'Call predict(), which sends the readings with every call' + ) + + def session_prediction_plan( + self, + request: Request, + outputs: tuple[str, ...], + pinned: Mapping[str, Any] | None = None, + ) -> Iterator[dict[str, Any]]: + raise UnsupportedRequestError(f'{self.spec.model_id} holds no session') + + def normalize( + self, + response: dict[str, Any], + outputs: tuple[str, ...], + payload: dict[str, Any] | None = None, + ) -> PredictionResult: + try: + parsed = PredictionResponse.from_dict(response) + except (ResponseContractError, TypeError, ValueError) as error: + raise StructuredDataError( + f'{self.spec.model_id} returned a response that does not ' + f'match the contract: {error}', + code='INVALID_RESPONSE', + ) from error + return self.answered( + parsed.predictions, + outputs, + payload, + model=parsed.model, + request_id=parsed.id, + metadata=parsed.metadata, + ) + + def answered( + self, + items: Sequence[PredictionItem], + outputs: tuple[str, ...], + payload: dict[str, Any] | None, + *, + model: str, + request_id: str = '', + metadata: Mapping[str, Any] | None = None, + ) -> PredictionResult: + return PredictionResult( + model=model, + request_id=request_id, + items=named_entities(items, payload), + metadata=dict(metadata or {}), + outputs=outputs, + ) + + def explain_error( + self, error: Exception, payload: dict[str, Any] + ) -> Exception: + r"""Nothing to add: a refusal here is about the readings, which the + caller handed over whole.""" + return error + + def close(self) -> None: + r"""Nothing to release.""" + + def _envelope( + self, request: Request, outputs: tuple[str, ...] + ) -> dict[str, Any]: + if request.payloads is not None: + raise UnsupportedRequestError( + f'this request was gathered for another model and carries ' + f'the sampled payloads that model reads; ' + f'{self.spec.model_id} is asked with Request.anomaly(...)' + ) + request.schema.check_consistent() + + table = request.schema[request.entity_table] + readings = request.predict.instance_table + + dtypes = { + name: (UNTYPED if column.untyped else column.dtype) + for name, column in table.columns.items() + } + for name in readings.columns: + if name in dtypes: + dtypes[name] = widen_tfm_dtype( + (dtypes[name], infer_tfm_dtype(readings[name])) + ) + dtypes = {name: dtype or 'string' for name, dtype in dtypes.items()} + + missing = [name for name in readings.columns if name not in dtypes] + if missing: + raise StructuredDataError( + f'schema does not describe column(s) {missing}', + code='INVALID_REQUEST', + ) + + instance_schema: dict[str, Any] = { + 'columns': { + name: { + 'dtype': dtypes[name], + 'stype': _meaning(table[name], dtypes[name]), + } + for name in readings.columns + } + } + key = table.primary_key + if key: + instance_schema['primary_key'] = ( + [key] if isinstance(key, str) else list(key) + ) + + envelope: dict[str, Any] = { + 'model': self.spec.model_id, + 'task': { + 'kind': request.task.kind, + 'target': { + 'column_name': request.task.target.column_name, + 'dtype': request.task.target.dtype, + }, + }, + 'schema': {'instance_table': instance_schema}, + 'context': { + 'instance_table': encode_table(readings.iloc[:0], dtypes) + }, + 'predict': {'instance_table': encode_table(readings, dtypes)}, + 'output': {'fields': list(outputs)}, + 'metadata': {'request_id': _request_id()}, + } + if request.inference: + envelope['inference'] = typed_inference( + json_ready(request.inference) + ) + return envelope + + +def _meaning(column: Any, dtype: str) -> str: + r"""What to declare a column as, for a model that reads one meaning. + + `coherent_stype` holds a placeholder meaning only where the dtype cannot + carry it, and a float can be categorical, so a column with nothing in it + to read stays categorical and this model then refuses it. The readings are + numbers and the model reconstructs numbers, so there the meaning follows + the dtype instead. + + Only that column, and only where nothing was said about it either. A + column that holds values carries a claim, and so does one a caller typed + by hand though it held none; both are left alone to be refused on their + own terms. + """ + if not column.untyped or column.stype_declared: + return coherent_stype(column.stype, dtype, column) + return MEANING_OF.get(dtype) or coherent_stype(column.stype, dtype, column) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py new file mode 100644 index 000000000..122fc3c7d --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Rules every single-table request obeys, whichever model answers it. + +A request with one table and no relationships is read the same way for every +such model: the caller's key goes back on each answer and is checked against +the batch it claims to belong to, and a column's declared meaning is held to +something its dtype can actually carry. Both were written for `kumo-tabular` +and neither is about it, so they live here rather than being reached for +across adapters or written a second time. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + +from sdm_client.errors import StructuredDataError +from sdm_client.schema import Column, supports_dtype +from sdm_client.wire import as_declared + +# What the contract allows a `PredictionItem.id` to be. +MAX_ID_LENGTH = 1024 + + +def key_named_by(value: str) -> str | None: + r"""The key an id names, or ``None`` if it names no key this can read. + + The endpoint answers under the list of primary-key values, and how it + renders that list is its own business: a key that travelled as a JSON + string comes back quoted, one that travelled as a number does not, and one + carrying a newline or a backslash comes back with that escaped. So the + rendering is read back into the value it stands for, and the value is what + gets compared. Taking the text apart by hand instead would refuse a correct + answer for any key an escape appears in. + + Read as JSON first, because JSON is what the endpoint answers with, and + only as a Python literal if that fails. The two agree on everything except + a character above U+FFFF: JSON writes it as a surrogate pair and + `ast.literal_eval` reads the halves as two separate characters that never + recombine, so a key containing one could never be matched back and the + request failed at the client's own id check. + + Anything not written that way is left alone rather than refused. The + contract calls this field a *service-generated* row identifier and names + ``row_index`` as what to align on, so an endpoint is entitled to answer + under a token of its own and refusing one would break a compliant + deployment. It bounds an id at 1024 characters, so anything longer is not + one either, and is declined before being parsed. + + A value that is not JSON is read as the Python rendering it must then be. + """ + import ast + + if len(value) > MAX_ID_LENGTH: + return None + named: Any = None + try: + named = json.loads(value) + except (ValueError, TypeError, RecursionError): + try: + named = ast.literal_eval(value) + except ( + ValueError, + SyntaxError, + TypeError, + MemoryError, + RecursionError, + ): + return None + if not isinstance(named, list) or len(named) != 1: + return None + return str(named[0]) + + +def named_entities( + items: Sequence[Any], payload: dict[str, Any] | None +) -> list[Any]: + r"""Put the caller's key back on each row, and check it belongs there. + + The endpoint answers under the stringified list of primary-key values, so + a row the caller asked about as ``1`` comes back as ``'[1]'``. The + relational path already returns the caller's own key, and + :mod:`sdm_client.result` tells callers to join on it, so a key that only + joins on one of the two models is a difference they have no reason to + expect. + + The key is read from the payload that produced this response, so a run + split into several calls maps each batch against its own rows. + + **A mismatch is an error, not a silent pass.** Rewriting an id is exactly + the operation that can misattribute an answer, so a row is checked against + the batch it claims to belong to before its id is replaced. The endpoint + answers under the list of key values and the relational endpoint answers + under the key itself, so a key is recognised in either shape. An id + recognisable as a key has to be the key of the row it was paired with. An + id in any other shape is not second-guessed: it may be a correlation token + of the endpoint's own. + + Those checks are about the batch rather than the key, so they also hold for + a request that has no key to put back: a composite key, or rows the caller + supplied without the key column, still have to be answered once each and + no more. Such a request has no value to restore. A null key is the same + case. ``str`` would turn it into ``'None'``, which + reads as a key that was asked about. Rows in a shape this cannot read at + all, a format other than ``arrays``, are left alone entirely. + """ + import dataclasses + + if payload is None: + return list(items) + table = payload.get('predict', {}).get('instance_table', {}) + rows = table.get('rows') + if not isinstance(rows, list): + return list(items) + columns = table.get('columns') or [] + + key = ( + payload.get('schema', {}).get('instance_table', {}).get('primary_key') + or [] + ) + named_key = key[0] if len(key) == 1 else None + key_dtype = ( + payload.get('schema', {}) + .get('instance_table', {}) + .get('columns', {}) + .get(named_key, {}) + .get('dtype') + if named_key + else None + ) + keys = ( + [as_declared(row[columns.index(named_key)], key_dtype) for row in rows] + if named_key and named_key in columns + else None + ) + keys_in_batch = {str(value) for value in keys or ()} + + named: list[Any] = [] + seen: set[int] = set() + for item in items: + index = item.row_index + if index is None: + raise StructuredDataError( + 'the response is missing a row_index, so its answer cannot be ' + 'attributed to a row', + code='INVALID_RESPONSE', + ) + if not 0 <= index < len(rows): + raise StructuredDataError( + f'the response names row_index {index}, and this batch asked ' + f'about {len(rows)} rows', + code='INVALID_RESPONSE', + ) + if index in seen: + raise StructuredDataError( + f'the response answers row {index} more than once', + code='INVALID_RESPONSE', + ) + if keys is not None and item.id is not None: + answered = str(item.id) + named_key = key_named_by(answered) + if named_key is None and answered in keys_in_batch: + named_key = answered + if named_key is not None and named_key != str(keys[index]): + raise StructuredDataError( + f'the response pairs row_index {index} with id ' + f'{item.id!r}, and this batch asked about ' + f'{keys[index]!r} there', + code='INVALID_RESPONSE', + ) + seen.add(index) + value = None if keys is None else keys[index] + named.append( + item if value is None else dataclasses.replace(item, id=value) + ) + + if seen != set(range(len(rows))): + missing = sorted(set(range(len(rows))) - seen) + raise StructuredDataError( + f'the response left row(s) {missing} unanswered', + code='INVALID_RESPONSE', + ) + return named + + +#: What a dtype means when nothing else says. Only reached for a column whose +#: meaning was never claimed, so it is the dtype's own reading rather than a + + +#: guess about the data. +MEANING_OF = { + 'int64': 'numerical', + 'int32': 'numerical', + 'float64': 'numerical', + 'float32': 'numerical', + 'bool': 'categorical', +} + + +def coherent_stype(stype: str, dtype: str, column: Column) -> str: + r"""A meaning the dtype can actually carry. + + An all-null context column is typed `categorical` because a schema has to + say something. The scored rows then decide its dtype, and if they hold + timestamps the column goes out as `timestamp[us]` meaning `categorical`, + which is not a thing a column can be, and is the same self-contradiction on + the wire that a scored frame re-typing a real column used to produce. + + Only the placeholder is overridden. A meaning the caller assigned is a + claim and is refused rather than quietly rewritten, before it reaches here. + + Every spelling of a timestamp counts, not one of them. The contract + writes ``timestamp[us]``, but a hand-built schema may say ``[ns]``, + ``[ms]`` or ``[s]``, and reading those as categorical puts exactly the + contradiction on the wire that this exists to prevent. + """ + if supports_dtype(stype, dtype) or column.stype_declared: + return stype + if dtype.startswith('timestamp'): + return 'timestamp' + return MEANING_OF.get(dtype, 'categorical') diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index 6b637ee7c..ac4c99692 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -34,6 +34,7 @@ ResponseContractError, ) +from sdm_client.adapters.single_table import coherent_stype, named_entities from sdm_client.base import ModelSpec from sdm_client.core.warnings import SDMWarning, caller_stacklevel from sdm_client.errors import StructuredDataError, UnsupportedRequestError @@ -41,10 +42,8 @@ from sdm_client.models import KUMO_RELATIONAL, KUMO_TABULAR from sdm_client.request import Request from sdm_client.result import PredictionResult -from sdm_client.schema import Column, supports_dtype from sdm_client.wire import ( UNTYPED, - as_declared, encode_table, infer_tfm_dtype, json_ready, @@ -67,9 +66,6 @@ _SESSION_PINNED = ('model', 'task', 'schema', 'context') _SESSION_SCORING = ('predict', 'output', 'inference', 'metadata') -# What the contract allows a `PredictionItem.id` to be. -_MAX_ID_LENGTH = 1024 - def _request_id() -> str: return f'sdm-{uuid.uuid4().hex[:12]}' @@ -164,7 +160,7 @@ def answered( request_id: str = '', metadata: Mapping[str, Any] | None = None, ) -> PredictionResult: - named = as_declared_target(_named_entities(items, payload), payload) + named = as_declared_target(named_entities(items, payload), payload) return PredictionResult( model=model, request_id=request_id, @@ -262,7 +258,7 @@ def _envelope( as_text.append(name) schema_columns[name] = { 'dtype': dtypes[name], - 'stype': _coherent_stype( + 'stype': coherent_stype( self._stype_for(request, declared, name), dtypes[name], table[name], @@ -416,200 +412,3 @@ def _declared_by(pinned: Mapping[str, Any] | None) -> dict[str, str] | None: for name, described in columns.items() if isinstance(described, Mapping) and described.get('dtype') } - - -def _key_named_by(value: str) -> str | None: - r"""The key an id names, or ``None`` if it names no key this can read. - - The endpoint answers under the list of primary-key values, and how it - renders that list is its own business: a key that travelled as a JSON - string comes back quoted, one that travelled as a number does not, and one - carrying a newline or a backslash comes back with that escaped. So the - rendering is read back into the value it stands for, and the value is what - gets compared. Taking the text apart by hand instead would refuse a correct - answer for any key an escape appears in. - - Read as JSON first, because JSON is what the endpoint answers with, and - only as a Python literal if that fails. The two agree on everything except - a character above U+FFFF: JSON writes it as a surrogate pair and - `ast.literal_eval` reads the halves as two separate characters that never - recombine, so a key containing one could never be matched back and the - request failed at the client's own id check. - - Anything not written that way is left alone rather than refused. The - contract calls this field a *service-generated* row identifier and names - ``row_index`` as what to align on, so an endpoint is entitled to answer - under a token of its own and refusing one would break a compliant - deployment. It bounds an id at 1024 characters, so anything longer is not - one either, and is declined before being parsed. - - A value that is not JSON is read as the Python rendering it must then be. - """ - import ast - import json - - if len(value) > _MAX_ID_LENGTH: - return None - named: Any = None - try: - named = json.loads(value) - except (ValueError, TypeError, RecursionError): - try: - named = ast.literal_eval(value) - except ( - ValueError, - SyntaxError, - TypeError, - MemoryError, - RecursionError, - ): - return None - if not isinstance(named, list) or len(named) != 1: - return None - return str(named[0]) - - -def _named_entities( - items: Sequence[Any], payload: dict[str, Any] | None -) -> list[Any]: - r"""Put the caller's key back on each row, and check it belongs there. - - The endpoint answers under the stringified list of primary-key values, so - a row the caller asked about as ``1`` comes back as ``'[1]'``. The - relational path already returns the caller's own key, and - :mod:`sdm_client.result` tells callers to join on it, so a key that only - joins on one of the two models is a difference they have no reason to - expect. - - The key is read from the payload that produced this response, so a run - split into several calls maps each batch against its own rows. - - **A mismatch is an error, not a silent pass.** Rewriting an id is exactly - the operation that can misattribute an answer, so a row is checked against - the batch it claims to belong to before its id is replaced. The endpoint - answers under the list of key values and the relational endpoint answers - under the key itself, so a key is recognised in either shape. An id - recognisable as a key has to be the key of the row it was paired with. An - id in any other shape is not second-guessed: it may be a correlation token - of the endpoint's own. - - Those checks are about the batch rather than the key, so they also hold for - a request that has no key to put back: a composite key, or rows the caller - supplied without the key column, still have to be answered once each and - no more. Such a request has no value to restore. A null key is the same - case. ``str`` would turn it into ``'None'``, which - reads as a key that was asked about. Rows in a shape this cannot read at - all, a format other than ``arrays``, are left alone entirely. - """ - import dataclasses - - if payload is None: - return list(items) - table = payload.get('predict', {}).get('instance_table', {}) - rows = table.get('rows') - if not isinstance(rows, list): - return list(items) - columns = table.get('columns') or [] - - key = ( - payload.get('schema', {}).get('instance_table', {}).get('primary_key') - or [] - ) - named_key = key[0] if len(key) == 1 else None - key_dtype = ( - payload.get('schema', {}) - .get('instance_table', {}) - .get('columns', {}) - .get(named_key, {}) - .get('dtype') - if named_key - else None - ) - keys = ( - [as_declared(row[columns.index(named_key)], key_dtype) for row in rows] - if named_key and named_key in columns - else None - ) - keys_in_batch = {str(value) for value in keys or ()} - - named: list[Any] = [] - seen: set[int] = set() - for item in items: - index = item.row_index - if index is None: - raise StructuredDataError( - 'the response is missing a row_index, so its answer cannot be ' - 'attributed to a row', - code='INVALID_RESPONSE', - ) - if not 0 <= index < len(rows): - raise StructuredDataError( - f'the response names row_index {index}, and this batch asked ' - f'about {len(rows)} rows', - code='INVALID_RESPONSE', - ) - if index in seen: - raise StructuredDataError( - f'the response answers row {index} more than once', - code='INVALID_RESPONSE', - ) - if keys is not None and item.id is not None: - answered = str(item.id) - named_key = _key_named_by(answered) - if named_key is None and answered in keys_in_batch: - named_key = answered - if named_key is not None and named_key != str(keys[index]): - raise StructuredDataError( - f'the response pairs row_index {index} with id ' - f'{item.id!r}, and this batch asked about ' - f'{keys[index]!r} there', - code='INVALID_RESPONSE', - ) - seen.add(index) - value = None if keys is None else keys[index] - named.append( - item if value is None else dataclasses.replace(item, id=value) - ) - - if seen != set(range(len(rows))): - missing = sorted(set(range(len(rows))) - seen) - raise StructuredDataError( - f'the response left row(s) {missing} unanswered', - code='INVALID_RESPONSE', - ) - return named - - -#: What a dtype means when nothing else says. Only reached for a column whose -#: meaning was never claimed, so it is the dtype's own reading rather than a -#: guess about the data. -_MEANING_OF = { - 'int64': 'numerical', - 'int32': 'numerical', - 'float64': 'numerical', - 'float32': 'numerical', - 'bool': 'categorical', -} - - -def _coherent_stype(stype: str, dtype: str, column: Column) -> str: - r"""A meaning the dtype can actually carry. - - An all-null context column is typed `categorical` because a schema has to - say something. The scored rows then decide its dtype, and if they hold - timestamps the column goes out as `timestamp[us]` meaning `categorical`, - which is not a thing a column can be, and is the same self-contradiction on - the wire that a scored frame re-typing a real column used to produce. - - Only the placeholder is overridden. A meaning the caller assigned is a - claim and is refused rather than quietly rewritten, before it reaches here. - """ - if supports_dtype(stype, dtype) or column.stype_declared: - return stype - # Every spelling of a timestamp, not one of them. The contract writes - # `timestamp[us]`, but a hand-built schema may say `[ns]`, `[ms]` or `[s]`, - # and reading those as categorical puts a dtype and a meaning on the wire - # that contradict each other. That is what this exists to prevent. - if dtype.startswith('timestamp'): - return 'timestamp' - return _MEANING_OF.get(dtype, 'categorical') diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 34c1ce818..8325e004e 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -82,6 +82,10 @@ def _adapter_for(spec: ModelSpec) -> ModelAdapter: from sdm_client.adapters.relational import RelationalAdapter return RelationalAdapter() + if spec.model_id == 'kumo-anomaly': + from sdm_client.adapters.anomaly import AnomalyAdapter + + return AnomalyAdapter() raise UnknownModelError(spec.model_id, list(MODELS)) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py new file mode 100644 index 000000000..e6e364a4c --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""``kumo-anomaly`` in process: one table of readings, scored as a series. + +This model is unlike the other two in the one way that matters here. It is +unsupervised, so there is no context to fit and no target column to name: it +reconstructs the rows it is given and scores each one by how badly the +reconstruction missed. That is why :class:`Anomaly` has no ``context`` or +``query``, and why a session over it is refused rather than pinned. + +The answer is three numbers per row, and the contract's ``PredictionItem`` has +nowhere to put three. ``prediction`` carries the score, which is the answer to +the question that was asked. The two labels ride in ``probabilities``, whose +name is wrong for them and whose shape is right: a mapping that reaches the +frame as one flat column each. The contract is generated from a spec that does +not describe this model, so widening it is not this client's to do. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from sdm_api.generated.tfm_api import PredictionItem + +from sdm_client.errors import StructuredDataError +from sdm_client.in_process.columns import ( + Shape, + check_columns, + key_columns, + stype_of, +) +from sdm_client.in_process.tables import values + +#: The thresholders the model applies, in the order it returns them. +THRESHOLDERS = ('scs', 'macs') + +#: How the published deployment samples. Reconstructions are combined by their +#: median, so fewer samples is a noisier score rather than a different one. +_SAMPLES = 30 +_STEPS = 20 + + +class Anomaly: + r"""Reads a ``kumo-anomaly`` request into the model.""" + + model_id = 'kumo-anomaly' + outputs = {'anomaly_detection': ('prediction', 'probabilities')} + + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model.""" + try: + _model_type() + except StructuredDataError: + return False + return True + + @staticmethod + def load(device: Any) -> Any: + return _model_type()(device=device) + + @staticmethod + def accepts(model: Any) -> bool: + r"""Whether ``model`` is this one. + + Asked of every model handed to the client, including ones this does + not serve, so an installed structured-data-models that predates + ``KumoAnomaly`` has to answer no rather than raise. Every published + release predates it today. + """ + try: + return isinstance(model, _model_type()) + except StructuredDataError: + return False + + def check_inference( + self, + options: Mapping[str, Any], + task: Mapping[str, Any], + settled: bool = False, + ) -> None: + r"""Refuse an inference option this model does not act on.""" + remaining = dict(options) + if remaining.pop('inference_config', {}): + raise StructuredDataError( + 'inference_config is reserved for API compatibility and must ' + 'be omitted or empty', + code='INVALID_REQUEST', + ) + unknown = sorted(set(remaining) - {'num_samples', 'num_steps'}) + if unknown: + raise StructuredDataError( + f'unsupported inference options: {unknown}', + code='INVALID_REQUEST', + ) + for name in ('num_samples', 'num_steps'): + value = remaining.get(name) + if value is None: + continue + if not isinstance(value, int) or isinstance(value, bool): + raise StructuredDataError( + f'{name} is {value!r}; it is a whole number of at least 1', + code='INVALID_REQUEST', + ) + if value < 1: + raise StructuredDataError( + f'{name} is {value}; it is a whole number of at least 1', + code='INVALID_REQUEST', + ) + + def rows(self, payload: Mapping[str, Any]) -> Any: + r"""The readings to score, as the model reads them. + + Every column is a channel of one multivariate series, so the whole + table is read rather than all but a target. A key column names the + row instead of measuring anything, so reconstructing it would score + rows by how surprising their identifiers are. + """ + spec = payload['schema']['instance_table'] + table = payload['predict']['instance_table'] + check_columns(table, spec, 'the rows to score') + keys = key_columns(spec) + stypes = { + name: stype_of(name, spec['columns'][name]) + for name in table['columns'] + if name not in keys + } + unread = sorted( + name for name, stype in stypes.items() if stype != 'numerical' + ) + if unread: + raise StructuredDataError( + f'{self.model_id} reconstructs numerical columns, and ' + f'{unread} is not one. A reading it cannot reconstruct has no ' + f'error to score, so leave it out of the rows you send', + code='UNSUPPORTED_DTYPE', + ) + if not stypes: + raise StructuredDataError( + f'{self.model_id} scores a row by reconstructing it, and ' + f'these rows carry no numerical column to reconstruct', + code='INVALID_REQUEST', + ) + return Shape.of(spec, stypes).tensor(table, 'the rows to score') + + @staticmethod + def row_ids( + table: Mapping[str, Any], spec: Mapping[str, Any] + ) -> list[str | None]: + r"""A key of several columns answers under the JSON of its values as + they were sent, which the client hands back as the row's id. + + Otherwise there is no id: each answer is matched to its row by + position, and the client puts the row's own key on it. That is also + what happens where the readings do not carry the key at all, which + is allowed: a key names the row and is not a channel, so there is + then no value to answer under. + """ + key = spec.get('primary_key') + if not isinstance(key, list) or len(key) < 2: + return [None] * len(table['rows']) + if not set(key) <= set(table['columns']): + return [None] * len(table['rows']) + cells = [values(table, name) for name in key] + return [ + json.dumps(list(values_of), separators=(',', ':'), default=str) + for values_of in zip(*cells, strict=True) + ] + + def answer( + self, + model: Any, + payload: Mapping[str, Any], + ids: Sequence[Any], + fields: Sequence[str], + generator: Any = None, + ) -> list[PredictionItem]: + r"""Score each row, under the id the request gave it. + + ``generator`` is what makes the same readings score the same twice. + Reconstruction is sampled, so without one an unchanged request comes + back with different numbers every call. It also says where the model + is: the rows are read into CPU tensors and the model may be + anywhere, and with no context to fit, the rows are all there is to + move to it. + + A score that is not a number is refused rather than returned, as the + other models refuse one. It is not a row that is unremarkable, and + handing it back hands a caller a frame they cannot threshold or + plot. + """ + options = payload.get('inference') or {} + rows = self.rows(payload) + if generator is not None: + rows = rows.to(generator.device) + out = model( + rows, + num_samples=int(options.get('num_samples') or _SAMPLES), + num_steps=int(options.get('num_steps') or _STEPS), + generator=generator, + ) + scored = out['mae'].numerical[..., 0] + if not bool(scored.isfinite().all()): + raise StructuredDataError( + 'the model answered with a malformed output', + code='INTERNAL_ERROR', + ) + scores = scored.tolist() + labelled = 'probabilities' in fields + labels = ( + { + name: out[name].categorical.decode()[..., 0].tolist() + for name in THRESHOLDERS + } + if labelled + else {} + ) + return [ + PredictionItem( + row_index=index, + id=ids[index] if index < len(ids) else None, + prediction=score, + probabilities={ + name: float(labels[name][index]) for name in THRESHOLDERS + } + if labelled + else None, + ) + for index, score in enumerate(scores) + ] + + +def _model_type() -> Any: + r"""``sdm.models.KumoAnomaly``, or why it could not be reached.""" + try: + from sdm.models import KumoAnomaly + except ImportError as error: + raise StructuredDataError( + 'this structured-data-models carries no KumoAnomaly. The model ' + 'is on its main and in no release, so until one ships it is ' + 'reached by installing that package from source', + code='MODEL_NOT_READY', + ) from error + return KumoAnomaly diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index 965295fe3..7250be495 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -22,10 +22,11 @@ import uuid from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from sdm_client.errors import MissingExtraError, StructuredDataError from sdm_client.explanations import GradientConverter +from sdm_client.in_process.anomaly import Anomaly from sdm_client.in_process.answers import ( QUANTILE_LEVELS, class_order, @@ -46,11 +47,53 @@ _SEED = 42 _ESTIMATORS = 1 -_SERVED = {served.model_id: served for served in (Tabular(), Relational())} +_SERVED = { + served.model_id: served for served in (Tabular(), Relational(), Anomaly()) +} + +#: What reads a request for one model. +Served = Tabular | Relational | Anomaly + + +@runtime_checkable +class ScoresRowsAlone(Protocol): + r"""A model shown no labelled rows, which therefore fits no context. + + It answers from the rows it is given and nothing carries over, so there + is no fitted context to pin and no session to hold. Declaring that as a + protocol rather than a flag is what lets the two shapes be told apart + where it matters, by what a reader actually has to offer. + """ + + def row_ids( + self, table: Mapping[str, Any], spec: Mapping[str, Any] + ) -> list[str | None]: ... + + def check_inference( + self, + options: Mapping[str, Any], + task: Mapping[str, Any], + settled: bool = False, + ) -> None: ... + + def answer( + self, + model: Any, + payload: Mapping[str, Any], + ids: Sequence[Any], + fields: Sequence[str], + generator: Any = None, + ) -> list[Any]: ... @dataclass class _Session: + r"""A fitted context, pinned. + + Only a model that fits one reaches here: :meth:`create_session` refuses + a model that does not, because there would be nothing to pin. + """ + served: Tabular | Relational task: Mapping[str, Any] schema: Mapping[str, Any] @@ -127,8 +170,9 @@ def list_models(self) -> list[str]: self._require_open() return [ model_id - for model_id in _SERVED - if self._loads or model_id in self._models + for model_id, served in _SERVED.items() + if (self._loads or model_id in self._models) + and served.installed() ] def predict( @@ -145,23 +189,32 @@ def predict( ) served, model = self._serving(payload['model']) _check_loaded_for(served, model, task['kind']) - context = served.context(payload) - query = served.query(context, payload, payload['schema']) - fitted = _fit(model, context) - answer = _run(fitted, query) - items = _items( - served, - task, - payload['schema'], - context, - payload, - answer, - fields, - ) - if 'explanation' in fields: - items = _explained( - items, fitted, query, context, task['kind'], payload, fields + if isinstance(served, ScoresRowsAlone): + items = _scored(served, model, payload, fields) + else: + context = served.context(payload) + query = served.query(context, payload, payload['schema']) + fitted = _fit(model, context) + answer = _run(fitted, query) + items = _items( + served, + task, + payload['schema'], + context, + payload, + answer, + fields, ) + if 'explanation' in fields: + items = _explained( + items, + fitted, + query, + context, + task['kind'], + payload, + fields, + ) return adapter.answered( items, outputs, @@ -176,8 +229,16 @@ def create_session( r"""Fit the context once, to be scored against until released.""" with self._running(): kind = payload['task']['kind'] - _fields(self._reader(payload['model']), kind, payload) - served, model = self._serving(payload['model']) + served = self._reader(payload['model']) + if isinstance(served, ScoresRowsAlone): + raise StructuredDataError( + f'{served.model_id} holds no session: it is shown no ' + f'labelled rows, so there is no fitted context to pin. ' + f'Call predict() instead', + code='SESSIONS_UNSUPPORTED', + ) + _fields(served, kind, payload) + _, model = self._serving(payload['model']) _check_loaded_for(served, model, kind) context = served.context(payload) fitted = _fit(model, context) @@ -250,7 +311,7 @@ def delete_session(self, session_id: str) -> None: self._require_open() self._sessions.pop(session_id, None) - def _reader(self, model_id: str) -> Tabular | Relational: + def _reader(self, model_id: str) -> Served: r"""What reads requests for ``model_id``, known without loading it.""" served = _SERVED.get(model_id) if served is None: @@ -264,7 +325,7 @@ def _not_served(self, model_id: str) -> StructuredDataError: code='INVALID_CONFIGURATION', ) - def _serving(self, model_id: str) -> tuple[Tabular | Relational, Any]: + def _serving(self, model_id: str) -> tuple[Served, Any]: r"""What reads requests for ``model_id``, and the model it runs. Runs are serialized, so a model is loaded once. The load is not held @@ -347,7 +408,7 @@ def _require_sdm() -> Any: return torch -def _served_by(model: Any) -> Tabular | Relational: +def _served_by(model: Any) -> Served: for served in _SERVED.values(): if served.accepts(model): return served @@ -371,7 +432,7 @@ def _device(device: Any, torch: Any) -> Any: ) from error -def _loaded(served: Tabular | Relational, device: Any) -> Any: +def _loaded(served: Served, device: Any) -> Any: try: return served.load(device) except Exception as error: @@ -387,7 +448,7 @@ def _request_id(payload: Mapping[str, Any]) -> str: return str((payload.get('metadata') or {}).get('request_id') or '') -def _check_task(served: Tabular | Relational, kind: str) -> None: +def _check_task(served: Served, kind: str) -> None: r"""Refuse a task this model does not answer.""" if kind not in served.outputs: raise StructuredDataError( @@ -397,14 +458,18 @@ def _check_task(served: Tabular | Relational, kind: str) -> None: ) -def _check_loaded_for( - served: Tabular | Relational, model: Any, kind: str -) -> None: +def _check_loaded_for(served: Served, model: Any, kind: str) -> None: r"""Refuse a task this model was loaded without. A model can be loaded for classification or regression alone, which a NIM's never is, so that is asked of the model rather than of its kind. + Not of a model that fits nothing: what was asked of the others is which + heads a checkpoint was packaged with, and a model that fits nothing has + no head to package. It answers the one task it is for or it is not that + model. """ + if isinstance(served, ScoresRowsAlone): + return needed = 'regression' if kind == 'regression' else 'classification' loaded = sorted(str(task) for task in model.tasks) if needed not in loaded: @@ -417,7 +482,7 @@ def _check_loaded_for( def _fields( - served: Tabular | Relational, + served: Served, kind: str, payload: Mapping[str, Any], ) -> list[str]: @@ -518,8 +583,33 @@ def _autocast(device: Any) -> Any: return torch.autocast(device_type='cuda', dtype=dtype) +def _scored( + served: ScoresRowsAlone, + model: Any, + payload: Mapping[str, Any], + fields: list[str], +) -> list[Any]: + r"""One answer per row, from a model with no context to fit.""" + import torch + + ids = served.row_ids( + payload['predict']['instance_table'], + payload['schema']['instance_table'], + ) + served.check_inference(payload.get('inference') or {}, payload['task']) + device = _placement(model) + with _autocast(device): + return served.answer( + model, + payload, + ids, + fields, + torch.Generator(device=device).manual_seed(_SEED), + ) + + def _items( - served: Tabular | Relational, + served: Served, task: Mapping[str, Any], schema: Mapping[str, Any], context: Context, diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/relational.py b/packages/structured-data-models-client/src/sdm_client/in_process/relational.py index fa6e4c74c..b0f3d1d15 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/relational.py @@ -61,6 +61,15 @@ class Relational: 'regression': ('prediction', 'quantiles', 'explanation'), } + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model. + + Every release does; `kumo-anomaly` is the one that may not, and + `list_models` asks this of each rather than of that one. + """ + return True + @staticmethod def load(device: Any) -> Any: from sdm.models import KumoRelational diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py b/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py index 8578f4533..45cd6eabe 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py @@ -33,6 +33,15 @@ class Tabular: 'regression': ('prediction', 'quantiles', 'explanation'), } + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model. + + Every release does; `kumo-anomaly` is the one that may not, and + `list_models` asks this of each rather than of that one. + """ + return True + @staticmethod def load(device: Any) -> Any: from sdm.models import KumoTabular diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 2e360f229..f4e79d2e3 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -32,7 +32,7 @@ from __future__ import annotations -from predictive_query import RELATIONAL, TABULAR +from predictive_query import ANOMALY, RELATIONAL, TABULAR from sdm_api import TFM_MODEL_KUMO_RELATIONAL, TFM_MODEL_KUMO_TABULAR from sdm_client.base import ModelSpec, RuntimeContract @@ -158,6 +158,33 @@ ) +# Named here rather than imported from the generated bindings, which carry +# only the ids the pinned spec's model enum declares. The spec does not +# describe a model that scores a row without being shown a labelled one, so +# there is no constant to import and hand-editing a generated file would be +# undone by the next regeneration. +TFM_MODEL_KUMO_ANOMALY = 'kumo-anomaly' + +#: The model that scores a row for how anomalous it is. +#: +#: `probabilities` carries the two thresholders' labels. That field is named +#: for what every other model puts in it, and this one puts something else +#: there because the contract has no third field for a row's answer; see +#: :mod:`sdm_client.in_process.anomaly`. It holds no session, because a +#: session pins a fitted context and this model fits none: every call +#: reconstructs the rows it was given and nothing carries over. +KUMO_ANOMALY = ModelSpec( + model_id=TFM_MODEL_KUMO_ANOMALY, + profile=ANOMALY, + runtime=RuntimeContract( + tasks=frozenset({'anomaly_detection'}), + outputs=frozenset({'prediction', 'probabilities'}), + sessions=False, + inference_options=frozenset({'num_samples', 'num_steps'}), + ), +) + + class Model: r"""Base for the model handles. Not useful on its own.""" @@ -205,10 +232,33 @@ class KumoRelational(Model): _spec = KUMO_RELATIONAL +class KumoAnomaly(Model): + r"""One table of readings in, a score per row out. + + Takes ``anomaly_detection`` only, and produces ``prediction``, the + reconstruction error for the row, and ``probabilities``, the two + thresholders' labels for it. It is unsupervised, so a request is built + with :meth:`Request.anomaly ` and names no + target. It holds no session, because there is no context to pin. + + Served in this process only, through + :meth:`SDMClient.in_process `: no + deployment answers for this id yet. + + Needs a structured-data-models that carries ``KumoAnomaly``, which no + release does yet. The ``in-process`` extra installs the latest release, so + until one ships this model is reached by installing that package from its + own main. + """ + + _spec = KUMO_ANOMALY + + #: Every model this client can send to, by id. MODELS = { KUMO_TABULAR.model_id: KUMO_TABULAR, KUMO_RELATIONAL.model_id: KUMO_RELATIONAL, + KUMO_ANOMALY.model_id: KUMO_ANOMALY, } diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index a09ac33da..900becc35 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -49,6 +49,12 @@ # Task kinds the contract carries. A model narrows this further: a tabular # model takes only the coarse 'classification', and its adapter says so. +# +# 'anomaly_detection' is the exception: the contract's enum does not carry it, +# because the spec these bindings are generated from does not describe a model +# that answers it. It is named here so a request can be built and refused by +# every model that does not serve it, which is what the rest of this tuple is +# for. Nothing is sent to a deployment that has not declared it. TASK_KINDS = ( 'classification', 'binary_classification', @@ -57,8 +63,15 @@ 'forecasting', 'temporal_link_prediction', 'ranking', + 'anomaly_detection', ) +#: The target of a question asked about the whole row rather than one column. +#: An anomaly score is joint across the channels it reconstructs, so naming +#: one column would promise a score specific to it that the model never +#: computes. +WHOLE_ROW = '*' + _CLASSIFICATION_KINDS = frozenset( {'classification', 'binary_classification', 'multiclass_classification'} ) @@ -377,6 +390,15 @@ def __init__( f'task must be one of {list(TASK_KINDS)}, got {task!r}', code='INVALID_REQUEST', ) + if task == 'anomaly_detection': + raise StructuredDataError( + f'anomaly_detection names no target and learns from no ' + f'labelled rows, and this builds a request that has both. ' + f'Asking it here would hand over a context and a target ' + f'{target!r} that nothing reads, and get back a score for the ' + f'whole row. Build it with Request.anomaly(readings) instead', + code='INVALID_REQUEST', + ) if target in context.columns and _has_container_cells(context[target]): # Said here because this is the only place that knows which column # is the target. SDM's own `Task.classification` covers "binary, @@ -542,6 +564,69 @@ def materialized( request.requested_inference = dict(requested_inference or {}) return request + @classmethod + def anomaly( + cls, + readings: pd.DataFrame, + *, + schema: Schema | None = None, + entity_table: str = 'instance', + inference: Mapping[str, Any] | None = None, + ) -> Request: + r"""Ask how anomalous each row is, with no labelled rows to learn from. + + Every other request here names a target and shows the model rows where + the answer is known. An anomaly model is unsupervised: it reconstructs + the readings it is given and scores each row by how badly the + reconstruction missed, so there is no column to name and nothing to + learn from. Those two arguments are absent rather than empty, because + a caller who passed them would be describing a question nobody asks. + + What travels instead is a target standing for the score, typed as + the score is because that is the one thing about it the answer + carries a type for, and a context table of the schema's columns and + no rows. The contract has no shape for a request with nothing to + learn from, and an empty table is what that looks like on the wire, + rather than a section left out of an envelope that requires one. + + Args: + readings: The rows to score. Every numerical column is read as a + channel of one multivariate series; a primary key names the + row and is not read as a channel. + schema: Override the inferred schema. Rarely needed. + entity_table: The name the instance table is known by. + inference: Gathering knobs passed to the adapter. + + Raises: + StructuredDataError: If ``readings`` is not a DataFrame. + + >>> request = Request.anomaly(readings) + >>> client.predict(request, model=KumoAnomaly()) + """ + readings = _require_frame(readings, 'readings') + if schema is None: + schema = Schema.from_data({entity_table: readings}) + _check_scored_values_mean_what_the_schema_says( + readings, schema, entity_table + ) + + request = cls.__new__(cls) + request.schema = schema + request.task = Task( + kind='anomaly_detection', + target=Target( + column_name=WHOLE_ROW, + dtype='float64', + table_name=entity_table, + ), + ) + request._context = Split(instance_table=readings.iloc[:0]) + request._predict = Split(instance_table=readings) + request.inference = dict(inference or {}) + request.requested_inference = dict(inference or {}) + request.entity_table = entity_table + return request + @property def _question(self) -> Question: r"""What this request asks. See :class:`Question`.""" diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py index cbe32cda0..bdda5ae67 100644 --- a/packages/structured-data-models-client/tests/conftest.py +++ b/packages/structured-data-models-client/tests/conftest.py @@ -32,6 +32,14 @@ def tabular(sdm): return sdm.models.KumoTabular(size='small', pretrained=False, device='cpu') +@pytest.fixture(scope='module') +def anomaly(sdm): + import torch + + torch.manual_seed(0) + return sdm.models.KumoAnomaly(pretrained=False, device='cpu') + + @pytest.fixture(scope='module') def relational(sdm): pytest.importorskip('sdm_relational_engine', reason='needs [relational]') diff --git a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py index 6a55fb51d..a8c08ce52 100644 --- a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py +++ b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py @@ -266,7 +266,11 @@ def load(**kwargs): ) with SDMClient.in_process() as client: assert client.url == 'in-process' - assert client.models() == ['kumo-tabular', 'kumo-relational'] + assert client.models() == [ + 'kumo-tabular', + 'kumo-relational', + 'kumo-anomaly', + ] assert loads == [] client.predict(request, model=KumoTabular()) client.predict(request, model=KumoTabular()) diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py new file mode 100644 index 000000000..33166774a --- /dev/null +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -0,0 +1,328 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``kumo-anomaly`` asked for a score, with nothing labelled to learn from. + +A small random-weight model stands in for the published one, so nothing is +downloaded and no GPU is needed. The sampling is turned down for the same +reason: reconstructions are combined by their median, so fewer samples is a +noisier score rather than a different kind of answer. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from sdm_client import ( + KumoAnomaly, + KumoTabular, + Request, + SDMClient, + StructuredDataError, + UnsupportedRequestError, +) + +pytestmark = pytest.mark.filterwarnings('ignore') + +FAST = {'num_samples': 2, 'num_steps': 4} +#: What the backend seeds every run with, named here so the direct call this +#: is compared against draws the same samples. +SEED = 42 +DISTURBED = slice(80, 90) + + +def readings(rows: int = 120) -> pd.DataFrame: + """A two-channel series with one stretch pushed off its own shape.""" + step = np.arange(rows, dtype=float) + frame = pd.DataFrame( + { + 'reading_id': np.arange(rows), + 'flow': np.sin(step / 9), + 'pressure': np.cos(step / 14), + } + ) + frame.loc[DISTURBED, ['flow', 'pressure']] += 4.0 + return frame + + +def scored(model, frame: pd.DataFrame, **inference) -> pd.DataFrame: + with SDMClient.in_process(model) as client: + request = Request.anomaly(frame, inference={**FAST, **inference}) + return client.predict(request, model=KumoAnomaly()).to_pandas() + + +def test_a_score_and_a_label_per_row_come_back(anomaly): + """The answer names the row, its score, and each thresholder's verdict.""" + frame = readings() + answer = scored(anomaly, frame) + + assert list(answer.columns) == [ + 'row_index', + 'id', + 'prediction', + 'probability_scs', + 'probability_macs', + ] + assert len(answer) == len(frame) + assert answer['row_index'].tolist() == list(range(len(frame))) + assert np.isfinite(answer['prediction']).all() + for name in ('probability_scs', 'probability_macs'): + assert set(answer[name].unique()) <= {0.0, 1.0} + + +def test_only_the_fields_that_were_asked_for_come_back(anomaly): + """Asking for the score alone is a smaller answer, not the same one with + the labels computed and sent anyway.""" + with SDMClient.in_process(anomaly) as client: + request = Request.anomaly(readings(), inference=FAST) + answer = client.predict( + request, model=KumoAnomaly(), outputs=['prediction'] + ).to_pandas() + + assert 'prediction' in answer.columns + assert not [name for name in answer.columns if name.startswith('prob')] + + +def test_the_same_readings_score_the_same_twice(anomaly): + """Reconstruction is sampled, so the answer is seeded. Without that an + unchanged request comes back with different numbers every call.""" + frame = readings() + + assert scored(anomaly, frame)['prediction'].tolist() == pytest.approx( + scored(anomaly, frame)['prediction'].tolist() + ) + + +def test_the_disturbed_stretch_scores_above_the_rest(anomaly): + """The score is what the model was asked for, so it has to track the + thing that was done to the data rather than merely be finite.""" + answer = scored(anomaly, readings()) + disturbed = answer['prediction'].iloc[DISTURBED] + rest = answer['prediction'].drop(answer.index[DISTURBED]) + + assert disturbed.mean() > rest.mean() + + +def test_the_answer_is_the_one_the_model_gives_when_called_directly( + sdm, anomaly +): + """Nothing is computed on the way through: the client shapes the request + and reads the answer back, and the numbers are the model's own.""" + import torch + + frame = readings() + answer = scored(anomaly, frame) + channels = frame[['flow', 'pressure']] + direct = anomaly( + sdm.TableTensor.from_pandas( + channels, dict.fromkeys(channels.columns, 'numerical') + ), + generator=torch.Generator().manual_seed(SEED), + **FAST, + ) + + torch.testing.assert_close( + torch.tensor(answer['prediction'].to_numpy()), + direct['mae'].numerical[..., 0].double(), + ) + + +def test_a_key_column_is_not_read_as_a_channel(anomaly): + """`reading_id` names the row. Reconstructing it would score a row by how + surprising its identifier is, which rises with every row.""" + frame = readings() + without = frame.drop(columns=['reading_id']) + + assert scored(anomaly, frame)['prediction'].tolist() == pytest.approx( + scored(anomaly, without)['prediction'].tolist() + ) + + +def test_each_answer_comes_back_under_the_row_s_own_key(anomaly): + """`sdm_client.result` tells callers to join on the key, so a key that + joins for one model and not another is a difference nobody expects.""" + frame = readings() + frame['reading_id'] += 1000 + answer = scored(anomaly, frame) + + assert answer['id'].tolist() == frame['reading_id'].tolist() + + +def test_readings_are_answered_though_the_schema_was_built_from_nulls( + anomaly, +): + """A column with nothing in it is declared categorical because a schema + has to say something. The readings then decide its dtype, and refusing + them for the placeholder's sake refuses a question that is answerable.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + blank = Schema.from_data({'instance': frame.iloc[:0].astype(object)}) + + with SDMClient.in_process(anomaly) as client: + request = Request.anomaly(frame, schema=blank, inference=FAST) + answer = client.predict(request, model=KumoAnomaly()).to_pandas() + + assert len(answer) == len(frame) + + +def test_a_meaning_the_caller_gave_a_column_that_holds_values_is_kept( + anomaly, +): + """The placeholder rule is for a column with nothing in it to read. A + column that holds values and says it is categorical says so on purpose, + and this model refuses that rather than reading it as numbers anyway.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame}) + schema['instance']['flow'].stype = 'categorical' + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(frame, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + +def test_readings_that_leave_out_a_composite_key_are_answered(anomaly): + """A key names the row and is not a channel, so a caller may leave it + out. Then there is no value to answer under and rows match by position.""" + from sdm_client import Schema + + frame = readings() + frame['site'] = 'north' + schema = Schema.from_data({'instance': frame}) + schema['instance'].primary_key = ['reading_id', 'site'] + + channels = frame[['flow', 'pressure']] + with SDMClient.in_process(anomaly) as client: + answer = client.predict( + Request.anomaly(channels, schema=schema, inference=FAST), + model=KumoAnomaly(), + ).to_pandas() + + assert len(answer) == len(frame) + assert answer['row_index'].tolist() == list(range(len(frame))) + + +def test_a_meaning_given_to_a_column_with_nothing_in_it_is_kept(anomaly): + """The placeholder is a meaning nobody chose. One a caller typed by hand + is a claim, whether or not the column it is on held anything to read.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame.iloc[:0].astype(object)}) + schema['instance']['flow'].stype = 'categorical' + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(frame, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + +def test_a_score_that_is_not_a_number_is_refused(anomaly): + """A row scored NaN is not a row that is unremarkable, and a frame of + them cannot be thresholded or plotted.""" + frame = pd.DataFrame({'flow': [1e100, 2e100] * 50}) + + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, frame) + assert 'malformed' in str(caught.value) + + +def test_a_column_the_model_cannot_reconstruct_is_refused(anomaly): + """A reading it cannot reconstruct has no error to score.""" + frame = readings() + frame['site'] = 'north' + + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, frame) + assert "'site'" in str(caught.value) or 'site' in str(caught.value) + + +def test_there_is_no_session_to_pin(anomaly): + """A session pins a fitted context, and this model fits none.""" + with SDMClient.in_process(anomaly) as client: + with pytest.raises( + (UnsupportedRequestError, StructuredDataError) + ) as caught: + client.fit(Request.anomaly(readings()), model=KumoAnomaly()) + assert 'session' in str(caught.value) + + +def test_the_supervised_constructor_will_not_build_this_question(anomaly): + """It takes a context and a target, and nothing reads either here. Built + that way the request came back with a whole-row score under a name the + caller had asked about, which is a different question answered quietly.""" + frame = readings() + + with pytest.raises(StructuredDataError) as caught: + Request( + context=frame, + predict=frame.drop(columns=['flow']), + target='flow', + task='anomaly_detection', + ) + assert 'Request.anomaly' in str(caught.value) + + +def test_readings_that_are_not_what_the_schema_says_are_refused(anomaly): + """A schema the caller brought says what a column holds. Readings that + hold something else are a different series under that name, and were + widened into one rather than refused.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame}) + flipped = frame.assign(flow=[True, False] * (len(frame) // 2)) + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(flipped, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + +def test_a_structured_data_models_without_this_model_still_answers( + sdm, monkeypatch +): + """Every published release predates `KumoAnomaly`, and this is asked of + every model handed to the client, including ones it does not serve. It + has to answer no there rather than raise the import.""" + monkeypatch.delattr(sdm.models, 'KumoAnomaly', raising=False) + + with pytest.raises(StructuredDataError) as caught: + SDMClient.in_process(object()) + assert caught.value.code == 'INVALID_CONFIGURATION' + + +def test_a_model_that_is_not_installed_is_not_offered(sdm, monkeypatch): + """`models()` says what this backend answers for. Naming one it cannot + load sends a caller to a refusal they were told to expect an answer to.""" + monkeypatch.delattr(sdm.models, 'KumoAnomaly', raising=False) + + with SDMClient.in_process() as client: + assert 'kumo-anomaly' not in client.models() + + +def test_another_model_is_not_asked_this_question(tabular): + """`anomaly_detection` is a task only one model here answers.""" + with SDMClient.in_process(tabular) as client: + with pytest.raises(StructuredDataError): + client.predict(Request.anomaly(readings()), model=KumoTabular()) + + +@pytest.mark.parametrize('option', ['num_samples', 'num_steps']) +@pytest.mark.parametrize('value', [0, -1, 1.5]) +def test_the_sampling_knobs_take_a_count(anomaly, option, value): + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, readings(), **{option: value}) + assert option in str(caught.value) diff --git a/packages/structured-data-models-client/tests/test_options_and_explanations.py b/packages/structured-data-models-client/tests/test_options_and_explanations.py index 4b3ad29a1..a460a6bba 100644 --- a/packages/structured-data-models-client/tests/test_options_and_explanations.py +++ b/packages/structured-data-models-client/tests/test_options_and_explanations.py @@ -31,7 +31,7 @@ Schema, ) from sdm_client.adapters.relational import RelationalAdapter -from sdm_client.adapters.tabular import _key_named_by +from sdm_client.adapters.single_table import key_named_by from sdm_client.errors import NimRequestError, StructuredDataError RNG = np.random.default_rng(11) @@ -287,19 +287,19 @@ def test_an_unrelated_error_is_returned_untouched(): def test_a_key_survives_the_json_the_endpoint_sends(key): """JSON writes a character above U+FFFF as a surrogate pair, and reading it as a Python literal takes the halves apart so they never recombine.""" - assert _key_named_by(json.dumps([key])) == key + assert key_named_by(json.dumps([key])) == key def test_a_key_survives_the_python_rendering_too(): """The other wording an endpoint may answer with.""" - assert _key_named_by(repr(['\U0001f600key'])) == '\U0001f600key' + assert key_named_by(repr(['\U0001f600key'])) == '\U0001f600key' @pytest.mark.parametrize( 'rendered', ['hello', '[1, 2]', '[]', '{"a": 1}', 'x' * 2000] ) def test_something_that_is_not_one_key_is_left_alone(rendered): - assert _key_named_by(rendered) is None + assert key_named_by(rendered) is None # --- a label with more classes than the threshold -------------------------- diff --git a/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py b/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py index 5c69d37ef..3c303b2ad 100644 --- a/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py +++ b/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py @@ -524,19 +524,19 @@ def test_the_fallback_leaves_a_declared_meaning_alone(): declared meaning that contradicts its values is refused before a request gets this far. The guard is what keeps the rewrite safe to have at all: it may correct a placeholder and must never overrule a caller.""" - from sdm_client.adapters.tabular import _coherent_stype + from sdm_client.adapters.single_table import coherent_stype from sdm_client.schema import Column column = Column(name='c', dtype='string', stype='categorical') object.__setattr__(column, 'untyped', True) - assert _coherent_stype('categorical', 'timestamp[us]', column) == ( + assert coherent_stype('categorical', 'timestamp[us]', column) == ( 'timestamp' ) object.__setattr__(column, 'stype_declared', True) - assert _coherent_stype('categorical', 'timestamp[us]', column) == ( + assert coherent_stype('categorical', 'timestamp[us]', column) == ( 'categorical' ) diff --git a/scripts/check_repo_hygiene.py b/scripts/check_repo_hygiene.py index c97527237..d86163e13 100644 --- a/scripts/check_repo_hygiene.py +++ b/scripts/check_repo_hygiene.py @@ -196,6 +196,13 @@ REPOSITORY_URL = re.compile(r'https://github\.com/NVIDIA/[A-Za-z0-9._-]+') +#: Repositories this one legitimately points at, as opposed to the names it +#: used to be published under. A dependency installed from its own source is +#: a URL a reader is meant to follow, so it is not a stale name for this one. +UPSTREAM_URLS = frozenset( + {'https://github.com/NVIDIA/structured-data-models.git'} +) + def tracked_files() -> list[Path]: import subprocess @@ -340,7 +347,7 @@ def check_repository_url() -> int: continue text = path.read_text(encoding='utf-8', errors='replace') for found in set(REPOSITORY_URL.findall(text)): - if found != expected: + if found != expected and found not in UPSTREAM_URLS: findings.append(f'{relative}: {found}') for finding in sorted(findings): diff --git a/scripts/check_wire_payloads.py b/scripts/check_wire_payloads.py index aa0084fb7..95358f350 100755 --- a/scripts/check_wire_payloads.py +++ b/scripts/check_wire_payloads.py @@ -145,6 +145,31 @@ def recorded_bodies(payloads: Path) -> list[tuple[str, dict[str, Any]]]: return bodies +#: Models this client builds requests for that the contract's enums do not +#: name, and the fields those enums reject them in. The contract describes +#: what a deployment serves, and no deployment serves these: they run in this +#: process only. Everything else about such a body is still policed, so an +#: envelope that is wrong in any other way still fails here. +NOT_IN_THE_CONTRACT = { + 'kumo-anomaly': {'model': 'kumo-anomaly', 'task.kind': 'anomaly_detection'} +} + + +def _declared_elsewhere(body: dict[str, Any], error: Any) -> bool: + r"""Whether ``error`` is only that the contract has not heard of this + model yet, rather than something wrong with the request. + + Matched on the value as well as the field. Forgiving the field alone + would accept ``anomaly_detectoin`` and every other misspelling of it, + which is the one thing this file exists to catch. + """ + exempt = NOT_IN_THE_CONTRACT.get(body.get('model')) + if exempt is None or error.validator != 'enum': + return False + where = '.'.join(str(part) for part in error.path) + return where in exempt and error.instance == exempt[where] + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -210,6 +235,9 @@ def main() -> int: {**defs[wanted], '$defs': defs}, format_checker=FormatChecker() ) errors = sorted(validator.iter_errors(body), key=lambda e: list(e.path)) + errors = [ + error for error in errors if not _declared_elsewhere(body, error) + ] if not errors: continue failed += 1 diff --git a/scripts/snapshot_wire_payloads.py b/scripts/snapshot_wire_payloads.py index 716112983..a27a07738 100644 --- a/scripts/snapshot_wire_payloads.py +++ b/scripts/snapshot_wire_payloads.py @@ -19,6 +19,7 @@ import pandas as pd from sdm_client import PredictiveQuery, Request, Schema +from sdm_client.adapters.anomaly import AnomalyAdapter from sdm_client.adapters.relational import RelationalAdapter from sdm_client.adapters.tabular import TabularAdapter @@ -124,6 +125,19 @@ def _tabular_cases() -> dict[str, list[dict[str, Any]]]: return recorded +def _anomaly_cases() -> dict[str, list[dict[str, Any]]]: + readings = _people()[['user_id', 'age', 'churn']] + adapter = AnomalyAdapter() + request = Request.anomaly(readings) + recorded: dict[str, list[dict[str, Any]]] = {} + for label, outputs in _output_sets(adapter, request): + recorded[f'anomaly/direct-readings{label}'] = [ + _normalize(payload) + for payload in adapter.prediction_plan(request, outputs) + ] + return recorded + + def _relational_cases() -> dict[str, list[dict[str, Any]]]: schema = Schema.from_data({'people': _people(), 'orders': _orders()}) request = PredictiveQuery( @@ -150,6 +164,7 @@ def _relational_available() -> bool: def collect() -> dict[str, Any]: recorded: dict[str, Any] = dict(_tabular_cases()) + recorded.update(_anomaly_cases()) if _relational_available(): recorded.update(_relational_cases()) return recorded diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index ff04d740b..1f777c1bf 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -116,6 +116,7 @@ "Explanations", "FittedModel", "GradientExplanations", + "KumoAnomaly", "KumoRelational", "KumoTabular", "MissingExtraError", @@ -200,6 +201,17 @@ }, "signature": "(entries: 'Mapping[Any, dict[str, Any]] | None' = None, *, keyed_by: 'tuple[str, ...]' = ('id',), hops: 'Mapping[Any, Mapping[str, Sequence[int | None]]] | None' = None) -> 'None'" }, + "KumoAnomaly": { + "bases": [ + "sdm_client.models.Model" + ], + "kind": "class", + "public_attributes": [ + "name" + ], + "public_methods": {}, + "signature": "()" + }, "KumoRelational": { "bases": [ "sdm_client.models.Model" @@ -308,6 +320,7 @@ "target_moves_with_time" ], "public_methods": { + "anomaly": "(readings: 'pd.DataFrame', *, schema: 'Schema | None' = None, entity_table: 'str' = 'instance', inference: 'Mapping[str, Any] | None' = None) -> 'Request'", "materialized": "(*, payloads: 'tuple[dict[str, Any], ...]', schema: 'Schema', query: 'str', indices: 'list[Any]', entity_table: 'str | None' = None, windowed: 'bool' = False, requested_inference: 'Mapping[str, Any] | None' = None) -> 'Request'" }, "signature": "(context: 'pd.DataFrame', predict: 'pd.DataFrame', *, target: 'str', task: 'str', related_tables: 'Mapping[str, pd.DataFrame] | None' = None, schema: 'Schema | None' = None, entity_table: 'str' = 'instance', inference: 'Mapping[str, Any] | None' = None, task_options: 'Mapping[str, Any] | None' = None) -> 'None'" diff --git a/tests/golden/wire_payloads.json b/tests/golden/wire_payloads.json index c4157f9e6..e1d7d1555 100644 --- a/tests/golden/wire_payloads.json +++ b/tests/golden/wire_payloads.json @@ -1,4 +1,253 @@ { + "anomaly/direct-readings": [ + { + "context": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [] + } + }, + "metadata": { + "request_id": "" + }, + "model": "kumo-anomaly", + "output": { + "fields": [ + "prediction" + ] + }, + "predict": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [ + [ + 0, + 20.0, + 0 + ], + [ + 1, + 21.0, + 1 + ], + [ + 2, + 22.0, + 0 + ], + [ + 3, + 23.0, + 1 + ], + [ + 4, + 24.0, + 0 + ], + [ + 5, + 25.0, + 1 + ], + [ + 6, + 26.0, + 0 + ], + [ + 7, + 27.0, + 1 + ], + [ + 8, + 28.0, + 0 + ], + [ + 9, + 29.0, + 1 + ], + [ + 10, + 30.0, + 0 + ], + [ + 11, + 31.0, + 1 + ] + ] + } + }, + "schema": { + "instance_table": { + "columns": { + "age": { + "dtype": "float64", + "stype": "numerical" + }, + "churn": { + "dtype": "int64", + "stype": "numerical" + }, + "user_id": { + "dtype": "int64", + "stype": "ID" + } + }, + "primary_key": [ + "user_id" + ] + } + }, + "task": { + "kind": "anomaly_detection", + "target": { + "column_name": "*", + "dtype": "float64" + } + } + } + ], + "anomaly/direct-readings#default-outputs": [ + { + "context": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [] + } + }, + "metadata": { + "request_id": "" + }, + "model": "kumo-anomaly", + "output": { + "fields": [ + "prediction", + "probabilities" + ] + }, + "predict": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [ + [ + 0, + 20.0, + 0 + ], + [ + 1, + 21.0, + 1 + ], + [ + 2, + 22.0, + 0 + ], + [ + 3, + 23.0, + 1 + ], + [ + 4, + 24.0, + 0 + ], + [ + 5, + 25.0, + 1 + ], + [ + 6, + 26.0, + 0 + ], + [ + 7, + 27.0, + 1 + ], + [ + 8, + 28.0, + 0 + ], + [ + 9, + 29.0, + 1 + ], + [ + 10, + 30.0, + 0 + ], + [ + 11, + 31.0, + 1 + ] + ] + } + }, + "schema": { + "instance_table": { + "columns": { + "age": { + "dtype": "float64", + "stype": "numerical" + }, + "churn": { + "dtype": "int64", + "stype": "numerical" + }, + "user_id": { + "dtype": "int64", + "stype": "ID" + } + }, + "primary_key": [ + "user_id" + ] + } + }, + "task": { + "kind": "anomaly_detection", + "target": { + "column_name": "*", + "dtype": "float64" + } + } + } + ], "relational/pql-count-aggregation": [ { "context": { diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 2c901ce48..8e10c3ef6 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -846,6 +846,56 @@ def test_each_request_is_read_as_the_one_it_is( written.write_text(json.dumps({'case': body})) assert _check_payloads(written).returncode == 0 + @needs_the_contract + @pytest.mark.parametrize( + ('mutate', 'said'), + [ + (lambda body: body.update(invented_key=1), 'invented_key'), + ( + lambda body: body['output'].update(fields=['predcition']), + 'predcition', + ), + (lambda body: body.pop('schema'), "'schema'"), + ( + lambda body: body['task'].update(kind='anomaly_detectoin'), + 'anomaly_detectoin', + ), + ( + lambda body: body.update(model='kumo-anomoly'), + 'kumo-anomoly', + ), + ], + ids=[ + 'an invented top-level key', + 'a field off the enum', + 'a dropped section', + 'a misspelt task kind', + 'a misspelt model id', + ], + ) + def test_a_model_the_contract_has_not_heard_of_is_policed_elsewhere( + self, tmp_path, mutate, said + ): + """`kumo-anomaly` is off the contract's model enum on purpose, and + that is the only thing forgiven. A body wrong in any other way still + fails, or the exemption would be a hole rather than a note.""" + recorded = json.loads((GOLDEN / 'wire_payloads.json').read_text()) + body = json.loads(json.dumps(recorded['anomaly/direct-readings'][0])) + assert body['model'] == 'kumo-anomaly' + mutate(body) + + written = tmp_path / 'wire_payloads.json' + written.write_text(json.dumps({'case': body})) + result = _check_payloads(written) + + assert result.returncode == 1 + assert said in result.stderr + + @needs_the_contract + def test_a_body_the_contract_has_not_heard_of_is_otherwise_accepted(self): + """The premise of the test above: untouched, it passes.""" + assert _check_payloads().returncode == 0 + @needs_the_contract def test_a_body_that_is_no_request_at_all_is_refused(self, tmp_path): written = tmp_path / 'wire_payloads.json'