From b395ec94379e43f352487a44af37cc5d9fa4633d Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:01:57 -0800 Subject: [PATCH 1/9] Score a row for how anomalous it is, with nothing labelled to learn from Adds `kumo-anomaly` to the models this client can be asked, served by structured-data-models in this process. It is unlike the two models already here in the way that shaped the change: it is unsupervised. There is no target column to name and no labelled rows to learn from, so `Request.anomaly(readings)` takes neither, and the in-process backend tells a model that fits a context from one that scores rows alone rather than asking every model for a context it may not have. A session is refused instead of pinned, because there is nothing fitted to pin. The model returns three numbers per row and `PredictionItem` has room for two kinds. `prediction` carries the reconstruction error, which is the answer to the question that was asked, and the two thresholders' labels ride in `probabilities`, whose name is wrong for them and whose shape is right. The contract is generated from a spec that does not describe this model, so widening it is not this client's to do; the recorded envelope is exempt from the model and task enums by name, and policed in every other respect. Reconstruction is sampled, so the run is seeded the way a fitted context is. Without that an unchanged request came back with different numbers each call. The models are on structured-data-models' main and in no release of it, so CI installs that commit rather than asking for the `in-process` extra, whose range names published versions. Pinning it in the package's own metadata would be rejected at upload, which would trade a test that runs for a release that cannot be published. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .github/workflows/ci.yml | 16 +- .../pyproject.toml | 6 +- .../src/sdm_client/__init__.py | 8 +- .../src/sdm_client/adapters/__init__.py | 3 +- .../src/sdm_client/adapters/anomaly.py | 197 ++++++++++++++ .../src/sdm_client/client.py | 4 + .../src/sdm_client/in_process/anomaly.py | 189 +++++++++++++ .../src/sdm_client/in_process/backend.py | 139 +++++++--- .../src/sdm_client/models.py | 50 +++- .../src/sdm_client/request.py | 72 +++++ .../tests/conftest.py | 8 + .../test_a_model_runs_in_this_process.py | 6 +- ...st_an_anomaly_is_scored_without_a_label.py | 162 ++++++++++++ scripts/check_wire_payloads.py | 20 ++ scripts/snapshot_wire_payloads.py | 15 ++ tests/golden/public_api.json | 13 + tests/golden/wire_payloads.json | 249 ++++++++++++++++++ tests/test_release_contract.py | 40 +++ 18 files changed, 1161 insertions(+), 36 deletions(-) create mode 100644 packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py create mode 100644 packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py create mode 100644 packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c7f5fd2b..3ad44bb38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,9 +166,23 @@ jobs: # index, since the default one resolves to a far larger CUDA build. - run: | pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install -e './packages/structured-data-models-client[test,in-process]' + pip install -e './packages/structured-data-models-client[test]' + pip install "structured-data-models @ git+https://github.com/NVIDIA/structured-data-models.git@${SDM_REF}" python -m pytest packages/structured-data-models-client/tests -q \ -m 'not live_nim and not live_warehouse' + env: + # The time-series models are on structured-data-models' main and in + # no release of it, so the `in-process` extra cannot reach them: its + # range names published versions, and main's own version string is + # below the floor of that range. This commit is installed instead, + # which is why the extra is not asked for here. + # + # It is pinned here rather than in the package's metadata on + # purpose. A direct URL in `optional-dependencies` is rejected at + # upload, so declaring it there would trade a test that runs for a + # release that cannot be published. Move the extra's range forward + # and delete this once a release carries the models. + SDM_REF: b5c36413dd30bcb93b39bfed2afdccf3e509781d # And the root suite, which holds the relational wire snapshots. They # only exercise anything with an engine installed, so running them in # the test job would prove nothing. diff --git a/packages/structured-data-models-client/pyproject.toml b/packages/structured-data-models-client/pyproject.toml index 9dfbb274b..920cd3f6e 100644 --- a/packages/structured-data-models-client/pyproject.toml +++ b/packages/structured-data-models-client/pyproject.toml @@ -78,7 +78,11 @@ s3 = ["structured-data-models-connectors[s3]==1.0.0"] # model rather than a backend to read, so `all` does not carry it; ask for it by # name. databricks-serving = ["databricks-sdk>=0.68.0,<1.0"] -# Running the models in this process rather than reaching a NIM. Like +# Running the models in this process rather than reaching a NIM. The range +# below names published releases, and none of them carries the time-series +# models, so `KumoAnomaly` needs that package installed from its own main +# until a release does. Widen this and drop the pin in `ci.yml` then. +# Like # `databricks-serving` it is a way to reach a model, and it brings PyTorch, so # `all` does not carry it. structured-data-models needs Python 3.11; on 3.10 # the extra installs nothing, and `SDMClient.in_process()` says why. The marker diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index e861ef39b..1b20c794c 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -35,7 +35,12 @@ Explanations, GradientExplanations, ) -from sdm_client.models import KumoRelational, KumoTabular, Model +from sdm_client.models import ( + KumoAnomaly, + KumoRelational, + KumoTabular, + Model, +) from sdm_client.query import PredictiveQuery from sdm_client.request import Request, Split, Target, Task from sdm_client.result import PredictionResult @@ -54,6 +59,7 @@ 'Explanations', 'FittedModel', 'GradientExplanations', + 'KumoAnomaly', 'KumoRelational', 'KumoTabular', 'MissingExtraError', diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py b/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py index 9519e2f24..de347bcb2 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/__init__.py @@ -4,6 +4,7 @@ """Per-model adapters. One module per model.""" +from sdm_client.adapters.anomaly import AnomalyAdapter from sdm_client.adapters.tabular import TabularAdapter -__all__ = ['TabularAdapter'] +__all__ = ['AnomalyAdapter', 'TabularAdapter'] diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py new file mode 100644 index 000000000..086d4e2f3 --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Shaping a request for ``kumo-anomaly``. + +Short, because this model is asked the least of any here. There is no target +column to type, no context to agree with the rows being scored, and no session +to pin, so what is left is the schema the readings are written against. + +The envelope still carries an empty ``context``. The contract requires that +section, and a model with nothing to learn from has a context with no rows +rather than no context, which is a thing the envelope can say. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator, Mapping, Sequence +from typing import Any + +from sdm_api.generated.tfm_api import ( + PredictionItem, + PredictionResponse, + ResponseContractError, +) + +from sdm_client.base import ModelSpec +from sdm_client.errors import StructuredDataError, UnsupportedRequestError +from sdm_client.models import KUMO_ANOMALY +from sdm_client.request import Request +from sdm_client.result import PredictionResult +from sdm_client.wire import ( + UNTYPED, + encode_table, + infer_tfm_dtype, + json_ready, + widen_tfm_dtype, +) +from sdm_client.wire.inference import typed_inference + +_DEFAULT_OUTPUTS = ('prediction', 'probabilities') + + +def _request_id() -> str: + return f'sdm-{uuid.uuid4().hex[:12]}' + + +class AnomalyAdapter: + r"""Shapes and reads ``kumo-anomaly`` requests.""" + + spec: ModelSpec = KUMO_ANOMALY + + def default_output(self, request: Request) -> tuple[str, ...]: + return _DEFAULT_OUTPUTS + + def prediction_plan( + self, request: Request, outputs: tuple[str, ...] + ) -> Iterator[dict[str, Any]]: + r"""One payload. The score for a row depends on the rows around it, + so splitting a series into batches would change the answer.""" + yield self._envelope(request, outputs) + + def session_plan( + self, request: Request, outputs: tuple[str, ...] + ) -> dict[str, Any]: + raise UnsupportedRequestError( + f'{self.spec.model_id} holds no session: it is shown no labelled ' + f'rows, so there is no fitted context to pin and score against. ' + f'Call predict(), which sends the readings with every call' + ) + + def session_prediction_plan( + self, + request: Request, + outputs: tuple[str, ...], + pinned: Mapping[str, Any] | None = None, + ) -> Iterator[dict[str, Any]]: + raise UnsupportedRequestError(f'{self.spec.model_id} holds no session') + + def normalize( + self, + response: dict[str, Any], + outputs: tuple[str, ...], + payload: dict[str, Any] | None = None, + ) -> PredictionResult: + try: + parsed = PredictionResponse.from_dict(response) + except (ResponseContractError, TypeError, ValueError) as error: + raise StructuredDataError( + f'{self.spec.model_id} returned a response that does not ' + f'match the contract: {error}', + code='INVALID_RESPONSE', + ) from error + return self.answered( + parsed.predictions, + outputs, + payload, + model=parsed.model, + request_id=parsed.id, + metadata=parsed.metadata, + ) + + def answered( + self, + items: Sequence[PredictionItem], + outputs: tuple[str, ...], + payload: dict[str, Any] | None, + *, + model: str, + request_id: str = '', + metadata: Mapping[str, Any] | None = None, + ) -> PredictionResult: + return PredictionResult( + model=model, + request_id=request_id, + items=items, + metadata=dict(metadata or {}), + outputs=outputs, + ) + + def explain_error( + self, error: Exception, payload: dict[str, Any] + ) -> Exception: + r"""Nothing to add: a refusal here is about the readings, which the + caller handed over whole.""" + return error + + def close(self) -> None: + r"""Nothing to release.""" + + def _envelope( + self, request: Request, outputs: tuple[str, ...] + ) -> dict[str, Any]: + if request.payloads is not None: + raise UnsupportedRequestError( + f'this request was gathered for another model and carries ' + f'the sampled payloads that model reads; ' + f'{self.spec.model_id} is asked with Request.anomaly(...)' + ) + request.schema.check_consistent() + + table = request.schema[request.entity_table] + readings = request.predict.instance_table + + dtypes = { + name: (UNTYPED if column.untyped else column.dtype) + for name, column in table.columns.items() + } + for name in readings.columns: + if name in dtypes: + dtypes[name] = widen_tfm_dtype( + (dtypes[name], infer_tfm_dtype(readings[name])) + ) + dtypes = {name: dtype or 'string' for name, dtype in dtypes.items()} + + missing = [name for name in readings.columns if name not in dtypes] + if missing: + raise StructuredDataError( + f'schema does not describe column(s) {missing}', + code='INVALID_REQUEST', + ) + + instance_schema: dict[str, Any] = { + 'columns': { + name: {'dtype': dtypes[name], 'stype': table[name].stype} + for name in readings.columns + } + } + key = table.primary_key + if key: + instance_schema['primary_key'] = ( + [key] if isinstance(key, str) else list(key) + ) + + envelope: dict[str, Any] = { + 'model': self.spec.model_id, + 'task': { + 'kind': request.task.kind, + 'target': { + 'column_name': request.task.target.column_name, + 'dtype': request.task.target.dtype, + }, + }, + 'schema': {'instance_table': instance_schema}, + 'context': { + 'instance_table': encode_table(readings.iloc[:0], dtypes) + }, + 'predict': {'instance_table': encode_table(readings, dtypes)}, + 'output': {'fields': list(outputs)}, + 'metadata': {'request_id': _request_id()}, + } + if request.inference: + envelope['inference'] = typed_inference( + json_ready(request.inference) + ) + return envelope diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 34c1ce818..8325e004e 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -82,6 +82,10 @@ def _adapter_for(spec: ModelSpec) -> ModelAdapter: from sdm_client.adapters.relational import RelationalAdapter return RelationalAdapter() + if spec.model_id == 'kumo-anomaly': + from sdm_client.adapters.anomaly import AnomalyAdapter + + return AnomalyAdapter() raise UnknownModelError(spec.model_id, list(MODELS)) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py new file mode 100644 index 000000000..b933b1b0f --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""``kumo-anomaly`` in process: one table of readings, scored as a series. + +This model is unlike the other two in the one way that matters here. It is +unsupervised, so there is no context to fit and no target column to name: it +reconstructs the rows it is given and scores each one by how badly the +reconstruction missed. That is why :class:`Anomaly` has no ``context`` or +``query``, and why a session over it is refused rather than pinned. + +The answer is three numbers per row, and the contract's ``PredictionItem`` has +nowhere to put three. ``prediction`` carries the score, which is the answer to +the question that was asked. The two labels ride in ``probabilities``, whose +name is wrong for them and whose shape is right: a mapping that reaches the +frame as one flat column each. The contract is generated from a spec that does +not describe this model, so widening it is not this client's to do. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from sdm_api.generated.tfm_api import PredictionItem + +from sdm_client.errors import StructuredDataError +from sdm_client.in_process.columns import ( + Shape, + check_columns, + key_columns, + stype_of, +) +from sdm_client.in_process.tables import values + +#: The thresholders the model applies, in the order it returns them. +THRESHOLDERS = ('scs', 'macs') + +#: How the published deployment samples. Reconstructions are combined by their +#: median, so fewer samples is a noisier score rather than a different one. +_SAMPLES = 30 +_STEPS = 20 + + +class Anomaly: + r"""Reads a ``kumo-anomaly`` request into the model.""" + + model_id = 'kumo-anomaly' + outputs = {'anomaly_detection': ('prediction', 'probabilities')} + + @staticmethod + def load(device: Any) -> Any: + from sdm.models import KumoAnomaly + + return KumoAnomaly(device=device) + + @staticmethod + def accepts(model: Any) -> bool: + from sdm.models import KumoAnomaly + + return isinstance(model, KumoAnomaly) + + def check_inference( + self, + options: Mapping[str, Any], + task: Mapping[str, Any], + settled: bool = False, + ) -> None: + r"""Refuse an inference option this model does not act on.""" + remaining = dict(options) + if remaining.pop('inference_config', {}): + raise StructuredDataError( + 'inference_config is reserved for API compatibility and must ' + 'be omitted or empty', + code='INVALID_REQUEST', + ) + unknown = sorted(set(remaining) - {'num_samples', 'num_steps'}) + if unknown: + raise StructuredDataError( + f'unsupported inference options: {unknown}', + code='INVALID_REQUEST', + ) + for name in ('num_samples', 'num_steps'): + value = remaining.get(name) + if value is None: + continue + if not isinstance(value, int) or isinstance(value, bool): + raise StructuredDataError( + f'{name} is {value!r}; it is a whole number of at least 1', + code='INVALID_REQUEST', + ) + if value < 1: + raise StructuredDataError( + f'{name} is {value}; it is a whole number of at least 1', + code='INVALID_REQUEST', + ) + + def rows(self, payload: Mapping[str, Any]) -> Any: + r"""The readings to score, as the model reads them. + + Every column is a channel of one multivariate series, so the whole + table is read rather than all but a target. A key column names the + row instead of measuring anything, so reconstructing it would score + rows by how surprising their identifiers are. + """ + spec = payload['schema']['instance_table'] + table = payload['predict']['instance_table'] + check_columns(table, spec, 'the rows to score') + keys = key_columns(spec) + stypes = { + name: stype_of(name, spec['columns'][name]) + for name in table['columns'] + if name not in keys + } + unread = sorted( + name for name, stype in stypes.items() if stype != 'numerical' + ) + if unread: + raise StructuredDataError( + f'{self.model_id} reconstructs numerical columns, and ' + f'{unread} is not one. A reading it cannot reconstruct has no ' + f'error to score, so leave it out of the rows you send', + code='UNSUPPORTED_DTYPE', + ) + if not stypes: + raise StructuredDataError( + f'{self.model_id} scores a row by reconstructing it, and ' + f'these rows carry no numerical column to reconstruct', + code='INVALID_REQUEST', + ) + return Shape.of(spec, stypes).tensor(table, 'the rows to score') + + @staticmethod + def row_ids( + table: Mapping[str, Any], spec: Mapping[str, Any] + ) -> list[str | None]: + r"""A key of several columns answers under the JSON of its values as + they were sent, which the client hands back as the row's id. + + Otherwise there is no id: each answer is matched to its row by + position, and the client puts the row's own key on it. + """ + key = spec.get('primary_key') + if not isinstance(key, list) or len(key) < 2: + return [None] * len(table['rows']) + cells = [values(table, name) for name in key] + return [ + json.dumps(list(values_of), separators=(',', ':'), default=str) + for values_of in zip(*cells, strict=True) + ] + + def answer( + self, + model: Any, + payload: Mapping[str, Any], + ids: Sequence[Any], + generator: Any = None, + ) -> list[PredictionItem]: + r"""Score each row, under the id the request gave it. + + ``generator`` is what makes the same readings score the same twice. + Reconstruction is sampled, so without one an unchanged request comes + back with different numbers every call. + """ + options = payload.get('inference') or {} + out = model( + self.rows(payload), + num_samples=int(options.get('num_samples') or _SAMPLES), + num_steps=int(options.get('num_steps') or _STEPS), + generator=generator, + ) + scores = out['mae'].numerical[..., 0].tolist() + labels = { + name: out[name].categorical.decode()[..., 0].tolist() + for name in THRESHOLDERS + } + return [ + PredictionItem( + row_index=index, + id=ids[index] if index < len(ids) else None, + prediction=score, + probabilities={ + name: float(labels[name][index]) for name in THRESHOLDERS + }, + ) + for index, score in enumerate(scores) + ] diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index 965295fe3..e73e87ea9 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -22,10 +22,11 @@ import uuid from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from sdm_client.errors import MissingExtraError, StructuredDataError from sdm_client.explanations import GradientConverter +from sdm_client.in_process.anomaly import Anomaly from sdm_client.in_process.answers import ( QUANTILE_LEVELS, class_order, @@ -46,11 +47,48 @@ _SEED = 42 _ESTIMATORS = 1 -_SERVED = {served.model_id: served for served in (Tabular(), Relational())} +_SERVED = { + served.model_id: served for served in (Tabular(), Relational(), Anomaly()) +} + +#: What reads a request for one model. +Served = Tabular | Relational | Anomaly + + +@runtime_checkable +class ScoresRowsAlone(Protocol): + r"""A model shown no labelled rows, which therefore fits no context. + + It answers from the rows it is given and nothing carries over, so there + is no fitted context to pin and no session to hold. Declaring that as a + protocol rather than a flag is what lets the two shapes be told apart + where it matters, by what a reader actually has to offer. + """ + + def row_ids( + self, table: Mapping[str, Any], spec: Mapping[str, Any] + ) -> list[str | None]: ... + + def check_inference( + self, + options: Mapping[str, Any], + task: Mapping[str, Any], + settled: bool = False, + ) -> None: ... + + def answer( + self, + model: Any, + payload: Mapping[str, Any], + ids: Sequence[Any], + generator: Any = None, + ) -> list[Any]: ... @dataclass class _Session: + # Only a model that fits a context reaches here: `create_session` + # refuses one that does not, because there would be nothing to pin. served: Tabular | Relational task: Mapping[str, Any] schema: Mapping[str, Any] @@ -145,23 +183,32 @@ def predict( ) served, model = self._serving(payload['model']) _check_loaded_for(served, model, task['kind']) - context = served.context(payload) - query = served.query(context, payload, payload['schema']) - fitted = _fit(model, context) - answer = _run(fitted, query) - items = _items( - served, - task, - payload['schema'], - context, - payload, - answer, - fields, - ) - if 'explanation' in fields: - items = _explained( - items, fitted, query, context, task['kind'], payload, fields + if isinstance(served, ScoresRowsAlone): + items = _scored(served, model, payload) + else: + context = served.context(payload) + query = served.query(context, payload, payload['schema']) + fitted = _fit(model, context) + answer = _run(fitted, query) + items = _items( + served, + task, + payload['schema'], + context, + payload, + answer, + fields, ) + if 'explanation' in fields: + items = _explained( + items, + fitted, + query, + context, + task['kind'], + payload, + fields, + ) return adapter.answered( items, outputs, @@ -176,8 +223,16 @@ def create_session( r"""Fit the context once, to be scored against until released.""" with self._running(): kind = payload['task']['kind'] - _fields(self._reader(payload['model']), kind, payload) - served, model = self._serving(payload['model']) + served = self._reader(payload['model']) + if isinstance(served, ScoresRowsAlone): + raise StructuredDataError( + f'{served.model_id} holds no session: it is shown no ' + f'labelled rows, so there is no fitted context to pin. ' + f'Call predict() instead', + code='SESSIONS_UNSUPPORTED', + ) + _fields(served, kind, payload) + _, model = self._serving(payload['model']) _check_loaded_for(served, model, kind) context = served.context(payload) fitted = _fit(model, context) @@ -250,7 +305,7 @@ def delete_session(self, session_id: str) -> None: self._require_open() self._sessions.pop(session_id, None) - def _reader(self, model_id: str) -> Tabular | Relational: + def _reader(self, model_id: str) -> Served: r"""What reads requests for ``model_id``, known without loading it.""" served = _SERVED.get(model_id) if served is None: @@ -264,7 +319,7 @@ def _not_served(self, model_id: str) -> StructuredDataError: code='INVALID_CONFIGURATION', ) - def _serving(self, model_id: str) -> tuple[Tabular | Relational, Any]: + def _serving(self, model_id: str) -> tuple[Served, Any]: r"""What reads requests for ``model_id``, and the model it runs. Runs are serialized, so a model is loaded once. The load is not held @@ -347,7 +402,7 @@ def _require_sdm() -> Any: return torch -def _served_by(model: Any) -> Tabular | Relational: +def _served_by(model: Any) -> Served: for served in _SERVED.values(): if served.accepts(model): return served @@ -371,7 +426,7 @@ def _device(device: Any, torch: Any) -> Any: ) from error -def _loaded(served: Tabular | Relational, device: Any) -> Any: +def _loaded(served: Served, device: Any) -> Any: try: return served.load(device) except Exception as error: @@ -387,7 +442,7 @@ def _request_id(payload: Mapping[str, Any]) -> str: return str((payload.get('metadata') or {}).get('request_id') or '') -def _check_task(served: Tabular | Relational, kind: str) -> None: +def _check_task(served: Served, kind: str) -> None: r"""Refuse a task this model does not answer.""" if kind not in served.outputs: raise StructuredDataError( @@ -397,14 +452,17 @@ def _check_task(served: Tabular | Relational, kind: str) -> None: ) -def _check_loaded_for( - served: Tabular | Relational, model: Any, kind: str -) -> None: +def _check_loaded_for(served: Served, model: Any, kind: str) -> None: r"""Refuse a task this model was loaded without. A model can be loaded for classification or regression alone, which a NIM's never is, so that is asked of the model rather than of its kind. """ + if isinstance(served, ScoresRowsAlone): + # `tasks` says which heads a checkpoint was packaged with, and a model + # that fits nothing has no head to package: it answers the one task it + # is for or it is not that model. + return needed = 'regression' if kind == 'regression' else 'classification' loaded = sorted(str(task) for task in model.tasks) if needed not in loaded: @@ -417,7 +475,7 @@ def _check_loaded_for( def _fields( - served: Tabular | Relational, + served: Served, kind: str, payload: Mapping[str, Any], ) -> list[str]: @@ -518,8 +576,29 @@ def _autocast(device: Any) -> Any: return torch.autocast(device_type='cuda', dtype=dtype) +def _scored( + served: ScoresRowsAlone, model: Any, payload: Mapping[str, Any] +) -> list[Any]: + r"""One answer per row, from a model with no context to fit.""" + import torch + + ids = served.row_ids( + payload['predict']['instance_table'], + payload['schema']['instance_table'], + ) + served.check_inference(payload.get('inference') or {}, payload['task']) + device = _placement(model) + with _autocast(device): + return served.answer( + model, + payload, + ids, + torch.Generator(device=device).manual_seed(_SEED), + ) + + def _items( - served: Tabular | Relational, + served: Served, task: Mapping[str, Any], schema: Mapping[str, Any], context: Context, diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 2e360f229..4161063b0 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -32,7 +32,7 @@ from __future__ import annotations -from predictive_query import RELATIONAL, TABULAR +from predictive_query import ANOMALY, RELATIONAL, TABULAR from sdm_api import TFM_MODEL_KUMO_RELATIONAL, TFM_MODEL_KUMO_TABULAR from sdm_client.base import ModelSpec, RuntimeContract @@ -158,6 +158,31 @@ ) +# Named here rather than imported from the generated bindings, which carry +# only the ids the pinned spec's model enum declares. The spec does not +# describe a model that scores a row without being shown a labelled one, so +# there is no constant to import and hand-editing a generated file would be +# undone by the next regeneration. +TFM_MODEL_KUMO_ANOMALY = 'kumo-anomaly' + +KUMO_ANOMALY = ModelSpec( + model_id=TFM_MODEL_KUMO_ANOMALY, + profile=ANOMALY, + runtime=RuntimeContract( + tasks=frozenset({'anomaly_detection'}), + # `probabilities` carries the two thresholders' labels. The field is + # named for what every other model puts in it, and this one puts + # something else there because the contract has no third field for a + # row's answer. See `sdm_client.in_process.anomaly`. + outputs=frozenset({'prediction', 'probabilities'}), + # A session pins a fitted context, and this model fits none: every + # call reconstructs the rows it was given and nothing carries over. + sessions=False, + inference_options=frozenset({'num_samples', 'num_steps'}), + ), +) + + class Model: r"""Base for the model handles. Not useful on its own.""" @@ -205,10 +230,33 @@ class KumoRelational(Model): _spec = KUMO_RELATIONAL +class KumoAnomaly(Model): + r"""One table of readings in, a score per row out. + + Takes ``anomaly_detection`` only, and produces ``prediction``, the + reconstruction error for the row, and ``probabilities``, the two + thresholders' labels for it. It is unsupervised, so a request is built + with :meth:`Request.anomaly ` and names no + target. It holds no session, because there is no context to pin. + + Served in this process only, through + :meth:`SDMClient.in_process `: no + deployment answers for this id yet. + + Needs a structured-data-models that carries ``KumoAnomaly``, which no + release does yet. The ``in-process`` extra installs the latest release, so + until one ships this model is reached by installing that package from its + own main. + """ + + _spec = KUMO_ANOMALY + + #: Every model this client can send to, by id. MODELS = { KUMO_TABULAR.model_id: KUMO_TABULAR, KUMO_RELATIONAL.model_id: KUMO_RELATIONAL, + KUMO_ANOMALY.model_id: KUMO_ANOMALY, } diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index a09ac33da..032093959 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -49,6 +49,12 @@ # Task kinds the contract carries. A model narrows this further: a tabular # model takes only the coarse 'classification', and its adapter says so. +# +# 'anomaly_detection' is the exception: the contract's enum does not carry it, +# because the spec these bindings are generated from does not describe a model +# that answers it. It is named here so a request can be built and refused by +# every model that does not serve it, which is what the rest of this tuple is +# for. Nothing is sent to a deployment that has not declared it. TASK_KINDS = ( 'classification', 'binary_classification', @@ -57,8 +63,15 @@ 'forecasting', 'temporal_link_prediction', 'ranking', + 'anomaly_detection', ) +#: The target of a question asked about the whole row rather than one column. +#: An anomaly score is joint across the channels it reconstructs, so naming +#: one column would promise a score specific to it that the model never +#: computes. +WHOLE_ROW = '*' + _CLASSIFICATION_KINDS = frozenset( {'classification', 'binary_classification', 'multiclass_classification'} ) @@ -542,6 +555,65 @@ def materialized( request.requested_inference = dict(requested_inference or {}) return request + @classmethod + def anomaly( + cls, + readings: pd.DataFrame, + *, + schema: Schema | None = None, + entity_table: str = 'instance', + inference: Mapping[str, Any] | None = None, + ) -> Request: + r"""Ask how anomalous each row is, with no labelled rows to learn from. + + Every other request here names a target and shows the model rows where + the answer is known. An anomaly model is unsupervised: it reconstructs + the readings it is given and scores each row by how badly the + reconstruction missed, so there is no column to name and nothing to + learn from. Those two arguments are absent rather than empty, because + a caller who passed them would be describing a question nobody asks. + + Args: + readings: The rows to score. Every numerical column is read as a + channel of one multivariate series; a primary key names the + row and is not read as a channel. + schema: Override the inferred schema. Rarely needed. + entity_table: The name the instance table is known by. + inference: Gathering knobs passed to the adapter. + + Raises: + StructuredDataError: If ``readings`` is not a DataFrame. + + >>> request = Request.anomaly(readings) + >>> client.predict(request, model=KumoAnomaly()) + """ + readings = _require_frame(readings, 'readings') + if schema is None: + schema = Schema.from_data({entity_table: readings}) + + request = cls.__new__(cls) + request.schema = schema + request.task = Task( + kind='anomaly_detection', + target=Target( + column_name=WHOLE_ROW, + # The score, which is what this target stands for and the one + # thing about it with a type the answer actually carries. + dtype='float64', + table_name=entity_table, + ), + ) + # The contract has no shape for a request with no context, and this + # one genuinely has none. An empty table carries the schema's columns + # and no rows, which is what "nothing to learn from" looks like on the + # wire, rather than a section left out of an envelope that requires it. + request._context = Split(instance_table=readings.iloc[:0]) + request._predict = Split(instance_table=readings) + request.inference = dict(inference or {}) + request.requested_inference = dict(inference or {}) + request.entity_table = entity_table + return request + @property def _question(self) -> Question: r"""What this request asks. See :class:`Question`.""" diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py index cbe32cda0..bdda5ae67 100644 --- a/packages/structured-data-models-client/tests/conftest.py +++ b/packages/structured-data-models-client/tests/conftest.py @@ -32,6 +32,14 @@ def tabular(sdm): return sdm.models.KumoTabular(size='small', pretrained=False, device='cpu') +@pytest.fixture(scope='module') +def anomaly(sdm): + import torch + + torch.manual_seed(0) + return sdm.models.KumoAnomaly(pretrained=False, device='cpu') + + @pytest.fixture(scope='module') def relational(sdm): pytest.importorskip('sdm_relational_engine', reason='needs [relational]') diff --git a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py index 6a55fb51d..a8c08ce52 100644 --- a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py +++ b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py @@ -266,7 +266,11 @@ def load(**kwargs): ) with SDMClient.in_process() as client: assert client.url == 'in-process' - assert client.models() == ['kumo-tabular', 'kumo-relational'] + assert client.models() == [ + 'kumo-tabular', + 'kumo-relational', + 'kumo-anomaly', + ] assert loads == [] client.predict(request, model=KumoTabular()) client.predict(request, model=KumoTabular()) diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py new file mode 100644 index 000000000..c0f04a91d --- /dev/null +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``kumo-anomaly`` asked for a score, with nothing labelled to learn from. + +A small random-weight model stands in for the published one, so nothing is +downloaded and no GPU is needed. The sampling is turned down for the same +reason: reconstructions are combined by their median, so fewer samples is a +noisier score rather than a different kind of answer. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from sdm_client import ( + KumoAnomaly, + KumoTabular, + Request, + SDMClient, + StructuredDataError, + UnsupportedRequestError, +) + +pytestmark = pytest.mark.filterwarnings('ignore') + +FAST = {'num_samples': 2, 'num_steps': 4} +#: What the backend seeds every run with, named here so the direct call this +#: is compared against draws the same samples. +SEED = 42 +DISTURBED = slice(80, 90) + + +def readings(rows: int = 120) -> pd.DataFrame: + """A two-channel series with one stretch pushed off its own shape.""" + step = np.arange(rows, dtype=float) + frame = pd.DataFrame( + { + 'reading_id': np.arange(rows), + 'flow': np.sin(step / 9), + 'pressure': np.cos(step / 14), + } + ) + frame.loc[DISTURBED, ['flow', 'pressure']] += 4.0 + return frame + + +def scored(model, frame: pd.DataFrame, **inference) -> pd.DataFrame: + with SDMClient.in_process(model) as client: + request = Request.anomaly(frame, inference={**FAST, **inference}) + return client.predict(request, model=KumoAnomaly()).to_pandas() + + +def test_a_score_and_a_label_per_row_come_back(anomaly): + """The answer names the row, its score, and each thresholder's verdict.""" + frame = readings() + answer = scored(anomaly, frame) + + assert list(answer.columns) == [ + 'row_index', + 'prediction', + 'probability_scs', + 'probability_macs', + ] + assert len(answer) == len(frame) + assert answer['row_index'].tolist() == list(range(len(frame))) + assert np.isfinite(answer['prediction']).all() + for name in ('probability_scs', 'probability_macs'): + assert set(answer[name].unique()) <= {0.0, 1.0} + + +def test_the_same_readings_score_the_same_twice(anomaly): + """Reconstruction is sampled, so the answer is seeded. Without that an + unchanged request comes back with different numbers every call.""" + frame = readings() + + assert scored(anomaly, frame)['prediction'].tolist() == pytest.approx( + scored(anomaly, frame)['prediction'].tolist() + ) + + +def test_the_disturbed_stretch_scores_above_the_rest(anomaly): + """The score is what the model was asked for, so it has to track the + thing that was done to the data rather than merely be finite.""" + answer = scored(anomaly, readings()) + disturbed = answer['prediction'].iloc[DISTURBED] + rest = answer['prediction'].drop(answer.index[DISTURBED]) + + assert disturbed.mean() > rest.mean() + + +def test_the_answer_is_the_one_the_model_gives_when_called_directly( + sdm, anomaly +): + """Nothing is computed on the way through: the client shapes the request + and reads the answer back, and the numbers are the model's own.""" + import torch + + frame = readings() + answer = scored(anomaly, frame) + channels = frame[['flow', 'pressure']] + direct = anomaly( + sdm.TableTensor.from_pandas( + channels, dict.fromkeys(channels.columns, 'numerical') + ), + generator=torch.Generator().manual_seed(SEED), + **FAST, + ) + + torch.testing.assert_close( + torch.tensor(answer['prediction'].to_numpy()), + direct['mae'].numerical[..., 0].double(), + ) + + +def test_a_key_column_is_not_read_as_a_channel(anomaly): + """`reading_id` names the row. Reconstructing it would score a row by how + surprising its identifier is, which rises with every row.""" + frame = readings() + without = frame.drop(columns=['reading_id']) + + assert scored(anomaly, frame)['prediction'].tolist() == pytest.approx( + scored(anomaly, without)['prediction'].tolist() + ) + + +def test_a_column_the_model_cannot_reconstruct_is_refused(anomaly): + """A reading it cannot reconstruct has no error to score.""" + frame = readings() + frame['site'] = 'north' + + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, frame) + assert "'site'" in str(caught.value) or 'site' in str(caught.value) + + +def test_there_is_no_session_to_pin(anomaly): + """A session pins a fitted context, and this model fits none.""" + with SDMClient.in_process(anomaly) as client: + with pytest.raises( + (UnsupportedRequestError, StructuredDataError) + ) as caught: + client.fit(Request.anomaly(readings()), model=KumoAnomaly()) + assert 'session' in str(caught.value) + + +def test_another_model_is_not_asked_this_question(tabular): + """`anomaly_detection` is a task only one model here answers.""" + with SDMClient.in_process(tabular) as client: + with pytest.raises(StructuredDataError): + client.predict(Request.anomaly(readings()), model=KumoTabular()) + + +@pytest.mark.parametrize('option', ['num_samples', 'num_steps']) +@pytest.mark.parametrize('value', [0, -1, 1.5]) +def test_the_sampling_knobs_take_a_count(anomaly, option, value): + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, readings(), **{option: value}) + assert option in str(caught.value) diff --git a/scripts/check_wire_payloads.py b/scripts/check_wire_payloads.py index aa0084fb7..bf90abeed 100755 --- a/scripts/check_wire_payloads.py +++ b/scripts/check_wire_payloads.py @@ -145,6 +145,23 @@ def recorded_bodies(payloads: Path) -> list[tuple[str, dict[str, Any]]]: return bodies +#: Models this client builds requests for that the contract's enums do not +#: name, and the fields those enums reject them in. The contract describes +#: what a deployment serves, and no deployment serves these: they run in this +#: process only. Everything else about such a body is still policed, so an +#: envelope that is wrong in any other way still fails here. +NOT_IN_THE_CONTRACT = {'kumo-anomaly': ('model', 'task.kind')} + + +def _declared_elsewhere(body: dict[str, Any], error: Any) -> bool: + r"""Whether ``error`` is only that the contract has not heard of this + model yet, rather than something wrong with the request.""" + exempt = NOT_IN_THE_CONTRACT.get(body.get('model')) + if exempt is None or error.validator != 'enum': + return False + return '.'.join(str(part) for part in error.path) in exempt + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -210,6 +227,9 @@ def main() -> int: {**defs[wanted], '$defs': defs}, format_checker=FormatChecker() ) errors = sorted(validator.iter_errors(body), key=lambda e: list(e.path)) + errors = [ + error for error in errors if not _declared_elsewhere(body, error) + ] if not errors: continue failed += 1 diff --git a/scripts/snapshot_wire_payloads.py b/scripts/snapshot_wire_payloads.py index 716112983..a27a07738 100644 --- a/scripts/snapshot_wire_payloads.py +++ b/scripts/snapshot_wire_payloads.py @@ -19,6 +19,7 @@ import pandas as pd from sdm_client import PredictiveQuery, Request, Schema +from sdm_client.adapters.anomaly import AnomalyAdapter from sdm_client.adapters.relational import RelationalAdapter from sdm_client.adapters.tabular import TabularAdapter @@ -124,6 +125,19 @@ def _tabular_cases() -> dict[str, list[dict[str, Any]]]: return recorded +def _anomaly_cases() -> dict[str, list[dict[str, Any]]]: + readings = _people()[['user_id', 'age', 'churn']] + adapter = AnomalyAdapter() + request = Request.anomaly(readings) + recorded: dict[str, list[dict[str, Any]]] = {} + for label, outputs in _output_sets(adapter, request): + recorded[f'anomaly/direct-readings{label}'] = [ + _normalize(payload) + for payload in adapter.prediction_plan(request, outputs) + ] + return recorded + + def _relational_cases() -> dict[str, list[dict[str, Any]]]: schema = Schema.from_data({'people': _people(), 'orders': _orders()}) request = PredictiveQuery( @@ -150,6 +164,7 @@ def _relational_available() -> bool: def collect() -> dict[str, Any]: recorded: dict[str, Any] = dict(_tabular_cases()) + recorded.update(_anomaly_cases()) if _relational_available(): recorded.update(_relational_cases()) return recorded diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index ff04d740b..1f777c1bf 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -116,6 +116,7 @@ "Explanations", "FittedModel", "GradientExplanations", + "KumoAnomaly", "KumoRelational", "KumoTabular", "MissingExtraError", @@ -200,6 +201,17 @@ }, "signature": "(entries: 'Mapping[Any, dict[str, Any]] | None' = None, *, keyed_by: 'tuple[str, ...]' = ('id',), hops: 'Mapping[Any, Mapping[str, Sequence[int | None]]] | None' = None) -> 'None'" }, + "KumoAnomaly": { + "bases": [ + "sdm_client.models.Model" + ], + "kind": "class", + "public_attributes": [ + "name" + ], + "public_methods": {}, + "signature": "()" + }, "KumoRelational": { "bases": [ "sdm_client.models.Model" @@ -308,6 +320,7 @@ "target_moves_with_time" ], "public_methods": { + "anomaly": "(readings: 'pd.DataFrame', *, schema: 'Schema | None' = None, entity_table: 'str' = 'instance', inference: 'Mapping[str, Any] | None' = None) -> 'Request'", "materialized": "(*, payloads: 'tuple[dict[str, Any], ...]', schema: 'Schema', query: 'str', indices: 'list[Any]', entity_table: 'str | None' = None, windowed: 'bool' = False, requested_inference: 'Mapping[str, Any] | None' = None) -> 'Request'" }, "signature": "(context: 'pd.DataFrame', predict: 'pd.DataFrame', *, target: 'str', task: 'str', related_tables: 'Mapping[str, pd.DataFrame] | None' = None, schema: 'Schema | None' = None, entity_table: 'str' = 'instance', inference: 'Mapping[str, Any] | None' = None, task_options: 'Mapping[str, Any] | None' = None) -> 'None'" diff --git a/tests/golden/wire_payloads.json b/tests/golden/wire_payloads.json index c4157f9e6..e1d7d1555 100644 --- a/tests/golden/wire_payloads.json +++ b/tests/golden/wire_payloads.json @@ -1,4 +1,253 @@ { + "anomaly/direct-readings": [ + { + "context": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [] + } + }, + "metadata": { + "request_id": "" + }, + "model": "kumo-anomaly", + "output": { + "fields": [ + "prediction" + ] + }, + "predict": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [ + [ + 0, + 20.0, + 0 + ], + [ + 1, + 21.0, + 1 + ], + [ + 2, + 22.0, + 0 + ], + [ + 3, + 23.0, + 1 + ], + [ + 4, + 24.0, + 0 + ], + [ + 5, + 25.0, + 1 + ], + [ + 6, + 26.0, + 0 + ], + [ + 7, + 27.0, + 1 + ], + [ + 8, + 28.0, + 0 + ], + [ + 9, + 29.0, + 1 + ], + [ + 10, + 30.0, + 0 + ], + [ + 11, + 31.0, + 1 + ] + ] + } + }, + "schema": { + "instance_table": { + "columns": { + "age": { + "dtype": "float64", + "stype": "numerical" + }, + "churn": { + "dtype": "int64", + "stype": "numerical" + }, + "user_id": { + "dtype": "int64", + "stype": "ID" + } + }, + "primary_key": [ + "user_id" + ] + } + }, + "task": { + "kind": "anomaly_detection", + "target": { + "column_name": "*", + "dtype": "float64" + } + } + } + ], + "anomaly/direct-readings#default-outputs": [ + { + "context": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [] + } + }, + "metadata": { + "request_id": "" + }, + "model": "kumo-anomaly", + "output": { + "fields": [ + "prediction", + "probabilities" + ] + }, + "predict": { + "instance_table": { + "columns": [ + "user_id", + "age", + "churn" + ], + "format": "arrays", + "rows": [ + [ + 0, + 20.0, + 0 + ], + [ + 1, + 21.0, + 1 + ], + [ + 2, + 22.0, + 0 + ], + [ + 3, + 23.0, + 1 + ], + [ + 4, + 24.0, + 0 + ], + [ + 5, + 25.0, + 1 + ], + [ + 6, + 26.0, + 0 + ], + [ + 7, + 27.0, + 1 + ], + [ + 8, + 28.0, + 0 + ], + [ + 9, + 29.0, + 1 + ], + [ + 10, + 30.0, + 0 + ], + [ + 11, + 31.0, + 1 + ] + ] + } + }, + "schema": { + "instance_table": { + "columns": { + "age": { + "dtype": "float64", + "stype": "numerical" + }, + "churn": { + "dtype": "int64", + "stype": "numerical" + }, + "user_id": { + "dtype": "int64", + "stype": "ID" + } + }, + "primary_key": [ + "user_id" + ] + } + }, + "task": { + "kind": "anomaly_detection", + "target": { + "column_name": "*", + "dtype": "float64" + } + } + } + ], "relational/pql-count-aggregation": [ { "context": { diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 2c901ce48..640e303c6 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -846,6 +846,46 @@ def test_each_request_is_read_as_the_one_it_is( written.write_text(json.dumps({'case': body})) assert _check_payloads(written).returncode == 0 + @needs_the_contract + @pytest.mark.parametrize( + ('mutate', 'said'), + [ + (lambda body: body.update(invented_key=1), 'invented_key'), + ( + lambda body: body['output'].update(fields=['predcition']), + 'predcition', + ), + (lambda body: body.pop('schema'), "'schema'"), + ], + ids=[ + 'an invented top-level key', + 'a field off the enum', + 'a dropped section', + ], + ) + def test_a_model_the_contract_has_not_heard_of_is_policed_elsewhere( + self, tmp_path, mutate, said + ): + """`kumo-anomaly` is off the contract's model enum on purpose, and + that is the only thing forgiven. A body wrong in any other way still + fails, or the exemption would be a hole rather than a note.""" + recorded = json.loads((GOLDEN / 'wire_payloads.json').read_text()) + body = json.loads(json.dumps(recorded['anomaly/direct-readings'][0])) + assert body['model'] == 'kumo-anomaly' + mutate(body) + + written = tmp_path / 'wire_payloads.json' + written.write_text(json.dumps({'case': body})) + result = _check_payloads(written) + + assert result.returncode == 1 + assert said in result.stderr + + @needs_the_contract + def test_a_body_the_contract_has_not_heard_of_is_otherwise_accepted(self): + """The premise of the test above: untouched, it passes.""" + assert _check_payloads().returncode == 0 + @needs_the_contract def test_a_body_that_is_no_request_at_all_is_refused(self, tmp_path): written = tmp_path / 'wire_payloads.json' From 23a3e9f3c75e82b5e35c505409e752f19f296334 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:13:10 -0800 Subject: [PATCH 2/9] Hold the anomaly path to the rules every single-table request obeys Four defects, each from asking what this adapter does that the tabular one already did. The caller's key never came back. `sdm_client.result` tells callers to join on it, so a key that joins for one model and not another is a difference nobody has reason to expect. A column the caller never typed was sent as the placeholder `categorical` a schema assigns an all-null column, and this model then refused it as a meaning it cannot reconstruct. A float can be categorical, so the existing reconciliation keeps it; for a model that reads one meaning, an undeclared one follows the dtype instead. A meaning the caller did assign is still a claim and is still refused on its own terms. The rows were read into CPU tensors and handed to a model that may be on a GPU. The fitted path moves its context; this one has no context, so the rows are all there is to move. The contract exemption forgave an enum error by field, which accepted `anomaly_detectoin` and every other misspelling. It now forgives the value as well, which is the one thing that file exists to catch. The two rules that were missing here were written for `kumo-tabular` and are about neither model, so they move to `adapters/single_table.py` rather than being reached for across adapters or written a second time. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/adapters/anomaly.py | 27 ++- .../src/sdm_client/adapters/single_table.py | 224 ++++++++++++++++++ .../src/sdm_client/adapters/tabular.py | 207 +--------------- .../src/sdm_client/in_process/anomaly.py | 8 +- ...st_an_anomaly_is_scored_without_a_label.py | 29 +++ .../tests/test_options_and_explanations.py | 8 +- .../tests/test_warehouse_and_scored_dtypes.py | 6 +- scripts/check_wire_payloads.py | 10 +- tests/golden/wire_payloads.json | 4 +- tests/test_release_contract.py | 10 + 10 files changed, 315 insertions(+), 218 deletions(-) create mode 100644 packages/structured-data-models-client/src/sdm_client/adapters/single_table.py diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py index 086d4e2f3..c2691c469 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py @@ -25,6 +25,11 @@ ResponseContractError, ) +from sdm_client.adapters.single_table import ( + MEANING_OF, + coherent_stype, + named_entities, +) from sdm_client.base import ModelSpec from sdm_client.errors import StructuredDataError, UnsupportedRequestError from sdm_client.models import KUMO_ANOMALY @@ -114,7 +119,7 @@ def answered( return PredictionResult( model=model, request_id=request_id, - items=items, + items=named_entities(items, payload), metadata=dict(metadata or {}), outputs=outputs, ) @@ -163,7 +168,10 @@ def _envelope( instance_schema: dict[str, Any] = { 'columns': { - name: {'dtype': dtypes[name], 'stype': table[name].stype} + name: { + 'dtype': dtypes[name], + 'stype': _meaning(table[name], dtypes[name]), + } for name in readings.columns } } @@ -195,3 +203,18 @@ def _envelope( json_ready(request.inference) ) return envelope + + +def _meaning(column: Any, dtype: str) -> str: + r"""What to declare a column as, for a model that reads one meaning. + + `coherent_stype` holds a placeholder meaning only where the dtype cannot + carry it, and a float can be categorical, so a column the caller never + typed stays categorical and this model then refuses it. The readings are + numbers and the model reconstructs numbers, so an undeclared meaning + follows the dtype here. A meaning the caller did assign is a claim, and is + left alone to be refused on its own terms. + """ + if column.stype_declared: + return str(column.stype) + return MEANING_OF.get(dtype) or coherent_stype(column.stype, dtype, column) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py new file mode 100644 index 000000000..9125584b3 --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Rules every single-table request obeys, whichever model answers it. + +A request with one table and no relationships is read the same way for every +such model: the caller's key goes back on each answer and is checked against +the batch it claims to belong to, and a column's declared meaning is held to +something its dtype can actually carry. Both were written for `kumo-tabular` +and neither is about it, so they live here rather than being reached for +across adapters or written a second time. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + +from sdm_client.errors import StructuredDataError +from sdm_client.schema import Column, supports_dtype +from sdm_client.wire import as_declared + +# What the contract allows a `PredictionItem.id` to be. +MAX_ID_LENGTH = 1024 + + +def key_named_by(value: str) -> str | None: + r"""The key an id names, or ``None`` if it names no key this can read. + + The endpoint answers under the list of primary-key values, and how it + renders that list is its own business: a key that travelled as a JSON + string comes back quoted, one that travelled as a number does not, and one + carrying a newline or a backslash comes back with that escaped. So the + rendering is read back into the value it stands for, and the value is what + gets compared. Taking the text apart by hand instead would refuse a correct + answer for any key an escape appears in. + + Read as JSON first, because JSON is what the endpoint answers with, and + only as a Python literal if that fails. The two agree on everything except + a character above U+FFFF: JSON writes it as a surrogate pair and + `ast.literal_eval` reads the halves as two separate characters that never + recombine, so a key containing one could never be matched back and the + request failed at the client's own id check. + + Anything not written that way is left alone rather than refused. The + contract calls this field a *service-generated* row identifier and names + ``row_index`` as what to align on, so an endpoint is entitled to answer + under a token of its own and refusing one would break a compliant + deployment. It bounds an id at 1024 characters, so anything longer is not + one either, and is declined before being parsed. + + A value that is not JSON is read as the Python rendering it must then be. + """ + import ast + + if len(value) > MAX_ID_LENGTH: + return None + named: Any = None + try: + named = json.loads(value) + except (ValueError, TypeError, RecursionError): + try: + named = ast.literal_eval(value) + except ( + ValueError, + SyntaxError, + TypeError, + MemoryError, + RecursionError, + ): + return None + if not isinstance(named, list) or len(named) != 1: + return None + return str(named[0]) + + +def named_entities( + items: Sequence[Any], payload: dict[str, Any] | None +) -> list[Any]: + r"""Put the caller's key back on each row, and check it belongs there. + + The endpoint answers under the stringified list of primary-key values, so + a row the caller asked about as ``1`` comes back as ``'[1]'``. The + relational path already returns the caller's own key, and + :mod:`sdm_client.result` tells callers to join on it, so a key that only + joins on one of the two models is a difference they have no reason to + expect. + + The key is read from the payload that produced this response, so a run + split into several calls maps each batch against its own rows. + + **A mismatch is an error, not a silent pass.** Rewriting an id is exactly + the operation that can misattribute an answer, so a row is checked against + the batch it claims to belong to before its id is replaced. The endpoint + answers under the list of key values and the relational endpoint answers + under the key itself, so a key is recognised in either shape. An id + recognisable as a key has to be the key of the row it was paired with. An + id in any other shape is not second-guessed: it may be a correlation token + of the endpoint's own. + + Those checks are about the batch rather than the key, so they also hold for + a request that has no key to put back: a composite key, or rows the caller + supplied without the key column, still have to be answered once each and + no more. Such a request has no value to restore. A null key is the same + case. ``str`` would turn it into ``'None'``, which + reads as a key that was asked about. Rows in a shape this cannot read at + all, a format other than ``arrays``, are left alone entirely. + """ + import dataclasses + + if payload is None: + return list(items) + table = payload.get('predict', {}).get('instance_table', {}) + rows = table.get('rows') + if not isinstance(rows, list): + return list(items) + columns = table.get('columns') or [] + + key = ( + payload.get('schema', {}).get('instance_table', {}).get('primary_key') + or [] + ) + named_key = key[0] if len(key) == 1 else None + key_dtype = ( + payload.get('schema', {}) + .get('instance_table', {}) + .get('columns', {}) + .get(named_key, {}) + .get('dtype') + if named_key + else None + ) + keys = ( + [as_declared(row[columns.index(named_key)], key_dtype) for row in rows] + if named_key and named_key in columns + else None + ) + keys_in_batch = {str(value) for value in keys or ()} + + named: list[Any] = [] + seen: set[int] = set() + for item in items: + index = item.row_index + if index is None: + raise StructuredDataError( + 'the response is missing a row_index, so its answer cannot be ' + 'attributed to a row', + code='INVALID_RESPONSE', + ) + if not 0 <= index < len(rows): + raise StructuredDataError( + f'the response names row_index {index}, and this batch asked ' + f'about {len(rows)} rows', + code='INVALID_RESPONSE', + ) + if index in seen: + raise StructuredDataError( + f'the response answers row {index} more than once', + code='INVALID_RESPONSE', + ) + if keys is not None and item.id is not None: + answered = str(item.id) + named_key = key_named_by(answered) + if named_key is None and answered in keys_in_batch: + named_key = answered + if named_key is not None and named_key != str(keys[index]): + raise StructuredDataError( + f'the response pairs row_index {index} with id ' + f'{item.id!r}, and this batch asked about ' + f'{keys[index]!r} there', + code='INVALID_RESPONSE', + ) + seen.add(index) + value = None if keys is None else keys[index] + named.append( + item if value is None else dataclasses.replace(item, id=value) + ) + + if seen != set(range(len(rows))): + missing = sorted(set(range(len(rows))) - seen) + raise StructuredDataError( + f'the response left row(s) {missing} unanswered', + code='INVALID_RESPONSE', + ) + return named + + +#: What a dtype means when nothing else says. Only reached for a column whose +#: meaning was never claimed, so it is the dtype's own reading rather than a + + +#: guess about the data. +MEANING_OF = { + 'int64': 'numerical', + 'int32': 'numerical', + 'float64': 'numerical', + 'float32': 'numerical', + 'bool': 'categorical', +} + + +def coherent_stype(stype: str, dtype: str, column: Column) -> str: + r"""A meaning the dtype can actually carry. + + An all-null context column is typed `categorical` because a schema has to + say something. The scored rows then decide its dtype, and if they hold + timestamps the column goes out as `timestamp[us]` meaning `categorical`, + which is not a thing a column can be, and is the same self-contradiction on + the wire that a scored frame re-typing a real column used to produce. + + Only the placeholder is overridden. A meaning the caller assigned is a + claim and is refused rather than quietly rewritten, before it reaches here. + """ + if supports_dtype(stype, dtype) or column.stype_declared: + return stype + # Every spelling of a timestamp, not one of them. The contract writes + # `timestamp[us]`, but a hand-built schema may say `[ns]`, `[ms]` or `[s]`, + # and reading those as categorical puts a dtype and a meaning on the wire + # that contradict each other. That is what this exists to prevent. + if dtype.startswith('timestamp'): + return 'timestamp' + return MEANING_OF.get(dtype, 'categorical') diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index 6b637ee7c..ac4c99692 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -34,6 +34,7 @@ ResponseContractError, ) +from sdm_client.adapters.single_table import coherent_stype, named_entities from sdm_client.base import ModelSpec from sdm_client.core.warnings import SDMWarning, caller_stacklevel from sdm_client.errors import StructuredDataError, UnsupportedRequestError @@ -41,10 +42,8 @@ from sdm_client.models import KUMO_RELATIONAL, KUMO_TABULAR from sdm_client.request import Request from sdm_client.result import PredictionResult -from sdm_client.schema import Column, supports_dtype from sdm_client.wire import ( UNTYPED, - as_declared, encode_table, infer_tfm_dtype, json_ready, @@ -67,9 +66,6 @@ _SESSION_PINNED = ('model', 'task', 'schema', 'context') _SESSION_SCORING = ('predict', 'output', 'inference', 'metadata') -# What the contract allows a `PredictionItem.id` to be. -_MAX_ID_LENGTH = 1024 - def _request_id() -> str: return f'sdm-{uuid.uuid4().hex[:12]}' @@ -164,7 +160,7 @@ def answered( request_id: str = '', metadata: Mapping[str, Any] | None = None, ) -> PredictionResult: - named = as_declared_target(_named_entities(items, payload), payload) + named = as_declared_target(named_entities(items, payload), payload) return PredictionResult( model=model, request_id=request_id, @@ -262,7 +258,7 @@ def _envelope( as_text.append(name) schema_columns[name] = { 'dtype': dtypes[name], - 'stype': _coherent_stype( + 'stype': coherent_stype( self._stype_for(request, declared, name), dtypes[name], table[name], @@ -416,200 +412,3 @@ def _declared_by(pinned: Mapping[str, Any] | None) -> dict[str, str] | None: for name, described in columns.items() if isinstance(described, Mapping) and described.get('dtype') } - - -def _key_named_by(value: str) -> str | None: - r"""The key an id names, or ``None`` if it names no key this can read. - - The endpoint answers under the list of primary-key values, and how it - renders that list is its own business: a key that travelled as a JSON - string comes back quoted, one that travelled as a number does not, and one - carrying a newline or a backslash comes back with that escaped. So the - rendering is read back into the value it stands for, and the value is what - gets compared. Taking the text apart by hand instead would refuse a correct - answer for any key an escape appears in. - - Read as JSON first, because JSON is what the endpoint answers with, and - only as a Python literal if that fails. The two agree on everything except - a character above U+FFFF: JSON writes it as a surrogate pair and - `ast.literal_eval` reads the halves as two separate characters that never - recombine, so a key containing one could never be matched back and the - request failed at the client's own id check. - - Anything not written that way is left alone rather than refused. The - contract calls this field a *service-generated* row identifier and names - ``row_index`` as what to align on, so an endpoint is entitled to answer - under a token of its own and refusing one would break a compliant - deployment. It bounds an id at 1024 characters, so anything longer is not - one either, and is declined before being parsed. - - A value that is not JSON is read as the Python rendering it must then be. - """ - import ast - import json - - if len(value) > _MAX_ID_LENGTH: - return None - named: Any = None - try: - named = json.loads(value) - except (ValueError, TypeError, RecursionError): - try: - named = ast.literal_eval(value) - except ( - ValueError, - SyntaxError, - TypeError, - MemoryError, - RecursionError, - ): - return None - if not isinstance(named, list) or len(named) != 1: - return None - return str(named[0]) - - -def _named_entities( - items: Sequence[Any], payload: dict[str, Any] | None -) -> list[Any]: - r"""Put the caller's key back on each row, and check it belongs there. - - The endpoint answers under the stringified list of primary-key values, so - a row the caller asked about as ``1`` comes back as ``'[1]'``. The - relational path already returns the caller's own key, and - :mod:`sdm_client.result` tells callers to join on it, so a key that only - joins on one of the two models is a difference they have no reason to - expect. - - The key is read from the payload that produced this response, so a run - split into several calls maps each batch against its own rows. - - **A mismatch is an error, not a silent pass.** Rewriting an id is exactly - the operation that can misattribute an answer, so a row is checked against - the batch it claims to belong to before its id is replaced. The endpoint - answers under the list of key values and the relational endpoint answers - under the key itself, so a key is recognised in either shape. An id - recognisable as a key has to be the key of the row it was paired with. An - id in any other shape is not second-guessed: it may be a correlation token - of the endpoint's own. - - Those checks are about the batch rather than the key, so they also hold for - a request that has no key to put back: a composite key, or rows the caller - supplied without the key column, still have to be answered once each and - no more. Such a request has no value to restore. A null key is the same - case. ``str`` would turn it into ``'None'``, which - reads as a key that was asked about. Rows in a shape this cannot read at - all, a format other than ``arrays``, are left alone entirely. - """ - import dataclasses - - if payload is None: - return list(items) - table = payload.get('predict', {}).get('instance_table', {}) - rows = table.get('rows') - if not isinstance(rows, list): - return list(items) - columns = table.get('columns') or [] - - key = ( - payload.get('schema', {}).get('instance_table', {}).get('primary_key') - or [] - ) - named_key = key[0] if len(key) == 1 else None - key_dtype = ( - payload.get('schema', {}) - .get('instance_table', {}) - .get('columns', {}) - .get(named_key, {}) - .get('dtype') - if named_key - else None - ) - keys = ( - [as_declared(row[columns.index(named_key)], key_dtype) for row in rows] - if named_key and named_key in columns - else None - ) - keys_in_batch = {str(value) for value in keys or ()} - - named: list[Any] = [] - seen: set[int] = set() - for item in items: - index = item.row_index - if index is None: - raise StructuredDataError( - 'the response is missing a row_index, so its answer cannot be ' - 'attributed to a row', - code='INVALID_RESPONSE', - ) - if not 0 <= index < len(rows): - raise StructuredDataError( - f'the response names row_index {index}, and this batch asked ' - f'about {len(rows)} rows', - code='INVALID_RESPONSE', - ) - if index in seen: - raise StructuredDataError( - f'the response answers row {index} more than once', - code='INVALID_RESPONSE', - ) - if keys is not None and item.id is not None: - answered = str(item.id) - named_key = _key_named_by(answered) - if named_key is None and answered in keys_in_batch: - named_key = answered - if named_key is not None and named_key != str(keys[index]): - raise StructuredDataError( - f'the response pairs row_index {index} with id ' - f'{item.id!r}, and this batch asked about ' - f'{keys[index]!r} there', - code='INVALID_RESPONSE', - ) - seen.add(index) - value = None if keys is None else keys[index] - named.append( - item if value is None else dataclasses.replace(item, id=value) - ) - - if seen != set(range(len(rows))): - missing = sorted(set(range(len(rows))) - seen) - raise StructuredDataError( - f'the response left row(s) {missing} unanswered', - code='INVALID_RESPONSE', - ) - return named - - -#: What a dtype means when nothing else says. Only reached for a column whose -#: meaning was never claimed, so it is the dtype's own reading rather than a -#: guess about the data. -_MEANING_OF = { - 'int64': 'numerical', - 'int32': 'numerical', - 'float64': 'numerical', - 'float32': 'numerical', - 'bool': 'categorical', -} - - -def _coherent_stype(stype: str, dtype: str, column: Column) -> str: - r"""A meaning the dtype can actually carry. - - An all-null context column is typed `categorical` because a schema has to - say something. The scored rows then decide its dtype, and if they hold - timestamps the column goes out as `timestamp[us]` meaning `categorical`, - which is not a thing a column can be, and is the same self-contradiction on - the wire that a scored frame re-typing a real column used to produce. - - Only the placeholder is overridden. A meaning the caller assigned is a - claim and is refused rather than quietly rewritten, before it reaches here. - """ - if supports_dtype(stype, dtype) or column.stype_declared: - return stype - # Every spelling of a timestamp, not one of them. The contract writes - # `timestamp[us]`, but a hand-built schema may say `[ns]`, `[ms]` or `[s]`, - # and reading those as categorical puts a dtype and a meaning on the wire - # that contradict each other. That is what this exists to prevent. - if dtype.startswith('timestamp'): - return 'timestamp' - return _MEANING_OF.get(dtype, 'categorical') diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index b933b1b0f..75317a7f0 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -165,8 +165,14 @@ def answer( back with different numbers every call. """ options = payload.get('inference') or {} + # The rows are read into CPU tensors, and the model may be anywhere. + # The fitted path moves its context the same way; this one has no + # context, so the rows are all there is to move. + rows = self.rows(payload) + if generator is not None: + rows = rows.to(generator.device) out = model( - self.rows(payload), + rows, num_samples=int(options.get('num_samples') or _SAMPLES), num_steps=int(options.get('num_steps') or _STEPS), generator=generator, diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index c0f04a91d..399ff9d12 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -61,6 +61,7 @@ def test_a_score_and_a_label_per_row_come_back(anomaly): assert list(answer.columns) == [ 'row_index', + 'id', 'prediction', 'probability_scs', 'probability_macs', @@ -127,6 +128,34 @@ def test_a_key_column_is_not_read_as_a_channel(anomaly): ) +def test_each_answer_comes_back_under_the_row_s_own_key(anomaly): + """`sdm_client.result` tells callers to join on the key, so a key that + joins for one model and not another is a difference nobody expects.""" + frame = readings() + frame['reading_id'] += 1000 + answer = scored(anomaly, frame) + + assert answer['id'].tolist() == frame['reading_id'].tolist() + + +def test_readings_are_answered_though_the_schema_was_built_from_nulls( + anomaly, +): + """A column with nothing in it is declared categorical because a schema + has to say something. The readings then decide its dtype, and refusing + them for the placeholder's sake refuses a question that is answerable.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + blank = Schema.from_data({'instance': frame.iloc[:0].astype(object)}) + + with SDMClient.in_process(anomaly) as client: + request = Request.anomaly(frame, schema=blank, inference=FAST) + answer = client.predict(request, model=KumoAnomaly()).to_pandas() + + assert len(answer) == len(frame) + + def test_a_column_the_model_cannot_reconstruct_is_refused(anomaly): """A reading it cannot reconstruct has no error to score.""" frame = readings() diff --git a/packages/structured-data-models-client/tests/test_options_and_explanations.py b/packages/structured-data-models-client/tests/test_options_and_explanations.py index 4b3ad29a1..a460a6bba 100644 --- a/packages/structured-data-models-client/tests/test_options_and_explanations.py +++ b/packages/structured-data-models-client/tests/test_options_and_explanations.py @@ -31,7 +31,7 @@ Schema, ) from sdm_client.adapters.relational import RelationalAdapter -from sdm_client.adapters.tabular import _key_named_by +from sdm_client.adapters.single_table import key_named_by from sdm_client.errors import NimRequestError, StructuredDataError RNG = np.random.default_rng(11) @@ -287,19 +287,19 @@ def test_an_unrelated_error_is_returned_untouched(): def test_a_key_survives_the_json_the_endpoint_sends(key): """JSON writes a character above U+FFFF as a surrogate pair, and reading it as a Python literal takes the halves apart so they never recombine.""" - assert _key_named_by(json.dumps([key])) == key + assert key_named_by(json.dumps([key])) == key def test_a_key_survives_the_python_rendering_too(): """The other wording an endpoint may answer with.""" - assert _key_named_by(repr(['\U0001f600key'])) == '\U0001f600key' + assert key_named_by(repr(['\U0001f600key'])) == '\U0001f600key' @pytest.mark.parametrize( 'rendered', ['hello', '[1, 2]', '[]', '{"a": 1}', 'x' * 2000] ) def test_something_that_is_not_one_key_is_left_alone(rendered): - assert _key_named_by(rendered) is None + assert key_named_by(rendered) is None # --- a label with more classes than the threshold -------------------------- diff --git a/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py b/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py index 5c69d37ef..3c303b2ad 100644 --- a/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py +++ b/packages/structured-data-models-client/tests/test_warehouse_and_scored_dtypes.py @@ -524,19 +524,19 @@ def test_the_fallback_leaves_a_declared_meaning_alone(): declared meaning that contradicts its values is refused before a request gets this far. The guard is what keeps the rewrite safe to have at all: it may correct a placeholder and must never overrule a caller.""" - from sdm_client.adapters.tabular import _coherent_stype + from sdm_client.adapters.single_table import coherent_stype from sdm_client.schema import Column column = Column(name='c', dtype='string', stype='categorical') object.__setattr__(column, 'untyped', True) - assert _coherent_stype('categorical', 'timestamp[us]', column) == ( + assert coherent_stype('categorical', 'timestamp[us]', column) == ( 'timestamp' ) object.__setattr__(column, 'stype_declared', True) - assert _coherent_stype('categorical', 'timestamp[us]', column) == ( + assert coherent_stype('categorical', 'timestamp[us]', column) == ( 'categorical' ) diff --git a/scripts/check_wire_payloads.py b/scripts/check_wire_payloads.py index bf90abeed..308ed221a 100755 --- a/scripts/check_wire_payloads.py +++ b/scripts/check_wire_payloads.py @@ -150,7 +150,9 @@ def recorded_bodies(payloads: Path) -> list[tuple[str, dict[str, Any]]]: #: what a deployment serves, and no deployment serves these: they run in this #: process only. Everything else about such a body is still policed, so an #: envelope that is wrong in any other way still fails here. -NOT_IN_THE_CONTRACT = {'kumo-anomaly': ('model', 'task.kind')} +NOT_IN_THE_CONTRACT = { + 'kumo-anomaly': {'model': 'kumo-anomaly', 'task.kind': 'anomaly_detection'} +} def _declared_elsewhere(body: dict[str, Any], error: Any) -> bool: @@ -159,7 +161,11 @@ def _declared_elsewhere(body: dict[str, Any], error: Any) -> bool: exempt = NOT_IN_THE_CONTRACT.get(body.get('model')) if exempt is None or error.validator != 'enum': return False - return '.'.join(str(part) for part in error.path) in exempt + where = '.'.join(str(part) for part in error.path) + # The value as well as the field. Forgiving the field alone would accept + # `anomaly_detectoin` and every other misspelling of it, which is the one + # thing this file exists to catch. + return where in exempt and error.instance == exempt[where] def main() -> int: diff --git a/tests/golden/wire_payloads.json b/tests/golden/wire_payloads.json index e1d7d1555..77a76e5c2 100644 --- a/tests/golden/wire_payloads.json +++ b/tests/golden/wire_payloads.json @@ -106,7 +106,7 @@ }, "user_id": { "dtype": "int64", - "stype": "ID" + "stype": "numerical" } }, "primary_key": [ @@ -231,7 +231,7 @@ }, "user_id": { "dtype": "int64", - "stype": "ID" + "stype": "numerical" } }, "primary_key": [ diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 640e303c6..8e10c3ef6 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -856,11 +856,21 @@ def test_each_request_is_read_as_the_one_it_is( 'predcition', ), (lambda body: body.pop('schema'), "'schema'"), + ( + lambda body: body['task'].update(kind='anomaly_detectoin'), + 'anomaly_detectoin', + ), + ( + lambda body: body.update(model='kumo-anomoly'), + 'kumo-anomoly', + ), ], ids=[ 'an invented top-level key', 'a field off the enum', 'a dropped section', + 'a misspelt task kind', + 'a misspelt model id', ], ) def test_a_model_the_contract_has_not_heard_of_is_policed_elsewhere( From a6af2079fa3f3b3dacf49b5a6854701b07a50d64 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:20:48 -0800 Subject: [PATCH 3/9] Ask for the score alone and get the score alone `probabilities` was built whichever fields were asked for, so a caller who wanted the score read a frame with two columns they had declined. The labels are now computed only when they were asked for, which is also the work they cost. And the hygiene gate refused the dependency URL added with this branch: it holds every repository link to the one these packages publish to, which is what catches a name this repo used to have. A dependency installed from its own source is a link a reader is meant to follow, so it is named as upstream rather than as this repository under another name. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/in_process/anomaly.py | 18 +++++++++++++----- .../src/sdm_client/in_process/backend.py | 9 +++++++-- ...est_an_anomaly_is_scored_without_a_label.py | 13 +++++++++++++ scripts/check_repo_hygiene.py | 9 ++++++++- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 75317a7f0..6e70a97af 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -156,6 +156,7 @@ def answer( model: Any, payload: Mapping[str, Any], ids: Sequence[Any], + fields: Sequence[str], generator: Any = None, ) -> list[PredictionItem]: r"""Score each row, under the id the request gave it. @@ -178,10 +179,15 @@ def answer( generator=generator, ) scores = out['mae'].numerical[..., 0].tolist() - labels = { - name: out[name].categorical.decode()[..., 0].tolist() - for name in THRESHOLDERS - } + labelled = 'probabilities' in fields + labels = ( + { + name: out[name].categorical.decode()[..., 0].tolist() + for name in THRESHOLDERS + } + if labelled + else {} + ) return [ PredictionItem( row_index=index, @@ -189,7 +195,9 @@ def answer( prediction=score, probabilities={ name: float(labels[name][index]) for name in THRESHOLDERS - }, + } + if labelled + else None, ) for index, score in enumerate(scores) ] diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index e73e87ea9..2c5f02149 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -81,6 +81,7 @@ def answer( model: Any, payload: Mapping[str, Any], ids: Sequence[Any], + fields: Sequence[str], generator: Any = None, ) -> list[Any]: ... @@ -184,7 +185,7 @@ def predict( served, model = self._serving(payload['model']) _check_loaded_for(served, model, task['kind']) if isinstance(served, ScoresRowsAlone): - items = _scored(served, model, payload) + items = _scored(served, model, payload, fields) else: context = served.context(payload) query = served.query(context, payload, payload['schema']) @@ -577,7 +578,10 @@ def _autocast(device: Any) -> Any: def _scored( - served: ScoresRowsAlone, model: Any, payload: Mapping[str, Any] + served: ScoresRowsAlone, + model: Any, + payload: Mapping[str, Any], + fields: list[str], ) -> list[Any]: r"""One answer per row, from a model with no context to fit.""" import torch @@ -593,6 +597,7 @@ def _scored( model, payload, ids, + fields, torch.Generator(device=device).manual_seed(_SEED), ) diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index 399ff9d12..05fa3bdb5 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -73,6 +73,19 @@ def test_a_score_and_a_label_per_row_come_back(anomaly): assert set(answer[name].unique()) <= {0.0, 1.0} +def test_only_the_fields_that_were_asked_for_come_back(anomaly): + """Asking for the score alone is a smaller answer, not the same one with + the labels computed and sent anyway.""" + with SDMClient.in_process(anomaly) as client: + request = Request.anomaly(readings(), inference=FAST) + answer = client.predict( + request, model=KumoAnomaly(), outputs=['prediction'] + ).to_pandas() + + assert 'prediction' in answer.columns + assert not [name for name in answer.columns if name.startswith('prob')] + + def test_the_same_readings_score_the_same_twice(anomaly): """Reconstruction is sampled, so the answer is seeded. Without that an unchanged request comes back with different numbers every call.""" diff --git a/scripts/check_repo_hygiene.py b/scripts/check_repo_hygiene.py index c97527237..d86163e13 100644 --- a/scripts/check_repo_hygiene.py +++ b/scripts/check_repo_hygiene.py @@ -196,6 +196,13 @@ REPOSITORY_URL = re.compile(r'https://github\.com/NVIDIA/[A-Za-z0-9._-]+') +#: Repositories this one legitimately points at, as opposed to the names it +#: used to be published under. A dependency installed from its own source is +#: a URL a reader is meant to follow, so it is not a stale name for this one. +UPSTREAM_URLS = frozenset( + {'https://github.com/NVIDIA/structured-data-models.git'} +) + def tracked_files() -> list[Path]: import subprocess @@ -340,7 +347,7 @@ def check_repository_url() -> int: continue text = path.read_text(encoding='utf-8', errors='replace') for found in set(REPOSITORY_URL.findall(text)): - if found != expected: + if found != expected and found not in UPSTREAM_URLS: findings.append(f'{relative}: {found}') for finding in sorted(findings): From 38e3e337eca39f39111902dd8fc6642e14e56532 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:32:49 -0800 Subject: [PATCH 4/9] Leave a meaning alone where the column had values to read The placeholder rule reached further than the case it was for. It read a column's meaning off its dtype whenever `stype_declared` was false, which a `Column` built through its constructor also is, so a caller who said `stype='categorical'` had it quietly read as numbers. `untyped` is the case that rule is about: nothing read, nothing said. A column that holds values carries a claim either way and is refused on its own terms. The recorded envelope shows it: an inferred key goes out as `ID` again. And a composite key the readings leave out is no longer a crash. The key names the row and is not a channel, so a caller may hand over the channels alone; there is then no value to answer under and rows match by position, which is what a single-column key already did. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/adapters/anomaly.py | 17 ++++---- .../src/sdm_client/in_process/anomaly.py | 5 +++ ...st_an_anomaly_is_scored_without_a_label.py | 41 +++++++++++++++++++ tests/golden/wire_payloads.json | 4 +- 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py index c2691c469..f5873f476 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py @@ -209,12 +209,15 @@ def _meaning(column: Any, dtype: str) -> str: r"""What to declare a column as, for a model that reads one meaning. `coherent_stype` holds a placeholder meaning only where the dtype cannot - carry it, and a float can be categorical, so a column the caller never - typed stays categorical and this model then refuses it. The readings are - numbers and the model reconstructs numbers, so an undeclared meaning - follows the dtype here. A meaning the caller did assign is a claim, and is - left alone to be refused on its own terms. + carry it, and a float can be categorical, so a column with nothing in it + to read stays categorical and this model then refuses it. The readings are + numbers and the model reconstructs numbers, so there the meaning follows + the dtype instead. + + Only that column. `untyped` is the one case where nothing was read and + nothing was said, so a meaning carried by a column that holds values is a + claim either way, and is left alone to be refused on its own terms. """ - if column.stype_declared: - return str(column.stype) + if not column.untyped: + return coherent_stype(column.stype, dtype, column) return MEANING_OF.get(dtype) or coherent_stype(column.stype, dtype, column) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 6e70a97af..3ea4ff82c 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -145,6 +145,11 @@ def row_ids( key = spec.get('primary_key') if not isinstance(key, list) or len(key) < 2: return [None] * len(table['rows']) + if not set(key) <= set(table['columns']): + # The readings need not carry the key: it names the row and is + # not a channel, so a caller may leave it out. Then there is no + # value to answer under and each answer is matched by position. + return [None] * len(table['rows']) cells = [values(table, name) for name in key] return [ json.dumps(list(values_of), separators=(',', ':'), default=str) diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index 05fa3bdb5..5c3d3781e 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -169,6 +169,47 @@ def test_readings_are_answered_though_the_schema_was_built_from_nulls( assert len(answer) == len(frame) +def test_a_meaning_the_caller_gave_a_column_that_holds_values_is_kept( + anomaly, +): + """The placeholder rule is for a column with nothing in it to read. A + column that holds values and says it is categorical says so on purpose, + and this model refuses that rather than reading it as numbers anyway.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame}) + schema['instance']['flow'].stype = 'categorical' + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(frame, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + +def test_readings_that_leave_out_a_composite_key_are_answered(anomaly): + """A key names the row and is not a channel, so a caller may leave it + out. Then there is no value to answer under and rows match by position.""" + from sdm_client import Schema + + frame = readings() + frame['site'] = 'north' + schema = Schema.from_data({'instance': frame}) + schema['instance'].primary_key = ['reading_id', 'site'] + + channels = frame[['flow', 'pressure']] + with SDMClient.in_process(anomaly) as client: + answer = client.predict( + Request.anomaly(channels, schema=schema, inference=FAST), + model=KumoAnomaly(), + ).to_pandas() + + assert len(answer) == len(frame) + assert answer['row_index'].tolist() == list(range(len(frame))) + + def test_a_column_the_model_cannot_reconstruct_is_refused(anomaly): """A reading it cannot reconstruct has no error to score.""" frame = readings() diff --git a/tests/golden/wire_payloads.json b/tests/golden/wire_payloads.json index 77a76e5c2..e1d7d1555 100644 --- a/tests/golden/wire_payloads.json +++ b/tests/golden/wire_payloads.json @@ -106,7 +106,7 @@ }, "user_id": { "dtype": "int64", - "stype": "numerical" + "stype": "ID" } }, "primary_key": [ @@ -231,7 +231,7 @@ }, "user_id": { "dtype": "int64", - "stype": "numerical" + "stype": "ID" } }, "primary_key": [ From 4493d1906b431f964a536d49f18b833b3f3e663b Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:43:31 -0800 Subject: [PATCH 5/9] Refuse a score that is not a number, and a meaning nobody asked to change `untyped` alone was still too wide: a caller can type a column that held nothing to read, and that is a claim like any other. The placeholder rule now applies only where nothing was read and nothing was said. And a score of NaN travelled back as an answer. The other models refuse a malformed output already; this path did not, so readings large enough to overflow the reconstruction came back as a frame of NaN that cannot be thresholded or plotted, with both labels reading false. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/adapters/anomaly.py | 9 ++++--- .../src/sdm_client/in_process/anomaly.py | 11 +++++++- ...st_an_anomaly_is_scored_without_a_label.py | 27 +++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py index f5873f476..7725e3a23 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/anomaly.py @@ -214,10 +214,11 @@ def _meaning(column: Any, dtype: str) -> str: numbers and the model reconstructs numbers, so there the meaning follows the dtype instead. - Only that column. `untyped` is the one case where nothing was read and - nothing was said, so a meaning carried by a column that holds values is a - claim either way, and is left alone to be refused on its own terms. + Only that column, and only where nothing was said about it either. A + column that holds values carries a claim, and so does one a caller typed + by hand though it held none; both are left alone to be refused on their + own terms. """ - if not column.untyped: + if not column.untyped or column.stype_declared: return coherent_stype(column.stype, dtype, column) return MEANING_OF.get(dtype) or coherent_stype(column.stype, dtype, column) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 3ea4ff82c..647a4fe90 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -183,7 +183,16 @@ def answer( num_steps=int(options.get('num_steps') or _STEPS), generator=generator, ) - scores = out['mae'].numerical[..., 0].tolist() + scored = out['mae'].numerical[..., 0] + if not bool(scored.isfinite().all()): + # The same refusal the other models make. A score that is not a + # number is not a row that is unremarkable, and returning one + # hands a caller a frame they cannot threshold or plot. + raise StructuredDataError( + 'the model answered with a malformed output', + code='INTERNAL_ERROR', + ) + scores = scored.tolist() labelled = 'probabilities' in fields labels = ( { diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index 5c3d3781e..914a41615 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -210,6 +210,33 @@ def test_readings_that_leave_out_a_composite_key_are_answered(anomaly): assert answer['row_index'].tolist() == list(range(len(frame))) +def test_a_meaning_given_to_a_column_with_nothing_in_it_is_kept(anomaly): + """The placeholder is a meaning nobody chose. One a caller typed by hand + is a claim, whether or not the column it is on held anything to read.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame.iloc[:0].astype(object)}) + schema['instance']['flow'].stype = 'categorical' + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(frame, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + +def test_a_score_that_is_not_a_number_is_refused(anomaly): + """A row scored NaN is not a row that is unremarkable, and a frame of + them cannot be thresholded or plotted.""" + frame = pd.DataFrame({'flow': [1e100, 2e100] * 50}) + + with pytest.raises(StructuredDataError) as caught: + scored(anomaly, frame) + assert 'malformed' in str(caught.value) + + def test_a_column_the_model_cannot_reconstruct_is_refused(anomaly): """A reading it cannot reconstruct has no error to score.""" frame = readings() From f9b5edf8daa93f0dd1dfaa8140bb4f1ea1fb3b1c Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 15:55:39 -0800 Subject: [PATCH 6/9] Refuse an unsupervised question built as a supervised one `Request(context=..., predict=..., target='flow', task='anomaly_detection')` was accepted, and then the adapter read neither the context nor the target: the caller got a whole-row reconstruction score under a question they had asked about one column. The constructor that takes those two arguments now says that this task has neither, and names the one that builds it. And `Request.anomaly` did not hold the readings to a schema the caller brought, so a boolean column declared as a float was widened into one and scored as zeros and ones. The supervised path already asks this of the rows it scores; this one asks it too. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/request.py | 15 ++++++++ ...st_an_anomaly_is_scored_without_a_label.py | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index 032093959..8aa8ddff5 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -390,6 +390,15 @@ def __init__( f'task must be one of {list(TASK_KINDS)}, got {task!r}', code='INVALID_REQUEST', ) + if task == 'anomaly_detection': + raise StructuredDataError( + f'anomaly_detection names no target and learns from no ' + f'labelled rows, and this builds a request that has both. ' + f'Asking it here would hand over a context and a target ' + f'{target!r} that nothing reads, and get back a score for the ' + f'whole row. Build it with Request.anomaly(readings) instead', + code='INVALID_REQUEST', + ) if target in context.columns and _has_container_cells(context[target]): # Said here because this is the only place that knows which column # is the target. SDM's own `Task.classification` covers "binary, @@ -590,6 +599,12 @@ def anomaly( readings = _require_frame(readings, 'readings') if schema is None: schema = Schema.from_data({entity_table: readings}) + # The same question the supervised path asks of the rows it scores. A + # schema the caller brought says what a column holds, and readings + # that hold something else are a different series under that name. + _check_scored_values_mean_what_the_schema_says( + readings, schema, entity_table + ) request = cls.__new__(cls) request.schema = schema diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index 914a41615..3fc0abc1c 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -257,6 +257,40 @@ def test_there_is_no_session_to_pin(anomaly): assert 'session' in str(caught.value) +def test_the_supervised_constructor_will_not_build_this_question(anomaly): + """It takes a context and a target, and nothing reads either here. Built + that way the request came back with a whole-row score under a name the + caller had asked about, which is a different question answered quietly.""" + frame = readings() + + with pytest.raises(StructuredDataError) as caught: + Request( + context=frame, + predict=frame.drop(columns=['flow']), + target='flow', + task='anomaly_detection', + ) + assert 'Request.anomaly' in str(caught.value) + + +def test_readings_that_are_not_what_the_schema_says_are_refused(anomaly): + """A schema the caller brought says what a column holds. Readings that + hold something else are a different series under that name, and were + widened into one rather than refused.""" + from sdm_client import Schema + + frame = readings()[['flow', 'pressure']] + schema = Schema.from_data({'instance': frame}) + flipped = frame.assign(flow=[True, False] * (len(frame) // 2)) + + with pytest.raises(StructuredDataError): + with SDMClient.in_process(anomaly) as client: + client.predict( + Request.anomaly(flipped, schema=schema, inference=FAST), + model=KumoAnomaly(), + ) + + def test_another_model_is_not_asked_this_question(tabular): """`anomaly_detection` is a task only one model here answers.""" with SDMClient.in_process(tabular) as client: From daf498838f93d86d030e2701c1592cbf932ec89b Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 16:05:40 -0800 Subject: [PATCH 7/9] Answer no, rather than raise, where this model is not installed `accepts` is asked of every model handed to the client, including ones it does not serve, and it imported `KumoAnomaly` unconditionally. Every published structured-data-models predates that export, so on the version the `in-process` extra actually installs, passing any unrelated object raised `ImportError` out of a call that promises a refusal naming what it takes. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/in_process/anomaly.py | 30 +++++++++++++++---- ...st_an_anomaly_is_scored_without_a_label.py | 13 ++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 647a4fe90..1d55637aa 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -52,15 +52,21 @@ class Anomaly: @staticmethod def load(device: Any) -> Any: - from sdm.models import KumoAnomaly - - return KumoAnomaly(device=device) + return _model_type()(device=device) @staticmethod def accepts(model: Any) -> bool: - from sdm.models import KumoAnomaly + r"""Whether ``model`` is this one. - return isinstance(model, KumoAnomaly) + Asked of every model handed to the client, including ones this does + not serve, so an installed structured-data-models that predates + ``KumoAnomaly`` has to answer no rather than raise. Every published + release predates it today. + """ + try: + return isinstance(model, _model_type()) + except StructuredDataError: + return False def check_inference( self, @@ -215,3 +221,17 @@ def answer( ) for index, score in enumerate(scores) ] + + +def _model_type() -> Any: + r"""``sdm.models.KumoAnomaly``, or why it could not be reached.""" + try: + from sdm.models import KumoAnomaly + except ImportError as error: + raise StructuredDataError( + 'this structured-data-models carries no KumoAnomaly. The model ' + 'is on its main and in no release, so until one ships it is ' + 'reached by installing that package from source', + code='MODEL_NOT_READY', + ) from error + return KumoAnomaly diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index 3fc0abc1c..b75509eee 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -291,6 +291,19 @@ def test_readings_that_are_not_what_the_schema_says_are_refused(anomaly): ) +def test_a_structured_data_models_without_this_model_still_answers( + sdm, monkeypatch +): + """Every published release predates `KumoAnomaly`, and this is asked of + every model handed to the client, including ones it does not serve. It + has to answer no there rather than raise the import.""" + monkeypatch.delattr(sdm.models, 'KumoAnomaly', raising=False) + + with pytest.raises(StructuredDataError) as caught: + SDMClient.in_process(object()) + assert caught.value.code == 'INVALID_CONFIGURATION' + + def test_another_model_is_not_asked_this_question(tabular): """`anomaly_detection` is a task only one model here answers.""" with SDMClient.in_process(tabular) as client: From 5d711ac6cfe8c2cc98ccb84a837feeed8cad2c6e Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Mon, 2 Nov 2026 16:16:40 -0800 Subject: [PATCH 8/9] Offer only the models this structured-data-models carries `models()` says what the backend answers for, and it named `kumo-anomaly` whether or not the installed structured-data-models has it. Every published release does not, so a caller on one was told to expect an answer and then refused when they asked for it. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/in_process/anomaly.py | 9 +++++++++ .../src/sdm_client/in_process/backend.py | 5 +++-- .../src/sdm_client/in_process/relational.py | 9 +++++++++ .../src/sdm_client/in_process/tabular.py | 9 +++++++++ .../tests/test_an_anomaly_is_scored_without_a_label.py | 9 +++++++++ 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 1d55637aa..60e3f4310 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -50,6 +50,15 @@ class Anomaly: model_id = 'kumo-anomaly' outputs = {'anomaly_detection': ('prediction', 'probabilities')} + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model.""" + try: + _model_type() + except StructuredDataError: + return False + return True + @staticmethod def load(device: Any) -> Any: return _model_type()(device=device) diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index 2c5f02149..ee5a67e70 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -166,8 +166,9 @@ def list_models(self) -> list[str]: self._require_open() return [ model_id - for model_id in _SERVED - if self._loads or model_id in self._models + for model_id, served in _SERVED.items() + if (self._loads or model_id in self._models) + and served.installed() ] def predict( diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/relational.py b/packages/structured-data-models-client/src/sdm_client/in_process/relational.py index fa6e4c74c..b0f3d1d15 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/relational.py @@ -61,6 +61,15 @@ class Relational: 'regression': ('prediction', 'quantiles', 'explanation'), } + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model. + + Every release does; `kumo-anomaly` is the one that may not, and + `list_models` asks this of each rather than of that one. + """ + return True + @staticmethod def load(device: Any) -> Any: from sdm.models import KumoRelational diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py b/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py index 8578f4533..45cd6eabe 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/tabular.py @@ -33,6 +33,15 @@ class Tabular: 'regression': ('prediction', 'quantiles', 'explanation'), } + @staticmethod + def installed() -> bool: + r"""Whether the structured-data-models here carries this model. + + Every release does; `kumo-anomaly` is the one that may not, and + `list_models` asks this of each rather than of that one. + """ + return True + @staticmethod def load(device: Any) -> Any: from sdm.models import KumoTabular diff --git a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py index b75509eee..33166774a 100644 --- a/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py +++ b/packages/structured-data-models-client/tests/test_an_anomaly_is_scored_without_a_label.py @@ -304,6 +304,15 @@ def test_a_structured_data_models_without_this_model_still_answers( assert caught.value.code == 'INVALID_CONFIGURATION' +def test_a_model_that_is_not_installed_is_not_offered(sdm, monkeypatch): + """`models()` says what this backend answers for. Naming one it cannot + load sends a caller to a refusal they were told to expect an answer to.""" + monkeypatch.delattr(sdm.models, 'KumoAnomaly', raising=False) + + with SDMClient.in_process() as client: + assert 'kumo-anomaly' not in client.models() + + def test_another_model_is_not_asked_this_question(tabular): """`anomaly_detection` is a task only one model here answers.""" with SDMClient.in_process(tabular) as client: From 020eeaf4129713f6829ec5f294eeed7be5980ab3 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Tue, 3 Nov 2026 13:53:31 -0800 Subject: [PATCH 9/9] Say it once, where the reader is already looking The reasoning sat in comments beside the code it was about, where a reader reaches it only by reading the body. It belongs in the docstring, which is what a reader opens first and what the documentation carries, so each one is folded into the docstring of the thing it explains, or into the documented constant where there is no docstring to fold it into. Nothing is dropped. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../src/sdm_client/adapters/single_table.py | 9 +++---- .../src/sdm_client/in_process/anomaly.py | 24 ++++++++++--------- .../src/sdm_client/in_process/backend.py | 15 ++++++++---- .../src/sdm_client/models.py | 14 ++++++----- .../src/sdm_client/request.py | 16 ++++++------- scripts/check_wire_payloads.py | 10 ++++---- 6 files changed, 49 insertions(+), 39 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py index 9125584b3..122fc3c7d 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/single_table.py @@ -212,13 +212,14 @@ def coherent_stype(stype: str, dtype: str, column: Column) -> str: Only the placeholder is overridden. A meaning the caller assigned is a claim and is refused rather than quietly rewritten, before it reaches here. + + Every spelling of a timestamp counts, not one of them. The contract + writes ``timestamp[us]``, but a hand-built schema may say ``[ns]``, + ``[ms]`` or ``[s]``, and reading those as categorical puts exactly the + contradiction on the wire that this exists to prevent. """ if supports_dtype(stype, dtype) or column.stype_declared: return stype - # Every spelling of a timestamp, not one of them. The contract writes - # `timestamp[us]`, but a hand-built schema may say `[ns]`, `[ms]` or `[s]`, - # and reading those as categorical puts a dtype and a meaning on the wire - # that contradict each other. That is what this exists to prevent. if dtype.startswith('timestamp'): return 'timestamp' return MEANING_OF.get(dtype, 'categorical') diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py index 60e3f4310..e6e364a4c 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/anomaly.py @@ -155,15 +155,15 @@ def row_ids( they were sent, which the client hands back as the row's id. Otherwise there is no id: each answer is matched to its row by - position, and the client puts the row's own key on it. + position, and the client puts the row's own key on it. That is also + what happens where the readings do not carry the key at all, which + is allowed: a key names the row and is not a channel, so there is + then no value to answer under. """ key = spec.get('primary_key') if not isinstance(key, list) or len(key) < 2: return [None] * len(table['rows']) if not set(key) <= set(table['columns']): - # The readings need not carry the key: it names the row and is - # not a channel, so a caller may leave it out. Then there is no - # value to answer under and each answer is matched by position. return [None] * len(table['rows']) cells = [values(table, name) for name in key] return [ @@ -183,12 +183,17 @@ def answer( ``generator`` is what makes the same readings score the same twice. Reconstruction is sampled, so without one an unchanged request comes - back with different numbers every call. + back with different numbers every call. It also says where the model + is: the rows are read into CPU tensors and the model may be + anywhere, and with no context to fit, the rows are all there is to + move to it. + + A score that is not a number is refused rather than returned, as the + other models refuse one. It is not a row that is unremarkable, and + handing it back hands a caller a frame they cannot threshold or + plot. """ options = payload.get('inference') or {} - # The rows are read into CPU tensors, and the model may be anywhere. - # The fitted path moves its context the same way; this one has no - # context, so the rows are all there is to move. rows = self.rows(payload) if generator is not None: rows = rows.to(generator.device) @@ -200,9 +205,6 @@ def answer( ) scored = out['mae'].numerical[..., 0] if not bool(scored.isfinite().all()): - # The same refusal the other models make. A score that is not a - # number is not a row that is unremarkable, and returning one - # hands a caller a frame they cannot threshold or plot. raise StructuredDataError( 'the model answered with a malformed output', code='INTERNAL_ERROR', diff --git a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py index ee5a67e70..7250be495 100644 --- a/packages/structured-data-models-client/src/sdm_client/in_process/backend.py +++ b/packages/structured-data-models-client/src/sdm_client/in_process/backend.py @@ -88,8 +88,12 @@ def answer( @dataclass class _Session: - # Only a model that fits a context reaches here: `create_session` - # refuses one that does not, because there would be nothing to pin. + r"""A fitted context, pinned. + + Only a model that fits one reaches here: :meth:`create_session` refuses + a model that does not, because there would be nothing to pin. + """ + served: Tabular | Relational task: Mapping[str, Any] schema: Mapping[str, Any] @@ -459,11 +463,12 @@ def _check_loaded_for(served: Served, model: Any, kind: str) -> None: A model can be loaded for classification or regression alone, which a NIM's never is, so that is asked of the model rather than of its kind. + Not of a model that fits nothing: what was asked of the others is which + heads a checkpoint was packaged with, and a model that fits nothing has + no head to package. It answers the one task it is for or it is not that + model. """ if isinstance(served, ScoresRowsAlone): - # `tasks` says which heads a checkpoint was packaged with, and a model - # that fits nothing has no head to package: it answers the one task it - # is for or it is not that model. return needed = 'regression' if kind == 'regression' else 'classification' loaded = sorted(str(task) for task in model.tasks) diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 4161063b0..f4e79d2e3 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -165,18 +165,20 @@ # undone by the next regeneration. TFM_MODEL_KUMO_ANOMALY = 'kumo-anomaly' +#: The model that scores a row for how anomalous it is. +#: +#: `probabilities` carries the two thresholders' labels. That field is named +#: for what every other model puts in it, and this one puts something else +#: there because the contract has no third field for a row's answer; see +#: :mod:`sdm_client.in_process.anomaly`. It holds no session, because a +#: session pins a fitted context and this model fits none: every call +#: reconstructs the rows it was given and nothing carries over. KUMO_ANOMALY = ModelSpec( model_id=TFM_MODEL_KUMO_ANOMALY, profile=ANOMALY, runtime=RuntimeContract( tasks=frozenset({'anomaly_detection'}), - # `probabilities` carries the two thresholders' labels. The field is - # named for what every other model puts in it, and this one puts - # something else there because the contract has no third field for a - # row's answer. See `sdm_client.in_process.anomaly`. outputs=frozenset({'prediction', 'probabilities'}), - # A session pins a fitted context, and this model fits none: every - # call reconstructs the rows it was given and nothing carries over. sessions=False, inference_options=frozenset({'num_samples', 'num_steps'}), ), diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index 8aa8ddff5..900becc35 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -582,6 +582,13 @@ def anomaly( learn from. Those two arguments are absent rather than empty, because a caller who passed them would be describing a question nobody asks. + What travels instead is a target standing for the score, typed as + the score is because that is the one thing about it the answer + carries a type for, and a context table of the schema's columns and + no rows. The contract has no shape for a request with nothing to + learn from, and an empty table is what that looks like on the wire, + rather than a section left out of an envelope that requires one. + Args: readings: The rows to score. Every numerical column is read as a channel of one multivariate series; a primary key names the @@ -599,9 +606,6 @@ def anomaly( readings = _require_frame(readings, 'readings') if schema is None: schema = Schema.from_data({entity_table: readings}) - # The same question the supervised path asks of the rows it scores. A - # schema the caller brought says what a column holds, and readings - # that hold something else are a different series under that name. _check_scored_values_mean_what_the_schema_says( readings, schema, entity_table ) @@ -612,16 +616,10 @@ def anomaly( kind='anomaly_detection', target=Target( column_name=WHOLE_ROW, - # The score, which is what this target stands for and the one - # thing about it with a type the answer actually carries. dtype='float64', table_name=entity_table, ), ) - # The contract has no shape for a request with no context, and this - # one genuinely has none. An empty table carries the schema's columns - # and no rows, which is what "nothing to learn from" looks like on the - # wire, rather than a section left out of an envelope that requires it. request._context = Split(instance_table=readings.iloc[:0]) request._predict = Split(instance_table=readings) request.inference = dict(inference or {}) diff --git a/scripts/check_wire_payloads.py b/scripts/check_wire_payloads.py index 308ed221a..95358f350 100755 --- a/scripts/check_wire_payloads.py +++ b/scripts/check_wire_payloads.py @@ -157,14 +157,16 @@ def recorded_bodies(payloads: Path) -> list[tuple[str, dict[str, Any]]]: def _declared_elsewhere(body: dict[str, Any], error: Any) -> bool: r"""Whether ``error`` is only that the contract has not heard of this - model yet, rather than something wrong with the request.""" + model yet, rather than something wrong with the request. + + Matched on the value as well as the field. Forgiving the field alone + would accept ``anomaly_detectoin`` and every other misspelling of it, + which is the one thing this file exists to catch. + """ exempt = NOT_IN_THE_CONTRACT.get(body.get('model')) if exempt is None or error.validator != 'enum': return False where = '.'.join(str(part) for part in error.path) - # The value as well as the field. Forgiving the field alone would accept - # `anomaly_detectoin` and every other misspelling of it, which is the one - # thing this file exists to catch. return where in exempt and error.instance == exempt[where]