diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d50a400..5f3bfe748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,13 +47,15 @@ these packages. All five distributions share one version and one entry. row, output and column, with each row's `hop` from the entity. - `result.explanations.columns` is a column analysis the client computes from the labelled context rows: each column split into cohorts, each cohort's - share and mean outcome, and a score per column, normalized to sum to 1. + share and mean outcome. `result.explanations.columns.to_pandas()` gives one line per table, hop, column, cohort and class. `predict()` computes it on each call that asks for explanations; a session computes it once at `fit()`, after `holdout` is taken out, and attaches it to every answer when `fit()` asked for `'explanation'`, otherwise only to a prediction whose query begins - `EXPLAIN`. It is `None` when explanations were not asked for. + `EXPLAIN`. It is `None` when explanations were not asked for. It describes + the labelled rows the request sends, not the rows gathered before the leak + check; the request no longer keeps a copy of the gathered context. - The `EXPLAIN` query prefix: `EXPLAIN PREDICT …` adds `'explanation'` to the call's outputs. A ranking refuses it; a forecast explains each step. diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index 549382c80..22acbefac 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -289,8 +289,9 @@ user_5 events 1 None 1 probability_yes kind_login -0.02 `result.explanations.columns` shows, for each column, how the outcome differs across groups of the labelled context rows. The client computes it from the -labelled rows it sends; the endpoint is not asked for it. It is `None` when -explanations were not asked for. +labelled rows the request sends; rows the leak check removed are not in it. +The endpoint is not asked for it. It is `None` when explanations were not +asked for. When it is computed: @@ -307,28 +308,26 @@ bins, categories into one cohort each (the tenth onward merged into `OTHER(*)`, missing values under `N/A`), timestamps into ranges of time before the anchor. Each cohort has its `share` of the entities and its `mean` outcome: the rate of each class for classification (classes by frequency, at -most 30), the plain mean for regression. A column's `score` is how much its -cohort means spread, weighted by share and averaged over classes; scores sum -to 1 across columns. +most 30), the plain mean for regression. | Model | Columns analysed | |---|---| | kumo-tabular | every column of the one table except the target, the primary key and timestamps (there is no anchor), reported under table `instance_table` | -| kumo-relational | every column of each related table except keys, per hop the table is reached at, plus `COUNT(*)`, the number of that table's rows linked to the entity. The entity's own table is not analysed. | +| kumo-relational | every column of each related table except keys, per hop the table is reached at, plus `COUNT(*)`, the number of that table's rows linked to the entity. The entity's own table is included, at hop 0; only the question rows themselves (the anchor and target) are not analysed. | On `kumo-relational` an entity's related rows at one hop are averaged first, then entities are grouped into cohorts. `result.explanations.columns.to_pandas()` gives one line per (table, hop, -column, cohort, class), highest `score` first. Values are illustrative, and +column, cohort, class), columns in context order. Values are illustrative, and the `no` lines are left out: ``` -table hop column stype cohort class share mean score -orders 1 COUNT(*) None [0 - 0] yes 0.30 0.12 0.41 -orders 1 COUNT(*) None (0 - 3] yes 0.45 0.38 0.41 -orders 1 COUNT(*) None (3 - 18+] yes 0.25 0.71 0.41 -orders 1 amount numerical [3 - 12] yes 0.50 0.22 0.27 +table hop column stype cohort class share mean +orders 1 COUNT(*) None [0 - 0] yes 0.30 0.12 +orders 1 COUNT(*) None (0 - 3] yes 0.45 0.38 +orders 1 COUNT(*) None (3 - 18+] yes 0.25 0.71 +orders 1 amount numerical [3 - 12] yes 0.50 0.22 ``` `class` is `None` for regression. `stype` is `None` on a `COUNT(*)` line. diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 34c1ce818..75140a3c0 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -595,6 +595,9 @@ def _answer_in_parts( # of a forecast is an ordinary regression, so the caller's fields are # what each part is asked for. wanted = _RANKING_PARTS if ranking else outputs + if request.parts: + # Refused before the analysis below, not after it. + self._prepare(request.parts[0], model, wanted) # A forecast's analysis reads every step's labelled rows at once. columns = ( _analysed_or_warned( @@ -1051,23 +1054,26 @@ class _LabelledRows: r"""The labelled rows a request's column analysis reads.""" _INSTANCE_ID = 'instance_id' + _ENTITY_REF = '__kumo_entity_ref' def __init__(self) -> None: # Each step's instance ids, renumbered so no two steps share one. self._ids: dict[tuple[int, Any], int] = {} - @staticmethod - def of(request: Request, planned: Mapping[str, Any]) -> dict[str, Any]: - r"""``planned``'s task and schema over the request's labelled context. + @classmethod + def of(cls, request: Request, planned: Mapping[str, Any]) -> dict[str, Any]: + r"""``planned``'s task over the labelled rows the request sends. - A gathered request carries its context as it was before the rows - answering the scored ones were dropped; any other request's planned - context is that context. + A request built from frames has no payloads; its planned context is + what it sends. """ - labelled = request._labelled_context + payloads = request.payloads + if not payloads: + return {**planned} return { **planned, - 'context': planned['context'] if labelled is None else labelled, + 'schema': cls._schema([payload['schema'] for payload in payloads]), + 'context': cls._sent(payloads), } def across(self, steps: Sequence[Request]) -> dict[str, Any]: @@ -1105,25 +1111,84 @@ def across(self, steps: Sequence[Request]) -> dict[str, Any]: ) for name in names } - # A step's schema describes only the tables that step reached. - schemas = [payload['schema'] for payload in each] + return { + **each[0], + 'schema': self._schema([payload['schema'] for payload in each]), + 'context': context, + } + + @classmethod + def _sent(cls, payloads: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + r"""The payloads' labelled contexts as one, each instance once. + + Every payload carries the same labelled draw, less its own scored + entities, under the same instance ids. An instance is keyed by entity + and anchor; the first payload sending it gives its row and its + related rows. + """ + seen: set[tuple[Any, Any]] = set() + instances: list[Any] = [] + related: dict[str, dict[str, Any]] = {} + for payload in payloads: + context = payload['context'] + table = context['instance_table'] + columns = list(table['columns']) + at = columns.index(cls._INSTANCE_ID) + entity = next( + index + for index, name in enumerate(columns) + if str(name).startswith(cls._ENTITY_REF) + ) + anchor_column = payload['task'].get('anchor_time_column') + anchor = ( + columns.index(anchor_column) + if anchor_column in columns + else None + ) + keys: set[tuple[Any, Any]] = set() + kept: set[Any] = set() + for row in table['rows']: + key = (row[entity], None if anchor is None else row[anchor]) + keys.add(key) + if key not in seen: + kept.add(row[at]) + instances.append(row) + seen |= keys + for name, rows in (context.get('related_tables') or {}).items(): + held = related.setdefault(name, {**rows, 'rows': []}) + where = list(rows['columns']).index(cls._INSTANCE_ID) + held['rows'].extend( + row for row in rows['rows'] if row[where] in kept + ) + first = payloads[0]['context'] + sent: dict[str, Any] = { + 'instance_table': {**first['instance_table'], 'rows': instances} + } + if any('related_tables' in payload['context'] for payload in payloads): + sent['related_tables'] = related + return sent + + @staticmethod + def _schema(schemas: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + r"""One schema describing every table any of ``schemas`` reached. + + A payload's schema describes only the tables that payload reached. + """ schema = dict(schemas[0]) - if any('related_tables' in step_schema for step_schema in schemas): + if any('related_tables' in each for each in schemas): described: dict[str, Any] = {} - for step_schema in schemas: - for name, table in ( - step_schema.get('related_tables') or {} - ).items(): + for each in schemas: + for name, table in (each.get('related_tables') or {}).items(): described.setdefault(name, table) schema['related_tables'] = described - if any('relationships' in step_schema for step_schema in schemas): + if any('relationships' in each for each in schemas): links: list[Any] = [] - for step_schema in schemas: - for link in step_schema.get('relationships') or []: + for each in schemas: + for link in each.get('relationships') or []: if link not in links: links.append(link) schema['relationships'] = links - return {**each[0], 'schema': schema, 'context': context} + return schema def _stacked( self, tables: Sequence[tuple[int, Mapping[str, Any]]] @@ -1531,10 +1596,21 @@ def _by_step( ), columns, ) + # Each step is its own call, and only the first step's id would otherwise + # survive. That is the one you do not have when a later step is the one + # that misbehaved. A step sent in several calls keeps every one of them. + metadata = { + 'steps': [step for _, step in rewritten.parts], + 'step_request_ids': [ + list(answer.metadata.get('batch_request_ids', [answer.request_id])) + for answer in answers + ], + } built = PredictionResult( model=answers[0].model if answers else '', request_id=answers[0].request_id if answers else '', items=items, + metadata=metadata, outputs=tuple(answers[0].outputs) if answers else (), explanations=explanations, ) diff --git a/packages/structured-data-models-client/src/sdm_client/explanations/column.py b/packages/structured-data-models-client/src/sdm_client/explanations/column.py index 656fbf688..617a7b184 100644 --- a/packages/structured-data-models-client/src/sdm_client/explanations/column.py +++ b/packages/structured-data-models-client/src/sdm_client/explanations/column.py @@ -26,23 +26,19 @@ class _Cohorts: cohorts: list[str] shares: np.ndarray # [cohorts] means: np.ndarray # [cohorts, outputs] - score: float class ColumnExplanations: r"""Per-column cohort statistics over a request's labelled rows. - Says which columns the outcome moves with, and how: the more a column's - cohorts differ in mean outcome, the higher its score. + Says which columns the outcome moves with, and how, cohort by cohort. Each analysed column is split into cohorts: numerical values and ``COUNT(*)`` (rows linked to an entity) into quantile bins, categories into one cohort each, timestamps into time-before-anchor buckets. Each cohort carries its population share and its mean outcome: one mean per class for classification (classes ordered by frequency in the context labels), - the plain mean for regression. A column's score is the population-weighted - standard deviation of its cohort means, averaged over classes, then - normalized so all scores sum to 1. + the plain mean for regression. Relational columns are analysed per entity: an entity's related rows at one hop are averaged first, then entities are grouped. The task table itself is @@ -94,21 +90,13 @@ def __init__(self, payload: Mapping[str, Any]) -> None: self._relational(payload, instance, outcome) else: self._tabular(payload, instance, outcome) - total = sum(column.score for column in self._columns) - if total > 0: - self._columns = [ - _Cohorts(**{**vars(column), 'score': column.score / total}) - for column in self._columns - ] - self._columns.sort(key=lambda column: column.score, reverse=True) def to_pandas(self) -> pd.DataFrame: r"""One row per (table, hop, column, cohort, class). - Columns by normalized score, highest first (ties keep context order: - tables as sent, hops ascending, ``COUNT(*)`` then columns as sent); - cohorts in their own order; classes by frequency. ``class`` is - ``None`` for regression. + Columns in context order (tables as sent, hops ascending, ``COUNT(*)`` + then columns as sent); cohorts in their own order; classes by + frequency. ``class`` is ``None`` for regression. """ classes = self._classes if self._classes is not None else [None] rows = [ @@ -121,7 +109,6 @@ def to_pandas(self) -> pd.DataFrame: label, float(column.shares[at]), float(column.means[at, k]), - column.score, ) for column in self._columns for at, cohort in enumerate(column.cohorts) @@ -138,7 +125,6 @@ def to_pandas(self) -> pd.DataFrame: 'class', 'share', 'mean', - 'score', ], ) @@ -279,7 +265,7 @@ def _table( assert anchor is not None result = self._times(data[at], linked, anchor, outcome) else: - index, labels, other = data + index, labels = data if kind == 'categorical': pairs = (linked, index[at]) else: @@ -288,15 +274,15 @@ def _table( position = np.searchsorted(rows, index[0][keep]) pairs = (linked[position], index[1][keep]) shares = self._shares(pairs, linked, len(labels)) - result = self._categories(shares, labels, other, outcome) + result = self._categories(shares, labels, outcome) self._add(name, hop, column, stypes[column], result) def _prepare(self, values: pd.Series, stype: str) -> tuple[str, Any] | None: r"""A column's per-row data, or ``None`` when it holds nothing to group. - A categorical becomes ``(index, labels, other)``: each row's cohort, - the cohort labels (N/A last) and which label is the merged tail. A - multicategorical's index is ``(row, cohort)`` pairs. + A categorical becomes ``(index, labels)``: each row's cohort and the + cohort labels (N/A last). A multicategorical's index is + ``(row, cohort)`` pairs. """ if stype == 'numerical': numbers = pd.to_numeric(values, errors='coerce').to_numpy(float) @@ -318,14 +304,11 @@ def _prepare(self, values: pd.Series, stype: str) -> tuple[str, Any] | None: return None mapping = self._by_frequency(present) labels = [str(value) for value in mapping] - other = np.zeros(len(labels) + 1, dtype=bool) tail = None if len(mapping) > self.MAX_CATEGORIES: mapping = mapping[: self.MAX_CATEGORIES - 1] labels = [*labels[: self.MAX_CATEGORIES - 1], self.OTHER] tail = self.MAX_CATEGORIES - 1 - other = np.zeros(self.MAX_CATEGORIES + 1, dtype=bool) - other[tail] = True labels.append(self.NA) position = {value: at for at, value in enumerate(mapping)} index = np.array( @@ -335,7 +318,7 @@ def _prepare(self, values: pd.Series, stype: str) -> tuple[str, Any] | None: ], dtype=int, ) - return 'categorical', (index, labels, other) + return 'categorical', (index, labels) @staticmethod def _category( @@ -361,11 +344,9 @@ def _multicategorical( rows = [at for at, cell in enumerate(cells) for _ in cell] index = [position[item] for cell in cells for item in cell] labels = [str(value) for value in mapping] - other = np.zeros(len(labels), dtype=bool) return 'multicategorical', ( (np.array(rows, dtype=int), np.array(index, dtype=int)), labels, - other, ) @staticmethod @@ -434,7 +415,7 @@ def _times( entity: np.ndarray, anchor: np.ndarray, outcome: np.ndarray, - ) -> tuple[list[str], np.ndarray, np.ndarray, float] | None: + ) -> tuple[list[str], np.ndarray, np.ndarray] | None: r"""Time before the entity's anchor, in ``NUM_COHORTS`` buckets.""" seconds = anchor[entity] - times valid = seconds >= 0 # NaN and future-looking rows drop out. @@ -451,9 +432,7 @@ def _times( for start, end in zip(edges[:-2], edges[1:-1], strict=True) ] labels.append(f'[{edges[-2]} {unit}, {edges[-1]} {unit}]') - return self._categories( - shares, labels, np.zeros(len(labels), dtype=bool), outcome - ) + return self._categories(shares, labels, outcome) @classmethod def _time_window(cls, seconds: int) -> tuple[int, str]: @@ -479,8 +458,8 @@ def _time_window(cls, seconds: int) -> tuple[int, str]: def _ranges( self, values: np.ndarray, is_count: bool, outcome: np.ndarray - ) -> tuple[list[str], np.ndarray, np.ndarray, float] | None: - r"""Entities in quantile bins: labels, shares, means and raw score.""" + ) -> tuple[list[str], np.ndarray, np.ndarray] | None: + r"""Entities in quantile bins: labels, shares and means.""" values = values.astype(float) valid = ~np.isnan(values) values, outcome = values[valid], outcome[valid] @@ -506,7 +485,7 @@ def _ranges( edges = np.concatenate( [bins[:1], (bins if zero_bin else bins[1:])[keep]] ) - text = [f'{value:.2f}'.removesuffix('.00') for value in edges.tolist()] + text = self._edge_labels(edges.tolist()) cohorts = [] for at, (start, end) in enumerate( zip(text[:-1], text[1:], strict=True) @@ -516,46 +495,58 @@ def _ranges( cohorts.append( f'[{start} - {end}]' if at == 0 else f'({start} - {end}]' ) - return cohorts, counts / counts.sum(), means, self._score(means, counts) + return cohorts, counts / counts.sum(), means + + @staticmethod + def _edge_labels(edges: list[float]) -> list[str]: + r"""Edges at the fewest decimals, from 2, that keep adjacent distinct + edges apart; an all-zero fraction is dropped.""" + for digits in range(2, 7): + text = [f'{value:.{digits}f}' for value in edges] + if all( + a != b or x == y + for a, b, x, y in zip( + text[:-1], text[1:], edges[:-1], edges[1:], strict=True + ) + ): + break + return [label.removesuffix('.' + '0' * digits) for label in text] def _categories( self, shares: np.ndarray, labels: list[str], - other: np.ndarray, outcome: np.ndarray, - ) -> tuple[list[str], np.ndarray, np.ndarray, float] | None: + ) -> tuple[list[str], np.ndarray, np.ndarray] | None: r"""Category cohorts, each entity weighted by its share in them.""" counts = shares.sum(axis=0) keep = counts > 0 if not keep.any(): return None sums = shares.T @ outcome - counts, other = counts[keep], other[keep] + counts = counts[keep] means = sums[keep] / counts[:, None] cohorts = [ label for label, kept in zip(labels, keep, strict=True) if kept ] - score = self._score(means[~other], counts[~other]) if len(cohorts) > self.NUM_COHORTS: - split = self.NUM_COHORTS - 1 - weight = counts[split:, None] / counts[split:].sum() - cohorts = [*cohorts[:split], self.OTHER] + # Missing values keep a row of their own, outside the merge. + na = 1 if cohorts[-1] == self.NA else 0 + split = self.NUM_COHORTS - 1 - na + end = len(cohorts) - na + weight = counts[split:end, None] / counts[split:end].sum() + cohorts = [*cohorts[:split], self.OTHER, *cohorts[end:]] means = np.concatenate( - [means[:split], (weight * means[split:]).sum(0, keepdims=True)] + [ + means[:split], + (weight * means[split:end]).sum(0, keepdims=True), + means[end:], + ] ) - counts = np.concatenate([counts[:split], [counts[split:].sum()]]) - return cohorts, counts / counts.sum(), means, score - - @staticmethod - def _score(means: np.ndarray, counts: np.ndarray) -> float: - r"""Population-weighted std (Bessel) of cohort means, class-averaged.""" - total = counts.sum() - if total <= 1: - return 0.0 - center = (means * counts[:, None]).sum(0) / total - spread = (counts[:, None] * (means - center) ** 2).sum(0) / (total - 1) - return float(np.sqrt(spread).mean()) + counts = np.concatenate( + [counts[:split], [counts[split:end].sum()], counts[end:]] + ) + return cohorts, counts / counts.sum(), means def _add( self, @@ -563,14 +554,11 @@ def _add( hop: int, column: str, stype: str | None, - result: tuple[list[str], np.ndarray, np.ndarray, float] | None, + result: tuple[list[str], np.ndarray, np.ndarray] | None, ) -> None: if result is None: return - cohorts, shares, means, score = result - self._columns.append( - _Cohorts(table, hop, column, stype, cohorts, shares, means, score) - ) + self._columns.append(_Cohorts(table, hop, column, stype, *result)) # Values ################################################################### diff --git a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py index 54b7f01ee..5ee85ccef 100644 --- a/packages/structured-data-models-client/src/sdm_client/materializers/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/materializers/relational.py @@ -303,9 +303,8 @@ def materialize( windowed = ( getattr(query.target_ast, 'date_offset_range', None) is not None ) - labelled = copy.deepcopy(payloads[0]['context']) _drop_context_that_answers_the_question(payloads, windowed=windowed) - request = Request.materialized( + return Request.materialized( payloads=payloads, windowed=windowed, schema=self.schema.snapshot(), @@ -321,8 +320,6 @@ def materialize( ) }, ) - request._labelled_context = labelled - return request def _gather( self, diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index a09ac33da..148a7cc7d 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -320,9 +320,6 @@ class Request: # it is how the client puts the parts' answers together, not something a # caller reads: `parts` and `asked` say what a caller needs to see. _decomposition: Decomposition | None = None - # The gathered labelled context as it was before the rows answering the - # scored ones were dropped from each payload; what column analysis reads. - _labelled_context: dict[str, Any] | None = None def __init__( self, diff --git a/packages/structured-data-models-client/tests/test_relational_explanations.py b/packages/structured-data-models-client/tests/test_relational_explanations.py index 9b9dd4b61..a7b9c393c 100644 --- a/packages/structured-data-models-client/tests/test_relational_explanations.py +++ b/packages/structured-data-models-client/tests/test_relational_explanations.py @@ -927,6 +927,48 @@ def test_forecast_names_an_unexplained_step(monkeypatch, relational_endpoint): assert any('step 2 carries none' in message for message in said), said +def test_a_forecast_keeps_every_steps_request_id( + monkeypatch, gradient_explanation +): + """A forecast's result records each step's request id under its step. + + Rules out the old behavior, where only the first step's id survived. + + Assumptions: + - Three steps, built by the real ``materialize()``; gathering is stood + in for, each step one call over users 5 and 7. + - The endpoint stamps call ``n`` (from 0) with the id ``call-n``. + """ + + class Stamping(RelationalEndpoint): + def predict(self, payload: dict) -> dict: + answer = super().predict(payload) + answer['id'] = f'call-{len(self.sent) - 1}' + return answer + + client = SDMClient(url='http://localhost:8000') + client._backend = NimBackend(Stamping(gradient_explanation, frozenset())) + query = FORECAST.removeprefix('EXPLAIN ').replace( + 'FORECAST 2', 'FORECAST 3' + ) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = client.predict( + relational_asked(monkeypatch, query, kind='regression'), + model=KumoRelational(), + outputs=['prediction'], + ) + + assert result.metadata['steps'] == [1, 2, 3] + assert result.metadata['step_request_ids'] == [ + ['call-0'], + ['call-1'], + ['call-2'], + ] + assert result.request_id == 'call-0' + + def test_one_class_refusal_names_the_column(monkeypatch, relational_endpoint): """A one-class context is refused naming the predicted column, not the wire's ``TARGET``, when the query starts with `EXPLAIN`. @@ -1250,18 +1292,21 @@ def test_a_relational_session_keeps_its_fit_time_analysis( assert label_means(other) == pytest.approx({'no': 1 / 2, 'yes': 1 / 2}) -def test_a_split_request_analyses_its_context_before_the_drop( +#: Five labelled entities, two of them the scored users 5 and 7. +GATHERED_AS = [(1, 'no'), (5, 'yes'), (7, 'no'), (2, 'yes'), (3, 'no')] + + +def test_a_split_request_analyses_the_rows_it_sends( monkeypatch, relational_endpoint ): - """A request gathered as two calls is analysed once, over the context as - gathered, before the scored entities' own rows were dropped from it. + """A request gathered as two calls is analysed once, over the labelled + rows it sends, after the scored entities' rows were dropped. Assumptions: - - Both calls share one context of five entities, two of them the scored - users 5 and 7: yes 2/5 as gathered, 1/3 once they are dropped. + - Both calls share one context object, ``GATHERED_AS``; the leak check + drops 5 and 7 from it, leaving 1, 2, 3 in both: no 2/3, yes 1/3. """ - gathered_as = [(1, 'no'), (5, 'yes'), (7, 'no'), (2, 'yes'), (3, 'no')] - first = labelled(gathered(5), gathered_as) + first = labelled(gathered(5), GATHERED_AS) second = gathered(7) second['context'] = first['context'] gathering(monkeypatch, (first, second)) @@ -1279,6 +1324,39 @@ def test_a_split_request_analyses_its_context_before_the_drop( [row[1] for row in sent['context']['instance_table']['rows']] for sent in endpoint.sent ] == [[1, 2, 3], [1, 2, 3]] + assert label_means(result) == pytest.approx({'no': 2 / 3, 'yes': 1 / 3}) + + +def test_a_split_request_analyses_each_sent_instance_once( + monkeypatch, relational_endpoint +): + """Calls carrying their own copies of one labelled draw are analysed over + the union of what they send, each instance counted once. + + Assumptions: + - Each call gets its own copy of ``GATHERED_AS`` under the same instance + ids, as the engine sends it; each loses only its own scored entity. + - Sent: call 1 has 1, 7, 2, 3 and call 2 has 1, 5, 2, 3. The union + 1, 5, 7, 2, 3 gives no 3/5, yes 2/5. Counting overlaps twice would + give yes 3/8; reading call 1 alone, 1/4. + """ + first = labelled(gathered(5), GATHERED_AS) + second = labelled(gathered(7), GATHERED_AS) + gathering(monkeypatch, (first, second)) + client, endpoint = relational_endpoint() + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = client.predict( + materialized(TIER), + model=KumoRelational(), + outputs=['prediction', 'explanation'], + ) + + assert [ + [row[1] for row in sent['context']['instance_table']['rows']] + for sent in endpoint.sent + ] == [[1, 7, 2, 3], [1, 5, 2, 3]] assert label_means(result) == pytest.approx({'no': 3 / 5, 'yes': 2 / 5}) @@ -1536,6 +1614,129 @@ def test_fractional_edges_keep_two_decimals(): assert labels == ['[0 - 23.71]', '(23.71 - 237.14]'] +def test_tiny_edges_gain_decimals_until_distinct(): + """Cohort edges too close for two decimals print with more, so every + cohort label is distinct and none is in scientific notation. + + Assumptions: + - Ten entities with ``tenure_days`` 0, 0.0001, ..., 0.0009, one row each. + - Hand-computed: the deciles step by 0.00009, so at two decimals every + edge would print as ``0``; each of the ten bins holds an entity. + """ + frame = numbered([0.0001 * at for at in range(10)]) + + labels = list(frame[frame['column'] == 'tenure_days']['cohort'].unique()) + assert len(labels) == 10 + assert all('e+' not in label and 'e-' not in label for label in labels) + + +def test_single_value_edges_keep_two_decimals(): + """A one-value column, whose two edges are equal, prints them at two + decimals rather than escalating. + + Assumptions: + - Two entities with ``tenure_days`` 0.3 each. + - Hand-computed: every decile is 0.3, so the column is one cohort. + """ + frame = numbered([0.3, 0.3]) + + labels = list(frame[frame['column'] == 'tenure_days']['cohort'].unique()) + assert labels == ['[0.30 - 0.30]'] + + +def categorised(kinds: int) -> pd.DataFrame: + """The ``cat`` cohorts of a 300-row regression table: every fifth row + has no ``cat`` and outcome 1, the rest cycle ``c0``.. over ``kinds`` + categories with outcome 0.""" + cat = [f'c{i % kinds}' if i % 5 else None for i in range(300)] + payload = { + 'task': {'kind': 'regression', 'target': {'column_name': 'y'}}, + 'schema': { + 'instance_table': { + 'primary_key': 'pk', + 'columns': { + 'pk': {'stype': 'ID'}, + 'cat': {'stype': 'categorical'}, + 'y': {'stype': 'numerical'}, + }, + } + }, + 'context': { + 'instance_table': { + 'columns': ['pk', 'cat', 'y'], + 'rows': [ + [i, value, 1.0 if value is None else 0.0] + for i, value in enumerate(cat) + ], + } + }, + } + frame = ColumnExplanations(payload).to_pandas() + return frame[frame['column'] == 'cat'] + + +def test_missing_values_keep_their_own_cohort_past_the_merge(): + """A categorical column with more cohorts than fit keeps its missing + values as an ``N/A`` row of their own, after ``OTHER(*)``, which merges + only rare categories. + + Assumptions: + - Twelve categories plus missing values: 13 cohorts, past the 10 shown. + - Hand-computed: missing rows are 60 of 300 (share 0.2), all with + outcome 1; every categorised row has outcome 0. + """ + cohorts = categorised(12) + + assert len(cohorts) == ColumnExplanations.NUM_COHORTS + assert list(cohorts['cohort'])[-2:] == ['OTHER(*)', 'N/A'] + na = cohorts[cohorts['cohort'] == 'N/A'] + assert na['share'].item() == pytest.approx(0.2) + assert na['mean'].item() == pytest.approx(1.0) + other = cohorts[cohorts['cohort'] == 'OTHER(*)'] + assert other['mean'].item() == pytest.approx(0.0) + + +def test_a_narrow_column_shows_missing_values_without_a_merge(): + """A categorical column within the cohorts shown lists every category + and ``N/A``, with no ``OTHER(*)``. + + Assumptions: + - Three categories plus missing values: 4 cohorts. + """ + cohorts = categorised(3) + + assert list(cohorts['cohort']) == ['c0', 'c1', 'c2', 'N/A'] + assert cohorts['mean'].iloc[-1] == pytest.approx(1.0) + + +def test_a_forecast_is_validated_before_it_is_analysed( + monkeypatch, relational_endpoint +): + """An explained forecast asking for an output no model gives is refused + before any column analysis starts. + + Assumptions: + - Two steps over users 5 and 7, gathering stood in for. + - The analysis is replaced by a spy that records each start. + """ + started: list[Any] = [] + monkeypatch.setattr( + 'sdm_client.client.ColumnExplanations', + lambda *args, **kwargs: started.append(args), + ) + client, _ = relational_endpoint() + request = relational_asked(monkeypatch, FORECAST, kind='regression') + + with pytest.raises(StructuredDataError): + client.predict( + request, + model=KumoRelational(), + outputs=['prediction', 'explanation', 'bogus'], + ) + + assert started == [] + + def test_a_forecast_not_explained_is_not_analysed( monkeypatch, relational_endpoint ):