From 1390d9d7f4d06e00a0056b3a0cd93d1cba77ca53 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Thu, 19 Nov 2026 09:26:16 +0000 Subject: [PATCH 1/2] Add the batch prediction explanations widget prediction.visualize() and PredictionExplanations.load(path).visualize() open the training widget's cohort and subgraph cards over a batch's retained PRED-stage explanations: batch identity and row count, an availability card per producer, a cohorts chart of predictions only, allowlisted sampled subgraphs with their recorded predictions, a static snapshot, and the existing HTML and PDF exports. Both widgets share the run reader, card builders and front end; a single tab shows without a tab strip. --- packages/kumo-engine/explainability.md | 19 +- .../kumo/engine/_explainability/__init__.py | 11 +- .../kumo/engine/_explainability/_columns.py | 74 ++++--- .../engine/_explainability/_prediction.py | 147 ++++++++++++++ .../kumo/engine/_explainability/_run.py | 23 ++- .../kumo/engine/_explainability/_snapshot.py | 67 +++++-- .../kumo/engine/_explainability/_subgraphs.py | 34 ++-- .../engine/_explainability/static/shell.js | 39 +++- .../_explainability/static/subgraphs.js | 33 ++-- .../engine/_explainability/static/widget.css | 6 + .../kumo/engine/explain/results.py | 12 ++ .../kumo/engine/trainer/trainer.py | 15 ++ .../test/test_prediction_explanations.py | 183 +++++++++++++++++- packages/kumo-engine/test/widget_exports.cjs | 43 +++- 14 files changed, 605 insertions(+), 101 deletions(-) create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_prediction.py diff --git a/packages/kumo-engine/explainability.md b/packages/kumo-engine/explainability.md index edfb91e9f..99042e9ea 100644 --- a/packages/kumo-engine/explainability.md +++ b/packages/kumo-engine/explainability.md @@ -13,6 +13,20 @@ The tuning metric opens first when recorded. Select another metric or toggle ind Each point represents a recorded epoch. Missing scores leave gaps, and epochs from the later refit are excluded. The widget reads `metric-history.json` from the published bundle or `auto_train_progress.json` from the retained run. The view appears when either source contains training or validation series. +Display `prediction.visualize()` for a batch prediction result to open the batch widget, with the same styling and graph controls on a single page. +It shows the batch, model and snapshot identities, a Cohorts chart with class selection and sampled subgraphs with their recorded predictions. +Cohort charts show average predictions and sampled population shares; a batch has no observed outcomes or evaluation metrics. +An unavailable, disabled, unsupported or failed producer is listed under **Explanation availability**, and the other producer's card still appears. + +```python +prediction = trainer.predict(snapshot, explanations=True) +prediction.visualize() + +# Reopen a retained prediction run, including its xai/manifest.json: +from kumo.engine import PredictionExplanations +PredictionExplanations.load("/path/to/prediction/run").visualize() +``` + Use **Export PDF** to print every chart view on the currently open page, including All metrics, Epoch history and Against baselines where available, then choose **Save as PDF** in the browser dialog. The export preserves the selected explanation and chart controls. The other tab is excluded. @@ -21,9 +35,10 @@ If the notebook blocks pop-ups, open the downloaded print page to save it as PDF Neither export requires a running kernel after saving. Call `.html()` on the returned view for self-contained HTML, or `.as_widget()` to request the optional anywidget transport explicitly. -The saved notebook snapshot includes the model summary, metrics and default Evaluation charts, followed by “To activate the widget, rerun this cell.” +The saved notebook snapshot includes the model summary, metrics and default Evaluation charts, or for a batch its identities, availability, default cohort chart and subgraph coverage, followed by “To activate the widget, rerun this cell.” Individual subgraphs are omitted from that static snapshot. -Opening the widget only reads retained artifacts; it never starts training or prediction. +Opening either widget only reads retained artifacts; it never starts training, prediction or explanation generation. +Retain the prediction run separately from exported predictions; a bare export cannot supply its explanations. ## Access the selected training run diff --git a/packages/kumo-engine/kumo/engine/_explainability/__init__.py b/packages/kumo-engine/kumo/engine/_explainability/__init__.py index f6f0d5c36..ad01f5efa 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/__init__.py +++ b/packages/kumo-engine/kumo/engine/_explainability/__init__.py @@ -6,7 +6,8 @@ The data side reads the Kumo files a training run leaves on disk and computes the chart values; the front end in ``static/`` draws them and derives the cutoff statistics from the transported counts. Open it with -``TrainingResult.visualize()``, or ``open_run`` for a retained run. +``TrainingResult.visualize()`` or ``open_run`` for a retained run; +``PredictionResult.visualize()`` shows a batch's cohorts and subgraphs. """ import json @@ -26,8 +27,9 @@ from kumo.engine._explainability._forecasting import forecast_card from kumo.engine._explainability._regression import predicted_card from kumo.engine._explainability._run import ( + CLASSIFICATION, FORECASTING, - LINK_PREDICTION, + REGRESSION, Run, load, mapping, @@ -72,7 +74,7 @@ def _static(name: str) -> str: def script(model_type: str) -> str: """The shared front end, the model type's own cards, then the mount.""" names = ['charts.js', 'shell.js', 'epoch-history.js', 'exports.js'] - if model_type != LINK_PREDICTION: + if model_type in (CLASSIFICATION, REGRESSION, FORECASTING): names.append(f'{model_type}.js') if model_type != FORECASTING: names.append('subgraphs.js') @@ -110,6 +112,9 @@ def __init__(self, result: 'TrainingResult') -> None: @cached_property def data(self) -> dict[str, Any]: """The payload the front end draws.""" + return self._build() + + def _build(self) -> dict[str, Any]: return build(load(self._result)) @property diff --git a/packages/kumo-engine/kumo/engine/_explainability/_columns.py b/packages/kumo-engine/kumo/engine/_explainability/_columns.py index 701132f5d..5c600dc33 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_columns.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_columns.py @@ -36,6 +36,8 @@ def cohorts(run: Run) -> Optional[list[ColumnCohorts]]: if (run.model_type not in (CLASSIFICATION, REGRESSION) or analysis is None or analysis.task_type.value != run.task_type): return None + if run.batch: + return analysis.cohorts or None entries = [c for c in analysis.cohorts if c.targets is not None] return entries if any(e.prediction_score > 0 for e in entries) else None @@ -112,44 +114,63 @@ def importance(run: Run) -> Optional[dict[str, Any]]: def group_view(run: Run, entry: ColumnCohorts, total_score: float, class_name: Optional[str] = None) -> dict[str, Any]: pops = np.asarray(entry.populations, dtype=float) - predictions, targets = entry.predictions, entry.targets - if isinstance(predictions, dict) and isinstance(targets, dict): - predictions, targets = predictions[class_name], targets[class_name] - pred = np.asarray(predictions, dtype=float) - actual = np.asarray(targets, dtype=float) - avg_actual = float((pops * actual).sum() / pops.sum()) - avg_pred = float((pops * pred).sum() / pops.sum()) + # A batch shows predictions only. + predictions, targets = entry.predictions, None if run.batch else entry.targets + if isinstance(predictions, dict): + predictions = predictions[class_name] + targets = targets and targets[class_name] + series = { + name: np.asarray(values, dtype=float) + for name, values in (('actual', targets), ('predicted', predictions)) + if values is not None + } + averages = { + name: float((pops * values).sum() / pops.sum()) + for name, values in series.items() + } eligible = np.where(pops >= GAP_MIN_SHARE)[0] gap = None - if len(eligible): + if len(eligible) and 'actual' in series: + pred, actual = series['predicted'], series['actual'] j = int(eligible[np.argmax(np.abs(pred - actual)[eligible])]) gap = dict( index=j, text=f'predicted {fmt.number(pred[j])} · actual ' f'{fmt.number(actual[j])}') stype = entry.stype or ('count' if entry.col_name == COUNT_COLUMN else '') - unit = (f'of {run.entities}' if entry.hop == 0 or entry.col_name - == COUNT_COLUMN else f'of sampled {entry.table_name} rows') - values = fmt.numbers([*pred, *actual, avg_actual, avg_pred]) - n = len(pred) - if class_name is not None: + unit = ('of sampled rows' if run.batch else f'of {run.entities}' + if entry.hop == 0 or entry.col_name == COUNT_COLUMN else + f'of sampled {entry.table_name} rows') + texts = fmt.numbers([v for values in series.values() + for v in values] + list(averages.values())) + n = len(pops) + if run.batch: + axis = ('average predicted value' if run.model_type == REGRESSION else + f'average predicted probability of {class_name or True}') + elif class_name is not None: axis = f'share {class_name}' elif run.model_type == CLASSIFICATION: axis = 'share True' else: axis = 'average ' + (run.target() or 'value') + groups = [] + for i in range(n): + group = dict(label=str(entry.cohorts[i]), + share_text=f'{fmt.percent(pops[i])} {unit}') + for k, (name, values) in enumerate(series.items()): + group.update({ + name: float(values[i]), + name + '_text': texts[k * n + i] + }) + groups.append(group) + share = entry.prediction_score / total_score if total_score else 0 return dict( header=(f'{entry.table_name}.{entry.col_name} · hop {entry.hop} · ' - f'{stype} · variation ' - f'{fmt.percent(entry.prediction_score / total_score)}'), - groups=[ - dict(label=str(entry.cohorts[i]), - share_text=f'{fmt.percent(pops[i])} {unit}', - predicted=float(pred[i]), predicted_text=values[i], - actual=float(actual[i]), actual_text=values[n + i]) - for i in range(n) - ], axis=axis, average_actual=avg_actual, average_predicted=avg_pred, - average_callout=(f'average actual {fmt.number(avg_actual)} · ' - f'average predicted {fmt.number(avg_pred)}'), + f'{stype} · variation {fmt.percent(share)}'), groups=groups, + axis=axis, **{ + 'average_' + name: v + for name, v in averages.items() + }, average_callout=' · '.join(f'average {name} {fmt.number(v)}' + for name, v in averages.items()), gap_callout=gap) @@ -172,6 +193,7 @@ def groups(run: Run) -> Optional[dict[str, Any]]: filter(None, (_link(entries, table, column) for table, column, _ in scores or [])), None) return dict( - title='Column groups', selectable=not scores, columns=views, - default=default or _key(max(entries, key=lambda e: e.prediction_score)), + title='Cohorts' if run.batch else 'Column groups', + selectable=not scores, columns=views, default=default + or _key(max(entries, key=lambda e: e.prediction_score)), coverage=run.explanation_coverage()) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_prediction.py b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py new file mode 100644 index 000000000..d129c118d --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The batch prediction widget: the training widget's cohort and subgraph +cards over one batch's retained PRED-stage explanations.""" + +from dataclasses import asdict +from typing import Any, Optional + +from pydantic import TypeAdapter + +from kumo.engine._explainability import ( + ExplainabilityView, + _columns, + _subgraphs, +) +from kumo.engine._explainability._run import _TASKS, Run +from kumo.engine._manifest import PredictionManifest +from kumo.engine.explain._prediction_manifest import read_record +from kumo.engine.explain.results import PredictionExplanations + +_REASONS: dict[Optional[str], str] = dict( + disabled='Generation was disabled for this batch.', + unsupported='Not supported for this prediction model.', + failed='Generation did not complete.') + +# Batch cards in order: card id, producer, its label, the card maker and the +# notice when available artifacts yield no card. +_CARDS = (('groups', 'cohorts', 'Cohorts', _columns.groups, + 'no readable column groups were recorded.'), + ('subgraphs', 'samples', 'Subgraphs', _subgraphs.card, + 'no readable saved examples are available.')) + + +def _task(source: PredictionExplanations, kind: str) -> Optional[str]: + """The task type in one producer's first readable artifact.""" + try: + if kind == 'cohorts': + return source.column_analysis().task_type.value + for reference in source.samples(): + store = reference.available and source.sample( + reference.artifact_id).subgraph_store + if store: + return store.task_type.value + except (OSError, ValueError): + pass + return None + + +def build(explanations: PredictionExplanations, + prediction: Optional[PredictionManifest], + run_id: Optional[str]) -> dict[str, Any]: + """Every number and text the batch widget draws.""" + try: + # Live metadata may hold nulls in typed fields; only a valid record + # of this batch is used. + prediction = TypeAdapter(PredictionManifest).validate_python( + asdict(prediction)) if prediction else None + except (TypeError, ValueError): + prediction = None + if prediction and run_id not in (None, prediction.id): + prediction = None + expected = (run_id, None, None) if prediction is None else ( + prediction.id, prediction.parents.model.id, + prediction.parents.snapshot.id) + notices, root = [], explanations.run_root + try: + record = read_record(root) if root else None + except (OSError, ValueError): + record = None + if record and any( + known not in (None, value) + for known, value in zip(expected, (record.run_id, record.model_id, + record.snapshot_id))): + notices.append( + 'Retained explanations do not match this prediction batch.') + record = None + source = (PredictionExplanations.load(root) + if record and root else PredictionExplanations.unavailable()) + availability = source.availability + # A producer status attached to the result outranks retained files. + for kind, state in explanations.availability.items(): + if state.reason in _REASONS: + availability[kind] = state + task = next( + filter(None, (_task(source, kind) for kind in ('cohorts', 'samples') + if availability[kind].available)), None) + provenance = record.result.provenance if record and record.result else None + run = Run(task or '', _TASKS.get(task, 'prediction'), None, {}, {}, {}, + None, False, False, None, source, batch=True, + cohort_rows=provenance and provenance.processed_cohort_rows) + cards = [] + for card_id, kind, label, make, empty in _CARDS: + state = availability[kind] + card = make(run) if state.available else None + if card is not None: + cards.append(dict(id=card_id, **card)) + else: + reason = (empty if state.available else _REASONS.get( + state.reason, 'Retained artifacts are unavailable.')) + notices.append(f'{label}: {reason}') + header: dict[str, Any] = dict(query='Batch prediction explanations') + prediction = prediction or (record.prediction if record else None) + identity = [('Batch', expected[0])] + if prediction is not None: + table, model = prediction.prediction_table, prediction.parents.model + rows = table.scored_rows if table.scored_rows is not None else table.rows + header['summary'] = (f'{rows:,} prediction rows' if rows is not None + else 'Prediction row count was not recorded.') + training = model.parents.training_table if model.parents else None + task = task or (training.task_type if training else None) + identity = [ + ('Batch', prediction.id), ('Model ID', model.id), + ('Snapshot ID', prediction.parents.snapshot.id), + ('Prediction type', + task.replace('_', ' ').capitalize() if task else 'Not recorded'), + ('Architecture', 'Not recorded in batch artifacts'), + ('Fit mode', model.fit_mode), + ('Batch started', prediction.timings + and prediction.timings.started_at) + ] + header['identity'] = [[k, str(v)] for k, v in identity if v] + return _subgraphs._json_value( + dict( + kind='prediction', model_type=run.model_type, entities=run.entities, + header=header, tabs=[ + dict( + id='explanation', label='Explanation', cards=[ + dict(id='status', title='Explanation availability', + messages=notices) + ] * bool(notices) + cards) + ])) + + +class PredictionExplainabilityView(ExplainabilityView): + """Read-only display of one batch's retained PRED-stage explanations.""" + def __init__(self, explanations: PredictionExplanations, + prediction: Optional[PredictionManifest] = None, + run_id: Optional[str] = None) -> None: + self._source = (explanations, prediction, run_id) + self._widget = None + + def _build(self) -> dict[str, Any]: + return build(*self._source) + + def __repr__(self) -> str: + return f'' diff --git a/packages/kumo-engine/kumo/engine/_explainability/_run.py b/packages/kumo-engine/kumo/engine/_explainability/_run.py index a90154783..59502e54a 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_run.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_run.py @@ -29,7 +29,10 @@ from kumoapi.explain.column_analysis import ColumnAnalysisOutput from kumoapi.explain.gradient import GraphGradientScore - from kumo.engine.explain import TrainingExplanations + from kumo.engine.explain import ( + PredictionExplanations, + TrainingExplanations, + ) from kumo.engine.trainer.trainer import TrainingResult CLASSIFICATION = 'classification' @@ -115,9 +118,13 @@ class Run: holdout_truncated: bool full_refit: bool history: Optional[dict[str, Any]] - explanations: 'TrainingExplanations' + explanations: 'TrainingExplanations | PredictionExplanations' table: Optional[str] = None snapshot: Optional[str] = None + # A batch prediction run records no targets; ``cohort_rows`` counts the + # rows its cohort pass processed, when recorded. + batch: bool = False + cohort_rows: Optional[int] = None @property def entities(self) -> str: @@ -169,6 +176,8 @@ def splits(self) -> dict[str, dict[str, Any]]: @cached_property def gradient(self) -> Optional['GraphGradientScore']: + if self.batch: + return None try: return self.explanations.gradient_scores() except (OSError, ValueError): @@ -234,9 +243,17 @@ def metrics_coverage(self) -> Optional[str]: def explanation_coverage(self) -> Optional[str]: """The explanation pass replays a fixed number of batches at a - quarter of the evaluation batch size.""" + quarter of the evaluation batch size; a batch's cohort pass records + the rows it processed.""" from kumo_core.training.runtime import LocalExplainRuntime + if self.batch: + return ('Cohort coverage was not recorded.' + if self.cohort_rows is None else + f'{self.cohort_rows:,} prediction rows processed for ' + 'cohorts. Population shares describe sampled rows with ' + 'recorded features, not the entire batch.') + batch = self._batch return self.covered(None if batch is None else int(LocalExplainRuntime.max_batches) * diff --git a/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py b/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py index 4021568be..50cdca600 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Script-free notebook snapshot of the default Evaluation views.""" +"""Script-free notebook snapshot of the default Evaluation views, or of a +batch's explanation cards.""" import math from html import escape @@ -21,13 +22,20 @@ def __init__(self, data: dict[str, Any]) -> None: self._marks: list[str] = [] self._text: list[str] = [] self._y = 12 - self._line(f'{data["model_type"].replace("_", " ").title()} evaluation', - size=20, bold=True) - for value in data['header'].values(): - if value: - self._line(value) - cards = next( - (t['cards'] for t in data['tabs'] if t['id'] == 'evaluation'), []) + batch = data.get('kind') == 'prediction' + self._title = 'Batch explanations' if batch else 'Training diagnostics' + self._line( + 'Batch explanations' if batch else + f'{data["model_type"].replace("_", " ").title()} evaluation', + size=20, bold=True) + for key, value in data['header'].items(): + for line in ([f'{k}: {v}' for k, v in value] if key == 'identity' + else [] if batch and key == 'query' else [value]): + if line: + self._line(line) + cards = next((t['cards'] for t in data['tabs'] + if t['id'] == ('explanation' if batch else 'evaluation')), + []) if not cards: self._line('No evaluation diagnostics were recorded.') for card in cards: @@ -36,8 +44,10 @@ def __init__(self, data: dict[str, Any]) -> None: f'') self._y += 8 self._line(card['title'], size=17, bold=True) - view = card['views'][0] - if view['id'] == 'history': + view = card.get('views', [{}])[0] + if card['id'] == 'groups': + self._cohorts(card) + elif view.get('id') == 'history': count = len(view['experiments']) self._line(f'Epoch history · {count} training experiment' + ('s' if count != 1 else '')) @@ -45,13 +55,16 @@ def __init__(self, data: dict[str, Any]) -> None: self._metrics(view['tiles']) elif card['id'] == 'cutoff': self._matrix(card) - elif view['id'] == 'heatmap': + elif view.get('id') == 'heatmap': self._heatmap(view) elif card['id'] in ('scores', 'predicted', 'forecast'): self._comparison(view) - for note in (view.get('coverage'), card.get('coverage')): - if note: - self._line(note) + for line in [ + view.get('coverage'), *card.get('messages', []), + card.get('coverage') + ]: + if line: + self._line(line) def _label(self, text: str, x: int, y: int, size: int = 14, anchor: str = 'start', bold: bool = False) -> None: @@ -157,6 +170,21 @@ def _heatmap(self, view: dict[str, Any]) -> None: f'{view["x"]} {view["columns"][peak_col]}, ' f'{view["y"]} {view["rows"][peak_row]}') + def _cohorts(self, card: dict[str, Any]) -> None: + """The default column's average prediction by group.""" + col = card['columns'][card['default']] + col = col['by_class'][0] if 'by_class' in col else col + self._comparison( + dict( + label=col['header'] + + (f" · class {col['label']}" if 'label' in col else ''), + y=col['axis'], series=['average predicted'], + reference_callout=col['average_callout'], tenths=[ + dict(label=f"{g['label']} · {g['share_text']}", + values=[g['predicted']], texts=[g['predicted_text']]) + for g in col['groups'] + ])) + def _comparison(self, view: dict[str, Any]) -> None: rows = view.get('tenths', view.get('steps', [])) self._line(view['label'] + ' · ' + view['y']) @@ -175,8 +203,13 @@ def _comparison(self, view: dict[str, Any]) -> None: scale = 230 / (high - low) if high > low else 0 zero = 112 - low * scale for row in rows: - self._y += 34 - self._label(row['label'], 16, self._y) + # A label wider than the bars' left margin takes its own line. + if len(row['label']) > 12: + self._line(row['label']) + self._y += 20 + else: + self._y += 34 + self._label(row['label'], 16, self._y) for i, (value, text) in enumerate(zip(row['values'], row['texts'])): y = self._y - 14 + i * 14 if isinstance(value, (int, float)) and math.isfinite(value): @@ -199,7 +232,7 @@ def svg(self, *, note: bool = True) -> str: f'Training diagnostics' + f'role="img">{self._title}' f'' + ''.join(self._marks) + message + '') diff --git a/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py b/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py index 5263fe708..fc3f6c193 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py @@ -81,24 +81,30 @@ def _sample(graph: Subgraph, task_type: str) -> Optional[dict[str, Any]]: def card(run: Run) -> Optional[dict[str, Any]]: - """Saved TEST examples; unavailable records never start model work.""" + """Saved TEST or batch examples; unavailable records never start model + work.""" if run.model_type == FORECASTING: return None try: - references = run.explanations.samples() + artifacts = run.explanations._cohort_artifacts('samples') + references = artifacts.samples() except (OSError, ValueError): return None - collector = posixpath.join(str(run.explanations.trial_root), - 'subgraph_collector', 'test') + collector = artifacts.collector_root # Each category keeps the collector's selection order. - categories = mapping( - mapping(read_json(posixpath.join( - collector, 'strategy_mappings.json'))).get('mappings')) + listed = {reference.artifact_id for reference in references} + categories = { + name: [i for i in ids if i in listed] + for name, ids in mapping( + mapping( + read_json(posixpath.join(collector, 'strategy_mappings.json'))). + get('mappings')).items() + } samples = [] for reference in references: try: sample = reference.available and _sample( - run.explanations.sample(reference.artifact_id), run.task_type) + artifacts.sample(reference.artifact_id), run.task_type) except (OSError, ValueError): sample = None if sample: @@ -115,7 +121,11 @@ def card(run: Run) -> Optional[dict[str, Any]]: if not samples: return None return _json_value( - dict(title='Example subgraph', samples=samples, task_type=run.task_type, - categories=list(categories) or ['saved'], - category_samples=categories, - unavailable_count=len(references) - len(samples))) + dict( + title='Example subgraph', samples=samples, task_type=run.task_type, + categories=list(categories) or ['saved'], + category_samples=categories, + unavailable_count=len(references) - len(samples), + coverage=f'{len(samples):,} retained example subgraphs. Examples ' + 'are a sampled subset of this prediction batch.' + if run.batch else None)) diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js index a66b1ba36..bfff1dd97 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js +++ b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js @@ -240,31 +240,33 @@ KX.cards.groups = (parent, card, ctx) => { chart.redraw(); }, {signal: columnCtx.signal}); } - legend(body, [{label: 'average actual', color: ROLE.actual}, - {label: 'average predicted', color: ROLE.model}]); + /* A batch records no actual values, so its groups have one series. */ + const series = [['actual', ROLE.actual], ['predicted', ROLE.model]].filter(([key]) => col['average_' + key] != null); + legend(body, series.map(([key, color]) => ({label: 'average ' + key, color}))); const chart = chartHost(body, (host, width) => { const rows = col.groups.map((grp, i) => ({ label: grp.label, sub: grp.share_text, - values: [grp.actual, grp.predicted], + values: series.map(([key]) => grp[key]), /* The gap callout carries this group's two values, so its bars are not labelled a second time. */ - texts: col.gap_callout && col.gap_callout.index === i ? [] : [grp.actual_text, grp.predicted_text], + texts: col.gap_callout && col.gap_callout.index === i ? [] : series.map(([key]) => grp[key + '_text']), band: col.gap_callout && col.gap_callout.index === i, })); - const vals = col.groups.flatMap(g => [g.actual, g.predicted]) - .concat([col.average_actual, col.average_predicted]).filter(v => v !== null && v !== undefined); + const vals = rows.flatMap(row => row.values) + .concat(series.map(([key]) => col['average_' + key])).filter(v => v !== null && v !== undefined); const lo = Math.min(0, ...vals), hi = Math.max(0, ...vals); const gap = col.gap_callout ? twoLines(col.gap_callout.text) : ''; const lane = gap ? Math.max(...gap.split('\n').map(l => l.length)) * 6.8 + 24 : 0; /* Bars start at zero; each average is its own dashed line in its series' colour, labelled at its top. */ - const res = rowsChart(host, width, {rows, series: [{color: ROLE.actual}, {color: ROLE.model}], + const res = rowsChart(host, width, {rows, series: series.map(([, color]) => ({color})), origin: 0, domain: [lo, hi * 1.04], axis: true, right: 56 + lane, top: 44, xTitle: col.axis, label: card.title}); const notes = S('g', {}, res.svg); const averages = (col.average_callout || '').split(' \u00b7 '); - [[col.average_actual, ROLE.actual], [col.average_predicted, ROLE.model]].forEach(([v, color], k) => { - if (v === null || v === undefined || !averages[k]) return; + series.forEach(([key, color], k) => { + const v = col['average_' + key]; + if (!averages[k]) return; const px = res.x(v); /* Drawn under the bars and their values, like a reference line. */ const line = S('path', {d: `M${px} 36V${res.rowY(rows.length)}`, style: {stroke: color, @@ -300,6 +302,10 @@ KX.cards.groups = (parent, card, ctx) => { return frame; }; +KX.cards.status = (parent, card, ctx) => cardFrame(parent, card, ctx, null, body => { + for (const message of card.messages) H('p', 'kx-coverage', body, message); +}); + /* ------------------------------------------------------------- Shell */ /* `saved` is the UI state the initializer was given (see _render.py): a @@ -315,7 +321,14 @@ function mount(host, data, signal, onDispose) { const head = H('div', 'kx-head', root); if (data.header.query) H('div', 'kx-query', head, data.header.query); for (const [key, value] of Object.entries(data.header)) { - if (key !== 'query' && value) H('div', 'kx-sub', head, value); + if (key === 'identity') { + const list = H('dl', 'kx-identity', head); + for (const [label, text] of value) { + const item = H('div', null, list); + H('dt', null, item, label); + H('dd', null, item, text); + } + } else if (key !== 'query' && value) H('div', 'kx-sub', head, value); } } @@ -361,6 +374,12 @@ function mount(host, data, signal, onDispose) { panels[next].button.focus(); }, {signal}); }); + /* One tab needs no tab strip. */ + if (panels.length === 1) { + tabs.remove(); + panels[0].panel.setAttribute('role', 'region'); + panels[0].panel.setAttribute('aria-label', panels[0].tab.label); + } select(panels[ctx.state.values.tab] ? ctx.state.values.tab : 0); return root; } diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js b/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js index 0658a8f9a..e3579e529 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js +++ b/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js @@ -267,24 +267,25 @@ KX.cards.subgraphs = (panel, payload, ctx) => { : 'All scored records have the same summed sensitivity; no relative size ranking.'; move(list.indexOf(sample), list.length); outcome.replaceChildren(); + /* Observed values come last in each list; a batch has none. */ + const observed = ctx.data.kind !== 'prediction'; + const pred = sample.prediction, actual = sample.target; if (['multiclass_classification', 'multilabel_classification'].includes(payload.task_type)) { - classOutcome(outcome, 'Predictions', sample.prediction, nodeOptions.signal); - classOutcome(outcome, 'Ground truth', sample.target, nodeOptions.signal); - } else if (ctx.data.model_type === 'classification') { - outcomeValue('Predicted probability of True', - typeof sample.prediction === 'number' ? (sample.prediction * 100).toFixed(1) + '%' : fmt(sample.prediction)); - outcomeValue('Observed outcome', ['False', 'True'][sample.target] ?? fmt(sample.target)); - } else if (ctx.data.model_type === 'regression') { - outcomeValue('Predicted value', fmt(sample.prediction)); - outcomeValue('Observed value', fmt(sample.target)); - outcomeValue('Prediction − observed', typeof sample.prediction === 'number' && typeof sample.target === 'number' - ? fmt(sample.prediction - sample.target) : 'Not recorded'); + for (const [title, values] of [['Predictions', pred], ['Ground truth', actual]].slice(0, observed ? 2 : 1)) + classOutcome(outcome, title, values, nodeOptions.signal); } else { - const pred = sample.prediction, actual = sample.target; - outcomeValue('Recommended records', fmt(pred?.length)); - outcomeValue('Observed records', fmt(actual?.length)); const targets = Array.isArray(actual) ? new Set(actual.map(String)) : null; - outcomeValue('Matches in recommendations', fmt(targets && Array.isArray(pred) ? pred.filter(x => targets.has(String(x))).length : null)); + const values = ctx.data.model_type === 'classification' ? [ + ['Predicted probability of True', typeof pred === 'number' ? (pred * 100).toFixed(1) + '%' : fmt(pred)], + ['Observed outcome', ['False', 'True'][actual] ?? fmt(actual)], + ] : ctx.data.model_type === 'regression' ? [ + ['Predicted value', fmt(pred)], ['Observed value', fmt(actual)], + ['Prediction − observed', typeof pred === 'number' && typeof actual === 'number' ? fmt(pred - actual) : 'Not recorded'], + ] : [ + ['Recommended records', fmt(pred?.length)], ['Observed records', fmt(actual?.length)], + ['Matches in recommendations', fmt(targets && Array.isArray(pred) ? pred.filter(x => targets.has(String(x))).length : null)], + ]; + for (const [label, value] of values.slice(0, observed ? undefined : 1)) outcomeValue(label, value); } if (sample.time != null) outcomeValue('Prediction date', String(sample.time).slice(0, 10)); const neighbors = new Map(sample.nodes.map(node => [node.id, []])); @@ -441,7 +442,7 @@ KX.cards.subgraphs = (panel, payload, ctx) => { addEdges(detachedSvg, detachedPositions); detached.forEach(n => addNode(n, detachedPositions.get(n.id), detachedSvg)); } - note.textContent = 'Saved, sampled context from the test explanation pass. ' + note.textContent = (payload.coverage || 'Saved, sampled context from the test explanation pass.') + ' ' + (sample.nodes.some(n => n.count != null) ? 'Grouped records summarize additional sampled records. ' : '') + (detached.length ? 'Records shown separately have no saved path to the prediction entity.' : ''); activate(ui.pinned); diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/widget.css b/packages/kumo-engine/kumo/engine/_explainability/static/widget.css index 9d7b0d087..e4a9c2ab5 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/widget.css +++ b/packages/kumo-engine/kumo/engine/_explainability/static/widget.css @@ -31,6 +31,12 @@ font-variant-numeric:tabular-nums;-webkit-font-smoothing:antialiased} .kx-query{font:600 20px/1.45 SFMono-Regular,Menlo,Consolas,"Liberation Mono",monospace; color:var(--kx-ink);overflow-wrap:anywhere;max-width:80ch} .kx-sub{margin-top:6px;font-size:13px;color:var(--kx-muted);overflow-wrap:anywhere} +.kx[data-kind="prediction"] .kx-query{font:700 24px/1.3 var(--kx-font)} +.kx[data-kind="prediction"] .kx-sub{font-size:15px;margin:8px 0 20px} +.kx-identity{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,250px),1fr));gap:14px 24px;margin:0;padding:0 0 4px} +.kx-identity>div{min-width:0} +.kx-identity dt{font-size:12px;color:var(--kx-muted);margin-bottom:3px} +.kx-identity dd{font-size:14px;font-weight:600;margin:0;overflow-wrap:anywhere} .kx-tabs{display:flex;flex-wrap:wrap;gap:28px;margin:0 2px 20px} .kx-tab{background:transparent;border:0;border-radius:0;padding:10px 0 12px;cursor:pointer; font-size:18px!important;font-weight:600!important;color:var(--kx-muted)!important;min-height:46px} diff --git a/packages/kumo-engine/kumo/engine/explain/results.py b/packages/kumo-engine/kumo/engine/explain/results.py index 1e76b1044..f549226ed 100644 --- a/packages/kumo-engine/kumo/engine/explain/results.py +++ b/packages/kumo-engine/kumo/engine/explain/results.py @@ -43,6 +43,10 @@ from matplotlib.axes import Axes from matplotlib.figure import Figure + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + @dataclass(frozen=True) class _CohortArtifacts: @@ -247,6 +251,14 @@ class PredictionExplanations(_CohortExplanations): _reason: UnavailableReason = 'unknown' _result: 'PredictionExplanationResult | None' = None + def visualize(self) -> 'PredictionExplainabilityView': + """Display this batch's saved explanations in the notebook widget.""" + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + + return PredictionExplainabilityView(self) + @classmethod def load(cls, run_root: str | Path) -> Self: """Reopen a validated retained run with its recorded producer outcomes. diff --git a/packages/kumo-engine/kumo/engine/trainer/trainer.py b/packages/kumo-engine/kumo/engine/trainer/trainer.py index e1618e294..cc5cb3d21 100644 --- a/packages/kumo-engine/kumo/engine/trainer/trainer.py +++ b/packages/kumo-engine/kumo/engine/trainer/trainer.py @@ -169,6 +169,9 @@ if TYPE_CHECKING: from kumo.engine._explainability import ExplainabilityView + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) from kumo.engine._monitoring import DriftReport, EvaluationResult logger = logging.getLogger(__name__) @@ -371,6 +374,18 @@ class PredictionResult: explanations: PredictionExplanations = field( default_factory=PredictionExplanations.unavailable) + def visualize(self) -> 'PredictionExplainabilityView': + """Display retained batch cohorts and subgraphs without running + prediction.""" + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + + manifest = (self.manifest if self.selection is not None + or self._exported is not None else None) + run_id = self.selection.job_id if self.selection is not None else None + return PredictionExplainabilityView(self.explanations, manifest, run_id) + @property def tracking(self) -> Optional[TrackingRef]: return self._tracking diff --git a/packages/kumo-engine/test/test_prediction_explanations.py b/packages/kumo-engine/test/test_prediction_explanations.py index 4bce84e60..1cec779ed 100644 --- a/packages/kumo-engine/test/test_prediction_explanations.py +++ b/packages/kumo-engine/test/test_prediction_explanations.py @@ -92,15 +92,15 @@ def _cohorts(root): return output -def _samples(root): +def _samples(root, task_type=TaskType.REGRESSION): collector = f'{root}/subgraph_collector/pred' save_json(EntityMappings(mappings={'first': ['7']}, orig_mappings={'first': ['current-user']}), f'{collector}/strategy_mappings.json') graph = Subgraph(subgraph_store=SubgraphStore( - root_id=7, task_type=TaskType.REGRESSION, - nodes={7: SubgraphNode(7, 'users', 'current-user')}, - edges={}, tables={'users': [7]})) + root_id=7, task_type=task_type, nodes={ + 7: SubgraphNode(7, 'users', 'current-user') + }, edges={}, tables={'users': [7]})) save_json(graph, f'{collector}/subgraph_json/7.json') @@ -327,6 +327,19 @@ def test_sample_availability_requires_recorded_graph(tmp_path, recorded_ids, xai.sample('7') if available: assert xai.sample('8').subgraph_store.root_id == 7 + # The widget shows only allowlisted samples, with their predictions. + filesystem.copy(f'{graphs}/8.json', f'{graphs}/99.json') + save_json( + EntityMappings( + mappings={'first': ['99', *recorded_ids]}, + orig_mappings={'first': ['x'] * (len(recorded_ids) + 1)}), + f'{collector}/strategy_mappings.json') + save_json({'prediction': .99}, f'{collector}/output/8.json') + card = xai.visualize().data['tabs'][0]['cards'][-1] + assert [sample['id'] for sample in card['samples']] == ['8'] + assert card['category_samples'] == {'first': list(recorded_ids)} + assert card['unavailable_count'] == int('7' in recorded_ids) + assert card['samples'][0]['prediction'] == .99 else: with pytest.raises(ExplanationUnavailableError): xai.sample('8') @@ -423,6 +436,16 @@ def test_copied_prediction_run_resolves_new_root(tmp_path): reopened = PredictionExplanations.load(copied) assert reopened.column_analysis().cohorts assert reopened.sample('7').subgraph_store.root_id == 7 + assert _ids(reopened.visualize()) == ['groups', 'subgraphs'] + for manifest in (None, '{}'): + path = copied / 'xai/manifest.json' + path.unlink() if manifest is None else path.write_text(manifest) + for outcome in (None, _complete()): + assert _ids( + PredictionExplanations.from_run( + copied, enabled=True, + result=outcome).visualize()) == ['status'] + assert _ids(PredictionExplanations.unavailable().visualize()) == ['status'] def test_prediction_finishes_before_replay(prediction, monkeypatch): @@ -471,3 +494,155 @@ def test_status_write_failure_preserves_prediction(prediction, monkeypatch, assert state.reason == ('failed' if enabled else 'disabled') if enabled: assert 'disk full' in state.generation_failure + view = result.visualize() + assert _ids(view) == ['status'] + assert view.data['header']['summary'] == '2 prediction rows' + assert view.data['header']['identity'][:3] == [[ + 'Batch', result.manifest.id + ], ['Model ID', 'model-id'], ['Snapshot ID', 'snapshot']] + if enabled: + # The failed status outranks retained files, a record of another + # batch stays hidden without the live manifest, and a matching + # record keeps the failure. + root = result.explanations.run_root + _cohorts(f'{root}/xai') + _samples(f'{root}/xai') + saved = _record(root, result=_complete()) + assert _ids(result.visualize()) == ['status'] + live = result.manifest + result._manifest_path.unlink() + view = replace( + result, explanations=PredictionExplanations.from_run( + root, enabled=True, result=_complete())).visualize() + assert _ids(view) == ['status'] + assert view.data['header']['identity'] == [['Batch', result.id]] + write_manifest(result._manifest_path, live) + write_record( + root, + saved.model_copy(update={ + 'run_id': live.id, + 'prediction': live + })) + assert 'Generation did not complete.' in result.visualize( + )._snapshot.plain() + + +def _ids(view): + return [card['id'] for card in view.data['tabs'][0]['cards']] + + +@pytest.mark.parametrize('task', [ + TaskType.BINARY_CLASSIFICATION, TaskType.REGRESSION, + TaskType.MULTICLASS_CLASSIFICATION, TaskType.MULTILABEL_CLASSIFICATION +]) +@pytest.mark.parametrize('null_field', [ + None, ('prediction_table', ), ('parents', ), ('parents', 'model'), + ('parents', 'snapshot') +]) +def test_prediction_widget_shows_cohorts_and_subgraphs(tmp_path, task, + null_field): + from kumo_core.train.artifact_manifest import PredictionArtifactType + + from kumo.engine.trainer.trainer import PredictionResult + + cohort = _cohorts(f'{tmp_path}/xai') + cohort.task_type = task + labeled = task in (TaskType.MULTICLASS_CLASSIFICATION, + TaskType.MULTILABEL_CLASSIFICATION) + if labeled: + cohort.cohorts[0].predictions = {'blue': [.2, .8], 'red': [.8, .2]} + save_json(cohort, + f'{tmp_path}/xai/column_analysis/pred/column_analysis.json') + _samples(f'{tmp_path}/xai', task) + record = _record(tmp_path, result=_complete()) + xai = PredictionExplanations.load(tmp_path) + selection = SimpleNamespace(job_id=record.run_id, + pointer_path=str(tmp_path / 'current.json')) + result = PredictionResult(selection, PredictionArtifactType.PREDICTIONS, + explanations=xai) + # Live metadata with a null typed field falls back to the saved record. + metadata = record.prediction.to_dict() + if null_field: + parent = metadata if len(null_field) == 1 else metadata[null_field[0]] + parent[null_field[-1]] = None + result._manifest_path.write_text(json.dumps(metadata)) + view = result.visualize() + data = view.data + assert data['kind'] == 'prediction' and len(data['tabs']) == 1 + assert data['header']['summary'] == '2 prediction rows' + assert dict( + data['header']['identity'])['Prediction type'] == (task.value.replace( + '_', ' ').capitalize()) + assert _ids(view) == ['groups', 'subgraphs'] + groups, graphs = data['tabs'][0]['cards'] + col = groups['columns'][groups['default']] + col = col['by_class'][0] if labeled else col + assert [g['predicted'] for g in col['groups']] == [.2, .8] + assert 'actual' not in col['groups'][0] and 'average_actual' not in col + assert graphs['samples'][0]['nodes'][0]['value'] == 'current-user' + json.dumps(data, allow_nan=False) + svg = view._repr_svg_() + assert 'Batch explanations' in svg and 'Batch: retained-run' in svg + assert 'young' in svg and 'average actual' not in svg and 'current-user' not in svg + assert 'To activate the widget, rerun this cell.' in view._snapshot.plain() + assert 'kumo-explainability' in view.html() + assert xai.visualize().data == data + + +@pytest.mark.parametrize('state', [ + 'disabled', 'failed_cohorts', 'failed_samples', 'unsupported', + 'samples_only', 'missing', 'malformed' +]) +@pytest.mark.parametrize('attached', ['loaded', 'unknown', 'complete']) +def test_prediction_widget_respects_producer_status(tmp_path, state, attached): + _cohorts(f'{tmp_path}/xai') + _samples(f'{tmp_path}/xai') + result = _complete() + if state.startswith('failed_'): + result = replace( + result, **{ + state.removeprefix('failed_'): + ProducerOutcome('failed', 'test failure') + }) + elif state == 'samples_only': + result = replace(result, cohorts=ProducerOutcome('unsupported')) + _samples(f'{tmp_path}/xai', TaskType.LINK_PREDICTION) + elif state == 'unsupported': + result = replace(result, cohorts=ProducerOutcome('unsupported'), + samples=ProducerOutcome('unsupported')) + _record(tmp_path, enabled=state != 'disabled', + result=None if state == 'disabled' else result) + path = tmp_path / 'xai/column_analysis/pred/column_analysis.json' + if state == 'missing': + path.unlink() + elif state == 'malformed': + path.write_text('{}') + source = (PredictionExplanations.load(tmp_path) + if attached == 'loaded' else PredictionExplanations.from_run( + tmp_path, enabled=True, + result=_complete() if attached == 'complete' else None)) + view = source.visualize() + ids = _ids(view) + assert ids[0] == 'status' + assert ('groups' in ids) == (state == 'failed_samples') + assert ('subgraphs' in ids) == (state in ('failed_cohorts', 'samples_only', + 'missing', 'malformed')) + assert ('1 retained example subgraphs.' + in view._snapshot.plain()) == ('subgraphs' in ids) + + +def test_prediction_widget_keeps_all_zero_variation_cohorts(tmp_path): + artifact = _cohorts(f'{tmp_path}/xai') + cohort = artifact.cohorts[0] + cohort.cohorts = [f'cohort-{i}' for i in range(12)] + cohort.populations = [1 / 12] * 12 + cohort.predictions = [0.] * 12 + cohort.prediction_score = 0. + save_json(artifact, + f'{tmp_path}/xai/column_analysis/pred/column_analysis.json') + _record(tmp_path, result=_complete()) + view = PredictionExplanations.load(tmp_path).visualize() + column = next(iter(view.data['tabs'][0]['cards'][-1]['columns'].values())) + assert len(column['groups']) == 12 and column['average_predicted'] == 0. + assert 'cohort-11' in view._repr_svg_() + json.dumps(view.data, allow_nan=False) diff --git a/packages/kumo-engine/test/widget_exports.cjs b/packages/kumo-engine/test/widget_exports.cjs index 934fbcc42..25cd1b0fb 100644 --- a/packages/kumo-engine/test/widget_exports.cjs +++ b/packages/kumo-engine/test/widget_exports.cjs @@ -8,7 +8,8 @@ // setting the HTML export or the print copy loses, or a missing printed view. // // Needs retained classification, forecasting and regression fits under -// KUMO_EXPLAINABILITY_RUNS/, a Python environment that +// KUMO_EXPLAINABILITY_RUNS/, optionally a retained batch prediction run +// with cohorts and subgraphs at KUMO_BATCH_EXPLANATIONS, a Python environment that // imports this checkout (PYTHON, default python3), and Playwright with its // browsers, from NODE_PATH or the locked package next to this file: // npm ci --prefix packages/kumo-engine/test/snowflake_notebook/browser @@ -23,17 +24,20 @@ const {chromium, firefox, webkit} = (() => { })(); const runs = process.env.KUMO_EXPLAINABILITY_RUNS; assert.ok(runs, 'Set KUMO_EXPLAINABILITY_RUNS to the retained runs (/scratch/trainers/*/runs/*-fit-*).'); -function widget(task, dir) { +function widget(task, dir, source = path.join(runs, task)) { const file = path.join(dir, task + '.html'); execFileSync(process.env.PYTHON || 'python3', ['-c', `import glob, sys from dataclasses import replace -from kumo.engine import Trainer +from kumo.engine import PredictionExplanations, Trainer from kumo.engine.explain import TrainingExplanations -run, = glob.glob(sys.argv[1] + '/scratch/trainers/*/runs/*-fit-*') -result = Trainer.load(run + '/published-model', accelerator='cpu').result -view = replace(result, explanations=TrainingExplanations.load(run)).visualize() +if '${task}' == 'batch': + view = PredictionExplanations.load(sys.argv[1]).visualize() +else: + run, = glob.glob(sys.argv[1] + '/scratch/trainers/*/runs/*-fit-*') + result = Trainer.load(run + '/published-model', accelerator='cpu').result + view = replace(result, explanations=TrainingExplanations.load(run)).visualize() open(sys.argv[2], 'w', encoding='utf-8').write('' + view.html())`, - path.join(runs, task), file], {cwd: path.join(__dirname, '..'), stdio: 'inherit'}); + source, file], {cwd: path.join(__dirname, '..'), stdio: 'inherit'}); return pathToFileURL(file).href; } async function plots(page, selector) { @@ -59,7 +63,8 @@ async function print(page) { async function save(page, filename) { const waiting = page.waitForEvent('download'); await page.getByRole('button', {name: 'Export HTML', exact: true}).click(); - await (await waiting).saveAs(filename); + const download = await waiting; await download.saveAs(filename); + return download.suggestedFilename(); } async function alter(filename, change) { const html = await fs.readFile(filename, 'utf8'); @@ -71,6 +76,7 @@ async function alter(filename, change) { const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'kumo-exports-')), results = []; try { const [training, forecast, regression] = ['classification', 'forecasting', 'regression'].map(task => widget(task, tmp)); + const batch = process.env.KUMO_BATCH_EXPLANATIONS && widget('batch', tmp, process.env.KUMO_BATCH_EXPLANATIONS); for (const engine of [chromium, firefox, webkit]) { const browser = await engine.launch(), context = await browser.newContext({acceptDownloads: true}); const errors = []; context.on('page', page => page.on('pageerror', error => errors.push(error.message))); @@ -128,6 +134,27 @@ async function alter(filename, change) { await alter(saved, payload => { payload.data.tabs = payload.data.tabs.filter(tab => tab.id === 'evaluation'); payload.saved.tab = 0; }); await reopened.goto(pathToFileURL(saved).href); const single = await print(reopened); assert.equal(await single.title(), 'Kumo · Evaluation'); await single.close(); + if (batch) { + // One panel without tabs; the selected column and pinned record survive both exports. + await page.goto(batch); + assert.equal(await page.getByRole('tab').count(), 0); + await page.locator('[data-card="groups"] select').first().selectOption({index: 1}); + await page.locator('g.kx-sg-node[data-root="false"]').first().dispatchEvent('click'); + const column = await page.locator('[data-card="groups"] select').first().inputValue(); + // Axis ticks follow the page width, which the saved page changes. + const bars = page => page.locator('[data-card="groups"] .kx-chart svg text:not(.kx-t-axis)').allTextContents(); + const cohort = await bars(page); + const details = await page.locator('.kx-sg-detail').textContent(); + assert.equal(await save(page, saved), 'kumo-prediction-explanations.html'); + await reopened.goto(pathToFileURL(saved).href); + assert.equal(await reopened.locator('[data-card="groups"] select').first().inputValue(), column); + assert.deepEqual(await bars(reopened), cohort); + assert.equal(await reopened.locator('.kx-sg-detail').textContent(), details); + const popup = await print(page); + assert.equal(await popup.title(), 'Kumo · Explanation'); + await plots(popup, '[data-card="groups"] .kx-chart svg'); await plots(popup, '.kx-sg-canvas > svg'); + assert.equal(await popup.locator('.kx-sg-detail').textContent(), details); await popup.close(); + } await page.goto(forecast); await save(page, saved); for (const values of [[-10, -5], [-10, 5]]) { await alter(saved, payload => { const totals = payload.data.tabs.flatMap(t => t.cards).find(c => c.id === 'forecast').views.find(v => v.id === 'totals'); From f88b08b37c0cb243166313fb63701666fb621e86 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Thu, 19 Nov 2026 09:50:54 +0000 Subject: [PATCH 2/2] Record batch sample predictions and fix batch widget audit findings The PRED replay now also writes the collector's output format, so a sampled subgraph from trainer.predict(..., explanations=True) shows its recorded prediction. A live manifest with a null typed field still has its surviving model and snapshot IDs checked against the retained record. An unreadable sampled graph no longer hides the readable ones when the task comes from samples. Only a batch drops the tab strip, so a training view restored from its bundle keeps its Evaluation tab. The widget status test reuses the producer-outcome parametrization. --- .../src/kumo_core/prediction_explanations.py | 2 +- .../tests/test_prediction_explanations.py | 3 + .../engine/_explainability/_prediction.py | 34 ++++++----- .../engine/_explainability/static/shell.js | 4 +- .../test/test_prediction_explanations.py | 56 +++++++++++++------ 5 files changed, 65 insertions(+), 34 deletions(-) diff --git a/packages/kumo-core/src/kumo_core/prediction_explanations.py b/packages/kumo-core/src/kumo_core/prediction_explanations.py index 883c69f7f..d2b91fc86 100644 --- a/packages/kumo-core/src/kumo_core/prediction_explanations.py +++ b/packages/kumo-core/src/kumo_core/prediction_explanations.py @@ -199,7 +199,7 @@ def explain_prediction( fit_mode_to_use=FitMode.fit, root_dir=output_dir, top_k=top_k, - collector_formats=['nested_json'], + collector_formats=['nested_json', 'output'], cohort_rows=options.cohort_rows, sample_rows=options.sample_rows, ) diff --git a/packages/kumo-core/tests/test_prediction_explanations.py b/packages/kumo-core/tests/test_prediction_explanations.py index a71a43bd3..206eae7c7 100644 --- a/packages/kumo-core/tests/test_prediction_explanations.py +++ b/packages/kumo-core/tests/test_prediction_explanations.py @@ -97,6 +97,9 @@ def test_prediction_requires_producer_completion(replay): result = run() assert result.cohorts.status == result.samples.status == 'failed' assert 'did not complete' in result.failure + # Each sampled graph keeps its recorded prediction beside it. + assert trainer.explain.call_args.kwargs['collector_formats'] == [ + 'nested_json', 'output'] @pytest.mark.parametrize('failed', ['cohorts', 'samples']) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_prediction.py b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py index d129c118d..980a2eddd 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_prediction.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py @@ -37,32 +37,40 @@ def _task(source: PredictionExplanations, kind: str) -> Optional[str]: try: if kind == 'cohorts': return source.column_analysis().task_type.value - for reference in source.samples(): + references = source.samples() + except (OSError, ValueError): + return None + for reference in references: + try: store = reference.available and source.sample( reference.artifact_id).subgraph_store - if store: - return store.task_type.value - except (OSError, ValueError): - pass + except (OSError, ValueError): + continue # an unreadable graph leaves the others usable + if store: + return store.task_type.value return None def build(explanations: PredictionExplanations, prediction: Optional[PredictionManifest], run_id: Optional[str]) -> dict[str, Any]: - """Every number and text the batch widget draws.""" + """The batch widget's payload.""" + # Live metadata may hold nulls in typed fields: every identity it still + # carries must match the retained record, and only a valid manifest of + # this batch is displayed. + parents = getattr(prediction, 'parents', None) + live = tuple( + getattr(item, 'id', None) + for item in (prediction, getattr(parents, 'model', None), + getattr(parents, 'snapshot', None))) + if run_id not in (None, live[0]): + live, prediction = (None, None, None), None + expected = (run_id or live[0], *live[1:]) try: - # Live metadata may hold nulls in typed fields; only a valid record - # of this batch is used. prediction = TypeAdapter(PredictionManifest).validate_python( asdict(prediction)) if prediction else None except (TypeError, ValueError): prediction = None - if prediction and run_id not in (None, prediction.id): - prediction = None - expected = (run_id, None, None) if prediction is None else ( - prediction.id, prediction.parents.model.id, - prediction.parents.snapshot.id) notices, root = [], explanations.run_root try: record = read_record(root) if root else None diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js index bfff1dd97..06c784fc8 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js +++ b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js @@ -374,8 +374,8 @@ function mount(host, data, signal, onDispose) { panels[next].button.focus(); }, {signal}); }); - /* One tab needs no tab strip. */ - if (panels.length === 1) { + /* A batch's single tab needs no tab strip; training keeps its tabs. */ + if (panels.length === 1 && data.kind === 'prediction') { tabs.remove(); panels[0].panel.setAttribute('role', 'region'); panels[0].panel.setAttribute('aria-label', panels[0].tab.label); diff --git a/packages/kumo-engine/test/test_prediction_explanations.py b/packages/kumo-engine/test/test_prediction_explanations.py index 1cec779ed..39ee88f5c 100644 --- a/packages/kumo-engine/test/test_prediction_explanations.py +++ b/packages/kumo-engine/test/test_prediction_explanations.py @@ -367,6 +367,7 @@ def test_prediction_run_and_export_absence(tmp_path): PredictionExplanationResult(ProducerOutcome('failed', 'cohort failed'), ProducerOutcome('failed', 'collector failed')), replace(_complete(), samples=ProducerOutcome('failed', 'collector failed')), + replace(_complete(), cohorts=ProducerOutcome('failed', 'cohort failed')), ]) def test_retained_producer_status_matches_live(tmp_path, outcomes): _cohorts(f'{tmp_path}/xai') @@ -376,6 +377,11 @@ def test_retained_producer_status_matches_live(tmp_path, outcomes): result=outcomes) reopened = PredictionExplanations.load(tmp_path) assert reopened.availability == live.availability + for view in (live.visualize(), reopened.visualize()): + ids = _ids(view) + assert ids[0] == 'status' + assert ('groups' in ids) == (outcomes.cohorts.status == 'complete') + assert ('subgraphs' in ids) == (outcomes.samples.status == 'complete') if outcomes.cohorts.status == 'complete': assert reopened.column_analysis().cohorts else: @@ -587,29 +593,45 @@ def test_prediction_widget_shows_cohorts_and_subgraphs(tmp_path, task, assert 'To activate the widget, rerun this cell.' in view._snapshot.plain() assert 'kumo-explainability' in view.html() assert xai.visualize().data == data + if null_field == ('prediction_table', ): + # A retained record of another model under this batch ID. + other = replace(record.prediction, parents=replace( + record.prediction.parents, model=stub('model', 'other-model'))) + write_record(str(tmp_path), record.model_copy(update={ + 'model_id': 'other-model', 'prediction': other})) + view = result.visualize() + assert _ids(view) == ['status'] + assert view.data['tabs'][0]['cards'][0]['messages'][0] == ( + 'Retained explanations do not match this prediction batch.') -@pytest.mark.parametrize('state', [ - 'disabled', 'failed_cohorts', 'failed_samples', 'unsupported', - 'samples_only', 'missing', 'malformed' -]) +@pytest.mark.parametrize( + 'state', ['disabled', 'samples_only', 'unreadable_graph', 'missing', + 'malformed']) @pytest.mark.parametrize('attached', ['loaded', 'unknown', 'complete']) def test_prediction_widget_respects_producer_status(tmp_path, state, attached): + """Status precedence and unreadable artifacts; producer outcomes are in + test_retained_producer_status_matches_live.""" _cohorts(f'{tmp_path}/xai') _samples(f'{tmp_path}/xai') result = _complete() - if state.startswith('failed_'): - result = replace( - result, **{ - state.removeprefix('failed_'): - ProducerOutcome('failed', 'test failure') - }) - elif state == 'samples_only': + if state in ('samples_only', 'unreadable_graph'): result = replace(result, cohorts=ProducerOutcome('unsupported')) _samples(f'{tmp_path}/xai', TaskType.LINK_PREDICTION) - elif state == 'unsupported': - result = replace(result, cohorts=ProducerOutcome('unsupported'), - samples=ProducerOutcome('unsupported')) + if state == 'unreadable_graph': + # An unreadable graph ordered before the readable one. + collector = tmp_path / 'xai/subgraph_collector/pred' + save_json(EntityMappings(mappings={'first': ['3', '7']}, + orig_mappings={'first': ['x', 'y']}), + str(collector / 'strategy_mappings.json')) + (collector / 'subgraph_json/3.json').write_text('not json') + result = replace( + result, samples=replace( + result.samples, artifacts=( + *result.samples.artifacts, + 'subgraph_collector/pred/subgraph_json/3.json')), + provenance=replace(result.provenance, selected_stage_ids=(0, 1), + collector_ids=('3', '7'))) _record(tmp_path, enabled=state != 'disabled', result=None if state == 'disabled' else result) path = tmp_path / 'xai/column_analysis/pred/column_analysis.json' @@ -623,10 +645,8 @@ def test_prediction_widget_respects_producer_status(tmp_path, state, attached): result=_complete() if attached == 'complete' else None)) view = source.visualize() ids = _ids(view) - assert ids[0] == 'status' - assert ('groups' in ids) == (state == 'failed_samples') - assert ('subgraphs' in ids) == (state in ('failed_cohorts', 'samples_only', - 'missing', 'malformed')) + assert ids[0] == 'status' and 'groups' not in ids + assert ('subgraphs' in ids) == (state != 'disabled') assert ('1 retained example subgraphs.' in view._snapshot.plain()) == ('subgraphs' in ids)