diff --git a/packages/kumo-core/src/kumo_core/prediction_explanations.py b/packages/kumo-core/src/kumo_core/prediction_explanations.py index 883c69f7f..d2b91fc86 100644 --- a/packages/kumo-core/src/kumo_core/prediction_explanations.py +++ b/packages/kumo-core/src/kumo_core/prediction_explanations.py @@ -199,7 +199,7 @@ def explain_prediction( fit_mode_to_use=FitMode.fit, root_dir=output_dir, top_k=top_k, - collector_formats=['nested_json'], + collector_formats=['nested_json', 'output'], cohort_rows=options.cohort_rows, sample_rows=options.sample_rows, ) diff --git a/packages/kumo-core/tests/test_prediction_explanations.py b/packages/kumo-core/tests/test_prediction_explanations.py index a71a43bd3..206eae7c7 100644 --- a/packages/kumo-core/tests/test_prediction_explanations.py +++ b/packages/kumo-core/tests/test_prediction_explanations.py @@ -97,6 +97,9 @@ def test_prediction_requires_producer_completion(replay): result = run() assert result.cohorts.status == result.samples.status == 'failed' assert 'did not complete' in result.failure + # Each sampled graph keeps its recorded prediction beside it. + assert trainer.explain.call_args.kwargs['collector_formats'] == [ + 'nested_json', 'output'] @pytest.mark.parametrize('failed', ['cohorts', 'samples']) diff --git a/packages/kumo-engine/explainability.md b/packages/kumo-engine/explainability.md index edfb91e9f..99042e9ea 100644 --- a/packages/kumo-engine/explainability.md +++ b/packages/kumo-engine/explainability.md @@ -13,6 +13,20 @@ The tuning metric opens first when recorded. Select another metric or toggle ind Each point represents a recorded epoch. Missing scores leave gaps, and epochs from the later refit are excluded. The widget reads `metric-history.json` from the published bundle or `auto_train_progress.json` from the retained run. The view appears when either source contains training or validation series. +Display `prediction.visualize()` for a batch prediction result to open the batch widget, with the same styling and graph controls on a single page. +It shows the batch, model and snapshot identities, a Cohorts chart with class selection and sampled subgraphs with their recorded predictions. +Cohort charts show average predictions and sampled population shares; a batch has no observed outcomes or evaluation metrics. +An unavailable, disabled, unsupported or failed producer is listed under **Explanation availability**, and the other producer's card still appears. + +```python +prediction = trainer.predict(snapshot, explanations=True) +prediction.visualize() + +# Reopen a retained prediction run, including its xai/manifest.json: +from kumo.engine import PredictionExplanations +PredictionExplanations.load("/path/to/prediction/run").visualize() +``` + Use **Export PDF** to print every chart view on the currently open page, including All metrics, Epoch history and Against baselines where available, then choose **Save as PDF** in the browser dialog. The export preserves the selected explanation and chart controls. The other tab is excluded. @@ -21,9 +35,10 @@ If the notebook blocks pop-ups, open the downloaded print page to save it as PDF Neither export requires a running kernel after saving. Call `.html()` on the returned view for self-contained HTML, or `.as_widget()` to request the optional anywidget transport explicitly. -The saved notebook snapshot includes the model summary, metrics and default Evaluation charts, followed by “To activate the widget, rerun this cell.” +The saved notebook snapshot includes the model summary, metrics and default Evaluation charts, or for a batch its identities, availability, default cohort chart and subgraph coverage, followed by “To activate the widget, rerun this cell.” Individual subgraphs are omitted from that static snapshot. -Opening the widget only reads retained artifacts; it never starts training or prediction. +Opening either widget only reads retained artifacts; it never starts training, prediction or explanation generation. +Retain the prediction run separately from exported predictions; a bare export cannot supply its explanations. ## Access the selected training run diff --git a/packages/kumo-engine/kumo/engine/_explainability/__init__.py b/packages/kumo-engine/kumo/engine/_explainability/__init__.py index f6f0d5c36..ad01f5efa 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/__init__.py +++ b/packages/kumo-engine/kumo/engine/_explainability/__init__.py @@ -6,7 +6,8 @@ The data side reads the Kumo files a training run leaves on disk and computes the chart values; the front end in ``static/`` draws them and derives the cutoff statistics from the transported counts. Open it with -``TrainingResult.visualize()``, or ``open_run`` for a retained run. +``TrainingResult.visualize()`` or ``open_run`` for a retained run; +``PredictionResult.visualize()`` shows a batch's cohorts and subgraphs. """ import json @@ -26,8 +27,9 @@ from kumo.engine._explainability._forecasting import forecast_card from kumo.engine._explainability._regression import predicted_card from kumo.engine._explainability._run import ( + CLASSIFICATION, FORECASTING, - LINK_PREDICTION, + REGRESSION, Run, load, mapping, @@ -72,7 +74,7 @@ def _static(name: str) -> str: def script(model_type: str) -> str: """The shared front end, the model type's own cards, then the mount.""" names = ['charts.js', 'shell.js', 'epoch-history.js', 'exports.js'] - if model_type != LINK_PREDICTION: + if model_type in (CLASSIFICATION, REGRESSION, FORECASTING): names.append(f'{model_type}.js') if model_type != FORECASTING: names.append('subgraphs.js') @@ -110,6 +112,9 @@ def __init__(self, result: 'TrainingResult') -> None: @cached_property def data(self) -> dict[str, Any]: """The payload the front end draws.""" + return self._build() + + def _build(self) -> dict[str, Any]: return build(load(self._result)) @property diff --git a/packages/kumo-engine/kumo/engine/_explainability/_columns.py b/packages/kumo-engine/kumo/engine/_explainability/_columns.py index 701132f5d..5c600dc33 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_columns.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_columns.py @@ -36,6 +36,8 @@ def cohorts(run: Run) -> Optional[list[ColumnCohorts]]: if (run.model_type not in (CLASSIFICATION, REGRESSION) or analysis is None or analysis.task_type.value != run.task_type): return None + if run.batch: + return analysis.cohorts or None entries = [c for c in analysis.cohorts if c.targets is not None] return entries if any(e.prediction_score > 0 for e in entries) else None @@ -112,44 +114,63 @@ def importance(run: Run) -> Optional[dict[str, Any]]: def group_view(run: Run, entry: ColumnCohorts, total_score: float, class_name: Optional[str] = None) -> dict[str, Any]: pops = np.asarray(entry.populations, dtype=float) - predictions, targets = entry.predictions, entry.targets - if isinstance(predictions, dict) and isinstance(targets, dict): - predictions, targets = predictions[class_name], targets[class_name] - pred = np.asarray(predictions, dtype=float) - actual = np.asarray(targets, dtype=float) - avg_actual = float((pops * actual).sum() / pops.sum()) - avg_pred = float((pops * pred).sum() / pops.sum()) + # A batch shows predictions only. + predictions, targets = entry.predictions, None if run.batch else entry.targets + if isinstance(predictions, dict): + predictions = predictions[class_name] + targets = targets and targets[class_name] + series = { + name: np.asarray(values, dtype=float) + for name, values in (('actual', targets), ('predicted', predictions)) + if values is not None + } + averages = { + name: float((pops * values).sum() / pops.sum()) + for name, values in series.items() + } eligible = np.where(pops >= GAP_MIN_SHARE)[0] gap = None - if len(eligible): + if len(eligible) and 'actual' in series: + pred, actual = series['predicted'], series['actual'] j = int(eligible[np.argmax(np.abs(pred - actual)[eligible])]) gap = dict( index=j, text=f'predicted {fmt.number(pred[j])} · actual ' f'{fmt.number(actual[j])}') stype = entry.stype or ('count' if entry.col_name == COUNT_COLUMN else '') - unit = (f'of {run.entities}' if entry.hop == 0 or entry.col_name - == COUNT_COLUMN else f'of sampled {entry.table_name} rows') - values = fmt.numbers([*pred, *actual, avg_actual, avg_pred]) - n = len(pred) - if class_name is not None: + unit = ('of sampled rows' if run.batch else f'of {run.entities}' + if entry.hop == 0 or entry.col_name == COUNT_COLUMN else + f'of sampled {entry.table_name} rows') + texts = fmt.numbers([v for values in series.values() + for v in values] + list(averages.values())) + n = len(pops) + if run.batch: + axis = ('average predicted value' if run.model_type == REGRESSION else + f'average predicted probability of {class_name or True}') + elif class_name is not None: axis = f'share {class_name}' elif run.model_type == CLASSIFICATION: axis = 'share True' else: axis = 'average ' + (run.target() or 'value') + groups = [] + for i in range(n): + group = dict(label=str(entry.cohorts[i]), + share_text=f'{fmt.percent(pops[i])} {unit}') + for k, (name, values) in enumerate(series.items()): + group.update({ + name: float(values[i]), + name + '_text': texts[k * n + i] + }) + groups.append(group) + share = entry.prediction_score / total_score if total_score else 0 return dict( header=(f'{entry.table_name}.{entry.col_name} · hop {entry.hop} · ' - f'{stype} · variation ' - f'{fmt.percent(entry.prediction_score / total_score)}'), - groups=[ - dict(label=str(entry.cohorts[i]), - share_text=f'{fmt.percent(pops[i])} {unit}', - predicted=float(pred[i]), predicted_text=values[i], - actual=float(actual[i]), actual_text=values[n + i]) - for i in range(n) - ], axis=axis, average_actual=avg_actual, average_predicted=avg_pred, - average_callout=(f'average actual {fmt.number(avg_actual)} · ' - f'average predicted {fmt.number(avg_pred)}'), + f'{stype} · variation {fmt.percent(share)}'), groups=groups, + axis=axis, **{ + 'average_' + name: v + for name, v in averages.items() + }, average_callout=' · '.join(f'average {name} {fmt.number(v)}' + for name, v in averages.items()), gap_callout=gap) @@ -172,6 +193,7 @@ def groups(run: Run) -> Optional[dict[str, Any]]: filter(None, (_link(entries, table, column) for table, column, _ in scores or [])), None) return dict( - title='Column groups', selectable=not scores, columns=views, - default=default or _key(max(entries, key=lambda e: e.prediction_score)), + title='Cohorts' if run.batch else 'Column groups', + selectable=not scores, columns=views, default=default + or _key(max(entries, key=lambda e: e.prediction_score)), coverage=run.explanation_coverage()) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_prediction.py b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py new file mode 100644 index 000000000..980a2eddd --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_prediction.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The batch prediction widget: the training widget's cohort and subgraph +cards over one batch's retained PRED-stage explanations.""" + +from dataclasses import asdict +from typing import Any, Optional + +from pydantic import TypeAdapter + +from kumo.engine._explainability import ( + ExplainabilityView, + _columns, + _subgraphs, +) +from kumo.engine._explainability._run import _TASKS, Run +from kumo.engine._manifest import PredictionManifest +from kumo.engine.explain._prediction_manifest import read_record +from kumo.engine.explain.results import PredictionExplanations + +_REASONS: dict[Optional[str], str] = dict( + disabled='Generation was disabled for this batch.', + unsupported='Not supported for this prediction model.', + failed='Generation did not complete.') + +# Batch cards in order: card id, producer, its label, the card maker and the +# notice when available artifacts yield no card. +_CARDS = (('groups', 'cohorts', 'Cohorts', _columns.groups, + 'no readable column groups were recorded.'), + ('subgraphs', 'samples', 'Subgraphs', _subgraphs.card, + 'no readable saved examples are available.')) + + +def _task(source: PredictionExplanations, kind: str) -> Optional[str]: + """The task type in one producer's first readable artifact.""" + try: + if kind == 'cohorts': + return source.column_analysis().task_type.value + references = source.samples() + except (OSError, ValueError): + return None + for reference in references: + try: + store = reference.available and source.sample( + reference.artifact_id).subgraph_store + except (OSError, ValueError): + continue # an unreadable graph leaves the others usable + if store: + return store.task_type.value + return None + + +def build(explanations: PredictionExplanations, + prediction: Optional[PredictionManifest], + run_id: Optional[str]) -> dict[str, Any]: + """The batch widget's payload.""" + # Live metadata may hold nulls in typed fields: every identity it still + # carries must match the retained record, and only a valid manifest of + # this batch is displayed. + parents = getattr(prediction, 'parents', None) + live = tuple( + getattr(item, 'id', None) + for item in (prediction, getattr(parents, 'model', None), + getattr(parents, 'snapshot', None))) + if run_id not in (None, live[0]): + live, prediction = (None, None, None), None + expected = (run_id or live[0], *live[1:]) + try: + prediction = TypeAdapter(PredictionManifest).validate_python( + asdict(prediction)) if prediction else None + except (TypeError, ValueError): + prediction = None + notices, root = [], explanations.run_root + try: + record = read_record(root) if root else None + except (OSError, ValueError): + record = None + if record and any( + known not in (None, value) + for known, value in zip(expected, (record.run_id, record.model_id, + record.snapshot_id))): + notices.append( + 'Retained explanations do not match this prediction batch.') + record = None + source = (PredictionExplanations.load(root) + if record and root else PredictionExplanations.unavailable()) + availability = source.availability + # A producer status attached to the result outranks retained files. + for kind, state in explanations.availability.items(): + if state.reason in _REASONS: + availability[kind] = state + task = next( + filter(None, (_task(source, kind) for kind in ('cohorts', 'samples') + if availability[kind].available)), None) + provenance = record.result.provenance if record and record.result else None + run = Run(task or '', _TASKS.get(task, 'prediction'), None, {}, {}, {}, + None, False, False, None, source, batch=True, + cohort_rows=provenance and provenance.processed_cohort_rows) + cards = [] + for card_id, kind, label, make, empty in _CARDS: + state = availability[kind] + card = make(run) if state.available else None + if card is not None: + cards.append(dict(id=card_id, **card)) + else: + reason = (empty if state.available else _REASONS.get( + state.reason, 'Retained artifacts are unavailable.')) + notices.append(f'{label}: {reason}') + header: dict[str, Any] = dict(query='Batch prediction explanations') + prediction = prediction or (record.prediction if record else None) + identity = [('Batch', expected[0])] + if prediction is not None: + table, model = prediction.prediction_table, prediction.parents.model + rows = table.scored_rows if table.scored_rows is not None else table.rows + header['summary'] = (f'{rows:,} prediction rows' if rows is not None + else 'Prediction row count was not recorded.') + training = model.parents.training_table if model.parents else None + task = task or (training.task_type if training else None) + identity = [ + ('Batch', prediction.id), ('Model ID', model.id), + ('Snapshot ID', prediction.parents.snapshot.id), + ('Prediction type', + task.replace('_', ' ').capitalize() if task else 'Not recorded'), + ('Architecture', 'Not recorded in batch artifacts'), + ('Fit mode', model.fit_mode), + ('Batch started', prediction.timings + and prediction.timings.started_at) + ] + header['identity'] = [[k, str(v)] for k, v in identity if v] + return _subgraphs._json_value( + dict( + kind='prediction', model_type=run.model_type, entities=run.entities, + header=header, tabs=[ + dict( + id='explanation', label='Explanation', cards=[ + dict(id='status', title='Explanation availability', + messages=notices) + ] * bool(notices) + cards) + ])) + + +class PredictionExplainabilityView(ExplainabilityView): + """Read-only display of one batch's retained PRED-stage explanations.""" + def __init__(self, explanations: PredictionExplanations, + prediction: Optional[PredictionManifest] = None, + run_id: Optional[str] = None) -> None: + self._source = (explanations, prediction, run_id) + self._widget = None + + def _build(self) -> dict[str, Any]: + return build(*self._source) + + def __repr__(self) -> str: + return f'' diff --git a/packages/kumo-engine/kumo/engine/_explainability/_run.py b/packages/kumo-engine/kumo/engine/_explainability/_run.py index a90154783..59502e54a 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_run.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_run.py @@ -29,7 +29,10 @@ from kumoapi.explain.column_analysis import ColumnAnalysisOutput from kumoapi.explain.gradient import GraphGradientScore - from kumo.engine.explain import TrainingExplanations + from kumo.engine.explain import ( + PredictionExplanations, + TrainingExplanations, + ) from kumo.engine.trainer.trainer import TrainingResult CLASSIFICATION = 'classification' @@ -115,9 +118,13 @@ class Run: holdout_truncated: bool full_refit: bool history: Optional[dict[str, Any]] - explanations: 'TrainingExplanations' + explanations: 'TrainingExplanations | PredictionExplanations' table: Optional[str] = None snapshot: Optional[str] = None + # A batch prediction run records no targets; ``cohort_rows`` counts the + # rows its cohort pass processed, when recorded. + batch: bool = False + cohort_rows: Optional[int] = None @property def entities(self) -> str: @@ -169,6 +176,8 @@ def splits(self) -> dict[str, dict[str, Any]]: @cached_property def gradient(self) -> Optional['GraphGradientScore']: + if self.batch: + return None try: return self.explanations.gradient_scores() except (OSError, ValueError): @@ -234,9 +243,17 @@ def metrics_coverage(self) -> Optional[str]: def explanation_coverage(self) -> Optional[str]: """The explanation pass replays a fixed number of batches at a - quarter of the evaluation batch size.""" + quarter of the evaluation batch size; a batch's cohort pass records + the rows it processed.""" from kumo_core.training.runtime import LocalExplainRuntime + if self.batch: + return ('Cohort coverage was not recorded.' + if self.cohort_rows is None else + f'{self.cohort_rows:,} prediction rows processed for ' + 'cohorts. Population shares describe sampled rows with ' + 'recorded features, not the entire batch.') + batch = self._batch return self.covered(None if batch is None else int(LocalExplainRuntime.max_batches) * diff --git a/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py b/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py index 4021568be..50cdca600 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_snapshot.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Script-free notebook snapshot of the default Evaluation views.""" +"""Script-free notebook snapshot of the default Evaluation views, or of a +batch's explanation cards.""" import math from html import escape @@ -21,13 +22,20 @@ def __init__(self, data: dict[str, Any]) -> None: self._marks: list[str] = [] self._text: list[str] = [] self._y = 12 - self._line(f'{data["model_type"].replace("_", " ").title()} evaluation', - size=20, bold=True) - for value in data['header'].values(): - if value: - self._line(value) - cards = next( - (t['cards'] for t in data['tabs'] if t['id'] == 'evaluation'), []) + batch = data.get('kind') == 'prediction' + self._title = 'Batch explanations' if batch else 'Training diagnostics' + self._line( + 'Batch explanations' if batch else + f'{data["model_type"].replace("_", " ").title()} evaluation', + size=20, bold=True) + for key, value in data['header'].items(): + for line in ([f'{k}: {v}' for k, v in value] if key == 'identity' + else [] if batch and key == 'query' else [value]): + if line: + self._line(line) + cards = next((t['cards'] for t in data['tabs'] + if t['id'] == ('explanation' if batch else 'evaluation')), + []) if not cards: self._line('No evaluation diagnostics were recorded.') for card in cards: @@ -36,8 +44,10 @@ def __init__(self, data: dict[str, Any]) -> None: f'') self._y += 8 self._line(card['title'], size=17, bold=True) - view = card['views'][0] - if view['id'] == 'history': + view = card.get('views', [{}])[0] + if card['id'] == 'groups': + self._cohorts(card) + elif view.get('id') == 'history': count = len(view['experiments']) self._line(f'Epoch history · {count} training experiment' + ('s' if count != 1 else '')) @@ -45,13 +55,16 @@ def __init__(self, data: dict[str, Any]) -> None: self._metrics(view['tiles']) elif card['id'] == 'cutoff': self._matrix(card) - elif view['id'] == 'heatmap': + elif view.get('id') == 'heatmap': self._heatmap(view) elif card['id'] in ('scores', 'predicted', 'forecast'): self._comparison(view) - for note in (view.get('coverage'), card.get('coverage')): - if note: - self._line(note) + for line in [ + view.get('coverage'), *card.get('messages', []), + card.get('coverage') + ]: + if line: + self._line(line) def _label(self, text: str, x: int, y: int, size: int = 14, anchor: str = 'start', bold: bool = False) -> None: @@ -157,6 +170,21 @@ def _heatmap(self, view: dict[str, Any]) -> None: f'{view["x"]} {view["columns"][peak_col]}, ' f'{view["y"]} {view["rows"][peak_row]}') + def _cohorts(self, card: dict[str, Any]) -> None: + """The default column's average prediction by group.""" + col = card['columns'][card['default']] + col = col['by_class'][0] if 'by_class' in col else col + self._comparison( + dict( + label=col['header'] + + (f" · class {col['label']}" if 'label' in col else ''), + y=col['axis'], series=['average predicted'], + reference_callout=col['average_callout'], tenths=[ + dict(label=f"{g['label']} · {g['share_text']}", + values=[g['predicted']], texts=[g['predicted_text']]) + for g in col['groups'] + ])) + def _comparison(self, view: dict[str, Any]) -> None: rows = view.get('tenths', view.get('steps', [])) self._line(view['label'] + ' · ' + view['y']) @@ -175,8 +203,13 @@ def _comparison(self, view: dict[str, Any]) -> None: scale = 230 / (high - low) if high > low else 0 zero = 112 - low * scale for row in rows: - self._y += 34 - self._label(row['label'], 16, self._y) + # A label wider than the bars' left margin takes its own line. + if len(row['label']) > 12: + self._line(row['label']) + self._y += 20 + else: + self._y += 34 + self._label(row['label'], 16, self._y) for i, (value, text) in enumerate(zip(row['values'], row['texts'])): y = self._y - 14 + i * 14 if isinstance(value, (int, float)) and math.isfinite(value): @@ -199,7 +232,7 @@ def svg(self, *, note: bool = True) -> str: f'Training diagnostics' + f'role="img">{self._title}' f'' + ''.join(self._marks) + message + '') diff --git a/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py b/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py index 5263fe708..fc3f6c193 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py +++ b/packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py @@ -81,24 +81,30 @@ def _sample(graph: Subgraph, task_type: str) -> Optional[dict[str, Any]]: def card(run: Run) -> Optional[dict[str, Any]]: - """Saved TEST examples; unavailable records never start model work.""" + """Saved TEST or batch examples; unavailable records never start model + work.""" if run.model_type == FORECASTING: return None try: - references = run.explanations.samples() + artifacts = run.explanations._cohort_artifacts('samples') + references = artifacts.samples() except (OSError, ValueError): return None - collector = posixpath.join(str(run.explanations.trial_root), - 'subgraph_collector', 'test') + collector = artifacts.collector_root # Each category keeps the collector's selection order. - categories = mapping( - mapping(read_json(posixpath.join( - collector, 'strategy_mappings.json'))).get('mappings')) + listed = {reference.artifact_id for reference in references} + categories = { + name: [i for i in ids if i in listed] + for name, ids in mapping( + mapping( + read_json(posixpath.join(collector, 'strategy_mappings.json'))). + get('mappings')).items() + } samples = [] for reference in references: try: sample = reference.available and _sample( - run.explanations.sample(reference.artifact_id), run.task_type) + artifacts.sample(reference.artifact_id), run.task_type) except (OSError, ValueError): sample = None if sample: @@ -115,7 +121,11 @@ def card(run: Run) -> Optional[dict[str, Any]]: if not samples: return None return _json_value( - dict(title='Example subgraph', samples=samples, task_type=run.task_type, - categories=list(categories) or ['saved'], - category_samples=categories, - unavailable_count=len(references) - len(samples))) + dict( + title='Example subgraph', samples=samples, task_type=run.task_type, + categories=list(categories) or ['saved'], + category_samples=categories, + unavailable_count=len(references) - len(samples), + coverage=f'{len(samples):,} retained example subgraphs. Examples ' + 'are a sampled subset of this prediction batch.' + if run.batch else None)) diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js index a66b1ba36..06c784fc8 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/shell.js +++ b/packages/kumo-engine/kumo/engine/_explainability/static/shell.js @@ -240,31 +240,33 @@ KX.cards.groups = (parent, card, ctx) => { chart.redraw(); }, {signal: columnCtx.signal}); } - legend(body, [{label: 'average actual', color: ROLE.actual}, - {label: 'average predicted', color: ROLE.model}]); + /* A batch records no actual values, so its groups have one series. */ + const series = [['actual', ROLE.actual], ['predicted', ROLE.model]].filter(([key]) => col['average_' + key] != null); + legend(body, series.map(([key, color]) => ({label: 'average ' + key, color}))); const chart = chartHost(body, (host, width) => { const rows = col.groups.map((grp, i) => ({ label: grp.label, sub: grp.share_text, - values: [grp.actual, grp.predicted], + values: series.map(([key]) => grp[key]), /* The gap callout carries this group's two values, so its bars are not labelled a second time. */ - texts: col.gap_callout && col.gap_callout.index === i ? [] : [grp.actual_text, grp.predicted_text], + texts: col.gap_callout && col.gap_callout.index === i ? [] : series.map(([key]) => grp[key + '_text']), band: col.gap_callout && col.gap_callout.index === i, })); - const vals = col.groups.flatMap(g => [g.actual, g.predicted]) - .concat([col.average_actual, col.average_predicted]).filter(v => v !== null && v !== undefined); + const vals = rows.flatMap(row => row.values) + .concat(series.map(([key]) => col['average_' + key])).filter(v => v !== null && v !== undefined); const lo = Math.min(0, ...vals), hi = Math.max(0, ...vals); const gap = col.gap_callout ? twoLines(col.gap_callout.text) : ''; const lane = gap ? Math.max(...gap.split('\n').map(l => l.length)) * 6.8 + 24 : 0; /* Bars start at zero; each average is its own dashed line in its series' colour, labelled at its top. */ - const res = rowsChart(host, width, {rows, series: [{color: ROLE.actual}, {color: ROLE.model}], + const res = rowsChart(host, width, {rows, series: series.map(([, color]) => ({color})), origin: 0, domain: [lo, hi * 1.04], axis: true, right: 56 + lane, top: 44, xTitle: col.axis, label: card.title}); const notes = S('g', {}, res.svg); const averages = (col.average_callout || '').split(' \u00b7 '); - [[col.average_actual, ROLE.actual], [col.average_predicted, ROLE.model]].forEach(([v, color], k) => { - if (v === null || v === undefined || !averages[k]) return; + series.forEach(([key, color], k) => { + const v = col['average_' + key]; + if (!averages[k]) return; const px = res.x(v); /* Drawn under the bars and their values, like a reference line. */ const line = S('path', {d: `M${px} 36V${res.rowY(rows.length)}`, style: {stroke: color, @@ -300,6 +302,10 @@ KX.cards.groups = (parent, card, ctx) => { return frame; }; +KX.cards.status = (parent, card, ctx) => cardFrame(parent, card, ctx, null, body => { + for (const message of card.messages) H('p', 'kx-coverage', body, message); +}); + /* ------------------------------------------------------------- Shell */ /* `saved` is the UI state the initializer was given (see _render.py): a @@ -315,7 +321,14 @@ function mount(host, data, signal, onDispose) { const head = H('div', 'kx-head', root); if (data.header.query) H('div', 'kx-query', head, data.header.query); for (const [key, value] of Object.entries(data.header)) { - if (key !== 'query' && value) H('div', 'kx-sub', head, value); + if (key === 'identity') { + const list = H('dl', 'kx-identity', head); + for (const [label, text] of value) { + const item = H('div', null, list); + H('dt', null, item, label); + H('dd', null, item, text); + } + } else if (key !== 'query' && value) H('div', 'kx-sub', head, value); } } @@ -361,6 +374,12 @@ function mount(host, data, signal, onDispose) { panels[next].button.focus(); }, {signal}); }); + /* A batch's single tab needs no tab strip; training keeps its tabs. */ + if (panels.length === 1 && data.kind === 'prediction') { + tabs.remove(); + panels[0].panel.setAttribute('role', 'region'); + panels[0].panel.setAttribute('aria-label', panels[0].tab.label); + } select(panels[ctx.state.values.tab] ? ctx.state.values.tab : 0); return root; } diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js b/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js index 0658a8f9a..e3579e529 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js +++ b/packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js @@ -267,24 +267,25 @@ KX.cards.subgraphs = (panel, payload, ctx) => { : 'All scored records have the same summed sensitivity; no relative size ranking.'; move(list.indexOf(sample), list.length); outcome.replaceChildren(); + /* Observed values come last in each list; a batch has none. */ + const observed = ctx.data.kind !== 'prediction'; + const pred = sample.prediction, actual = sample.target; if (['multiclass_classification', 'multilabel_classification'].includes(payload.task_type)) { - classOutcome(outcome, 'Predictions', sample.prediction, nodeOptions.signal); - classOutcome(outcome, 'Ground truth', sample.target, nodeOptions.signal); - } else if (ctx.data.model_type === 'classification') { - outcomeValue('Predicted probability of True', - typeof sample.prediction === 'number' ? (sample.prediction * 100).toFixed(1) + '%' : fmt(sample.prediction)); - outcomeValue('Observed outcome', ['False', 'True'][sample.target] ?? fmt(sample.target)); - } else if (ctx.data.model_type === 'regression') { - outcomeValue('Predicted value', fmt(sample.prediction)); - outcomeValue('Observed value', fmt(sample.target)); - outcomeValue('Prediction − observed', typeof sample.prediction === 'number' && typeof sample.target === 'number' - ? fmt(sample.prediction - sample.target) : 'Not recorded'); + for (const [title, values] of [['Predictions', pred], ['Ground truth', actual]].slice(0, observed ? 2 : 1)) + classOutcome(outcome, title, values, nodeOptions.signal); } else { - const pred = sample.prediction, actual = sample.target; - outcomeValue('Recommended records', fmt(pred?.length)); - outcomeValue('Observed records', fmt(actual?.length)); const targets = Array.isArray(actual) ? new Set(actual.map(String)) : null; - outcomeValue('Matches in recommendations', fmt(targets && Array.isArray(pred) ? pred.filter(x => targets.has(String(x))).length : null)); + const values = ctx.data.model_type === 'classification' ? [ + ['Predicted probability of True', typeof pred === 'number' ? (pred * 100).toFixed(1) + '%' : fmt(pred)], + ['Observed outcome', ['False', 'True'][actual] ?? fmt(actual)], + ] : ctx.data.model_type === 'regression' ? [ + ['Predicted value', fmt(pred)], ['Observed value', fmt(actual)], + ['Prediction − observed', typeof pred === 'number' && typeof actual === 'number' ? fmt(pred - actual) : 'Not recorded'], + ] : [ + ['Recommended records', fmt(pred?.length)], ['Observed records', fmt(actual?.length)], + ['Matches in recommendations', fmt(targets && Array.isArray(pred) ? pred.filter(x => targets.has(String(x))).length : null)], + ]; + for (const [label, value] of values.slice(0, observed ? undefined : 1)) outcomeValue(label, value); } if (sample.time != null) outcomeValue('Prediction date', String(sample.time).slice(0, 10)); const neighbors = new Map(sample.nodes.map(node => [node.id, []])); @@ -441,7 +442,7 @@ KX.cards.subgraphs = (panel, payload, ctx) => { addEdges(detachedSvg, detachedPositions); detached.forEach(n => addNode(n, detachedPositions.get(n.id), detachedSvg)); } - note.textContent = 'Saved, sampled context from the test explanation pass. ' + note.textContent = (payload.coverage || 'Saved, sampled context from the test explanation pass.') + ' ' + (sample.nodes.some(n => n.count != null) ? 'Grouped records summarize additional sampled records. ' : '') + (detached.length ? 'Records shown separately have no saved path to the prediction entity.' : ''); activate(ui.pinned); diff --git a/packages/kumo-engine/kumo/engine/_explainability/static/widget.css b/packages/kumo-engine/kumo/engine/_explainability/static/widget.css index 9d7b0d087..e4a9c2ab5 100644 --- a/packages/kumo-engine/kumo/engine/_explainability/static/widget.css +++ b/packages/kumo-engine/kumo/engine/_explainability/static/widget.css @@ -31,6 +31,12 @@ font-variant-numeric:tabular-nums;-webkit-font-smoothing:antialiased} .kx-query{font:600 20px/1.45 SFMono-Regular,Menlo,Consolas,"Liberation Mono",monospace; color:var(--kx-ink);overflow-wrap:anywhere;max-width:80ch} .kx-sub{margin-top:6px;font-size:13px;color:var(--kx-muted);overflow-wrap:anywhere} +.kx[data-kind="prediction"] .kx-query{font:700 24px/1.3 var(--kx-font)} +.kx[data-kind="prediction"] .kx-sub{font-size:15px;margin:8px 0 20px} +.kx-identity{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,250px),1fr));gap:14px 24px;margin:0;padding:0 0 4px} +.kx-identity>div{min-width:0} +.kx-identity dt{font-size:12px;color:var(--kx-muted);margin-bottom:3px} +.kx-identity dd{font-size:14px;font-weight:600;margin:0;overflow-wrap:anywhere} .kx-tabs{display:flex;flex-wrap:wrap;gap:28px;margin:0 2px 20px} .kx-tab{background:transparent;border:0;border-radius:0;padding:10px 0 12px;cursor:pointer; font-size:18px!important;font-weight:600!important;color:var(--kx-muted)!important;min-height:46px} diff --git a/packages/kumo-engine/kumo/engine/explain/results.py b/packages/kumo-engine/kumo/engine/explain/results.py index 1e76b1044..f549226ed 100644 --- a/packages/kumo-engine/kumo/engine/explain/results.py +++ b/packages/kumo-engine/kumo/engine/explain/results.py @@ -43,6 +43,10 @@ from matplotlib.axes import Axes from matplotlib.figure import Figure + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + @dataclass(frozen=True) class _CohortArtifacts: @@ -247,6 +251,14 @@ class PredictionExplanations(_CohortExplanations): _reason: UnavailableReason = 'unknown' _result: 'PredictionExplanationResult | None' = None + def visualize(self) -> 'PredictionExplainabilityView': + """Display this batch's saved explanations in the notebook widget.""" + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + + return PredictionExplainabilityView(self) + @classmethod def load(cls, run_root: str | Path) -> Self: """Reopen a validated retained run with its recorded producer outcomes. diff --git a/packages/kumo-engine/kumo/engine/trainer/trainer.py b/packages/kumo-engine/kumo/engine/trainer/trainer.py index e1618e294..cc5cb3d21 100644 --- a/packages/kumo-engine/kumo/engine/trainer/trainer.py +++ b/packages/kumo-engine/kumo/engine/trainer/trainer.py @@ -169,6 +169,9 @@ if TYPE_CHECKING: from kumo.engine._explainability import ExplainabilityView + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) from kumo.engine._monitoring import DriftReport, EvaluationResult logger = logging.getLogger(__name__) @@ -371,6 +374,18 @@ class PredictionResult: explanations: PredictionExplanations = field( default_factory=PredictionExplanations.unavailable) + def visualize(self) -> 'PredictionExplainabilityView': + """Display retained batch cohorts and subgraphs without running + prediction.""" + from kumo.engine._explainability._prediction import ( + PredictionExplainabilityView, + ) + + manifest = (self.manifest if self.selection is not None + or self._exported is not None else None) + run_id = self.selection.job_id if self.selection is not None else None + return PredictionExplainabilityView(self.explanations, manifest, run_id) + @property def tracking(self) -> Optional[TrackingRef]: return self._tracking diff --git a/packages/kumo-engine/test/test_prediction_explanations.py b/packages/kumo-engine/test/test_prediction_explanations.py index 4bce84e60..39ee88f5c 100644 --- a/packages/kumo-engine/test/test_prediction_explanations.py +++ b/packages/kumo-engine/test/test_prediction_explanations.py @@ -92,15 +92,15 @@ def _cohorts(root): return output -def _samples(root): +def _samples(root, task_type=TaskType.REGRESSION): collector = f'{root}/subgraph_collector/pred' save_json(EntityMappings(mappings={'first': ['7']}, orig_mappings={'first': ['current-user']}), f'{collector}/strategy_mappings.json') graph = Subgraph(subgraph_store=SubgraphStore( - root_id=7, task_type=TaskType.REGRESSION, - nodes={7: SubgraphNode(7, 'users', 'current-user')}, - edges={}, tables={'users': [7]})) + root_id=7, task_type=task_type, nodes={ + 7: SubgraphNode(7, 'users', 'current-user') + }, edges={}, tables={'users': [7]})) save_json(graph, f'{collector}/subgraph_json/7.json') @@ -327,6 +327,19 @@ def test_sample_availability_requires_recorded_graph(tmp_path, recorded_ids, xai.sample('7') if available: assert xai.sample('8').subgraph_store.root_id == 7 + # The widget shows only allowlisted samples, with their predictions. + filesystem.copy(f'{graphs}/8.json', f'{graphs}/99.json') + save_json( + EntityMappings( + mappings={'first': ['99', *recorded_ids]}, + orig_mappings={'first': ['x'] * (len(recorded_ids) + 1)}), + f'{collector}/strategy_mappings.json') + save_json({'prediction': .99}, f'{collector}/output/8.json') + card = xai.visualize().data['tabs'][0]['cards'][-1] + assert [sample['id'] for sample in card['samples']] == ['8'] + assert card['category_samples'] == {'first': list(recorded_ids)} + assert card['unavailable_count'] == int('7' in recorded_ids) + assert card['samples'][0]['prediction'] == .99 else: with pytest.raises(ExplanationUnavailableError): xai.sample('8') @@ -354,6 +367,7 @@ def test_prediction_run_and_export_absence(tmp_path): PredictionExplanationResult(ProducerOutcome('failed', 'cohort failed'), ProducerOutcome('failed', 'collector failed')), replace(_complete(), samples=ProducerOutcome('failed', 'collector failed')), + replace(_complete(), cohorts=ProducerOutcome('failed', 'cohort failed')), ]) def test_retained_producer_status_matches_live(tmp_path, outcomes): _cohorts(f'{tmp_path}/xai') @@ -363,6 +377,11 @@ def test_retained_producer_status_matches_live(tmp_path, outcomes): result=outcomes) reopened = PredictionExplanations.load(tmp_path) assert reopened.availability == live.availability + for view in (live.visualize(), reopened.visualize()): + ids = _ids(view) + assert ids[0] == 'status' + assert ('groups' in ids) == (outcomes.cohorts.status == 'complete') + assert ('subgraphs' in ids) == (outcomes.samples.status == 'complete') if outcomes.cohorts.status == 'complete': assert reopened.column_analysis().cohorts else: @@ -423,6 +442,16 @@ def test_copied_prediction_run_resolves_new_root(tmp_path): reopened = PredictionExplanations.load(copied) assert reopened.column_analysis().cohorts assert reopened.sample('7').subgraph_store.root_id == 7 + assert _ids(reopened.visualize()) == ['groups', 'subgraphs'] + for manifest in (None, '{}'): + path = copied / 'xai/manifest.json' + path.unlink() if manifest is None else path.write_text(manifest) + for outcome in (None, _complete()): + assert _ids( + PredictionExplanations.from_run( + copied, enabled=True, + result=outcome).visualize()) == ['status'] + assert _ids(PredictionExplanations.unavailable().visualize()) == ['status'] def test_prediction_finishes_before_replay(prediction, monkeypatch): @@ -471,3 +500,169 @@ def test_status_write_failure_preserves_prediction(prediction, monkeypatch, assert state.reason == ('failed' if enabled else 'disabled') if enabled: assert 'disk full' in state.generation_failure + view = result.visualize() + assert _ids(view) == ['status'] + assert view.data['header']['summary'] == '2 prediction rows' + assert view.data['header']['identity'][:3] == [[ + 'Batch', result.manifest.id + ], ['Model ID', 'model-id'], ['Snapshot ID', 'snapshot']] + if enabled: + # The failed status outranks retained files, a record of another + # batch stays hidden without the live manifest, and a matching + # record keeps the failure. + root = result.explanations.run_root + _cohorts(f'{root}/xai') + _samples(f'{root}/xai') + saved = _record(root, result=_complete()) + assert _ids(result.visualize()) == ['status'] + live = result.manifest + result._manifest_path.unlink() + view = replace( + result, explanations=PredictionExplanations.from_run( + root, enabled=True, result=_complete())).visualize() + assert _ids(view) == ['status'] + assert view.data['header']['identity'] == [['Batch', result.id]] + write_manifest(result._manifest_path, live) + write_record( + root, + saved.model_copy(update={ + 'run_id': live.id, + 'prediction': live + })) + assert 'Generation did not complete.' in result.visualize( + )._snapshot.plain() + + +def _ids(view): + return [card['id'] for card in view.data['tabs'][0]['cards']] + + +@pytest.mark.parametrize('task', [ + TaskType.BINARY_CLASSIFICATION, TaskType.REGRESSION, + TaskType.MULTICLASS_CLASSIFICATION, TaskType.MULTILABEL_CLASSIFICATION +]) +@pytest.mark.parametrize('null_field', [ + None, ('prediction_table', ), ('parents', ), ('parents', 'model'), + ('parents', 'snapshot') +]) +def test_prediction_widget_shows_cohorts_and_subgraphs(tmp_path, task, + null_field): + from kumo_core.train.artifact_manifest import PredictionArtifactType + + from kumo.engine.trainer.trainer import PredictionResult + + cohort = _cohorts(f'{tmp_path}/xai') + cohort.task_type = task + labeled = task in (TaskType.MULTICLASS_CLASSIFICATION, + TaskType.MULTILABEL_CLASSIFICATION) + if labeled: + cohort.cohorts[0].predictions = {'blue': [.2, .8], 'red': [.8, .2]} + save_json(cohort, + f'{tmp_path}/xai/column_analysis/pred/column_analysis.json') + _samples(f'{tmp_path}/xai', task) + record = _record(tmp_path, result=_complete()) + xai = PredictionExplanations.load(tmp_path) + selection = SimpleNamespace(job_id=record.run_id, + pointer_path=str(tmp_path / 'current.json')) + result = PredictionResult(selection, PredictionArtifactType.PREDICTIONS, + explanations=xai) + # Live metadata with a null typed field falls back to the saved record. + metadata = record.prediction.to_dict() + if null_field: + parent = metadata if len(null_field) == 1 else metadata[null_field[0]] + parent[null_field[-1]] = None + result._manifest_path.write_text(json.dumps(metadata)) + view = result.visualize() + data = view.data + assert data['kind'] == 'prediction' and len(data['tabs']) == 1 + assert data['header']['summary'] == '2 prediction rows' + assert dict( + data['header']['identity'])['Prediction type'] == (task.value.replace( + '_', ' ').capitalize()) + assert _ids(view) == ['groups', 'subgraphs'] + groups, graphs = data['tabs'][0]['cards'] + col = groups['columns'][groups['default']] + col = col['by_class'][0] if labeled else col + assert [g['predicted'] for g in col['groups']] == [.2, .8] + assert 'actual' not in col['groups'][0] and 'average_actual' not in col + assert graphs['samples'][0]['nodes'][0]['value'] == 'current-user' + json.dumps(data, allow_nan=False) + svg = view._repr_svg_() + assert 'Batch explanations' in svg and 'Batch: retained-run' in svg + assert 'young' in svg and 'average actual' not in svg and 'current-user' not in svg + assert 'To activate the widget, rerun this cell.' in view._snapshot.plain() + assert 'kumo-explainability' in view.html() + assert xai.visualize().data == data + if null_field == ('prediction_table', ): + # A retained record of another model under this batch ID. + other = replace(record.prediction, parents=replace( + record.prediction.parents, model=stub('model', 'other-model'))) + write_record(str(tmp_path), record.model_copy(update={ + 'model_id': 'other-model', 'prediction': other})) + view = result.visualize() + assert _ids(view) == ['status'] + assert view.data['tabs'][0]['cards'][0]['messages'][0] == ( + 'Retained explanations do not match this prediction batch.') + + +@pytest.mark.parametrize( + 'state', ['disabled', 'samples_only', 'unreadable_graph', 'missing', + 'malformed']) +@pytest.mark.parametrize('attached', ['loaded', 'unknown', 'complete']) +def test_prediction_widget_respects_producer_status(tmp_path, state, attached): + """Status precedence and unreadable artifacts; producer outcomes are in + test_retained_producer_status_matches_live.""" + _cohorts(f'{tmp_path}/xai') + _samples(f'{tmp_path}/xai') + result = _complete() + if state in ('samples_only', 'unreadable_graph'): + result = replace(result, cohorts=ProducerOutcome('unsupported')) + _samples(f'{tmp_path}/xai', TaskType.LINK_PREDICTION) + if state == 'unreadable_graph': + # An unreadable graph ordered before the readable one. + collector = tmp_path / 'xai/subgraph_collector/pred' + save_json(EntityMappings(mappings={'first': ['3', '7']}, + orig_mappings={'first': ['x', 'y']}), + str(collector / 'strategy_mappings.json')) + (collector / 'subgraph_json/3.json').write_text('not json') + result = replace( + result, samples=replace( + result.samples, artifacts=( + *result.samples.artifacts, + 'subgraph_collector/pred/subgraph_json/3.json')), + provenance=replace(result.provenance, selected_stage_ids=(0, 1), + collector_ids=('3', '7'))) + _record(tmp_path, enabled=state != 'disabled', + result=None if state == 'disabled' else result) + path = tmp_path / 'xai/column_analysis/pred/column_analysis.json' + if state == 'missing': + path.unlink() + elif state == 'malformed': + path.write_text('{}') + source = (PredictionExplanations.load(tmp_path) + if attached == 'loaded' else PredictionExplanations.from_run( + tmp_path, enabled=True, + result=_complete() if attached == 'complete' else None)) + view = source.visualize() + ids = _ids(view) + assert ids[0] == 'status' and 'groups' not in ids + assert ('subgraphs' in ids) == (state != 'disabled') + assert ('1 retained example subgraphs.' + in view._snapshot.plain()) == ('subgraphs' in ids) + + +def test_prediction_widget_keeps_all_zero_variation_cohorts(tmp_path): + artifact = _cohorts(f'{tmp_path}/xai') + cohort = artifact.cohorts[0] + cohort.cohorts = [f'cohort-{i}' for i in range(12)] + cohort.populations = [1 / 12] * 12 + cohort.predictions = [0.] * 12 + cohort.prediction_score = 0. + save_json(artifact, + f'{tmp_path}/xai/column_analysis/pred/column_analysis.json') + _record(tmp_path, result=_complete()) + view = PredictionExplanations.load(tmp_path).visualize() + column = next(iter(view.data['tabs'][0]['cards'][-1]['columns'].values())) + assert len(column['groups']) == 12 and column['average_predicted'] == 0. + assert 'cohort-11' in view._repr_svg_() + json.dumps(view.data, allow_nan=False) diff --git a/packages/kumo-engine/test/widget_exports.cjs b/packages/kumo-engine/test/widget_exports.cjs index 934fbcc42..25cd1b0fb 100644 --- a/packages/kumo-engine/test/widget_exports.cjs +++ b/packages/kumo-engine/test/widget_exports.cjs @@ -8,7 +8,8 @@ // setting the HTML export or the print copy loses, or a missing printed view. // // Needs retained classification, forecasting and regression fits under -// KUMO_EXPLAINABILITY_RUNS/, a Python environment that +// KUMO_EXPLAINABILITY_RUNS/, optionally a retained batch prediction run +// with cohorts and subgraphs at KUMO_BATCH_EXPLANATIONS, a Python environment that // imports this checkout (PYTHON, default python3), and Playwright with its // browsers, from NODE_PATH or the locked package next to this file: // npm ci --prefix packages/kumo-engine/test/snowflake_notebook/browser @@ -23,17 +24,20 @@ const {chromium, firefox, webkit} = (() => { })(); const runs = process.env.KUMO_EXPLAINABILITY_RUNS; assert.ok(runs, 'Set KUMO_EXPLAINABILITY_RUNS to the retained runs (/scratch/trainers/*/runs/*-fit-*).'); -function widget(task, dir) { +function widget(task, dir, source = path.join(runs, task)) { const file = path.join(dir, task + '.html'); execFileSync(process.env.PYTHON || 'python3', ['-c', `import glob, sys from dataclasses import replace -from kumo.engine import Trainer +from kumo.engine import PredictionExplanations, Trainer from kumo.engine.explain import TrainingExplanations -run, = glob.glob(sys.argv[1] + '/scratch/trainers/*/runs/*-fit-*') -result = Trainer.load(run + '/published-model', accelerator='cpu').result -view = replace(result, explanations=TrainingExplanations.load(run)).visualize() +if '${task}' == 'batch': + view = PredictionExplanations.load(sys.argv[1]).visualize() +else: + run, = glob.glob(sys.argv[1] + '/scratch/trainers/*/runs/*-fit-*') + result = Trainer.load(run + '/published-model', accelerator='cpu').result + view = replace(result, explanations=TrainingExplanations.load(run)).visualize() open(sys.argv[2], 'w', encoding='utf-8').write('' + view.html())`, - path.join(runs, task), file], {cwd: path.join(__dirname, '..'), stdio: 'inherit'}); + source, file], {cwd: path.join(__dirname, '..'), stdio: 'inherit'}); return pathToFileURL(file).href; } async function plots(page, selector) { @@ -59,7 +63,8 @@ async function print(page) { async function save(page, filename) { const waiting = page.waitForEvent('download'); await page.getByRole('button', {name: 'Export HTML', exact: true}).click(); - await (await waiting).saveAs(filename); + const download = await waiting; await download.saveAs(filename); + return download.suggestedFilename(); } async function alter(filename, change) { const html = await fs.readFile(filename, 'utf8'); @@ -71,6 +76,7 @@ async function alter(filename, change) { const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'kumo-exports-')), results = []; try { const [training, forecast, regression] = ['classification', 'forecasting', 'regression'].map(task => widget(task, tmp)); + const batch = process.env.KUMO_BATCH_EXPLANATIONS && widget('batch', tmp, process.env.KUMO_BATCH_EXPLANATIONS); for (const engine of [chromium, firefox, webkit]) { const browser = await engine.launch(), context = await browser.newContext({acceptDownloads: true}); const errors = []; context.on('page', page => page.on('pageerror', error => errors.push(error.message))); @@ -128,6 +134,27 @@ async function alter(filename, change) { await alter(saved, payload => { payload.data.tabs = payload.data.tabs.filter(tab => tab.id === 'evaluation'); payload.saved.tab = 0; }); await reopened.goto(pathToFileURL(saved).href); const single = await print(reopened); assert.equal(await single.title(), 'Kumo · Evaluation'); await single.close(); + if (batch) { + // One panel without tabs; the selected column and pinned record survive both exports. + await page.goto(batch); + assert.equal(await page.getByRole('tab').count(), 0); + await page.locator('[data-card="groups"] select').first().selectOption({index: 1}); + await page.locator('g.kx-sg-node[data-root="false"]').first().dispatchEvent('click'); + const column = await page.locator('[data-card="groups"] select').first().inputValue(); + // Axis ticks follow the page width, which the saved page changes. + const bars = page => page.locator('[data-card="groups"] .kx-chart svg text:not(.kx-t-axis)').allTextContents(); + const cohort = await bars(page); + const details = await page.locator('.kx-sg-detail').textContent(); + assert.equal(await save(page, saved), 'kumo-prediction-explanations.html'); + await reopened.goto(pathToFileURL(saved).href); + assert.equal(await reopened.locator('[data-card="groups"] select').first().inputValue(), column); + assert.deepEqual(await bars(reopened), cohort); + assert.equal(await reopened.locator('.kx-sg-detail').textContent(), details); + const popup = await print(page); + assert.equal(await popup.title(), 'Kumo · Explanation'); + await plots(popup, '[data-card="groups"] .kx-chart svg'); await plots(popup, '.kx-sg-canvas > svg'); + assert.equal(await popup.locator('.kx-sg-detail').textContent(), details); await popup.close(); + } await page.goto(forecast); await save(page, saved); for (const values of [[-10, -5], [-10, 5]]) { await alter(saved, payload => { const totals = payload.data.tabs.flatMap(t => t.cards).find(c => c.id === 'forecast').views.find(v => v.id === 'totals');