From 0cde40b5c91080ededf628c9ff459b34eb92516a Mon Sep 17 00:00:00 2001 From: devanj08 Date: Thu, 19 Nov 2026 06:47:22 +0000 Subject: [PATCH] Add training diagnostics and explainability widget --- packages/kumo-engine/explainability.md | 22 +- .../kumo/engine/_explainability/__init__.py | 100 ++ .../kumo/engine/_explainability/_cards.py | 81 + .../engine/_explainability/_classification.py | 347 ++++ .../kumo/engine/_explainability/_columns.py | 292 ++++ .../kumo/engine/_explainability/_coverage.py | 108 ++ .../kumo/engine/_explainability/_files.py | 195 +++ .../engine/_explainability/_forecasting.py | 166 ++ .../kumo/engine/_explainability/_format.py | 75 + .../kumo/engine/_explainability/_header.py | 114 ++ .../kumo/engine/_explainability/_holdout.py | 32 + .../kumo/engine/_explainability/_metrics.py | 545 +++++++ .../engine/_explainability/_regression.py | 372 +++++ .../kumo/engine/_explainability/_render.py | 67 + .../kumo/engine/_explainability/_run.py | 87 + .../kumo/engine/_explainability/_snapshot.py | 225 +++ .../kumo/engine/_explainability/_subgraphs.py | 184 +++ .../engine/_explainability/static/charts.js | 721 +++++++++ .../_explainability/static/classification.js | 340 ++++ .../_explainability/static/epoch-history.js | 118 ++ .../engine/_explainability/static/exports.js | 219 +++ .../_explainability/static/forecasting.js | 55 + .../_explainability/static/regression.js | 241 +++ .../engine/_explainability/static/shell.js | 376 +++++ .../_explainability/static/subgraphs.js | 741 +++++++++ .../engine/_explainability/static/widget.css | 304 ++++ packages/kumo-engine/kumo/engine/_widget.py | 20 +- .../kumo/engine/trainer/trainer.py | 14 + packages/kumo-engine/pyproject.toml | 1 + .../kumo-engine/test/test_explainability.py | 1426 +++++++++++++++++ packages/kumo-engine/test/widget_exports.cjs | 143 ++ .../kumo-engine/test/widget_validation.md | 30 + 32 files changed, 7757 insertions(+), 4 deletions(-) create mode 100644 packages/kumo-engine/kumo/engine/_explainability/__init__.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_cards.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_classification.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_columns.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_coverage.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_files.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_forecasting.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_format.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_header.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_holdout.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_metrics.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_regression.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_render.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_run.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_snapshot.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/_subgraphs.py create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/charts.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/classification.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/epoch-history.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/exports.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/forecasting.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/regression.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/shell.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/subgraphs.js create mode 100644 packages/kumo-engine/kumo/engine/_explainability/static/widget.css create mode 100644 packages/kumo-engine/test/test_explainability.py create mode 100644 packages/kumo-engine/test/widget_exports.cjs create mode 100644 packages/kumo-engine/test/widget_validation.md diff --git a/packages/kumo-engine/explainability.md b/packages/kumo-engine/explainability.md index 4ddc13287..edfb91e9f 100644 --- a/packages/kumo-engine/explainability.md +++ b/packages/kumo-engine/explainability.md @@ -5,6 +5,26 @@ It does not retrain a model. Batch prediction can separately generate diagnostic for its current inputs with `Trainer.predict(..., explanations=True)`, described below. Explanation generation can fail without invalidating training or prediction. +## Notebook widgets + +Display `training_result.visualize()` to inspect training Evaluation and Explanation views. +Under **Evaluation**, choose **Metrics → Epoch history** to compare training and validation scores across the experiments that preceded model selection. +The tuning metric opens first when recorded. Select another metric or toggle individual experiments to compare their curves on the same scale. +Each point represents a recorded epoch. Missing scores leave gaps, and epochs from the later refit are excluded. +The widget reads `metric-history.json` from the published bundle or `auto_train_progress.json` from the retained run. The view appears when either source contains training or validation series. + +Use **Export PDF** to print every chart view on the currently open page, including All metrics, Epoch history and Against baselines where available, then choose **Save as PDF** in the browser dialog. +The export preserves the selected explanation and chart controls. +The other tab is excluded. +If the notebook blocks pop-ups, open the downloaded print page to save it as PDF. +**Export HTML** downloads the complete interactive widget with its current chart and customer selections, including the saved data needed to explore other records offline. +Neither export requires a running kernel after saving. + +Call `.html()` on the returned view for self-contained HTML, or `.as_widget()` to request the optional anywidget transport explicitly. +The saved notebook snapshot includes the model summary, metrics and default Evaluation charts, followed by “To activate the widget, rerun this cell.” +Individual subgraphs are omitted from that static snapshot. +Opening the widget only reads retained artifacts; it never starts training or prediction. + ## Access the selected training run Capture the result returned by your existing training call: @@ -126,7 +146,7 @@ aggregated nodes. A reference can exist for a missing graph; check `available`. identifiers organized by sampling category; they are not interchangeable with collector IDs. Without those mappings, arbitrary original-key lookup is not available. These bounded samples are training context, not explanations for -every entity. The API does not render interactive graphs. +every entity. The notebook widget renders these saved graphs interactively. ## Availability and validation diff --git a/packages/kumo-engine/kumo/engine/_explainability/__init__.py b/packages/kumo-engine/kumo/engine/_explainability/__init__.py new file mode 100644 index 000000000..8ebf0c5ad --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/__init__.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Post-training explainability widget for one trained model. + +The data side reads the Kumo files a training run leaves on disk and computes +every number; the front end in ``static/`` only draws them. Open it with +``TrainingResult.visualize()``. +""" + +from dataclasses import replace +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from kumo.engine import _widget +from kumo.engine._explainability import _cards, _files, _render, _run +from kumo.engine._explainability._snapshot import Snapshot + +if TYPE_CHECKING: + from kumo.engine.trainer.trainer import TrainingResult + + +class ExplainabilityView: + """The explainability widget of one trained model. + + Displaying it in a notebook cell renders the widget; ``html()`` returns + the same self-contained HTML for saving to a file. + """ + def __init__(self, files: _files.RunFiles) -> None: + self._files = files + self._widget: Any = None + + @cached_property + def data(self) -> dict[str, Any]: + """Every number and text the widget draws.""" + return _cards.build(_run.load(self._files)) + + @property + def model_type(self) -> str: + return self.data['model_type'] + + @cached_property + def _snapshot(self) -> Snapshot: + return Snapshot(self.data) + + def html(self) -> str: + """The widget as one self-contained HTML fragment.""" + return _widget.render_widget( + **_render.specification(self.data, self._snapshot.html())) + + def as_widget(self) -> Any: + """Return a cached anywidget view, for hosts that need one.""" + if self._widget is None or getattr(self._widget, 'comm', 1) is None: + self._widget = _widget.render_anywidget( + fallback_mime=self._snapshot.mime, + **_render.specification(self.data, self._snapshot.html())) + return self._widget + + def _repr_html_(self) -> str: + return self.html() + + def _repr_svg_(self) -> str: + return self._snapshot.svg() + + def _repr_pretty_(self, printer: Any, cycle: bool) -> None: + printer.text(self._snapshot.plain()) + + def _ipython_display_(self) -> None: + if not _widget.is_snowflake_workspace(): + raise NotImplementedError + from IPython.display import display + + try: + widget = self.as_widget() + except ImportError: + bundle = self._snapshot.mime() + hint = ('Interactive display requires ipywidgets and anywidget ' + 'version 0.9 or later in the 0.x series.') + bundle['text/html'] += f'

{hint}

' + bundle['text/plain'] += '\n' + hint + display(bundle, raw=True) + else: + display(widget) + + def __repr__(self) -> str: + return f'' + + +def for_result(result: 'TrainingResult') -> ExplainabilityView: + files = _files.locate(str(result.bundle_dir), result.explanations.run_root, + result.explanations.trial_root) + return ExplainabilityView( + replace( + files, holdout_path=str(result.holdout_path) if result.holdout_path + else None, holdout_truncated=result.holdout_truncated)) + + +def open_run(run_root: str) -> ExplainabilityView: + """The widget for a retained run whose bundle is in the run directory.""" + return ExplainabilityView(_files.open_run(run_root)) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_cards.py b/packages/kumo-engine/kumo/engine/_explainability/_cards.py new file mode 100644 index 000000000..963704944 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_cards.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The cards each model type shows, in order, on its two tabs. + +A card whose inputs are missing is left out; a tab with no card is left out. +""" + +from collections.abc import Callable +from typing import Any, Optional + +from kumo.engine._explainability import ( + _classification, + _columns, + _metrics, + _subgraphs, +) +from kumo.engine._explainability._forecasting import forecast_card +from kumo.engine._explainability._header import header +from kumo.engine._explainability._regression import predicted_card +from kumo.engine._explainability._run import ( + CLASS_LABELED, + CLASSIFICATION, + FORECASTING, + LINK_PREDICTION, + REGRESSION, + Run, +) + +Card = Callable[[Run], Optional[dict[str, Any]]] + +EVALUATION: dict[str, list[tuple[str, Card]]] = { + CLASSIFICATION: [ + ('metrics', _metrics.card), + ('cutoff', _classification.cutoff_card), + ('scores', _classification.scores_card), + ], + REGRESSION: [('metrics', _metrics.card), ('predicted', predicted_card)], + FORECASTING: [('metrics', _metrics.card), ('forecast', forecast_card)], + LINK_PREDICTION: [('metrics', _metrics.card)], +} + +EXPLANATION: dict[str, list[tuple[str, Card]]] = { + CLASSIFICATION: [('subgraphs', _subgraphs.card), + ('importance', _columns.importance), + ('groups', _columns.groups)], + REGRESSION: [('subgraphs', _subgraphs.card), + ('importance', _columns.importance), + ('groups', _columns.groups)], + FORECASTING: [('importance', _columns.importance)], + LINK_PREDICTION: [('subgraphs', _subgraphs.card), + ('importance', _columns.importance)], +} + + +def _cards(run: Run, spec: list[tuple[str, Card]]) -> list[dict[str, Any]]: + out = [] + for card_id, build in spec: + if run.task_type in CLASS_LABELED and card_id in ('cutoff', 'scores'): + continue + card = build(run) + if card is not None: + out.append({'id': card_id, **card}) + return out + + +def build(run: Run) -> dict[str, Any]: + """Every number and text the widget draws for one run.""" + tabs = [] + for tab_id, label, spec in (('evaluation', 'Evaluation', EVALUATION), + ('explanation', 'Explanation', EXPLANATION)): + cards = _cards(run, spec[run.model_type]) + if cards: + tabs.append({'id': tab_id, 'label': label, 'cards': cards}) + return { + 'model_type': run.model_type, + 'task_type': run.task_type, + 'entities': run.entities, + 'header': header(run), + 'tabs': tabs + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_classification.py b/packages/kumo-engine/kumo/engine/_explainability/_classification.py new file mode 100644 index 000000000..79db9eb38 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_classification.py @@ -0,0 +1,347 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Binary classification cards: Classification cutoff and Scores against +outcomes, both from the published model's holdout predictions. + +The cutoff card carries every count and text for the 101 grid cutoffs of +both positive classes, so the front end only selects what the slider names. +""" + +import math +from typing import Any, Optional + +import numpy as np + +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._holdout import ( + equal_count_groups, + finite, +) +from kumo.engine._explainability._run import Run + +GRID = [round(i / 100, 2) for i in range(101)] +DEFAULT_CUTOFF = 50 +SCORE_BIN = 0.02 +CLASSES = ('True', 'False') + + +def _ratio(numerator: int, denominator: int, empty: float = 1.0) -> float: + return numerator / denominator if denominator else empty + + +def _text(value: float) -> str: + """A rate in ``[0, 1]`` to three significant figures, zero included.""" + if not math.isfinite(value): + return 'N/A' + return '0.00' if value == 0 else fmt.number(value) + + +def counts(scores: np.ndarray, positive: np.ndarray) -> list[list[int]]: + """``[tp, fp, fn, tn]`` at each grid cutoff, counting ``score > t``.""" + out = [] + for t in GRID: + flagged = scores > t + out.append([ + int((flagged & positive).sum()), + int((flagged & ~positive).sum()), + int((~flagged & positive).sum()), + int((~flagged & ~positive).sum()), + ]) + return out + + +def preset(rows: list[list[int]], share: float) -> int: + """Grid index whose flagged share is closest to the positive share.""" + n = sum(rows[0]) + gaps = [abs((tp + fp) / n - share) for tp, fp, _, _ in rows] + return gaps.index(min(gaps)) + + +def _curves(scores: np.ndarray, positive: np.ndarray) -> dict[str, Any]: + n = len(scores) + order = np.argsort(-scores, kind='mergesort') + ranked, hits = scores[order], positive[order] + ends = np.r_[np.where(np.diff(ranked))[0], n - 1] + tps = np.cumsum(hits)[ends].astype(float) + fps = (ends + 1) - tps + n_pos, n_neg = tps[-1], fps[-1] + out: dict[str, Any] = {} + if n_pos > 0: + recall = tps / n_pos + precision = tps / (tps + fps) + ap = float(np.sum(np.diff(np.r_[0.0, recall]) * precision)) + out['ap'] = ap + if n_neg > 0: + # Trapezoid area, which counts tied scores half, for runs whose + # published AUROC is not stored. + fpr = np.r_[0.0, fps / n_neg] + tpr = np.r_[0.0, recall] + out['auroc'] = float(np.sum( + np.diff(fpr) * (tpr[1:] + tpr[:-1]) / 2)) + return out + + +def _at(row: list[int], n: int, name: str, cutoff: float) -> dict[str, Any]: + tp, fp, fn, tn = row + precision = _ratio(tp, tp + fp) + recall = _ratio(tp, tp + fn, empty=0.0) + accuracy = _ratio(tp + tn, n) + flagged = (tp + fp) / n + fpr = _ratio(fp, fp + tn) + f1 = _ratio(2 * tp, 2 * tp + fp + fn, empty=0.0) + t = fmt.cutoff(cutoff) + # The score histogram's axis is the first class's probability, so its + # cutoff line is labelled with the value on that axis (W5). + on_axis = fmt.cutoff(cutoff if name == CLASSES[0] else 1 - cutoff) + recall_share = fmt.percent(recall) if math.isfinite(recall) else 'N/A' + return { + 'counts': + row, + 'cells': [f'{fmt.count(c)} · {fmt.percent(c / n)}' for c in row], + 'strip': [ + { + 'label': 'precision', + 'text': _text(precision) + }, + { + 'label': 'recall', + 'text': _text(recall) + }, + { + 'label': 'accuracy', + 'text': _text(accuracy) + }, + { + 'label': 'flagged', + 'text': fmt.percent(flagged) + }, + { + 'label': 'F1', + 'text': _text(f1) + }, + ], + 'flagged': + flagged, + 'precision': + finite(precision), + 'recall': + finite(recall), + 'fpr': + finite(fpr), + 'matrix_callout': + f'{fmt.percent(flagged)} flagged at cutoff {t}', + 'roc_callout': + f'cutoff {t} · TPR {_text(recall)} · FPR {_text(fpr)}', + 'pr_callout': + f'cutoff {t} · precision {_text(precision)} · recall {_text(recall)}', + 'gain_callout': + f'{fmt.percent(flagged)} flagged · {recall_share} of {name} captured', + 'line_callout': + f'cutoff {on_axis}', + } + + +def _class(run: Run, name: str, auroc: Optional[float]) -> dict[str, Any]: + holdout = run.holdout + assert holdout is not None + target = holdout['TARGET'].astype(bool).to_numpy() + positive = target if name == 'True' else ~target + scores = holdout[f'{name}_PROB'].to_numpy(dtype=float) + n = len(scores) + share = float(positive.mean()) + rows = counts(holdout['True_PROB'].to_numpy(dtype=float), target) + if name == 'False': + # Match ClassificationEvaluationResponse's reversed class counts. + rows = [[tn, fn, fp, tp] for tp, fp, fn, tn in reversed(rows)] + curves = _curves(scores, positive) + if auroc is None: + auroc = curves.get('auroc') + other = 'False' if name == 'True' else 'True' + cutoffs = [_at(row, n, name, t) for row, t in zip(rows, GRID)] + # UI curves use the same threshold records as the cutoff readout. + for key, x, y in (('roc', 'fpr', 'recall'), ('pr', 'recall', 'precision'), + ('gain', 'flagged', 'recall')): + curves[key] = [[round(point[x], 2), + round(point[y], 2)] for point in cutoffs] + return { + 'name': name, + 'other': other, + 'share': share, + 'preset': preset(rows, share), + 'cutoffs': cutoffs, + 'roc': curves.get('roc'), + 'pr': curves.get('pr'), + 'gain': curves.get('gain'), + 'auroc': auroc, + 'auroc_callout': + None if auroc is None else f'AUROC {fmt.number(auroc)}', + 'ap': curves.get('ap'), + 'ap_callout': + None if 'ap' not in curves else f'AP {fmt.number(curves["ap"])}', + 'base_rate_callout': f'base rate {fmt.number(share)}', + 'ideal_callout': f'ideal bend {fmt.percent(share)}', + 'matrix_rows': [f'actual {name}', f'actual {other}'], + 'matrix_columns': [f'predicted {name}', f'predicted {other}'], + } + + +def cutoff_card(run: Run) -> Optional[dict[str, Any]]: + holdout = run.holdout + if holdout is None or len(holdout) == 0: + return None + published = run.metrics.get('test_refit_auroc', + run.metrics.get('test_auroc')) + classes = [_class(run, name, published) for name in CLASSES] + return { + 'title': + 'Classification cutoff', + 'grid': + GRID, + 'grid_text': [fmt.cutoff(t) for t in GRID], + 'default': + DEFAULT_CUTOFF, + 'classes': + classes, + 'views': [ + { + 'id': 'matrix', + 'label': 'Confusion matrix' + }, + { + 'id': 'roc', + 'label': 'ROC', + 'x': 'false positive rate', + 'y': 'true positive rate', + 'diagonal': 'chance' + }, + { + 'id': 'pr', + 'label': 'Precision-recall', + 'x': 'recall', + 'y': 'precision' + }, + { + 'id': 'gain', + 'label': 'Cumulative gain', + 'x': f'share of {run.entities} flagged', + 'y': 'share of positives captured', + 'diagonal': 'random' + }, + ], + 'coverage': + run.coverage.holdout_line(), + } + + +def _tenths(run: Run, scores: np.ndarray, target: np.ndarray) -> dict[str, Any]: + holdout = run.holdout + assert holdout is not None + group = equal_count_groups(scores, holdout['ENTITY'].to_numpy()) + occupied = np.unique(group) + rows = [] + for k in occupied: + mask = group == k + rows.append((int(mask.sum()), float(scores[mask].mean()), + float(target[mask].mean()))) + gaps = [abs(p - o) for _, p, o in rows] + worst = gaps.index(max(gaps)) + share = float(target.mean()) + _, p, o = rows[worst] + return { + 'id': + 'tenths', + 'label': + 'By score tenth', + 'x': + 'score tenth, lowest to highest', + 'y': + 'share True', + 'series': ['average score', 'observed share True'], + 'tenths': [{ + 'label': str(occupied[k] + 1), + 'rows': n, + 'score': p_, + 'score_text': fmt.number(p_), + 'observed': o_, + 'observed_text': fmt.number(o_), + } for k, (n, p_, o_) in enumerate(rows)], + 'reference': + share, + 'reference_callout': + f'all {run.entities}: {fmt.number(share)}', + 'highlight': + worst, + 'gap_callout': + f'tenth {occupied[worst] + 1}: predicted {fmt.number(p)} · observed ' + f'{fmt.number(o)}', + } + + +def _distribution(run: Run, scores: np.ndarray, + target: np.ndarray) -> dict[str, Any]: + low = math.floor(scores.min() / SCORE_BIN + 1e-9) + high = math.ceil(scores.max() / SCORE_BIN - 1e-9) + high = max(high, low + 1) + edges = np.round(np.arange(low, high + 1) * SCORE_BIN, 2) + true_counts, _ = np.histogram(scores[target], edges) + false_counts, _ = np.histogram(scores[~target], edges) + total = true_counts + false_counts + tallest = int(np.argmax(total)) + n = len(scores) + medians = { + 'True': float(np.median(scores[target])) if target.any() else None, + 'False': float(np.median(scores[~target])) if (~target).any() else None, + } + return { + 'id': + 'outcome', + 'label': + 'Scores by outcome', + 'x': + 'predicted probability of True', + 'y': + run.entities, + 'edges': [float(e) for e in edges], + 'series': [ + { + 'name': + 'actual True', + 'counts': [int(c) for c in true_counts], + 'median': + medians['True'], + 'median_callout': (f'median {fmt.number(medians["True"])}' + if medians['True'] is not None else None), + }, + { + 'name': + 'actual False', + 'counts': [int(c) for c in false_counts], + 'median': + medians['False'], + 'median_callout': (f'median {fmt.number(medians["False"])}' + if medians['False'] is not None else None), + }, + ], + 'tallest': + tallest, + 'tallest_callout': + (f'{fmt.count(total[tallest])} {run.entities} ' + f'({fmt.percent(total[tallest] / n)}) score ' + f'{fmt.cutoff(edges[tallest])} to {fmt.cutoff(edges[tallest + 1])}'), + } + + +def scores_card(run: Run) -> Optional[dict[str, Any]]: + holdout = run.holdout + if holdout is None or len(holdout) == 0: + return None + target = holdout['TARGET'].astype(bool).to_numpy() + scores = holdout['True_PROB'].to_numpy(dtype=float) + return { + 'title': 'Scores against outcomes', + 'views': + [_tenths(run, scores, target), + _distribution(run, scores, target)], + 'coverage': run.coverage.holdout_line(), + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_columns.py b/packages/kumo-engine/kumo/engine/_explainability/_columns.py new file mode 100644 index 000000000..a8a43ffe7 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_columns.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Column importance and Column groups, from Kumo's own explanation pass. + +Both read the evaluated fit's explanation files: gradient sensitivity per +input column (``explain_test/scores.csv``) and per-column value groups +(``column_analysis/test/column_analysis.json``). +""" + +import posixpath +from dataclasses import asdict +from typing import Any, Optional + +import numpy as np + +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._header import target +from kumo.engine._explainability._run import ( + CLASSIFICATION, + REGRESSION, + Run, +) +from kumo.engine.explain._artifacts import read_cohorts, read_gradient + +TOP = 10 +MAX_GROUPS = 10 +DOMINANT_GROUP = 0.9 +GAP_MIN_SHARE = 0.05 +COUNT_COLUMN = 'COUNT(*)' + + +def _trial_file(run: Run, *parts: str) -> Optional[str]: + trial = run.files.trial + return None if trial is None else posixpath.join(trial, *parts) + + +def _scores(run: Run) -> Optional[list[tuple[str, str, float]]]: + path = _trial_file(run, 'explain_test', 'scores.csv') + if path is None: + return None + try: + scores = read_gradient(path) + except (OSError, ValueError): + return None + out = [(table, column, score) for table, values in scores.tables.items() + for column, score in values.columns.items()] + if any(score < 0 for _, _, score in out): + return None + out.sort(key=lambda r: (-r[2], r[0], r[1])) + return out if sum(r[2] for r in out) > 0 else None + + +def cohorts(run: Run) -> Optional[list[dict[str, Any]]]: + if run.model_type not in (CLASSIFICATION, REGRESSION): + return None + path = _trial_file(run, 'column_analysis', 'test', 'column_analysis.json') + if path is None: + return None + try: + artifact = read_cohorts(path) + except (OSError, ValueError): + return None + if artifact.task_type.value != run.task_type: + return None + entries = [asdict(c) for c in artifact.cohorts if c.targets is not None] + return entries if any(e['prediction_score'] > 0 for e in entries) else None + + +def _key(entry: dict[str, Any]) -> str: + return f"{entry['table_name']}|{entry['hop']}|{entry['col_name']}" + + +def _label(table: str, column: str) -> str: + return f'{table}.{column}' + + +def _dominant(entry: dict[str, Any]) -> Optional[str]: + """I3: the value group holding 90% or more of the column's population.""" + pops = np.asarray(entry['populations'], dtype=float) + total = pops.sum() + if total <= 0: + return None + j = int(np.argmax(pops)) + share = pops[j] / total + if share < DOMINANT_GROUP: + return None + return f"{fmt.percent(share)} in {entry['cohorts'][j]}" + + +def _ranking( + items: list[tuple[str, str, float, Optional[str]]], + dominant: Optional[dict[str, Optional[str]]] = None) -> dict[str, Any]: + """Bars as shares of the summed score, top ten then the remainder.""" + total = sum(score for _, _, score, _ in items) + bars = [] + for table, column, score, group in items: + bar = { + 'table': table, + 'column': column, + 'label': _label(table, column), + 'share': score / total, + 'text': fmt.percent(score / total), + 'group': group, + } + if dominant is not None and group is not None: + bar['callout'] = dominant.get(group) + bars.append(bar) + rest = bars[TOP:] + remainder = None + if rest: + share = sum(bar['share'] for bar in rest) + remainder = { + 'label': f'{len(rest)} other columns', + 'share': share, + 'text': fmt.percent(share), + } + by_table: dict[str, float] = {} + for bar in bars: + by_table[bar['table']] = by_table.get(bar['table'], 0) + bar['share'] + tables = sorted(by_table.items(), key=lambda kv: (-kv[1], kv[0])) + return { + 'bars': + bars, + 'top': + min(TOP, len(bars)), + 'remainder': + remainder, + 'top_callout': + f"{fmt.percent(bars[0]['share'])} of total", + 'tables': [{ + 'table': t, + 'share': s, + 'text': f'{t} {fmt.percent(s)}' + } for t, s in tables], + } + + +def _first_entry(entries: list[dict[str, Any]], table: str, + column: str) -> Optional[dict[str, Any]]: + matches = [ + e for e in entries + if e['table_name'] == table and e['col_name'] == column + ] + return min(matches, key=lambda e: e['hop']) if matches else None + + +def importance(run: Run) -> Optional[dict[str, Any]]: + scores = _scores(run) + if scores is None: + return None + entries = cohorts(run) + views = [] + linked = [] + for table, column, score in scores: + entry = None if entries is None else _first_entry( + entries, table, column) + linked.append( + (table, column, score, None if entry is None else _key(entry))) + sensitivity = _ranking(linked) + sensitivity['legend'] = True + views.append({'id': 'sensitivity', 'label': 'Sensitivity', **sensitivity}) + if entries is not None: + ranked = sorted( + entries, key=lambda e: (-float(e['prediction_score']), + _label(e['table_name'], e['col_name']))) + dominant = {_key(e): _dominant(e) for e in entries} + variation = _ranking([(e['table_name'], e['col_name'], + float(e['prediction_score']), _key(e)) + for e in ranked], dominant) + variation['legend'] = False + views.append({ + 'id': 'variation', + 'label': 'Variation across groups', + **variation + }) + return { + 'title': 'Column importance', + 'views': views, + 'coverage': run.coverage.line(run.coverage.explanation), + } + + +def _share_unit(run: Run, entry: dict[str, Any]) -> str: + if entry['hop'] == 0 or entry['col_name'] == COUNT_COLUMN: + return f'of {run.entities}' + return f"of sampled {entry['table_name']} rows" + + +def group_view(run: Run, entry: dict[str, Any], total_score: float, + class_name: Optional[str] = None) -> dict[str, Any]: + names = [str(c) for c in entry['cohorts']][:MAX_GROUPS] + pops = np.asarray(entry['populations'], dtype=float)[:MAX_GROUPS] + predictions, targets = entry['predictions'], entry['targets'] + if class_name is not None: + predictions, targets = predictions[class_name], targets[class_name] + pred = np.asarray(predictions, dtype=float)[:MAX_GROUPS] + actual = np.asarray(targets, dtype=float)[:MAX_GROUPS] + weight = pops.sum() + avg_actual = float((pops * actual).sum() / weight) if weight else None + avg_pred = float((pops * pred).sum() / weight) if weight else None + eligible = np.where(pops >= GAP_MIN_SHARE)[0] + gap = None + if len(eligible): + j = int(eligible[np.argmax(np.abs(pred - actual)[eligible])]) + gap = { + 'index': + j, + 'text': + f'predicted {fmt.number(pred[j])} · actual ' + f'{fmt.number(actual[j])}' + } + stype = entry.get('stype') or ('count' + if entry['col_name'] == COUNT_COLUMN else '') + share = float(entry['prediction_score']) / total_score + header = (f"{_label(entry['table_name'], entry['col_name'])} · hop " + f"{entry['hop']} · {stype} · variation " + f"{fmt.percent(share)}") + unit = _share_unit(run, entry) + values = fmt.numbers( + list(pred) + list(actual) + + [v for v in (avg_actual, avg_pred) if v is not None]) + n = len(names) + return { + 'key': + _key(entry), + 'header': + header, + 'groups': [{ + 'label': names[i], + 'share': float(pops[i]), + 'share_text': f'{fmt.percent(pops[i])} {unit}', + 'predicted': float(pred[i]), + 'predicted_text': values[i], + 'actual': float(actual[i]), + 'actual_text': values[n + i], + } for i in range(n)], + 'axis': + _axis(run) if class_name is None else f'share {class_name}', + 'average_actual': + avg_actual, + 'average_predicted': + avg_pred, + 'average_callout': + None if avg_actual is None or avg_pred is None else + (f'average actual {fmt.number(avg_actual)} · ' + f'average predicted {fmt.number(avg_pred)}'), + 'gap_callout': + gap, + } + + +def _axis(run: Run) -> str: + """The group chart's value axis: what both bar series average.""" + if run.model_type == CLASSIFICATION: + return 'share True' + return 'average ' + (target(run) or 'value') + + +def groups(run: Run) -> Optional[dict[str, Any]]: + entries = cohorts(run) + if entries is None: + return None + total = sum(float(e['prediction_score']) for e in entries) + views = {} + for entry in entries: + if isinstance(entry['predictions'], dict): + view = { + 'by_class': [{ + 'label': label, + **group_view(run, entry, total, label) + } for label in entry['predictions']] + } + else: + view = group_view(run, entry, total) + views[_key(entry)] = view + default = None + scores = _scores(run) + for table, column, _ in scores or []: + entry = _first_entry(entries, table, column) + if entry is not None: + default = _key(entry) + break + if default is None: + default = _key(max(entries, key=lambda e: float(e['prediction_score']))) + return { + 'title': 'Column groups', + 'selectable': not scores, + 'default': default, + 'columns': views, + 'coverage': run.coverage.line(run.coverage.explanation), + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_coverage.py b/packages/kumo-engine/kumo/engine/_explainability/_coverage.py new file mode 100644 index 000000000..27fd7ea05 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_coverage.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Detect when Kumo evaluated or explained only the head of the test split. + +Kumo stops test evaluation after ``max_test_steps`` batches, and its +explanation pass replays at most a fixed number of batches at a quarter of the +evaluation batch size. When either cap leaves rows out, the affected cards +carry one fixed line; when no cap applies, they carry nothing. +""" + +import posixpath +from dataclasses import dataclass +from typing import Any, Optional + +import pandas as pd + +from kumo.engine._explainability import _files +from kumo.engine._explainability._format import count + + +@dataclass(frozen=True) +class Coverage: + """Test rows behind each family of cards, when fewer than the split.""" + + test_rows: Optional[int] + metrics: Optional[int] = None + holdout: Optional[int] = None + explanation: Optional[int] = None + holdout_truncated: bool = False + + def line(self, covered: Optional[int]) -> Optional[str]: + if covered is None or self.test_rows is None: + return None + if covered >= self.test_rows: + return None + return (f'based on the first {count(covered)} of ' + f'{count(self.test_rows)} test rows') + + def holdout_line(self) -> Optional[str]: + line = self.line(self.holdout) + if self.holdout_truncated: + return ( + line + '; retained holdout is truncated' if line else + 'Based on a truncated retained holdout; full coverage is unavailable.' + ) + return line + + +def _int(value: Any) -> Optional[int]: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return int(value) + + +def eval_batch_size(files: _files.RunFiles) -> Optional[int]: + """The published model's evaluation batch size, as the trainer resolves it.""" + config = _files.read_json( + posixpath.join(files.bundle, 'model', 'config.json')) + if not isinstance(config, dict): + return None + for key in ('eval_loader_kwargs', 'loader_kwargs'): + kwargs = config.get(key) + if isinstance(kwargs, dict) and _int(kwargs.get('batch_size')): + return _int(kwargs['batch_size']) + return None + + +def explanation_batches() -> int: + from kumo_core.training.runtime import LocalExplainRuntime + + return int(LocalExplainRuntime.max_batches) + + +def holdout_seeds(frame: pd.DataFrame) -> int: + """Test-table rows behind a holdout frame. + + A forecasting holdout carries one row per horizon step, so its test rows + are its distinct entity and anchor pairs. + """ + if 'ANCHOR_TIMESTAMP' in frame.columns and 'ENTITY' in frame.columns: + return int(len(frame[['ENTITY', 'ANCHOR_TIMESTAMP']].drop_duplicates())) + return int(len(frame)) + + +def detect(files: _files.RunFiles, + holdout: Optional[pd.DataFrame] = None) -> Coverage: + from kumo.engine.trainer._spec import HOLDOUT_META_ROLE + + stats = _files.split_stats(files) + test = (stats or {}).get('test_table_stats') or {} + test_rows = _int(test.get('num_train_rows')) + seeds = None if holdout is None else holdout_seeds(holdout) + meta = _files.read_json(_files.bundle_file(files, HOLDOUT_META_ROLE)) + truncated = (files.holdout_truncated is True + or isinstance(meta, dict) and meta.get('truncated') is True) + if test_rows is None: + return Coverage(None, holdout=seeds, holdout_truncated=truncated) + batch = eval_batch_size(files) + optimization = _files.plan(files).get('optimization') or {} + steps = _int(optimization.get('max_test_steps')) + metrics = None + if batch is not None and steps is not None and steps > 0: + metrics = min(test_rows, steps * batch) + explanation = None + if batch is not None: + explanation = min(test_rows, explanation_batches() * max(batch // 4, 1)) + return Coverage(test_rows, metrics, seeds, explanation, truncated) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_files.py b/packages/kumo-engine/kumo/engine/_explainability/_files.py new file mode 100644 index 000000000..6f9aeac1c --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_files.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Locate and read the Kumo files one training run leaves on disk. + +Every reader returns ``None`` for a missing or unreadable optional file, so a +card whose inputs are absent is omitted instead of failing the widget. +""" + +import io +import json +import logging +import posixpath +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +import pandas as pd +from kumo_core.io import filesystem + +from kumo.engine.explain._artifacts import artifact_root, resolve_run +from kumo.engine.explain.errors import InvalidRunError + +logger = logging.getLogger(__name__) + +REFIT_SUFFIX = '_refit_trainval' + + +@dataclass(frozen=True) +class RunFiles: + """Directories of one run, in the shorthand the specification uses. + + ``bundle`` is the published model (``PUB``). ``trial`` is the evaluated + model's trial directory: the train-plus-validation refit when one ran, + otherwise the selected trial. ``first`` is the selected trial's first fit. + ``table`` is the training-table directory and ``snapshot`` the graph + snapshot. Directories that a bundle alone does not carry are ``None``. + """ + + bundle: str + trial: Optional[str] = None + first: Optional[str] = None + table: Optional[str] = None + snapshot: Optional[str] = None + holdout_path: Optional[str] = None + holdout_truncated: Optional[bool] = None + + @property + def refit(self) -> bool: + return self.trial is not None and self.trial.endswith(REFIT_SUFFIX) + + +def _join(base: Optional[str], *parts: str) -> Optional[str]: + return None if base is None else posixpath.join(base, *parts) + + +def _present(path: Optional[str]) -> Optional[str]: + if not isinstance(path, str): + return None + try: + return path if filesystem.exists(path) else None + except OSError: + return None + + +def locate(bundle_dir: str, run_root: Optional[str] = None, + trial_root: Optional[str] = None) -> RunFiles: + """Resolve the run directories around a published bundle.""" + bundle = artifact_root(bundle_dir) + if trial_root is not None: + root = artifact_root(run_root) if run_root is not None else None + trial = artifact_root(trial_root) + elif run_root is None: + return RunFiles(bundle) + else: + try: + root, trial = resolve_run(run_root) + except InvalidRunError as error: + logger.debug('Training run not readable: %s', error) + return RunFiles(bundle) + first = trial.removesuffix(REFIT_SUFFIX) + manifest = read_json(_join(root, 'task_dataset', 'runtime_manifest.json')) + table = snapshot = None + if isinstance(manifest, dict): + stages, graph = manifest.get('stages'), manifest.get('graph') + test = stages.get('test') if isinstance(stages, dict) else None + table = _present(test.get('index_path')) if isinstance(test, + dict) else None + snapshot = _present(graph.get('index_path')) if isinstance( + graph, dict) else None + return RunFiles(bundle, _present(trial), _present(first), table, snapshot) + + +def open_run(run_root: str) -> RunFiles: + """Locate a retained run whose bundle is ``/published-model``.""" + root = artifact_root(run_root) + return locate(posixpath.join(root, 'published-model'), root) + + +def read_json(path: Optional[str]) -> Any: + if _present(path) is None: + return None + assert path is not None + try: + with filesystem.open_file(path, 'rt', encoding='utf-8') as src: + return json.load(src) + except (OSError, ValueError) as error: + logger.debug('Unreadable JSON %s: %s', path, error) + return None + + +def read_parquet(path: Optional[str], + columns: Optional[list[str]] = None) -> Optional[pd.DataFrame]: + if _present(path) is None: + return None + assert path is not None + try: + with filesystem.open_file(path, 'rb') as src: + return pd.read_parquet(io.BytesIO(src.read()), columns=columns) + except (OSError, ValueError, KeyError) as error: + # A missing column is a missing optional input, not a widget error. + logger.debug('Unreadable Parquet %s: %s', path, error) + return None + + +def read_parquet_dir(path: Optional[str], + columns: list[str]) -> Optional[pd.DataFrame]: + """Concatenate every Parquet part in a directory.""" + if _present(path) is None: + return None + assert path is not None + parts = sorted(name for name in filesystem.listdir(path) + if name.endswith('.parquet')) + frames = [ + read_parquet(posixpath.join(path, posixpath.basename(name)), columns) + for name in parts + ] + if not frames or any(frame is None for frame in frames): + return None + return pd.concat(frames, ignore_index=True) + + +def plan(files: RunFiles) -> dict[str, Any]: + value = read_json( + posixpath.join(files.bundle, 'model', 'resolved-model-plan.json')) + return value if isinstance(value, dict) else {} + + +def task_type(files: RunFiles) -> Optional[str]: + schema = read_json(posixpath.join(files.bundle, 'prediction-schema.json')) + if isinstance(schema, dict) and isinstance(schema.get('task_type'), str): + return schema['task_type'] + return None + + +def entity_table(files: RunFiles) -> Optional[str]: + """The table named in the query's ``FOR EACH``.""" + spec = read_json(posixpath.join(files.bundle, 'training-spec.json')) + query = spec.get('predictive_query') if isinstance(spec, dict) else None + if isinstance(query, str): + match = re.search(r'\bFOR\s+EACH\s+([A-Za-z_][\w]*)\.', query, + flags=re.IGNORECASE) + if match: + return match.group(1) + return None + + +def split_stats(files: RunFiles) -> Optional[dict[str, Any]]: + value = read_json(_join(files.table, 'split_stats.json')) + return ({ + key: item + for key, item in value.items() if isinstance(item, dict) + } if isinstance(value, dict) else None) + + +def bundle_file(files: RunFiles, role: str) -> Optional[str]: + from kumo.engine.trainer._spec import listed_file + + try: + bundle = Path(files.bundle) + path = listed_file(bundle, role) + return (str(path) if path is not None + and path.resolve().is_relative_to(bundle.resolve()) else None) + except (OSError, ValueError, TypeError, AttributeError, KeyError): + return None + + +def holdout(files: RunFiles, columns: list[str]) -> Optional[pd.DataFrame]: + from kumo.engine.trainer._spec import HOLDOUT_ROLE + + frame = read_parquet(files.holdout_path, columns) + if frame is not None: + return frame + return read_parquet(bundle_file(files, HOLDOUT_ROLE), columns) diff --git a/packages/kumo-engine/kumo/engine/_explainability/_forecasting.py b/packages/kumo-engine/kumo/engine/_explainability/_forecasting.py new file mode 100644 index 000000000..b3184cceb --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_forecasting.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Forecasting card: Forecast against actual, from the published model's +holdout predictions, one row per entity and forecast step.""" + +from typing import Any, Optional + +import numpy as np +import pandas as pd + +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._holdout import equal_count_groups +from kumo.engine._explainability._run import Run + +# Forecast step lengths in days and the word for one step. +_PERIODS = ((1, 'day'), (7, 'week'), (28, 'month'), (29, 'month'), + (30, 'month'), (31, 'month'), (365, 'year'), (366, 'year')) + + +def period(steps: pd.Series) -> str: + """The calendar word for the spacing of the forecast steps.""" + stamps = pd.to_datetime(steps.drop_duplicates()).sort_values() + days = stamps.diff().dropna().dt.days + if days.empty: + return 'step' + typical = int(days.median()) + for length, word in _PERIODS: + if typical == length: + return word + return 'step' + + +def _weeks(frame: pd.DataFrame, word: str) -> dict[str, Any]: + per = frame.groupby('TIMESTAMP', sort=True).agg(mae=('abs_error', 'mean'), + zero=('abs_actual', 'mean'), + bias=('error', 'mean')) + texts = fmt.numbers(list(per['mae']) + list(per['zero'])) + n = len(per) + return { + 'id': + 'horizon', + 'label': + f'By horizon {word}', + 'x': + f'forecast {word}', + 'y': + 'mean absolute error', + 'series': ['model', 'predict zero'], + 'steps': [{ + 'label': fmt.date(str(stamp)), + 'model': float(row.mae), + 'model_text': texts[i], + 'zero': float(row.zero), + 'zero_text': texts[n + i], + 'bias_callout': f'bias {fmt.signed(float(row.bias))}', + } for i, (stamp, row) in enumerate(per.iterrows())], + } + + +def _totals(frame: pd.DataFrame, word: str, entities: str) -> dict[str, Any]: + per = frame.groupby('TIMESTAMP', + sort=True).agg(actual=('TARGET', 'sum'), + forecast=('TARGET_PRED', 'sum')) + actual, forecast = float(per['actual'].sum()), float(per['forecast'].sum()) + callout = None + if actual > 0: + callout = (f'forecast total {fmt.percent(forecast / actual)} of ' + f'actual over {len(per)} {word}s') + values = list(per['actual']) + list(per['forecast']) + target = frame['TARGET'].to_numpy() + if np.allclose(target, np.round(target), rtol=0, atol=1e-3): + # A count target totals to whole units; the forecast is shown on the + # same footing. + totals = [fmt.count(round(v)) for v in values] + else: + totals = fmt.numbers(values) + n = len(per) + return { + 'id': + 'totals', + 'label': + f'{word.capitalize()}ly totals' + if word in ('week', 'month', 'year') else 'Totals by step', + 'x': + f'forecast {word}', + 'y': + f'total over all {entities}', + 'series': ['actual', 'forecast'], + 'steps': [{ + 'label': fmt.date(str(stamp)), + 'actual': float(row.actual), + 'actual_text': totals[i], + 'forecast': float(row.forecast), + 'forecast_text': totals[n + i], + } for i, (stamp, row) in enumerate(per.iterrows())], + 'total_callout': + callout, + } + + +def _tenths(run: Run, frame: pd.DataFrame) -> Optional[dict[str, Any]]: + per = frame.groupby('ENTITY', + sort=True).agg(actual=('TARGET', 'sum'), + forecast=('TARGET_PRED', 'sum')) + total = float(per['actual'].sum()) + if total <= 0: + return None + group = equal_count_groups(per['forecast'].to_numpy(dtype=float), + per.index.to_numpy()) + rows = [] + for k in np.unique(group): + mask = group == k + share = float(per['actual'].to_numpy()[mask].sum()) / total + rows.append({ + 'label': str(k + 1), + 'entities': int(mask.sum()), + 'share': share, + 'share_text': fmt.percent(share), + }) + top = rows[-1] + return { + 'id': + 'tenths', + 'label': + 'By forecast tenth', + 'x': + 'forecast tenth, lowest to highest', + 'y': + 'share of actual units', + 'tenths': + rows, + 'reference': + 0.1, + 'reference_callout': + 'equal share 10%', + 'top_callout': + f'{top["share_text"]} of actual units · ' + f'{fmt.count(top["entities"])} {run.entities}', + } + + +def forecast_card(run: Run) -> Optional[dict[str, Any]]: + holdout = run.holdout + if holdout is None or len(holdout) == 0: + return None + frame = holdout.copy() + frame['TARGET'] = frame['TARGET'].astype(float) + frame['TARGET_PRED'] = frame['TARGET_PRED'].astype(float) + frame = frame[np.isfinite(frame['TARGET']) + & np.isfinite(frame['TARGET_PRED'])] + if frame.empty: + return None + frame['error'] = frame['TARGET_PRED'] - frame['TARGET'] + frame['abs_error'] = frame['error'].abs() + frame['abs_actual'] = frame['TARGET'].abs() + word = period(frame['TIMESTAMP']) + views = [_weeks(frame, word), _totals(frame, word, run.entities)] + tenths = _tenths(run, frame) + if tenths is not None: + views.append(tenths) + return { + 'title': 'Forecast against actual', + 'views': views, + 'coverage': run.coverage.holdout_line(), + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_format.py b/packages/kumo-engine/kumo/engine/_explainability/_format.py new file mode 100644 index 000000000..e56e79895 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_format.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Number formats shared by every card. + +Magnitudes of 10 or more get one decimal, smaller values three significant +figures, counts thousands separators, shares one-decimal percentages. +""" + +import math +from collections.abc import Iterable + +MINUS = '−' + + +def _plain(value: float) -> str: + rounded = float(f'{value:.3g}') + if rounded == 0: + return '0' + if abs(rounded) >= 10: + return f'{value:,.1f}' + digits = 2 - math.floor(math.log10(abs(rounded))) + return f'{rounded:.{max(digits, 0)}f}' + + +def number(value: float) -> str: + """One decimal at magnitude 10 or more, else three significant figures.""" + if not math.isfinite(value): + return 'N/A' + text = f'{value:,.1f}' if abs(value) >= 10 else _plain(value) + return text.replace('-', MINUS) + + +def signed(value: float) -> str: + """`number` with an explicit sign, for differences.""" + text = number(value) + if text in ('0', '0.0', 'N/A') or text.startswith(MINUS): + return text + return '+' + text + + +def numbers(values: Iterable[float]) -> list[str]: + """Format values that share one row or axis on one scale. + + When any value reaches magnitude 10 the whole row gets one decimal, so a + row such as ``-6.8 / 21.6 / 577.0`` keeps a single precision. + """ + items = list(values) + finite = [abs(v) for v in items if math.isfinite(v)] + if finite and max(finite) >= 10: + return [(f'{v:,.1f}'.replace('-', MINUS) if math.isfinite(v) else 'N/A') + for v in items] + return [number(v) for v in items] + + +def count(value: int) -> str: + return f'{int(value):,}' + + +def percent(share: float) -> str: + """A share in ``[0, 1]`` as a one-decimal percentage.""" + return f'{share * 100:.1f}%'.replace('-', MINUS) + + +def multiple(value: float) -> str: + return f'{value:.1f}×' + + +def cutoff(value: float) -> str: + return f'{value:.2f}' + + +def date(value: str) -> str: + """The date part of an ISO timestamp.""" + return str(value)[:10] diff --git a/packages/kumo-engine/kumo/engine/_explainability/_header.py b/packages/kumo-engine/kumo/engine/_explainability/_header.py new file mode 100644 index 000000000..4672a9930 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_header.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The widget header: the predictive query and one line about the holdout.""" + +import posixpath +import re +from typing import Any, Optional + +from kumo.engine._explainability import _files +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._forecasting import period +from kumo.engine._explainability._metrics import catalog, fits, positive_share +from kumo.engine._explainability._run import ( + CLASSIFICATION, + FORECASTING, + LINK_PREDICTION, + REGRESSION, + Run, +) + + +def query(run: Run) -> Optional[str]: + spec = _files.read_json( + posixpath.join(run.files.bundle, 'training-spec.json')) + value = spec.get('predictive_query') if isinstance(spec, dict) else None + return ' '.join(value.split()) if isinstance(value, str) else None + + +_TARGET = re.compile( + r'^PREDICT\s+(.+?)\s+(?:RANK\s+TOP\b|FORECAST\b|FOR\s+EACH\b)', + re.IGNORECASE) + + +def target(run: Run, short: bool = False) -> Optional[str]: + """The query's target expression, for axis titles. + + ``short`` drops the time window, so ``SUM(transactions.price, 0, 30, + days)`` reads ``SUM(transactions.price)``. + """ + text = query(run) + match = _TARGET.match(text) if text else None + if not match: + return None + expr = match.group(1) + if short: + expr = re.sub(r'\(([^,()]*),[^()]*\)', r'(\1)', expr, count=1) + return expr + + +def _anchor(run: Run) -> Optional[str]: + stats = _files.split_stats(run.files) or {} + split = stats.get('test_table_stats') + if isinstance(split, dict) and split.get('anchor_date'): + return f'anchor {fmt.date(split["anchor_date"])}' + return None + + +def _holdout(run: Run) -> list[str]: + frame = run.holdout + if frame is None or not len(frame): + return [] + if run.model_type == FORECASTING: + entities = frame['ENTITY'].nunique() + steps = frame['TIMESTAMP'].nunique() + word = period(frame['TIMESTAMP']) + unit = word if steps == 1 else f'{word}s' + return [ + f'{fmt.count(entities)} {run.entities} × {steps} {unit} = ' + f'{fmt.count(len(frame))} rows' + ] + return [f'{fmt.count(len(frame))} {run.entities}'] + + +def _detail(run: Run) -> list[str]: + if run.model_type == CLASSIFICATION: + share = positive_share(run) + return [] if share is None else [f'{fmt.percent(share)} True'] + if run.model_type == REGRESSION and run.holdout is not None and len( + run.holdout): + zero = float((run.holdout['TARGET'] == 0).mean()) + return [f'{fmt.percent(zero)} with actual 0'] if zero >= 0.1 else [] + if run.model_type == LINK_PREDICTION: + cat = catalog(run) + if cat is None: + return [] + return [ + f'{fmt.number(cat.mean_targets)} items each', + f'catalog {fmt.count(cat.size)} {cat.table}' + ] + return [] + + +def _link_entities(run: Run) -> list[str]: + stats = _files.split_stats(run.files) or {} + split = stats.get('test_table_stats') + rows = split.get('num_train_rows') if isinstance(split, dict) else None + return [f'{fmt.count(rows)} {run.entities}'] if isinstance(rows, + int) else [] + + +def header(run: Run) -> dict[str, Any]: + """The query and a holdout line; either may be missing.""" + parts = (_link_entities(run) + if run.model_type == LINK_PREDICTION else _holdout(run)) + anchor = _anchor(run) + if parts and anchor: + parts.append(anchor) + parts += _detail(run) + line = ('Holdout: ' + ' · '.join(parts)) if parts else None + if fits(run).full_refit: + note = 'Diagnostics and explanations describe the model before full refit.' + line = f'{line} · {note}' if line else note + return {'query': query(run), 'holdout': line} diff --git a/packages/kumo-engine/kumo/engine/_explainability/_holdout.py b/packages/kumo-engine/kumo/engine/_explainability/_holdout.py new file mode 100644 index 000000000..39d986a55 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_holdout.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared grouping, histogram limits and finite-value helpers for holdouts.""" + +import math +from typing import Optional + +import numpy as np + +TENTHS = 10 +# Trailing histogram bins holding this many rows or fewer fold into one +# final bin, the boundary of the Kumo UI's Hide Outliers control. +FOLD_AT_MOST = 10 + + +def equal_count_groups(values: np.ndarray, entities: np.ndarray, + groups: int = TENTHS) -> np.ndarray: + """Zero-based group of each row, ascending by value then entity. + + Row ``i`` in sorted order goes to group ``floor(groups * i / n)``, so + group sizes differ by at most one and the last group holds the highest. + """ + n = len(values) + order = np.lexsort((entities, values)) + out = np.empty(n, dtype=int) + out[order] = np.arange(n) * groups // max(n, 1) + return out + + +def finite(value: float) -> Optional[float]: + return value if math.isfinite(value) else None diff --git a/packages/kumo-engine/kumo/engine/_explainability/_metrics.py b/packages/kumo-engine/kumo/engine/_explainability/_metrics.py new file mode 100644 index 000000000..65348ce4d --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_metrics.py @@ -0,0 +1,545 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The Metrics card, shared by every model type. + +Holdout scores describe the evaluated fit, before any full-data refit. +Training and validation scores use the original run's last epoch. +""" + +import math +import posixpath +import re +import sys +from dataclasses import dataclass +from typing import Any, Optional + +import numpy as np + +from kumo.engine._explainability import _files +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._run import ( + CLASS_LABELED, + CLASSIFICATION, + FORECASTING, + LINK_PREDICTION, + REGRESSION, + Run, +) + +HIGHER_IS_BETTER = {'auroc', 'ap', 'auprc', 'map', 'recall', 'precision', 'f1'} +LOWER_IS_BETTER = {'mae', 'mse', 'rmse', 'loss'} + +_NAMES = { + 'auroc': 'AUROC', + 'ap': 'AP', + 'auprc': 'AUPRC', + 'mae': 'MAE', + 'mse': 'MSE', + 'rmse': 'RMSE', + 'loss': 'Loss', + 'map': 'MAP', + 'recall': 'Recall', + 'precision': 'Precision', + 'f1': 'F1', + 'acc': 'Accuracy', + 'accuracy': 'Accuracy', + 'smape': 'SMAPE', + 'r2': 'R²', + 'mape': 'MAPE', + 'ndcg': 'NDCG', +} +_RANKING_ORDER = ('f1', 'map', 'precision', 'recall') +_LOCALITY = 'locality_score' + + +def family(metric: str) -> str: + return metric.split('@')[0] + + +def top_k(metric: str) -> Optional[int]: + match = re.fullmatch(r'[a-z0-9_]+@(\d+)', metric) + return int(match.group(1)) if match else None + + +def display_name(metric: str) -> str: + name = _NAMES.get(family(metric), family(metric).upper()) + k = top_k(metric) + return name if k is None else f'{name}@{k}' + + +def higher_is_better(metric: str) -> Optional[bool]: + if family(metric) in HIGHER_IS_BETTER: + return True + if family(metric) in LOWER_IS_BETTER: + return False + return None + + +@dataclass(frozen=True) +class Fits: + """Retained holdout scores and the fit they describe.""" + + holdout: dict[str, float] + refit: bool + full_refit: bool + + +def _prefixed(values: Any, prefix: str) -> dict[str, float]: + if not isinstance(values, dict): + return {} + return { + key.removeprefix(prefix): float(value) + for key, value in values.items() + if key.startswith(prefix) and isinstance(value, (int, float)) + and not isinstance(value, bool) and math.isfinite(value) + } + + +def fits(run: Run) -> Fits: + refit_holdout = _prefixed(run.metrics, 'test_refit_') + refit = run.files.refit or bool(refit_holdout) + holdout = refit_holdout if refit else { + k: v + for k, v in _prefixed(run.metrics, 'test_').items() + if not k.startswith('refit_') + } + provenance = _files.read_json( + posixpath.join(run.files.bundle, '_provenance.json')) + config = _files.read_json( + posixpath.join(run.files.bundle, 'model', 'config.json')) + refit_cfg = config.get('refit_cfg') if isinstance(config, dict) else None + full_refit = (provenance.get('fit_mode') == 'refit_full' + if isinstance(provenance, dict) and provenance.get('fit_mode') + else isinstance(refit_cfg, dict) + and refit_cfg.get('refitted_full') is True) + return Fits(holdout, refit, full_refit) + + +# ---------------------------------------------------------------- baselines + + +@dataclass(frozen=True) +class Baseline: + name: str + value: float + + +def _regression_baselines(run: Run) -> dict[str, list[Baseline]]: + if run.holdout is None: + return {} + actual = run.holdout['TARGET'].to_numpy(dtype=float) + if actual.size == 0: + return {} + out: dict[str, list[Baseline]] = {} + for name, pred in (('predict zero', np.zeros_like(actual)), + ('predict the mean', np.full_like(actual, + actual.mean()))): + error = pred - actual + values = { + 'mae': float(np.abs(error).mean()), + 'mse': float((error**2).mean()), + 'rmse': float(np.sqrt((error**2).mean())), + } + for metric, value in values.items(): + out.setdefault(metric, []).append(Baseline(name, value)) + return out + + +def positive_share(run: Run) -> Optional[float]: + if run.task_type in CLASS_LABELED: + return None + if run.holdout is None or len(run.holdout) == 0: + return None + return float(run.holdout['TARGET'].astype(bool).mean()) + + +def _classification_baselines(run: Run) -> dict[str, list[Baseline]]: + out = {'auroc': [Baseline('chance', 0.5)]} + share = positive_share(run) + if share is not None: + out['ap'] = [Baseline('base rate', share)] + out['auprc'] = [Baseline('base rate', share)] + return out + + +@dataclass(frozen=True) +class Catalog: + size: int + mean_targets: float + table: str + + +def catalog(run: Run) -> Optional[Catalog]: + """Catalog size and mean holdout list length for random baselines.""" + files = run.files + if files.table is None or files.snapshot is None: + return None + manifest = _files.read_json( + posixpath.join(files.table, 'diskgraph_manifest.json')) + stats = _files.read_json(posixpath.join(files.snapshot, + 'ingest_stats.json')) + if not isinstance(manifest, dict) or not isinstance(stats, dict): + return None + rhs = (manifest.get('task_dataset') or {}).get('rhs_node_type') + counts = (stats.get('stats') or {}).get('node_counts') or {} + size = counts.get(rhs) if isinstance(rhs, str) else None + seeds = _files.read_parquet_dir(posixpath.join(files.table, 'test_seeds'), + ['TARGET1']) + if not isinstance(size, + int) or size <= 0 or seeds is None or not len(seeds): + return None + lengths = seeds['TARGET1'].map(len).to_numpy(dtype=float) + return Catalog(size, float(lengths.mean()), str(rhs)) + + +def _random_baselines(run: Run) -> dict[str, list[Baseline]]: + cat = catalog(run) + if cat is None: + return {} + out: dict[str, list[Baseline]] = {} + for metric in run.training_job.get('metrics') or []: + k = top_k(metric) + if k is None: + continue + if family(metric) == 'recall': + out[metric] = [Baseline('random', k / cat.size)] + elif family(metric) == 'precision': + out[metric] = [Baseline('random', cat.mean_targets / cat.size)] + return out + + +def baselines(run: Run) -> dict[str, list[Baseline]]: + if run.model_type == CLASSIFICATION: + return {} if run.task_type in CLASS_LABELED else _classification_baselines( + run) + if run.model_type in (REGRESSION, FORECASTING): + return _regression_baselines(run) + return _random_baselines(run) + + +# --------------------------------------------------------------------- card + + +def tile_metrics(run: Run) -> list[str]: + job = run.training_job + metrics = [m for m in job.get('metrics') or [] if isinstance(m, str)] + tune = job.get('tune_metric') + if not isinstance(tune, str): + tune = metrics[0] if metrics else None + if tune is None: + return [] + if run.model_type == LINK_PREDICTION: + k = top_k(tune) + order = [tune + ] + ([] if k is None else [f'recall@{k}', f'precision@{k}']) + else: + names = set(metrics) + order = [tune] + [ + m for m in metrics if not (m == 'auprc' and 'ap' in names) + and not (m == 'mse' and 'rmse' in names) + ] + seen: list[str] = [] + for metric in order: + if metric not in seen: + seen.append(metric) + return seen[:3] + + +def _best(metric: str, items: list[Baseline]) -> Optional[Baseline]: + direction = higher_is_better(metric) + if not items or direction is None: + return None + return (max if direction else min)(items, key=lambda b: b.value) + + +def tile(run: Run, fit: Fits, metric: str, + base: dict[str, list[Baseline]]) -> Optional[dict[str, Any]]: + holdout = fit.holdout.get(metric) + if holdout is None: + return None + training = _prefixed(run.metrics, 'train_').get(metric) + items = base.get(metric, []) + callout = None + if run.model_type == LINK_PREDICTION: + rand = items[0] if items else None + if rand is not None and rand.value > 0: + callout = f'{fmt.multiple(holdout / rand.value)} random' + else: + best = _best(metric, items) + if best is not None: + callout = f'{fmt.signed(holdout - best.value)} vs {best.name}' + gap = None if training is None else holdout - training + return { + 'metric': + metric, + 'name': + display_name(metric), + 'tune': + metric == run.training_job.get('tune_metric'), + 'holdout': + holdout, + 'holdout_text': + fmt.number(holdout), + 'holdout_label': + 'holdout · before full refit' if fit.full_refit else 'holdout', + 'training': + training, + 'training_text': + None if training is None else fmt.number(training), + 'training_scope': + 'Last training epoch; predictions use the selected checkpoint.', + 'gap': + gap, + 'gap_text': + None if gap is None else f'holdout minus training: {fmt.signed(gap)}', + 'baselines': [{ + 'name': b.name, + 'value': b.value, + 'text': f'{b.name} {fmt.number(b.value)}' + } for b in items], + 'callout': + callout, + } + + +def table_metrics(run: Run, fit: Fits) -> list[str]: + metrics = [ + m for m in run.training_job.get('metrics') or [] if isinstance(m, str) + ] + if run.model_type != LINK_PREDICTION: + return metrics + + def order(metric: str) -> tuple[int, int]: + fam = family(metric) + rank = (_RANKING_ORDER.index(fam) + if fam in _RANKING_ORDER else len(_RANKING_ORDER)) + return rank, top_k(metric) or 0 + + return sorted(metrics, key=order) + + +def _split(stats: Optional[dict[str, Any]], name: str) -> dict[str, Any]: + value = (stats or {}).get(f'{name}_table_stats') + return value if isinstance(value, dict) else {} + + +def _rows(split: dict[str, Any]) -> Optional[int]: + value = split.get('num_train_rows') + return value if isinstance(value, int) else None + + +def _detail_anchor(split: dict[str, Any]) -> Optional[str]: + rows = _rows(split) + if rows is None or not split.get('anchor_date'): + return None + return f'{fmt.count(rows)} rows · {fmt.date(split["anchor_date"])}' + + +def _detail_range(split: dict[str, Any]) -> Optional[str]: + rows = _rows(split) + if rows is None or not split.get('min_date') or not split.get('max_date'): + return None + return (f'{fmt.count(rows)} rows · {fmt.date(split["min_date"])} to ' + f'{fmt.date(split["max_date"])}') + + +def table(run: Run, fit: Fits) -> dict[str, Any]: + stats = _files.split_stats(run.files) + train, val, test = (_split(stats, s) for s in ('train', 'val', 'test')) + holdout_detail = _detail_anchor(test) + groups = [{ + 'title': + 'Model before full refit' if fit.full_refit else 'Published model', + 'span': 1 + }, { + 'title': 'Training run · last epoch', + 'span': 2 + }] + columns = [ + { + 'label': 'Holdout', + 'detail': holdout_detail, + 'values': fit.holdout + }, + { + 'label': 'Validation', + 'detail': _detail_anchor(val), + 'values': _prefixed(run.metrics, 'val_') + }, + { + 'label': 'Training', + 'detail': _detail_range(train), + 'values': _prefixed(run.metrics, 'train_') + }, + ] + tune = run.training_job.get('tune_metric') + rows = [] + for metric in table_metrics(run, fit): + values = [column['values'].get(metric) for column in columns] + if all(v is None for v in values): + continue + rows.append({ + 'metric': + metric, + 'name': + display_name(metric), + 'tune': + metric == tune, + 'values': + values, + 'texts': [None if v is None else fmt.number(v) for v in values] + }) + if run.model_type == LINK_PREDICTION: + values: list[Optional[float]] = [None] * len(columns) + holdouts = [i for i, c in enumerate(columns) if c['label'] == 'Holdout'] + sources = [ + f'test_refit_{_LOCALITY}' if fit.refit else f'test_{_LOCALITY}' + ] + for index, key in zip(holdouts, sources): + values[index] = run.metrics.get(key) + if any(v is not None for v in values): + rows.append({ + 'metric': + _LOCALITY, + 'name': + 'Locality score', + 'tune': + False, + 'values': + values, + 'texts': [None if v is None else fmt.number(v) for v in values] + }) + # Drop columns no row fills, such as training values a bundle lacks. + keep = [ + i for i in range(len(columns)) + if any(row['values'][i] is not None for row in rows) + ] + start = 0 + kept_groups = [] + for group in groups: + span = len([i for i in keep if start <= i < start + group['span']]) + if span: + kept_groups.append({**group, 'span': span}) + start += group['span'] + for row in rows: + row['values'] = [row['values'][i] for i in keep] + row['texts'] = [row['texts'][i] for i in keep] + return { + 'groups': + kept_groups, + 'columns': [{ + 'label': columns[i]['label'], + 'detail': columns[i]['detail'] + } for i in keep], + 'rows': + rows, + } + + +def _history(run: Run) -> Optional[dict[str, Any]]: + """Search-trial scalars with the same tags and epoch labels as the UI.""" + history = _files.read_json(_files.bundle_file(run.files, 'metric_history')) + if history is None and run.files.first is not None: + history = _files.read_json( + posixpath.join(posixpath.dirname(run.files.first), + 'auto_train_progress.json')) + trials = history.get('trial_progress') if isinstance(history, + dict) else None + if not isinstance(trials, dict): + return None + experiments = [] + tags: set[str] = set() + + # Object.keys in the web client enumerates array indices first, then + # named trials in their original order. + def trial_order(key: str) -> int: + if re.fullmatch(r'0|[1-9][0-9]{0,9}', key): + return min(int(key), 2**32 - 1) + return 2**32 - 1 + + for key in sorted(trials, key=trial_order): + trial = trials[key] + records = trial.get('metrics') if isinstance(trial, dict) else None + if not isinstance(records, dict): + continue + series: dict[str, list[tuple[int, Optional[float]]]] = {} + for epoch, record in records.items(): + if not re.fullmatch(r'[0-9]{1,10}', epoch) or not isinstance( + record, dict): + continue + for field, prefix in (('train_metrics', 'train_'), + ('validation_metrics', 'val_')): + values = record.get(field) + if not isinstance(values, dict): + continue + for tag, value in values.items(): + if not tag.startswith(prefix) or '_per_label_' in tag: + continue + number = value if ( + isinstance(value, (float, int)) + and not isinstance(value, bool) + and abs(value) <= sys.float_info.max) else None + series.setdefault(tag, []).append((int(epoch), number)) + if not series: + continue + for tag, points in series.items(): + points.sort(key=lambda point: point[0]) + # The TensorBoard hook adjusts each series independently. + if points[0][0] == 0: + series[tag] = [(epoch + 1, value) for epoch, value in points] + tags.update(series) + experiments.append(series) + if not experiments: + return None + metrics = list(dict.fromkeys(tag.split('_', 1)[1] for tag in sorted(tags))) + tune = run.training_job.get('tune_metric') + if tune in metrics: + metrics.remove(tune) + metrics.insert(0, tune) + return { + 'id': 'history', + 'label': 'Epoch history', + 'metrics': metrics, + 'experiments': experiments + } + + +def card(run: Run) -> Optional[dict[str, Any]]: + fit = fits(run) + base = baselines(run) + tiles = [ + t for m in tile_metrics(run) + if (t := tile(run, fit, m, base)) is not None + ] + views = [] + if tiles: + views.extend([ + { + 'id': + 'baselines', + 'label': ('Against random' if run.model_type == LINK_PREDICTION + else 'Against baselines'), + 'tiles': + tiles + }, + { + 'id': 'table', + 'label': 'All metrics', + 'table': table(run, fit) + }, + ]) + history = _history(run) + if history is not None: + views.append(history) + if not views: + return None + coverage = run.coverage.line(run.coverage.metrics) + if run.coverage.holdout_truncated: + coverage = ' · '.join( + filter(None, [coverage, run.coverage.holdout_line()])) + return { + 'title': 'Metrics', + 'views': views, + 'coverage': coverage if tiles else None, + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_regression.py b/packages/kumo-engine/kumo/engine/_explainability/_regression.py new file mode 100644 index 000000000..4779cb909 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_regression.py @@ -0,0 +1,372 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regression card: Predicted against actual, from the published model's +holdout predictions and the training table's label histogram bin width.""" + +import math +from dataclasses import fields +from typing import Any, Optional + +import numpy as np + +from kumo.engine._explainability import _files +from kumo.engine._explainability import _format as fmt +from kumo.engine._explainability._header import target +from kumo.engine._explainability._holdout import FOLD_AT_MOST, equal_count_groups +from kumo.engine._explainability._run import Run + +ZERO_BAND_SHARE = 0.10 +BANDS = 5 +STATS = (('min', 'min'), ('p25', '25th percentile'), ('median', 'median'), + ('mean', 'mean'), ('p75', '75th percentile'), ('max', 'max')) + + +def _tenths(run: Run, actual: np.ndarray, pred: np.ndarray, + entities: np.ndarray) -> dict[str, Any]: + group = equal_count_groups(pred, entities) + occupied = np.unique(group) + means = [(float(pred[group == k].mean()), float(actual[group == k].mean())) + for k in occupied] + reference = float(actual.mean()) + texts = fmt.numbers([v for pair in means for v in pair] + [reference]) + gaps = [abs(p - a) for p, a in means] + worst = gaps.index(max(gaps)) + return { + 'id': + 'tenths', + 'label': + 'By predicted tenth', + 'x': + 'predicted tenth, lowest to highest', + 'y': + 'average ' + (target(run, short=True) or 'value'), + 'series': ['average prediction', 'average actual'], + 'tenths': [{ + 'label': str(occupied[k] + 1), + 'predicted': p, + 'predicted_text': texts[2 * k], + 'actual': a, + 'actual_text': texts[2 * k + 1], + } for k, (p, a) in enumerate(means)], + 'reference': + reference, + 'reference_callout': + f'all {run.entities}: {texts[-1]}', + 'highlight': + worst, + 'gap_callout': + f'tenth {occupied[worst] + 1}: predicted {texts[2 * worst]} · actual ' + f'{texts[2 * worst + 1]}', + } + + +def _bands(run: Run, actual: np.ndarray, pred: np.ndarray, + entities: np.ndarray) -> dict[str, Any]: + n = len(actual) + band = np.full(n, -1) + zero = actual == 0 + labels: list[str] = [] + if zero.mean() >= ZERO_BAND_SHARE: + band[zero] = 0 + labels.append('0') + rest, parts, first = np.where(~zero)[0], BANDS - 1, 1 + else: + rest, parts, first = np.arange(n), BANDS, 0 + if len(rest): + group = equal_count_groups(actual[rest], entities[rest], parts) + band[rest] = first + group + error = pred - actual + squared = error**2 + total = float(squared.sum()) + rows = [] + for b in range(first + parts if len(rest) else first): + mask = band == b + if not mask.any(): + continue + if b >= first: + low, high = fmt.numbers([actual[mask].min(), actual[mask].max()]) + label = f'{low} to {high}' + else: + label = labels[0] + share = float(squared[mask].sum()) / total if total else 0.0 + median = float(np.median(error[mask])) + rows.append({ + 'label': label, + 'rows': int(mask.sum()), + 'rows_text': f'{fmt.count(mask.sum())} {run.entities}', + 'median': median, + 'median_text': fmt.signed(median), + 'error_share': share, + 'error_share_text': fmt.percent(share), + }) + worst = max(range(len(rows)), key=lambda i: rows[i]['error_share']) + return { + 'id': + 'bands', + 'label': + 'By actual value', + 'x': + 'actual value', + 'y': + 'median of predicted minus actual', + 'bands': + rows, + 'highlight': + worst, + 'share_callout': + f'{fmt.percent(rows[worst]["error_share"])} of squared error', + } + + +def bin_width(run: Run) -> Optional[float]: + stats = _files.split_stats(run.files) or {} + test = stats.get('test_table_stats') or {} + distribution = test.get('target_distribution') + histogram = distribution.get('histogram') if isinstance(distribution, + dict) else None + width = histogram.get('bin_width') if isinstance(histogram, dict) else None + if isinstance(width, (int, float)) and not isinstance( + width, bool) and math.isfinite(width) and width > 0: + return float(width) + return None + + +def _bin(values: np.ndarray, width: float) -> np.ndarray: + return np.floor(values / width).astype(int) + + +def _edges(width: float, kept: int, folded: bool, start: int = 0) -> list[str]: + edges = fmt.numbers([(start + i) * width for i in range(kept + 1)]) + labels = [f'{edges[i]} to {edges[i + 1]}' for i in range(kept)] + if folded: + labels.append(f'≥ {edges[kept]}') + return labels + + +def _heatmap(run: Run, width: Optional[float]) -> Optional[dict[str, Any]]: + from kumo_core.train.output import EvalOutput, TargetPredictionHeatMap + + saved = _files.read_json( + _files._join(run.files.trial, 'test_target_prediction_heatmap', + 'heatmap.json')) + heatmap = None + if isinstance(saved, dict): + try: + heatmap = TargetPredictionHeatMap(**{ + f.name: saved[f.name] + for f in fields(TargetPredictionHeatMap) + }) + bounds = (heatmap.cell_width, heatmap.label_bin_start, + heatmap.label_bin_end, heatmap.prediction_bin_start, + heatmap.prediction_bin_end) + if (not all(math.isfinite(v) + for v in bounds) or heatmap.cell_width <= 0 + or heatmap.label_bin_end < heatmap.label_bin_start + or heatmap.prediction_bin_end < heatmap.prediction_bin_start + or not heatmap.counts + or len(heatmap.cells) != len(heatmap.counts) + or any(not isinstance(n, int) or n < 0 + for n in heatmap.counts) or sum(heatmap.counts) == 0 + or any(not (heatmap.label_bin_start <= x <= heatmap. + label_bin_end and heatmap.prediction_bin_start + <= y <= heatmap.prediction_bin_end) + for x, y in heatmap.cells)): + heatmap = None + except (KeyError, TypeError, ValueError): + heatmap = None + coverage = None + if heatmap is None: + # Match the producer's 10,000-row limit; keep reopened views stable. + if run.holdout is None or width is None: + return None + frame = run.holdout[['TARGET', 'TARGET_PRED']].copy() + frame = frame[np.isfinite(frame['TARGET']) + & np.isfinite(frame['TARGET_PRED'])] + if frame.empty: + return None + sample = frame.sample(min(len(frame), 10_000), random_state=0) + heatmap = EvalOutput._generate_prediction_heatmap( + sample, 'TARGET', 'TARGET_PRED', width, '') + coverage = (f'Based on {fmt.count(len(sample))} of ' + f'{fmt.count(len(frame))} retained holdout rows.') + # Size the aligned axes before allocating, including constant-axis cases. + start = min(heatmap.label_bin_start, heatmap.prediction_bin_start) + x_span = heatmap.label_bin_end - start + y_span = heatmap.prediction_bin_end - start + cell_width = max(heatmap.cell_width, + math.sqrt(x_span) * math.sqrt(y_span) / 100, + max(x_span, y_span) / 9999) + while True: + columns = math.floor(x_span / cell_width + 1e-9) + 1 + rows = math.floor(y_span / cell_width + 1e-9) + 1 + if columns * rows <= 10_000: + break + cell_width *= max(1.01, math.sqrt(columns * rows / 10_000)) + xs = start + np.arange(columns) * cell_width + ys = start + np.arange(rows) * cell_width + cells = np.zeros((rows, columns), dtype=int) + for (x, y), count in zip(heatmap.cells, heatmap.counts): + row = min(len(ys) - 1, max(0, round((y - ys[0]) / cell_width))) + col = min(len(xs) - 1, max(0, round((x - xs[0]) / cell_width))) + cells[row, col] += count + + def view(matrix: np.ndarray) -> dict[str, Any]: + rows, columns = matrix.shape + x_edges = [float(xs[0] + i * cell_width) for i in range(columns + 1)] + y_edges = [float(ys[0] + i * cell_width) for i in range(rows + 1)] + row, col = np.unravel_index(int(np.argmax(matrix)), matrix.shape) + total = sum(heatmap.counts) + return { + 'columns': [ + f'{fmt.number(a)} to {fmt.number(b)}' + for a, b in zip(x_edges, x_edges[1:]) + ], + 'rows': [ + f'{fmt.number(a)} to {fmt.number(b)}' + for a, b in zip(y_edges, y_edges[1:]) + ], + 'x_edges': + x_edges, + 'y_edges': + y_edges, + 'width': + cell_width, + 'counts': + matrix.tolist(), + 'densest': [int(row), int(col)], + 'densest_callout': + f'{fmt.count(matrix[row, col])} {run.entities} ' + f'({fmt.percent(matrix[row, col] / total)})', + } + + def kept(counts: np.ndarray) -> int: + indices = np.flatnonzero(counts > 10) + return int(indices[-1] + 1) if len(indices) else len(counts) + + trimmed = cells[:kept(cells.sum(axis=1)), :kept(cells.sum(axis=0))] + return { + 'id': 'heatmap', + 'label': 'Heatmap', + 'x': 'actual value', + 'y': 'predicted value', + **view(cells), + 'trimmed': view(trimmed), + 'coverage': coverage, + 'diagonal': 'predicted = actual', + } + + +def _stats(values: np.ndarray) -> dict[str, Any]: + numbers = { + 'min': float(values.min()), + 'p25': float(np.percentile(values, 25)), + 'median': float(np.median(values)), + 'mean': float(values.mean()), + 'p75': float(np.percentile(values, 75)), + 'max': float(values.max()), + } + texts = fmt.numbers([numbers[key] for key, _ in STATS]) + return { + 'values': + numbers, + 'items': [{ + 'label': label, + 'text': text + } for (_, label), text in zip(STATS, texts)], + } + + +def _distributions(run: Run, actual: np.ndarray, pred: np.ndarray, + width: float) -> dict[str, Any]: + a_bin, p_bin = _bin(actual, width), _bin(pred, width) + start = int(min(a_bin.min(), p_bin.min())) + a_bin, p_bin = a_bin - start, p_bin - start + indices, inverse = np.unique(np.r_[a_bin, p_bin], return_inverse=True) + size = int(indices[-1]) + 1 + a_counts = np.bincount(inverse[:len(actual)], minlength=len(indices)) + p_counts = np.bincount(inverse[len(actual):], minlength=len(indices)) + heavy = indices[a_counts + p_counts > FOLD_AT_MOST] + kept = max(1, int(heavy[-1]) + 1) if len(heavy) else 1 + folded = kept < size + if folded: + keep = indices < kept + a_counts = np.r_[a_counts[keep], a_counts[~keep].sum()] + p_counts = np.r_[p_counts[keep], p_counts[~keep].sum()] + indices = np.r_[indices[keep], kept] + bin_count = kept + int(folded) + sparse = {} + if bin_count <= 1000: + dense = np.zeros((2, bin_count), dtype=int) + dense[:, indices] = [a_counts, p_counts] + a_counts, p_counts = dense + labels = _edges(width, kept, folded, start) + else: + sparse = {'indices': indices.tolist(), 'bin_count': bin_count} + labels = [_edges(width, 1, False, start + int(i))[0] for i in indices] + if folded: + labels[-1] = f'≥ {fmt.number((start + kept) * width)}' + a_stats, p_stats = _stats(actual), _stats(pred) + return { + 'id': + 'distributions', + 'label': + 'Distributions', + 'x': + target(run) or 'value', + 'y': + run.entities, + 'bins': + labels, + **sparse, + 'start': + start * width, + 'width': + width, + 'series': [ + { + 'name': 'actual', + 'counts': [int(c) for c in a_counts], + 'stats': a_stats, + 'max_callout': f'highest actual {a_stats["items"][-1]["text"]}', + }, + { + 'name': 'predicted', + 'counts': [int(c) for c in p_counts], + 'stats': p_stats, + 'max_callout': + f'highest prediction {p_stats["items"][-1]["text"]}', + }, + ], + } + + +def predicted_card(run: Run) -> Optional[dict[str, Any]]: + views = [] + width = bin_width(run) + holdout = run.holdout + actual = pred = None + if holdout is not None and len(holdout): + actual = holdout['TARGET'].to_numpy(dtype=float) + pred = holdout['TARGET_PRED'].to_numpy(dtype=float) + keep = np.isfinite(actual) & np.isfinite(pred) + actual, pred = actual[keep], pred[keep] + if len(actual): + entities = holdout['ENTITY'].to_numpy()[keep] + views.extend([ + _tenths(run, actual, pred, entities), + _bands(run, actual, pred, entities) + ]) + heatmap = _heatmap(run, width) + if heatmap is not None: + views.append(heatmap) + if width is not None and actual is not None and pred is not None and len( + actual): + views.append(_distributions(run, actual, pred, width)) + if not views: + return None + return { + 'title': 'Predicted against actual', + 'views': views, + 'coverage': run.coverage.holdout_line(), + } diff --git a/packages/kumo-engine/kumo/engine/_explainability/_render.py b/packages/kumo-engine/kumo/engine/_explainability/_render.py new file mode 100644 index 000000000..c4bc7ae72 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_explainability/_render.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Notebook rendering: the computed data plus the static front end, through +the same widget layer as the graph visualization.""" + +import json +from functools import cache +from importlib.resources import files as resource_files +from typing import Any + +from kumo.engine._explainability._run import ( + CLASSIFICATION, + FORECASTING, + REGRESSION, +) + +# Shared front end first, then the file of the model type's own cards. +_SHARED = ('charts.js', 'shell.js', 'epoch-history.js', 'exports.js') +_TYPE_SCRIPT = { + CLASSIFICATION: 'classification.js', + REGRESSION: 'regression.js', + FORECASTING: 'forecasting.js', +} + + +@cache +def _static(name: str) -> str: + return (resource_files('kumo.engine._explainability') / 'static' / + name).read_text(encoding='utf-8') + + +def style() -> str: + return _static('widget.css') + + +def script(model_type: str) -> str: + names: list[str] = list(_SHARED) + if model_type in _TYPE_SCRIPT: + names.append(_TYPE_SCRIPT[model_type]) + if model_type != FORECASTING: + names.append('subgraphs.js') + code = '\n'.join(_static(name) for name in names) + # The widget layer runs the script inside one function; keep it from + # closing its own ') == 2 # the data and the code, nothing else + if shutil.which('node'): + # Validate the transported source, including script-tag escaping. + source = html.split('', + 1)[0] + subprocess.run(['node', '--check'], input=source, text=True, + capture_output=True, check=True) + + assert view._repr_html_().startswith('
{note}

' in html + assert note in list(ElementTree.fromstring(view._repr_svg_()).itertext()) + + +def test_anywidget_keeps_static_output_with_widget_model(tmp_path): + pytest.importorskip('anywidget') + run = _run_dir(tmp_path, test_rows=3, batch=64, max_test_steps=2000) + view = open_run(str(run)) + widget = view.as_widget() + try: + bundle, _ = widget._repr_mimebundle_() + assert bundle['application/vnd.jupyter.widget-view+json'][ + 'model_id'] == widget.model_id + assert '', { + 'missing': float('nan') + }], + 'score': None + }, + } + } + }, + } + _write_json(root / 'subgraph_json' / '0.json', graph) + if failure != 'missing': + other = json.loads(json.dumps(graph)) + if failure == 'dangling': + other['subgraph_store']['edges'] = {'0': {'related': [99]}} + _write_json(root / 'subgraph_json' / '1.json', other) + if failure == 'malformed': + (root / 'subgraph_json' / '1.json').write_text('not json') + result = _subgraphs.card(run) + assert result is not None + assert len(result['samples']) == (2 if failure is None else 1) + assert result['unavailable_count'] == (0 if failure is None else 1) + sample = result['samples'][0] + assert sample['categories'] == ['random'] + assert sample['original_ids'] == {'random': 'account-x'} + assert sample['prediction'] is None and sample['target'] is None + assert sample['nodes'][0]['value'] == '9007199254740993' + assert [f['score'] for f in sample['nodes'][0]['features']] == [0.0, None] + assert result['entity_label'] == 'users' + json.dumps(result, allow_nan=False) + assert sample['nodes'][0]['features'][1]['value'][1] == {'missing': None} + assert _subgraphs.card(_run.load(_files.locate(run.files.bundle))) is None diff --git a/packages/kumo-engine/test/widget_exports.cjs b/packages/kumo-engine/test/widget_exports.cjs new file mode 100644 index 000000000..c3687dbf8 --- /dev/null +++ b/packages/kumo-engine/test/widget_exports.cjs @@ -0,0 +1,143 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// NODE_PATH must provide Playwright and its browsers. Inputs are retained widget HTML. +// Usage: node widget_exports.cjs training.html [forecasting.html] [regression.html] +const {chromium, firefox, webkit} = require('playwright'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'), os = require('node:os'), path = require('node:path'); +const {pathToFileURL} = require('node:url'); +const [training, forecast, regression] = process.argv.slice(2); +assert.ok(training, 'Provide a binary training widget with epoch history and importance.'); +async function plots(page, selector) { + const geometry = await page.locator(selector).evaluateAll(svgs => svgs.map(svg => { + const shapes = [...svg.querySelectorAll('path[d], rect, circle')]; + return {count: shapes.length, valid: shapes.every(shape => { + const box = shape.getBBox(); + return [box.x, box.y, box.width, box.height].every(Number.isFinite) && + !/NaN|Infinity/.test([...shape.attributes].map(a => a.value).join(' ')); + })}; + })); + assert.ok(geometry.length, 'Expected plotted SVGs, not only headings.'); + assert.ok(geometry.every(g => g.count > 0 && g.valid), JSON.stringify(geometry)); + return geometry.length; +} +async function print(page) { + const waiting = page.waitForEvent('popup'); + await page.getByRole('button', {name: 'Export PDF', exact: true}).evaluate(button => button.click()); + const popup = await waiting; await popup.waitForLoadState(); + assert.equal(await popup.locator('.kx-panel').count(), 1); + return popup; +} +async function save(page, filename) { + const waiting = page.waitForEvent('download'); + await page.getByRole('button', {name: 'Export HTML', exact: true}).click(); + await (await waiting).saveAs(filename); +} +async function alter(filename, change) { + const html = await fs.readFile(filename, 'utf8'); + await fs.writeFile(filename, html.replace(/(]+id="kumo-export-data"[^>]*>)([\s\S]*?)(<\/script>)/, + (_, start, json, end) => { const payload = JSON.parse(json); change(payload); + return start + JSON.stringify(payload).replaceAll('<', '\\u003c') + end; })); +} +(async () => { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'kumo-exports-')), results = []; + try { for (const engine of [chromium, firefox, webkit]) { + const browser = await engine.launch(), context = await browser.newContext({acceptDownloads: true}); + const errors = []; context.on('page', page => page.on('pageerror', error => errors.push(error.message))); + try { + const page = await context.newPage(); await page.goto(pathToFileURL(path.resolve(training)).href); + await page.getByRole('tab', {name: 'Explanation', exact: true}).click(); + const importance = page.locator('[data-card="importance"]'); + await importance.locator('[aria-label^="Show all"]').click(); + await importance.locator('g[role="button"]').nth(11).click(); + const expanded = await importance.locator('svg[role="img"]').textContent(); + const saved = path.join(tmp, engine.name() + '.html'); await save(page, saved); + const reopened = await context.newPage(); await reopened.goto(pathToFileURL(saved).href); + assert.equal(await reopened.locator('[data-card="importance"] svg[role="img"]').textContent(), expanded); + await page.getByRole('tab', {name: 'Evaluation', exact: true}).click(); + let metric, points; + for (const selected of ['Epoch history', 'All metrics']) { + await page.getByRole('button', {name: selected, exact: true}).click(); + if (selected === 'Epoch history') { + const picker = page.locator('.kx-history-controls select'); + await picker.selectOption({index: (await picker.locator('option').count()) - 1}, {force: true}); + metric = await picker.inputValue(); + await page.locator('.kx-history-legend input').nth(1).uncheck(); + assert.equal(await plots(page, '.kx-history-plots svg'), 2); + points = await page.locator('.kx-history-plots circle title').allTextContents(); + assert.ok(points.length > 0); + } + await page.evaluate(() => scrollTo(0, 100)); + const position = await page.evaluate(() => [scrollX, scrollY]); + const popup = await print(page); + assert.equal(await plots(popup, '.kx-history-plots svg'), 2); + await plots(popup, '.kx-chart svg'); + if (metric) { + assert.equal(await popup.locator('.kx-history-legend input').nth(1).isChecked(), false); + assert.equal(await popup.locator('.kx-history-controls select').inputValue(), metric); + assert.deepEqual(await popup.locator('.kx-history-plots circle title').allTextContents(), points); + } + for (const label of ['All metrics', 'Epoch history', 'Against baselines', 'Confusion matrix', 'ROC', 'Precision-recall', 'Cumulative gain']) + assert.ok((await popup.locator('body').textContent()).includes(label), label); + await page.waitForTimeout(200); + assert.deepEqual(await page.evaluate(() => [scrollX, scrollY]), position); + await popup.close(); + } + await save(page, saved); await reopened.goto(pathToFileURL(saved).href); + await reopened.getByRole('button', {name: 'Epoch history', exact: true}).click(); + assert.equal(await reopened.locator('.kx-history-controls select').inputValue(), metric); + assert.equal(await reopened.locator('.kx-history-legend input').nth(1).isChecked(), false); + assert.deepEqual(await reopened.locator('.kx-history-plots circle title').allTextContents(), points); + await alter(saved, payload => { payload.data.tabs = payload.data.tabs.filter(tab => tab.id === 'evaluation'); payload.saved.tab = 0; }); + await reopened.goto(pathToFileURL(saved).href); const single = await print(reopened); + assert.equal(await single.title(), 'Kumo · Evaluation'); await single.close(); + if (forecast) { + await page.goto(pathToFileURL(path.resolve(forecast)).href); await save(page, saved); + for (const values of [[-10, -5], [-10, 5]]) { + await alter(saved, payload => { const totals = payload.data.tabs.flatMap(t => t.cards).find(c => c.id === 'forecast').views.find(v => v.id === 'totals'); + totals.steps.forEach(step => { step.actual = values[0]; step.forecast = values[1]; }); payload.saved = null; }); + await reopened.goto(pathToFileURL(saved).href); + await reopened.locator('[data-card="forecast"] button[data-view="totals"]').click(); + await plots(reopened, '[data-card="forecast"] .kx-chart svg'); + assert.ok(await reopened.locator('[data-card="forecast"] .kx-chart svg').evaluate(svg => { + const height = svg.viewBox.baseVal.height, points = [...svg.querySelectorAll('circle')]; + return points.length > 0 && points.every(p => +p.getAttribute('cy') >= 0 && +p.getAttribute('cy') <= height); + }), 'Signed forecast markers must remain inside the plotted SVG.'); + } + } + if (regression) { + await page.goto(pathToFileURL(path.resolve(regression)).href); + await page.locator('[data-card="predicted"] [data-view="heatmap"]').click(); + await page.getByLabel('Hide outliers', {exact: true}).check(); + await plots(page, '.kx-heatmap svg'); + const heatmap = await page.locator('.kx-heatmap svg').textContent(); + await page.locator('[data-card="predicted"] [data-view="tenths"]').click(); + await save(page, saved); await reopened.goto(pathToFileURL(saved).href); + await reopened.locator('[data-card="predicted"] [data-view="heatmap"]').click(); + assert.equal(await reopened.getByLabel('Hide outliers', {exact: true}).isChecked(), true); + assert.equal(await reopened.locator('.kx-heatmap svg').textContent(), heatmap); + const popup = await print(page); + assert.equal(await popup.getByLabel('Hide outliers', {exact: true}).isChecked(), true); + assert.equal(await popup.locator('.kx-heatmap svg').textContent(), heatmap); + await plots(popup, '.kx-heatmap svg'); await popup.close(); + await alter(saved, payload => { + const view = payload.data.tabs.flatMap(t => t.cards).find(c => c.id === 'predicted').views.find(v => v.id === 'distributions'); + Object.assign(view, {start: 10000, width: 0.02, bins: ['10000', '20001'], indices: [0, 500050], bin_count: 500051}); + view.series.forEach(series => { series.counts = [12, 24]; series.stats.values.max = 20001; }); + }); + await reopened.goto(pathToFileURL(saved).href); + await reopened.locator('[data-card="predicted"] [data-view="distributions"]').click(); + await plots(reopened, '[data-card="predicted"] .kx-chart svg'); + assert.ok(await reopened.locator('[data-card="predicted"] .kx-chart svg').evaluate(svg => { + const bars = [...svg.querySelectorAll('rect')]; + return bars.length === 4 && bars.every(bar => +bar.getAttribute('width') >= 0) && + +bars[1].getAttribute('x') - +bars[0].getAttribute('x') > 100 && svg.querySelectorAll('path').length < 100; + }), 'Sparse bins must keep their numeric positions without unbounded ticks.'); + const sparse = await print(reopened); await plots(sparse, '.kx-chart svg'); await sparse.close(); + } + assert.deepEqual(errors, []); results.push({browser: engine.name(), expandedImportance: true, selectedAndUnselectedHistoryPlots: true, singleTabTitle: true, signedForecast: !!forecast, hiddenHeatmapSetting: !!regression, sparseDistributions: !!regression}); + } finally { await browser.close(); } + } console.log(JSON.stringify(results, null, 2)); } finally { await fs.rm(tmp, {recursive: true, force: true}); } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/packages/kumo-engine/test/widget_validation.md b/packages/kumo-engine/test/widget_validation.md new file mode 100644 index 000000000..dde121b0b --- /dev/null +++ b/packages/kumo-engine/test/widget_validation.md @@ -0,0 +1,30 @@ +# Widget validation + +## Training widget + +Validated on Linux on 2026-11-19 using retained classification, regression, forecasting and link-prediction artifacts plus synthetic edge cases. +No training or prediction jobs ran. + +| Check | Result | +| --- | --- | +| `python -m pytest packages/kumo-engine/test/test_explainability.py -q` | 128 passed | +| Changed Python files: flake8 | Passed | +| Frontend JavaScript: `node --check` | Passed | +| Chromium, Firefox and WebKit export checks | Passed in all three browsers | + +The browser check verifies two plotted epoch SVGs when history is selected and when it is not, preserving the selected metric and experiments across view switches and matching plotted values to the live chart. +It also checks all active-page views, expanded importance and linked-column HTML restoration, unchanged notebook scroll position, the single-tab Evaluation title, all-negative/mixed forecast values inside the SVG, hidden heatmap outlier settings, and sparse distribution geometry spanning 500,051 bins. +Separated-range and constant-axis heatmaps conserve their counts within a 10,000-cell allocation bound. +The original PDF regression reproduced as two live epoch plots and zero printed plots; the corrected export contains both. + +Run the browser checks against HTML from retained binary training, forecasting and regression views: + +```sh +NODE_PATH= node packages/kumo-engine/test/widget_exports.cjs training.html forecasting.html regression.html +``` + +Save UTF-8 HTML documents with a `` declaration, wrapping the fragment returned by this checkout’s `kumo.engine._explainability.open_run(retained_run).html()`; the classification input must include epoch history and more than ten importance entries. +The script launches installed Chromium, Firefox and WebKit browsers and fails on browser errors or missing/invalid plot geometry. + +Frontend source fingerprint (SHA-256 over sorted static filenames, a null byte and each file’s contents): `bc1ab908778b4a5db6e6bd65bae117e50269cadfe2f0ee6e5bc76b2baadfa0a0`. +These checks cover local browser behavior, including offline HTML and the printable document; they do not prove native notebook print dialogs, comment-overlay placement or every host/OS combination.