diff --git a/.gitattributes b/.gitattributes
index 9b4730c3..a3169da1 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -37,3 +37,6 @@
# Keep this small, reviewed, output-free walkthrough readable in code reviews.
/examples/imaging-pipeline/run_imaging_pipeline.ipynb -filter diff merge text eol=lf
+
+# Model Alerts is reviewed and committed without notebook outputs.
+/examples/model-alerts/model-alerts.ipynb -filter diff merge text eol=lf
diff --git a/examples/model-alerts/PUBLISHING.md b/examples/model-alerts/PUBLISHING.md
new file mode 100644
index 00000000..a5463f48
--- /dev/null
+++ b/examples/model-alerts/PUBLISHING.md
@@ -0,0 +1,119 @@
+
+
+
+# Export and future publication
+
+Model Alerts is source plus a local static export workflow. Merging this
+example alone does not add it to versioned GitHub Pages. No public application
+server is needed: future hosting must consume the static artifacts only.
+
+## Execute, verify and render
+
+1. Select a complete source revision and preserve it throughout execution.
+ Provision a CUDA 13 environment as described in [README.md](README.md).
+ Run `run.py --profile full-reference` into a new directory outside Git,
+ using the shared GPU lock. Keep earlier failed/invalidated runs intact.
+2. Inspect `execution.json`, `study/freeze.json`, `study/report.json` and
+ scientific control outputs. Verify every `source_sha256` against the
+ matching example source, and every `output_sha256` against the named
+ artifact using SHA-256. The executed notebook must contain every cell,
+ warning and figure with no execution errors. A quick-smoke receipt is API
+ evidence only, even when execution succeeds.
+3. Use a separate CPU documentation environment with current Quarto, following
+ [the guide build instructions](../../docs/README.md). Install this example's
+ optional requirements and the repository's optional developer tools there.
+ Export into another new directory:
+
+ ```sh
+ /path/to/docs-env/bin/python examples/model-alerts/preview.py \
+ --executed /absolute/path/full-reference/model-alerts.executed.ipynb \
+ --output /absolute/path/new-model-alerts-render \
+ --source-ref PUSHED_SOURCE_COMMIT
+ ```
+
+ Replace `PUSHED_SOURCE_COMMIT` with the exact reviewed source SHA. The
+ renderer verifies executed/source hashes before reusing GPU cell outputs,
+ executes the ordinary CPU guide chapters, applies the current developer
+ version postprocessor and runs `docs-check`. Includes remain build inputs
+ outside published resource trees. No central publisher is changed.
+ `--allow-smoke` enables an explicitly labeled engineering preview only.
+4. Serve only the rendered book, never its parent containing data/weights:
+
+ ```sh
+ python -m http.server 8797 --bind 0.0.0.0 \
+ --directory /absolute/path/new-model-alerts-render/guide/_book
+ ```
+
+ Open `http://localhost:8797/model-alerts.html`. Verify every control, term
+ hover/focus/touch/Escape behavior, equations and light/dark theme. Check
+ exact pixel values and unchanged numerical data under display changes.
+ Repeat with scientific runtime network access blocked, zero WebSockets,
+ `file://` use, mobile/desktop and DPR 1/2, and a nested
+ `/cuPhoton/main/model-alerts.html` route. Verify visible GitHub source links.
+5. Audit final HTML and public receipts for private paths, account/machine
+ identifiers, unintended remote dependencies and restricted assets. Measure
+ source/artifact bytes and inspect arrays for avoidable duplication before
+ considering any precision change. Do not ship training data or weights
+ inside HTML.
+
+## Package reviewed static assets
+
+After qualification, package the entire book so navigation and downloads work:
+
+```sh
+export CUPHOTON_MODEL_ALERTS_EXPORT=/absolute/path/reviewed-export
+mkdir "$CUPHOTON_MODEL_ALERTS_EXPORT"
+tar -czf "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-guide.tar.gz" \
+ -C /absolute/path/model-alerts-render/guide/_book .
+cp /absolute/path/full-reference/model-alerts.executed.ipynb \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/"
+cp /absolute/path/full-reference/execution.json \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-execution.json"
+cp /absolute/path/full-reference/model-alerts-source.zip \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/"
+(
+ cd "$CUPHOTON_MODEL_ALERTS_EXPORT"
+ sha256sum model-alerts-guide.tar.gz model-alerts.executed.ipynb \
+ model-alerts-execution.json model-alerts-source.zip > model-alerts-SHA256SUMS
+)
+```
+
+Verify checksums after copying. An archive extracted locally must still open
+without an application server. Python's static HTTP server is optional.
+
+The following is a **future upload action**, after separate authorization and
+review of a matching existing Release tag:
+
+```sh
+gh release upload RELEASE_TAG --repo NVIDIA/cuPhoton \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-guide.tar.gz" \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts.executed.ipynb" \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-execution.json" \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-source.zip" \
+ "$CUPHOTON_MODEL_ALERTS_EXPORT/model-alerts-SHA256SUMS"
+```
+
+This task does not upload assets, merge, deploy or publish a software release.
+Keep existing Release assets intact; avoid replacement flags.
+
+## Versioned Pages integration gap
+
+The current CPU builder executes ordinary chapters with global `--execute`.
+It needs a receipt-verified pre-executed GPU notebook path, while continuing
+normal CPU execution. The local preview demonstrates staging that distinction;
+it does not change automatic publication.
+
+A future integration must stage this chapter only for revisions containing
+its source, rewrite source links to the selected tag/ref, and pass all current
+version postprocessing and documentation checks. Main and eligible release
+versions need durable checksummed storage for the large static export and its
+companions. The current publisher commits rendered files to `gh-pages`;
+Release-asset upload by itself does not prevent that growth. Restore verified
+static assets into temporary deployment trees rather than silently committing
+large HTML or falling back to a mutable latest release.
+
+Qualify a main build, an eligible release and an older release without this
+example. Check navigation/search/version switching, missing or corrupted
+assets, source links and preservation of existing versions. Expected future
+routes are `main/model-alerts.html` and `vX.Y.Z/model-alerts.html` beneath the
+cuPhoton Pages site. Normal publication/deployment remains a separate action.
diff --git a/examples/model-alerts/README.md b/examples/model-alerts/README.md
new file mode 100644
index 00000000..2bfcc8c6
--- /dev/null
+++ b/examples/model-alerts/README.md
@@ -0,0 +1,147 @@
+
+
+
+# Model Alerts — Which changes deserve attention?
+
+Generate independent reference/search exposures, detect positive xPois
+residuals, and train fresh native xScan pair/triplet classifiers. Follow the
+notebook from electron counts through labels, grouped splits, operating
+points, mistakes and a human review queue. Expanded science panels derive
+the assumptions; terminology links support hover, keyboard focus and touch.
+
+Every image, source, detector defect and noise realization is generated from
+declared analytic models. No survey imagery, calibration products, old demo
+datasets or pretrained checkpoints are inputs. The classifier results apply
+to this modeled population. They do not establish survey performance or
+calibrated astrophysical probabilities.
+
+## Execute in a clean environment
+
+Use Linux, Python 3.12 or newer supported by cuPhoton, and a CUDA 13-capable
+GPU with its driver. From a complete cuPhoton source checkout:
+
+```sh
+uv venv --python 3.12 .venv-model-alerts
+uv pip install --python .venv-model-alerts/bin/python -e '.[gpu]'
+uv pip install --python .venv-model-alerts/bin/python \
+ -r examples/model-alerts/requirements.txt
+```
+
+Use the appropriate CUDA 13 PyTorch index for your platform if it is needed
+by your package installation. Keep these example dependencies optional;
+ordinary cuPhoton installations do not install them. Bokeh, Jupyter and
+Matplotlib are used for visualization/execution; Beautiful Soup supports the
+repository's documentation checks. No third-party source is vendored.
+
+Choose an output directory outside the checkout that does not yet exist.
+Every session sharing a GPU must use the same advisory lock. This process
+retains that lock for the complete scientific workload:
+
+```sh
+export CUPHOTON_MODEL_ALERTS_GPU_LOCK=/absolute/path/shared/gpu0.lock
+.venv-model-alerts/bin/python examples/model-alerts/run.py \
+ --profile quick-smoke --output /absolute/path/new-api-smoke \
+ --gpu-lock "$CUPHOTON_MODEL_ALERTS_GPU_LOCK"
+.venv-model-alerts/bin/python examples/model-alerts/run.py \
+ --profile full-reference --output /absolute/path/new-full-reference \
+ --gpu-lock "$CUPHOTON_MODEL_ALERTS_GPU_LOCK"
+```
+
+Create the shared lock's parent directory first. Check actual GPU ownership
+as well as the lock before starting; do not interrupt another workload.
+Provision dependencies before blocking network access for scientific
+execution. Generation, training and inference make no network requests.
+
+Quick smoke uses small field counts and one training epoch. It exercises the
+APIs and export path and cannot substantiate scientific performance. Full
+reference starts with 180 training and 60 validation fields, reserves 60 test
+fields, and adds 60 shifted fields. Every field has three epochs. Development
+counts double if the primary validation field-bootstrap half-width exceeds
+five percentage points. All reference models complete 12 training epochs;
+minimum validation loss selects the checkpoint. Validation fields are reused
+for checkpoint and threshold selection, so the precision rule is a conditional
+development heuristic. Runtime and artifact sizes follow the actual counts.
+
+The runner snapshots source, executes a fresh notebook kernel and preserves
+stdout, warnings and figures. It writes `model-alerts.executed.ipynb`,
+`model-alerts-source.zip`, `execution.json`, and the separate `study/` data,
+weights and scientific receipts. The committed notebook stays output-free.
+`CUPHOTON_MODEL_ALERTS_OUTPUT` and `CUPHOTON_MODEL_ALERTS_PROFILE` are set by the
+runner for its kernel; a direct notebook session must set them and the shared
+lock explicitly.
+
+## Scientific controls and frozen evaluation
+
+The ontology precedes training. Unmatched detections stay negative; movers
+are separate astrophysical challenges. Ambiguous associations remain counted
+in coverage, separately from overlapping causes. Injection retention separates
+losses from detection, eligibility exclusions and classification. Training excludes those two special categories from primary
+binary fitting and evaluation states the same coverage boundary.
+
+Native xScan validation and label-provenance checks guard independent fields.
+All shared template noise, persistent defects and epochs remain together.
+Model inputs contain only measured image channels or explicitly enabled
+measured xFit features. Scaling is fitted on training data. Pair/triplet use
+the same candidate pool and three initialization seeds. Controls include
+SNR-only and measured-statistics baselines, one fixed training-label
+permutation, nested development learning curves and a rejected invalid split.
+xFit fusion uses native exact-candidate/stamp correspondence checks and
+reports failed-fit coverage. Raw nonfinite and repeated-extreme fractions
+are measured by binary class to check trivial generator signatures.
+
+Each model/seed uses the highest inclusive score threshold achieving at least
+95% validation recall, including tied scores. The final endpoint is test FPR
+at that frozen threshold; achieved test and shift recall are reported
+separately. Field bootstrap preserves epoch dependence and pairs comparisons
+on the same fields. Seed outcomes are shown individually. Both evaluation
+populations include family, SNR and overlapping-cause breakdowns. Local
+registration offsets vary by star; brightening hosts are randomized among
+non-mismatch stars. The development receipt counts bipolar registration
+residuals. All generator mixture amplitudes are declared in `Design`.
+
+`study/freeze.json` precedes reserved-field generation. Never retune after
+unblinding. If a bug invalidates final evaluation, retain that attempt and set
+fresh disjoint `CUPHOTON_MODEL_ALERTS_TEST_SEED_BASE` and
+`CUPHOTON_MODEL_ALERTS_SHIFT_SEED_BASE` blocks for a new full reference. Smoke
+uses different default blocks from the full reference. Unexpected permutation
+predictability stops full execution before freeze for diagnosis.
+
+## Static interactions
+
+| Live feature | Static equivalent or local operation |
+| --- | --- |
+| Image panels, linked pan/zoom and tapped pixels | Reused xScan Bokeh image/inspector primitives with browser callbacks |
+| Raw versus display transformations | Browser-only transforms; exact raw values retained |
+| Candidate navigation and family/outcome filters | Saved deterministic gallery of at most 96 candidates |
+| Score/SNR and failure inspection | Complete evaluation score plot and stratified failure gallery |
+| Threshold and queue choices | Saved scores for all eligible evaluation candidates |
+| Prevalence changes | Fixed-class-conditional-mixture projection from complete TPR/FPR |
+| Annotation persistence | Optional local `cuphoton xscan review-bokeh --review-dir /path/to/generated-review` after preparing a queue from this generated dataset |
+| Training, inference or new data | Local execution only; browser controls do not rerun them |
+
+The gallery is selected by round-robin confusion category, SNR bin and family,
+then candidate-ID order. Empty categories remain empty. Metrics and queue
+projections use complete evaluation arrays, never gallery counts. The HTML
+contains gallery pixels and evaluation statistics; full training arrays and
+weights are separate artifacts.
+
+The source reuses Model Sky's public notebook presentation code and the
+existing xScan raw-comparison/Alard-Lupton display helpers. It does not read
+live demo fixtures or open a Bokeh server. Existing server viewer behavior
+is unchanged. See [PUBLISHING.md](PUBLISHING.md) for Quarto export, source/hash
+verification, packaging and the future versioned Pages integration work.
+
+## Check source and interactions
+
+```sh
+PYTHONPATH=src .venv-model-alerts/bin/python -m pytest \
+ examples/model-alerts/tests
+CUPHOTON_MODEL_ALERTS_CHROMIUM=/absolute/path/to/chromium \
+ PYTHONPATH=src .venv-model-alerts/bin/python -m pytest \
+ examples/model-alerts/tests/test_static_browser.py
+ruff check examples/model-alerts
+```
+
+The browser regression runs standalone HTML with network APIs blocked.
+Complete qualification also covers the final rendered chapter on desktop and
+mobile, themes, DPR, keyboard/touch, source links and a versioned static route.
diff --git a/examples/model-alerts/assets/guide-head.html b/examples/model-alerts/assets/guide-head.html
new file mode 100644
index 00000000..fbb3b456
--- /dev/null
+++ b/examples/model-alerts/assets/guide-head.html
@@ -0,0 +1,31 @@
+
+
+
diff --git a/examples/model-alerts/assets/guide-include.html b/examples/model-alerts/assets/guide-include.html
new file mode 100644
index 00000000..f1d49963
--- /dev/null
+++ b/examples/model-alerts/assets/guide-include.html
@@ -0,0 +1,3 @@
+
+
+
diff --git a/examples/model-alerts/assets/guide.css b/examples/model-alerts/assets/guide.css
new file mode 100644
index 00000000..4f6af8c9
--- /dev/null
+++ b/examples/model-alerts/assets/guide.css
@@ -0,0 +1,133 @@
+/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+/* SPDX-License-Identifier: Apache-2.0 */
+
+body {
+ --model-alerts-font: ui-sans-serif, system-ui, sans-serif;
+ --model-alerts-mono: ui-monospace, Consolas, monospace;
+}
+
+body.quarto-dark {
+ color-scheme: dark;
+ --model-alerts-background: #0b1008;
+ --model-alerts-panel: #121a0e;
+ --model-alerts-panel-raised: #1a2514;
+ --model-alerts-text: #f0f4eb;
+ --model-alerts-green: #76b900;
+ --model-alerts-accent: #b5ed39;
+ --model-alerts-line: #2b3b20;
+ --model-alerts-plot-background: #10180d;
+ --model-alerts-plot-text: #f0f4eb;
+ --model-alerts-plot-muted: #b7c9a9;
+ --model-alerts-plot-border: #34472b;
+ --model-alerts-plot-cyan: #67d5e5;
+ --model-alerts-plot-pink: #ed91bd;
+ --model-alerts-font: "NVIDIA Sans", ui-sans-serif, system-ui, -apple-system,
+ BlinkMacSystemFont, "Segoe UI", sans-serif;
+ --model-alerts-mono: ui-monospace, "Cascadia Code", "SFMono-Regular", Consolas,
+ "Liberation Mono", monospace;
+}
+
+body.quarto-light {
+ --model-alerts-green: #76b900;
+ color-scheme: light;
+ --model-alerts-background: #f7f9f4;
+ --model-alerts-panel: #ffffff;
+ --model-alerts-panel-raised: #edf3e6;
+ --model-alerts-text: #20291b;
+ --model-alerts-accent: #426900;
+ --model-alerts-line: #ccd7c2;
+ --model-alerts-plot-background: #ffffff;
+ --model-alerts-plot-text: #20291b;
+ --model-alerts-plot-muted: #4f6043;
+ --model-alerts-plot-border: #ccd7c2;
+ --model-alerts-plot-cyan: #007f99;
+ --model-alerts-plot-pink: #ab3d75;
+}
+
+main.content a.model-alerts-glossary-term {
+ color: inherit;
+ text-decoration: underline dotted var(--model-alerts-accent);
+ text-underline-offset: 0.2em;
+ text-decoration-thickness: 1.5px;
+ cursor: help;
+}
+main.content a.model-alerts-glossary-term:hover,
+main.content a.model-alerts-glossary-term:focus-visible { color: var(--model-alerts-accent); }
+.model-alerts-glossary-term:focus-visible, .model-alerts-science summary:focus-visible {
+ outline: 2px solid var(--model-alerts-accent);
+ outline-offset: 4px;
+ border-radius: 2px;
+}
+main.content .model-alerts-glossary dt {
+ margin-top: 1.25rem;
+ font-weight: 700;
+ scroll-margin-top: 7rem;
+}
+main.content .model-alerts-glossary dd { margin: 0.3rem 0 0; max-width: 95ch; }
+.model-alerts-glossary dt:target, .model-alerts-glossary dt:focus {
+ color: var(--model-alerts-accent);
+ outline: 2px solid var(--model-alerts-accent);
+ outline-offset: 4px;
+}
+#model-alerts-glossary-tip {
+ position: fixed;
+ z-index: 10000;
+ box-sizing: border-box;
+ width: max-content;
+ max-width: min(25rem, calc(100vw - 24px));
+ max-height: calc(100dvh - 24px);
+ overflow-y: auto;
+ padding: 14px 16px;
+ color: var(--model-alerts-text);
+ background: var(--model-alerts-panel-raised);
+ border: 1px solid var(--model-alerts-accent);
+ border-radius: 8px;
+ box-shadow: 0 5px 24px rgb(0 0 0 / 35%);
+ font: 15px/1.55 var(--model-alerts-font);
+ overflow-wrap: break-word;
+}
+#model-alerts-glossary-tip[hidden] { display: none; }
+#model-alerts-glossary-tip strong { display: block; margin-bottom: 0.35em; color: var(--model-alerts-accent); }
+main.content details.model-alerts-science {
+ margin: 1.15rem 0;
+ padding: 0.8rem 1.1rem;
+ border: 1px solid var(--model-alerts-line);
+ border-left: 3px solid var(--model-alerts-green);
+ border-radius: 6px;
+ background: var(--model-alerts-panel);
+}
+.model-alerts-science summary { cursor: pointer; font-weight: 650; color: var(--model-alerts-accent); }
+.model-alerts-science[open] summary { margin-bottom: 1rem; }
+.model-alerts-equation {
+ margin: 1rem 0;
+ padding: 0.7rem 1rem;
+ overflow-x: auto;
+ color: var(--model-alerts-text);
+ background: var(--model-alerts-panel-raised);
+ border-radius: 4px;
+ font: 1.05rem/1.8 var(--model-alerts-mono);
+}
+@media print { #model-alerts-glossary-tip { display: none; } }
+
+.cell, .cell-output-display { min-width: 0; max-width: 100%; }
+.cell-output-display { overflow-x: auto; }
+.model-alerts-story-card { border: 1px solid var(--model-alerts-line); border-radius: 10px; background: var(--model-alerts-panel); overflow: hidden; }
+.model-alerts-story-grid { grid-template-columns: repeat(2, minmax(0, 1fr)) !important; }
+@media (max-width: 575px) {
+ .model-alerts-story-grid { grid-template-columns: 1fr !important; }
+}
+
+/* Keep the compact guide navbar inside a phone viewport. */
+@media (max-width: 575px) {
+ #quarto-header .navbar-container { flex-wrap: nowrap; min-width: 0; }
+ #quarto-header .navbar-brand-container { min-width: 0; flex-shrink: 1; }
+ #quarto-header .navbar-brand { margin-right: .35rem; }
+ #quarto-header .navbar-logo { max-height: 22px; }
+ #quarto-header .navbar-title { font-size: 1rem; }
+}
+.model-alerts-table { max-width: 100%; overflow-x: auto; margin: 1rem 0; }
+.model-alerts-table table { width: 100%; border-collapse: collapse; }
+.model-alerts-table th, .model-alerts-table td {
+ padding: .4rem .6rem; border-bottom: 1px solid var(--model-alerts-line);
+ text-align: left; vertical-align: top;
+}
diff --git a/examples/model-alerts/assets/guide.js b/examples/model-alerts/assets/guide.js
new file mode 100644
index 00000000..eb97d892
--- /dev/null
+++ b/examples/model-alerts/assets/guide.js
@@ -0,0 +1,240 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+(() => {
+ "use strict";
+ const root = document.body;
+ const original = new WeakMap();
+ const sourceColors = new WeakMap();
+ const styled = new WeakSet();
+ const discovered = new WeakSet();
+ const uiTypes = new Set(["Div", "Button", "CheckboxGroup", "RadioButtonGroup", "Slider", "Select", "Toolbar"]);
+
+ // Variables cross shadow boundaries and follow dynamic readouts without
+ // rewriting scientific callbacks or their text.
+ const uiCSS = `
+ :host { color: var(--model-alerts-plot-text) !important; color-scheme: inherit;
+ --bokeh-icon-color: var(--model-alerts-plot-muted);
+ --bokeh-icon-color-disabled: var(--model-alerts-plot-border); }
+ .bk-btn, .bk-input { color: var(--model-alerts-plot-text); background: var(--model-alerts-panel-raised);
+ border-color: var(--model-alerts-plot-border); }
+ .bk-btn:hover { background: var(--model-alerts-panel); }
+ .bk-btn.bk-active, .bk-btn-primary,
+ .bk-btn.bk-active:hover, .bk-btn-primary:hover { color: var(--model-alerts-background);
+ background: var(--model-alerts-accent); border-color: var(--model-alerts-accent); }
+ .bk-slider-title, .bk-input-group { color: var(--model-alerts-plot-text); }
+ .bk-toolbar-button { background: var(--model-alerts-panel-raised); }
+ input { accent-color: var(--model-alerts-accent); }
+ [style*="color:#f0f4eb"], [style*="color: #f0f4eb"],
+ [style*="color:#172b45"], [style*="color: #172b45"] { color: var(--model-alerts-plot-text) !important; }
+ [style*="color:#b7c9a9"], [style*="color: #b7c9a9"],
+ [style*="color:#52647c"], [style*="color: #52647c"] { color: var(--model-alerts-plot-muted) !important; }
+ [style*="background:#182310"], [style*="background:#f0f6fa"] {
+ background: var(--model-alerts-panel-raised) !important; }
+ `;
+
+ function remember(model, property) {
+ if (!original.has(model)) original.set(model, new Map());
+ const values = original.get(model);
+ if (!values.has(property)) values.set(property, model[property]);
+ return values.get(property);
+ }
+ function palette() {
+ const css = getComputedStyle(root);
+ const color = (name, fallback) => css.getPropertyValue(`--model-alerts-plot-${name}`).trim() || fallback;
+ return {
+ background: color("background", "#ffffff"), text: color("text", "#20291b"),
+ muted: color("muted", "#4f6043"), border: color("border", "#ccd7c2"),
+ cyan: color("cyan", "#007f99"), pink: color("pink", "#ab3d75"),
+ };
+ }
+
+ function styleDocument(doc, light, colors) {
+ const replacements = new Map([
+ ["#67d5e5", colors.cyan], ["#007f99", colors.cyan], ["#00bed6", colors.cyan],
+ ["#ed91bd", colors.pink], ["#ab3d75", colors.pink],
+ ["#82938a", colors.muted], ["#93a6b8", colors.muted], ["#d97a00", "#a35600"],
+ ]);
+ const replace = (value) => typeof value === "string"
+ ? (replacements.get(value.toLowerCase()) || value) : value;
+ for (const model of doc._all_models.values()) {
+ const type = model.type;
+ // Never theme image pixels, color maps, limits or palettes.
+ if (["Image", "ImageRGBA", "LinearColorMapper"].includes(type)) continue;
+ const set = (property, value, restoreDark = false) => {
+ if (!(property in model.properties)) return;
+ const before = remember(model, property);
+ const after = restoreDark && !light ? before : value;
+ if (model[property] !== after) model[property] = after;
+ };
+ if (type === "Figure" || type === "Plot") {
+ set("background_fill_color", colors.background);
+ set("border_fill_color", colors.background);
+ set("outline_line_color", colors.border);
+ } else if (type.endsWith("Axis")) {
+ set("axis_label_text_color", colors.muted);
+ set("major_label_text_color", colors.muted);
+ set("axis_line_color", colors.border);
+ set("major_tick_line_color", colors.border);
+ } else if (type === "Grid") {
+ set("grid_line_color", colors.border);
+ } else if (type === "Title") {
+ set("text_color", colors.text);
+ } else if (type === "ColorBar") {
+ set("background_fill_color", colors.background);
+ set("major_label_text_color", colors.muted);
+ set("title_text_color", colors.text);
+ set("major_tick_line_color", colors.muted);
+ } else if (type === "Legend") {
+ set("background_fill_color", colors.background);
+ set("label_text_color", colors.text);
+ set("title_text_color", colors.text);
+ set("border_line_color", colors.border);
+ }
+ // Change only known presentation colors, leaving field transforms intact.
+ for (const property of ["line_color", "fill_color"]) {
+ if (!(property in model.properties)) continue;
+ const before = remember(model, property);
+ if (typeof before === "string") {
+ set(property, replace(before));
+ } else if (before && typeof before.value === "string") {
+ const value = replace(before.value);
+ set(property, value === before.value ? before : {...before, value});
+ }
+ }
+ if (type === "ColumnDataSource" && Array.isArray(model.data.color)
+ && model.data.color.every((value) => typeof value === "string")) {
+ if (!sourceColors.has(model)) sourceColors.set(model, model.data.color.slice());
+ const colors = sourceColors.get(model).map(replace);
+ if (colors.some((value, index) => value !== model.data.color[index])) {
+ // Numerical arrays and selections retain their current objects.
+ model.data = {...model.data, color: colors};
+ }
+ }
+ if (uiTypes.has(type) && !styled.has(model)) {
+ model.stylesheets = [...model.stylesheets, uiCSS];
+ styled.add(model);
+ }
+ }
+ }
+
+ function apply(theme) {
+ root.dataset.modelAlertsTheme = theme;
+ const light = theme === "light";
+ const colors = palette();
+ for (const doc of window.Bokeh?.documents || []) {
+ styleDocument(doc, light, colors);
+ discovered.add(doc);
+ }
+ }
+
+ const sync = () => apply(document.body.classList.contains("quarto-dark") ? "dark" : "light");
+ sync();
+ new MutationObserver(sync).observe(document.body, {attributes: true, attributeFilter: ["class"]});
+
+ // Bokeh initializes asynchronously. Discover each document once, stop after
+ // ten seconds, and let subsequent theme clicks cover any later documents.
+ let attempts = 80;
+ function discoverDocuments() {
+ const colors = palette();
+ for (const doc of window.Bokeh?.documents || []) {
+ if (!discovered.has(doc)) {
+ styleDocument(doc, root.dataset.modelAlertsTheme === "light", colors);
+ discovered.add(doc);
+ }
+ }
+ if (--attempts > 0) window.setTimeout(discoverDocuments, 125);
+ }
+ discoverDocuments();
+})();
+
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Definitions remain ordinary document content. Clicking, touching, printing
+// and disabled JavaScript all retain the same internal definition links.
+(() => {
+ const links = document.querySelectorAll('a.model-alerts-glossary-term');
+ if (!links.length) return;
+ const tip = document.createElement('div');
+ tip.id = 'model-alerts-glossary-tip';
+ tip.setAttribute('role', 'tooltip');
+ tip.hidden = true;
+ document.body.append(tip);
+ let active = null;
+ let hideTimer;
+
+ function hide() {
+ window.clearTimeout(hideTimer);
+ active?.removeAttribute('aria-describedby');
+ active = null;
+ tip.hidden = true;
+ }
+
+ function position() {
+ if (!active || tip.hidden) return;
+ const anchor = active.getBoundingClientRect();
+ const box = tip.getBoundingClientRect();
+ const margin = 12;
+ const left = Math.max(margin, Math.min(
+ anchor.left + anchor.width / 2 - box.width / 2,
+ window.innerWidth - box.width - margin,
+ ));
+ const below = anchor.bottom + 9;
+ const above = anchor.top - box.height - 9;
+ const top = below + box.height <= window.innerHeight - margin ? below : above;
+ tip.style.left = `${left}px`;
+ tip.style.top = `${Math.max(margin, Math.min(top, window.innerHeight - box.height - margin))}px`;
+ }
+
+ function show(link) {
+ window.clearTimeout(hideTimer);
+ const term = document.getElementById(link.hash.slice(1));
+ const definition = term?.nextElementSibling;
+ if (definition?.tagName !== 'DD') return;
+ active?.removeAttribute('aria-describedby');
+ active = link;
+ const heading = document.createElement('strong');
+ heading.textContent = term.textContent;
+ const body = document.createElement('span');
+ body.textContent = definition.textContent;
+ tip.replaceChildren(heading, body);
+ tip.hidden = false;
+ link.setAttribute('aria-describedby', tip.id);
+ position();
+ }
+
+ function later() {
+ window.clearTimeout(hideTimer);
+ hideTimer = window.setTimeout(hide, 180);
+ }
+
+ for (const link of links) {
+ // Avoid a second native title popup only once this enhancement exists.
+ link.removeAttribute('title');
+ link.addEventListener('pointerenter', (event) => {
+ if (event.pointerType !== 'touch') show(link);
+ });
+ link.addEventListener('pointerleave', () => {
+ if (document.activeElement !== link) later();
+ });
+ link.addEventListener('focus', () => show(link));
+ link.addEventListener('blur', later);
+ link.addEventListener('click', () => {
+ hide();
+ const term = document.getElementById(link.hash.slice(1));
+ // Preserve native anchor navigation/history; Back restores the reader.
+ window.setTimeout(() => term?.focus({ preventScroll: true }), 0);
+ });
+ }
+ tip.addEventListener('pointerenter', () => window.clearTimeout(hideTimer));
+ tip.addEventListener('pointerleave', later);
+ document.addEventListener('keydown', (event) => {
+ if (event.key === 'Escape') hide();
+ });
+ document.addEventListener('pointerdown', (event) => {
+ if (active && !active.contains(event.target) && !tip.contains(event.target)) hide();
+ });
+ document.addEventListener('scroll', position, true);
+ window.addEventListener('resize', position);
+})();
diff --git a/examples/model-alerts/baselines.py b/examples/model-alerts/baselines.py
new file mode 100644
index 00000000..50447310
--- /dev/null
+++ b/examples/model-alerts/baselines.py
@@ -0,0 +1,226 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Measured-image controls; no injection metadata enters the features."""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.optimize import minimize
+from scipy.special import expit
+
+FEATURE_NAMES = (
+ "measured_detection_snr",
+ "positive_core_fraction",
+ "negative_absolute_flux_fraction",
+ "difference_peak_to_rms",
+ "search_template_correlation",
+ "nonfinite_fraction",
+ "repeated_extreme_fraction",
+)
+
+
+def measured_features(images, snr):
+ """Simple statistics from complete, unclipped raw electron cutouts.
+
+ Measure nonfinite pixels and repeated extrema rather than assuming they
+ are absent. Tied extrema can indicate clipping or quantization; they do
+ not prove either. Missing pixels are zero-filled only for this diagnostic
+ calculation; generated model inputs are required to be finite.
+ """
+ images = np.asarray(images, dtype=float)
+ snr = np.asarray(snr, dtype=float)
+ if (
+ images.ndim != 4
+ or images.shape[1] != 3
+ or snr.shape != (len(images),)
+ ):
+ raise ValueError("candidate triplets and one measured SNR required")
+ nonfinite = (~np.isfinite(images)).mean(axis=(1, 2, 3))
+ images = np.where(np.isfinite(images), images, 0)
+ planes = images.reshape(len(images), 3, -1)
+ low = (planes == planes.min(axis=2, keepdims=True)).sum(axis=2)
+ high = (planes == planes.max(axis=2, keepdims=True)).sum(axis=2)
+ repeated = np.minimum(
+ planes.shape[2], np.maximum(low - 1, 0) + np.maximum(high - 1, 0)
+ ).sum(axis=1) / (3 * planes.shape[2])
+ search, reference, difference = images[:, 0], images[:, 1], images[:, 2]
+ positive = np.maximum(difference, 0)
+ absolute = np.abs(difference).sum(axis=(1, 2)) + 1e-12
+ center = difference.shape[-1] // 2
+ core = positive[:, center - 2 : center + 3, center - 2 : center + 3].sum(
+ axis=(1, 2)
+ )
+ rms = np.sqrt(np.mean(difference**2, axis=(1, 2))) + 1e-12
+ a = search.reshape(len(search), -1)
+ b = reference.reshape(len(reference), -1)
+ a = a - a.mean(axis=1, keepdims=True)
+ b = b - b.mean(axis=1, keepdims=True)
+ correlation = (a * b).sum(axis=1) / np.sqrt(
+ np.maximum((a * a).sum(axis=1) * (b * b).sum(axis=1), 1e-24)
+ )
+ return np.column_stack(
+ (
+ snr,
+ core / absolute,
+ np.maximum(-difference, 0).sum(axis=(1, 2)) / absolute,
+ difference.max(axis=(1, 2)) / rms,
+ correlation,
+ nonfinite,
+ repeated,
+ )
+ )
+
+
+class LogisticBaseline:
+ """L2 logistic regression with training-only standardization.
+
+ Fixed regularization is a teaching choice, not tuned on test data.
+ Objective averages Bernoulli negative log likelihood plus L2/2n.
+ """
+
+ def fit(self, features, labels):
+ features, labels = np.asarray(features), np.asarray(labels)
+ if features.ndim != 2 or labels.shape != (len(features),):
+ raise ValueError("feature matrix and matching labels required")
+ if set(np.unique(labels)) != {0, 1}:
+ raise ValueError("both binary classes required")
+ self.center = features.mean(axis=0)
+ self.scale = features.std(axis=0)
+ self.scale[self.scale == 0] = 1
+ x = np.column_stack(
+ (np.ones(len(features)), (features - self.center) / self.scale)
+ )
+
+ def objective(weights):
+ logits = x @ weights
+ penalty = weights.copy()
+ penalty[0] = 0
+ loss = np.mean(np.logaddexp(0, logits) - labels * logits)
+ loss += 0.5 * np.dot(penalty, penalty) / len(labels)
+ gradient = x.T @ (expit(logits) - labels) / len(
+ labels
+ ) + penalty / len(labels)
+ return loss, gradient
+
+ result = minimize(
+ objective,
+ np.zeros(x.shape[1]),
+ jac=True,
+ method="L-BFGS-B",
+ options={"maxiter": 1000, "gtol": 1e-8},
+ )
+ if not result.success:
+ raise RuntimeError(
+ f"baseline optimization failed: {result.message}"
+ )
+ self.weights = result.x
+ return self
+
+ def predict(self, features):
+ x = np.column_stack(
+ (
+ np.ones(len(features)),
+ (np.asarray(features) - self.center) / self.scale,
+ )
+ )
+ return expit(x @ self.weights)
+
+ def receipt(self):
+ return dict(
+ center=self.center.tolist(),
+ scale=self.scale.tolist(),
+ weights=self.weights.tolist(),
+ l2_penalty=1,
+ fitted_split="train",
+ )
+
+
+def development_baselines(raw_dir, prepared_dir, output):
+ """Fit both baselines on training, freeze their validation thresholds."""
+ import json
+ from pathlib import Path
+
+ from evaluation import confusion, paired_field_bootstrap, recall_threshold
+
+ raw_dir, prepared_dir, output = map(Path, (raw_dir, prepared_dir, output))
+ if output.exists():
+ raise ValueError("baseline output must not exist")
+ normalization = json.loads(
+ (prepared_dir / "normalization.json").read_text()
+ )
+ selected = normalization["selected_raw_indices"]
+ images = np.stack(
+ [
+ np.load(raw_dir / f"{name}.npy", mmap_mode="r")[selected]
+ for name in ("search", "template", "difference")
+ ],
+ axis=1,
+ )
+ rows = [
+ json.loads(line)
+ for line in (prepared_dir / "metadata.jsonl").read_text().splitlines()
+ ]
+ features = measured_features(images, [r["snr"] for r in rows])
+ split = np.load(prepared_dir / "split.npy")
+ labels = np.load(prepared_dir / "labels.npy")
+ fields = np.array([r["field"] for r in rows])
+ output.mkdir(parents=True)
+ receipts = {}
+ for name, columns in (
+ ("snr_only", [0]),
+ ("measured_statistics", list(range(len(FEATURE_NAMES)))),
+ ):
+ model = LogisticBaseline().fit(
+ features[split == 0][:, columns], labels[split == 0]
+ )
+ scores = model.predict(features[:, columns])
+ validation = split == 1
+ threshold = recall_threshold(labels[validation], scores[validation])
+ interval = paired_field_bootstrap(
+ labels[validation],
+ scores[validation],
+ scores[validation],
+ fields[validation],
+ threshold,
+ threshold,
+ )
+ receipts[name] = dict(
+ model=model.receipt(),
+ threshold=threshold,
+ feature_names=[FEATURE_NAMES[i] for i in columns],
+ training_signature_audit={
+ str(label): {
+ FEATURE_NAMES[i]: dict(
+ minimum=float(
+ features[
+ (split == 0) & (labels == label), i
+ ].min()
+ ),
+ maximum=float(
+ features[
+ (split == 0) & (labels == label), i
+ ].max()
+ ),
+ nonzero=int(
+ np.count_nonzero(
+ features[(split == 0) & (labels == label), i]
+ )
+ ),
+ )
+ for i in (-2, -1)
+ }
+ for label in (0, 1)
+ },
+ validation=confusion(
+ labels[validation], scores[validation], threshold
+ ),
+ validation_fpr_interval=interval["a"],
+ )
+ np.savez(
+ output / f"{name}.npz", scores=scores, labels=labels, split=split
+ )
+ (output / "baselines.json").write_text(
+ json.dumps(receipts, indent=2) + "\n"
+ )
+ return receipts
diff --git a/examples/model-alerts/controls.py b/examples/model-alerts/controls.py
new file mode 100644
index 00000000..34cee516
--- /dev/null
+++ b/examples/model-alerts/controls.py
@@ -0,0 +1,127 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Development controls that retain native provenance guards."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+ARRAYS = ("search", "template", "difference", "labels", "split")
+
+
+def copy_rows(source, output, selected):
+ """Copy explicit rows to a fresh directory and retain provenance."""
+ source, output = Path(source), Path(output)
+ if output.exists():
+ raise ValueError("control output must not exist")
+ rows = [
+ json.loads(line)
+ for line in (source / "metadata.jsonl").read_text().splitlines()
+ ]
+ selected = np.asarray(selected, dtype=int)
+ if selected.ndim != 1 or len(selected) == 0:
+ raise ValueError("nonempty row selection required")
+ output.mkdir(parents=True)
+ for name in ARRAYS:
+ np.save(
+ output / f"{name}.npy",
+ np.load(source / f"{name}.npy", mmap_mode="r")[selected],
+ )
+ if (source / "variance.npy").is_file():
+ np.save(
+ output / "variance.npy",
+ np.load(source / "variance.npy", mmap_mode="r")[selected],
+ )
+ chosen = [rows[i] for i in selected]
+ (output / "metadata.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in chosen)
+ )
+ return chosen
+
+
+def learning_curve_subset(raw, output, field_count):
+ """Select first field IDs; keep validation fixed and omit all test rows.
+
+ Run prepare_training on this raw subset so its normalization is fit on
+ only the subset training fields. The final test is never consulted.
+ """
+ raw = Path(raw)
+ rows = [
+ json.loads(line)
+ for line in (raw / "metadata.jsonl").read_text().splitlines()
+ ]
+ fields = sorted({r["field"] for r in rows if r["split"] == "train"})
+ if not 1 <= field_count <= len(fields):
+ raise ValueError("field count must fit the development training pool")
+ selected_fields = set(fields[:field_count])
+ selected = [
+ i
+ for i, r in enumerate(rows)
+ if r["split"] == "val"
+ or (r["split"] == "train" and r["field"] in selected_fields)
+ ]
+ copy_rows(raw, output, selected)
+ return selected_fields
+
+
+def permutation_control(prepared, output, *, seed=991):
+ """Permute training labels once; retain validation truth.
+
+ A permutation is not a new scientific label. Every changed row explicitly
+ records its original label and the negative-control provenance. Validation
+ labels and images stay unchanged. Never select a seed from results.
+ """
+ from cuphoton.xscan.training import require_training_label_provenance
+
+ prepared, output = Path(prepared), Path(output)
+ split = np.load(prepared / "split.npy")
+ selected = np.flatnonzero(split != 2)
+ rows = copy_rows(prepared, output, selected)
+ labels = np.load(output / "labels.npy")
+ train = np.flatnonzero(np.load(output / "split.npy") == 0)
+ labels[train] = np.random.default_rng(seed).permutation(labels[train])
+ for i in train:
+ rows[i]["original_label"] = rows[i]["label"]
+ rows[i]["label"] = int(labels[i])
+ rows[i]["label_source"] = "permuted_training_negative_control"
+ np.save(output / "labels.npy", labels)
+ (output / "metadata.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in rows)
+ )
+ require_training_label_provenance(output)
+ return dict(
+ seed=seed, permuted_rows=len(train), validation_truth_unchanged=True
+ )
+
+
+def rejected_split_control(prepared, output):
+ """Prove that native training refuses candidate-level field leakage."""
+ from cuphoton.xscan.training import require_training_label_provenance
+
+ prepared, output = Path(prepared), Path(output)
+ split = np.load(prepared / "split.npy")
+ rows = copy_rows(prepared, output, np.flatnonzero(split != 2))
+ split = np.load(output / "split.npy")
+ train = np.flatnonzero(split == 0)
+ first = int(train[0])
+ if sum(r["split_group"] == rows[first]["split_group"] for r in rows) < 2:
+ raise ValueError("need multiple detections from a training field")
+ split[first] = 1
+ rows[first]["split"] = "val"
+ np.save(output / "split.npy", split)
+ (output / "metadata.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in rows)
+ )
+ try:
+ require_training_label_provenance(output)
+ except ValueError as error:
+ if "group leakage" not in str(error):
+ raise
+ return dict(rejected=True, reason=str(error))
+ raise AssertionError(
+ "native provenance guard accepted invalid field split"
+ )
diff --git a/examples/model-alerts/evaluation.py b/examples/model-alerts/evaluation.py
new file mode 100644
index 00000000..e6d78750
--- /dev/null
+++ b/examples/model-alerts/evaluation.py
@@ -0,0 +1,167 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Frozen operating points and field-based uncertainty for Model Alerts."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def checked(labels, scores):
+ labels, scores = np.asarray(labels), np.asarray(scores, dtype=float)
+ if labels.ndim != 1 or labels.shape != scores.shape or not len(labels):
+ raise ValueError(
+ "labels and scores must be nonempty matching vectors"
+ )
+ if not np.isin(labels, [0, 1]).all() or not np.isfinite(scores).all():
+ raise ValueError("binary labels and finite scores are required")
+ if np.any((scores < 0) | (scores > 1)):
+ raise ValueError("scores must lie in [0, 1]")
+ return labels.astype(bool), scores
+
+
+def recall_threshold(labels, scores, recall=0.95):
+ """Highest inclusive threshold satisfying the requested validation recall.
+
+ The kth positive order statistic retains all tied scores. Negative scores
+ cannot increase this threshold without losing the required positives.
+ """
+ labels, scores = checked(labels, scores)
+ if not 0 < recall <= 1 or not labels.any():
+ raise ValueError("recall must be in (0, 1] and positives must exist")
+ positive = np.sort(scores[labels])[::-1]
+ return float(positive[int(np.ceil(recall * len(positive))) - 1])
+
+
+def confusion(labels, scores, threshold):
+ labels, scores = checked(labels, scores)
+ if not np.isfinite(threshold):
+ raise ValueError("threshold must be finite")
+ retained = scores >= threshold
+ tp = int(np.sum(labels & retained))
+ fp = int(np.sum(~labels & retained))
+ fn = int(np.sum(labels & ~retained))
+ tn = int(np.sum(~labels & ~retained))
+ return dict(
+ tp=tp,
+ fp=fp,
+ fn=fn,
+ tn=tn,
+ recall=tp / (tp + fn) if tp + fn else None,
+ fpr=fp / (fp + tn) if fp + tn else None,
+ brier=float(np.mean((scores - labels) ** 2)),
+ )
+
+
+def queue_projection(tpr, fpr, prevalence, count):
+ """Expected queue under fixed class-conditional score distributions."""
+ values = np.asarray([tpr, fpr, prevalence], dtype=float)
+ if not np.isfinite(values).all() or np.any((values < 0) | (values > 1)):
+ raise ValueError("rates and prevalence must lie in [0, 1]")
+ if not np.isfinite(count) or count < 0:
+ raise ValueError("population count must be finite and nonnegative")
+ real, bogus = prevalence * tpr, (1 - prevalence) * fpr
+ return dict(
+ queue=count * (real + bogus),
+ completeness=tpr,
+ purity=real / (real + bogus) if real + bogus else None,
+ )
+
+
+def paired_field_bootstrap(
+ labels,
+ scores_a,
+ scores_b,
+ fields,
+ threshold_a,
+ threshold_b,
+ *,
+ seed=731,
+ draws=2000,
+):
+ """Field intervals for FPR and paired A minus B at frozen thresholds.
+
+ Sample fields with replacement, preserving every candidate and epoch in
+ each selected field. Replicates lacking negatives have undefined FPR and
+ are counted explicitly rather than replaced by zero.
+ """
+ labels, scores_a = checked(labels, scores_a)
+ _, scores_b = checked(labels, scores_b)
+ fields = np.asarray(fields)
+ if fields.shape != labels.shape or draws < 1:
+ raise ValueError(
+ "matching field identifiers and positive draws required"
+ )
+ unique, inverse = np.unique(fields, return_inverse=True)
+ negatives = np.bincount(inverse, weights=~labels)
+ false_a = np.bincount(
+ inverse, weights=(~labels & (scores_a >= threshold_a))
+ )
+ false_b = np.bincount(
+ inverse, weights=(~labels & (scores_b >= threshold_b))
+ )
+ rng = np.random.default_rng(seed)
+ outcomes = []
+ for _ in range(draws):
+ selected = rng.integers(len(unique), size=len(unique))
+ denominator = negatives[selected].sum()
+ if denominator:
+ a = false_a[selected].sum() / denominator
+ b = false_b[selected].sum() / denominator
+ outcomes.append((a, b, a - b))
+ if not outcomes:
+ raise ValueError("no bootstrap replicate contains negatives")
+ intervals = np.quantile(outcomes, [0.025, 0.975], axis=0).T
+ return dict(
+ zip(("a", "b", "a_minus_b"), intervals.tolist(), strict=False)
+ ) | {
+ "draws": draws,
+ "undefined_draws": draws - len(outcomes),
+ "fields": len(unique),
+ "seed": seed,
+ }
+
+
+def field_rate_intervals(
+ labels, scores, fields, threshold, *, seed=731, draws=2000
+):
+ """Field intervals for recall/FPR at a frozen inclusive threshold."""
+ labels, scores = checked(labels, scores)
+ fields = np.asarray(fields)
+ if fields.shape != labels.shape or draws < 1:
+ raise ValueError(
+ "matching field IDs and positive draw count required"
+ )
+ _, inverse = np.unique(fields, return_inverse=True)
+ counts = np.array(
+ [
+ np.bincount(inverse, weights=values)
+ for values in (
+ labels,
+ ~labels,
+ labels & (scores >= threshold),
+ ~labels & (scores >= threshold),
+ )
+ ]
+ )
+ rng = np.random.default_rng(seed)
+ rates = {"recall": [], "fpr": []}
+ for _ in range(draws):
+ sample = rng.integers(counts.shape[1], size=counts.shape[1])
+ positive, negative, tp, fp = counts[:, sample].sum(axis=1)
+ if positive:
+ rates["recall"].append(tp / positive)
+ if negative:
+ rates["fpr"].append(fp / negative)
+ return {
+ name: dict(
+ interval=np.quantile(values, [0.025, 0.975]).tolist()
+ if values
+ else None,
+ undefined_draws=draws - len(values),
+ draws=draws,
+ seed=seed,
+ )
+ for name, values in rates.items()
+ }
diff --git a/examples/model-alerts/fusion.py b/examples/model-alerts/fusion.py
new file mode 100644
index 00000000..a6d795eb
--- /dev/null
+++ b/examples/model-alerts/fusion.py
@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Exact-candidate, exact-stamp xFit feature investigation."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+
+
+def fit_feature_bundle(dataset, output, *, backend="cupy"):
+ """Use the native exporter, fit CLI and feature correspondence guards.
+
+ The caller must retain the shared GPU lock for CUDA backends. Inputs are
+ explicit generated datasets only, never demo fixtures. Failed/invalid
+ fits remain represented by the native feature gates.
+ """
+ from cuphoton.xscan.xfit_features import (
+ TRANSFORM_CONSTANTS,
+ build_xfit_feature_bundle,
+ export_xfit_input,
+ load_xfit_feature_matrix,
+ )
+
+ dataset, output = Path(dataset), Path(output)
+ if output.exists():
+ raise ValueError("xFit investigation output must not exist")
+ variance = dataset / "variance.npy"
+ if not variance.is_file():
+ raise ValueError("variance plane required for this investigation")
+ output.mkdir(parents=True)
+ archive, run, bundle = (
+ output / "input.npz",
+ output / "fit",
+ output / "features",
+ )
+ exported = export_xfit_input(
+ dataset_dir=dataset,
+ output_path=archive,
+ variance_path=variance,
+ image_unit="normalized_electron",
+ )
+ with (
+ (output / "fit.stdout").open("w") as stdout,
+ (output / "fit.stderr").open("w") as stderr,
+ ):
+ subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "cuphoton",
+ "xfit",
+ "fit-dipoles",
+ "--input",
+ str(archive),
+ "--output-dir",
+ str(run),
+ "--model",
+ "gaussian",
+ "--mode",
+ "difference",
+ "--backend",
+ backend,
+ "--compute-dtype",
+ "float64",
+ ],
+ stdout=stdout,
+ stderr=stderr,
+ check=True,
+ )
+ build_xfit_feature_bundle(
+ dataset_dir=dataset,
+ xfit_run_dir=run,
+ output_dir=bundle,
+ missing_policy="indicator",
+ )
+ matrix = load_xfit_feature_matrix(dataset_dir=dataset, feature_dir=bundle)
+ names = list(matrix.feature_names)
+ receipt = dict(
+ rows=len(matrix.values),
+ backend=backend,
+ feature_scaling=dict(
+ fitted_split=None,
+ strategy="fixed native row-local transforms",
+ constants=dict(TRANSFORM_CONSTANTS),
+ ),
+ source_verification=exported["source_hash_verification"],
+ native_join_diagnostics=matrix.join_diagnostics,
+ )
+ for name in ("fit_present", "fit_valid", "uncertainty_valid"):
+ values = matrix.values[:, names.index(name)]
+ receipt[name] = dict(
+ available=int(np.sum(values > 0.5)),
+ unavailable=int(np.sum(values <= 0.5)),
+ fraction=float(np.mean(values > 0.5)),
+ )
+ (output / "coverage.json").write_text(
+ json.dumps(receipt, indent=2) + "\n"
+ )
+ return bundle, receipt
diff --git a/examples/model-alerts/generator.py b/examples/model-alerts/generator.py
new file mode 100644
index 00000000..9aba677c
--- /dev/null
+++ b/examples/model-alerts/generator.py
@@ -0,0 +1,546 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Independent electron-count exposures and measured xPois candidates.
+
+No catalog, image, calibration file, checkpoint or network input is accepted.
+Truth is returned separately and used only after detection for association.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+import numpy as np
+from scipy import ndimage
+from scipy.special import erf
+
+
+@dataclass(frozen=True)
+class Design:
+ size: int = 128
+ stamp_size: int = 31
+ epochs: int = 3
+ stars: int = 22
+ changes: int = 5
+ detection_threshold: float = 4.0
+ association_radius: float = 3.0
+ read_noise: float = 4.0
+ calibration_size: int = 64
+ star_flux_min: float = 400.0
+ star_flux_max: float = 18000.0
+ hot_flux_min: float = 300.0
+ hot_flux_max: float = 1500.0
+ cosmic_flux_min: float = 200.0
+ cosmic_flux_max: float = 900.0
+ moving_flux: float = 1800.0
+ global_offset_sigma: float = 0.2
+ local_offset_sigma: float = 0.35
+ mismatch_factor: float = 1.25
+ column_flux_min: float = 100.0
+ column_flux_max: float = 400.0
+ column_half_length: int = 5
+ root_seed: int = 20261115
+
+ def __post_init__(self):
+ if self.calibration_size < 2:
+ raise ValueError(
+ "calibration frames must contain at least 2x2 pixels"
+ )
+ if self.stamp_size < 3 or self.stamp_size % 2 != 1:
+ raise ValueError("stamp size must be odd and at least 3")
+ if self.size <= self.stamp_size + 20:
+ raise ValueError(
+ "field must leave a complete stamp/kernel margin"
+ )
+ if min(self.epochs, self.stars, self.changes) < 1:
+ raise ValueError(
+ "positive epoch, star and change counts required"
+ )
+ if self.changes > self.stars:
+ raise ValueError("brightening positions require enough stars")
+ if (
+ min(
+ self.detection_threshold,
+ self.association_radius,
+ self.read_noise,
+ )
+ <= 0
+ ):
+ raise ValueError(
+ "positive threshold, radius and read noise required"
+ )
+
+
+DEFAULT_DESIGN = Design()
+
+
+def rng_for(design, field, stream):
+ """Disjoint deterministic streams; field identity owns every epoch."""
+ return np.random.default_rng(
+ np.random.SeedSequence([design.root_seed, int(field), int(stream)])
+ )
+
+
+def psf(shape, x, y, sigma):
+ """Pixel-integrated circular Gaussian with unit infinite-plane flux."""
+ if not np.isfinite([x, y, sigma]).all() or sigma <= 0:
+ raise ValueError("finite position and positive sigma required")
+ yy, xx = np.indices(shape)
+ scale = np.sqrt(2) * sigma
+ px = 0.5 * (erf((xx + 0.5 - x) / scale) - erf((xx - 0.5 - x) / scale))
+ py = 0.5 * (erf((yy + 0.5 - y) / scale) - erf((yy - 0.5 - y) / scale))
+ return px * py
+
+
+def expose(expected, rng, read_noise):
+ """Poisson electrons plus independent Gaussian readout noise, gain=1."""
+ if not np.isfinite(expected).all() or np.any(expected < 0):
+ raise ValueError("expected electrons must be finite and nonnegative")
+ return rng.poisson(expected).astype(float) + rng.normal(
+ 0, read_noise, expected.shape
+ )
+
+
+def calibration_sample(background, read_noise, rng, *, size=64):
+ """Estimate sky/read variance from independent calibration frames."""
+ shape = (size, size)
+ sky = expose(np.full(shape, background), rng, read_noise)
+ dark_a = rng.normal(0, read_noise, shape)
+ dark_b = rng.normal(0, read_noise, shape)
+ return dict(
+ background=float(sky.mean()),
+ read_variance=float(np.var(dark_a - dark_b, ddof=1) / 2),
+ pixels=size * size,
+ )
+
+
+def input_variances(exposure):
+ """Measured source intensity plus independently estimated calibration."""
+ values = []
+ for name, prefix in (("search", "search"), ("template", "reference")):
+ background = exposure[prefix + "_background"]
+ values.append(
+ np.maximum(exposure[name] - background, 0)
+ + background
+ + exposure[prefix + "_read_variance"]
+ )
+ return tuple(values)
+
+
+def field_exposures(field, *, design=DEFAULT_DESIGN, shifted=False):
+ """One shared template and independent search noise per epoch.
+
+ Teaching parameters: uniform backgrounds 60--140 electrons/pixel;
+ reference sigma 0.9--1.2 pixels; search sigma 1.2--1.9 pixels. Shift uses
+ background 180--300 and search sigma 1.9--2.6, plus unseen column bleed.
+ Expected change SNR is log-uniform 4--35 under a background-only Gaussian
+ matched-filter approximation. It is an injection parameter, never a
+ measured detection significance or a model feature.
+ """
+ rng = rng_for(design, field, 0)
+ shape = (design.size, design.size)
+ margin = design.stamp_size // 2 + 9
+ positions = rng.uniform(margin, design.size - margin, (design.stars, 2))
+ fluxes = np.exp(
+ rng.uniform(
+ np.log(design.star_flux_min),
+ np.log(design.star_flux_max),
+ design.stars,
+ )
+ )
+ reference_sigma = rng.uniform(0.9, 1.2)
+ background = rng.uniform(60, 140)
+ reference_calibration = calibration_sample(
+ background,
+ design.read_noise,
+ rng_for(design, field, 1000),
+ size=design.calibration_size,
+ )
+ reference_mean = np.full(shape, background)
+ for (x, y), flux in zip(positions, fluxes, strict=False):
+ reference_mean += flux * psf(shape, x, y, reference_sigma)
+ # Detector dark-current defect is present in both reference and search.
+ hot_x, hot_y = rng.integers(margin, design.size - margin, 2)
+ hot_flux = rng.uniform(design.hot_flux_min, design.hot_flux_max)
+ reference_mean[hot_y, hot_x] += hot_flux
+ template = expose(
+ reference_mean, rng_for(design, field, 1), design.read_noise
+ )
+ for epoch in range(design.epochs):
+ erng = rng_for(design, field, 10 + epoch)
+ sigma = erng.uniform(1.9, 2.6) if shifted else erng.uniform(1.2, 1.9)
+ sky = erng.uniform(180, 300) if shifted else erng.uniform(60, 140)
+ expected = np.full(shape, sky)
+ causes = []
+ # A single constant kernel can absorb a common translation, but not
+ # independent local astrometric errors for different static sources.
+ offsets = erng.normal(0, design.global_offset_sigma, 2) + erng.normal(
+ 0, design.local_offset_sigma, (design.stars, 2)
+ )
+ mismatch = np.arange(design.stars) % 7 == 0
+ host_pool = np.flatnonzero(~mismatch)
+ host_count = (design.changes + 1) // 2
+ if host_count > len(host_pool):
+ raise ValueError(
+ "not enough non-mismatch stars for brightening hosts"
+ )
+ hosts = erng.choice(host_pool, host_count, replace=False)
+ for index, ((x, y), flux) in enumerate(
+ zip(positions, fluxes, strict=False)
+ ):
+ dx, dy = offsets[index]
+ local_sigma = sigma * (
+ design.mismatch_factor if mismatch[index] else 1
+ )
+ expected += flux * psf(shape, x + dx, y + dy, local_sigma)
+ causes.append(
+ dict(
+ x=float(x + dx),
+ y=float(y + dy),
+ family="psf_mismatch"
+ if index % 7 == 0
+ else "misregistration",
+ positive=False,
+ challenge=False,
+ )
+ )
+ changes = erng.uniform(
+ margin, design.size - margin, (design.changes, 2)
+ )
+ for index, (x, y) in enumerate(changes):
+ snr = np.exp(erng.uniform(np.log(4), np.log(35)))
+ flux = snr * np.sqrt(
+ 4
+ * np.pi
+ * sigma**2
+ * (sky + background + 2 * design.read_noise**2)
+ )
+ # Half brighten an existing source; half appear on blank sky.
+ if index % 2 == 0:
+ host = hosts[index // 2]
+ x, y = positions[host] + offsets[host]
+ expected += flux * psf(shape, x, y, sigma)
+ causes.append(
+ dict(
+ x=float(x),
+ y=float(y),
+ family="brightening" if index % 2 == 0 else "appearance",
+ positive=True,
+ challenge=False,
+ expected_snr=float(snr),
+ injection_id=f"{field}:{epoch}:change-{index}",
+ )
+ )
+ # One untrailed mover is a held-out astrophysical challenge per epoch.
+ x, y = erng.uniform(margin, design.size - margin, 2)
+ expected += design.moving_flux * psf(shape, x, y, sigma)
+ causes.append(
+ dict(
+ x=float(x),
+ y=float(y),
+ family="moving_source",
+ positive=True,
+ challenge=True,
+ )
+ )
+ expected[hot_y, hot_x] += hot_flux
+ search = expose(expected, erng, design.read_noise)
+ causes.append(
+ dict(
+ x=float(hot_x),
+ y=float(hot_y),
+ family="bad_pixel",
+ positive=False,
+ challenge=False,
+ )
+ )
+ x, y = erng.integers(margin, design.size - margin, 2)
+ search[y, x : x + 3] += erng.uniform(
+ design.cosmic_flux_min, design.cosmic_flux_max, 3
+ )
+ causes.append(
+ dict(
+ x=float(x + 1),
+ y=float(y),
+ family="cosmic_ray",
+ positive=False,
+ challenge=False,
+ )
+ )
+ if shifted:
+ x, y = erng.integers(margin, design.size - margin, 2)
+ extent = design.column_half_length
+ search[y - extent : y + extent + 1, x] += erng.uniform(
+ design.column_flux_min, design.column_flux_max
+ )
+ causes.append(
+ dict(
+ x=float(x),
+ y=float(y),
+ family="column_bleed",
+ segment_half_length=extent,
+ positive=False,
+ challenge=False,
+ )
+ )
+ search_calibration = calibration_sample(
+ sky,
+ design.read_noise,
+ rng_for(design, field, 1100 + epoch),
+ size=design.calibration_size,
+ )
+ yield dict(
+ template=template.copy(),
+ search=search,
+ reference_background=reference_calibration["background"],
+ reference_read_variance=reference_calibration["read_variance"],
+ search_background=search_calibration["background"],
+ search_read_variance=search_calibration["read_variance"],
+ field=int(field),
+ epoch=epoch,
+ causes=causes,
+ )
+
+
+def detect(exposure, *, design=DEFAULT_DESIGN, backend="cpu"):
+ """Fit xPois without truth masks, then find positive filtered maxima.
+
+ Variances use measured images, assumed unit gain and independently
+ estimated read-noise/sky calibration. Kernel uncertainty and spatial
+ covariance are omitted;
+ resulting SNR is a diagnostic rather than a calibrated significance.
+ """
+ from cuphoton.xpois import (
+ GaussianBasisComponent,
+ solve_constant_kernel,
+ standardize_constant_kernel_residual,
+ )
+
+ search, template = exposure["search"], exposure["template"]
+ variance, ref_variance = input_variances(exposure)
+ fit = solve_constant_kernel(
+ template,
+ search,
+ [
+ GaussianBasisComponent(sigma=0.7, degree=2),
+ GaussianBasisComponent(sigma=1.5, degree=2),
+ GaussianBasisComponent(sigma=2.5, degree=0),
+ ],
+ variance=variance,
+ kernel_shape=(15, 15),
+ backend=backend,
+ )
+ noise = standardize_constant_kernel_residual(
+ fit.residual,
+ kernel=fit.kernel,
+ target_variance=variance,
+ reference_variance=ref_variance,
+ )
+ # Fixed detection filter, independent of the injection seeing truth.
+ axis = np.arange(-4, 5)
+ filt = np.exp(-(axis[:, None] ** 2 + axis[None, :] ** 2) / (2 * 1.4**2))
+ filt /= filt.sum()
+ filtered = ndimage.convolve(fit.residual, filt, mode="constant")
+ vfiltered = ndimage.convolve(
+ np.nan_to_num(noise.marginal_residual_variance),
+ filt**2,
+ mode="constant",
+ )
+ snr = filtered / np.sqrt(np.maximum(vfiltered, 1e-12))
+ valid = ndimage.binary_erosion(
+ noise.valid_mask, iterations=design.stamp_size // 2
+ )
+ half = design.stamp_size // 2
+ valid[:half] = valid[-half:] = False
+ valid[:, :half] = valid[:, -half:] = False
+ selected = (
+ valid
+ & (snr >= design.detection_threshold)
+ & (snr == ndimage.maximum_filter(snr, size=7))
+ )
+ locations = np.column_stack(np.where(selected))
+ stamps, metadata = [], []
+ for y, x in locations:
+ cut = np.s_[y - half : y + half + 1, x - half : x + half + 1]
+ stamps.append(
+ np.stack([search[cut], template[cut], fit.residual[cut]])
+ )
+ metadata.append(dict(x=int(x), y=int(y), snr=float(snr[y, x])))
+ return stamps, metadata, fit, noise
+
+
+def associate(candidate, causes, radius):
+ """Associate truth after detection and flag coverage limits."""
+
+ def distance(cause):
+ dx = candidate["x"] - cause["x"]
+ dy = max(
+ abs(candidate["y"] - cause["y"])
+ - cause.get("segment_half_length", 0),
+ 0,
+ )
+ return np.hypot(dx, dy)
+
+ near = [c for c in causes if distance(c) <= radius]
+ near.sort(key=distance)
+ real = [c for c in near if c["positive"]]
+ artifacts = [
+ c
+ for c in near
+ if c["family"] in ("cosmic_ray", "bad_pixel", "column_bleed")
+ ]
+ ambiguous = len(real) > 1 or bool(real and artifacts)
+ return dict(
+ associated_injections=[
+ c["injection_id"] for c in real if "injection_id" in c
+ ],
+ label=int(bool(real)),
+ family=real[0]["family"]
+ if real
+ else (near[0]["family"] if near else "noise"),
+ ambiguous=ambiguous,
+ overlapping_causes=len(near) > 1,
+ challenge=any(c["challenge"] for c in real),
+ )
+
+
+def write_dataset(
+ output, fields, *, design=DEFAULT_DESIGN, shifted=False, backend="cpu"
+):
+ """Create native xScan arrays; refuse occupied paths before any writes.
+
+ ``fields`` is an explicit mapping of field identifier to split name. The
+ caller owns the development/freeze/test schedule; this function never
+ chooses or extends seed blocks on the basis of classification outcomes.
+ """
+ from cuphoton.xscan.dataset import validate_dataset_dir
+ from cuphoton.xscan.training import require_training_label_provenance
+
+ output = Path(output).resolve()
+ if output.exists() and (not output.is_dir() or any(output.iterdir())):
+ raise ValueError("dataset output must be absent or empty")
+ if not fields or not set(fields.values()) <= {"train", "val", "test"}:
+ raise ValueError("explicit field-to-split mapping required")
+ if design.stamp_size % 2 != 1 or design.stamp_size < 3:
+ raise ValueError("stamp size must be odd and at least 3")
+ if any(type(field) is not int or field < 0 for field in fields):
+ raise ValueError("field identifiers must be nonnegative integers")
+ images, rows, recovery, variances = [], [], [], []
+ for field, split in fields.items():
+ for exposure in field_exposures(
+ field, design=design, shifted=shifted
+ ):
+ stamps, candidates, fit, noise = detect(
+ exposure, design=design, backend=backend
+ )
+ positives = [
+ c
+ for c in exposure["causes"]
+ if c["positive"] and not c["challenge"]
+ ]
+ recovered = sum(
+ any(
+ np.hypot(c["x"] - t["x"], c["y"] - t["y"])
+ <= design.association_radius
+ for c in candidates
+ )
+ for t in positives
+ )
+ recovery.append(
+ dict(
+ field=field,
+ epoch=exposure["epoch"],
+ injected=len(positives),
+ injection_families={
+ c["injection_id"]: c["family"] for c in positives
+ },
+ calibration=dict(
+ reference_background=exposure["reference_background"],
+ search_background=exposure["search_background"],
+ reference_read_variance=exposure[
+ "reference_read_variance"
+ ],
+ search_read_variance=exposure["search_read_variance"],
+ pixels=design.calibration_size**2,
+ ),
+ detected=recovered,
+ )
+ )
+ for stamp, candidate in zip(stamps, candidates, strict=False):
+ truth = associate(
+ candidate, exposure["causes"], design.association_radius
+ )
+ index = len(rows)
+ rows.append(
+ dict(
+ candidate_id=index,
+ band="modeled_broadband",
+ ccd_id=f"detector-{field}",
+ exposure_id=f"field-{field}-epoch-{exposure['epoch']}",
+ sample_index=index,
+ field=field,
+ epoch=exposure["epoch"],
+ split=split,
+ split_group=f"field-{field}",
+ label_source="analytic_injection",
+ target_label_available=True,
+ **candidate,
+ **truth,
+ )
+ )
+ images.append(stamp)
+ half = design.stamp_size // 2
+ x, y = candidate["x"], candidate["y"]
+ variances.append(
+ noise.marginal_residual_variance[
+ y - half : y + half + 1, x - half : x + half + 1
+ ]
+ )
+ if not images:
+ raise ValueError("design produced no detected candidates")
+ images = np.asarray(images, dtype=np.float32)
+ digests = [hashlib.sha256(s.tobytes()).hexdigest() for s in images]
+ if len(set(digests)) != len(digests):
+ raise ValueError("duplicate candidate content detected")
+ output.mkdir(parents=True, exist_ok=True)
+ for index, name in enumerate(("search", "template", "difference")):
+ np.save(output / f"{name}.npy", images[:, index], allow_pickle=False)
+ np.save(output / "variance.npy", np.asarray(variances, dtype=np.float64))
+ np.save(
+ output / "labels.npy",
+ np.array([r["label"] for r in rows], dtype=np.int64),
+ )
+ np.save(
+ output / "split.npy",
+ np.array(
+ [("train", "val", "test").index(r["split"]) for r in rows],
+ dtype=np.int64,
+ ),
+ )
+ (output / "metadata.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in rows)
+ )
+ (output / "generation.json").write_text(
+ json.dumps(
+ dict(
+ design=asdict(design),
+ fields=fields,
+ shifted=shifted,
+ backend=backend,
+ recovery=recovery,
+ candidates=len(rows),
+ unique_stamp_hashes=len(
+ {hashlib.sha256(s.tobytes()).hexdigest() for s in images}
+ ),
+ ),
+ indent=2,
+ )
+ + "\n"
+ )
+ validate_dataset_dir(output, dataset_kind="prepared")
+ require_training_label_provenance(output)
+ return rows
diff --git a/examples/model-alerts/model-alerts.ipynb b/examples/model-alerts/model-alerts.ipynb
new file mode 100644
index 00000000..f6ad4315
--- /dev/null
+++ b/examples/model-alerts/model-alerts.ipynb
@@ -0,0 +1,927 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "b401a46e",
+ "metadata": {},
+ "source": [
+ "---\n",
+ "title: Model Alerts\n",
+ "subtitle: Which changes deserve attention?\n",
+ "execute:\n",
+ " warning: true\n",
+ "---\n",
+ "\n",
+ "\n",
+ "\n",
+ "A new image contains a bright spot. Is it a changing astronomical source,\n",
+ "a detector defect, or the imperfect cancellation of a steady star? Model\n",
+ "Alerts follows that decision from electrons in two exposures to the list a\n",
+ "person reviews. We generate independent fields, detect positive changes\n",
+ "with **xPois**, and train **xScan** from random initialization. **xFit** supplies\n",
+ "an additional measured-feature investigation.\n",
+ "\n",
+ "[View the source notebook in NVIDIA/cuPhoton](https://github.com/NVIDIA/cuPhoton/blob/main/examples/model-alerts/model-alerts.ipynb).\n",
+ "\n",
+ "Every image, source, defect and noise realization is generated here from\n",
+ "declared analytic models. No observed survey images or pretrained models\n",
+ "enter the experiment. The results describe this modeled population. They do\n",
+ "not establish survey performance or calibrated probabilities for the sky.\n",
+ "\n",
+ "Read the expanded **Science Explanations** for assumptions, units and\n",
+ "limitations. Follow a terminology link for a definition; keyboard focus and\n",
+ "pointer hover also reveal it in the HTML. All plot controls explore saved\n",
+ "arrays and scores in your browser. They do not train a new classifier."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e15677e3",
+ "metadata": {},
+ "source": [
+ "## The decision comes before the classifier\n",
+ "\n",
+ "We call a detected PSF-like appearance or brightening **positive**. Detector\n",
+ "artifacts, noise peaks and residual light from static sources are **negative**.\n",
+ "The first task is to define those words, before showing a model any labels.\n",
+ "\n",
+ "An untrailed moving source is a genuine astrophysical challenge. We report\n",
+ "it separately, rather than teaching the model that all motion is bogus.\n",
+ "Overlapping changes and artifacts receive ambiguity flags. The complete\n",
+ "candidate inventory retains them, while primary binary metrics state their\n",
+ "coverage after excluding ambiguous associations and these challenges.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "A label answers a scientific question about a detected candidate, not about\n",
+ "an arbitrary image location. The distribution is conditional on passing the\n",
+ "positive-detection rule. Pair classification therefore operates on\n",
+ "subtraction-selected candidates even though its input omits the difference\n",
+ "channel. Fading sources require a separate negative-polarity detection and\n",
+ "labeling extension; positive-only recovery does not measure that task.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "338d6b07",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import json\n",
+ "import os\n",
+ "from pathlib import Path\n",
+ "\n",
+ "import numpy as np\n",
+ "from bokeh.io import output_notebook, show\n",
+ "from bokeh.resources import INLINE\n",
+ "\n",
+ "from generator import Design, psf\n",
+ "from reference import execute_reference\n",
+ "from views import candidate_explorer, performance_diagnostics, training_diagnostics\n",
+ "from reporting import table, model_rows\n",
+ "\n",
+ "output_notebook(resources=INLINE)\n",
+ "root = Path(os.environ.get(\"CUPHOTON_MODEL_ALERTS_OUTPUT\", \"results\"))\n",
+ "profile = os.environ.get(\"CUPHOTON_MODEL_ALERTS_PROFILE\", \"quick-smoke\")\n",
+ "print(\"Profile:\", profile)\n",
+ "print(\"Quick smoke exercises APIs; scientific performance requires full reference.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bf5e5184",
+ "metadata": {},
+ "source": [
+ "## Count electrons, then add independent noise\n",
+ "\n",
+ "Our detector has unit gain: image values are electrons per pixel. A star\n",
+ "has total expected flux $F$ electrons and a pixel-integrated circular Gaussian\n",
+ "PSF. Reference seeing has $\\sigma=0.9$–$1.2$ pixels; ordinary search seeing\n",
+ "has $\\sigma=1.2$–$1.9$ pixels. Background is uniform within an exposure and\n",
+ "varies from 60 to 140 electrons per pixel. Read noise is 4 electrons RMS.\n",
+ "These are teaching choices, not instrument measurements.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "For pixel $p$, the expectation is\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$\\mu_p=b+\\sum_j F_j\\int_{p}P(x-x_j,y-y_j;\\sigma_j)\\,dx\\,dy.$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "The PSF has units pixel$^{-2}$ and integrates to one on the infinite plane.\n",
+ "The generator integrates it over each pixel using error functions rather\n",
+ "than sampling only the center. Exposures obey\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$I_p\\sim\\operatorname{Poisson}(\\mu_p)+\\mathcal N(0,\\sigma_r^2),\\qquad\n",
+ "\\operatorname{Var}(I_p)=\\mu_p+\\sigma_r^2.$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "The Poisson term is electron counting noise. Read noise is independent of it\n",
+ "and of every other exposure. Finite image boundaries can truncate flux;\n",
+ "injected source positions leave a declared margin. There is no saturation,\n",
+ "clipping or masked-pixel generator signature. Registered detector-plane\n",
+ "coordinates suffice here; no observed WCS or calibration product is used.\n",
+ "\n",
+ " \n",
+ "The declared artifact mixture has one persistent hot pixel (300–1500 expected electrons), one three-pixel cosmic ray (200–900 electrons per pixel), and one untrailed moving-source challenge (1800 expected electrons) per search epoch. Static stars span 400–18000 expected electrons. Registration combines a common 0.2-pixel Gaussian offset with independent 0.35-pixel offsets per star. Every seventh star has a 1.25× search PSF width. Brightening hosts are randomly chosen each epoch from the other stars, using the same PSF as their hosts. Shift-only column bleed spans 11 pixels with 100–400 added electrons per pixel; association uses distance to the full segment. These are teaching parameters, recorded in the design receipt.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02726f57",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "design = Design()\n",
+ "y, x = np.indices((71, 71))\n",
+ "unit_psf = psf((71, 71), 34.2, 35.3, 1.5)\n",
+ "print(\"Integrated finite-field PSF flux:\", unit_psf.sum())\n",
+ "print(\"Pixel centroid:\", (unit_psf*x).sum(), (unit_psf*y).sum())\n",
+ "print(\"Declared generator:\", design)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "343747ce",
+ "metadata": {},
+ "source": [
+ "## Independent fields own their repeated observations\n",
+ "\n",
+ "The reference design starts with 300 fields and three search epochs per\n",
+ "field: 180 training fields, 60 validation fields and 60 reserved test fields.\n",
+ "Each field owns its stars, template noise, persistent defects and repeated\n",
+ "epochs. The specified shift adds 60 independent fields. A development pilot\n",
+ "runs before any reserved test field is generated.\n",
+ "\n",
+ "The seed audit reads every generated development field, including fields with no detections, plus the CPU/CUDA agreement field. Reserved blocks and generator stream roles must be disjoint.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Repeated exposures of one field share sky structure and a noisy template.\n",
+ "Splitting their candidates independently would let the learner recognize the\n",
+ "same structure on both sides of an evaluation boundary. We split fields\n",
+ "first, then generate their observations from separate deterministic streams.\n",
+ "Native provenance guards reject field identities crossing splits. We also\n",
+ "audit seeds and duplicate candidate bytes. Different row IDs do not establish\n",
+ "independence. If the primary validation field-bootstrap interval has\n",
+ "half-width above five percentage points, we double the development design\n",
+ "and reassess before freezing final counts and seed blocks.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7d4858d0",
+ "metadata": {},
+ "source": [
+ "## Create the residual that selects candidates\n",
+ "\n",
+ "xPois fits a Gaussian-polynomial convolution kernel and differential\n",
+ "background from measured images. It does not use injected source locations\n",
+ "as fitting masks. The residual is search minus the matched reference.\n",
+ "We filter it with a fixed Gaussian of $\\sigma=1.4$ pixels, then select local\n",
+ "positive maxima above the development detection threshold. Cutouts are\n",
+ "centered on those measured maxima.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "The image model and residual convention are\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$S\\simeq R\\otimes K+B,\\qquad D=S-(R\\otimes K+B).$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Assumed unit gain, independent read-noise/background calibration and measured source\n",
+ "intensity estimate input variances. For a fixed fitted kernel,\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$V_D\\simeq V_S+V_R\\otimes K^2,\\quad\n",
+ "q=\\frac{D\\otimes H}{\\sqrt{V_D\\otimes H^2}}.$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "$q$ is a marginal detection diagnostic. It omits spatial covariance and\n",
+ "kernel-fit uncertainty; it is not a calibrated Gaussian significance. The\n",
+ "threshold is four in this diagnostic and the association radius is three\n",
+ "pixels, both declared development choices and frozen before final testing.\n",
+ "A complete stamp/kernel footprint is required. The same selection rule is\n",
+ "used for pair, triplet, baseline and fusion experiments.\n",
+ "\n",
+ "Background is estimated from an independent 64Ă—64 blank-sky calibration\n",
+ "frame matched to each exposure. Read variance is estimated as half the sample\n",
+ "variance of the difference of two independent zero-light readout frames.\n",
+ "These calibration noises use streams separate from search/template noise;\n",
+ "4096 calibration pixels support each estimate. Finite calibration-estimate\n",
+ "uncertainty is omitted from the marginal variance diagnostic. Exact generator\n",
+ "background/read-noise values do not enter the measurement pipeline.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "77b119c4",
+ "metadata": {},
+ "source": [
+ "## Truth labels detections; it never selects the easy ones\n",
+ "\n",
+ "After detection, we associate peaks to injected changes within the frozen\n",
+ "radius. Unmatched detections remain negative examples. Bad pixels, cosmic\n",
+ "rays, registration errors and local PSF mismatch all contribute candidates.\n",
+ "A real change overlapping a detector artifact, or several associated real\n",
+ "changes, is flagged as ambiguous.\n",
+ "\n",
+ "Negative-family attribution uses the nearest associated cause; a genuine change takes label priority. Overlapping causes are counted separately from ambiguous associations. A brightening of a static star can overlap that star without making its binary label ambiguous. The persistent hot pixel contributes Poisson electrons in both the shared reference and every search exposure.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Position association is an evaluation convention with finite coverage. A\n",
+ "chance noise maximum near an injection can be associated; a badly displaced\n",
+ "real peak can fall outside the radius. Report injection recovery, ambiguous\n",
+ "counts and excluded challenge counts alongside classification metrics. A\n",
+ "positive detected near a static star may include overlapping static residual\n",
+ "light; the overlap flag makes that limitation visible. Do not use truth\n",
+ "coordinates, family, filenames, field identity or split assignment as model\n",
+ "features.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b5b849b9",
+ "metadata": {},
+ "source": [
+ "## Build a native xScan dataset\n",
+ "\n",
+ "The canonical arrays are search, `template`, `difference`, binary labels\n",
+ "and split assignments, with candidate metadata stored separately. Raw\n",
+ "cutouts remain in electron units for inspection. Model inputs use shared\n",
+ "search/reference centering and scaling fitted only on eligible training\n",
+ "rows; difference scaling is also fitted only on training rows.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "A normalization estimate is itself a learned quantity. Fitting it on all\n",
+ "rows would expose held-out data even without their labels. The fitted\n",
+ "training constants are saved and applied unchanged to validation, test and\n",
+ "shifted fields. Pair and triplet use the same detection pool and compatible\n",
+ "channel scales. xFit receives corresponding variance divided by the square\n",
+ "of the difference-channel scale. Exact-stamp hashes are calculated in the\n",
+ "units actually passed to the native fusion workflow.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5c4bbf90",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The reference helper executes generation, native guards, pilot training,\n",
+ "# required controls, precision assessment, freeze and final inference.\n",
+ "# The executing kernel holds the shared GPU lease for this entire call.\n",
+ "reference = execute_reference(root, profile=profile,\n",
+ " device=os.environ.get(\"CUPHOTON_MODEL_ALERTS_DEVICE\", \"cuda:0\"))\n",
+ "table(reference[\"design\"], caption=\"Executed design\")\n",
+ "table([dict(population=k, **v) for k,v in reference[\"coverage\"].items()],\n",
+ " caption=\"Detection and metric coverage\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9eb30934",
+ "metadata": {},
+ "source": [
+ "## Learn from images, from random weights\n",
+ "\n",
+ "We compare native xScan pair and triplet transformer models initialized\n",
+ "from scratch. The pair sees search and reference; the triplet also sees the\n",
+ "xPois residual. Three initialization seeds are shown separately for each\n",
+ "principal model. The same fields, candidates and optimization settings\n",
+ "support a paired comparison.\n",
+ "\n",
+ "All reference models and the permutation control complete the declared 12-epoch budget. We retain the checkpoint with lowest validation loss. The same validation fields select checkpoints and operating thresholds; their intervals are conditional development diagnostics and the precision expansion rule is a heuristic. Independent reserved fields supply the final endpoints.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Training minimizes binary cross entropy from logits $z_i$,\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$\\mathcal L=-\\frac1n\\sum_i[y_i\\log\\sigma(z_i)+(1-y_i)\\log(1-\\sigma(z_i))].$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "The sigmoid converts a logit to a score between zero and one. It does not\n",
+ "make that score a calibrated astrophysical probability. Checkpoints are\n",
+ "selected using validation loss. Our teaching configuration uses three\n",
+ "encoder stages with embedding dimensions 16, 32 and 64, one block per stage,\n",
+ "12 epochs and Adam-style native optimizer settings. These are example\n",
+ "choices, not a reproduction of a paper's published accuracy. Quick smoke\n",
+ "uses one epoch and fewer fields and cannot substantiate performance.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "90fd9098",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table([dict(model=f\"{r['mode']}-{r['seed']}\", threshold=r[\"threshold\"],\n",
+ " validation_auc=r[\"validation_roc_auc\"], **r[\"validation\"])\n",
+ " for r in reference[\"training\"]], caption=\"Validation operating points\")\n",
+ "show(training_diagnostics(root, reference))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0292f471",
+ "metadata": {},
+ "source": [
+ "## Can a simple measured statistic explain the result?\n",
+ "\n",
+ "SNR-only logistic regression is the first baseline. A second baseline adds\n",
+ "positive core concentration, negative residual fraction, peak-to-RMS ratio\n",
+ "and search/reference correlation. Nonfinite and repeated-extreme fractions are measured from raw pixels. Feature standardization and logistic coefficients\n",
+ "are fitted on training rows only.\n",
+ "\n",
+ "We measure nonfinite-pixel fraction and repeated-extreme fraction in every raw triplet and report their training ranges separately by class. Repeated extrema can expose clipping or quantization but do not establish its cause. No missing or clipped pixels are intentionally inserted.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "A high score can reflect a trivial generator cue rather than source\n",
+ "morphology. The baselines expose how much signal comes from brightness or\n",
+ "simple shape statistics. No family or injection-SNR column enters them.\n",
+ "The expected injection SNR is log-uniform from four to 35 under an\n",
+ "approximate background-dominated Gaussian calculation; measured detection\n",
+ "SNR differs because of source noise, fitted subtraction and selection.\n",
+ "Performance is conditional on this brightness and artifact mixture.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7f6d5462",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table([dict(model=k, threshold=v[\"threshold\"], **v[\"validation\"])\n",
+ " for k,v in reference[\"baselines\"].items()], caption=\"Measured baselines\")\n",
+ "table(reference[\"baselines\"][\"measured_statistics\"][\"training_signature_audit\"],\n",
+ " caption=\"Measured training signature audit by binary class\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1a0ab425",
+ "metadata": {},
+ "source": [
+ "## Ask the experiment to fail in a known way\n",
+ "\n",
+ "We permute training labels once with a recorded seed and leave validation\n",
+ "truth untouched. We also deliberately split candidates from one field\n",
+ "across training and validation: the native provenance guard must reject that\n",
+ "dataset before training. A development-only learning curve uses nested field\n",
+ "subsets with their own training-only normalization.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "The permutation control tests whether genuine validation structure can be\n",
+ "predicted from randomly assigned training targets. Unexpected predictability\n",
+ "requires diagnosis, not a search for a more reassuring permutation seed.\n",
+ "The invalid split is a guard demonstration and is never used to claim a\n",
+ "performance gain. Learning curves help identify data limitation but cannot\n",
+ "choose settings after final-test unblinding. Three initialization seeds show\n",
+ "optimization sensitivity; they are not three independent sky surveys.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4518eeab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table(dict(invalid_split_rejected=reference[\"controls\"][\"invalid_split\"][\"rejected\"],\n",
+ " permutation_validation_auc=reference[\"controls\"][\"permutation\"][\"validation_roc_auc\"],\n",
+ " permutation_review_required=reference[\"controls\"][\"permutation_review_required\"],\n",
+ " registration_diagnostic=reference[\"controls\"][\"registration_diagnostic\"],\n",
+ " cpu_cuda_inference=reference[\"controls\"].get(\"cpu_cuda_inference\", {}),\n",
+ " cpu_cuda_agreement=reference[\"controls\"][\"cpu_cuda_agreement\"]),\n",
+ " caption=\"Scientific controls\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2040a072",
+ "metadata": {},
+ "source": [
+ "## Does a measured xFit description add information?\n",
+ "\n",
+ "The fusion investigation exports exact candidate difference stamps and\n",
+ "row-aligned variance, fits the Gaussian difference model with xFit, then\n",
+ "builds the native portable feature bundle. Candidate IDs and stamp hashes\n",
+ "must correspond exactly. Native gates retain failed or invalid fits.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "An ideal single positive source has no negative lobe, while registration\n",
+ "residuals can resemble positive/negative dipoles. Fitted shape, separation,\n",
+ "fit quality and uncertainty can therefore add measured evidence. The\n",
+ "Gaussian fit is an approximation to pixel-integrated sources and imperfect\n",
+ "subtraction. A failed fit is an outcome with coverage implications, not a\n",
+ "reason to discard a difficult candidate. Report `fit_present`, `fit_valid`\n",
+ "and uncertainty coverage, along with fused classification metrics. Weights\n",
+ "are marginal-variance approximations and omit residual covariance.\n",
+ "\n",
+ " \n",
+ "Native xFit feature transforms use fixed documented constants and row-local measurements, not a population-fitted scaler. The coverage receipt records those constants. Only image normalization uses training-estimated quantities.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5cd4f84c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table(reference[\"fusion\"][\"development_coverage\"], caption=\"xFit coverage\")\n",
+ "table(reference[\"fusion\"][\"validation\"][\"validation\"], caption=\"Fusion validation\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4eec4819",
+ "metadata": {},
+ "source": [
+ "## Choose a validation operating point with tied scores\n",
+ "\n",
+ "For every model and seed, we choose the highest threshold whose inclusive\n",
+ "score >= threshold rule retains at least 95% of validation positives.\n",
+ "Checkpoints, normalization and thresholds are frozen before generating or\n",
+ "inferring the final reserved test set. The primary endpoint is test\n",
+ "false-positive rate at that frozen threshold.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Sort the $n_+$ positive validation scores in descending order. With\n",
+ "$k=\\lceil0.95n_+\\rceil$, choose $t=s_{(k)}$. Including every tie at $t$ may\n",
+ "retain more than 95%; raising it would lose required positive examples.\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$\\operatorname{TPR}(t)=\\frac{TP}{TP+FN},\\qquad\n",
+ "\\operatorname{FPR}(t)=\\frac{FP}{FP+TN}.$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "The native training workflow's accuracy-selected threshold is not this\n",
+ "operating point. Test recall is an achieved result, not automatically 95%.\n",
+ "Invalidating bugs require a new freeze and fresh reserved seeds; earlier\n",
+ "attempts remain recorded. Exploratory browser sliders do not alter the\n",
+ "frozen endpoint reported in the scientific tables.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "25ae1d84",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table({k:reference[\"freeze\"][k] for k in\n",
+ " (\"train_fields\",\"validation_fields\",\"test_fields\",\"shifted_fields\",\n",
+ " \"test_seed_block\",\"shift_seed_block\")}, caption=\"Frozen field counts and seed blocks\")\n",
+ "table(model_rows(reference[\"endpoints\"]), caption=\"Frozen-threshold test endpoints\")\n",
+ "table([dict(population=population, model=model, injection_family=family, **counts)\n",
+ " for population, outcomes in ((\"test\", reference[\"endpoints\"]), (\"shift\", reference[\"shift\"]))\n",
+ " for model, metrics in outcomes.items()\n",
+ " for family, counts in metrics[\"injections\"][\"by_family\"].items()],\n",
+ " caption=\"Injection-family recovery and retention\", collapsed=True)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a8e4b24f",
+ "metadata": {},
+ "source": [
+ "## Separate detection recovery from classification retention\n",
+ "\n",
+ "Some injected changes never pass detection. A classifier cannot recover\n",
+ "those missing candidates. We report detected associated injections divided\n",
+ "by all injected ordinary changes, and final retained associated injections\n",
+ "divided by that same injection denominator. Candidate-level recall has a\n",
+ "different denominator: eligible detected positives.\n",
+ "\n",
+ "The receipt decomposes detected injections into those with an eligible binary candidate and those detected only in excluded ambiguous/challenge associations. Classifier-rejected eligible injections are a separate loss. Each injection is counted once even if it matches several detections. Thus detection-to-retention loss is not attributed entirely to the classifier.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "An injection can create a complicated residual and a detection can overlap\n",
+ "multiple causes. Injection recovery is counted by injection identity;\n",
+ "classification confusion counts are counted by candidate identity. Neither\n",
+ "quantity may silently replace the other. The detection threshold and\n",
+ "association convention are part of the complete experiment, not fixed facts\n",
+ "of the sky. Coverage tables identify movers, ambiguous associations and\n",
+ "binary-eligible candidates for every split.\n",
+ "\n",
+ " \n",
+ "Association recovery does not prove that the change caused detection: a brightening can share a host registration residual. We report injected, detected and retained counts separately for appearances and brightenings. Appearances avoid this host-source residual confounding.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "baac929e",
+ "metadata": {},
+ "source": [
+ "## Read curves and reliability at the modeled prevalence\n",
+ "\n",
+ "ROC tracks achieved recall against false-positive rate as score thresholds\n",
+ "change. Precision–recall makes the class mixture more visible. Reliability\n",
+ "compares score averages with observed positive fractions in occupied bins;\n",
+ "Brier score averages squared score errors. We show each initialization seed\n",
+ "rather than hiding outcomes behind a mean.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Brier score is\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$\\mathrm{BS}=\\frac1N\\sum_i(s_i-y_i)^2.$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Empty reliability bins contain no evidence and remain empty. A small Brier\n",
+ "score or a visually good reliability curve on this generator does not\n",
+ "calibrate scores for an observed survey. ROC/PR curves and reliability use\n",
+ "all eligible evaluation candidates, not the curated image gallery. The\n",
+ "reported operating point remains the validation-frozen threshold.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a02ceee6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "evaluation = np.load(root / \"evaluation.npz\", allow_pickle=False)\n",
+ "predictions = {name: evaluation[key] for name, key in reference[\"score_arrays\"].items()}\n",
+ "show(performance_diagnostics(evaluation[\"labels\"], predictions))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "332536e2",
+ "metadata": {},
+ "source": [
+ "## Uncertainty follows independent fields\n",
+ "\n",
+ "We bootstrap fields, carrying every candidate and epoch in each sampled\n",
+ "field together. Pair and triplet differences use the same resampled fields.\n",
+ "The three initialization outcomes remain separate. Intervals are conditional\n",
+ "on trained checkpoints, frozen thresholds and the specified modeled mixture.\n",
+ "\n",
+ "Both reserved test and specified-shift populations receive field-bootstrap recall/FPR intervals. The validation sample is reused for checkpoint and threshold selection; its interval does not account for that selection.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Independent-candidate bootstrap would overstate the information in repeated\n",
+ "observations sharing a template. Field bootstrap samples field identities\n",
+ "with replacement; repeated draws multiply all their candidates. Replicates\n",
+ "without negatives have undefined FPR and are explicitly counted. The\n",
+ "validation half-width rule is a development precision check, not a\n",
+ "post-selection confidence guarantee. These percentile intervals do not\n",
+ "include all training-population, generator-model or threshold-selection\n",
+ "uncertainty.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "156fbea3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table([dict(population=population, model=model,\n",
+ " recall_interval=metrics[\"recall\"][\"interval\"],\n",
+ " fpr_interval=metrics[\"fpr\"][\"interval\"])\n",
+ " for population in (\"test\", \"shift\")\n",
+ " for model, metrics in reference[\"uncertainty\"][population+\"_model_field_intervals\"].items()],\n",
+ " caption=\"Field-bootstrap recall and FPR intervals\")\n",
+ "table([dict(seed=seed, comparison=\"pair minus triplet test FPR\",\n",
+ " pair_interval=reference[\"uncertainty\"][str(seed)][\"a\"],\n",
+ " triplet_interval=reference[\"uncertainty\"][str(seed)][\"b\"],\n",
+ " difference_interval=reference[\"uncertainty\"][str(seed)][\"a_minus_b\"])\n",
+ " for seed in (11, 23, 47) if str(seed) in reference[\"uncertainty\"]],\n",
+ " caption=\"Paired comparisons on the same resampled fields\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "98b75920",
+ "metadata": {},
+ "source": [
+ "## Explore the complete queue, inspect a bounded gallery\n",
+ "\n",
+ "Search, reference and signed difference panels share pan, zoom and tapped\n",
+ "pixel selection. Exact raw pixel readouts remain in electron units even when\n",
+ "display units change. Navigate candidates or filter by family and confusion\n",
+ "outcome. The gallery contains at most 96 candidates selected by a recorded\n",
+ "round-robin confusion/SNR/family rule. Empty categories remain empty.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "A human chooses a review budget as well as a desired completeness. At\n",
+ "assumed positive prevalence $\\pi$, holding class-conditional score mixtures\n",
+ "fixed,\n",
+ "\n",
+ "\n",
+ "\n",
+ "$$\\mathrm{purity}=\\frac{\\pi\\,TPR}{\\pi\\,TPR+(1-\\pi)\\,FPR},\\qquad\n",
+ "N_{\\mathrm{queue}}=N[\\pi\\,TPR+(1-\\pi)\\,FPR].$$\n",
+ "\n",
+ "
\n",
+ "\n",
+ "The browser recomputes these quantities from complete saved evaluation\n",
+ "scores. It does not retrain or rerun inference. A prevalence change is not\n",
+ "an arbitrary distribution shift: the projection assumes the positive and\n",
+ "negative subpopulations keep their score distributions. Gallery counts are\n",
+ "never substituted for population counts. Display transforms and page themes\n",
+ "must leave raw data and equivalent scientific color scales unchanged.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "408d8a43",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gallery = np.load(root / \"gallery.npz\", allow_pickle=False)\n",
+ "rows = json.loads((root / \"gallery-rows.json\").read_text())\n",
+ "# The reference stores explicit original gallery indices and the full score pool.\n",
+ "explorer, gallery_rule = candidate_explorer(\n",
+ " gallery[\"images\"], rows, gallery[\"labels\"], gallery[\"scores\"],\n",
+ " reference[\"inspector_threshold\"], preselected=True,\n",
+ " evaluation=dict(labels=evaluation[\"labels\"],\n",
+ " scores=evaluation[reference[\"inspector_score_key\"]],\n",
+ " snr=evaluation[\"snr\"],\n",
+ " candidate_id=evaluation[\"candidate_id\"],\n",
+ " family=evaluation[\"family\"]))\n",
+ "show(explorer)\n",
+ "print(json.dumps(reference[\"gallery_rule\"], indent=2))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c9f19797",
+ "metadata": {},
+ "source": [
+ "## Look at high-confidence mistakes\n",
+ "\n",
+ "A confident false positive can reveal a subtraction residual that resembles\n",
+ "a PSF. A confident false negative can reveal source crowding, displacement,\n",
+ "low contrast or a shape outside the training mixture. Inspect errors by\n",
+ "measured SNR and artifact family without discarding inconvenient categories.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "Confidence is not correctness. A score near zero or one may reflect an\n",
+ "unfamiliar example being pushed far from the decision boundary. The\n",
+ "stratified gallery intentionally overrepresents useful error types; it is\n",
+ "not a random sample and must not be used to estimate error rates. Empty\n",
+ "error categories are reported honestly instead of filled with another\n",
+ "model's mistakes. Family/SNR tables use the entire eligible evaluation set.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "296cfc40",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table([dict(population=population, model=model, family=family, **metrics)\n",
+ " for population, outcomes in reference[\"breakdowns\"].items()\n",
+ " for model, groups in outcomes.items()\n",
+ " for family, metrics in groups[\"family\"].items()],\n",
+ " caption=\"Complete-evaluation family breakdowns\", collapsed=True)\n",
+ "table([dict(population=population, model=model, measured_snr_bin=snr, **metrics)\n",
+ " for population, outcomes in reference[\"breakdowns\"].items()\n",
+ " for model, groups in outcomes.items()\n",
+ " for snr, metrics in groups[\"snr\"].items()],\n",
+ " caption=\"Complete-evaluation SNR breakdowns\", collapsed=True)\n",
+ "table([dict(population=population, model=model, overlapping_causes=overlap, **metrics)\n",
+ " for population, outcomes in reference[\"breakdowns\"].items()\n",
+ " for model, groups in outcomes.items()\n",
+ " for overlap, metrics in groups[\"overlap\"].items()],\n",
+ " caption=\"Coverage of overlapping causes\", collapsed=True)\n",
+ "table([dict(population=population, model=model, **r)\n",
+ " for population, outcomes in reference[\"high_confidence_mistakes\"].items()\n",
+ " for model, mistakes in outcomes.items() for r in mistakes],\n",
+ " caption=\"High-confidence mistakes\", collapsed=True)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1ab1c1f5",
+ "metadata": {},
+ "source": [
+ "## Try one specified distribution shift\n",
+ "\n",
+ "The shift test changes search seeing to $\\sigma=1.9$–$2.6$ pixels and\n",
+ "background to 180–300 electrons per pixel, and adds an unseen column-bleed\n",
+ "artifact. The reference exposure model and classifier thresholds stay fixed.\n",
+ "We report achieved recall, FPR and coverage on these additional fields.\n",
+ "\n",
+ "This joint shift changes seeing, background and artifact mixture together. It also increases the mismatch with the fixed 1.4-pixel detection filter, so it cannot identify separate causal contributions or establish general robustness.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "This shift combines broader sources, larger background fluctuations and a\n",
+ "new negative family. It can reveal one kind of fragility, but cannot support\n",
+ "a claim of general robustness. Changing prevalence alone would not create\n",
+ "these image differences. No shifted labels or images choose normalization,\n",
+ "architecture, checkpoints, association radius or thresholds. Movers remain\n",
+ "a separately identified astrophysical challenge rather than becoming bogus\n",
+ "training labels.\n",
+ "\n",
+ " \n",
+ "An untrailed mover is image-identical to an appearance in one search/reference pair. Its retention is an ontology consistency check; these images do not measure motion. Separate shift-family and overlap tables expose the unseen artifact and association coverage.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5ad286d3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table(model_rows(reference[\"shift\"]), caption=\"Specified shift endpoints\")\n",
+ "table([dict(population=population, model=model, **counts)\n",
+ " for population, results in reference[\"moving_source_challenge\"].items()\n",
+ " for model, counts in results.items()], caption=\"Moving-source challenge retention\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9aea0675",
+ "metadata": {},
+ "source": [
+ "## Reproduce the result, then decide what it supports\n",
+ "\n",
+ "Reference receipts record generator settings, seed blocks, source and\n",
+ "artifact hashes, dependencies, hardware descriptions and frozen operating\n",
+ "points. Check them before comparing exports. Training data, weights and\n",
+ "executed notebooks remain outside Git and outside the standalone HTML.\n",
+ "The full reference includes all required investigations; quick smoke\n",
+ "exercises APIs and has no scientific performance qualification.\n",
+ "\n",
+ "\n",
+ "Science Explanation
\n",
+ "\n",
+ "After provisioning dependencies, execution must work with network access\n",
+ "blocked. The public HTML contains the bounded gallery and complete evaluation\n",
+ "score/statistic arrays, not the full training set or checkpoints. Its controls\n",
+ "need no Bokeh server, Python kernel, WebSocket or API. Source ZIPs, executed\n",
+ "notebooks and exported HTML are separate build artifacts. See README and\n",
+ "PUBLISHING for commands and the remaining versioned-Pages integration gap.\n",
+ "Persistent human annotations, if wanted, use the optional local native review\n",
+ "workflow on the newly generated queue. Publication is a later action.\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bcdc90a2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table({k:reference[\"provenance\"][k] for k in\n",
+ " (\"cuphoton_revision\",\"hardware\",\"dependencies\")}, caption=\"Execution provenance\")\n",
+ "table(reference[\"artifact_sizes\"], caption=\"Saved artifact bytes\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8cc59919",
+ "metadata": {},
+ "source": [
+ "## Terminology\n",
+ "\n",
+ "
\n",
+ "- PSF
- Point-spread function: the pixel distribution of an unresolved source; its integral gives source flux.
\n",
+ "- reference
- An earlier noisy exposure shared by repeated search epochs of one independent field.
\n",
+ "- search
- The newer exposure inspected for positive astrophysical changes.
\n",
+ "- electron
- The detector count unit here; unit gain equates recorded counts and electrons.
\n",
+ "- flux
- Expected total source electrons integrated over the PSF, before finite-image truncation.
\n",
+ "- background
- Expected diffuse electrons per pixel, separate from astronomical source flux.
\n",
+ "- seeing
- Apparent atmospheric source width, modeled here by a Gaussian sigma in pixels.
\n",
+ "- Poisson
- Counting distribution whose variance equals its expected electron count.
\n",
+ "- read noise
- Independent Gaussian detector readout fluctuation with a declared RMS in electrons.
\n",
+ "- kernel
- Fitted convolution weights mapping the reference PSF and photometric scale toward the search exposure.
\n",
+ "- residual
- Search minus its fitted matched-reference-plus-background model.
\n",
+ "- variance
- Expected squared fluctuation in the image unit; variance scales quadratically under unit changes.
\n",
+ "- covariance
- Correlated fluctuations between pixels or observations; omitted from the marginal diagnostic.
\n",
+ "- SNR
- Signal-to-noise ratio; measured detection SNR here is an approximate marginal diagnostic.
\n",
+ "- candidate
- A measured local positive residual maximum passing the frozen detection rule.
\n",
+ "- association
- A truth-only positional rule connecting a measured detection to injected causes.
\n",
+ "- ambiguous
- A detection associated with several genuine changes or a real change overlapping a detector artifact.
\n",
+ "- field
- Independent generated sky/detector scene owning its template, sources, defects and repeated epochs.
\n",
+ "- epoch
- One search observation of a field, with independent search noise.
\n",
+ "- split
- Assignment of independent fields to training, validation or reserved final test roles.
\n",
+ "- provenance
- Recorded origins, identities, labels, seeds and hashes supporting reproducible input relationships.
\n",
+ "- normalization
- Centering/scaling fitted only on training data and applied unchanged to held-out data.
\n",
+ "- logit
- Unbounded classifier output transformed by the sigmoid to a score.
\n",
+ "- score
- Classifier output between zero and one; it need not be a calibrated astrophysical probability.
\n",
+ "- checkpoint
- Saved model parameters selected using development validation loss.
\n",
+ "- baseline
- A simpler measured-data model used to test whether complex learning adds useful information.
\n",
+ "- permutation
- A fixed random reassignment of training labels for a negative-control investigation.
\n",
+ "- fusion
- Combining measured xFit scalar features with image features through native xScan interfaces.
\n",
+ "- threshold
- The inclusive score cutoff used to retain candidates for review.
\n",
+ "- recall
- Retained eligible detected positives divided by all eligible detected positives.
\n",
+ "- false-positive rate
- Retained negatives divided by all eligible negatives, at the specified frozen threshold.
\n",
+ "- precision
- Retained positives divided by all retained candidates; also called purity.
\n",
+ "- prevalence
- Fraction of positives in the candidate population before the classifier threshold.
\n",
+ "- Brier score
- Mean squared difference between a score and its binary label.
\n",
+ "- reliability
- Agreement of average scores with observed positive fractions in occupied score bins.
\n",
+ "- bootstrap
- Resampling independent fields with replacement to quantify specified conditional sampling uncertainty.
\n",
+ "- distribution shift
- A specified change to image or population-generating conditions relative to development.
\n",
+ "- coverage
- The fraction/count of observations for which a stated method or metric is defined and included.
\n",
+ "- cosmic ray
- A modeled compact detector charge event with a morphology distinct from the optical PSF.
\n",
+ "- misregistration
- An offset between reference and search source positions that can create dipolar residuals.
\n",
+ "- training epoch
- One complete pass through the training candidates, distinct from a search-observation epoch.
\n",
+ "- detection threshold
- Minimum positive filtered residual SNR for creating a candidate; distinct from the classifier score threshold.
\n",
+ "
"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/model-alerts/pixel-readout.js b/examples/model-alerts/pixel-readout.js
new file mode 100644
index 00000000..f2118b00
--- /dev/null
+++ b/examples/model-alerts/pixel-readout.js
@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+const x = Math.floor(cb_obj.x), y = Math.floor(cb_obj.y);
+if (!Number.isFinite(x) || !Number.isFinite(y)) return;
+if (x < 0 || y < 0 || x >= size || y >= size) return;
+if (sources.some(s => !s.data.raw_flat?.length)) {
+ exact.text = 'No selected candidate.';
+ selection.data = {x:[], y:[], width:[], height:[]};
+ for (const source of label_sources) {
+ source.data = {x:[], y:[], x_offset:[], y_offset:[], text:[]};
+ }
+ return;
+}
+exact.text = `Raw electrons at (${x}, ${y}): ` + sources.map(
+ s => String(s.data.raw_flat[0][y * size + x])
+).join(' | ');
diff --git a/examples/model-alerts/preview.py b/examples/model-alerts/preview.py
new file mode 100644
index 00000000..390b7167
--- /dev/null
+++ b/examples/model-alerts/preview.py
@@ -0,0 +1,265 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Render a local guide preview with a completed Model Alerts reference.
+
+The source notebook stays output-free. This stages its executed counterpart
+in a copy of the guide, leaving the checkout and publication workflows alone.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from urllib.parse import quote
+from zipfile import ZIP_DEFLATED, ZipFile
+
+import nbformat
+import yaml
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[1]
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--executed", type=Path, required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument(
+ "--site-url", default="https://nvidia.github.io/cuPhoton/"
+ )
+ parser.add_argument(
+ "--source-ref",
+ default=None,
+ help="GitHub branch, tag or commit for the visible notebook link",
+ )
+ parser.add_argument(
+ "--allow-smoke",
+ action="store_true",
+ help="Render an explicitly labeled API-only smoke preview",
+ )
+ options = parser.parse_args()
+ output = options.output.resolve()
+ if output == REPO or output.is_relative_to(REPO):
+ parser.error("--output must be outside the example source directory")
+ if output.exists() and any(output.iterdir()):
+ parser.error("--output must be absent or empty")
+
+ executed = options.executed.resolve()
+ receipt = json.loads(executed.with_name("execution.json").read_text())
+ if receipt["status"] != "passed":
+ raise ValueError("The reference execution has not passed")
+ if receipt["profile"] != "full-reference" and not options.allow_smoke:
+ raise ValueError(
+ "scientific preview requires full-reference; "
+ "use --allow-smoke only for API review"
+ )
+ expected_notebook = receipt["output_sha256"][executed.name]
+ if hashlib.sha256(executed.read_bytes()).hexdigest() != expected_notebook:
+ raise ValueError("Executed notebook does not match its receipt")
+ for name, expected in receipt["source_sha256"].items():
+ actual = hashlib.sha256((HERE / name).read_bytes()).hexdigest()
+ if actual != expected:
+ raise ValueError(f"Reference source no longer matches: {name}")
+ notebook = nbformat.read(executed, as_version=4)
+ code = [cell for cell in notebook.cells if cell.cell_type == "code"]
+ if len(code) != 14 or any(cell.execution_count is None for cell in code):
+ raise ValueError("A completed 14-cell reference is required")
+ if any(o.output_type == "error" for c in code for o in c.outputs):
+ raise ValueError("Reference contains an execution error")
+ if (
+ sum(
+ "application/vnd.bokehjs_exec.v0+json" in o.get("data", {})
+ for c in code
+ for o in c.outputs
+ )
+ != 3
+ ):
+ raise ValueError("Reference must contain three interactive documents")
+
+ guide = output / "guide"
+ shutil.copytree(
+ REPO / "docs/guide",
+ guide,
+ ignore=shutil.ignore_patterns(
+ "_book", "_freeze", ".quarto", "__pycache__"
+ ),
+ )
+ shutil.copytree(
+ HERE / "assets",
+ guide / "assets/model-alerts",
+ ignore=shutil.ignore_patterns("*.html"),
+ )
+ includes = output / "includes"
+ includes.mkdir()
+ for name in ("guide-head.html", "guide-include.html"):
+ shutil.copy2(HERE / "assets" / name, includes / name)
+ (guide / "_environment").write_text(f"CUPHOTON_REPO={REPO}\n")
+ config_path = guide / "_quarto.yml"
+ config = yaml.safe_load(config_path.read_text())
+ config["book"]["site-url"] = options.site_url
+ config["book"]["chapters"].insert(
+ 2, {"part": "Examples", "chapters": ["model-alerts.ipynb"]}
+ )
+ config["project"].setdefault("resources", []).append("source/**")
+ config_path.write_text(yaml.safe_dump(config, sort_keys=False))
+ index = guide / "index.qmd"
+ index.write_text(
+ index.read_text().replace(
+ "## About this guide",
+ "## Model Alerts\n\n"
+ "[Which changes deserve attention?](model-alerts.ipynb): "
+ "generate independent fields, train fresh classifiers, "
+ "and inspect "
+ "a modeled human review queue. The notebook includes executed "
+ "cell outputs, interactive figures, terminology definitions "
+ "and expandable scientific explanations.\n\n"
+ "## About this guide",
+ )
+ )
+
+ for cell in notebook.cells:
+ if cell.cell_type == "markdown":
+ cell.source = cell.source.replace(
+ "assets/story-", "assets/model-alerts/story-"
+ )
+ first = notebook.cells[0]
+ match = re.match(r"\A---\n(.*?)\n---\n", first.source, flags=re.S)
+ metadata = yaml.safe_load(match[1]) if match else {}
+ body = first.source[match.end() :] if match else first.source
+ body = re.sub(
+ r"\[View the source notebook in NVIDIA/cuPhoton\]"
+ r"\([^)]*examples/model-alerts/model-alerts\.ipynb\)\.?\n*",
+ "",
+ body,
+ )
+ if receipt["profile"] != "full-reference":
+ body = (
+ "::: {.callout-warning}\nAPI smoke preview only. "
+ "These small runs do not qualify scientific performance.\n:::\n\n"
+ + body
+ )
+ metadata.update(
+ title="Model Alerts", subtitle="Which changes deserve attention?"
+ )
+ metadata["execute"] = {"enabled": False, "warning": True}
+ metadata["number-sections"] = False
+ metadata["format"] = {
+ "html": {
+ "css": "assets/model-alerts/guide.css",
+ "include-in-header": "../includes/guide-head.html",
+ "include-after-body": "../includes/guide-include.html",
+ "embed-resources": True,
+ "html-math-method": "mathml",
+ }
+ }
+ metadata["code-links"] = [
+ {
+ "text": "Download source notebook",
+ "icon": "file-code",
+ "href": "source/model-alerts/model-alerts.ipynb",
+ },
+ {
+ "text": "Download example sources",
+ "icon": "download",
+ "href": "source/model-alerts.zip",
+ },
+ {
+ "text": "cuPhoton on GitHub",
+ "icon": "github",
+ "href": "https://github.com/NVIDIA/cuPhoton",
+ },
+ ]
+ source_ref = options.source_ref or receipt["cuphoton_revision"]
+ source_link = (
+ "[View the source Jupyter notebook in NVIDIA/cuPhoton]"
+ "(https://github.com/NVIDIA/cuPhoton/blob/"
+ f"{quote(source_ref, safe='')}/"
+ "examples/model-alerts/model-alerts.ipynb).\n\n"
+ )
+ first.source = (
+ "---\n"
+ + yaml.safe_dump(metadata, sort_keys=False)
+ + "---\n\n"
+ + source_link
+ + body.lstrip()
+ )
+ nbformat.write(notebook, guide / "model-alerts.ipynb")
+
+ sources = output / "example-sources/model-alerts"
+ shutil.copytree(
+ HERE,
+ sources,
+ ignore=shutil.ignore_patterns(
+ "__pycache__",
+ ".pytest_cache",
+ ".ruff_cache",
+ "data",
+ "results",
+ "*.executed.ipynb",
+ "model-alerts-output",
+ "model-alerts-guide-output",
+ ),
+ )
+ shutil.copy2(REPO / "LICENSE", sources / "LICENSE")
+ # HTML includes are build inputs, not standalone documentation pages.
+ shutil.copytree(
+ sources,
+ guide / "source/model-alerts",
+ ignore=shutil.ignore_patterns("*.html"),
+ )
+ with ZipFile(
+ guide / "source/model-alerts.zip", "w", ZIP_DEFLATED
+ ) as archive:
+ for path in sorted(sources.rglob("*")):
+ if path.is_file():
+ archive.write(
+ path,
+ "model-alerts/" + path.relative_to(sources).as_posix(),
+ )
+
+ env = os.environ.copy()
+ env.update(
+ QUARTO_PYTHON=sys.executable,
+ CUPHOTON_REPO=str(REPO),
+ CUPHOTON_XREP_TORCH_DEVICE="cpu",
+ CUDA_VISIBLE_DEVICES="",
+ MPLBACKEND="Agg",
+ PATH=str(Path(sys.executable).parent) + os.pathsep + env["PATH"],
+ )
+ # QMD CPU examples execute; the prepared IPYNB retains its saved outputs.
+ subprocess.run(
+ ["quarto", "render", str(guide), "--no-cache"], env=env, check=True
+ )
+ # Reuse current developer transforms/checks without changing publication.
+ sys.path.insert(0, str(REPO / "dev/src"))
+ from cuphoton_dev.publisher import docs_check
+ from cuphoton_dev.transforms import postprocess
+
+ revision = subprocess.check_output(
+ ["git", "rev-parse", "HEAD"], cwd=REPO, text=True
+ ).strip()
+ postprocess(
+ guide / "_book",
+ version="main",
+ source_sha=revision,
+ controller_sha=revision,
+ site_url=options.site_url,
+ renderer="quarto "
+ + subprocess.check_output(["quarto", "--version"], text=True).strip(),
+ )
+ checked = docs_check(guide / "_book")
+ (output / "docs-check.json").write_text(
+ json.dumps(checked, indent=2) + "\n"
+ )
+ print(guide / "_book/model-alerts.html")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/model-alerts/reference.py b/examples/model-alerts/reference.py
new file mode 100644
index 00000000..3818a420
--- /dev/null
+++ b/examples/model-alerts/reference.py
@@ -0,0 +1,766 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Development, freeze and independent evaluation for Model Alerts."""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import importlib.metadata
+import json
+import os
+import re
+import subprocess
+from dataclasses import asdict
+from pathlib import Path
+
+import numpy as np
+from baselines import LogisticBaseline, measured_features
+from controls import (
+ learning_curve_subset,
+ permutation_control,
+ rejected_split_control,
+)
+from evaluation import confusion, field_rate_intervals, paired_field_bootstrap
+from fusion import fit_feature_bundle
+from generator import Design, detect, field_exposures, write_dataset
+from run_study import pilot
+from study import prepare_training, train_development
+from views import gallery_indices
+
+
+def write_json(path, value):
+ Path(path).write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")
+
+
+def rows_at(path):
+ return [
+ json.loads(line)
+ for line in (Path(path) / "metadata.jsonl").read_text().splitlines()
+ ]
+
+
+def sha(path):
+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def content_hashes(dataset):
+ """Hash full raw triplets across partitions, independent of row IDs."""
+ arrays = [
+ np.load(Path(dataset) / f"{name}.npy", mmap_mode="r")
+ for name in ("search", "template", "difference")
+ ]
+ return [
+ hashlib.sha256(np.stack(triplet).tobytes()).hexdigest()
+ for triplet in zip(*arrays, strict=True)
+ ]
+
+
+def infer(run, dataset, mode, *, feature_dir=None, device="cuda:0"):
+ import torch
+
+ from cuphoton.xscan.dataset import StampDataset
+ from cuphoton.xscan.training import (
+ load_model_from_checkpoint,
+ predict_dataset,
+ )
+
+ model, _, performance = load_model_from_checkpoint(
+ Path(run), device=torch.device(device)
+ )
+ data = StampDataset(
+ Path(dataset),
+ input_mode=mode,
+ split="test",
+ xfit_feature_dir=feature_dir,
+ )
+ result = predict_dataset(
+ model=model,
+ dataset=data,
+ batch_size=64,
+ device=torch.device(device),
+ performance=performance,
+ )
+ expected_rows = rows_at(dataset)
+ expected_indices = data.indices.tolist()
+ np.testing.assert_array_equal(
+ result["labels"],
+ np.load(Path(dataset) / "labels.npy")[expected_indices],
+ )
+ if [r["candidate_id"] for r in result["metadata_rows"]] != [
+ expected_rows[i]["candidate_id"] for i in expected_indices
+ ]:
+ raise AssertionError("native inference row identities changed order")
+ return result["probabilities"]
+
+
+def _experiment(root, *, smoke, device, revision):
+ design = Design()
+ agreement = {}
+ if device.startswith("cuda"):
+ exposure = next(field_exposures(9000, design=design))
+ cpu_stamps, cpu_rows, cpu_fit, _ = detect(
+ exposure, design=design, backend="cpu"
+ )
+ gpu_stamps, gpu_rows, gpu_fit, _ = detect(
+ exposure, design=design, backend="cupy"
+ )
+ valid = np.isfinite(cpu_fit.residual)
+ np.testing.assert_array_equal(valid, np.isfinite(gpu_fit.residual))
+ np.testing.assert_allclose(
+ cpu_fit.residual[valid],
+ gpu_fit.residual[valid],
+ rtol=1e-8,
+ atol=1e-7,
+ )
+ if [(r["x"], r["y"]) for r in cpu_rows] != [
+ (r["x"], r["y"]) for r in gpu_rows
+ ]:
+ raise AssertionError(
+ "CPU/CUDA detected candidate locations disagree"
+ )
+ np.testing.assert_allclose(
+ cpu_stamps, gpu_stamps, rtol=1e-8, atol=1e-7
+ )
+ agreement = dict(
+ backend_pair=["cpu", "cupy"],
+ field=9000,
+ candidate_count=len(cpu_rows),
+ valid_residual_pixels=int(valid.sum()),
+ relative_tolerance=1e-8,
+ absolute_tolerance=1e-7,
+ maximum_residual_difference=float(
+ np.max(
+ np.abs(cpu_fit.residual[valid] - gpu_fit.residual[valid])
+ )
+ ),
+ )
+ multiplier = 1
+ while True:
+ development = root / f"development-{multiplier}"
+ result = pilot(
+ development, smoke=smoke, device=device, multiplier=multiplier
+ )
+ if smoke or not result["precision_requires_expansion"]:
+ break
+ print(
+ "Development precision requires doubling fields before freeze.",
+ flush=True,
+ )
+ multiplier *= 2
+ if multiplier > 16:
+ raise RuntimeError(
+ "precision remains inadequate; diagnose development study"
+ )
+ prepared, raw = development / "prepared", development / "raw"
+ epochs = 1 if smoke else 12
+ controls = dict(
+ registration_diagnostic=result["registration_diagnostic"],
+ cpu_cuda_agreement=agreement,
+ invalid_split=rejected_split_control(
+ prepared, root / "invalid-split"
+ ),
+ )
+ permuted = root / "permuted"
+ controls["permutation_design"] = permutation_control(prepared, permuted)
+ controls["permutation"] = train_development(
+ permuted,
+ root / "permutation-run",
+ mode="triplet",
+ seed=11,
+ epochs=epochs,
+ device=device,
+ )
+ controls["permutation_review_required"] = (
+ abs(controls["permutation"]["validation_roc_auc"] - 0.5) > 0.15
+ )
+ if not smoke and controls["permutation_review_required"]:
+ raise RuntimeError(
+ "permutation control unexpectedly predictive; "
+ "diagnose before final freeze, do not change its seed"
+ )
+ controls["learning_curve"] = []
+ for divisor in (4, 2):
+ count = max(2, result["train_fields"] // divisor)
+ subset_raw, subset = (
+ root / f"curve-raw-{divisor}",
+ root / f"curve-prepared-{divisor}",
+ )
+ learning_curve_subset(raw, subset_raw, count)
+ prepare_training(subset_raw, subset)
+ curve = train_development(
+ subset,
+ root / f"curve-run-{divisor}",
+ mode="triplet",
+ seed=11,
+ epochs=epochs,
+ device=device,
+ )
+ controls["learning_curve"].append(
+ dict(training_fields=count, **curve)
+ )
+ feature_dir, fit_coverage = fit_feature_bundle(
+ prepared,
+ root / "fusion-development",
+ backend="cupy" if device.startswith("cuda") else "numpy",
+ )
+ fusion_result = train_development(
+ prepared,
+ root / "fusion-run",
+ mode="triplet",
+ seed=11,
+ epochs=epochs,
+ device=device,
+ xfit_feature_dir=feature_dir,
+ )
+ normalization = json.loads((prepared / "normalization.json").read_text())
+ test_count = 3 if smoke else 60 * multiplier
+ shift_count = 3 if smoke else 60
+ test_base = int(
+ os.environ.get(
+ "CUPHOTON_MODEL_ALERTS_TEST_SEED_BASE",
+ "40000" if smoke else "20000",
+ )
+ )
+ shift_base = int(
+ os.environ.get(
+ "CUPHOTON_MODEL_ALERTS_SHIFT_SEED_BASE",
+ "50000" if smoke else "30000",
+ )
+ )
+ # Read every generated field, including zero-detection fields and earlier
+ # precision pilots. Candidate metadata alone would miss empty fields.
+ development_fields = {9000}
+ for generation_path in root.glob("development-*/raw/generation.json"):
+ generation = json.loads(generation_path.read_text())
+ development_fields.update(map(int, generation["fields"]))
+ test_fields = set(range(test_base, test_base + test_count))
+ shift_fields = set(range(shift_base, shift_base + shift_count))
+ if (
+ min(test_base, shift_base) < 0
+ or test_fields & shift_fields
+ or (test_fields | shift_fields) & development_fields
+ ):
+ raise ValueError(
+ "reserved seed blocks overlap development or each other"
+ )
+ streams = [
+ 0,
+ 1,
+ *range(10, 10 + design.epochs),
+ 1000,
+ *range(1100, 1100 + design.epochs),
+ ]
+ if len(streams) != len(set(streams)):
+ raise ValueError("generator stream roles overlap")
+ # Reserved blocks are declared before their first generation. Separate
+ # Smoke uses separate blocks; invalidated full reruns need fresh blocks.
+ freeze = dict(
+ design=asdict(design),
+ seed_audit=dict(
+ development_fields=sorted(development_fields),
+ development_field_count=len(development_fields),
+ stream_roles=streams,
+ seed_scheme="SeedSequence(root_seed, field, stream)",
+ reserved_blocks_disjoint=True,
+ ),
+ train_fields=result["train_fields"],
+ validation_fields=result["validation_fields"],
+ test_fields=test_count,
+ shifted_fields=shift_count,
+ test_seed_block=[test_base, test_base + test_count],
+ shift_seed_block=[shift_base, shift_base + shift_count],
+ normalization=normalization,
+ operating_points=result["outcomes"],
+ baselines=result["baselines"],
+ fusion_operating_point=fusion_result,
+ checkpoints={
+ str(p.relative_to(root)): sha(p)
+ for p in root.rglob("checkpoint.pt")
+ },
+ source_sha256={
+ str(p.relative_to(Path(__file__).parent)): sha(p)
+ for p in Path(__file__).parent.rglob("*")
+ if p.is_file()
+ and p.suffix in (".py", ".js", ".css", ".html", ".ipynb")
+ and "__pycache__" not in p.parts
+ },
+ )
+ write_json(root / "freeze.json", freeze)
+ # Nothing above this line generates or infers the reserved populations.
+ print(
+ "Frozen models and thresholds; generating reserved populations.",
+ flush=True,
+ )
+ report = dict(
+ development_directory=str(development.relative_to(root)),
+ design=dict(
+ profile="quick-smoke" if smoke else "full-reference",
+ scientific_performance_qualified=not smoke,
+ **asdict(design),
+ train_fields=result["train_fields"],
+ validation_fields=result["validation_fields"],
+ test_fields=test_count,
+ shifted_fields=shift_count,
+ ),
+ training=result["outcomes"],
+ baselines=result["baselines"],
+ controls=controls,
+ fusion=dict(
+ development_coverage=fit_coverage, validation=fusion_result
+ ),
+ freeze=freeze,
+ coverage={
+ split: dict(
+ all_detected=len(split_rows),
+ eligible=sum(
+ not r["ambiguous"] and not r["challenge"]
+ for r in split_rows
+ ),
+ ambiguous=sum(r["ambiguous"] for r in split_rows),
+ overlapping_causes=sum(
+ r["overlapping_causes"] for r in split_rows
+ ),
+ moving_challenge=sum(
+ r["challenge"] and not r["ambiguous"] for r in split_rows
+ ),
+ )
+ for split in ("train", "val")
+ for split_rows in [
+ [r for r in rows_at(raw) if r["split"] == split]
+ ]
+ },
+ endpoints={},
+ uncertainty={},
+ shift={},
+ moving_source_challenge={},
+ breakdowns={"test": {}, "shift": {}},
+ high_confidence_mistakes={"test": {}, "shift": {}},
+ )
+ seen_content = set(content_hashes(raw))
+ content_audit = {"development": len(seen_content)}
+ for population, fields, shifted in (
+ ("test", {test_base + i: "test" for i in range(test_count)}, False),
+ ("shift", {shift_base + i: "test" for i in range(shift_count)}, True),
+ ):
+ population_raw, population_data = (
+ root / f"{population}-raw",
+ root / f"{population}-prepared",
+ )
+ write_dataset(
+ population_raw,
+ fields,
+ design=design,
+ shifted=shifted,
+ backend="cpu",
+ )
+ digests = content_hashes(population_raw)
+ if seen_content.intersection(digests) or len(set(digests)) != len(
+ digests
+ ):
+ raise ValueError(
+ "duplicate candidate content across evaluation partitions"
+ )
+ seen_content.update(digests)
+ content_audit[population] = len(digests)
+ prepare_training(
+ population_raw,
+ population_data,
+ normalization=normalization,
+ eligible_only=False,
+ )
+ population_features, coverage = fit_feature_bundle(
+ population_data,
+ root / f"fusion-{population}",
+ backend="cupy" if device.startswith("cuda") else "numpy",
+ )
+ rows = rows_at(population_data)
+ labels = np.load(population_data / "labels.npy")
+ groups = np.array([r["field"] for r in rows])
+ eligible = np.array(
+ [not r["ambiguous"] and not r["challenge"] for r in rows]
+ )
+ movers = np.array(
+ [r["challenge"] and not r["ambiguous"] for r in rows]
+ )
+ raw_images = np.stack(
+ [
+ np.load(population_raw / f"{n}.npy", mmap_mode="r")
+ for n in ("search", "template", "difference")
+ ],
+ axis=1,
+ )
+ scores, thresholds = {}, {}
+ for outcome in result["outcomes"]:
+ key = f"{outcome['mode']}-{outcome['seed']}"
+ scores[key] = infer(
+ development / key,
+ population_data,
+ outcome["mode"],
+ device=device,
+ )
+ thresholds[key] = outcome["threshold"]
+ if population == "test" and device.startswith("cuda"):
+ cpu_scores = infer(
+ development / "triplet-11",
+ population_data,
+ "triplet",
+ device="cpu",
+ )
+ gpu_scores = scores["triplet-11"]
+ np.testing.assert_allclose(
+ cpu_scores, gpu_scores, rtol=2e-4, atol=2e-5
+ )
+ report["controls"]["cpu_cuda_inference"] = dict(
+ mode="triplet",
+ seed=11,
+ rows=len(cpu_scores),
+ maximum_score_difference=float(
+ np.max(np.abs(cpu_scores - gpu_scores))
+ ),
+ threshold_decision_flips=int(
+ np.sum(
+ (cpu_scores >= thresholds["triplet-11"])
+ != (gpu_scores >= thresholds["triplet-11"])
+ )
+ ),
+ relative_tolerance=2e-4,
+ absolute_tolerance=2e-5,
+ )
+ scores["fusion-11"] = infer(
+ root / "fusion-run",
+ population_data,
+ "triplet",
+ feature_dir=population_features,
+ device=device,
+ )
+ thresholds["fusion-11"] = fusion_result["threshold"]
+ measured = measured_features(raw_images, [r["snr"] for r in rows])
+ for name, baseline in result["baselines"].items():
+ model = LogisticBaseline()
+ for attribute in ("center", "scale", "weights"):
+ setattr(
+ model, attribute, np.array(baseline["model"][attribute])
+ )
+ columns = (
+ [0] if name == "snr_only" else list(range(measured.shape[1]))
+ )
+ scores[name] = model.predict(measured[:, columns])
+ thresholds[name] = baseline["threshold"]
+ metrics = {
+ key: confusion(labels[eligible], value[eligible], thresholds[key])
+ for key, value in scores.items()
+ }
+ report["coverage"][population] = dict(
+ all_detected=len(rows),
+ eligible=int(eligible.sum()),
+ ambiguous=sum(r["ambiguous"] for r in rows),
+ overlapping_causes=sum(r["overlapping_causes"] for r in rows),
+ moving_challenge=int(movers.sum()),
+ fit=coverage,
+ )
+ report["moving_source_challenge"][population] = {
+ key: dict(
+ count=int(movers.sum()),
+ retained=int(np.sum(value[movers] >= thresholds[key])),
+ )
+ for key, value in scores.items()
+ }
+ generation = json.loads(
+ (population_raw / "generation.json").read_text()
+ )
+ injection_families = {
+ identity: family
+ for recovery in generation["recovery"]
+ for identity, family in recovery["injection_families"].items()
+ }
+ injected = sum(r["injected"] for r in generation["recovery"])
+ detected = sum(r["detected"] for r in generation["recovery"])
+ detected_ids = {
+ identity for r in rows for identity in r["associated_injections"]
+ }
+ eligible_ids = {
+ identity
+ for i, r in enumerate(rows)
+ if eligible[i]
+ for identity in r["associated_injections"]
+ }
+ if len(detected_ids) != detected:
+ raise AssertionError(
+ "recovery and candidate association disagree"
+ )
+ for key, value in scores.items():
+ retained_ids = {
+ identity
+ for i, r in enumerate(rows)
+ if eligible[i] and value[i] >= thresholds[key]
+ for identity in r["associated_injections"]
+ }
+ metrics[key]["injections"] = dict(
+ injected=injected,
+ detected=detected,
+ eligible_detected=len(eligible_ids),
+ ineligible_only_detected=len(detected_ids - eligible_ids),
+ classifier_rejected_eligible=len(eligible_ids - retained_ids),
+ retained=len(retained_ids),
+ detection_recovery=detected / injected,
+ retained_fraction=len(retained_ids) / injected,
+ by_family={
+ family: dict(
+ injected=len(identities),
+ detected=len(identities & detected_ids),
+ eligible_detected=len(identities & eligible_ids),
+ ineligible_only_detected=len(
+ identities & (detected_ids - eligible_ids)
+ ),
+ classifier_rejected_eligible=len(
+ identities & (eligible_ids - retained_ids)
+ ),
+ retained=len(identities & retained_ids),
+ detection_recovery=len(identities & detected_ids)
+ / len(identities),
+ retained_fraction=len(identities & retained_ids)
+ / len(identities),
+ )
+ for family in sorted(set(injection_families.values()))
+ for identities in [
+ {
+ identity
+ for identity, f in injection_families.items()
+ if f == family
+ }
+ ]
+ },
+ )
+ target = "endpoints" if population == "test" else "shift"
+ report[target] = metrics
+ report["uncertainty"][population + "_model_field_intervals"] = {
+ key: field_rate_intervals(
+ labels[eligible],
+ value[eligible],
+ groups[eligible],
+ thresholds[key],
+ )
+ for key, value in scores.items()
+ }
+ if population == "test":
+ for outcome in result["outcomes"]:
+ if outcome["mode"] != "pair":
+ continue
+ seed = outcome["seed"]
+ a = f"pair-{seed}"
+ b = f"triplet-{seed}"
+ report["uncertainty"][str(seed)] = paired_field_bootstrap(
+ labels[eligible],
+ scores[a][eligible],
+ scores[b][eligible],
+ groups[eligible],
+ thresholds[a],
+ thresholds[b],
+ )
+ selected_rows = [
+ r for r, keep in zip(rows, eligible, strict=True) if keep
+ ]
+ selected_scores = {
+ key: value[eligible] for key, value in scores.items()
+ }
+ selected_labels = labels[eligible]
+ score_key = "triplet-11"
+ gallery = gallery_indices(
+ selected_rows,
+ selected_labels,
+ selected_scores[score_key],
+ thresholds[score_key],
+ )
+ np.savez(
+ root / "evaluation.npz",
+ labels=selected_labels,
+ snr=np.array([r["snr"] for r in selected_rows]),
+ candidate_id=np.array(
+ [str(r["candidate_id"]) for r in selected_rows]
+ ),
+ family=np.array([r["family"] for r in selected_rows]),
+ **selected_scores,
+ )
+ np.savez(
+ root / "gallery.npz",
+ images=raw_images[eligible][gallery],
+ labels=selected_labels[gallery],
+ scores=selected_scores[score_key][gallery],
+ )
+ write_json(
+ root / "gallery-rows.json",
+ [selected_rows[i] for i in gallery],
+ )
+ report["score_arrays"] = {key: key for key in scores}
+ report["inspector_threshold"] = thresholds[score_key]
+ report["inspector_score_key"] = score_key
+ report["gallery_rule"] = dict(
+ indices=gallery.tolist(),
+ maximum=96,
+ rule="round-robin confusion/SNR/family, candidate-ID order",
+ )
+ selected_rows = [
+ r for r, keep in zip(rows, eligible, strict=True) if keep
+ ]
+ selected_scores = {
+ key: value[eligible] for key, value in scores.items()
+ }
+ selected_labels = labels[eligible]
+ for key, value in selected_scores.items():
+ by_family = {}
+ for family in sorted({r["family"] for r in selected_rows}):
+ mask = np.array(
+ [r["family"] == family for r in selected_rows]
+ )
+ by_family[family] = confusion(
+ selected_labels[mask], value[mask], thresholds[key]
+ )
+ by_snr = {}
+ snr = np.array([r["snr"] for r in selected_rows])
+ for lo, hi in ((4, 5), (5, 10), (10, 20), (20, float("inf"))):
+ mask = (snr >= lo) & (snr < hi)
+ by_snr[f"{lo}:{hi}"] = (
+ confusion(
+ selected_labels[mask],
+ value[mask],
+ thresholds[key],
+ )
+ if mask.any()
+ else dict(count=0)
+ )
+ report["breakdowns"][population][key] = dict(
+ family=by_family,
+ snr=by_snr,
+ overlap={
+ str(overlap): confusion(
+ selected_labels[mask], value[mask], thresholds[key]
+ )
+ for overlap in (False, True)
+ for mask in [
+ np.array(
+ [
+ r["overlapping_causes"] == overlap
+ for r in selected_rows
+ ]
+ )
+ ]
+ if mask.any()
+ },
+ )
+ wrong = (value >= thresholds[key]) != selected_labels.astype(bool)
+ order = np.argsort(
+ -np.abs(value - selected_labels), kind="stable"
+ )
+ report["high_confidence_mistakes"][population][key] = [
+ dict(
+ candidate_id=selected_rows[i]["candidate_id"],
+ score=float(value[i]),
+ label=int(selected_labels[i]),
+ family=selected_rows[i]["family"],
+ )
+ for i in order
+ if wrong[i]
+ ][:12]
+ import torch
+
+ report["provenance"] = dict(
+ hardware={
+ "accelerator": torch.cuda.get_device_name(0)
+ if device.startswith("cuda")
+ else "CPU"
+ },
+ source_sha256=freeze["source_sha256"],
+ cuphoton_revision=revision,
+ dependencies={
+ name: importlib.metadata.version(name)
+ for name in (
+ "numpy",
+ "scipy",
+ "torch",
+ "astropy",
+ "bokeh",
+ "pyarrow",
+ )
+ },
+ artifact_sha256={
+ str(p.relative_to(root)): sha(p) for p in root.glob("*.npz")
+ },
+ content_audit=dict(
+ partition_counts=content_audit, all_triplets_unique=True
+ ),
+ network_required_after_provisioning=False,
+ )
+ if device.startswith("cuda"):
+ report["provenance"]["dependencies"]["cupy-cuda13x"] = (
+ importlib.metadata.version("cupy-cuda13x")
+ )
+ report["artifact_sizes"] = {
+ str(p.relative_to(root)): p.stat().st_size
+ for p in root.glob("*")
+ if p.is_file()
+ }
+ write_json(root / "report.json", report)
+ return report
+
+
+def execute_reference(output, *, profile="quick-smoke", device="cuda:0"):
+ """Execute once in a new directory, retaining the shared GPU lease."""
+ from cuphoton.xscan.training import resolve_device
+
+ device = str(resolve_device(device))
+ if profile not in ("quick-smoke", "full-reference"):
+ raise ValueError("profile must be quick-smoke or full-reference")
+ if profile == "full-reference" and not device.startswith("cuda"):
+ raise ValueError(
+ "full-reference qualification requires CUDA execution"
+ )
+ root = Path(output).resolve()
+ source = Path(__file__).resolve().parent
+ if repo := os.environ.get("CUPHOTON_MODEL_ALERTS_SOURCE_REPO"):
+ source = Path(repo).resolve()
+ else:
+ # Extracted ZIPs may have no Git ancestor. Never assume a fixed depth.
+ source = next(
+ (p for p in source.parents if (p / ".git").exists()), source
+ )
+ if (
+ root == source
+ or root.is_relative_to(source)
+ or root == Path(root.anchor)
+ ):
+ raise ValueError(
+ "output must be outside example sources and filesystem root"
+ )
+ if root.exists():
+ raise ValueError("reference output must not exist")
+ lock_path = os.environ.get("CUPHOTON_MODEL_ALERTS_GPU_LOCK")
+ if device.startswith("cuda") and not lock_path:
+ raise ValueError(
+ "CUDA execution requires CUPHOTON_MODEL_ALERTS_GPU_LOCK"
+ )
+ revision = os.environ.get("CUPHOTON_MODEL_ALERTS_REVISION")
+ if not revision:
+ revision = subprocess.check_output(
+ ["git", "rev-parse", "HEAD"], cwd=source, text=True
+ ).strip()
+ if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
+ raise ValueError(
+ "cuPhoton revision must be a complete Git commit SHA"
+ )
+ root.mkdir(parents=True)
+ if device.startswith("cuda"):
+ with Path(lock_path).open("a") as lock:
+ print("Waiting for shared GPU lock", flush=True)
+ fcntl.flock(lock, fcntl.LOCK_EX)
+ print("Acquired shared GPU lock", flush=True)
+ return _experiment(
+ root,
+ smoke=profile == "quick-smoke",
+ device=device,
+ revision=revision,
+ )
+ return _experiment(
+ root, smoke=profile == "quick-smoke", device=device, revision=revision
+ )
diff --git a/examples/model-alerts/reporting.py b/examples/model-alerts/reporting.py
new file mode 100644
index 00000000..6eb86a37
--- /dev/null
+++ b/examples/model-alerts/reporting.py
@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Scientific tables; full machine-readable inventories stay in receipts."""
+
+from __future__ import annotations
+
+import html
+import json
+
+
+def table(records, *, caption, collapsed=False):
+ """Display explicit records without Pandas or implicit data reduction."""
+ from IPython.display import HTML, display
+
+ if isinstance(records, dict):
+ records = [dict(quantity=k, value=v) for k, v in records.items()]
+ records = list(records)
+ if not records:
+ display(HTML(f"{html.escape(caption)}: no entries.
"))
+ return
+ columns = list(dict.fromkeys(k for row in records for k in row))
+
+ def cell(value):
+ if value is None:
+ return "undefined"
+ if isinstance(value, float):
+ return f"{value:.6g}"
+ if isinstance(value, (dict, list)):
+ return html.escape(json.dumps(value, ensure_ascii=False))
+ return html.escape(str(value))
+
+ text = (
+ ''
+ + html.escape(caption)
+ + ""
+ )
+ text += (
+ "".join(
+ '| ' + html.escape(k) + " | " for k in columns
+ )
+ + "
"
+ )
+ for row in records:
+ text += (
+ ""
+ + "".join("| " + cell(row.get(k)) + " | " for k in columns)
+ + "
"
+ )
+ text += "
"
+ if collapsed:
+ text = (
+ ""
+ + html.escape(caption)
+ + "
"
+ + text
+ + " "
+ )
+ display(HTML(text))
+
+
+def model_rows(outcomes):
+ """Flatten model outcomes with explicit confusion denominators."""
+ rows = []
+ for name, result in outcomes.items():
+ rows.append(
+ dict(
+ model=name,
+ **{k: v for k, v in result.items() if k != "injections"},
+ **{
+ k: v
+ for k, v in result.get("injections", {}).items()
+ if k != "by_family"
+ },
+ )
+ )
+ return rows
diff --git a/examples/model-alerts/requirements.txt b/examples/model-alerts/requirements.txt
new file mode 100644
index 00000000..a2024a59
--- /dev/null
+++ b/examples/model-alerts/requirements.txt
@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Optional example additions to a CUDA 13 cuPhoton installation.
+bokeh==3.9.1
+ipykernel==7.4.0
+nbclient==0.11.0
+nbformat>=5.10.4,<6
+# Local guide export and the current repository's developer postprocessor.
+beautifulsoup4>=4.14,<5
+matplotlib>=3.10,<4
diff --git a/examples/model-alerts/review-controls.js b/examples/model-alerts/review-controls.js
new file mode 100644
index 00000000..0eb47252
--- /dev/null
+++ b/examples/model-alerts/review-controls.js
@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+const t=threshold.value;
+threshold_line.location=t;
+let tp=0,fp=0,fn=0,tn=0;
+for(let k=0;k=t;
+ if(positive) {if(keep)tp++;else fn++;} else {if(keep)fp++;else tn++;}
+}
+const tpr=tp+fn ? tp/(tp+fn) : NaN, fpr=fp+tn ? fp/(fp+tn) : NaN;
+const real=prevalence.value*tpr, bogus=(1-prevalence.value)*fpr;
+queue.text=`Complete evaluation: TP ${tp}, FP ${fp}, FN ${fn}, TN ${tn}. `+
+ `Recall ${tpr.toFixed(4)}, FPR ${fpr.toFixed(4)}. `+
+ `Expected queue per 10,000: ${(10000*(real+bogus)).toFixed(1)}; purity ${real+bogus ? (real/(real+bogus)).toFixed(4) : 'undefined'}. `+
+ `Prevalence projection holds class-conditional mixtures fixed. Sliders explore saved scores.`;
+let i=Math.min(index.value, bank.data.score.length-1), found=-1;
+for(let k=0;k=t, positive=bank.data.label[j]===1;
+ const outcome=(positive===keep ? 'T':'F')+(keep?'P':'N');
+ if((family.value==='all'||family.value===bank.data.family[j]) && (category.value==='all'||category.value===outcome)) {found=j;break;}
+}
+if(found<0) {
+ metadata.text='No gallery candidates match these filters. Complete metrics remain above.';
+ for(const source of sources) {
+ source.data={image:[],flat:[],raw_flat:[],x:[],y:[],dw:[],dh:[]};
+ }
+ for(const source of label_sources) {
+ source.data={x:[],y:[],x_offset:[],y_offset:[],text:[]};
+ }
+ selection.data={x:[],y:[],width:[],height:[]};
+ exact.text='No selected candidate.';
+ state.data={candidate:[-1]};
+ return;
+}
+i=found;
+if(index.value!==found)index.value=found;
+metadata.text=`Candidate ${bank.data.identifier[i]}, gallery ${i+1}/${bank.data.score.length}; `+
+ `family ${bank.data.family[i]}; label ${bank.data.label[i]}; score ${bank.data.score[i]}; measured SNR ${bank.data.snr[i]}.`;
+const candidate_changed=state.data.candidate[0]!==i;
+const display_changed=cb_obj===transform;
+if(!candidate_changed && !display_changed)return;
+state.data={candidate:[i]};
+const keys=['search','template','difference'];
+for(let p=0;p<3;p++) {
+ const values=Array.from(bank.data[keys[p]][i]);
+ const sorted=values.slice().sort((a,b)=>a-b), median=sorted[Math.floor(sorted.length/2)];
+ const deviations=values.map(v=>Math.abs(v-median)).sort((a,b)=>a-b);
+ let sigma=1.4826*deviations[Math.floor(deviations.length/2)];
+ if(!(sigma>0)) {
+ const mean=values.reduce((a,b)=>a+b,0)/values.length;
+ sigma=Math.sqrt(values.reduce((a,b)=>a+(b-mean)**2,0)/values.length)||1;
+ }
+ const quantile=(q)=>{
+ const position=q*(sorted.length-1),i=Math.floor(position),f=position-i;
+ return sorted[i]+f*(sorted[Math.min(i+1,sorted.length-1)]-sorted[i]);
+ };
+ let low=quantile(.01), high=quantile(.99);
+ if(p===2) {high=Math.max(Math.abs(low),Math.abs(high));low=-high;}
+ const convert=(v)=>transform.value==='raw'?v:transform.value==='robust_z'?(v-median)/sigma:(v-(median-3*sigma))/(6*sigma);
+ const display=values.map(convert);
+ const matrix=Bokeh.require("core/util/ndarray").ndarray(
+ new Float64Array(display), {dtype:"float64", shape:[size,size]});
+ sources[p].data={image:[matrix],flat:[display],raw_flat:[values],robust_center:[median],robust_sigma:[sigma],display_center:[convert(median)],display_sigma:[transform.value==='raw'?sigma:transform.value==='robust_z'?1:1/6],transform:[transform.value],x:[0],y:[0],dw:[size],dh:[size]};
+ mappers[p].low=convert(low);mappers[p].high=convert(high);
+ sources[p].change.emit();
+ label_sources[p].data={x:[],y:[],x_offset:[],y_offset:[],text:[]};
+}
+if(candidate_changed) {
+ selection.data={x:[],y:[],width:[],height:[]};
+ exact.text='Tap any image for exact raw pixel values.';
+} else if(selection.data.x.length) {
+ const point={x:selection.data.x[0],y:selection.data.y[0]};
+ // Native taps toggle the same pixel off; clear before refreshing its labels.
+ selection.data={x:[],y:[],width:[],height:[]};
+ inspector.execute(point);
+}
diff --git a/examples/model-alerts/run.py b/examples/model-alerts/run.py
new file mode 100644
index 00000000..6f84c094
--- /dev/null
+++ b/examples/model-alerts/run.py
@@ -0,0 +1,252 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Snapshot and freshly execute the complete Model Alerts notebook."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import zipfile
+from pathlib import Path
+
+import nbformat
+from jupyter_client import AsyncKernelManager
+from nbclient import NotebookClient
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[1]
+
+
+def digest(path):
+ with Path(path).open("rb") as stream:
+ return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def source_files():
+ return sorted(
+ p
+ for p in HERE.rglob("*")
+ if p.is_file()
+ and "__pycache__" not in p.parts
+ and ".pytest_cache" not in p.parts
+ and p.suffix
+ in (".py", ".js", ".css", ".html", ".md", ".ipynb", ".txt")
+ )
+
+
+def source_status():
+ return subprocess.check_output(
+ [
+ "git",
+ "status",
+ "--porcelain",
+ "--untracked-files=all",
+ "--",
+ "src",
+ "examples/model-alerts",
+ ".gitattributes",
+ ],
+ cwd=REPO,
+ text=True,
+ ).strip()
+
+
+def source_revision():
+ return subprocess.check_output(
+ ["git", "rev-parse", "HEAD"], cwd=REPO, text=True
+ ).strip()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument(
+ "--profile",
+ choices=["quick-smoke", "full-reference"],
+ default="quick-smoke",
+ )
+ parser.add_argument("--gpu-lock", type=Path)
+ parser.add_argument(
+ "--device", choices=["cpu", "cuda:0"], default="cuda:0"
+ )
+ args = parser.parse_args()
+ output = args.output.resolve()
+ if output.exists() or output == REPO or output.is_relative_to(REPO):
+ parser.error(
+ "--output must be a new directory outside example sources"
+ )
+ if args.device.startswith("cuda") and args.gpu_lock is None:
+ parser.error("CUDA execution requires --gpu-lock")
+ if args.gpu_lock is not None and not args.gpu_lock.parent.is_dir():
+ parser.error("shared GPU lock parent must already exist")
+ revision = source_revision()
+ if source_status():
+ parser.error(
+ "commit example and cuPhoton source edits before execution"
+ )
+ files = source_files()
+ hashes = {str(p.relative_to(HERE)): digest(p) for p in files}
+ output.mkdir(parents=True)
+ snapshot = output / "source"
+ for p in files:
+ destination = snapshot / p.relative_to(HERE)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(p, destination)
+ with zipfile.ZipFile(
+ output / "model-alerts-source.zip", "w", zipfile.ZIP_DEFLATED
+ ) as archive:
+ for p in files:
+ archive.write(p, "model-alerts/" + str(p.relative_to(HERE)))
+ archive.write(REPO / "LICENSE", "model-alerts/LICENSE")
+ notebook = nbformat.read(snapshot / "model-alerts.ipynb", as_version=4)
+ manager = AsyncKernelManager(kernel_name="python3")
+ manager.kernel_spec.argv = [
+ sys.executable,
+ "-m",
+ "ipykernel_launcher",
+ "-f",
+ "{connection_file}",
+ ]
+ client = NotebookClient(
+ notebook,
+ timeout=None if args.profile == "full-reference" else 7200,
+ km=manager,
+ resources={"metadata": {"path": str(snapshot)}},
+ record_timing=True,
+ )
+ environment = os.environ.copy()
+ environment.update(
+ CUPHOTON_MODEL_ALERTS_OUTPUT=str(output / "study"),
+ CUPHOTON_MODEL_ALERTS_SOURCE_REPO=str(REPO),
+ CUPHOTON_MODEL_ALERTS_PROFILE=args.profile,
+ CUPHOTON_MODEL_ALERTS_GPU_LOCK=str(args.gpu_lock.resolve())
+ if args.gpu_lock is not None
+ else "",
+ CUPHOTON_MODEL_ALERTS_DEVICE=args.device,
+ PYTHONPATH=os.pathsep.join(
+ [
+ str(REPO / "src"),
+ str(snapshot),
+ *[
+ str(Path(p).resolve())
+ for p in environment.get("PYTHONPATH", "").split(
+ os.pathsep
+ )
+ if p
+ ],
+ ]
+ ),
+ CUPHOTON_MODEL_ALERTS_REVISION=revision,
+ )
+ started = time.monotonic()
+ try:
+ client.execute(env=environment, cwd=str(snapshot), cleanup_kc=True)
+ finally:
+ # Preserve warnings and numbers; replace only machine-local prefixes,
+ # following the Model Sky execution workflow.
+ for cell in notebook.cells:
+ if cell.cell_type != "code":
+ continue
+ for item in cell.outputs:
+ replacements = [
+ (str(output), ""),
+ (str(REPO), ""),
+ (sys.prefix, ""),
+ (str(Path(sys.prefix).resolve()), ""),
+ (str(Path.home()), ""),
+ ]
+
+ def portable(text, replacements=tuple(replacements)):
+ for prefix, label in sorted(
+ replacements, key=lambda p: -len(p[0])
+ ):
+ text = text.replace(prefix, label)
+ return text
+
+ if item.output_type == "stream":
+ item.text = portable(item.text)
+ elif item.output_type in ("display_data", "execute_result"):
+ for mime in ("text/plain", "text/html", "text/markdown"):
+ if isinstance(item.data.get(mime), str):
+ item.data[mime] = portable(item.data[mime])
+ nbformat.write(notebook, output / "model-alerts.executed.ipynb")
+ errors = [
+ o
+ for c in notebook.cells
+ if c.cell_type == "code"
+ for o in c.outputs
+ if o.output_type == "error"
+ ]
+ if errors:
+ raise RuntimeError("notebook execution contains errors")
+ if source_status() or source_revision() != revision:
+ raise RuntimeError(
+ "cuPhoton source changed during reference execution"
+ )
+ if any(
+ digest(HERE / name) != expected for name, expected in hashes.items()
+ ):
+ raise RuntimeError("source changed during reference execution")
+ report = json.loads((output / "study/report.json").read_text())
+ receipt = dict(
+ status="passed",
+ profile=args.profile,
+ elapsed_seconds=time.monotonic() - started,
+ source_sha256=hashes,
+ cuphoton_revision=revision,
+ dependencies={
+ name: importlib.metadata.version(name)
+ for name in (
+ "numpy",
+ "scipy",
+ "torch",
+ "bokeh",
+ "nbclient",
+ "nbformat",
+ )
+ },
+ output_sha256={
+ name: digest(output / name)
+ for name in (
+ "model-alerts.executed.ipynb",
+ "model-alerts-source.zip",
+ "study/report.json",
+ "study/freeze.json",
+ )
+ },
+ code_cells=sum(c.cell_type == "code" for c in notebook.cells),
+ study_profile=report["design"],
+ artifact_sha256={
+ str(p.relative_to(output)): digest(p)
+ for p in output.rglob("*")
+ if p.is_file()
+ and "__pycache__" not in p.parts
+ and p.suffix
+ in (
+ ".npy",
+ ".npz",
+ ".pt",
+ ".parquet",
+ ".jsonl",
+ ".json",
+ ".ipynb",
+ ".zip",
+ )
+ },
+ )
+ (output / "execution.json").write_text(
+ json.dumps(receipt, indent=2) + "\n"
+ )
+ print(json.dumps(receipt, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/model-alerts/run_study.py b/examples/model-alerts/run_study.py
new file mode 100644
index 00000000..15792382
--- /dev/null
+++ b/examples/model-alerts/run_study.py
@@ -0,0 +1,145 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Run the development pilot; final test seeds are never accessed here."""
+
+from __future__ import annotations
+
+import argparse
+import fcntl
+import hashlib
+import json
+from pathlib import Path
+
+import numpy as np
+from baselines import development_baselines
+from generator import Design, write_dataset
+from study import prepare_training, train_development
+
+
+def pilot(output, *, smoke=False, device="cuda:0", multiplier=1):
+ output = Path(output).resolve()
+ if output.exists():
+ raise ValueError("pilot output must not exist")
+ output.mkdir(parents=True)
+ train_count, val_count = (6, 3) if smoke else (180, 60)
+ train_count *= multiplier
+ val_count *= multiplier
+ fields = {i: "train" for i in range(train_count)}
+ fields.update({10000 + i: "val" for i in range(val_count)})
+ raw, prepared = output / "raw", output / "prepared"
+ design = Design()
+ write_dataset(raw, fields, design=design, backend="cpu")
+ raw_rows = [
+ json.loads(line)
+ for line in (raw / "metadata.jsonl").read_text().splitlines()
+ ]
+ difference = np.load(raw / "difference.npy", mmap_mode="r")
+ mask = np.array(
+ [
+ r["family"] == "misregistration"
+ and not r["label"]
+ and not r["ambiguous"]
+ for r in raw_rows
+ ]
+ )
+ center = design.stamp_size // 2
+ cut = np.s_[center - 4 : center + 5, center - 4 : center + 5]
+ selected = difference[mask][:, cut[0], cut[1]]
+ variance = np.load(raw / "variance.npy", mmap_mode="r")[mask][
+ :, cut[0], cut[1]
+ ]
+ standardized = selected / np.sqrt(variance)
+ bipolar = (standardized.max(axis=(1, 2)) >= 5) & (
+ standardized.min(axis=(1, 2)) <= -5
+ )
+ registration_diagnostic = dict(
+ candidates=int(mask.sum()),
+ bipolar_candidates=int(bipolar.sum()),
+ criterion=(
+ "central 9x9 has both signs at >=5 marginal pixel sigma; "
+ "descriptive, not calibrated detection significance"
+ ),
+ )
+ if not smoke and not bipolar.any():
+ raise RuntimeError(
+ "development contains no bipolar registration residuals"
+ )
+ prepare_training(raw, prepared)
+ baselines = development_baselines(raw, prepared, output / "baselines")
+ outcomes = []
+ for mode in ("pair", "triplet"):
+ for seed in (11,) if smoke else (11, 23, 47):
+ print(f"Training {mode}, seed {seed}", flush=True)
+ outcomes.append(
+ train_development(
+ prepared,
+ output / f"{mode}-{seed}",
+ mode=mode,
+ seed=seed,
+ epochs=1 if smoke else 12,
+ device=device,
+ )
+ )
+ half_widths = [
+ (r["validation_fpr_interval"][1] - r["validation_fpr_interval"][0])
+ / 2
+ for r in outcomes
+ ]
+ receipt = dict(
+ profile="quick-smoke" if smoke else "development",
+ scientific_performance_qualified=False,
+ train_fields=train_count,
+ validation_fields=val_count,
+ epochs_per_field=design.epochs,
+ outcomes=outcomes,
+ baselines=baselines,
+ precision_requires_expansion=max(half_widths) > 0.05,
+ maximum_validation_half_width=max(half_widths),
+ source_sha256={
+ p.name: hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in Path(__file__).parent.glob("*.py")
+ },
+ registration_diagnostic=registration_diagnostic,
+ final_test_generated=False,
+ )
+ (output / "pilot.json").write_text(json.dumps(receipt, indent=2) + "\n")
+ return receipt
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument(
+ "--profile",
+ choices=["quick-smoke", "development"],
+ default="quick-smoke",
+ )
+ parser.add_argument("--device", default="cuda:0")
+ parser.add_argument("--gpu-lock", type=Path)
+ args = parser.parse_args()
+ from cuphoton.xscan.training import resolve_device
+
+ device = str(resolve_device(args.device))
+ if device.startswith("cuda"):
+ if args.gpu_lock is None:
+ parser.error("CUDA runs require an explicit shared --gpu-lock")
+ with args.gpu_lock.open("a") as lock:
+ print("Waiting for shared GPU lock", flush=True)
+ fcntl.flock(lock, fcntl.LOCK_EX)
+ print("Acquired shared GPU lock", flush=True)
+ pilot(
+ args.output,
+ smoke=args.profile == "quick-smoke",
+ device=device,
+ )
+ else:
+ pilot(
+ args.output,
+ smoke=args.profile == "quick-smoke",
+ device=device,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/model-alerts/study.py b/examples/model-alerts/study.py
new file mode 100644
index 00000000..de6c538f
--- /dev/null
+++ b/examples/model-alerts/study.py
@@ -0,0 +1,209 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Development-only study preparation using the native xScan training API."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+from evaluation import confusion, paired_field_bootstrap, recall_threshold
+
+
+def prepare_training(
+ raw_dir, output, *, normalization=None, eligible_only=True
+):
+ """Keep binary unambiguous non-challenge rows; fit scaling on train only.
+
+ Original complete detection products remain in raw_dir for coverage and
+ challenge reporting. Row identities survive selection unchanged.
+ """
+ from cuphoton.xscan.dataset import validate_dataset_dir
+ from cuphoton.xscan.training import require_training_label_provenance
+
+ raw_dir, output = Path(raw_dir), Path(output)
+ if output.exists() and (not output.is_dir() or any(output.iterdir())):
+ raise ValueError("training output must be absent or empty")
+ rows = [
+ json.loads(line)
+ for line in (raw_dir / "metadata.jsonl").read_text().splitlines()
+ ]
+ eligible = np.array(
+ [
+ (not r["ambiguous"] and not r["challenge"]) or not eligible_only
+ for r in rows
+ ]
+ )
+ selected = np.flatnonzero(eligible)
+ split = np.load(raw_dir / "split.npy")[selected]
+ arrays = [
+ np.load(raw_dir / f"{name}.npy", mmap_mode="r")[selected]
+ for name in ("search", "template", "difference")
+ ]
+ if normalization is None:
+ if not np.any(split == 0):
+ raise ValueError("training rows required to fit scaling")
+ pair_train = np.concatenate(
+ [a[split == 0].ravel() for a in arrays[:2]]
+ )
+ pair_center = float(np.median(pair_train))
+ pair_scale = float(np.std(pair_train))
+ diff_scale = float(np.std(arrays[2][split == 0]))
+ else:
+ pair_center = float(normalization["pair_center"])
+ pair_scale = float(normalization["pair_scale"])
+ diff_scale = float(normalization["difference_scale"])
+ if min(pair_scale, diff_scale) <= 0:
+ raise ValueError("nondegenerate training arrays required")
+ output.mkdir(parents=True, exist_ok=True)
+ for index, (name, array) in enumerate(
+ zip(("search", "template", "difference"), arrays, strict=False)
+ ):
+ center, scale = (
+ (pair_center, pair_scale) if index < 2 else (0, diff_scale)
+ )
+ np.save(
+ output / f"{name}.npy",
+ ((array - center) / scale).astype(np.float32),
+ )
+ variance_path = raw_dir / "variance.npy"
+ if variance_path.is_file():
+ np.save(
+ output / "variance.npy",
+ np.load(variance_path, mmap_mode="r")[selected] / diff_scale**2,
+ )
+ np.save(output / "labels.npy", np.load(raw_dir / "labels.npy")[selected])
+ np.save(output / "split.npy", split)
+ (output / "metadata.jsonl").write_text(
+ "".join(json.dumps(rows[i]) + "\n" for i in selected)
+ )
+ receipt = dict(
+ pair_center=pair_center,
+ pair_scale=pair_scale,
+ difference_center=0,
+ difference_scale=diff_scale,
+ fitted_split="train",
+ raw_rows=len(rows),
+ selected_rows=len(selected),
+ ambiguous_rows=sum(r["ambiguous"] for r in rows),
+ overlapping_cause_rows=sum(r["overlapping_causes"] for r in rows),
+ challenge_rows=sum(r["challenge"] for r in rows),
+ selected_raw_indices=selected.tolist(),
+ )
+ (output / "normalization.json").write_text(
+ json.dumps(receipt, indent=2) + "\n"
+ )
+ validate_dataset_dir(output)
+ require_training_label_provenance(output)
+ return receipt
+
+
+def train_development(
+ dataset_dir,
+ run_dir,
+ *,
+ mode,
+ seed,
+ epochs=12,
+ device="cuda:0",
+ xfit_feature_dir=None,
+):
+ """Fit native xScan from scratch and freeze a validation threshold.
+
+ This function never accesses a test split or generates reserved fields.
+ """
+ from cuphoton.xscan.config import ModelConfig, TrainingConfig
+ from cuphoton.xscan.dataset import StampDataset
+ from cuphoton.xscan.training import (
+ load_model_from_checkpoint,
+ predict_dataset,
+ resolve_device,
+ train_classifier,
+ )
+
+ resolved_device = resolve_device(device)
+ run_dir, dataset_dir = Path(run_dir), Path(dataset_dir)
+ if run_dir.exists():
+ raise ValueError("run directory must not exist")
+ run_dir.mkdir(parents=True)
+ size = np.load(dataset_dir / "search.npy", mmap_mode="r").shape[-1]
+ config = TrainingConfig(
+ dataset_dir=str(dataset_dir),
+ epochs=epochs,
+ batch_size=64,
+ learning_rate=3e-4,
+ seed=seed,
+ device=str(resolved_device),
+ early_stopping_metric="val_loss",
+ early_stopping_patience=epochs,
+ use_xfit_features=xfit_feature_dir is not None,
+ xfit_feature_dir=str(xfit_feature_dir) if xfit_feature_dir else None,
+ model=ModelConfig(
+ input_mode=mode,
+ image_size=size,
+ depths=[1, 1, 1],
+ num_heads=[1, 2, 4],
+ embed_dims=[16, 32, 64],
+ decoder_embedding_dim=16,
+ pos_dim=8,
+ drop_rate=0.1,
+ attn_drop=0.1,
+ ),
+ )
+ summary = train_classifier(config, run_dir=run_dir)
+ if summary["epochs_completed"] != epochs:
+ raise AssertionError(
+ "native training did not complete the declared epoch budget"
+ )
+ model, _, performance = load_model_from_checkpoint(
+ run_dir, device=resolved_device
+ )
+ validation = StampDataset(
+ dataset_dir,
+ input_mode=mode,
+ split="val",
+ xfit_feature_dir=xfit_feature_dir,
+ )
+ prediction = predict_dataset(
+ model=model,
+ dataset=validation,
+ batch_size=64,
+ device=resolved_device,
+ performance=performance,
+ )
+ labels, scores = prediction["labels"], prediction["probabilities"]
+ threshold = recall_threshold(labels, scores)
+ fields = np.array([r["field"] for r in prediction["metadata_rows"]])
+ interval = paired_field_bootstrap(
+ labels, scores, scores, fields, threshold, threshold
+ )
+ from cuphoton.xscan.metrics import auc, roc_curve
+
+ fpr, tpr, _ = roc_curve(labels, scores)
+ receipt = dict(
+ validation_roc_auc=auc(fpr, tpr),
+ mode=mode,
+ seed=seed,
+ threshold=threshold,
+ validation=confusion(labels, scores, threshold),
+ validation_fpr_interval=interval["a"],
+ validation_fields=int(len(np.unique(fields))),
+ epochs_completed=summary["epochs_completed"],
+ early_stopping=summary["early_stopping"],
+ selection_split="val",
+ threshold_split="val",
+ validation_reused=True,
+ uncertainty_scope="conditional development precision heuristic",
+ )
+ np.savez(
+ run_dir / "validation.npz",
+ labels=labels,
+ scores=scores,
+ fields=fields,
+ )
+ (run_dir / "operating-point.json").write_text(
+ json.dumps(receipt, indent=2) + "\n"
+ )
+ return receipt
diff --git a/examples/model-alerts/tests/test_baselines.py b/examples/model-alerts/tests/test_baselines.py
new file mode 100644
index 00000000..6a3c025c
--- /dev/null
+++ b/examples/model-alerts/tests/test_baselines.py
@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from baselines import LogisticBaseline, measured_features
+
+
+def test_baseline_scaling_is_frozen_and_constant_features_are_safe():
+ features = np.column_stack((np.linspace(-2, 2, 100), np.zeros(100)))
+ labels = np.r_[np.zeros(50), np.ones(50)]
+ model = LogisticBaseline().fit(features, labels)
+ saved = model.receipt()
+ scores = model.predict(features)
+ assert np.mean((scores >= 0.5) == labels) == 1
+ model.predict(features + 1000)
+ assert model.receipt() == saved
+ assert np.isfinite(scores).all()
+
+
+def test_measured_statistics_have_no_mask_or_clipping_signature():
+ images = np.random.default_rng(3).normal(size=(5, 3, 31, 31))
+ features = measured_features(images, np.arange(5) + 4)
+ assert features.shape == (5, 7)
+ assert np.isfinite(features).all()
+ np.testing.assert_array_equal(features[:, -2:], 0)
+
+
+def test_signature_statistics_detect_clipped_and_missing_pixels():
+ images = np.random.default_rng(3).normal(size=(2, 3, 31, 31))
+ images[0] = np.clip(images[0], -1, 1)
+ images[1, 0, 0, 0] = np.nan
+ features = measured_features(images, [4, 5])
+ assert features[0, -1] > 0.2
+ assert features[1, -2] > 0
+ assert features[0, -2] == 0
+ assert np.isfinite(features).all()
diff --git a/examples/model-alerts/tests/test_evaluation.py b/examples/model-alerts/tests/test_evaluation.py
new file mode 100644
index 00000000..5080213d
--- /dev/null
+++ b/examples/model-alerts/tests/test_evaluation.py
@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from evaluation import (
+ confusion,
+ paired_field_bootstrap,
+ queue_projection,
+ recall_threshold,
+)
+
+
+def test_threshold_is_highest_feasible_and_includes_ties():
+ labels = np.r_[np.ones(20), np.zeros(4)]
+ scores = np.r_[np.linspace(0.2, 0.9, 18), 0.1, 0.1, 0.1, 0.3, 0.5, 0.8]
+ threshold = recall_threshold(labels, scores)
+ assert threshold == 0.1
+ assert confusion(labels, scores, threshold)["recall"] == 1
+ assert (
+ confusion(labels, scores, np.nextafter(threshold, 1))["recall"] < 0.95
+ )
+
+
+def test_paired_bootstrap_retains_field_dependence():
+ labels = np.zeros(8)
+ fields = np.repeat(["a", "b"], 4)
+ scores = np.repeat([1.0, 0.0], 4)
+ result = paired_field_bootstrap(labels, scores, scores, fields, 0.5, 0.5)
+ assert result["a"] == [0, 1]
+ assert result["a_minus_b"] == [0, 0]
+
+
+def test_empty_queue_has_undefined_purity():
+ assert queue_projection(0, 0, 0.01, 100)["purity"] is None
+ assert queue_projection(0.9, 0.1, 0.01, 100)["queue"] == pytest.approx(
+ 10.8
+ )
+
+
+@pytest.mark.parametrize(
+ "labels,scores",
+ [([1], [float("nan")]), ([2], [0.5]), ([], []), ([1, 0], [0.5])],
+)
+def test_reject_invalid_evaluation(labels, scores):
+ with pytest.raises(ValueError):
+ recall_threshold(labels, scores)
+
+
+def test_field_rate_intervals_keep_perfect_decisions_and_count_undefined():
+ from evaluation import field_rate_intervals
+
+ result = field_rate_intervals(
+ [1, 1, 0, 0], [0.8, 0.9, 0.1, 0.2], ["a", "a", "b", "b"], 0.5
+ )
+ assert result["recall"]["interval"] == [1, 1]
+ assert result["fpr"]["interval"] == [0, 0]
+ assert result["recall"]["undefined_draws"] > 0
+ assert result["fpr"]["undefined_draws"] > 0
diff --git a/examples/model-alerts/tests/test_execution.py b/examples/model-alerts/tests/test_execution.py
new file mode 100644
index 00000000..86f883f9
--- /dev/null
+++ b/examples/model-alerts/tests/test_execution.py
@@ -0,0 +1,249 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Check execution guards and receipts without a reference study."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import nbformat
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import reference
+import run
+import run_study
+import study
+
+
+def git(repo, *args):
+ return subprocess.check_output(
+ ["git", *args], cwd=repo, text=True, stderr=subprocess.PIPE
+ ).strip()
+
+
+@pytest.fixture
+def source_repo(tmp_path):
+ repo = tmp_path / "repo"
+ example = repo / "examples/model-alerts"
+ example.mkdir(parents=True)
+ (repo / "src").mkdir()
+ (repo / "src/package.py").write_text("VALUE = 1\n")
+ (repo / "LICENSE").write_text("test license\n")
+ (example / "reference.py").write_text("# example\n")
+ nbformat.write(nbformat.v4.new_notebook(), example / "model-alerts.ipynb")
+ git(repo, "init", "--initial-branch=main")
+ git(repo, "config", "user.name", "Test")
+ git(repo, "config", "user.email", "test@example.org")
+ git(repo, "config", "commit.gpgsign", "false")
+ git(repo, "config", "core.hooksPath", "/dev/null")
+ git(repo, "add", ".")
+ git(repo, "commit", "-m", "Initial source")
+ return repo
+
+
+@pytest.mark.parametrize("override", [None, "a" * 40, "invalid"])
+def test_reference_resolves_revision_before_experiment(
+ source_repo, tmp_path, monkeypatch, override
+):
+ monkeypatch.setattr(
+ reference,
+ "__file__",
+ str(source_repo / "examples/model-alerts/reference.py"),
+ )
+ monkeypatch.delenv("CUPHOTON_MODEL_ALERTS_SOURCE_REPO", raising=False)
+ monkeypatch.delenv("CUPHOTON_MODEL_ALERTS_REVISION", raising=False)
+ if override is not None:
+ monkeypatch.setenv("CUPHOTON_MODEL_ALERTS_REVISION", override)
+ monkeypatch.chdir(tmp_path)
+ called = []
+ monkeypatch.setattr(
+ reference, "_experiment", lambda root, **kwargs: called.append(kwargs)
+ )
+ output = tmp_path / "reference"
+ if override == "invalid":
+ with pytest.raises(ValueError, match="complete Git commit SHA"):
+ reference.execute_reference(output, device="cpu")
+ assert not output.exists()
+ assert not called
+ else:
+ reference.execute_reference(output, device="cpu")
+ assert called[0]["revision"] == (
+ override or git(source_repo, "rev-parse", "HEAD")
+ )
+
+
+@pytest.mark.parametrize("change", [None, "package", "revision"])
+def test_runner_rejects_source_changes_before_passed_receipt(
+ source_repo, tmp_path, monkeypatch, change
+):
+ initial_revision = git(source_repo, "rev-parse", "HEAD")
+ output = tmp_path / "result"
+ monkeypatch.setattr(run, "REPO", source_repo)
+ monkeypatch.setattr(run, "HERE", source_repo / "examples/model-alerts")
+ monkeypatch.setattr(
+ sys, "argv", ["run.py", "--output", str(output), "--device", "cpu"]
+ )
+ monkeypatch.setattr(
+ run,
+ "AsyncKernelManager",
+ lambda **kwargs: SimpleNamespace(
+ has_kernel=False, kernel_spec=SimpleNamespace(argv=[])
+ ),
+ )
+
+ def execute(**kwargs):
+ assert (
+ kwargs["env"]["CUPHOTON_MODEL_ALERTS_REVISION"]
+ == initial_revision
+ )
+ if change == "package":
+ (source_repo / "src/package.py").write_text("VALUE = 2\n")
+ elif change == "revision":
+ git(source_repo, "commit", "--allow-empty", "-m", "New revision")
+ (output / "study").mkdir()
+ (output / "study/report.json").write_text('{"design": {}}')
+ (output / "study/freeze.json").write_text("{}")
+
+ monkeypatch.setattr(
+ run,
+ "NotebookClient",
+ lambda *args, **kwargs: SimpleNamespace(execute=execute),
+ )
+ monkeypatch.setattr(
+ run.importlib.metadata, "version", lambda name: "test"
+ )
+ if change:
+ with pytest.raises(RuntimeError, match="source changed"):
+ run.main()
+ assert not (output / "execution.json").exists()
+ else:
+ run.main()
+ receipt = json.loads((output / "execution.json").read_text())
+ assert receipt["status"] == "passed"
+ assert receipt["cuphoton_revision"] == initial_revision
+
+
+def test_runner_times_out_stalled_cell_and_releases_kernel(
+ source_repo, tmp_path, monkeypatch
+):
+ from nbclient.exceptions import CellTimeoutError
+
+ example = source_repo / "examples/model-alerts"
+ notebook = nbformat.v4.new_notebook(
+ cells=[nbformat.v4.new_code_cell("import time; time.sleep(30)")]
+ )
+ nbformat.write(notebook, example / "model-alerts.ipynb")
+ git(source_repo, "add", ".")
+ git(source_repo, "commit", "-m", "Stalled notebook")
+ output = tmp_path / "timeout"
+ monkeypatch.setattr(run, "REPO", source_repo)
+ monkeypatch.setattr(run, "HERE", example)
+ monkeypatch.setattr(
+ sys, "argv", ["run.py", "--output", str(output), "--device", "cpu"]
+ )
+ managers = []
+ notebook_client = run.NotebookClient
+
+ def timed_client(*args, **kwargs):
+ kwargs["timeout"] = 1
+ client = notebook_client(*args, **kwargs)
+ managers.append(kwargs["km"])
+ return client
+
+ monkeypatch.setattr(run, "NotebookClient", timed_client)
+ with pytest.raises(CellTimeoutError):
+ run.main()
+ assert not managers[0].has_kernel
+ assert (output / "model-alerts.executed.ipynb").is_file()
+ assert not (output / "execution.json").exists()
+
+
+@pytest.mark.parametrize(
+ "cuda,with_lock", [(True, True), (True, False), (False, False)]
+)
+def test_auto_device_obeys_gpu_lock(tmp_path, monkeypatch, cuda, with_lock):
+ import torch
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: cuda)
+ args = [
+ "run_study.py",
+ "--output",
+ str(tmp_path / "pilot"),
+ "--device",
+ "auto",
+ ]
+ if with_lock:
+ args.extend(["--gpu-lock", str(tmp_path / "gpu.lock")])
+ monkeypatch.setattr(sys, "argv", args)
+ events = []
+ monkeypatch.setattr(
+ run_study.fcntl, "flock", lambda *args: events.append("lock")
+ )
+ monkeypatch.setattr(
+ run_study,
+ "pilot",
+ lambda *args, **kwargs: events.append(kwargs["device"]),
+ )
+ if cuda and not with_lock:
+ with pytest.raises(SystemExit):
+ run_study.main()
+ assert events == []
+ else:
+ run_study.main()
+ assert events == (["lock", "cuda"] if cuda else ["cpu"])
+
+
+def test_auto_device_is_resolved_for_training_and_validation(
+ tmp_path, monkeypatch
+):
+ import torch
+
+ from cuphoton.xscan import dataset, training
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ data = tmp_path / "data"
+ data.mkdir()
+ np.save(data / "search.npy", np.zeros((2, 31, 31)))
+ devices = []
+
+ def train(config, **kwargs):
+ devices.append(config.device)
+ return dict(epochs_completed=1, early_stopping={})
+
+ def load(*args, device):
+ devices.append(str(device))
+ return object(), None, None
+
+ def predict(**kwargs):
+ devices.append(str(kwargs["device"]))
+ return dict(
+ labels=np.array([0, 1]),
+ probabilities=np.array([0.1, 0.9]),
+ metadata_rows=[dict(field=1), dict(field=2)],
+ )
+
+ monkeypatch.setattr(training, "train_classifier", train)
+ monkeypatch.setattr(training, "load_model_from_checkpoint", load)
+ monkeypatch.setattr(training, "predict_dataset", predict)
+ monkeypatch.setattr(
+ dataset, "StampDataset", lambda *args, **kwargs: object()
+ )
+ monkeypatch.setattr(
+ study, "paired_field_bootstrap", lambda *args: {"a": [0, 1]}
+ )
+ result = study.train_development(
+ data,
+ tmp_path / "train",
+ mode="pair",
+ seed=11,
+ epochs=1,
+ device="auto",
+ )
+ assert devices == ["cpu", "cpu", "cpu"]
+ assert result["epochs_completed"] == 1
+ assert (tmp_path / "train/validation.npz").is_file()
diff --git a/examples/model-alerts/tests/test_gallery.py b/examples/model-alerts/tests/test_gallery.py
new file mode 100644
index 00000000..24cbb6d3
--- /dev/null
+++ b/examples/model-alerts/tests/test_gallery.py
@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from views import gallery_indices
+
+
+def test_gallery_is_deterministic_bounded_and_keeps_rare_errors():
+ rows = [
+ dict(candidate_id=i, snr=8, family="appearance") for i in range(200)
+ ]
+ labels = np.ones(200, dtype=int)
+ scores = np.ones(200)
+ scores[150] = 0.01
+ first = gallery_indices(rows, labels, scores, 0.5)
+ assert len(first) == 96 and len(set(first)) == 96
+ assert 150 in first
+ np.testing.assert_array_equal(
+ first, gallery_indices(rows, labels, scores, 0.5)
+ )
+
+
+def test_saved_gallery_preserves_receipt_order():
+ import pytest
+
+ pytest.importorskip("bokeh")
+ from views import candidate_explorer
+
+ rows = [
+ dict(candidate_id=i, snr=8.0, family="appearance")
+ for i in (90, 3, 17, 8)
+ ]
+ images = np.random.default_rng(318).normal(size=(4, 3, 31, 31))
+ labels, scores = np.array([1, 0, 1, 0]), np.array([0.9, 0.6, 0.3, 0.1])
+ with pytest.raises(ValueError, match="complete evaluation"):
+ candidate_explorer(
+ images, rows, labels, scores, 0.5, preselected=True
+ )
+ _, receipt = candidate_explorer(
+ images,
+ rows,
+ labels,
+ scores,
+ 0.5,
+ preselected=True,
+ evaluation=dict(
+ labels=labels,
+ scores=scores,
+ snr=[r["snr"] for r in rows],
+ candidate_id=[r["candidate_id"] for r in rows],
+ family=[r["family"] for r in rows],
+ ),
+ )
+ assert receipt["indices"] == [0, 1, 2, 3]
diff --git a/examples/model-alerts/tests/test_generator.py b/examples/model-alerts/tests/test_generator.py
new file mode 100644
index 00000000..3fa2a5a4
--- /dev/null
+++ b/examples/model-alerts/tests/test_generator.py
@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from generator import associate, expose, field_exposures, psf
+
+
+def test_pixel_integrated_flux_and_centroid():
+ y, x = np.indices((71, 71))
+ image = psf(x.shape, 34.2, 35.3, 1.5)
+ assert image.sum() == pytest.approx(1, abs=1e-12)
+ assert (image * x).sum() == pytest.approx(34.2, abs=1e-6)
+ assert (image * y).sum() == pytest.approx(35.3, abs=1e-6)
+ assert np.all(image >= 0)
+
+
+def test_poisson_plus_read_variance():
+ expected = np.full((800, 800), 100.0)
+ values = expose(expected, np.random.default_rng(982), 4)
+ assert abs(values.mean() - 100) < 0.06
+ assert abs(values.var() - 116) < 0.6
+
+
+def test_field_owns_template_and_seeds():
+ first = list(field_exposures(41))
+ again = list(field_exposures(41))
+ other = list(field_exposures(42))
+ for a, b in zip(first, again, strict=False):
+ np.testing.assert_array_equal(a["search"], b["search"])
+ np.testing.assert_array_equal(a["template"], first[0]["template"])
+ assert not np.array_equal(first[0]["search"], first[1]["search"])
+ assert not np.array_equal(first[0]["template"], other[0]["template"])
+ assert all(c["family"] != "column_bleed" for c in first[0]["causes"])
+ assert any(
+ c["family"] == "column_bleed"
+ for c in next(field_exposures(42, shifted=True))["causes"]
+ )
+
+
+def test_unmatched_noise_and_mover_ontology():
+ assert associate(dict(x=1, y=2), [], 3)["label"] == 0
+ mover = dict(
+ x=1, y=2, positive=True, family="moving_source", challenge=True
+ )
+ result = associate(dict(x=1, y=2), [mover], 3)
+ assert result["label"] == 1 and result["challenge"]
+ artifact = dict(
+ x=2, y=2, positive=False, family="cosmic_ray", challenge=False
+ )
+ assert associate(dict(x=1, y=2), [mover, artifact], 3)["ambiguous"]
+
+
+def test_detection_uses_images_without_truth():
+ from generator import detect
+
+ exposure = next(field_exposures(10))
+ stamps, rows, _, _ = detect(exposure)
+ exposure["causes"] = []
+ repeated_stamps, repeated_rows, _, _ = detect(exposure)
+ assert rows == repeated_rows and len(rows) > 0
+ for stamp, repeated, row in zip(
+ stamps, repeated_stamps, rows, strict=True
+ ):
+ np.testing.assert_array_equal(stamp, repeated)
+ assert stamp.shape == (3, 31, 31)
+ assert row["snr"] >= 4
+ assert stamp[0, 15, 15] == exposure["search"][row["y"], row["x"]]
+
+
+def test_independent_calibration_is_estimated_not_generator_truth():
+ from generator import calibration_sample
+
+ calibration = calibration_sample(100.0, 4.0, np.random.default_rng(785))
+ assert 99 < calibration["background"] < 101
+ assert 14 < calibration["read_variance"] < 18
+ assert calibration["background"] != 100
+ assert calibration["read_variance"] != 16
+ assert calibration["pixels"] == 4096
+
+
+def test_persistent_detector_defect_is_present_in_reference_and_search():
+ exposures = list(field_exposures(41))
+ for exposure in exposures:
+ cause = next(
+ c for c in exposure["causes"] if c["family"] == "bad_pixel"
+ )
+ x, y = int(cause["x"]), int(cause["y"])
+ for name in ("template", "search"):
+ image = exposure[name]
+ assert (
+ image[y, x] - np.median(image[y - 2 : y + 3, x - 2 : x + 3])
+ > 200
+ )
+
+
+def test_nearest_cause_defines_negative_family_independent_of_input_order():
+ far = dict(
+ x=2, y=0, positive=False, family="misregistration", challenge=False
+ )
+ near = dict(
+ x=0.1, y=0, positive=False, family="cosmic_ray", challenge=False
+ )
+ for causes in ([far, near], [near, far]):
+ result = associate(dict(x=0, y=0), causes, 3)
+ assert result["family"] == "cosmic_ray"
+ assert result["overlapping_causes"] and not result["ambiguous"]
+
+
+def test_local_registration_varies_by_star_and_hosts_avoid_mismatch():
+ exposures = list(field_exposures(82))
+ first = np.array([[c["x"], c["y"]] for c in exposures[0]["causes"][:22]])
+ second = np.array([[c["x"], c["y"]] for c in exposures[1]["causes"][:22]])
+ assert np.all((second - first).std(axis=0) > 0.1)
+ for exposure in exposures:
+ static = exposure["causes"][:22]
+ for change in exposure["causes"]:
+ if change["family"] == "brightening":
+ host = min(
+ static,
+ key=lambda c: np.hypot(
+ c["x"] - change["x"], c["y"] - change["y"]
+ ),
+ )
+ assert host["x"] == change["x"] and host["y"] == change["y"]
+ assert host["family"] != "psf_mismatch"
+
+
+def test_extended_column_association_includes_endpoints():
+ column = dict(
+ x=10,
+ y=10,
+ segment_half_length=5,
+ positive=False,
+ family="column_bleed",
+ challenge=False,
+ )
+ assert (
+ associate(dict(x=10, y=15), [column], 3)["family"] == "column_bleed"
+ )
+ assert associate(dict(x=10, y=19), [column], 3)["family"] == "noise"
diff --git a/examples/model-alerts/tests/test_native_dataset.py b/examples/model-alerts/tests/test_native_dataset.py
new file mode 100644
index 00000000..b77c9ecc
--- /dev/null
+++ b/examples/model-alerts/tests/test_native_dataset.py
@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+pytest.importorskip("torch")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from generator import Design, write_dataset
+from study import prepare_training
+
+
+def test_native_provenance_and_training_only_normalization(tmp_path):
+ from cuphoton.xscan.training import require_training_label_provenance
+
+ raw, first, second = (tmp_path / n for n in ("raw", "first", "second"))
+ write_dataset(raw, {10: "train", 11: "val"}, design=Design(epochs=1))
+ receipt = prepare_training(raw, first)
+ assert require_training_label_provenance(first)["ok"]
+ for name in ("search", "template", "difference"):
+ values = np.load(raw / f"{name}.npy")
+ values[np.load(raw / "split.npy") == 1] += 100000
+ np.save(raw / f"{name}.npy", values)
+ perturbed = prepare_training(raw, second)
+ for key in ("pair_center", "pair_scale", "difference_scale"):
+ assert perturbed[key] == receipt[key]
+ # Deliberately invalid split: retain a field's identity while moving only
+ # one candidate across splits. The native training guard must reject it.
+ rows = [
+ json.loads(line)
+ for line in (first / "metadata.jsonl").read_text().splitlines()
+ ]
+ split = np.load(first / "split.npy")
+ rows[0]["split"] = "val"
+ split[0] = 1
+ np.save(first / "split.npy", split)
+ (first / "metadata.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in rows)
+ )
+ with pytest.raises(ValueError, match="group leakage"):
+ require_training_label_provenance(first)
diff --git a/examples/model-alerts/tests/test_static_browser.py b/examples/model-alerts/tests/test_static_browser.py
new file mode 100644
index 00000000..1f72265a
--- /dev/null
+++ b/examples/model-alerts/tests/test_static_browser.py
@@ -0,0 +1,152 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Real Chromium regression for offline Bokeh image updates."""
+
+import html
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from views import candidate_explorer
+
+
+def test_static_callback_keeps_raw_pixels_and_handles_empty_filter(tmp_path):
+ chromium = os.environ.get("CUPHOTON_MODEL_ALERTS_CHROMIUM")
+ if not chromium:
+ pytest.skip("set CUPHOTON_MODEL_ALERTS_CHROMIUM to test real browser")
+ from bokeh.embed import file_html
+ from bokeh.resources import INLINE
+
+ rows = [
+ dict(candidate_id=i, snr=8.0, family="a" if i % 2 == 0 else "b")
+ for i in range(4)
+ ]
+ layout, _ = candidate_explorer(
+ np.random.default_rng(13).normal(size=(4, 3, 31, 31)),
+ rows,
+ np.array([0, 1, 0, 1]),
+ np.array([0.1, 0.9, 0.6, 0.3]),
+ 0.5,
+ evaluation=dict(
+ labels=np.array([0, 1, 0, 1, 0, 1, 0, 1]),
+ scores=np.array([0.1, 0.9, 0.6, 0.3, 0.1, 0.9, 0.6, 0.3]),
+ snr=np.full(8, 8.0),
+ candidate_id=np.arange(8).astype(str),
+ family=["appearance"] * 8,
+ ),
+ )
+ page = file_html(layout, INLINE, "Model Alerts browser regression")
+ capture = """"""
+ exercise = """"""
+ page = page.replace("", "" + capture, 1)
+ before, after = page.rsplit("", 1)
+ path = tmp_path / "offline.html"
+ path.write_text(before + exercise + "" + after)
+ result = subprocess.run(
+ [
+ chromium,
+ "--headless",
+ "--no-sandbox",
+ "--disable-gpu",
+ "--dump-dom",
+ "--virtual-time-budget=5000",
+ "--host-resolver-rules=MAP * ~NOTFOUND",
+ path.as_uri(),
+ ],
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=30,
+ )
+ matches = re.findall(
+ r'(.*?)
', result.stdout, re.S
+ )
+ assert matches, result.stderr
+ report = json.loads(html.unescape(matches[-1]))
+ assert report == dict(
+ unchanged=True,
+ transformed=True,
+ scales_unchanged=True,
+ selection_persists=True,
+ population=True,
+ previous_filtered=True,
+ empty=True,
+ empty_tap=True,
+ errors=[],
+ network=0,
+ )
diff --git a/examples/model-alerts/views.py b/examples/model-alerts/views.py
new file mode 100644
index 00000000..12fa5f53
--- /dev/null
+++ b/examples/model-alerts/views.py
@@ -0,0 +1,532 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Browser-only candidate review using existing xScan image helpers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+
+
+def gallery_indices(rows, labels, scores, threshold, maximum=96):
+ """Round-robin confusion/SNR/family strata, stable candidate-ID ordering.
+
+ Classification metrics never use this deliberately stratified gallery.
+ Empty strata stay empty. The selection is frozen at the validation
+ operating point, even when the exploratory threshold slider changes.
+ """
+ from collections import defaultdict
+
+ strata = defaultdict(list)
+ for i, (row, label, score) in enumerate(
+ zip(rows, labels, scores, strict=True)
+ ):
+ category = (
+ "T" if bool(label) == bool(score >= threshold) else "F"
+ ) + ("P" if score >= threshold else "N")
+ snr_bin = int(np.digitize(row["snr"], [5, 10, 20]))
+ strata[(category, snr_bin, row["family"])].append(i)
+ for values in strata.values():
+ values.sort(key=lambda i: str(rows[i]["candidate_id"]))
+ chosen = []
+ depth = 0
+ while len(chosen) < min(maximum, len(rows)):
+ for key in sorted(strata):
+ if depth < len(strata[key]):
+ chosen.append(strata[key][depth])
+ if len(chosen) == maximum:
+ break
+ depth += 1
+ return np.array(chosen, dtype=int)
+
+
+def candidate_explorer(
+ images,
+ rows,
+ labels,
+ scores,
+ threshold,
+ *,
+ evaluation=None,
+ preselected=False,
+):
+ """All statistics from complete evaluation; at most 96 image triplets."""
+ from bokeh.events import Tap
+ from bokeh.layouts import column, row
+ from bokeh.models import (
+ Button,
+ ColumnDataSource,
+ CustomJS,
+ Div,
+ LinearColorMapper,
+ Range1d,
+ Select,
+ Slider,
+ Span,
+ )
+ from bokeh.palettes import Greys256, RdBu11
+
+ from cuphoton.xscan import alard_lupton_experiment_review as lab
+ from cuphoton.xscan import raw_compare_review as raw
+
+ labels, scores = np.asarray(labels), np.asarray(scores)
+ if images.shape[:2] != (len(rows), 3) or len(rows) == 0:
+ raise ValueError("nonempty candidate triplets required")
+ if images.shape[2] != images.shape[3]:
+ raise ValueError("square stamps required")
+ if preselected and len(rows) > 96:
+ raise ValueError("saved gallery exceeds the 96-candidate limit")
+ if preselected and evaluation is None:
+ raise ValueError(
+ "preselected galleries require complete evaluation arrays"
+ )
+ chosen = (
+ np.arange(len(rows))
+ if preselected
+ else gallery_indices(rows, labels, scores, threshold)
+ )
+ stamps = images[chosen]
+ size = images.shape[-1]
+ bank = ColumnDataSource(
+ dict(
+ search=[s[0].ravel() for s in stamps],
+ template=[s[1].ravel() for s in stamps],
+ difference=[s[2].ravel() for s in stamps],
+ identifier=[str(rows[i]["candidate_id"]) for i in chosen],
+ family=[rows[i]["family"] for i in chosen],
+ snr=[rows[i]["snr"] for i in chosen],
+ label=labels[chosen],
+ score=scores[chosen],
+ )
+ )
+ if evaluation is None:
+ evaluation = dict(
+ labels=labels,
+ scores=scores,
+ snr=[r["snr"] for r in rows],
+ candidate_id=[str(r["candidate_id"]) for r in rows],
+ family=[r["family"] for r in rows],
+ )
+ population_labels = np.asarray(evaluation["labels"])
+ population_scores = np.asarray(evaluation["scores"])
+ from evaluation import checked
+
+ checked(population_labels, population_scores)
+ if any(
+ len(evaluation[key]) != len(population_labels)
+ for key in ("snr", "candidate_id", "family")
+ ):
+ raise ValueError("complete evaluation columns must be row-aligned")
+ metrics = ColumnDataSource(
+ dict(
+ label=population_labels,
+ score=population_scores,
+ snr=evaluation["snr"],
+ candidate=evaluation["candidate_id"],
+ family=evaluation["family"],
+ color=[
+ "#007f99" if label else "#ab3d75"
+ for label in population_labels
+ ],
+ )
+ )
+ from bokeh.plotting import figure
+
+ score_plot = figure(
+ title="Every evaluation candidate: score versus SNR",
+ height=280,
+ sizing_mode="stretch_width",
+ x_axis_type="log",
+ x_axis_label="Measured detection SNR (marginal diagnostic)",
+ y_axis_label="Saved classifier score",
+ tools="pan,wheel_zoom,reset,hover",
+ tooltips=[
+ ("Candidate", "@candidate"),
+ ("Family", "@family"),
+ ("Label", "@label"),
+ ("SNR", "@snr"),
+ ("Score", "@score"),
+ ],
+ )
+ score_plot.scatter(
+ x="snr", y="score", source=metrics, color="color", size=5, alpha=0.55
+ )
+ threshold_line = Span(
+ location=threshold,
+ dimension="width",
+ line_color="#777777",
+ line_dash="dashed",
+ )
+ score_plot.add_layout(threshold_line)
+ index = Slider(
+ start=0,
+ end=max(1, len(chosen) - 1),
+ value=0,
+ step=1,
+ title="Gallery position",
+ )
+ threshold_control = Slider(
+ start=0,
+ end=1,
+ value=float(threshold),
+ step=0.001,
+ title="Exploratory score threshold",
+ )
+ prevalence = Slider(
+ start=0.001,
+ end=0.5,
+ value=0.01,
+ step=0.001,
+ title="Assumed positive prevalence",
+ )
+ family = Select(
+ title="Gallery family",
+ value="all",
+ options=["all"] + sorted(set(bank.data["family"])),
+ )
+ category = Select(
+ title="Gallery outcome at current threshold",
+ value="all",
+ options=["all", "TP", "FP", "FN", "TN"],
+ )
+ transform = Select(
+ title="Display units",
+ value="raw",
+ options=["raw", "robust_z", "unit_robust"],
+ )
+ previous, following = (
+ Button(label="Previous candidate"),
+ Button(label="Next candidate"),
+ )
+ state = ColumnDataSource(dict(candidate=[0]))
+ restore = Button(label="Restore frozen threshold")
+ restore.js_on_click(
+ CustomJS(
+ args=dict(threshold=threshold_control, frozen=float(threshold)),
+ code="threshold.value=frozen;",
+ )
+ )
+ metadata, queue, exact = (
+ Div(),
+ Div(),
+ Div(text="Tap any image for exact raw pixel values."),
+ )
+ selection = ColumnDataSource(dict(x=[], y=[], width=[], height=[]))
+ sources, mappers, label_sources, plots = [], [], [], []
+ xr, yr = Range1d(0, size), Range1d(0, size)
+ panel_names = ["Search", "Reference", "xPois difference"]
+ for p, title in enumerate(panel_names):
+ values = stamps[0, p]
+ source = ColumnDataSource(
+ lab._image_payload(values, values, transform_label="raw")
+ )
+ low, high = np.percentile(values, [1, 99])
+ if p == 2:
+ high = max(abs(low), abs(high))
+ low = -high
+ mapper = LinearColorMapper(
+ palette=Greys256 if p < 2 else list(reversed(RdBu11)),
+ low=float(low),
+ high=float(high),
+ )
+ label_source = raw._pixel_label_source()
+ plot = raw._image_figure(
+ title,
+ source,
+ mapper,
+ selection,
+ label_source,
+ shared_x_range=xr,
+ shared_y_range=yr,
+ stamp_width=size,
+ stamp_height=size,
+ )
+ sources.append(source)
+ mappers.append(mapper)
+ label_sources.append(label_source)
+ plots.append(plot)
+ inspector = lab._pixel_inspector_callback(
+ labels=panel_names,
+ sources=sources,
+ mappers=mappers,
+ label_sources=label_sources,
+ selection_source=selection,
+ stamp_width=size,
+ stamp_height=size,
+ )
+ for plot in plots:
+ plot.js_on_event(Tap, inspector)
+ plot.js_on_event(
+ Tap,
+ CustomJS(
+ args=dict(
+ sources=sources,
+ exact=exact,
+ size=size,
+ selection=selection,
+ label_sources=label_sources,
+ ),
+ code=(
+ Path(__file__).with_name("pixel-readout.js")
+ ).read_text(),
+ ),
+ )
+ update = CustomJS(
+ args=dict(
+ bank=bank,
+ metrics=metrics,
+ state=state,
+ inspector=inspector,
+ index=index,
+ threshold=threshold_control,
+ threshold_line=threshold_line,
+ prevalence=prevalence,
+ family=family,
+ category=category,
+ transform=transform,
+ sources=sources,
+ mappers=mappers,
+ metadata=metadata,
+ queue=queue,
+ exact=exact,
+ selection=selection,
+ label_sources=label_sources,
+ size=size,
+ ),
+ code=(Path(__file__).with_name("review-controls.js")).read_text(),
+ )
+ for widget, prop in [
+ (index, "value"),
+ (threshold_control, "value"),
+ (prevalence, "value"),
+ (family, "value"),
+ (category, "value"),
+ (transform, "value"),
+ ]:
+ widget.js_on_change(prop, update)
+ for button, delta in [(previous, -1), (following, 1)]:
+ button.js_on_click(
+ CustomJS(
+ args=dict(
+ index=index,
+ delta=delta,
+ bank=bank,
+ threshold=threshold_control,
+ family=family,
+ category=category,
+ ),
+ code="""
+const matches=[];
+for(let j=0;j=threshold.value;
+ const positive=bank.data.label[j]===1;
+ const outcome=(positive===keep?'T':'F')+(keep?'P':'N');
+ if((family.value==='all'||family.value===bank.data.family[j]) &&
+ (category.value==='all'||category.value===outcome))matches.push(j);
+}
+if(!matches.length)return;
+let position=matches.indexOf(index.value);
+if(position<0)position=delta>0?-1:0;
+index.value=matches[(position+delta+matches.length)%matches.length];
+""",
+ )
+ )
+ # Initial complete statistics are filled without waiting for an event.
+ from evaluation import confusion
+
+ summary = confusion(population_labels, population_scores, threshold)
+ queue.text = (
+ f"Frozen validation threshold {threshold:.8g}; "
+ f"complete evaluation: {summary}. "
+ "Move a slider to explore the queue."
+ )
+ metadata.text = (
+ f"Candidate {bank.data['identifier'][0]}, gallery 1/{len(chosen)}; "
+ f"family {bank.data['family'][0]}; label {labels[chosen][0]}; "
+ f"score {scores[chosen][0]}; measured SNR {bank.data['snr'][0]}. "
+ f"Gallery from {len(population_labels)} evaluation candidates."
+ )
+ return column(
+ queue,
+ score_plot,
+ row(threshold_control, prevalence, restore),
+ row(family, category, transform),
+ row(previous, index, following),
+ metadata,
+ row(*plots),
+ exact,
+ sizing_mode="stretch_width",
+ ), dict(
+ indices=chosen.tolist(),
+ rule="round-robin confusion/SNR/family, candidate-ID order",
+ maximum=96,
+ )
+
+
+def performance_diagnostics(labels, predictions):
+ """ROC/PR/reliability for complete modeled evaluation, each seed separate.
+
+ ``predictions`` maps a display name to a saved score vector. Empty
+ reliability bins are omitted rather than presented as zero-frequency
+ measurements. Curves do not select or change operating thresholds.
+ """
+ from bokeh.layouts import column
+ from bokeh.models import ColumnDataSource, Div
+ from bokeh.palettes import Category10
+ from bokeh.plotting import figure
+
+ from cuphoton.xscan.metrics import (
+ brier_score,
+ precision_recall_curve,
+ reliability_curve,
+ roc_curve,
+ )
+
+ roc = figure(
+ title="ROC: complete candidate pool",
+ height=300,
+ sizing_mode="stretch_width",
+ x_axis_label="False-positive rate",
+ y_axis_label="Achieved recall",
+ tools="pan,wheel_zoom,reset,save",
+ )
+ pr = figure(
+ title="Precision–recall: modeled prevalence",
+ height=300,
+ sizing_mode="stretch_width",
+ x_axis_label="Recall",
+ y_axis_label="Precision",
+ tools="pan,wheel_zoom,reset,save",
+ )
+ reliability = figure(
+ title="Reliability on modeled data",
+ height=300,
+ sizing_mode="stretch_width",
+ x_axis_label="Mean saved score in bin",
+ y_axis_label="Observed positive fraction in bin",
+ tools="pan,wheel_zoom,reset,save,hover",
+ tooltips=[("Model/seed", "@name"), ("Bin count", "@count")],
+ )
+ reliability.line([0, 1], [0, 1], line_dash="dashed", color="#777777")
+ briers = []
+ for index, (name, scores) in enumerate(predictions.items()):
+ color = Category10[10][index % 10]
+ fpr, tpr, _ = roc_curve(labels, scores)
+ precision, recall, _ = precision_recall_curve(labels, scores)
+ roc.line(fpr, tpr, legend_label=name, color=color, line_width=2)
+ pr.line(
+ recall, precision, legend_label=name, color=color, line_width=2
+ )
+ bins = reliability_curve(labels, scores)
+ source = ColumnDataSource(
+ dict(
+ score=[b["mean_probability"] for b in bins],
+ frequency=[b["empirical_positive_rate"] for b in bins],
+ count=[b["count"] for b in bins],
+ name=[name] * len(bins),
+ )
+ )
+ reliability.scatter(
+ "score",
+ "frequency",
+ source=source,
+ color=color,
+ legend_label=name,
+ size=7,
+ )
+ briers.append(f"{name}: {brier_score(labels, scores):.5f}")
+ for plot in (roc, pr, reliability):
+ plot.legend.click_policy = "hide"
+ return column(
+ Div(
+ text="Brier scores (lower is better): "
+ + "; ".join(briers)
+ + ". Simulated reliability does not establish probabilities "
+ "for an observed sky."
+ ),
+ roc,
+ pr,
+ reliability,
+ sizing_mode="stretch_width",
+ )
+
+
+def training_diagnostics(root, reference):
+ """Saved losses and development learning curve, without refitting."""
+ import json
+
+ from bokeh.layouts import column
+ from bokeh.palettes import Category10
+ from bokeh.plotting import figure
+
+ root = Path(root)
+ losses = figure(
+ title="Training and validation loss, each seed",
+ x_axis_label="Epoch",
+ y_axis_label="Binary cross entropy",
+ height=300,
+ sizing_mode="stretch_width",
+ )
+ for index, result in enumerate(reference["training"]):
+ name = f"{result['mode']}-{result['seed']}"
+ history = json.loads(
+ (
+ root
+ / reference["development_directory"]
+ / name
+ / "history.json"
+ ).read_text()
+ )
+ color = Category10[10][index % 10]
+ epochs = [r["epoch"] for r in history]
+ losses.line(
+ epochs,
+ [r["train_loss"] for r in history],
+ color=color,
+ legend_label=name + " train",
+ line_dash="dashed",
+ )
+ losses.line(
+ epochs,
+ [r["val_loss"] for r in history],
+ color=color,
+ legend_label=name + " validation",
+ line_width=2,
+ )
+ losses.scatter(
+ epochs, [r["train_loss"] for r in history], color=color, size=5
+ )
+ losses.scatter(
+ epochs,
+ [r["val_loss"] for r in history],
+ color=color,
+ marker="square",
+ size=5,
+ )
+ losses.legend.click_policy = "hide"
+ curve = figure(
+ title="Development learning curve: triplet seed 11",
+ x_axis_label="Independent training fields",
+ y_axis_label="Validation FPR at validation-selected recall threshold",
+ height=300,
+ sizing_mode="stretch_width",
+ )
+ points = [
+ (r["training_fields"], r["validation"]["fpr"])
+ for r in reference["controls"]["learning_curve"]
+ ]
+ full = next(
+ r
+ for r in reference["training"]
+ if r["mode"] == "triplet" and r["seed"] == 11
+ )
+ points.append(
+ (reference["design"]["train_fields"], full["validation"]["fpr"])
+ )
+ points.sort()
+ x, y = zip(*points, strict=True)
+ curve.line(x, y, color="#007f99")
+ curve.scatter(x, y, color="#007f99", size=9)
+ return column(losses, curve, sizing_mode="stretch_width")