From d179f27b1f479729f82f7273f7160bb44f9a4afa Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 09:51:53 +0000 Subject: [PATCH 1/5] Retire the old Bead calibration: the PSF calibration's bead average replaces it The plugin (Localize/Bead calibration), its window (calibrate/qt_gui.py), its assistant actions and its pages are removed. The bead average is the PSF calibration window's, which already had every page, the bead table and the assistant's actions; its method's explanation moves to a widgetless help page, Panels/Bead average. smappy-calibrate opens the PSF calibration window (headless --out unchanged). The tests of the use_calibration hand-over and of the bead settings' forms are kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014JegwAEzHHExiXkwvXgA2j --- CLAUDE.md | 6 +- GUI.md | 15 +- docs/ai-integration.md | 2 +- docs/bead_calibration.md | 5 + docs/dual_color_calibration.md | 4 +- docs/uipsf/roadmap.md | 5 +- src/smappy/assistant/tools.py | 6 +- src/smappy/calibrate/psf_window.py | 11 +- src/smappy/calibrate/qt_gui.py | 651 ------------------ src/smappy/cli/calibrate.py | 14 +- src/smappy/datasets.py | 2 +- .../{Bead calibration.md => Bead average.md} | 95 +-- src/smappy/docs/panels/PSF calibration.md | 18 +- src/smappy/gui/add_button.py | 4 +- src/smappy/gui/app.py | 27 +- src/smappy/gui/assistant_actions.py | 5 +- src/smappy/gui/window_actions.py | 177 +---- src/smappy/plugins/__init__.py | 2 +- src/smappy/plugins/bead_calibration.py | 162 ----- .../plugins/docs/Localize/Bead calibration.md | 59 -- .../plugins/docs/Localize/PSF calibration.md | 9 +- .../plugins/docs/Localize/Spline 3D 2C.md | 2 +- src/smappy/plugins/docs/Localize/Spline 3D.md | 14 +- tests/test_calibrate_qt.py | 155 +---- tests/test_calibration_plugins.py | 24 +- tests/test_panel_docs.py | 4 +- tests/test_window_actions.py | 85 +-- tests/test_workspace.py | 4 +- 28 files changed, 113 insertions(+), 1454 deletions(-) delete mode 100644 src/smappy/calibrate/qt_gui.py rename src/smappy/docs/panels/{Bead calibration.md => Bead average.md} (92%) delete mode 100644 src/smappy/plugins/bead_calibration.py delete mode 100644 src/smappy/plugins/docs/Localize/Bead calibration.md diff --git a/CLAUDE.md b/CLAUDE.md index d1b95672..0c6facca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ updates it too. What the menus and buttons do is the assistant's through or plugin window button is added there as an action, or to `LEFT_OUT` with the reason -- `tests/test_assistant_gui.py` fails until it is. What is inside a window `open_window` opens -- its fields and buttons -- is -`gui/window_actions.py`'s ("Bead calibration/Detect and calibrate"), or the +`gui/window_actions.py`'s ("PSF calibration/Detect"), or the window is in `WINDOWS_WITHOUT_ACTIONS` with the reason -- `tests/test_window_actions.py` fails until it is. @@ -227,7 +227,7 @@ shape, so read the one closest to what you are writing: | puts one MINFLUX measurement onto another by its MBM beads | `mbm_registration.py` (*MBM registration*: each bead's position at the start, the median of its points less the beads' drift curve, `start_positions`; beads paired by a vote on their shifts, never by name -- slots are reused; a translation, or Kabsch's rigid map from three beads, in `smappy/mbm_registration.py`; kept as a named transformation for *Transform*, `Result.transformations`) | | writes what was measured per region onto its localizations | `region_column.py` (*Regions to attribute*: `region_runs.write_values`, NaN outside, never over a column of the localizations' own; a block's *write values* calls the same) | | runs steps once per region, writes columns back | `smappy/blocks.py` (a `chain.Block`; the regions through `region_runs.regions_of` -- ROIs or an id column, one interface; `RegionScratch`, `WriteBack`, *write values*; per-cluster results in `Session.id_regions`; the ROI project holds one, `ROIProject.block`) | -| opens an interactive window of the GUI's, and runs without it | `bead_calibration.py` (`Plugin.window`: the button opens what the GUI registered in `session.window_openers`; `form_in_window`: its settings are the window's, for a run without it -- `run_plugin`, a chain, a script -- while the panel shows the launcher alone); `psf_calibration.py` the same for the PSF calibration window, `calibrate/psf_window.py`, whose inputs the window and the plugin's `run` gather alike in `calibrate/psf_run.py`, whose samples come first (`psf_methods.SAMPLES`: beads, molecules on the coverslip in z sweeps, in situ) and narrow its methods (`SAMPLE_METHODS`) -- bead average, uipsf's pupil model, in situ from blinking molecules: `calibrate/insitu_run.py`; a dual sample's *channels* part the fitters' (`DualColorSettings.channels`, `MoleculeStackSettings.channels`), its *registration* how the map is measured or (molecules) a file; molecules on the coverslip, known z per frame, `calibrate/molecule_stack.py`, `molecule_dual.py` (a split chip) and `molecule_stack_run.py` on uipsf's `CoverslipStackGeometry` and `workflows/molecule_stack.py`, `docs/uipsf/molecule-stacks.md`; the elliptical Gaussian's width curves for Astigmatic z, `calibrate/elliptical.py`; and the *4Pi* mode, beside single channel and dual colour -- the pupil model and the bead average of 4Pi bead scans read through the channel layer, saved for Spline 4Pi, `calibrate/fourpi_beads.py`, `psf_methods.for_mode`) are Qt-free entries in `calibrate/psf_methods.py` -- each a *Detect beads* and a *Calibrate*, its pages and, after a run, the model against the data in `calibrate/model_views.py` | +| opens an interactive window of the GUI's, and runs without it | `psf_calibration.py` (`Plugin.window`: the button opens what the GUI registered in `session.window_openers`; `form_in_window`: its settings are the window's, for a run without it -- `run_plugin`, a chain, a script -- while the panel shows the launcher alone): the PSF calibration window, `calibrate/psf_window.py` (the old *Bead calibration* window it replaced was retired in 2026-11; its method is the *bead average*, `calibrate/core.py` and `calibrate/dual.py`, explained on `docs/panels/Bead average.md`), whose inputs the window and the plugin's `run` gather alike in `calibrate/psf_run.py`, whose samples come first (`psf_methods.SAMPLES`: beads, molecules on the coverslip in z sweeps, in situ) and narrow its methods (`SAMPLE_METHODS`) -- bead average, uipsf's pupil model, in situ from blinking molecules: `calibrate/insitu_run.py`; a dual sample's *channels* part the fitters' (`DualColorSettings.channels`, `MoleculeStackSettings.channels`), its *registration* how the map is measured or (molecules) a file; molecules on the coverslip, known z per frame, `calibrate/molecule_stack.py`, `molecule_dual.py` (a split chip) and `molecule_stack_run.py` on uipsf's `CoverslipStackGeometry` and `workflows/molecule_stack.py`, `docs/uipsf/molecule-stacks.md`; the elliptical Gaussian's width curves for Astigmatic z, `calibrate/elliptical.py`; and the *4Pi* mode, beside single channel and dual colour -- the pupil model and the bead average of 4Pi bead scans read through the channel layer, saved for Spline 4Pi, `calibrate/fourpi_beads.py`, `psf_methods.for_mode`) are Qt-free entries in `calibrate/psf_methods.py` -- each a *Detect beads* and a *Calibrate*, its pages and, after a run, the model against the data in `calibrate/model_views.py` | | hands the session to a new instance | `smappy.handoff` (*File > Open in a new window*: the table through shared memory, the rest as JSON, `smappy-gui --handoff`; the same layers and views as a saved file's session block, `handoff.session_block` in `/gui/session`, put back with the windows, clamped to the screens there are, by *File > Open and restore* and *Restore saved session*, `gui/restore.py`) | | is run by MicroClaw during an acquisition | `gui/live_session.py` (the GUI on a fit another program controls through `Context.stop` and `writer_finished`; the package and its protocol runner are `ries-lab/smappy-microclaw`) | @@ -439,7 +439,7 @@ understand the algorithm without reading the code. The format is in `src/smappy/docs/__init__.py`; `Analysis/Drift/RCC.md` is the example to copy. The tabs and windows that are not plugins (the Render tab, the Regions tab, -the ROI manager, the per-region steps, the 3D view, the bead calibration, the batch window) have pages too, +the ROI manager, the per-region steps, the 3D view, the PSF calibration, the batch window) have pages too, in `src/smappy/docs/panels/`, with `title`, `summary` and `widget` in the front matter and a *Controls* section in place of *Parameters*: one `### label` per control, the label exactly as the panel shows it. diff --git a/GUI.md b/GUI.md index eac6b330..0f1dfcba 100644 --- a/GUI.md +++ b/GUI.md @@ -202,13 +202,14 @@ reasoning is here so that it does not have to be re-derived. in the plugin's section and draws the curve, with the date of the run in the tooltip. Both drift plugins do it; a plugin that keeps nothing offers its figure for as long as the session lasts, as before. -* **Bead calibration is a Qt window in this process** (`calibrate/qt_gui.py`), - opened from the Localize tab or the Tools menu (and on its own by - `smappy-calibrate`). It holds only widgets: the work is - `calibrate.core` and `calibrate.dual`, and the five plot pages are +* **PSF calibration is a Qt window in this process** (`calibrate/psf_window.py`), + opened from the Localize tab (and on its own by `smappy-calibrate`). It + holds only widgets: the work is `calibrate.psf_methods` -- for the bead + average `calibrate.core` and `calibrate.dual`, whose plot pages are `calibrate.plots`, functions of the result and of the selected and excluded beads that draw onto a matplotlib figure. Single channel and dual - colour both run. A saved calibration goes straight into the Spline 3D + colour both run. (It replaced the *Bead calibration* window, + `calibrate/qt_gui.py`, retired in 2026-11.) A saved calibration goes straight into the Spline 3D fitter's settings -- or, for a dual-colour one, into Spline 3D 2C's. ## Code map @@ -228,8 +229,8 @@ reasoning is here so that it does not have to be re-derived. smappy/gui/view3d.py the 3D window, its panel and mouse smappy/gui/roi_tab.py the ROI tab: geometry, find, evaluate smappy/gui/roi_window.py the ROI manager window: four quadrants - smappy/calibrate/qt_gui.py bead calibration in Qt: widgets only - smappy/calibrate/plots.py its pages, drawn onto a matplotlib Figure + smappy/calibrate/psf_window.py PSF calibration in Qt: widgets only + smappy/calibrate/plots.py the bead average's pages, drawn onto a matplotlib Figure smappy/roi_manager/link.py the ROI project backed by the session smappy/gui/params.py Settings dataclass -> form widget, and back smappy/gui/widgets.py CollapsibleSection diff --git a/docs/ai-integration.md b/docs/ai-integration.md index cfb4da84..f3407e7e 100644 --- a/docs/ai-integration.md +++ b/docs/ai-integration.md @@ -226,7 +226,7 @@ Qt-free, returning JSON; the protocol layer only lists and calls them. microscope manager and the camera editor so far -- and `tests/test_window_actions.py` fails on one that has neither. Without a window the same calibrations are `run_plugin`'s: - *Localize/Bead calibration* and *Localize/PSF calibration* run from + *Localize/PSF calibration* (the old *Bead calibration* was retired in 2026-11) runs from their settings (`Plugin.form_in_window`: the window's settings, which the panel does not show), detect, calibrate and save as the window does with nothing marked by hand, through the same code (`calibrate/psf_run.py`), diff --git a/docs/bead_calibration.md b/docs/bead_calibration.md index c5609bed..1cacda59 100644 --- a/docs/bead_calibration.md +++ b/docs/bead_calibration.md @@ -1,5 +1,10 @@ # Bead spline calibration +> **2026-11:** the *Bead calibration* window this page describes was retired. Its method is the +> *bead average* of the *PSF calibration* window (`calibrate/psf_window.py`), explained for users on the +> help page *Bead average*; `smappy-calibrate --out` still runs it headless, and `smappy-calibrate` alone +> opens the PSF calibration window. The window's controls below are history. + Build one single-channel PSF (point spread function: the image one emitter makes on the camera, at every height) from all selected bead stacks, as a spline calibration (also called a bead calibration, cspline or experimental diff --git a/docs/dual_color_calibration.md b/docs/dual_color_calibration.md index 16c0a8da..234a6e25 100644 --- a/docs/dual_color_calibration.md +++ b/docs/dual_color_calibration.md @@ -11,8 +11,8 @@ frame in one global fit, pages under Help > Plugin documentation. ## Use -Open the *Bead calibration* section of the Localize tab, -or run `smappy-calibrate`. Set **mode → Dual colour**. The *channels* +Open the *PSF calibration* section of the Localize tab (sample *beads in a z stack*, +method *bead average*), or run `smappy-calibrate`. Set **mode → Dual colour**. The *channels* section says where the two channels lie (`ArrangementSettings`, the same part the two-colour fitters, *Calibrate transform* and the dual molecule stacks ask it in): on *the microscope's*, the microscope's entry -- its arms' diff --git a/docs/uipsf/roadmap.md b/docs/uipsf/roadmap.md index 8cf58130..796f2ca8 100644 --- a/docs/uipsf/roadmap.md +++ b/docs/uipsf/roadmap.md @@ -56,8 +56,9 @@ when a milestone lands or an item moves. *PSF calibration*'s optics on *auto*, a calibration's microscope state and the fit's warning); next, the found-settings line where files are opened, manual elements (the splitting dichroic), then the manager from the menu. -3. **Retire *Bead calibration*:** the default GUI and the tutorials (`fitting_3d`, `two_colour_3d`) use *PSF - calibration* (October 2026); the plugin and its window stay in the tree for now, to be removed (M6). +3. **Done: *Bead calibration* retired** (November 2026): the plugin, its window (`calibrate/qt_gui.py`) and + its pages are gone; the bead average is *PSF calibration*'s, its method explained on the help page + *Bead average*, and `smappy-calibrate` opens the PSF calibration window. 4. **Done: the 2ch stall** (architecture.md, "Why Zenodo 2ch stalled"). Not the indices: the start put 50 of 88 dim channel-0 backgrounds at softplus' 10⁻³ floor, the saturation guard then shortened every parameter's step for three of them (α = 2·10⁻⁴) until the step test stopped stage 1, and `torch.polar`'s diff --git a/src/smappy/assistant/tools.py b/src/smappy/assistant/tools.py index 9d454caa..b3993a8d 100644 --- a/src/smappy/assistant/tools.py +++ b/src/smappy/assistant/tools.py @@ -1403,12 +1403,12 @@ def open_plugin(self, path: str, settings: Optional[Dict[str, Any]] = None @tool("Open one of SMAPpy's own windows that is not a plugin: the ROI " "manager (on an ROI, given its number), the 3D view, the batch " - "window, the bead or PSF calibration, the microscope manager, the " + "window, the PSF calibration, the microscope manager, the " "camera editor, the help window. session_state lists them under " "'windows'. Changes nothing, so it works in suggest mode too. " - "What is inside the bead and PSF calibration windows -- files, " + "What is inside the PSF calibration window -- files, " "settings, Detect, Calibrate, Save -- is list_actions' " - "('Bead calibration', 'PSF calibration'); open_window is not needed first.", + "('PSF calibration'); open_window is not needed first.", needs="windows", name={"type": "string", "description": "e.g. 'ROI manager', '3D view'"}, roi={"type": "integer", diff --git a/src/smappy/calibrate/psf_window.py b/src/smappy/calibrate/psf_window.py index 7f5cb6e8..a1421b05 100644 --- a/src/smappy/calibrate/psf_window.py +++ b/src/smappy/calibrate/psf_window.py @@ -44,7 +44,7 @@ from ..gui.params import SettingsForm from ..gui.widgets import CollapsibleSection from .core import CalibrationSettings -from .dual import DualColorSettings +from .dual import DualColorSettings, bead_specs from .elliptical import EllipticalCalibrationSettings from .input import acquisition_kind, discover_acquisitions, is_fourpi_scan from .psf_methods import (AVERAGE_ONLY, METHODS, SAMPLES, LearningProgress, duration, for_mode, methods_of, modes_of, @@ -54,7 +54,6 @@ from .molecule_stack import MoleculeStackSettings from .pupil import BeadSettings, OpticsSettings, RunSettings, SplineSettings from .medium import FIELDS as MEDIUM_FIELDS -from .qt_gui import _bead_specs def torch_missing(): @@ -308,7 +307,7 @@ def choice(label, items, slot): hlayout.setContentsMargins(0, 0, 0, 0) self.detection_forms = {} for cls in (CalibrationSettings, DualColorSettings): - form = SettingsForm(cls, _bead_specs(cls)) + form = SettingsForm(cls, bead_specs(cls)) hlayout.addWidget(form) self.detection_forms[cls] = form # in situ: its own choices (the focus, the start, the coverslip, the rounds) and, under more, how the @@ -1517,14 +1516,16 @@ def _use_in_fitter(self) -> None: self.status_bar.showMessage(f"the {fitter} now uses {Path(self.run_result.saved).name}") -def show_psf_calibration(paths=()): - """Open the window, making a QApplication if there is none.""" +def show_psf_calibration(paths=(), dual: bool = False): + """Open the window, making a QApplication if there is none (`smappy-calibrate` without --out).""" from PySide6.QtWidgets import QApplication from ..gui import make_app standalone = QApplication.instance() is None app = make_app() window = PSFCalibrationWindow(paths) + if dual: + window.mode_box.setCurrentText("Dual colour") window.show() if standalone: app.exec() diff --git a/src/smappy/calibrate/qt_gui.py b/src/smappy/calibrate/qt_gui.py deleted file mode 100644 index cff429ec..00000000 --- a/src/smappy/calibrate/qt_gui.py +++ /dev/null @@ -1,651 +0,0 @@ -"""The bead PSF calibration window. - -Only widgets live here. The work is `calibrate.core` (single channel) and -`calibrate.dual` (dual colour), run on a worker thread; the pages are -`calibrate.plots`, functions of the result and of which beads are selected -and excluded, so this window only decides which page is in front and hands -them the table's state. The mode selector picks single channel or dual -colour, and switching keeps the files and the settings both modes share. -""" -from __future__ import annotations - -from dataclasses import fields -from pathlib import Path -from typing import List, Optional - -import numpy as np -from PySide6.QtCore import QObject, Qt, QThread, Signal -from PySide6.QtGui import QKeySequence, QShortcut -from PySide6.QtWidgets import (QAbstractItemView, QCheckBox, QComboBox, QFileDialog, - QHBoxLayout, QLabel, QListWidget, - QMainWindow, QMessageBox, QPushButton, QScrollArea, - QSplitter, QTabWidget, QTableWidget, QTableWidgetItem, - QVBoxLayout, QWidget) - -from ..gui.params import SettingsForm -from ..gui.widgets import CollapsibleSection -from . import plots -from .core import (CalibrationSettings, build_calibration, calibration_save_defaults, - calibration_save_path, collect_beads, doubles) -from .dual import DualColorSettings, build_dual_calibration, collect_dual_beads, mirror_axis -from .input import discover_acquisitions - -PAGES = ("Overview", "Transformation", "Fit quality", "Field diagnostics", "Bead diagnostics", "CRLB") -COLUMNS = ("bead", "stack", "x, y (px)", "brightness", "dz (nm)", "correlation", - "residual", "state") - - -class _Worker(QObject): - done = Signal(str, object) - progress = Signal(str) - - def __init__(self, kind, function): - super().__init__() - self.kind, self.function = kind, function - - def run(self): - try: - self.done.emit(self.kind, self.function()) - except Exception as error: # shown, never fatal - self.done.emit("error", error) - - -def _bead_specs(cls): - """The form of a calibration's settings, as a bead calibration reads them (`dual.bead_specs`).""" - from .dual import bead_specs - return bead_specs(cls) - - -class CalibrationWindow(QMainWindow): - """Files and settings on the left, plots and the bead table on the right.""" - - # the path a saved calibration was written to, and whether it is dual - # colour -- which decides *which* fitter it belongs in - calibrated = Signal(str, bool) - - def __init__(self, paths=(), settings=None, parent=None): - super().__init__(parent) - from ..gui.help_window import attach_help - attach_help(self, 'Panels/Bead calibration') - self.setWindowFlag(Qt.Window, True) - self.setWindowTitle("SMAPpy - bead calibration") - self.setAcceptDrops(True) - self.paths: List[str] = [str(p) for p in paths] - self.result = None - self.diagnostics = None - self.profile_data = None - self.showing_quality = False - self.excluded: set = set() - self.busy = False - self._thread = None - # a job the assistant started (`gui/window_actions.py`) fails into the status line and - # `last_error`, never a modal box: one would hold the assistant's queue on the GUI thread - self.quiet_job = False - self._job_quiet = False - self.last_error = "" - self.mode = "Single channel" - self.stack_viewer: Optional[QWidget] = None - - control = self._controls(settings) - scroll = QScrollArea(widgetResizable=True) - scroll.setWidget(control) - scroll.setFrameShape(QScrollArea.NoFrame) - scroll.setMinimumWidth(360) - scroll.setMaximumWidth(430) - - self.tabs = QTabWidget() - self.pages = [] - for title in PAGES: - from matplotlib.figure import Figure - - from ..gui.canvas import canvas_class - figure = Figure(figsize=(9, 6), constrained_layout=True) - canvas = canvas_class()(figure) - canvas.mpl_connect("pick_event", self.pick_pair) - self.tabs.addTab(canvas, title) - self.pages.append((figure, canvas)) - self.tabs.currentChanged.connect(lambda _: self.redraw()) - - self.bead_table = QTableWidget(0, len(COLUMNS)) - self.bead_table.setHorizontalHeaderLabels(COLUMNS) - self.bead_table.verticalHeader().setVisible(False) - self.bead_table.setSelectionBehavior(QAbstractItemView.SelectRows) - self.bead_table.setSelectionMode(QAbstractItemView.ExtendedSelection) - self.bead_table.setEditTriggers(QAbstractItemView.NoEditTriggers) - self.bead_table.itemSelectionChanged.connect(self.redraw) - self.bead_table.setMinimumHeight(140) - exclude = QShortcut(QKeySequence(Qt.Key_Space), self) - exclude.setContext(Qt.WindowShortcut) - exclude.activated.connect(self.toggle_excluded) - - right = QSplitter(Qt.Vertical) - right.addWidget(self.tabs) - table_side = QWidget() - tlayout = QVBoxLayout(table_side) - tlayout.setContentsMargins(2, 2, 2, 2) - tlayout.addWidget(QLabel("beads - select rows to inspect; space excludes or " - "includes them, then Recalculate")) - tlayout.addWidget(self.bead_table) - row = QHBoxLayout() - self.exclude_button = QPushButton("Exclude / include") - self.exclude_button.clicked.connect(self.toggle_excluded) - self.rebuild_button = QPushButton("Recalculate") - self.rebuild_button.setToolTip("build the calibration again without the excluded beads") - self.rebuild_button.clicked.connect(self.rebuild) - row.addWidget(self.exclude_button) - row.addWidget(self.rebuild_button) - row.addStretch(1) - tlayout.addLayout(row) - right.addWidget(table_side) - right.setSizes([600, 220]) - - whole = QSplitter(Qt.Horizontal) - whole.addWidget(scroll) - whole.addWidget(right) - whole.setSizes([390, 1000]) - self.setCentralWidget(whole) - self.status_bar = self.statusBar() - self.resize(1400, 900) - self.update_mode() - self._refresh_files() - - # ------------------------------------------------------------ controls - def _controls(self, settings) -> QWidget: - panel = QWidget() - layout = QVBoxLayout(panel) - layout.setContentsMargins(6, 6, 6, 6) - layout.setSpacing(6) - - files = QWidget() - flayout = QVBoxLayout(files) - flayout.setContentsMargins(0, 0, 0, 0) - self.file_list = QListWidget() - self.file_list.setSelectionMode(QAbstractItemView.ExtendedSelection) - self.file_list.setMaximumHeight(120) - flayout.addWidget(self.file_list) - brow = QHBoxLayout() - from ..gui.add_button import AddButton - self.add_button = AddButton("Bead stacks", "Image stacks (*.tif *.tiff *.ome.tif);;" - "All files (*)") - self.add_button.chosen.connect(self.add_chosen) - brow.addWidget(self.add_button) - remove = QPushButton("Remove") - remove.clicked.connect(self.remove_paths) - brow.addWidget(remove) - flayout.addLayout(brow) - self.subdirectories = QCheckBox("search subdirectories") - self.subdirectories.setChecked(True) - flayout.addWidget(self.subdirectories) - layout.addWidget(CollapsibleSection("bead stacks", files, expanded=True)) - - mode_row = QHBoxLayout() - mode_row.addWidget(QLabel("mode")) - self.mode_box = QComboBox() - self.mode_box.addItems(["Single channel", "Dual colour"]) - self.mode_box.currentTextChanged.connect(self.mode_changed) - mode_row.addWidget(self.mode_box, 1) - layout.addLayout(mode_row) - - # both forms are built once and one is hidden, so switching mode keeps - # what was typed - self.settings_holder = QWidget() - self.settings_layout = QVBoxLayout(self.settings_holder) - self.settings_layout.setContentsMargins(0, 0, 0, 0) - self.forms = {} - for cls in (CalibrationSettings, DualColorSettings): - form = SettingsForm(cls, _bead_specs(cls)) - self.settings_layout.addWidget(form) - self.forms[cls] = form - if settings is not None: - self._apply_settings(settings) - self._show_form() - layout.addWidget(CollapsibleSection("settings", self.settings_holder, expanded=True)) - - self.run_button = QPushButton("Detect + calibrate") - self.run_button.clicked.connect(self.run) - layout.addWidget(self.run_button) - self.quality_button = QPushButton("Calculate fit quality") - self.quality_button.clicked.connect(self.fit_quality) - layout.addWidget(self.quality_button) - self.save_button = QPushButton("Save calibration...") - self.save_button.clicked.connect(self.save) - layout.addWidget(self.save_button) - self.use_button = QPushButton("Use in the Spline 3D fitter") - self.use_button.setToolTip("put the saved calibration into the fitter's " - "calibration field") - self.use_button.clicked.connect(self._use_in_fitter) - self.use_button.setEnabled(False) - layout.addWidget(self.use_button) - - self.browse_button = QPushButton("Browse average stack...") - self.browse_button.setToolTip("look through the averaged bead stack slice " - "by slice, in its own window") - self.browse_button.clicked.connect(self.browse_stack) - self.browse_button.setEnabled(False) - layout.addWidget(self.browse_button) - layout.addStretch(1) - return panel - - def _apply_settings(self, settings) -> None: - """Put what a caller passed into whichever forms know those fields.""" - given = {f.name: getattr(settings, f.name) for f in fields(settings)} - for cls, form in self.forms.items(): - known = {f.name for f in fields(cls)} - form.set(cls(**{k: v for k, v in given.items() if k in known})) - - def _show_form(self) -> None: - """Show the mode's form, carrying the shared fields over.""" - wanted = DualColorSettings if self.is_dual else CalibrationSettings - other = CalibrationSettings if self.is_dual else DualColorSettings - shared = {f.name for f in fields(CalibrationSettings)} - try: - values = self.forms[other].value() - except (ValueError, RuntimeError): - values = None - if values is not None: - for name in shared: - field_widget = self.forms[wanted].fields.get(name) - if field_widget is not None: - field_widget.set(getattr(values, name)) - for cls, form in self.forms.items(): - form.setVisible(cls is wanted) - self.form = self.forms[wanted] - - # ---------------------------------------------------------------- mode - @property - def is_dual(self) -> bool: - return self.mode == "Dual colour" - - def mode_changed(self, text: str) -> None: - if self.busy: - self.mode_box.setCurrentText(self.mode) - return - self.mode = text - self.result = self.diagnostics = self.profile_data = None - self.showing_quality = False - self.excluded.clear() - self.bead_table.setRowCount(0) - self.browse_button.setEnabled(False) - self._show_form() - self.update_mode() - for figure, canvas in self.pages: - figure.clear() - canvas.draw_idle() - self.status_bar.showMessage("mode changed; files and settings kept - " - "detect and calibrate again") - - def update_mode(self) -> None: - for index in (1, 3): # transformation pages: dual only - self.tabs.setTabEnabled(index, self.is_dual) - # the button says where the calibration goes, which the mode decides - # (`app.use_calibration`) - self.use_button.setText("Use in the Spline 3D 2C fitter" if self.is_dual - else "Use in the Spline 3D fitter") - - # --------------------------------------------------------------- files - def _refresh_files(self) -> None: - self.file_list.clear() - for path in self.paths: - self.file_list.addItem(path) - if hasattr(self, "status_bar"): - self.status_bar.showMessage(f"{len(self.paths)} bead stack path(s)") - - def add_paths(self, paths) -> None: - for path in paths: - if path and path not in self.paths: - self.paths.append(str(path)) - self._refresh_files() - - def add_chosen(self, paths) -> None: - """Files and folders from *Add...* or a drop: each searched for acquisitions (`datasets.find`), - into subfolders with *search subdirectories*; one in which nothing is found is added as it is.""" - found = [] - for path in paths: - try: - hits = [str(p) for p in discover_acquisitions( - [path], recursive=self.subdirectories.isChecked())] - except Exception as error: - self.error(error) - return - found += hits or [str(path)] - self.add_paths(found) - - def dragEnterEvent(self, event) -> None: - from ..gui import drops - if drops.accepts(event.mimeData()): - event.acceptProposedAction() - - def dropEvent(self, event) -> None: - from ..gui import drops - event.acceptProposedAction() - self.add_chosen([str(p) for p in drops.local_paths(event.mimeData())]) - - def remove_paths(self) -> None: - for item in self.file_list.selectedItems(): - if item.text() in self.paths: - self.paths.remove(item.text()) - self._refresh_files() - - # ---------------------------------------------------------------- work - def settings(self): - return self.form.value() - - def error(self, exc, quiet: bool = False) -> None: - self.last_error = f"{type(exc).__name__}: {exc}" - self.status_bar.showMessage(self.last_error) - if not (quiet or self.quiet_job): - QMessageBox.warning(self, "calibration", self.last_error) - - def _work(self, kind: str, function) -> None: - if self.busy: - return - self.busy = True - self._job_quiet, self.quiet_job = self.quiet_job, False - self.last_error = "" - for button in (self.run_button, self.quality_button, self.rebuild_button): - button.setEnabled(False) - # the last job's thread may still be winding down (its `done` comes before its loop has - # quit): replacing a QThread still running aborts the program, as the DECODE training - # window did after every prefit (`train_window.TrainWindow._work`) - if self._thread is not None and self._thread.isRunning(): - self._thread.quit() - self._thread.wait() - self._thread = QThread() - self._thread.setStackSize(32 * 1024 * 1024) - self._worker = _Worker(kind, function) - self._worker.moveToThread(self._thread) - self._thread.started.connect(self._worker.run) - self._worker.done.connect(self._finished) - self._worker.done.connect(self._thread.quit) - self._thread.start() - - def _finished(self, kind: str, value) -> None: - self.busy = False - quiet, self._job_quiet = self._job_quiet, False - for button in (self.run_button, self.quality_button, self.rebuild_button): - button.setEnabled(True) - if kind == "error": - self.error(value, quiet) - return - if kind == "result": - self.result = value - self.excluded = {i for i, reason in enumerate(value.reasons) - if reason == "manual exclusion"} - self.diagnostics = self.profile_data = None - self.showing_quality = False - self.use_button.setEnabled(False) - self.browse_button.setEnabled(True) - self.refresh_table() - self._refresh_stack_viewer() - self._show_resolved_settings(value) - self.status_bar.showMessage(f"{int(np.sum(value.accepted))} of " - f"{len(value.beads.records)} beads used") - elif kind == "quality": - self.diagnostics, self.profile_data = value - self.showing_quality = True - self.tabs.setCurrentIndex(2) - self.redraw() - - def run(self) -> None: - if not self.paths: - QMessageBox.information(self, "no stacks", "add bead stacks first") - return - settings = self.settings() - paths = list(self.paths) - dual = self.is_dual - self.excluded.clear() - self.tabs.setCurrentIndex(0) - self.status_bar.showMessage("detecting beads...") - - def job(): - if dual: - beads = collect_dual_beads(paths, settings) - return build_dual_calibration(beads) - beads = collect_beads(paths, settings) - # the beads marked doubles start excluded, as by hand: the table lets them in again - return build_calibration(beads, excluded=doubles(beads.records)) - - self._work("result", job) - - def rebuild(self) -> None: - if self.result is None: - return - beads = self.result.beads - excluded = sorted(self.excluded) - dual = self.is_dual - - def job(): - if dual: - return build_dual_calibration(beads, excluded=excluded) - return build_calibration(beads, excluded=excluded) - - self.status_bar.showMessage(f"rebuilding without {len(excluded)} bead(s)...") - self._work("result", job) - - def fit_quality(self) -> None: - """Refit the calibration beads with the fitter the data will meet. - - Single channel that is the spline fitter on one ROI per bead; dual - colour it is `smappy.dualfit`'s global fit over both channels at once, - sharing x, y and z -- the two-colour workflow itself, run backwards on - the beads. A pair therefore has one z, not one per channel. - """ - if self.result is None: - return - result = self.result - dual = self.is_dual - - def job(): - from .validation import (aligned_midline_profiles, - fit_bead_diagnostics, - fit_paired_bead_diagnostics, paired_profiles) - if dual: - fitted = fit_paired_bead_diagnostics(result) - result.registration.refits = {"global": fitted} - return fitted, paired_profiles(result) - return (fit_bead_diagnostics(result), aligned_midline_profiles(result)) - - self.status_bar.showMessage("fitting the beads back with the " - + ("two-channel" if dual else "spline") - + " fitter...") - self._work("quality", job) - - # -------------------------------------------------------------- table - def selected_beads(self) -> List[int]: - return sorted({index.row() for index in self.bead_table.selectionModel().selectedRows()}) - - def toggle_excluded(self) -> None: - for i in self.selected_beads(): - self.excluded.symmetric_difference_update({i}) - self.refresh_table() - self.redraw() - - def refresh_table(self) -> None: - result = self.result - if result is None: - self.bead_table.setRowCount(0) - return - chosen = self.selected_beads() - self.bead_table.blockSignals(True) - self.bead_table.setRowCount(len(result.beads.records)) - for i, record in enumerate(result.beads.records): - state = "manual exclusion" if i in self.excluded else result.reasons[i] - accepted = getattr(result, "transform_accepted", None) - if accepted is not None and i not in self.excluded and accepted[i]: - state = ("PSF + transformation" if result.accepted[i] - else "transformation only: " + state) - values = (i, record["stack"] + 1, f"{record['x_px']}, {record['y_px']}", - f"{record['brightness_adu']:.4g}", - f"{result.shifts[i, 0] * result.calibration.dz:.1f}", - f"{result.correlations[i]:.3f}", f"{result.residuals[i]:.3f}", state) - for column, value in enumerate(values): - item = QTableWidgetItem(str(value)) - if i in self.excluded: - item.setForeground(Qt.gray) - self.bead_table.setItem(i, column, item) - self.bead_table.resizeColumnsToContents() - self.bead_table.blockSignals(False) - for i in chosen: - if i < self.bead_table.rowCount(): - self.bead_table.selectRow(i) - - # ------------------------------------------------------------ drawing - def redraw(self) -> None: - """Draw the page in front; a page is drawn when it is looked at.""" - if self.result is None: - return - page = self.tabs.currentIndex() - figure, canvas = self.pages[page] - result, selected, excluded = self.result, self.selected_beads(), self.excluded - try: - if page == 0: - draw = plots.draw_pairs if self.is_dual else plots.draw_overview - draw(figure, result, selected, excluded) - elif page == 1: - plots.draw_transformation(figure, result, selected, excluded) - elif page == 3: - plots.draw_field(figure, result, selected, excluded) - elif page == 2: - if self.diagnostics is None: - return - if self.is_dual: - plots.draw_paired_fit_quality(figure, result, self.diagnostics, - self.profile_data, selected) - else: - self.status_bar.showMessage(plots.draw_fit_quality( - figure, self.diagnostics, self.profile_data, selected)) - elif page == 4: - plots.draw_beads(figure, result, selected, excluded, dual=self.is_dual) - else: - self.status_bar.showMessage(plots.draw_crlb(figure, result, dual=self.is_dual)) - except Exception as error: - self.status_bar.showMessage(f"could not draw: {type(error).__name__}: {error}") - return - canvas.draw_idle() - - def pick_pair(self, event) -> None: - ids = getattr(event.artist, "pair_ids", None) - if ids is None or not len(ids): - return - index = int(np.atleast_1d(ids)[0]) - if index < self.bead_table.rowCount(): - self.bead_table.selectRow(index) - - def _show_resolved_settings(self, result) -> None: - """Put the values a run actually used behind its *auto* fields. - - The settings that say "auto" resolve against the bead stacks -- the z - step from the acquisition, the saturation from the camera's integer - type, the split from the image width -- and until a run has happened - there is nothing to resolve them against. Afterwards there is, so the - fields say what they did rather than only that they decided. - """ - beads = result.beads - channels = getattr(beads, 'channels', None) # dual: the two halves - hints = {'dz_nm': getattr(channels[0] if channels else beads, - 'dz_nm', None)} - records = list(getattr(beads, 'records', ()) or ()) - if records: - # a pair's record wraps one per channel; both saw the same camera - first = records[0].get('channel_records', [records[0]])[0] - hints['saturation_adu'] = first.get('saturation_adu') - geometry = getattr(result.calibration, 'geometry', None) - if isinstance(geometry, dict) and 'channels' in geometry: - # the first pixel of the second rectangle along the split: where the halves met - from .dual import geometry_layout - chip = geometry_layout(geometry) - along = 0 if 'right-left' in geometry['layout'] else 1 - hints['channels.split_position'] = max(r[along] for r in chip.regions) - self.form.set_hints({k: v for k, v in hints.items() if v is not None}) - - # -------------------------------------------------------- stack viewer - def average_stacks(self): - """The averaged bead stack(s) to browse, and what to call them. - - The *raw* average, not the smoothed spline model: it is what the - beads actually gave, so a bad bead or a registration error still shows - in it. Dual colour has one per channel, on a shared z grid. - """ - result = self.result - if result is None: - return [], [], None - channels = getattr(getattr(result, "registration", None), - "channel_raw_psfs", None) - if channels is not None and len(channels) > 1: - return (list(channels), ["main", "secondary"], - mirror_axis(result.beads.geometry)) - return [np.asarray(result.raw_psf)], ["average bead stack"], None - - def browse_stack(self) -> None: - volumes, titles, mirror = self.average_stacks() - if not volumes: - QMessageBox.information(self, "no stack", "calibrate first") - return - from .stack_view import StackViewer - calibration = self.result.calibration - if self.stack_viewer is None: - self.stack_viewer = StackViewer(volumes, calibration, titles, mirror, self) - else: - self.stack_viewer.calibration = calibration - self.stack_viewer.mirror_axis = mirror - self.stack_viewer.set_volumes(volumes, titles) - self.stack_viewer.show() - self.stack_viewer.raise_() - - def _refresh_stack_viewer(self) -> None: - """A recalculated result belongs in an already open viewer.""" - if self.stack_viewer is not None and self.stack_viewer.isVisible(): - self.browse_stack() - - # ------------------------------------------------------------- saving - def save(self) -> None: - if self.result is None: - return - folder, name = calibration_save_defaults(self.paths, dual=self.is_dual) - path, _ = QFileDialog.getSaveFileName(self, "Save calibration", - str(Path(folder) / (name + ".h5")), - "Calibration (*.h5 *.hdf5)") - if not path: - return - try: - self.save_to(path) - except Exception as error: - self.error(error) - - def save_to(self, path) -> str: - """Save the calibration at ``path`` and hand it to its fitter: *Save calibration...* after its - dialog, and the assistant's *Save* (`gui/window_actions.py`), which raises what went wrong.""" - path = str(path) - if not path.lower().endswith(".hdf5"): - # one .h5, whether the name was typed without it or a dialog added another - path = str(calibration_save_path(path)) - self.result.save(path, overwrite=True) - self.saved_path = path - self.saved_dual = self.is_dual - self.use_button.setEnabled(True) - self.status_bar.showMessage(f"saved {path}") - self.calibrated.emit(path, self.is_dual) - return path - - def _use_in_fitter(self) -> None: - path = getattr(self, "saved_path", None) - if not path: - return - dual = getattr(self, "saved_dual", self.is_dual) - self.calibrated.emit(path, dual) - fitter = "Spline 3D 2C" if dual else "Spline 3D" - self.status_bar.showMessage(f"the {fitter} fitter now uses {Path(path).name}") - - -def show_calibration_qt(paths=(), settings=None): - """Open the window, making a QApplication if there is none.""" - from PySide6.QtWidgets import QApplication - - from ..gui import make_app - standalone = QApplication.instance() is None - app = make_app() - window = CalibrationWindow(paths, settings) - window.show() - if standalone: - app.exec() - return window diff --git a/src/smappy/cli/calibrate.py b/src/smappy/cli/calibrate.py index 580ce516..635f6626 100644 --- a/src/smappy/cli/calibrate.py +++ b/src/smappy/cli/calibrate.py @@ -1,9 +1,15 @@ -"""Create a single-channel or split-frame dual-color spline PSF calibration.""" +"""Create a single-channel or split-frame dual-color spline PSF calibration. + +With --out it runs headless: the bead average of `calibrate.core` / `calibrate.dual`, from the +options below. Without, it opens the PSF calibration window on the paths (in dual colour with +--layout); the window's own settings then apply, not these options -- the old bead calibration +window that took them was retired in 2026-11. +""" import argparse def main(): - p = argparse.ArgumentParser(description=__doc__) + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) p.add_argument('paths', nargs='*', help='acquisitions or directories to pool') p.add_argument('--out', help='headless run: save a native calibration HDF5') p.add_argument('--dz', type=float, help='override objective z spacing in nm') @@ -45,8 +51,8 @@ def main(): result.save(args.out) print(f'Saved {args.out}: {result.accepted.sum()} accepted beads') else: - from ..calibrate.qt_gui import show_calibration_qt - show_calibration_qt(args.paths, settings) + from ..calibrate.psf_window import show_psf_calibration + show_psf_calibration(args.paths, dual=bool(args.layout)) if __name__ == '__main__': diff --git a/src/smappy/datasets.py b/src/smappy/datasets.py index b95bd01b..828f5095 100644 --- a/src/smappy/datasets.py +++ b/src/smappy/datasets.py @@ -1,7 +1,7 @@ """What a file or a folder holds: the data sets in it, of one kind. One engine for every place that is handed files and folders -- the batch -(its folder rules, run and watched), the PSF and bead calibration windows -- +(its folder rules, run and watched), the PSF calibration window -- so they agree on what one data set is. The batch had a plain ``rglob("*.tif")`` of its own, and fitted a Micro-Manager series that had grown into ``x.ome.tif`` and ``x_1.ome.tif`` as two acquisitions, an NDTiff diff --git a/src/smappy/docs/panels/Bead calibration.md b/src/smappy/docs/panels/Bead average.md similarity index 92% rename from src/smappy/docs/panels/Bead calibration.md rename to src/smappy/docs/panels/Bead average.md index 0de22d8e..8067c68a 100644 --- a/src/smappy/docs/panels/Bead calibration.md +++ b/src/smappy/docs/panels/Bead average.md @@ -1,8 +1,7 @@ --- -title: Bead calibration -summary: Turns z-stacks of fluorescent beads into the measured 3D PSF model the Spline 3D fitters use, and for a split camera into two models and the transformation between the halves. -widget: smappy.calibrate.qt_gui.CalibrationWindow -covers: [smappy.calibrate.core.find_peaks, smappy.calibrate.core.merged_neighbours, smappy.calibrate.core.judge_shapes, smappy.calibrate.core.shape_scores, smappy.calibrate.core.measure_peaks, smappy.calibrate.core.select_beads, smappy.calibrate.core.brightness_centre, smappy.calibrate.core.detect_beads, smappy.calibrate.dual.resolved_geometry, smappy.calibrate.dual.resolve_arrangement, smappy.calibrate.dual.bead_registration, smappy.calibrate.core.collect_beads, smappy.calibrate.core.estimate_shift, smappy.calibrate.core.subvoxel_peak, smappy.calibrate.core.shift_volume, smappy.calibrate.core.robust_shape_error, smappy.calibrate.core.build_calibration, smappy.calibrate.core._single_model, smappy.calibrate.core.spline_coefficients, smappy.calibrate.core.positive_pair_models, smappy.calibrate.core.FOCAL_PLANES, smappy.calibrate.dual.collect_dual_beads, smappy.calibrate.dual.fit_dual_transform, smappy.calibrate.dual.robust_projective, smappy.calibrate.dual.refine_projective, smappy.calibrate.dual.build_dual_calibration, smappy.calibrate.validation.fit_bead_diagnostics, smappy.calibrate.validation.fit_paired_bead_diagnostics, smappy.calibrate.qt_gui.CalibrationWindow] +title: Bead average +summary: The PSF calibration window's bead average -- turns z-stacks of fluorescent beads into the measured 3D PSF model the Spline 3D fitters use, and for a split camera into two models and the transformation between the halves. +covers: [smappy.calibrate.core.find_peaks, smappy.calibrate.core.merged_neighbours, smappy.calibrate.core.judge_shapes, smappy.calibrate.core.shape_scores, smappy.calibrate.core.measure_peaks, smappy.calibrate.core.select_beads, smappy.calibrate.core.brightness_centre, smappy.calibrate.core.detect_beads, smappy.calibrate.dual.resolved_geometry, smappy.calibrate.dual.resolve_arrangement, smappy.calibrate.dual.bead_registration, smappy.calibrate.core.collect_beads, smappy.calibrate.core.estimate_shift, smappy.calibrate.core.subvoxel_peak, smappy.calibrate.core.shift_volume, smappy.calibrate.core.robust_shape_error, smappy.calibrate.core.build_calibration, smappy.calibrate.core._single_model, smappy.calibrate.core.spline_coefficients, smappy.calibrate.core.positive_pair_models, smappy.calibrate.core.FOCAL_PLANES, smappy.calibrate.dual.collect_dual_beads, smappy.calibrate.dual.fit_dual_transform, smappy.calibrate.dual.robust_projective, smappy.calibrate.dual.refine_projective, smappy.calibrate.dual.build_dual_calibration, smappy.calibrate.validation.fit_bead_diagnostics, smappy.calibrate.validation.fit_paired_bead_diagnostics] --- ## What it does @@ -13,9 +12,9 @@ textbook Gaussian: a cylindrical lens stretches it, the objective adds its own aberrations, and both differ from one microscope to the next. The reliable way to know it is to measure it. -This window measures it from **beads**: sub-diffraction fluorescent beads +This method measures it from **beads**: sub-diffraction fluorescent beads stuck to a coverslip, imaged while the objective is stepped through focus. -Each bead gives a small 3D image of the PSF (a *bead stack*). The window +Each bead gives a small 3D image of the PSF (a *bead stack*). It finds the beads, cuts them out, aligns them onto one another, averages them, smooths the average and turns it into a cubic-spline model -- a [spline calibration](plugin:Panels/Glossary) (also called a bead calibration, @@ -25,12 +24,12 @@ The model is saved as an `.h5` file, which the [Spline 3D](plugin:Localize/Spline 3D) fitter reads; how the fit uses it is explained on that page. -It has two modes (*mode*): +It has two modes (the window's *mode*): * **Single channel**: one camera image, one PSF model. * **Dual colour**: a camera whose chip is split into two halves, each seeing the sample through a different filter (two colours) or at a different focus - (biplane). The window then makes **two** PSF models, one per half, + (biplane). It then makes **two** PSF models, one per half, normalised together so that they keep the beads' split of the light, and measures the **transformation** that says where a point in one half appears in the other -- the [registration](plugin:Panels/Glossary) (also @@ -59,8 +58,12 @@ What it needs: the calibration is *ROI-local* and only applies to data of exactly the same image size (a warning says so). -It opens from the *Bead calibration* section of the Localize tab, or on its -own as `smappy-calibrate`. +It is the *bead average* of the [PSF calibration](plugin:Panels/PSF calibration) +window -- sample *beads in a z stack*, method *bead average* -- whose other +methods learn a model of the optics instead; that page explains the window +itself: adding stacks, *Detect beads*, *Calibrate*, marking beads, saving. +`smappy-calibrate --out FILE` runs the same calibration without a window. +(Until 2026-11 it had a window of its own, *Bead calibration*.) ## How it works @@ -193,9 +196,9 @@ axes[3].set_yticklabels([]); axes[3].tick_params(labelsize=7) **6. The spline.** The model is interpolated with cubic polynomials between the grid points, so that the fitter can evaluate it, and its slopes, at any -sub-pixel position and height. This is what *Save calibration...* writes. +sub-pixel position and height. This is what the window saves. -**7. Check it.** *Calculate fit quality* fits the calibration's own beads, +**7. Check it.** The *Fit quality* page fits the calibration's own beads, plane by plane, with the same fitter the data will meet, and compares the fitted z with the objective's position. A good calibration gives a straight line of slope 1. The check is on the beads that made the model, so it shows @@ -482,36 +485,11 @@ listed under *Differences from SMAP*: a Gaussian smoothing in place of a smoothing B-spline, a linear correlation, a leave-one-out shape rejection, and the joint normalisation of the two halves. -## Controls +## Settings -### bead stacks -The files or folders of bead z-stacks (TIFF or OME-TIFF). All stacks are -pooled into one calibration, so they must share the z step. - -### Add... -Adds stacks. On a Mac it takes files and folders together, several at -once; elsewhere files only. Files and folders can also be dropped on the -window. - -### Select several folders... -In the arrow beside *Add...*: adds folders, several at once. - -### Remove -Removes the selected entries from the list. - -### search subdirectories -Whether an added folder is searched for acquisitions in its subfolders -too, or only at its top. - -### mode -*Single channel* or *Dual colour*. Switching keeps the files and the -settings the two modes share, clears the result, and shows or hides the -split-camera settings; the *Transformation* and *Field diagnostics* pages -exist only in dual colour. - -### settings -The calibration's settings. The common ones are shown; *more* has the -rest, whose defaults rarely need changing. +The settings of the window's *beads* part (and, in dual colour, its *channels* +and *registration*), as the bead average reads them. The common ones are shown; +*more* has the rest, whose defaults rarely need changing. ### bead box The model's width, in camera pixels; odd, at least 7. It must hold the @@ -643,29 +621,10 @@ The second round's tolerance on each axis, in pixels. Good bead data sits well below 0.1 pixel; if too few pairs pass, look at the *Transformation* page before raising it. -### Detect + calibrate -Runs everything: detection, extraction, alignment, rejection, the model (and -in dual colour the pairs and the transformation). It runs in the -background; the status bar says how many beads were used. +## Output -### Calculate fit quality -Refits the beads with the model just built and draws the *Fit quality* -page. - -### Save calibration... -Writes the calibration as `.h5`, by default named after the bead data's -folder and saved beside it. Opened from the main window, saving also puts -it into the fitter's *calibration* field -- Spline 3D for a single-channel -calibration, Spline 3D 2C for a dual-colour one. - -### Use in the Spline 3D fitter -Puts the calibration last saved into the fitter again, for when the field -was changed since. In dual-colour mode the button says *Spline 3D 2C*. - -### Browse average stack... -Opens the averaged bead stack -- the raw average, before smoothing, so that -a bad bead or a misalignment still shows -- slice by slice in its own -window; in dual colour both halves side by side. +The pages the window shows after a bead average. Select a bead in the table +to see its own stack. ### Overview The stack's projection with its beads (green used, red rejected), the @@ -683,7 +642,7 @@ limit* box. A tight, round cloud centred on zero is good; a cloud with a tail or two clusters means wrong pairs or the wrong layout. ### Fit quality -After *Calculate fit quality*: fitted against expected z for every bead and +Fitted against expected z for every bead and for the average, the centred error, and the lateral and axial profiles of the beads against the model. Good: a straight line of slope 1 over the useful range and a centred error of a few nanometres; the ends of the range, @@ -724,14 +683,6 @@ with its share of them, and the axis is scaled to the solid lines. Where one half's bound climbs and the other's does not, that half carries the measurement there. -### Exclude / include -Marks the selected beads in the table as excluded (or includes them again); -space does the same. Takes effect on *Recalculate*. - -### Recalculate -Builds the calibration again from the beads already found, without the -excluded ones. In dual colour it refits the transformation too. - ## Differences from SMAP Based on SMAP's bead calibrator, `calibrate3D_GUI_g` in `fit3Dcspline` diff --git a/src/smappy/docs/panels/PSF calibration.md b/src/smappy/docs/panels/PSF calibration.md index 8b804158..7b40b44c 100644 --- a/src/smappy/docs/panels/PSF calibration.md +++ b/src/smappy/docs/panels/PSF calibration.md @@ -14,8 +14,8 @@ three samples (*sample*), each offering the methods that fit it (*method*): * **beads in a z stack**: fluorescent beads on the coverslip, imaged while the objective steps through focus. Methods: the **bead average** -- the beads aligned onto one another, averaged and smoothed, the - [bead calibration](plugin:Panels/Bead calibration)'s method, explained on - its page -- the **pupil model (uiPSF)**, the PSF learnt as a **model of + the old bead calibration's method, explained on + [its page](plugin:Panels/Bead average) -- the **pupil model (uiPSF)**, the PSF learnt as a **model of the optics**, explained here, and the elliptical Gaussian; * **molecules on the coverslip in a z stack**: single dye molecules stuck to the coverslip in blinking buffer, imaged through repeated z sweeps. The @@ -88,8 +88,8 @@ GPU where PyTorch finds one (NVIDIA, or Apple's). ## How it works This section and the next describe the pupil model from beads; in situ -learning has its own section after them, and the bead average's are on the -[bead calibration's page](plugin:Panels/Bead calibration). +learning has its own section after them, and the bead average's are on +[its own page](plugin:Panels/Bead average). **Looking at the stacks.** The *Stacks* page shows each added file as it was recorded, one plane at a time, with a slider for the file and one for the @@ -1013,8 +1013,8 @@ grid, so the run takes less memory and time. ### beads How beads are found, and for the bead average how they are aligned and -averaged: the bead calibration's settings, which its -[page](plugin:Panels/Bead calibration) explains. The pupil model reads only +averaged: the bead average's settings, which its +[page](plugin:Panels/Bead average) explains. The pupil model reads only the finding; *bead box* is also the size of the boxes it learns from and of its spline. @@ -1220,7 +1220,7 @@ calibration or a registration from a movie. Empty, it is measured from the molecules' own detections, on the channels *channels* cuts the chip into; the learning refines it either way. Dual-colour beads have a *registration* too, folded: how they are paired and their map fitted, always from the beads -themselves (the [Bead calibration](plugin:Panels/Bead calibration)'s +themselves (the [bead average](plugin:Panels/Bead average)'s *registration*). ### z proxy @@ -1409,8 +1409,8 @@ diagnostics*, *CRLB*), explained on its page. The pupil model's: in one table. A held value has no CRLB. *Copy* puts the table on the clipboard, tab-separated, for a spreadsheet. - **CRLB**: how precisely a fit with the saved spline could localize a - molecule at each height, x, y and z in nm, as on the bead calibration's - page ([Bead calibration](plugin:Panels/Bead calibration), *CRLB*). The + molecule at each height, x, y and z in nm, as on the bead average's + page ([Bead average](plugin:Panels/Bead average), *CRLB*). The pupil model's, in situ's and the molecules' too: before the calibration is saved, the spline is sampled for the page as saving would, with the default *spline* settings. diff --git a/src/smappy/gui/add_button.py b/src/smappy/gui/add_button.py index f18c4737..b6f21c31 100644 --- a/src/smappy/gui/add_button.py +++ b/src/smappy/gui/add_button.py @@ -1,7 +1,7 @@ """*Add...*: files and folders, the one button every list of inputs has. -The batch window, the PSF calibration window and the bead calibration each -had their own pair, *Add files...* and *Add folder...*, which read as if +The batch window, the PSF calibration window and the old bead calibration +each had their own pair, *Add files...* and *Add folder...*, which read as if only the second took folders and differed in what a folder gave. Here it is one button. Its click opens the system's dialog, which on a Mac takes files and folders together, several at once (`mac_panel`); its menu holds *Select diff --git a/src/smappy/gui/app.py b/src/smappy/gui/app.py index 817ff480..abad0e9d 100644 --- a/src/smappy/gui/app.py +++ b/src/smappy/gui/app.py @@ -395,7 +395,6 @@ def __init__(self, session: Session, render: "RenderWindow"): QApplication.instance().focusWindowChanged.connect(self._on_focus_window) view = _add_menu(self.menuBar(), "View", self) self.view3d_window = None - self.calibration_window = None self.psf_calibration_window = None self.decode_training_window = None # the render window, brought to the front; Ctrl+2 beside the 3D @@ -413,7 +412,7 @@ def __init__(self, session: Session, render: "RenderWindow"): self.batch_window = None self._fill_plugins_menu() self.plugins_menu.aboutToShow.connect(self._fill_plugins_menu) - # The per-region steps and the bead calibration are not in this menu: + # The per-region steps and the PSF calibration are not in this menu: # each has one way in where it is used (the Regions tab; the Localize # tab's section). The ROI manager is a window one looks at, as the 3D # view is, so it is in View beside it; Ctrl+R from this window and @@ -428,10 +427,6 @@ def __init__(self, session: Session, render: "RenderWindow"): # offers to run in a new instance instead (`PluginPanel._ask_live`) session.window_openers["new window"] = \ lambda run=None: self.open_in_new_window(run=run) - # the Localize tab's Bead calibration section (`Plugin.window`): the - # window asks single or dual itself, and opens as it was left - session.window_openers["bead calibration"] = \ - lambda settings=None: self.open_calibration() # *PSF calibration*: uipsf's pupil model, a window of its own (`calibrate.psf_window`) session.window_openers["psf calibration"] = \ lambda settings=None: self.open_psf_calibration() @@ -629,7 +624,6 @@ def roi_manager(roi_id=None): return {"ROI manager": roi_manager, "3D view": lambda roi_id=None: self.show_3d(), "batch window": lambda roi_id=None: self.open_batch(), - "bead calibration": lambda roi_id=None: self.open_calibration(), "PSF calibration": lambda roi_id=None: self.open_psf_calibration(), "DECODE training": lambda roi_id=None: self.open_decode_training(), "microscope manager": lambda roi_id=None: self.open_microscope_manager(), @@ -1854,26 +1848,9 @@ def open_bead_selector(self, settings=None, plugin=None, reply=None) -> None: window.show() window.raise_() - def open_calibration(self, dual: Optional[bool] = None) -> None: - """The calibration window: bead stacks in, a spline calibration out. - - A window of its own, like the 3D viewer, but in this process, so a - saved calibration can go straight into the Spline 3D fitter. ``dual`` - sets the mode; None leaves it as it was. - """ - from ..calibrate.qt_gui import CalibrationWindow - if self.calibration_window is None: - self.calibration_window = CalibrationWindow(parent=self) - self.calibration_window.calibrated.connect(self.use_calibration) - if dual is not None: - self.calibration_window.mode_box.setCurrentText( - "Dual colour" if dual else "Single channel") - self.calibration_window.show() - self.calibration_window.raise_() - def open_psf_calibration(self) -> None: """The PSF calibration window, made once and opened as it was left; what it saves goes to the - fitter it belongs to, as the bead calibration's does.""" + fitter it belongs to.""" from ..calibrate.psf_window import PSFCalibrationWindow if self.psf_calibration_window is None: self.psf_calibration_window = PSFCalibrationWindow(parent=self) diff --git a/src/smappy/gui/assistant_actions.py b/src/smappy/gui/assistant_actions.py index caf48ea2..be3deda0 100644 --- a/src/smappy/gui/assistant_actions.py +++ b/src/smappy/gui/assistant_actions.py @@ -14,7 +14,7 @@ without anyone deciding it. The Plugins menu is `open_plugin`'s and `run_plugin`'s, and the windows are `open_window`'s too; what is inside a window -- its fields and buttons -- is `window_actions.py`'s, by the window's -name ("Bead calibration/Detect and calibrate"), or a reason it has none. +name ("PSF calibration/Detect"), or a reason it has none. """ from __future__ import annotations @@ -60,8 +60,7 @@ WINDOW_BUTTONS = { "open files": "File/Open", "add files": "File/Add", - "bead calibration": "Bead calibration/State", # and the window's other actions - "psf calibration": "PSF calibration/State", + "psf calibration": "PSF calibration/State", # and the window's other actions "decode training": "open_window", "training hosts": "File/Preferences", "decode model": "describe_plugin", # its notice: the model in a line, or how it differs diff --git a/src/smappy/gui/window_actions.py b/src/smappy/gui/window_actions.py index 19f4c337..1292fe43 100644 --- a/src/smappy/gui/window_actions.py +++ b/src/smappy/gui/window_actions.py @@ -1,4 +1,4 @@ -"""The assistant's actions inside SMAPpy's own windows: the bead and the PSF calibration. +"""The assistant's actions inside SMAPpy's own windows: the PSF calibration. `open_window` opened these and nothing more, so "calibrate these beads" ended at an empty window the assistant could neither fill nor press (a user's report, 2026-11-12). Each action @@ -11,7 +11,7 @@ window's *State*. A job runs in the window's worker as a click's does, so an action that starts one returns at once: *State* says when it is done, and a job the assistant started fails into the window's status and ``last_error``, never a modal box, which would hold the assistant's queue -(`CalibrationWindow.quiet_job`). The calculations are the windows' Qt-free methods +(`PSFCalibrationWindow.quiet_job`). The calculations are the window's Qt-free methods (`calibrate.core`, `calibrate.dual`, `calibrate.psf_methods`), which a script calls without a window. @@ -170,174 +170,6 @@ def _idle(window) -> None: raise ToolError("the window is busy with a job: State says when it is done") -# ---------------------------------------------------------- bead calibration -def bead_calibration_actions(control) -> List[Action]: - """The bead calibration window's controls (`calibrate/qt_gui.CalibrationWindow`).""" - name = "Bead calibration" - - def win(): - control.open_calibration() - return control.calibration_window - - def state(): - w = win() - result = w.result - out: Dict[str, Any] = { - "files": list(w.paths), "mode": w.mode, "modes": ["Single channel", "Dual colour"], - "settings": _values(w.form), "busy": w.busy, "status": _status(w), - "page": w.tabs.tabText(w.tabs.currentIndex()), - "pages": [t for i, t in enumerate(_pages(w.tabs)) if w.tabs.isTabEnabled(i)]} - if w.last_error: - out["error"] = w.last_error - if result is not None: - import numpy as np - out["result"] = {"beads": len(result.beads.records), - "used": int(np.sum(result.accepted)), - "excluded by hand": sorted(int(b) for b in w.excluded), - "fit quality": w.diagnostics is not None, - "saved": getattr(w, "saved_path", None)} - out["beads"] = _table(w.bead_table) - return out - - def add(path, subdirectories=True): - w = win() - _idle(w) - w.add_paths(_found(_paths(path), bool(subdirectories))) - return {"files": list(w.paths)} - - def remove(path=None): - w = win() - _idle(w) - if path is None: - w.paths.clear() - else: - gone = {str(Path(p).expanduser()) for p in ([path] if isinstance(path, str) else path)} - unknown = sorted(gone - set(w.paths)) - if unknown: - raise ToolError(f"{unknown[0]} is not in the list; it has {', '.join(w.paths) or 'none'}") - w.paths[:] = [p for p in w.paths if p not in gone] - w._refresh_files() - return {"files": list(w.paths)} - - def mode(mode): - w = win() - _idle(w) - chosen = {"single channel": "Single channel", "single": "Single channel", - "dual colour": "Dual colour", "dual color": "Dual colour", - "dual": "Dual colour"}.get(str(mode).strip().lower()) - if chosen is None: - raise ToolError(f"mode: no {mode!r}; it is 'Single channel' or 'Dual colour'") - w.mode_box.setCurrentText(chosen) - return {"mode": w.mode, "settings": _values(w.form)} - - def settings(values): - w = win() - _idle(w) - return {"settings": _set_form(w.form, values, "settings")} - - def calibrate(): - w = win() - _idle(w) - if not w.paths: - raise ToolError(f"no bead stacks: {name}/Add files first") - try: - w.settings() - except ValueError as error: - raise ToolError(f"a setting does not read: {error}") from None - return _started(w, w.run, "detecting and calibrating") - - def exclude(beads, exclude=True): - w = win() - _idle(w) - if w.result is None: - raise ToolError(f"no calibration yet: {name}/Detect and calibrate first") - rows = _select(w.bead_table, beads) - w.excluded = (w.excluded | set(rows)) if exclude else (w.excluded - set(rows)) - w.refresh_table() - w.redraw() - return {"excluded": sorted(w.excluded), - "then": f"{name}/Recalculate builds the calibration without them"} - - def recalculate(): - w = win() - _idle(w) - if w.result is None: - raise ToolError(f"no calibration yet: {name}/Detect and calibrate first") - return _started(w, w.rebuild, f"recalculating without {len(w.excluded)} bead(s)") - - def quality(): - w = win() - _idle(w) - if w.result is None: - raise ToolError(f"no calibration yet: {name}/Detect and calibrate first") - return _started(w, w.fit_quality, "fitting the beads back") - - def page(page): - w = win() - return {"page": _show_page(w.tabs, page)} - - def save(path=None): - from ..calibrate.core import calibration_save_defaults - w = win() - _idle(w) - if w.result is None: - raise ToolError(f"no calibration yet: {name}/Detect and calibrate first") - folder, stem = calibration_save_defaults(w.paths, dual=w.is_dual) - output = control._assistant().server.assistant.output - target = _save_target(path, Path(folder) / (stem + ".h5"), output, w.paths) - try: - saved = w.save_to(target) - except Exception as error: - raise ToolError(f"not saved: {type(error).__name__}: {error}") from None - fitter = "Spline 3D 2C" if w.is_dual else "Spline 3D" - return {"saved": saved, "fitter": f"the {fitter} fitter now uses it"} - - def use(): - w = win() - if not getattr(w, "saved_path", None): - raise ToolError(f"nothing saved yet: {name}/Save first") - w._use_in_fitter() - return {"fitter": _status(w)} - - return [ - Action(f"{name}/State", "The bead calibration window as it stands: its stacks, mode, " - "settings (by the dotted names Settings takes), whether a job runs, the status line " - "or the last error, and after a calibration how many beads it used and the bead " - "table. Opens the window.", state, shows=True), - Action(f"{name}/Add files", "Bead z-stacks into the window's list: each file or folder " - "searched for acquisitions, into subfolders unless subdirectories is false.", - add, {"path": PATHS, "subdirectories": {"type": "boolean"}}, ("path",), shows=True), - Action(f"{name}/Remove files", "Stacks out of the list, by path; without a path, all.", - remove, {"path": PATHS}, shows=True), - Action(f"{name}/Mode", "Single channel or Dual colour (a split chip, registered): the " - "files and the shared settings are kept, a result dropped.", mode, - {"mode": {"type": "string", "enum": ["Single channel", "Dual colour"]}}, - ("mode",), shows=True), - Action(f"{name}/Settings", "Set some of the mode's settings, by the dotted names State " - "lists; the rest stay. read_doc('Panels/Bead calibration') explains them.", - settings, {"values": VALUES}, ("values",), shows=True), - Action(f"{name}/Detect and calibrate", "Detect + calibrate: find the beads, average them " - "into a spline PSF; the beads marked doubles start excluded. Runs in the window; " - "returns at once. Nothing is saved.", calibrate, shows=True), - Action(f"{name}/Exclude beads", "Leave beads out (exclude=false lets them in again), by " - "their row in State's bead table, then Recalculate.", exclude, - {"beads": BEADS, "exclude": {"type": "boolean"}}, ("beads",), shows=True), - Action(f"{name}/Recalculate", "Build the calibration again from the same beads without " - "the excluded ones; returns at once.", recalculate, shows=True), - Action(f"{name}/Calculate fit quality", "Fit the beads back with the fitter the data will " - "meet, for the Fit quality page; returns at once.", quality, shows=True), - Action(f"{name}/Show page", "Bring a page of plots to the front: Overview, " - "Transformation, Fit quality, Field diagnostics, Bead diagnostics, CRLB.", page, - {"page": {"type": "string"}}, ("page",), shows=True), - Action(f"{name}/Save", "Save the calibration (.h5) -- by default beside the stacks, as " - "the window's dialog offers; a path given is under the output folder or the " - "stacks' folder -- and put it into the Spline 3D (or 2C) fitter.", save, - {"path": {"type": "string", "description": "the .h5 to write; default beside the stacks"}}), - Action(f"{name}/Use in fitter", "Put the saved calibration into the Spline 3D fitter " - "(dual colour: Spline 3D 2C) again.", use), - ] - - # ----------------------------------------------------------- PSF calibration def _psf_parts(w) -> Dict[str, Any]: """The PSF window's forms by the name its sections give them, those the chosen method reads @@ -655,7 +487,7 @@ def keep_pupil(): def window_actions(control) -> Dict[str, Action]: """Every window's actions, by name, for `assistant_actions.actions_of`.""" out: Dict[str, Action] = {} - for make in (bead_calibration_actions, psf_calibration_actions): + for make in (psf_calibration_actions,): out.update({a.name: a for a in make(control)}) return out @@ -674,5 +506,4 @@ def window_actions(control) -> Dict[str, Action]: } #: The windows that have actions, by the name `open_window` gives them and the prefix of theirs. -WINDOWS_WITH_ACTIONS = {"bead calibration": "Bead calibration", - "PSF calibration": "PSF calibration"} +WINDOWS_WITH_ACTIONS = {"PSF calibration": "PSF calibration"} diff --git a/src/smappy/plugins/__init__.py b/src/smappy/plugins/__init__.py index 727fb9c6..542c0d3e 100644 --- a/src/smappy/plugins/__init__.py +++ b/src/smappy/plugins/__init__.py @@ -1001,7 +1001,7 @@ class Plugin: # A window of the GUI's that this plugin opens instead of running: the # panel's button becomes *Open* and hands the settings to whatever the # GUI registered under this name (`Session.window_openers`). For a tool - # that is interactive through and through -- the bead calibration, with + # that is interactive through and through -- the PSF calibration, with # its bead table -- and which a form and a Run button cannot be; the # plugin is what lets it sit in a tab, and it imports no Qt to do so. window: Optional[str] = None diff --git a/src/smappy/plugins/bead_calibration.py b/src/smappy/plugins/bead_calibration.py deleted file mode 100644 index 3c0c3798..00000000 --- a/src/smappy/plugins/bead_calibration.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Bead calibration: the calibration window, as a section of the Localize tab, and a run without it. - -By hand it is a launcher (`Plugin.window`): the calibration is made by looking -at the beads -- a table to exclude them from, a stack to browse, pairs to pick -on the transformation plot -- and that is a window, which holds every setting, -so the panel shows the launcher alone (`form_in_window`). Being a plugin is -what lets it be pinned to a tab, or taken out of one, like the fitters beside -it. - -Without a window -- the assistant's `run_plugin`, a chain, a batch, a script --- `run` makes what the window's *Detect + calibrate* makes, from these -settings and through the same functions (`calibrate.core`, `calibrate.dual`): -the beads marked doubles left out in single channel, as the window starts -them; then, on request, *Calculate fit quality*; then the calibration saved -where the window's *Save calibration...* offers, or at *save as*. Until -2026-11 it refused, and an assistant could open the window and do nothing in -it. -""" -from __future__ import annotations - -from dataclasses import dataclass, fields -from pathlib import Path - -from ..calibrate.dual import DualColorSettings -from . import Context, FormState, Plot, Plugin, Result, param, register - -PAGES = ("Overview", "Transformation", "Fit quality", "Field diagnostics", "Bead diagnostics", "CRLB") - - -@dataclass -class BeadCalibrationSettings: - """The window's settings, for a run without it: the window asks single or dual colour itself, - and its form is the one the GUI shows (`form_in_window`).""" - stacks: str = param("", label="bead stacks", kind="open_file", - help="a bead z-stack, or a folder of them: every acquisition found in it is used") - subdirectories: bool = param(True, label="search subdirectories", - help="a folder's subfolders searched too") - mode: str = param("Single channel", label="mode", choices=("Single channel", "Dual colour"), - help="one channel, or a split camera's two registered onto each other") - beads: DualColorSettings = param(default_factory=DualColorSettings, label="settings") - fit_quality: bool = param(False, label="fit quality", - help="fit the beads back with the fitter the data will meet, for the " - "Fit quality page; takes as long as the calibration again") - save_as: str = param("", label="save as", kind="save_file", - help="where the calibration is written; empty: beside the stacks, named as " - "the window's Save offers") - - -def bead_paths(stacks: str, subdirectories: bool = True): - """The acquisitions in ``stacks``, as the window's *Add...* finds them (`psf_calibration.stack_paths`).""" - from .psf_calibration import stack_paths - return stack_paths(stacks, subdirectories) - - -def single(settings: DualColorSettings): - """The single-channel settings within the dual ones: `DualColorSettings` is `CalibrationSettings` - with the channels and the registration, and the window keeps one form of each.""" - from ..calibrate.core import CalibrationSettings - return CalibrationSettings(**{f.name: getattr(settings, f.name) for f in fields(CalibrationSettings)}) - - -def calibrate_beads(paths, settings: DualColorSettings, dual: bool, progress=None): - """What the window's *Detect + calibrate* runs: the beads collected and averaged into a spline, - in single channel the beads marked doubles excluded (the window's table lets them in again).""" - from ..calibrate.core import build_calibration, collect_beads, doubles - from ..calibrate.dual import build_dual_calibration, collect_dual_beads - if dual: - return build_dual_calibration(collect_dual_beads(paths, settings, progress), progress=progress) - beads = collect_beads(paths, single(settings), progress) - return build_calibration(beads, excluded=doubles(beads.records), progress=progress) - - -def fit_quality(result, dual: bool): - """*Calculate fit quality*: the beads fitted back, and the profiles the Fit quality page draws.""" - from ..calibrate.validation import (aligned_midline_profiles, fit_bead_diagnostics, - fit_paired_bead_diagnostics, paired_profiles) - if dual: - fitted = fit_paired_bead_diagnostics(result) - result.registration.refits = {"global": fitted} - return fitted, paired_profiles(result) - return fit_bead_diagnostics(result), aligned_midline_profiles(result) - - -def save_path(paths, dual: bool, save_as: str = "") -> str: - """Where the calibration goes: ``save_as``, or beside the stacks as the window's dialog offers -- - one ``.h5``, whatever the name was typed with (`core.calibration_save_path`).""" - from ..calibrate.core import calibration_save_defaults, calibration_save_path - if save_as: - path = str(Path(save_as).expanduser()) - else: - folder, name = calibration_save_defaults(paths, dual=dual) - path = str(Path(folder) / (name + ".h5")) - return path if path.lower().endswith(".hdf5") else str(calibration_save_path(path)) - - -def pages(result, dual: bool, quality=None) -> dict: - """The window's pages as the result's figures, nothing selected (`calibrate.plots`).""" - from ..calibrate import plots - excluded = {i for i, reason in enumerate(result.reasons) if reason == "manual exclusion"} - out = {"Overview": (plots.draw_pairs if dual else plots.draw_overview, (result, [], excluded))} - if dual: - out["Transformation"] = (plots.draw_transformation, (result, [], excluded)) - out["Field diagnostics"] = (plots.draw_field, (result, [], excluded)) - if quality is not None: - diagnostics, profiles = quality - out["Fit quality"] = ((plots.draw_paired_fit_quality, (result, diagnostics, profiles, [])) - if dual else (plots.draw_fit_quality, (diagnostics, profiles, []))) - out["Bead diagnostics"] = (lambda figure, *a: plots.draw_beads(figure, *a, dual=dual), - (result, [], excluded)) - out["CRLB"] = (lambda figure, *a: plots.draw_crlb(figure, *a, dual=dual), (result,)) - return {name: Plot(lambda figure, d=draw, a=args: d(figure, *a), panels=2, size=(9.0, 6.0), - summary=False) - for name, (draw, args) in sorted(out.items(), key=lambda kv: PAGES.index(kv[0]))} - - -@register("Localize/Bead calibration") -class BeadCalibration(Plugin): - """An experimental spline PSF from bead z-stacks, in its own window.""" - description = ("build a spline PSF calibration for Spline 3D or Spline 3D 2C " - "from bead z-stacks; opens its own window") - changes_locs = False - Settings = BeadCalibrationSettings - # 2: no *mode*: the window's own combo is the one place it is chosen - # 3: faint specks no longer cost a bead its isolation (`core.select_beads`): more beads are kept - # 4: a neighbour the detection's smoothing merged into a bead costs it its isolation, and a bead - # shaped unlike the others is marked a double and left out (`core.merged_neighbours`, `judge_shapes`) - # 5: a run without the window, from settings (the mode among them again, for that run only) - version = "5" - window = "bead calibration" - form_in_window = True - - @classmethod - def specs(cls): - from ..calibrate.dual import bead_specs - from . import param_specs - specs = param_specs(cls.Settings) - specs["beads"].children = bead_specs(DualColorSettings) - return specs - - def describe(self, ctx, settings: BeadCalibrationSettings) -> FormState: - dual = settings.mode == "Dual colour" - return FormState(shown={"beads.channels": dual, "beads.registration": dual}) - - def run(self, ctx: Context, settings: BeadCalibrationSettings) -> Result: - dual = settings.mode == "Dual colour" - paths = bead_paths(settings.stacks, settings.subdirectories) - ctx.report(f"detecting beads in {len(paths)} stack(s)...") - result = calibrate_beads(paths, settings.beads, dual, ctx.report) - quality = None - if settings.fit_quality: - ctx.report("fitting the beads back...") - quality = fit_quality(result, dual) - path = save_path(paths, dual, settings.save_as) - result.save(path, overwrite=True) - import numpy as np - used, n = int(np.sum(result.accepted)), len(result.beads.records) - fitter = "Spline 3D 2C" if dual else "Spline 3D" - text = (f"{settings.mode}, {len(paths)} stack(s): {used} of {n} beads used\n" - f"saved {path}: for the {fitter} fitter's calibration") - return Result(text=text, settings=settings, plots=pages(result, dual, quality), - data={"files": paths, "mode": settings.mode, "beads": n, "used": used, - "saved": path, "fitter": f"Localize/{fitter}"}) diff --git a/src/smappy/plugins/docs/Localize/Bead calibration.md b/src/smappy/plugins/docs/Localize/Bead calibration.md deleted file mode 100644 index 8f94fb54..00000000 --- a/src/smappy/plugins/docs/Localize/Bead calibration.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -version: "5" -covers: [smappy.calibrate.core.calibrate, smappy.calibrate.dual.calibrate_dual, - smappy.plugins.bead_calibration.calibrate_beads, smappy.plugins.bead_calibration.save_path] ---- - -## What it does - -A 3D fit needs to know what a molecule looks like at every height: the -**point spread function** (PSF). The bead calibration measures it from -fluorescent beads on a coverslip, imaged while the objective steps through -focus, and saves it as the calibration file -- a -[spline calibration](plugin:Panels/Glossary), also called a cspline or -experimental PSF model -- that -[Spline 3D](plugin:Localize/Spline 3D) and -[Spline 3D 2C](plugin:Localize/Spline 3D 2C) read. - -By hand, this section only opens the [bead calibration window](plugin:Panels/Bead calibration), -with the *Open* button in its title bar; -everything else -- single channel or dual colour, the bead stacks, the -settings, excluding beads and checking the result -- happens in that window, -and its page explains how the calibration is made. The window opens as it -was last left. The section is here so that the calibration sits beside the -fitters that use it; like any other section it can be moved or taken out of -the tab (right-click its title). - -Choose *Dual colour* in the window for a camera whose chip is split into two -halves, each seeing one dye (a [splitter](plugin:Panels/Glossary), also -called a dual-view): the calibration then holds a PSF for each half and the -map between them (the [registration](plugin:Panels/Glossary), also called -channel alignment), for Spline 3D 2C. - -## Without the window - -The settings below are for a calibration made without the window: by the -AI assistant (`run_plugin`), in a [chain](plugin:Panels/Batch window), or from a -script. The section in the tab never shows them: the window has its own. -Such a run does what the window's *Detect + calibrate* does with nothing -changed by hand. In single channel, the beads marked doubles are left out, -as the window starts them. With *fit quality* ticked, it also does -*Calculate fit quality*. The calibration is then saved where the window's -*Save calibration...* would offer: beside the stacks, named after them and -the date. *save as* puts it somewhere else. The figures are the window's -pages, with no bead selected. - -A run without the window does not put the calibration into the fitter. Set -the fitter's *calibration* to the file it reports. - -## Parameters - -### stacks - -A file is one stack. A folder is searched as the window's *Add...* searches -it, so a folder holding several stacks calibrates from all of them. - -### beads - -The window's settings, by the same names. *channels* and *registration* -are read only in dual colour. diff --git a/src/smappy/plugins/docs/Localize/PSF calibration.md b/src/smappy/plugins/docs/Localize/PSF calibration.md index 3142ad8e..61f9f6ed 100644 --- a/src/smappy/plugins/docs/Localize/PSF calibration.md +++ b/src/smappy/plugins/docs/Localize/PSF calibration.md @@ -8,8 +8,8 @@ covers: [smappy.calibrate.pupil.run_calibration, smappy.calibrate.insitu_run.run A 3D fit needs to know what one molecule looks like at every height: the **point spread function** (PSF). The PSF calibration makes it from bead -z-stacks by the **bead average** of the -[bead calibration](plugin:Localize/Bead calibration), or a **model of the +z-stacks by the **bead average** +([its page](plugin:Panels/Bead average)), or a **model of the microscope's optics** learnt from the same stacks: the objective's pupil, with its aberrations written as Zernike modes, from which the PSF at any height follows. This is uiPSF (Liu et al. 2024, [doi:10.1038/s41592-024-02282-x](https://doi.org/10.1038/s41592-024-02282-x)), @@ -22,15 +22,14 @@ the spherical aberration that grows with depth above all. A model is smooth without smoothing, says which aberrations the microscope has and how sure it is of each, and shows where it does not fit the data. Saved, it is turned into the same kind of -[spline calibration](plugin:Panels/Glossary) the bead calibration writes, so +[spline calibration](plugin:Panels/Glossary) the bead average writes, so [Spline 3D](plugin:Localize/Spline 3D) and [Spline 3D 2C](plugin:Localize/Spline 3D 2C) read either alike, and the model is kept inside the same file. By hand, this section only opens the [PSF calibration window](plugin:Panels/PSF calibration), with the *Open* button in its title bar; the window's page explains the -methods, their settings and what they show. It replaces the *Bead calibration* section in the shipped GUI; *Bead calibration* -is still in the plugin tree (Plugins > Localize) for now. +methods, their settings and what they show. It replaces the *Bead calibration* plugin and window, retired in 2026-11. The pupil model, from beads or in situ, needs PyTorch, an optional part of SMAPpy; without it the window says what to type to install it. diff --git a/src/smappy/plugins/docs/Localize/Spline 3D 2C.md b/src/smappy/plugins/docs/Localize/Spline 3D 2C.md index ab8ff070..6b35324a 100644 --- a/src/smappy/plugins/docs/Localize/Spline 3D 2C.md +++ b/src/smappy/plugins/docs/Localize/Spline 3D 2C.md @@ -44,7 +44,7 @@ What it needs: * **A dual-colour [bead calibration](plugin:Panels/Glossary)** (also called a spline calibration or experimental PSF model) of this microscope, made with - the bead calibration window in its *Dual colour* mode (the *Bead + the [PSF calibration](plugin:Panels/PSF calibration) window in its *Dual colour* mode (the *PSF calibration* section of the Localize tab opens it): z-stacks of beads that are seen in both halves. It holds everything the fit needs about the two halves -- which half is which, the transformation between them (the [registration](plugin:Panels/Glossary), diff --git a/src/smappy/plugins/docs/Localize/Spline 3D.md b/src/smappy/plugins/docs/Localize/Spline 3D.md index 4c653758..e77ca2c0 100644 --- a/src/smappy/plugins/docs/Localize/Spline 3D.md +++ b/src/smappy/plugins/docs/Localize/Spline 3D.md @@ -244,8 +244,8 @@ point from the box's corner, in x, y and z, $$\mathrm{PSF}(x, y, z) = \sum_{p=0}^{3} \sum_{q=0}^{3} \sum_{r=0}^{3} a_{pqr}\, w^{p}\, v^{q}\, u^{r} ,$$ so 64 coefficients per box, stored as an array of shape -$(64, n_z, n_y, n_x)$ in the order $16p + 4q + r$. The Bead calibration -tool computes them by not-a-knot cubic interpolation of the averaged bead +$(64, n_z, n_y, n_x)$ in the order $16p + 4q + r$. The bead average +computes them by not-a-knot cubic interpolation of the averaged bead stack along each axis in turn, after normalising it so that the brightest plane sums to 1 over the calibration ROI; $N$ is then the photons in that plane. A SMAP file brings its own coefficients. @@ -272,7 +272,7 @@ does with z -- which is what makes the z precision curve of the figure above come out of the fit rather than from a formula. A calibration made from few, noisy beads has bumps of its own that the fitter takes for information, and then reports a z precision that is too good; smoothing the calibration along -z (the Bead calibration's setting) is what prevents it. +z (the bead average's *smoothing z*) is what prevents it. **z.** The spline's z index $k$ is converted with the calibration's plane spacing $dz$ and its focal plane $k_0$ (`z0` in the file), @@ -315,11 +315,11 @@ the rounding of single precision (checked on simulated splines). Whether SMAP flipped the beads is read from its own record of it, `EMon`, not from its setting `emmirror`, whose "no mirror" (2) was once taken here for a flip; a file with no `EMon` is taken as flipped if `emmirror` is 1 or 3. -Calibrations from the Bead calibration tool are never mirrored. +Calibrations from the bead average are never mirrored. **EM gain.** A calibration records whether the beads were taken with [EM gain](plugin:Panels/Glossary) -- the electron multiplication of an EMCCD -camera -- the Bead calibration tool reads it from the bead files' metadata, a +camera -- the PSF calibration reads it from the bead files' metadata, a SMAP calibration carries it -- and if the data's camera says otherwise, the spline is mirrored in x to match: the EM register of many EMCCDs reads out mirrored, so a model from beads on the other port (the conventional @@ -401,7 +401,7 @@ is in the repository's `docs/uipsf/field-dependence.md`. **Compared with the paper.** The method is that of [Li et al. 2018](https://doi.org/10.1038/nmeth.4661); what the code does -differently (the first two concern the Bead calibration tool; a SMAP +differently (the first two concern the bead average; a SMAP calibration brings its own coefficients): * **The beads' background and scale.** The paper subtracts each bead stack's @@ -624,7 +624,7 @@ main changes: accepts several starts, keeping the fit with the best likelihood. * **No refractive-index correction.** SMAP's fitter has an optional factor (0.8 by default when switched on); here z stays in objective nanometres. -* **Calibrations of its own.** The Bead calibration tool follows SMAP's +* **Calibrations of its own.** The bead average follows SMAP's `calibrate3D_g`, but uses a linear rather than circular correlation to align beads, and crops z to where all accepted beads overlap, which can make its z range narrower than SMAP's. diff --git a/tests/test_calibrate_qt.py b/tests/test_calibrate_qt.py index 4e9a0f40..d9596383 100644 --- a/tests/test_calibrate_qt.py +++ b/tests/test_calibrate_qt.py @@ -1,11 +1,13 @@ -"""The calibration window: its widgets, and the pages `calibrate.plots` draws in it.""" +"""A saved calibration handed to its fitter (`ControlWindow.use_calibration`, which the PSF calibration +window's *Save* calls), and the bead settings' forms. The old bead calibration window these tests +began with was retired in 2026-11; the PSF calibration window's are in `tests/uipsf_tests/test_psf_window.py`.""" import pytest pytest.importorskip("PySide6") matplotlib = pytest.importorskip("matplotlib") matplotlib.use("Agg") -from smappy.calibrate.core import CalibrationSettings, build_calibration # noqa: E402 +from smappy.calibrate.core import CalibrationSettings # noqa: E402 from smappy.calibrate.dual import DualColorSettings # noqa: E402 from smappy.plugins import param_specs # noqa: E402 @@ -27,155 +29,6 @@ def test_settings_forms_cover_both_dataclasses_on_this_python(): assert dual["roi_size"].info.label == "bead box" # inherited, labelled -def test_window_draws_a_real_result_and_offers_it(app, tmp_path): - import sys - sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) - from test_bead_calibration import synthetic_collection - from smappy.calibrate.qt_gui import CalibrationWindow - - window = CalibrationWindow() - result = build_calibration(synthetic_collection()) - window._finished("result", result) - assert window.bead_table.rowCount() == len(result.beads.records) - assert len(window.pages[0][0].axes) > 0 # the overview drew - - window.bead_table.selectRow(2) - window.toggle_excluded() - assert window.excluded == {2} - window.toggle_excluded() - assert window.excluded == set() - - # the fit-quality page, through the same path the worker uses - from smappy.calibrate.validation import aligned_midline_profiles, fit_bead_diagnostics - window._finished("quality", (fit_bead_diagnostics(result), aligned_midline_profiles(result))) - assert window.tabs.currentIndex() == 2 and len(window.pages[2][0].axes) > 0 - - # the averaged stack browses in its own window, with the view controls in it - window.browse_stack() - assert window.stack_viewer.slice_count() == result.raw_psf.shape[0] - window.stack_viewer.orientation_box.setCurrentText("XZ") - assert window.stack_viewer.slice_count() == result.raw_psf.shape[1] - - path = tmp_path / "cal.h5" - result.save(path) - seen = [] - window.calibrated.connect(lambda p, dual: seen.append((p, dual))) - window.saved_path = str(path) - window._use_in_fitter() - assert seen == [(str(path), False)] - - -def test_a_dual_calibration_is_offered_to_the_two_channel_fitter(app, tmp_path): - from smappy.calibrate.qt_gui import CalibrationWindow - window = CalibrationWindow() - window.mode_box.setCurrentText("Dual colour") - seen = [] - window.calibrated.connect(lambda p, dual: seen.append((p, dual))) - window.saved_path = str(tmp_path / "cal.h5") - window._use_in_fitter() - assert seen == [(str(tmp_path / "cal.h5"), True)] - # and the button says so - assert window.use_button.text() == "Use in the Spline 3D 2C fitter" - window.mode_box.setCurrentText("Single channel") - assert window.use_button.text() == "Use in the Spline 3D fitter" - - -def test_the_dual_form_is_the_channels_and_the_movie_registrations_pairing_with_a_bead_slides_defaults(app): - """Where the channels lie (`ArrangementSettings`, the microscope's unless chosen) apart from how - the beads are paired (`RegisterSettings`), and of that only what a bead calibration reads.""" - from smappy.calibrate.dual import BEAD_UNUSED, bead_registration - from smappy.calibrate.qt_gui import CalibrationWindow - window = CalibrationWindow() - window.mode_box.setCurrentText("Dual colour") - pairing = window.form.fields["registration"] - assert not set(BEAD_UNUSED) & set(pairing.fields) - assert "min_pairs" in pairing.fields and "layout" not in pairing.fields - placed = window.form.fields["channels"] - assert {"layout", "main_channel", "split_position"} <= set(placed.fields) - settings = window.settings() - assert settings.registration == bead_registration() - assert settings.channels.layout == "microscope" - assert settings.registration.min_pairs == 8 - placed.fields["layout"].set("right-left") - assert window.settings().channels.layout == "right-left" - - -def test_mode_switch_keeps_the_shared_settings(app): - from smappy.calibrate.qt_gui import CalibrationWindow - window = CalibrationWindow(settings=CalibrationSettings(roi_size=19, smooth_z_nm=15.0)) - assert window.settings().roi_size == 19 - window.mode_box.setCurrentText("Dual colour") - dual = window.settings() - assert isinstance(dual, DualColorSettings) - assert dual.roi_size == 19 and dual.smooth_z_nm == 15.0 - window.mode_box.setCurrentText("Single channel") - assert window.settings().roi_size == 19 - - -def test_the_dual_fit_quality_page_refits_with_the_two_channel_fitter(app): - """The crash was that the window took the single-channel path for 2C. - - A dual result's beads have two channels each, so there is no one volume to - refit; the page now goes through `smappy.dualfit`'s global fitter, which is - what a two-colour dataset is fitted with. - """ - import sys - sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) - from test_dual_calibration import synthetic - from smappy.calibrate.dual import calibrate_dual - from smappy.calibrate.qt_gui import CalibrationWindow - from smappy.calibrate.validation import fit_paired_bead_diagnostics, paired_profiles - - stack, settings = synthetic("up-down mirrored", "upper") - result = calibrate_dual([stack], settings) - window = CalibrationWindow() - window.mode_box.setCurrentText("Dual colour") - window._finished("result", result) - assert window.bead_table.rowCount() == len(result.beads.records) - - window._finished("quality", (fit_paired_bead_diagnostics(result), - paired_profiles(result))) - assert window.tabs.currentIndex() == 2 - axes = window.pages[2][0].axes - assert len(axes) >= 8 # two rows of four, plus a bar - assert "two-channel fit" in axes[0].get_title() - - # every page draws for a dual result, not only the two the crash reached - for page in range(len(window.pages)): - window.tabs.setCurrentIndex(page) - window.redraw() - assert len(window.pages[page][0].axes) > 0, page - window.tabs.setCurrentIndex(2) - window.redraw() - - # the averaged stacks browse as a pair, with the mirror known - window.browse_stack() - assert len(window.stack_viewer.volumes) == 2 - assert window.stack_viewer.linked_box.isVisible() - window.stack_viewer.native_box.setChecked(True) - assert window.stack_viewer.shown_volumes()[1].shape == \ - window.stack_viewer.volumes[1].shape - - -def test_an_auto_setting_shows_what_the_run_resolved_it_to(app): - """"auto" says a decision will be made; afterwards it says which one.""" - from smappy.gui.widgets import note_colour - import sys - sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) - from test_bead_calibration import synthetic_collection - from smappy.calibrate.qt_gui import CalibrationWindow - - window = CalibrationWindow() - step = window.form.fields["dz_nm"] - assert step.auto.isChecked() and step.widget.text() == "" - - result = build_calibration(synthetic_collection()) - window._finished("result", result) - assert step.auto.isChecked() # still auto ... - assert step.value() is None # ... and still not a setting - assert float(step.widget.text()) == pytest.approx(result.beads.dz_nm) - assert note_colour() in step.widget.styleSheet() - @pytest.fixture def control(app, tmp_path, monkeypatch): diff --git a/tests/test_calibration_plugins.py b/tests/test_calibration_plugins.py index 4485fc18..0ade2b50 100644 --- a/tests/test_calibration_plugins.py +++ b/tests/test_calibration_plugins.py @@ -1,4 +1,4 @@ -"""The bead and PSF calibration plugins run without their windows: from settings, as the assistant's +"""The PSF calibration plugin runs without its window: from settings, as the assistant's `run_plugin`, a chain or a script runs them (before 2026-11 both only refused).""" import importlib.util @@ -35,27 +35,9 @@ def _drawn(result): assert figure.axes, f"{name} drew nothing" -def test_the_bead_calibration_runs_without_its_window_and_saves_beside_the_stacks(beads): - from smappy import calibration_list - result = run("Localize/Bead calibration", {"stacks": str(beads), "beads.dz_nm": 20}) - assert result.data["beads"] >= 4 and 0 < result.data["used"] <= result.data["beads"] - saved = result.data["saved"] - assert saved.endswith(".h5") and str(beads.parent) in saved - assert calibration_list.describe(saved) is not None # a calibration the fitters read - assert result.data["fitter"] == "Localize/Spline 3D" - assert list(result.plots) == ["Overview", "Bead diagnostics", "CRLB"] - _drawn(result) - - -def test_save_as_puts_the_bead_calibration_where_it_is_asked(beads, tmp_path): - result = run("Localize/Bead calibration", {"stacks": str(beads), "beads.dz_nm": 20, - "save_as": str(tmp_path / "mine")}) - assert result.data["saved"] == str(tmp_path / "mine.h5") and (tmp_path / "mine.h5").exists() - - def test_a_calibration_without_stacks_says_what_to_give(tmp_path): with pytest.raises(ValueError, match="give the stacks"): - run("Localize/Bead calibration", {}) + run("Localize/PSF calibration", {}) with pytest.raises(ValueError, match="no file or folder"): run("Localize/PSF calibration", {"stacks": str(tmp_path / "nowhere")}) @@ -93,7 +75,7 @@ def test_a_typed_pixel_reaches_the_camera_the_beads_are_cut_with(): def test_the_panels_show_the_launcher_alone_and_the_assistant_lists_the_settings(): from smappy.assistant.tools import Assistant - for path in ("Localize/Bead calibration", "Localize/PSF calibration"): + for path in ("Localize/PSF calibration",): cls = get(path) assert cls.form_in_window and cls.window described = Assistant().call("describe_plugin", {"path": path}) diff --git a/tests/test_panel_docs.py b/tests/test_panel_docs.py index 8734c1bd..323fadba 100644 --- a/tests/test_panel_docs.py +++ b/tests/test_panel_docs.py @@ -16,13 +16,13 @@ from smappy import docs PANELS = {"Panels/Render tab", "Panels/Regions tab", "Panels/ROI manager", - "Panels/3D view", "Panels/Bead calibration", "Panels/PSF calibration", + "Panels/3D view", "Panels/PSF calibration", "Panels/Bead average", "Panels/Batch window", "Panels/Camera editor", "Panels/Glossary", "Panels/Your own plugins", "Panels/Bead selector", "Panels/Microscope manager", "Panels/Assistant", "Panels/DECODE training", "Panels/Training hosts"} # pages beside the panels that belong to no widget, so name no controls -WIDGETLESS = {"Panels/Glossary", "Panels/Your own plugins"} +WIDGETLESS = {"Panels/Glossary", "Panels/Your own plugins", "Panels/Bead average"} def controls(page_text: str): diff --git a/tests/test_window_actions.py b/tests/test_window_actions.py index 0276ff5b..c7602a84 100644 --- a/tests/test_window_actions.py +++ b/tests/test_window_actions.py @@ -1,8 +1,5 @@ -"""The assistant inside SMAPpy's own windows (`gui/window_actions.py`): the bead calibration it -could open and not use (a user's report, 2026-11-12), and the PSF calibration.""" -import time - -import numpy as np +"""The assistant inside SMAPpy's own windows (`gui/window_actions.py`): the calibration window it +could open and not use (a user's report, 2026-11-12), now the PSF calibration.""" import pytest import tifffile @@ -15,17 +12,6 @@ def app(): return QApplication.instance() or QApplication([]) -def wait_for(condition, seconds=60.0): - from PySide6.QtWidgets import QApplication - end = time.monotonic() + seconds - while time.monotonic() < end: - QApplication.processEvents() - if condition(): - return True - time.sleep(0.01) - return False - - @pytest.fixture def control(app, tmp_path): from smappy.gui.app import ControlWindow, RenderWindow @@ -39,13 +25,13 @@ def control(app, tmp_path): a.output.mkdir() a.mode = "act" yield window, a - for w in (window.calibration_window, window.psf_calibration_window): + for w in (window.psf_calibration_window,): # a job's thread is still winding down after its `done`: one destroyed running aborts thread = getattr(w, "_thread", None) if thread is not None: thread.quit() thread.wait() - for w in (window.calibration_window, window.psf_calibration_window, render, window): + for w in (window.psf_calibration_window, render, window): if w is not None: w.close() @@ -65,7 +51,7 @@ def _bead_folder(tmp_path, n_stacks=2): def test_every_window_the_assistant_opens_has_actions_inside_or_a_reason(control): - """`open_window` alone opened the bead calibration and left the assistant outside it.""" + """`open_window` alone opened a calibration window and left the assistant outside it.""" from smappy.gui.window_actions import WINDOWS_WITH_ACTIONS, WINDOWS_WITHOUT_ACTIONS window, a = control opened = set(window._assistant_windows()) @@ -76,67 +62,6 @@ def test_every_window_the_assistant_opens_has_actions_inside_or_a_reason(control assert set(WINDOWS_WITH_ACTIONS.values()) <= names -def test_the_assistant_calibrates_beads_in_the_bead_calibration_window_and_saves_it(control, tmp_path): - window, a = control - names = [x["name"] for x in a.call("list_actions", {"prefix": "Bead calibration"})] - assert {"Bead calibration/Add files", "Bead calibration/Detect and calibrate", - "Bead calibration/Save"} <= set(names) - with pytest.raises(Exception, match="no bead stacks"): - act(a, "Bead calibration/Detect and calibrate") - added = act(a, "Bead calibration/Add files", path=str(_bead_folder(tmp_path)))["answer"] - assert len(added["files"]) == 2 - assert window.calibration_window.isVisible() - with pytest.raises(Exception, match="has no 'bead_size'"): - act(a, "Bead calibration/Settings", values={"bead_size": 3}) - settings = act(a, "Bead calibration/Settings", values={"dz_nm": 20})["answer"]["settings"] - assert settings["dz_nm"] == 20 - - started = act(a, "Bead calibration/Detect and calibrate")["answer"] - assert started["started"] - w = window.calibration_window - assert wait_for(lambda: not w.busy, 120) - state = act(a, "Bead calibration/State")["answer"] - assert "error" not in state, state - n = state["result"]["beads"] - assert n >= 4 and 0 < state["result"]["used"] <= n - assert state["beads"]["columns"][0] == "bead" and len(state["beads"]["rows"]) == n - - act(a, "Bead calibration/Exclude beads", beads=[0, 1]) - act(a, "Bead calibration/Recalculate") - assert wait_for(lambda: not w.busy, 120) - state = act(a, "Bead calibration/State")["answer"] - assert state["result"]["excluded by hand"] == [0, 1] - assert act(a, "Bead calibration/Show page", page="bead diag")["answer"] == { - "page": "Bead diagnostics"} - - # a file is written only by a proposal in suggest mode, and only where the assistant may - a.mode = "suggest" - assert act(a, "Bead calibration/Save")["proposed"]["tool"] == "do_action" - assert act(a, "Bead calibration/State")["answer"]["busy"] is False # reading is not proposed - a.mode = "act" - with pytest.raises(Exception, match="outside the output folder"): - act(a, "Bead calibration/Save", path=str(tmp_path / "elsewhere" / "cal.h5")) - seen = [] - w.calibrated.connect(lambda path, dual: seen.append((path, dual))) - saved = act(a, "Bead calibration/Save", path="cal")["answer"] - assert saved["saved"] == str(a.output / "cal.h5") and (a.output / "cal.h5").exists() - assert seen == [(saved["saved"], False)] and "Spline 3D fitter" in saved["fitter"] - - -def test_a_failed_job_the_assistant_started_is_said_in_state_not_in_a_box(control, tmp_path): - """A modal box would hold the assistant's queue on the GUI thread until someone closed it.""" - window, a = control - junk = tmp_path / "junk.tif" - tifffile.imwrite(junk, np.zeros((3, 4, 4), np.uint16)) - act(a, "Bead calibration/Add files", path=str(junk)) - act(a, "Bead calibration/Detect and calibrate") - w = window.calibration_window - assert wait_for(lambda: not w.busy, 60) - from PySide6.QtWidgets import QApplication - assert QApplication.activeModalWidget() is None - assert act(a, "Bead calibration/State")["answer"]["error"] - - def test_the_assistant_sets_up_the_psf_calibration_window(control, tmp_path): window, a = control folder = _bead_folder(tmp_path, n_stacks=1) diff --git a/tests/test_workspace.py b/tests/test_workspace.py index 8ed02b7f..c184a521 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -201,7 +201,7 @@ def test_the_window_opens_the_shipped_tabs(window): localize = tab_named(window, "Localize") titles = {s.title for s in localize.sections} assert {"PSF calibration", "Gaussian 2D", "Spline 3D"} <= titles - # the old Bead calibration is a plugin still, no longer pinned: the PSF calibration does what it did + # the old Bead calibration was retired (2026-11): the PSF calibration does what it did assert "Bead calibration" not in titles @@ -223,7 +223,7 @@ def test_the_psf_calibration_section_opens_the_window_as_it_was_left(window): def test_the_menus_have_no_second_way_into_what_a_tab_opens(window): - """The bead calibration is the Localize tab's, the per-region steps the + """The PSF calibration is the Localize tab's, the per-region steps the Regions tab's. The ROI manager is a window to look at, as the 3D view is, and is in View, once, with Ctrl+R. The Plugins menu is the whole plugin tree, every plugin in it, and not counted.""" From e95455ec063c3cf790e322c93e14a302d2a16198 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 10:12:02 +0000 Subject: [PATCH 2/5] uipsf in situ: no molecule chosen far below the coverslip; the twin test tested on the minimal pupil From round 2, once the coverslip edge has set the focus, a unit localised more than 3 errors below the coverslip (its CRLB times the edge's scatter factor, with the edge's own error) is not chosen (`insitu_rounds.below_coverslip`, `RoundSettings.below_coverslip_sd`): the z bins below 0 had filled with molecules the model placed badly. The twin test already mirrored held modes; a test now pins it for the minimal pupil, in situ's default. The roadmap's in situ line and the architecture's known flaws brought up to date. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014JegwAEzHHExiXkwvXgA2j --- docs/uipsf/architecture.md | 6 +++++- docs/uipsf/roadmap.md | 7 +++++-- src/uipsf/workflows/insitu_rounds.py | 27 ++++++++++++++++++++++--- tests/uipsf_tests/test_held_pupil.py | 18 +++++++++++++++++ tests/uipsf_tests/test_insitu_rounds.py | 20 +++++++++++++++++- 5 files changed, 71 insertions(+), 7 deletions(-) diff --git a/docs/uipsf/architecture.md b/docs/uipsf/architecture.md index 9941cf5d..d5d963bd 100644 --- a/docs/uipsf/architecture.md +++ b/docs/uipsf/architecture.md @@ -1819,7 +1819,11 @@ molecule, motion within a 200 ms frame (a linker), Cy3B's emission against the 5 **Known flaws of the scripts' variants**: the twin test mirrors the learnt modes but not a held map, so its verdict is not fair with a held magnitude or phase (a −200 nats "prefer twin" there is that); the z bins below -the coverslip fill with poorly localised molecules (heights are not bounded at 0). +the coverslip fill with poorly localised molecules (heights are not bounded at 0). *Both fixed since*: the +twin test mirrors held modes too (`insitu.twin_model`, for the Zernike and the minimal pupil, tested in +`test_held_pupil.py`), and from the second round a molecule localised more than 3 errors below the coverslip +is not chosen once the edge has set the focus (`insitu_rounds.below_coverslip`, `RoundSettings.below_coverslip_sd`; +November 2026, not yet run again on the M2 pairs). **Open**: which focus is right (held and free magnitude differ by ~100 nm on January; needs a sample of known height); why σ in situ is below the beads' (bead size?); the residual 1.3 per pixel of clean DNA-PAINT diff --git a/docs/uipsf/roadmap.md b/docs/uipsf/roadmap.md index 796f2ca8..b208016b 100644 --- a/docs/uipsf/roadmap.md +++ b/docs/uipsf/roadmap.md @@ -126,8 +126,11 @@ when a milestone lands or an item moves. (architecture.md, "The region size, the blur and the background"): 15 px put January on a second, nearly equal solution with 0.3 rad more astigmatism and a round blur; 25 px (3 µm, now *auto*) with σ free finds the beads' astigmatism and an anisotropic blur from both starts; a large region needs care on high, uneven - backgrounds (haze rejection or a planar background). Next: a sloped background, - the symmetric bead magnitude and the order choice in the PSF calibration window. Later: summed blinks against frames; a sparse pile-up on the coverslip, where the + backgrounds (haze rejection or a planar background). In the window since: the start's magnitude held + (*hold*) and order 6 on *auto* (*Zernike order*, which includes secondary spherical, Noll 22); the twin test + fair with held modes and heights below the coverslip not chosen (November 2026). A sloped background + (`background.Planar`) and a stricter shape test after round 1 only after a benchmark on real data: each adds + freedom to an already noisy fit. Later: summed blinks against frames; a sparse pile-up on the coverslip, where the edge is weak; speed (a round ~2 min). Zenodo `1ch_smlm_tubulin` and simulations. **On the minimal model** (November 2026, architecture.md, "In situ on the minimal model"): γ held from the microscope (`microscope_db.gamma_of`; a nominal 0.36 only as a warned last resort), no SAF diff --git a/src/uipsf/workflows/insitu_rounds.py b/src/uipsf/workflows/insitu_rounds.py index bd3dcb11..f08f20b6 100644 --- a/src/uipsf/workflows/insitu_rounds.py +++ b/src/uipsf/workflows/insitu_rounds.py @@ -32,7 +32,9 @@ off at high NA, so it is measured: the lowest molecules localised again with the focus moved (`_measured_ratio`), and from the second correction on a secant step on the rounds' (focus, edge) pairs (`_focus_per_height`). This automates v1's ``backgroundROI``, a region the user marked as -lying on the coverslip. +lying on the coverslip. Once the edge has set the focus, a molecule localised below the coverslip by more +than ``below_coverslip_sd`` (3) of its errors is not chosen (`below_coverslip`): none is there, and those the +model put there are the ones it fits worst. """ from __future__ import annotations @@ -78,6 +80,10 @@ class RoundSettings: # only units whose height above the coverslip lies in (low, high) nm are chosen from round 2 (None: all): to # learn from molecules near the coverslip or deep in the sample apart (M2, October 2026) height_range_nm: tuple | None = None + # a unit localised below the coverslip by more than this many of its errors is not chosen (None: all are), + # once the edge sets the focus: no molecule is there, and the z bins below 0 filled with molecules the model + # placed badly -- the October scripts' known flaw (architecture.md, "In situ on matched M2 pairs") + below_coverslip_sd: float | None = 3.0 # off when nothing of the pupil is learnt (every aberration held from beads): the sign is the beads' twin_test: bool = True @@ -88,6 +94,7 @@ class Edge: sd_nm: float spike: float # the fraction of the window's molecules on the coverslip n: int # heights in the window + kappa: float = 1.0 # how many times its CRLB a height scatters, as the edge fitted it def coverslip_edge(h, sd, window: float = 0.3, max_blur: float | None = None) -> Edge | None: @@ -142,8 +149,17 @@ def nll(p, spike=True): key=lambda r: r.fun) spiked = flat.fun - both.fun > SPIKE_NATS best, f = (both, nll) if spiked else (flat, lambda p: nll(p, False)) - w = unpack(best.x, spiked)[3] - return Edge(float(best.x[0]), _sd_of_first(f, best.x, float(np.median(s0_i))), float(w), len(x)) + _, kappa, _, w = unpack(best.x, spiked) + return Edge(float(best.x[0]), _sd_of_first(f, best.x, float(np.median(s0_i))), float(w), len(x), float(kappa)) + + +def below_coverslip(h, sd, edge: Edge, k: float = 3.0) -> np.ndarray: + """Which heights above the coverslip ``h`` (the edge's shift taken off), with CRLBs ``sd``, lie below it by + more than ``k`` errors: each height's scatter, κ times its CRLB as the edge measured it, and the edge's own + error together. A molecule on the coverslip scatters around 0 and stays; one placed far below by a model + that does not fit it (a pair, a neighbour, the background's haze) is out.""" + h, sd = np.asarray(h, float), np.asarray(sd, float) + return h < -k * np.hypot(edge.kappa * sd, edge.sd_nm) def _sd_of_first(f, x, scale: float) -> float: @@ -327,9 +343,14 @@ def calibrate_insitu(model, rois, start: dict | None = None, noise=None, setting # most of round 2's choice, and learning then rejected or lost them. rejected = [None] * U centre = in_focus_height(current, float(shared["geometry.focus"])) + # only once the edge has set the focus: before, 0 is not where the coverslip is + below = (below_coverslip(loc["h"] - shift, loc["sd_h"], edge, s.below_coverslip_sd) + if prior is not None and s.below_coverslip_sd is not None else np.zeros(len(pool), bool)) for k, u in enumerate(pool): if not loc["xy_offset_px"][k] <= s.max_xy_offset_px: rejected[u] = "off centre" + elif below[k]: + rejected[u] = "below coverslip" elif s.height_range_nm is not None and not ( s.height_range_nm[0] <= loc["h"][k] - shift <= s.height_range_nm[1]): rejected[u] = "height range" diff --git a/tests/uipsf_tests/test_held_pupil.py b/tests/uipsf_tests/test_held_pupil.py index 1194ab18..39e0ff79 100644 --- a/tests/uipsf_tests/test_held_pupil.py +++ b/tests/uipsf_tests/test_held_pupil.py @@ -4,6 +4,7 @@ import uipsf from uipsf.export import build, describe, load_model, save_model +from uipsf.model.pupil import MinimalPupil from uipsf.workflows import learn_beads from uipsf.workflows.insitu import twin, twin_model from uipsf_tests.conftest import small_beads @@ -83,3 +84,20 @@ def test_a_model_file_keeps_the_held_modes(tmp_path): xyz = torch.tensor([[0.0, 0.0, 0.0], [30.0, -20.0, 300.0]]) assert torch.allclose(m.psf(xyz), cal.psf(xyz), atol=1e-7) assert (22, 0.05, 0.0) in m.zernikes("phase") and (4, -0.1, 0.0) in m.zernikes("magnitude") + + +def test_the_twin_of_a_minimal_pupil_holding_the_high_orders_is_the_twin_of_the_whole(): + # in situ's default: the minimal pupil, γ and the phase above the order held from a bead model; the twin test + # must mirror the held phase too, or it compares a mirrored low order with unmirrored high ones + model, _, _ = small_beads(OPTICS, n=2, dtype=f64) + grid = model.psf.grid + whole = MinimalPupil(grid, n_max=6, held_gamma=0.3, saf_damping=False) + part = MinimalPupil(grid, n_max=6, held_gamma=0.3, saf_damping=False, + held_phase={j: v for j, v in PHASE.items() if j > 10}) + model.psf.pupil = part + mirrored = twin_model(model).psf.pupil(twin(model, values(part, PHASE, {}))) + model.psf.pupil = whole + expected = whole(twin(model, values(whole, PHASE, {}))) + assert torch.allclose(mirrored[0], expected[0]) and torch.allclose(mirrored[1], expected[1]) + # and the twin is not the pupil itself: astigmatism and spherical flip + assert not torch.allclose(mirrored[1], whole(values(whole, PHASE, {}))[1]) diff --git a/tests/uipsf_tests/test_insitu_rounds.py b/tests/uipsf_tests/test_insitu_rounds.py index 737a5584..3cc67192 100644 --- a/tests/uipsf_tests/test_insitu_rounds.py +++ b/tests/uipsf_tests/test_insitu_rounds.py @@ -6,7 +6,7 @@ import pytest import torch -from uipsf.workflows.insitu_rounds import _focus_per_height, _moved, coverslip_edge +from uipsf.workflows.insitu_rounds import _focus_per_height, _moved, below_coverslip, coverslip_edge def heights(n=1500, on_coverslip=0.2, shift=120.0, sd=30.0, scatter=1.2, outliers=0.01, seed=0): @@ -37,6 +37,24 @@ def test_without_molecules_on_the_coverslip_the_flat_edge_is_found_and_a_spike_r assert abs(np.mean([e.shift_nm for e in found]) - 120.0) < 10 +def test_molecules_on_the_coverslip_stay_and_those_placed_far_below_it_are_out(): + rng = np.random.default_rng(3) + n = 1500 + t = rng.uniform(0, 1400, n) + on = rng.random(n) < 0.2 + t[on] = 0.0 + sd = 30.0 * (0.6 + 0.8 * rng.random(n)) + h = t + 120.0 + rng.normal(0, 1.2 * sd) + bad = rng.random(n) < 0.03 # misplaced by a model that does not fit them + h[bad] -= rng.uniform(250, 500, bad.sum()) + edge = coverslip_edge(h, sd) + assert edge.kappa >= 1.0 + out = below_coverslip(h - edge.shift_nm, sd, edge) + assert out[on & ~bad].mean() < 0.01 + assert out[bad & (t < 100)].mean() > 0.9 + assert not below_coverslip(np.array([-10.0]), np.array([30.0]), edge)[0] + + def test_too_few_heights_give_no_edge(): assert coverslip_edge(np.arange(10.0), np.ones(10)) is None From 3b88299c396b94ec6a924721f350dcffbe14eb95 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 10:34:55 +0000 Subject: [PATCH 3/5] PSF calibration: says where the pupil model learns, and warns of a CPU-only PyTorch beside an NVIDIA card The line Detect writes now ends with where learning runs (uipsf.device.describe: the graphics card by name, Apple's GPU or the CPU). A torch built for the CPU only on a machine with an NVIDIA card (torch_install.cpu_build_on_nvidia, from its version's local label: a plain pip install on Windows brings it) learnt on the CPU without a word; the window now says so and offers Install PyTorch, whose command then reinstalls the card's build from the CUDA index (command(replace=True)), and asks for a restart where the old build is already loaded. Neither blocks a run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014JegwAEzHHExiXkwvXgA2j --- src/smappy/calibrate/psf_window.py | 52 +++++++++++++++--- src/smappy/docs/panels/PSF calibration.md | 17 ++++-- src/smappy/torch_install.py | 51 +++++++++++++++++- src/uipsf/device.py | 15 ++++++ tests/test_torch_install.py | 65 +++++++++++++++++++++++ 5 files changed, 185 insertions(+), 15 deletions(-) diff --git a/src/smappy/calibrate/psf_window.py b/src/smappy/calibrate/psf_window.py index a1421b05..85bfcbc7 100644 --- a/src/smappy/calibrate/psf_window.py +++ b/src/smappy/calibrate/psf_window.py @@ -66,6 +66,27 @@ def torch_missing(): f"into this Python; or, in a terminal:\n{shell_line(torch_install.command())}") +def torch_slow(): + """What the window says when torch works but leaves the NVIDIA card unused (`torch_install.replacement`), + with the command that brings the card's build; None otherwise.""" + from .. import torch_install + reason = torch_install.replacement() + if reason is None: + return None + return (f"{reason}. Install PyTorch replaces it with the build for the card; or, in a terminal:\n" + f"{shell_line(torch_install.command(replace=True))}") + + +def learning_device() -> str: + """"on the graphics card (…)", "on Apple's GPU" or "on the CPU" (`uipsf.device.describe`); empty where torch + cannot be loaded, which the notice says already.""" + try: + from uipsf.device import describe + return describe() + except Exception: + return "" + + def shell_line(parts, windows: Optional[bool] = None) -> str: """``parts`` as a line to paste into a terminal. @@ -351,6 +372,7 @@ def choice(label, items, slot): self.install_button.clicked.connect(self.install_torch) layout.addWidget(self.install_button) self._installer = None + self._device = None # where the pupil model learns, found once (`learning_device`) self.detect_button = QPushButton("Detect beads") self.detect_button.setToolTip("find the beads, ring them on the Stacks page and say how long " @@ -443,10 +465,12 @@ def _show_method(self) -> None: form.set_shown({name: reads["averaging"] for name in AVERAGE_ONLY if name in shared}) pupil = self.method.key in ("pupil", "insitu", "molecules") or getattr(self.method, "needs_torch", False) missing = torch_missing() - self.notice.setText(missing or "") - self.notice.setVisible(pupil and missing is not None) - # shown only while torch is missing or too old: nothing to press once it is right - self.install_button.setVisible(pupil and (missing is not None or self._installer is not None)) + # a CPU build on a machine with an NVIDIA card works, slowly: said and offered, never blocking + note = missing or torch_slow() + self.notice.setText(note or "") + self.notice.setVisible(pupil and note is not None) + # shown only while torch is missing, too old or the CPU's beside a card: nothing to press once it is right + self.install_button.setVisible(pupil and (note is not None or self._installer is not None)) self.stop_button.setVisible(pupil) self.browse_button.setVisible(self.method.has_stack) self.toggle_button.setVisible(getattr(self.method, "marks", True)) @@ -1073,7 +1097,13 @@ def _show_detected_text(self) -> None: """The detection's line, with the time calibrating the beads as marked takes where the method knows it (`detected_text`): excluding beads shortens it.""" text_of = getattr(self.method, "detected_text", None) - self.found_label.setText(text_of(self.detected, self.marks) if text_of else self.detected.text) + text = text_of(self.detected, self.marks) if text_of else self.detected.text + if self.method.key in ("pupil", "insitu", "molecules") or getattr(self.method, "needs_torch", False): + if self._device is None: + self._device = learning_device() + if self._device: + text += f"; learnt {self._device}" + self.found_label.setText(text) def detect(self) -> None: """Find the beads and say how many and how long calibrating them will take; nothing is learnt.""" @@ -1456,7 +1486,8 @@ def install_torch(self) -> None: self._installer.kill() return from .. import torch_install - program, *arguments = torch_install.command() + self._replacing = torch_install.cpu_build_on_nvidia() + program, *arguments = torch_install.command(replace=self._replacing) process = QProcess(self) process.setProcessChannelMode(QProcess.MergedChannels) process.readyReadStandardOutput.connect(self._installer_output) @@ -1478,8 +1509,13 @@ def _installer_finished(self, code, _status=None) -> None: self._installer = None self.install_button.setText("Install PyTorch") missing = torch_missing() - self.status_bar.showMessage("PyTorch is installed: the pupil model is ready" if missing is None - else f"the install stopped (exit code {code}); {missing.splitlines()[0]}") + if missing is not None: + self.status_bar.showMessage(f"the install stopped (exit code {code}); {missing.splitlines()[0]}") + elif getattr(self, "_replacing", False) and "torch" in sys.modules: + # the CPU build is loaded in this process already: the card's is used from the next start + self.status_bar.showMessage("PyTorch for the card is installed: restart SMAPpy to learn on it") + else: + self.status_bar.showMessage("PyTorch is installed: the pupil model is ready") self._show_method() # ------------------------------------------------------------- saving diff --git a/src/smappy/docs/panels/PSF calibration.md b/src/smappy/docs/panels/PSF calibration.md index 7b40b44c..9f5de39d 100644 --- a/src/smappy/docs/panels/PSF calibration.md +++ b/src/smappy/docs/panels/PSF calibration.md @@ -2,7 +2,7 @@ title: PSF calibration summary: Makes the PSF calibration the Spline 3D fitters read, from beads in a z stack, from single molecules on the coverslip in a z stack, or in situ from the blinking molecules of a measurement -- the bead average or a physical model of the microscope's pupil learnt with uiPSF v2 -- or, as a fourth, the curves of an elliptical Gaussian's widths in z that Astigmatic z reads, or, in the 4Pi mode, a 4Pi model for Spline 4Pi from 4Pi bead scans, and shows what each found and how well it fits. widget: smappy.calibrate.psf_window.PSFCalibrationWindow -covers: [smappy.calibrate.pupil.half_split_uncertainty, smappy.calibrate.pupil.fit_quality, smappy.calibrate.plots.draw_pupil_fit_quality, smappy.torch_install.command, smappy.calibrate.psf_methods.BeadAverage, smappy.calibrate.psf_methods.PupilModel, smappy.calibrate.pupil.run_calibration, smappy.calibrate.pupil.bead_rois, smappy.calibrate.pupil.single_rois, smappy.calibrate.pupil.dual_rois, smappy.calibrate.pupil.spline_calibration, smappy.calibrate.pupil.dual_calibration, smappy.calibrate.pupil.save, smappy.calibrate.pupil.OpticsSettings, smappy.calibrate.pupil.BeadSettings, smappy.calibrate.pupil.learns_tilt, uipsf.diagnostics.figures.coverslip_tilt, smappy.calibrate.core.find_stack_beads, smappy.calibrate.psf_window.PSFCalibrationWindow, smappy.calibrate.pupil.estimate_seconds, smappy.calibrate.psf_methods.LearningProgress, smappy.calibrate.model_views.average_views, smappy.calibrate.model_views.pupil_views, smappy.calibrate.model_views.draw_model_vs_data, smappy.calibrate.model_views.planes, smappy.calibrate.stack_browser.StackBrowser, smappy.calibrate.stack_view.StackViewer, smappy.calibrate.pupil.covered_z_nm, smappy.calibrate.psf_methods.InSitu, smappy.calibrate.psf_methods.EllipticalGaussian, smappy.calibrate.elliptical.bead_widths, smappy.calibrate.elliptical.align_beads, smappy.calibrate.elliptical.calibrate_widths, smappy.calibrate.elliptical.save_path, smappy.calibrate.elliptical.EllipticalCalibrationSettings, smappy.calibrate.insitu_run.run_insitu, smappy.calibrate.insitu_run.movie_rois, smappy.calibrate.insitu_run.block_order, smappy.calibrate.insitu_run.enough, smappy.calibrate.insitu_run.start_astigmatism, smappy.calibrate.insitu_run.held_pupil, smappy.microscope_db.gamma_of, smappy.calibrate.insitu.quotas, smappy.calibrate.insitu_run.insitu_spline, smappy.calibrate.insitu_run.InSituModelSettings, smappy.calibrate.insitu.InSituSettings, smappy.calibrate.insitu.select, smappy.calibrate.insitu_views.draw_z_bins, smappy.calibrate.insitu_views.draw_rounds, smappy.calibrate.insitu_views.model_vs_data, uipsf.workflows.insitu_rounds.calibrate_insitu, uipsf.workflows.insitu_rounds.coverslip_edge, uipsf.workflows.insitu_rounds.focus_from_data, uipsf.workflows.insitu.twin, smappy.calibrate.psf_methods.FourPiPupil, smappy.calibrate.psf_methods.FourPiAverage, smappy.calibrate.psf_methods.for_mode, smappy.calibrate.fourpi_beads.average_model, smappy.fourpi.iab_from_beads, smappy.calibrate.fourpi_beads.scan_beads, smappy.calibrate.fourpi_beads.read_scan, smappy.calibrate.fourpi_beads.scan_layout, smappy.calibrate.fourpi_beads.scan_step_nm, smappy.calibrate.fourpi_beads.bead_fits, smappy.calibrate.fourpi_beads.FourPiBeadSettings, smappy.calibrate.psf_methods.MoleculeStack, smappy.calibrate.psf_methods.MoleculeElliptical, smappy.calibrate.psf_methods.methods_of, smappy.calibrate.molecule_stack.molecule_rois, smappy.calibrate.molecule_stack.read_file, smappy.calibrate.molecule_stack.stage_z_um, smappy.calibrate.molecule_stack.split_sweeps, smappy.calibrate.molecule_stack.width_curve, smappy.calibrate.molecule_stack.focus_of, smappy.calibrate.molecule_stack.detect, smappy.calibrate.molecule_stack.select, smappy.calibrate.molecule_stack.draw, smappy.calibrate.molecule_stack.illumination_radius, smappy.calibrate.molecule_stack.molecule_widths, smappy.calibrate.molecule_stack.MoleculeStackSettings, smappy.calibrate.molecule_dual.dual_molecule_rois, smappy.calibrate.molecule_dual.registration_from_molecules, smappy.calibrate.molecule_stack_run.run_molecule_stack, smappy.calibrate.molecule_stack_run.spline_range, smappy.calibrate.molecule_stack_run.estimate_seconds, smappy.calibrate.molecule_views.draw_selection, smappy.calibrate.molecule_views.draw_z_check, smappy.calibrate.molecule_views.draw_foci, smappy.calibrate.molecule_views.model_vs_data, uipsf.workflows.molecule_stack.learn_molecule_stack, uipsf.workflows.molecule_stack.z_check, uipsf.workflows.molecule_stack.molecule_chunk, uipsf.model.geometry.CoverslipStackGeometry] +covers: [smappy.calibrate.pupil.half_split_uncertainty, smappy.calibrate.pupil.fit_quality, smappy.calibrate.plots.draw_pupil_fit_quality, smappy.torch_install.command, smappy.torch_install.cpu_build_on_nvidia, uipsf.device.describe, smappy.calibrate.psf_methods.BeadAverage, smappy.calibrate.psf_methods.PupilModel, smappy.calibrate.pupil.run_calibration, smappy.calibrate.pupil.bead_rois, smappy.calibrate.pupil.single_rois, smappy.calibrate.pupil.dual_rois, smappy.calibrate.pupil.spline_calibration, smappy.calibrate.pupil.dual_calibration, smappy.calibrate.pupil.save, smappy.calibrate.pupil.OpticsSettings, smappy.calibrate.pupil.BeadSettings, smappy.calibrate.pupil.learns_tilt, uipsf.diagnostics.figures.coverslip_tilt, smappy.calibrate.core.find_stack_beads, smappy.calibrate.psf_window.PSFCalibrationWindow, smappy.calibrate.pupil.estimate_seconds, smappy.calibrate.psf_methods.LearningProgress, smappy.calibrate.model_views.average_views, smappy.calibrate.model_views.pupil_views, smappy.calibrate.model_views.draw_model_vs_data, smappy.calibrate.model_views.planes, smappy.calibrate.stack_browser.StackBrowser, smappy.calibrate.stack_view.StackViewer, smappy.calibrate.pupil.covered_z_nm, smappy.calibrate.psf_methods.InSitu, smappy.calibrate.psf_methods.EllipticalGaussian, smappy.calibrate.elliptical.bead_widths, smappy.calibrate.elliptical.align_beads, smappy.calibrate.elliptical.calibrate_widths, smappy.calibrate.elliptical.save_path, smappy.calibrate.elliptical.EllipticalCalibrationSettings, smappy.calibrate.insitu_run.run_insitu, smappy.calibrate.insitu_run.movie_rois, smappy.calibrate.insitu_run.block_order, smappy.calibrate.insitu_run.enough, smappy.calibrate.insitu_run.start_astigmatism, smappy.calibrate.insitu_run.held_pupil, smappy.microscope_db.gamma_of, smappy.calibrate.insitu.quotas, smappy.calibrate.insitu_run.insitu_spline, smappy.calibrate.insitu_run.InSituModelSettings, smappy.calibrate.insitu.InSituSettings, smappy.calibrate.insitu.select, smappy.calibrate.insitu_views.draw_z_bins, smappy.calibrate.insitu_views.draw_rounds, smappy.calibrate.insitu_views.model_vs_data, uipsf.workflows.insitu_rounds.calibrate_insitu, uipsf.workflows.insitu_rounds.coverslip_edge, uipsf.workflows.insitu_rounds.focus_from_data, uipsf.workflows.insitu.twin, smappy.calibrate.psf_methods.FourPiPupil, smappy.calibrate.psf_methods.FourPiAverage, smappy.calibrate.psf_methods.for_mode, smappy.calibrate.fourpi_beads.average_model, smappy.fourpi.iab_from_beads, smappy.calibrate.fourpi_beads.scan_beads, smappy.calibrate.fourpi_beads.read_scan, smappy.calibrate.fourpi_beads.scan_layout, smappy.calibrate.fourpi_beads.scan_step_nm, smappy.calibrate.fourpi_beads.bead_fits, smappy.calibrate.fourpi_beads.FourPiBeadSettings, smappy.calibrate.psf_methods.MoleculeStack, smappy.calibrate.psf_methods.MoleculeElliptical, smappy.calibrate.psf_methods.methods_of, smappy.calibrate.molecule_stack.molecule_rois, smappy.calibrate.molecule_stack.read_file, smappy.calibrate.molecule_stack.stage_z_um, smappy.calibrate.molecule_stack.split_sweeps, smappy.calibrate.molecule_stack.width_curve, smappy.calibrate.molecule_stack.focus_of, smappy.calibrate.molecule_stack.detect, smappy.calibrate.molecule_stack.select, smappy.calibrate.molecule_stack.draw, smappy.calibrate.molecule_stack.illumination_radius, smappy.calibrate.molecule_stack.molecule_widths, smappy.calibrate.molecule_stack.MoleculeStackSettings, smappy.calibrate.molecule_dual.dual_molecule_rois, smappy.calibrate.molecule_dual.registration_from_molecules, smappy.calibrate.molecule_stack_run.run_molecule_stack, smappy.calibrate.molecule_stack_run.spline_range, smappy.calibrate.molecule_stack_run.estimate_seconds, smappy.calibrate.molecule_views.draw_selection, smappy.calibrate.molecule_views.draw_z_check, smappy.calibrate.molecule_views.draw_foci, smappy.calibrate.molecule_views.model_vs_data, uipsf.workflows.molecule_stack.learn_molecule_stack, uipsf.workflows.molecule_stack.z_check, uipsf.workflows.molecule_stack.molecule_chunk, uipsf.model.geometry.CoverslipStackGeometry] --- ## What it does @@ -83,7 +83,12 @@ with one too old, *Install PyTorch* installs the right build for the computer (for Apple's GPU, an NVIDIA card or the CPU) into the Python running SMAPpy; the window also shows the command to type instead. The bead average works without it. It uses the -GPU where PyTorch finds one (NVIDIA, or Apple's). +GPU where PyTorch finds one (NVIDIA, or Apple's), and the line *Detect* writes +says where: *learnt on the graphics card*, *on Apple's GPU* or *on the CPU*. +A PyTorch built for the CPU only on a computer with an NVIDIA card -- a plain +`pip install torch` on Windows brings that one -- works, several times +slower; the window says so and offers *Install PyTorch* to replace it with +the card's build. ## How it works @@ -1266,9 +1271,11 @@ beads can be minutes away. Nothing is saved, and the beads are neither scored nor drawn against the model. ### Install PyTorch -Shown with the pupil model while PyTorch is missing or too old: installs it -into the Python running SMAPpy, with pip's progress in the status line; -pressed again, it stops. The pupil model can be used as soon as it is done. +Shown with the pupil model while PyTorch is missing or too old, or built for +the CPU only beside an NVIDIA card: installs it into the Python running +SMAPpy, with pip's progress in the status line; pressed again, it stops. The +pupil model can be used as soon as it is done -- after replacing a CPU build +once SMAPpy is started again, if a calibration had already loaded the old one. ### Save as... Saves the calibration under another name, or one made with *save model* off. diff --git a/src/smappy/torch_install.py b/src/smappy/torch_install.py index 0f5933ec..a55b808b 100644 --- a/src/smappy/torch_install.py +++ b/src/smappy/torch_install.py @@ -20,6 +20,12 @@ Nothing here imports torch: what is installed is read from its package metadata, so asking costs nothing and an old torch is not loaded into the process before it is replaced. + +**A build for the CPU on a machine with an NVIDIA card** (`cpu_build_on_nvidia`) works, and learns several +times slower without saying so: uipsf takes the card only when torch was built for it (`uipsf.device`). It +happens when torch came from elsewhere than the window -- a plain ``pip install torch`` on Windows, whose PyPI +wheel is CPU only. The window then warns and offers the button again (`replacement`); its command replaces +the build rather than finding the installed version good enough. """ from __future__ import annotations @@ -56,6 +62,38 @@ def problem() -> Optional[str]: return None +def _installed() -> Optional[str]: + from importlib import metadata + try: + return metadata.version("torch") + except metadata.PackageNotFoundError: + return None + + +def cpu_build_on_nvidia(version: Optional[str] = None, system: Optional[str] = None, + nvidia: Optional[bool] = None) -> bool: + """Whether the installed torch is built for the CPU only on a machine with an NVIDIA card (module docstring), + read from its version's local label: PyTorch's CPU index marks its wheels ``+cpu``; on Windows a CUDA build + is always ``+cu…`` (PyPI's wheel, without a label, is CPU only), on Linux PyPI's unlabelled wheel is CUDA's.""" + version = _installed() if version is None else version + system = system or platform.system() + nvidia = has_nvidia() if nvidia is None else nvidia + if not version or not nvidia or system == "Darwin": + return False + label = version.partition("+")[2].lower() + if label == "cpu": + return True + return system == "Windows" and not label.startswith("cu") + + +def replacement() -> Optional[str]: + """What the window says when torch works but leaves the NVIDIA card unused; None when it does not.""" + if problem() is not None or not cpu_build_on_nvidia(): + return None + return (f"PyTorch {_installed()} is built for the CPU only, though this computer has an NVIDIA card: the pupil " + f"model learns on the CPU, several times slower") + + def has_nvidia() -> bool: return shutil.which("nvidia-smi") is not None @@ -84,11 +122,20 @@ def installer(has_pip: Optional[bool] = None, uv: Optional[str] = None) -> List[ def command(system: Optional[str] = None, nvidia: Optional[bool] = None, - start: Optional[List[str]] = None) -> List[str]: - """The install command, as a list for a process, that installs the right torch here.""" + start: Optional[List[str]] = None, replace: bool = False) -> List[str]: + """The install command, as a list for a process, that installs the right torch here. + + ``replace``: a CPU build is there and the card's is wanted (`cpu_build_on_nvidia`). ``--upgrade`` alone + can keep it -- PyPI's unlabelled 2.x sorts below a 2.x+cpu, and an installed version newer than the CUDA + index's is "satisfied" -- so the build is reinstalled from the CUDA index. On Windows without its + dependencies, which a CUDA wheel there shares with the CPU one (it carries CUDA inside); on Linux the CUDA + wheel needs NVIDIA's packages, so only the higher local label (+cu126 above +cpu) moves it.""" system = system or platform.system() nvidia = has_nvidia() if nvidia is None else nvidia pip = (start or installer()) + ["--upgrade", f"torch>={'.'.join(map(str, MINIMUM))}"] + if replace and nvidia and system in ("Windows", "Linux"): + extra = ["--force-reinstall", "--no-deps"] if system == "Windows" else [] + return pip + extra + ["--index-url", CUDA_INDEX] if system == "Linux" and not nvidia: return pip + ["--index-url", CPU_INDEX] if system == "Windows" and nvidia: diff --git a/src/uipsf/device.py b/src/uipsf/device.py index 6a545638..16627696 100644 --- a/src/uipsf/device.py +++ b/src/uipsf/device.py @@ -32,6 +32,21 @@ def default_device() -> torch.device: return torch.device("cpu") +def describe(device=None) -> str: + """Where the learning runs, for a person: "on the graphics card (NVIDIA GeForce RTX 3070 Ti)", "on Apple's + GPU" or "on the CPU". The PSF calibration window says it after *Detect*: a CPU build of torch on a + machine with a card learnt on the CPU, several times slower, and nothing said so.""" + d = default_device() if device is None else torch.device(device) + if d.type == "cuda": + try: + return f"on the graphics card ({torch.cuda.get_device_name(d)})" + except Exception: # a name is a courtesy, never a reason to fail + return "on the graphics card" + if d.type == "mps": + return "on Apple's GPU" + return "on the CPU" + + def resolve(device=None, dtype: torch.dtype = torch.float32) -> tuple[torch.device, torch.dtype]: """The device to use, refusing float64 where the hardware has none (MPS).""" device = default_device() if device is None else torch.device(device) diff --git a/tests/test_torch_install.py b/tests/test_torch_install.py index 5d4c3c57..357274b3 100644 --- a/tests/test_torch_install.py +++ b/tests/test_torch_install.py @@ -52,3 +52,68 @@ def test_the_window_offers_the_install_only_for_the_pupil_model_and_only_while_t window._show_method() assert not window.install_button.isVisibleTo(window) and window.run_button.isEnabled() window.close() + + +def test_a_cpu_build_on_a_machine_with_an_nvidia_card_is_found_from_its_version(): + found = torch_install.cpu_build_on_nvidia + assert found("2.14.1", "Windows", nvidia=True) # PyPI's Windows wheel is the CPU's + assert not found("2.14.1+cu126", "Windows", nvidia=True) + assert found("2.14.1+cpu", "Linux", nvidia=True) + assert not found("2.14.1", "Linux", nvidia=True) # PyPI's Linux wheel is CUDA's + assert not found("2.14.1", "Windows", nvidia=False) # no card: the CPU is all there is + assert not found("2.14.1", "Darwin", nvidia=False) + + +def test_replacing_a_cpu_build_reinstalls_from_the_cuda_index(): + windows = torch_install.command("Windows", nvidia=True, replace=True) + assert windows[-4:] == ["--force-reinstall", "--no-deps", "--index-url", torch_install.CUDA_INDEX] + linux = torch_install.command("Linux", nvidia=True, replace=True) + assert linux[-2:] == ["--index-url", torch_install.CUDA_INDEX] and "--no-deps" not in linux + assert torch_install.command("Darwin", nvidia=False, replace=True) == torch_install.command("Darwin", nvidia=False) + + +def test_the_window_warns_of_a_cpu_build_beside_a_card_offers_the_install_and_still_runs(monkeypatch): + import pytest + pytest.importorskip("PySide6") + from PySide6.QtWidgets import QApplication + + from smappy.calibrate.psf_window import PSFCalibrationWindow + QApplication.instance() or QApplication([]) + monkeypatch.setattr(torch_install, "problem", lambda: None) + monkeypatch.setattr(torch_install, "_installed", lambda: "2.14.1") + monkeypatch.setattr(torch_install, "has_nvidia", lambda: True) + monkeypatch.setattr(torch_install.platform, "system", lambda: "Windows") + window = PSFCalibrationWindow() + window.method_box.setCurrentText("pupil model (uiPSF)") + assert window.install_button.isVisibleTo(window) and window.run_button.isEnabled() + assert "built for the CPU only" in window.notice.text() and torch_install.CUDA_INDEX in window.notice.text() + monkeypatch.setattr(torch_install, "has_nvidia", lambda: False) + window._show_method() + assert not window.install_button.isVisibleTo(window) and not window.notice.isVisibleTo(window) + window.close() + + +def test_after_detect_the_window_says_where_the_pupil_model_learns(monkeypatch): + import pytest + pytest.importorskip("PySide6") + from PySide6.QtWidgets import QApplication + + from smappy.calibrate import psf_window + from smappy.calibrate.psf_methods import Detected + QApplication.instance() or QApplication([]) + monkeypatch.setattr(torch_install, "problem", lambda: None) + monkeypatch.setattr(psf_window, "learning_device", lambda: "on the graphics card (a test card)") + window = psf_window.PSFCalibrationWindow() + window.method_box.setCurrentText("pupil model (uiPSF)") + window.method = type("M", (), {"key": "pupil"})() # its own text, not a method's detected_text + window.detected = Detected(None, "12 beads to learn from") + window._show_detected_text() + assert window.found_label.text() == "12 beads to learn from; learnt on the graphics card (a test card)" + window.close() + + +def test_the_device_is_described_for_a_person(): + import pytest + pytest.importorskip("torch") + from uipsf.device import describe + assert describe("cpu") == "on the CPU" and describe("mps") == "on Apple's GPU" From 59f6ecd57738d55f1dd7287133d512a286492396 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 12:11:31 +0000 Subject: [PATCH 4/5] uipsf M2 pairs: scripts for the tilt, the focus drift and the score per height, from the molecules' heights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit heights.py localises every molecule of the nucleus movie once per model and saves height, CRLB, ½ Σ deviance, chip position and frame; tilt_from_edge.py fits the coverslip's edge per tile and a plane through it (a real 1.6° tilt is ~28 nm per µm), edge_vs_time.py the edge per block of frames, score_by_height.py two models' scores per height bin on the unseen molecules. Smoke-tested here on synthetic files (a planted 1.6° tilt read as 1.63°, a planted drift and a height-dependent score recovered) and heights.py on a small simulated movie; not yet run on M2's data, which are not in git. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014JegwAEzHHExiXkwvXgA2j --- .../uipsf/real/m2_pairs/edge_vs_time.py | 98 ++++++++++++++ benchmarks/uipsf/real/m2_pairs/heights.py | 112 ++++++++++++++++ .../uipsf/real/m2_pairs/score_by_height.py | 83 ++++++++++++ .../uipsf/real/m2_pairs/tilt_from_edge.py | 121 ++++++++++++++++++ docs/uipsf/architecture.md | 12 ++ 5 files changed, 426 insertions(+) create mode 100644 benchmarks/uipsf/real/m2_pairs/edge_vs_time.py create mode 100644 benchmarks/uipsf/real/m2_pairs/heights.py create mode 100644 benchmarks/uipsf/real/m2_pairs/score_by_height.py create mode 100644 benchmarks/uipsf/real/m2_pairs/tilt_from_edge.py diff --git a/benchmarks/uipsf/real/m2_pairs/edge_vs_time.py b/benchmarks/uipsf/real/m2_pairs/edge_vs_time.py new file mode 100644 index 00000000..a841caa4 --- /dev/null +++ b/benchmarks/uipsf/real/m2_pairs/edge_vs_time.py @@ -0,0 +1,98 @@ +"""Did the focus drift over the movie? The coverslip's edge per block of time, one model held. + + python edge_vs_time.py OUTDIR MODEL [MODEL ...] [--blocks 10] [--json FILE] + +The nucleus movie is ~72 000 frames under the focus lock. The in situ models learn one focus for all of it; if +the focus wandered, the molecules' heights wandered with it, the depth signal that tells the even aberrations +apart is blurred, and the learning may put the blur into the pupil (an open hypothesis of architecture.md, "In +situ against molecule stacks on M2"). With the pupil held, a drift of the focus shows directly: the molecules +on the coverslip are found at a height that moves. Each block's edge (`coverslip_edge`) is that height. + +Per model (``heights_.npz`` from `heights.py`): the units ordered by movie and frame, cut into ``blocks`` +of equal count, each block's edge with its error, the spike fraction, and the median and 90th percentile of its +heights; then a weighted line through the edges, in nm per 10 000 frames. The frame is counted across the movies +(each movie's frames after the previous movie's last frame seen), so the line's axis is the acquisition's order. + +What to read: edges that stay within ~10 nm say the focus held; a trend or a jump of tens of nm says it did not, +and then a focus per block (or per movie) in the learning is the next thing to try. The median height follows +where the molecules are as well as the focus, so it is shown, not fitted. +""" +import argparse +import json +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from heights import coverslip_edge, load # noqa: E402 + + +def acquisition_frame(frame, source) -> np.ndarray: + """The frame counted across the movies: each movie after the previous one's last frame seen.""" + frame, source = np.asarray(frame, int), np.asarray(source, int) + out = frame.copy() + offset = 0 + for s in np.unique(source): + sel = source == s + out[sel] = frame[sel] + offset + offset += int(frame[sel].max()) + 1 + return out + + +def block_edges(t, h, sd, blocks: int): + order = np.argsort(t, kind="stable") + rows = [] + for part in np.array_split(order, blocks): + e = coverslip_edge(h[part], sd[part]) + if e is None or not np.isfinite(e.sd_nm): + continue + rows.append(dict(frames=(int(t[part].min()), int(t[part].max())), t=float(np.median(t[part])), + edge_nm=e.shift_nm, sd_nm=e.sd_nm, spike=e.spike, n=len(part), + median_nm=float(np.nanmedian(h[part])), p90_nm=float(np.nanpercentile(h[part], 90)))) + return rows + + +def trend(rows): + """Weighted line edge = a + b·t: b in nm per 10 000 frames, its error (χ²-scaled), χ² per degree of freedom.""" + t = np.array([r["t"] for r in rows]) / 1e4 + z = np.array([r["edge_nm"] for r in rows]) + w = 1 / np.maximum(np.array([r["sd_nm"] for r in rows]), 1.0) ** 2 + A = np.c_[np.ones_like(t), t] + cov = np.linalg.inv(A.T @ (A * w[:, None])) + coef = cov @ (A.T @ (w * z)) + dof = max(len(z) - 2, 1) + chi2 = float(np.sum(w * (z - A @ coef) ** 2) / dof) + return float(coef[1]), float(np.sqrt(cov[1, 1] * max(chi2, 1.0))), chi2 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("out") + ap.add_argument("names", nargs="+") + ap.add_argument("--blocks", type=int, default=10) + ap.add_argument("--json") + a = ap.parse_args() + for name in a.names: + d = load(a.out, name) + t = acquisition_frame(d["frame"], d["source"]) + rows = block_edges(t, d["h"], d["sd_h"], a.blocks) + if len(rows) < 3: + print(f"{name}: only {len(rows)} blocks with an edge; fewer --blocks") + continue + slope, err, chi2 = trend(rows) + edges = np.array([r["edge_nm"] for r in rows]) + print(f"\n{name}: {len(rows)} blocks; edge {slope:+.1f} ± {err:.1f} nm per 10 000 frames, range " + f"{edges.min():+.0f} to {edges.max():+.0f} nm, χ²/dof about the line {chi2:.1f}") + for r in rows: + print(f" frames {r['frames'][0]:6d}-{r['frames'][1]:6d}: edge {r['edge_nm']:+7.1f} ± {r['sd_nm']:5.1f} nm" + f" spike {r['spike']:.2f} median {r['median_nm']:6.0f} p90 {r['p90_nm']:6.0f} n {r['n']}") + if a.json: + with open(a.json, "a") as f: + f.write(json.dumps(dict(kind="edge_vs_time", model=name, blocks=a.blocks, + slope_nm_per_10k_frames=slope, slope_err=err, chi2_dof=chi2, + block_rows=rows)) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/uipsf/real/m2_pairs/heights.py b/benchmarks/uipsf/real/m2_pairs/heights.py new file mode 100644 index 00000000..a23a1289 --- /dev/null +++ b/benchmarks/uipsf/real/m2_pairs/heights.py @@ -0,0 +1,112 @@ +"""Every molecule of M2's nucleus movie localised once per model, saved, for the analyses that read heights. + + python heights.py OUTDIR MODEL [MODEL ...] [--states "used,held out,not drawn"] + +Writes ``OUTDIR/heights_.npz`` per model: each unit's id and state, its height above the coverslip +``h`` (nm, the in situ geometry's), the height's CRLB ``sd_h``, ½ Σ deviance (the score `cross_score.py` uses), +photons, its place on the chip (``x_px``, ``y_px``) and its frame and movie. Each unit is localised from +`STARTS_NM` with x, y, z, photons and background free, the best start kept (`cross_score.insitu_scores`'s +procedure). A MODEL is a saved ``_3dcal.h5`` in OUTDIR; an in situ model uses its own focus, so a stack +model enters through the in situ run that held every aberration of it (``insitu_all_n1345_ab_n1345``). + +`movie_rois` with its defaults is deterministic, so every model's file holds the same units in the same order +(``units`` is checked by the readers). Units the selection rejected are not localised: they are pairs, +neighbours or off centre, and would only add noise to an edge. + +The readers: `tilt_from_edge.py` (the coverslip's edge per tile of the field), `edge_vs_time.py` (per time +block) and `score_by_height.py` (two models' scores per height bin). +""" +import argparse +import glob +import os + +import numpy as np +import torch + +# the data are not in git: M2_DATA names the folder that holds 261112_new's sets +ROOT = os.environ.get("M2_DATA", os.path.expanduser("~/Library/Mobile Documents/com~apple~CloudDocs/Vienna/Data/" + "smappy test/261112_new")) +MOVIE = "261112_MR_Intercalib2_BGAAF647_50ms_30i_nucleus_1/*_MMStack_Pos0.ome.tif" +PIXEL_NM = (127.0, 116.0) # iXon on M2's astigmatic path (docs/uipsf/microscopes.md) +STARTS_NM = (-200.0, 200.0, 600.0, 1000.0, 1400.0) +STATES = ("used", "held out", "not drawn") + + +def localise_all(model, shared: dict, batch, noise, starts=STARTS_NM) -> dict: + """Each emitter of ``batch`` localised from every height in ``starts``, the best (lowest ½ Σ deviance) + kept: {"h", "sd_h", "half_deviance", "photons"}, numpy [E].""" + from uipsf.solve import Problem + from uipsf.workflows.insitu import INSITU_CHUNK + from uipsf.workflows.scoring import localise + + E = batch.n_emitters + out = {k: np.full(E, np.nan) for k in ("h", "sd_h", "photons")} + out["half_deviance"] = np.full(E, np.inf) + for z0 in starts: + xyz = torch.zeros(E, 3, dtype=torch.float64) + xyz[:, 2] = z0 + fit, problem = localise(model, shared, batch, noise, start={"emitter.xyz": xyz}, chunk_size=INSITU_CHUNK) + dev = Problem(model, noise, problem.batch, INSITU_CHUNK).deviance(fit.params) + e = 0.5 * dev.reshape(E, -1).sum(1).double().cpu().numpy() + phys = fit.params.physical() + photons = phys["emitter.photons"] if "emitter.photons" in phys else phys["emitter.photons_per_slice"].mean(1) + better = e < out["half_deviance"] + out["half_deviance"][better] = e[better] + out["h"][better] = phys["emitter.xyz"][:, 2].detach().cpu().double().numpy()[better] + out["sd_h"][better] = fit.std["emitter.xyz"][:, 2].detach().cpu().double().numpy()[better] + out["photons"][better] = photons.detach().cpu().double().numpy()[better] + return out + + +def unit_table(rois, ids) -> dict: + """Where and when each unit ``ids`` of ``rois`` (`InSituROIs`) is: its state, chip pixel, first frame, + movie.""" + u = rois.units + source = np.zeros(len(u.reason), int) if u.source is None else np.asarray(u.source) + return {"units": np.asarray(ids), "state": np.array([rois.state[i] for i in ids]), + "x_px": np.asarray(u.x_px)[ids], "y_px": np.asarray(u.y_px)[ids], + "frame": np.array([int(np.min(u.frames[i])) for i in ids]), "source": source[ids]} + + +def path(out: str, name: str) -> str: + return os.path.join(out, f"heights_{name}.npz") + + +def load(out: str, name: str) -> dict: + with np.load(path(out, name), allow_pickle=False) as f: + return {k: f[k] for k in f.files} + + +def coverslip_edge(h, sd): + """The heights' lower edge (`uipsf.workflows.insitu_rounds.coverslip_edge`, the rounds' own fit), or None.""" + from uipsf.workflows.insitu_rounds import coverslip_edge as edge + ok = np.isfinite(h) & np.isfinite(sd) & (sd > 0) + return edge(np.asarray(h)[ok], np.asarray(sd)[ok]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("out") + ap.add_argument("names", nargs="+") + ap.add_argument("--states", default=",".join(STATES), help="comma-separated unit states to localise") + a = ap.parse_args() + from smappy.calibrate.insitu_run import movie_rois + from smappy.calibrate.pupil import load_start + + rois = movie_rois(sorted(glob.glob(os.path.join(ROOT, MOVIE)))) + states = [s.strip() for s in a.states.split(",")] + ids = np.sort(np.concatenate([rois.ids(s) for s in states])) + batch, noise = rois.batch(ids=ids) + table = unit_table(rois, ids) + print(f"{len(ids)} units ({', '.join(f'{s}: {len(rois.ids(s))}' for s in states)})", flush=True) + for name in a.names: + m = load_start(os.path.join(a.out, f"{name}_3dcal.h5")) + shared = {k: torch.as_tensor(v) for k, v in m.shared.items()} + found = localise_all(m.model, shared, batch, noise) + np.savez(path(a.out, name), **table, **found, focus_nm=float(torch.as_tensor(shared["geometry.focus"]))) + print(f" {name:44s} heights p5/50/95 {np.nanpercentile(found['h'], [5, 50, 95]).round(0).tolist()} nm, " + f"½Σdeviance {found['half_deviance'].mean():.2f} per unit -> {path(a.out, name)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/uipsf/real/m2_pairs/score_by_height.py b/benchmarks/uipsf/real/m2_pairs/score_by_height.py new file mode 100644 index 00000000..ff2acdc3 --- /dev/null +++ b/benchmarks/uipsf/real/m2_pairs/score_by_height.py @@ -0,0 +1,83 @@ +"""Where in z does each pupil fit the cell better? Models' scores on the same molecules, per height bin. + + python score_by_height.py OUTDIR REFERENCE MODEL [MODEL ...] [--bins -200,0,200,...] [--json FILE] + +`cross_score.py` found that on the nucleus movie's 223 *held out* molecules the in situ pupils beat the stack's +(held, the focus learnt: ``insitu_all_n1345_ab_n1345``) by 1.75-2.75 nats per molecule, while on the 4105 +*not drawn* -- the z bins' surplus, crowded where molecules are plentiful -- they lose by 0.36-0.50. Which +heights the surplus crowds at, and so where the stack's pupil is the better one, was not checked. This script +does: an excess the in situ pupils carry because of depth (the index, the nucleus) would win deep and lose near +the glass; one from something else would not depend on height. + +Every MODEL's ½ Σ deviance per molecule (``heights_.npz`` from `heights.py`), each molecule placed in a +height bin by REFERENCE's height (so a bin holds the same molecules for every model), and per bin the mean +difference against the first MODEL, with its standard error and count, for the *held out* and *not drawn* +molecules apart and together. The *used* are left out: the in situ models learnt from them. + +Negative: the model fits that bin's molecules better than the first MODEL (nats per molecule; 1 ≈ moving each +molecule by 1.4 CRLB). +""" +import argparse +import json +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from heights import load # noqa: E402 + +BINS_NM = (-200.0, 0.0, 200.0, 400.0, 600.0, 800.0, 1000.0, 1200.0, 1600.0) +UNSEEN = ("held out", "not drawn") + + +def by_bin(height, delta, edges): + """Per bin of ``height``: (count, mean of ``delta``, its standard error).""" + idx = np.digitize(height, edges) - 1 + out = [] + for b in range(len(edges) - 1): + d = delta[(idx == b) & np.isfinite(delta)] + out.append((len(d), float(d.mean()) if len(d) else float("nan"), + float(d.std(ddof=1) / np.sqrt(len(d))) if len(d) > 1 else float("nan"))) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("out") + ap.add_argument("reference", help="the model whose heights place the molecules in bins") + ap.add_argument("names", nargs="+", help="models to compare; differences against the first") + ap.add_argument("--bins", default=",".join(f"{b:g}" for b in BINS_NM)) + ap.add_argument("--json") + a = ap.parse_args() + edges = np.array([float(b) for b in a.bins.split(",")]) + ref = load(a.out, a.reference) + data = {n: load(a.out, n) for n in a.names} + for n, d in data.items(): + if not np.array_equal(d["units"], ref["units"]): + raise SystemExit(f"{n} holds other units than {a.reference}: run heights.py with the same --states") + state, height = ref["state"], ref["h"] + base = data[a.names[0]]["half_deviance"] + labels = [f"{edges[b]:g}..{edges[b + 1]:g}" for b in range(len(edges) - 1)] + for n in a.names[1:]: + delta = data[n]["half_deviance"] - base + print(f"\n{n} - {a.names[0]}, nats per molecule by {a.reference}'s height (nm):") + print(f" {'height':>12s} " + " ".join(f"{s + ' (n)':>22s}" for s in (*UNSEEN, "both"))) + parts = {s: by_bin(height[state == s], delta[state == s], edges) for s in UNSEEN} + unseen = np.isin(state, UNSEEN) + parts["both"] = by_bin(height[unseen], delta[unseen], edges) + for b, label in enumerate(labels): + cells = [] + for s in (*UNSEEN, "both"): + k, m, se = parts[s][b] + cells.append(f"{m:+7.2f} ± {se:4.2f} ({k:5d})" if k else f"{'-':>22s}") + print(f" {label:>12s} " + " ".join(cells)) + if a.json: + with open(a.json, "a") as f: + f.write(json.dumps(dict(kind="score_by_height", model=n, against=a.names[0], + reference=a.reference, bins=edges.tolist(), + **{s.replace(" ", "_"): parts[s] for s in parts})) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/uipsf/real/m2_pairs/tilt_from_edge.py b/benchmarks/uipsf/real/m2_pairs/tilt_from_edge.py new file mode 100644 index 00000000..af2bb050 --- /dev/null +++ b/benchmarks/uipsf/real/m2_pairs/tilt_from_edge.py @@ -0,0 +1,121 @@ +"""Is the coverslip tilted? Its edge measured per tile of the field, a plane through the tiles, no pupil tilt. + + python tilt_from_edge.py OUTDIR MODEL [MODEL ...] [--tiles 3] [--stack STACK_MODEL] [--json FILE] + +The in situ runs on M2's nucleus movie learn a coverslip tilt of 1.6° about y (2.9° with every aberration of +the stack held), which takes most of their excess astigmatism, coma and spherical over the molecule stack +(architecture.md, "In situ against molecule stacks on M2"). That is a tilt of the interfaces in the *pupil*. +A real tilt would also raise the coverslip across the field: the field is ~26 µm wide, so 1.6° is ~28 nm per µm, +~700 nm across. The molecules' heights show that without any pupil: the lowest molecules of each tile lie on the +coverslip, and their edge (`coverslip_edge`, the rounds' own fit) moves with it. + +Per model (``heights_.npz`` from `heights.py`): the field cut into ``tiles`` × ``tiles``, each tile's edge +and its error, then a weighted plane through the tiles, edge = a + b_x·u + b_y·v (u, v in µm from the field's +centre). Reported: b_x, b_y in nm per µm with their errors (scaled by the fit's χ² where it exceeds 1), the +slope's size as an angle, and each tile's edge. A model without a learnt tilt should be used (the edge then +carries the whole tilt); the edge's slope barely depends on the pupil, since it is the molecules' positions. + +``--stack``: the molecule stack's own focal-plane slope (``geometry.plane``, nm per µm, `CoverslipStackGeometry`) +printed beside it -- the coverslip's tilt as the z sweeps saw it, two hours later. + +What to read: a slope of ~28-50 nm per µm along x would say the tilt is real; a slope within a few nm per µm +of 0 says the learnt tilt stands in for something else (the nucleus' index, focus drift, the pores' own +orientation). A tile without molecules on the glass (its spike fraction near 0) gives the lower edge of the +sample instead: inside a thick nucleus that is not the coverslip, so look at the spike column. +""" +import argparse +import json +import math +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from heights import PIXEL_NM, coverslip_edge, load # noqa: E402 + +MIN_PER_TILE = 60 + + +def tile_edges(x_px, y_px, h, sd, tiles: int, pixel_nm=PIXEL_NM): + """Each tile's (u, v) centre in µm from the field's centre, its edge, error, spike fraction and count.""" + x0, x1 = np.nanmin(x_px), np.nanmax(x_px) + 1e-9 + y0, y1 = np.nanmin(y_px), np.nanmax(y_px) + 1e-9 + cx, cy = (x0 + x1) / 2, (y0 + y1) / 2 + ix = np.minimum(((x_px - x0) / (x1 - x0) * tiles).astype(int), tiles - 1) + iy = np.minimum(((y_px - y0) / (y1 - y0) * tiles).astype(int), tiles - 1) + rows = [] + for j in range(tiles): + for i in range(tiles): + sel = (ix == i) & (iy == j) + if sel.sum() < MIN_PER_TILE: + continue + e = coverslip_edge(h[sel], sd[sel]) + if e is None or not np.isfinite(e.sd_nm): + continue + u = (np.mean(x_px[sel]) - cx) * pixel_nm[0] / 1000 + v = (np.mean(y_px[sel]) - cy) * pixel_nm[1] / 1000 + rows.append(dict(tile=(i, j), u_um=float(u), v_um=float(v), edge_nm=e.shift_nm, sd_nm=e.sd_nm, + spike=e.spike, n=int(sel.sum()))) + return rows + + +def plane(rows): + """Weighted least squares edge = a + b_x u + b_y v: (a, b_x, b_y), their errors and χ² per degree of freedom.""" + u = np.array([r["u_um"] for r in rows]) + v = np.array([r["v_um"] for r in rows]) + z = np.array([r["edge_nm"] for r in rows]) + w = 1 / np.maximum(np.array([r["sd_nm"] for r in rows]), 1.0) ** 2 + A = np.c_[np.ones_like(u), u, v] + cov = np.linalg.inv(A.T @ (A * w[:, None])) + coef = cov @ (A.T @ (w * z)) + dof = max(len(z) - 3, 1) + chi2 = float(np.sum(w * (z - A @ coef) ** 2) / dof) + err = np.sqrt(np.diag(cov) * max(chi2, 1.0)) + return coef, err, chi2 + + +def angle_deg(bx, by) -> float: + return math.degrees(math.atan(math.hypot(bx, by) / 1000)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("out") + ap.add_argument("names", nargs="+") + ap.add_argument("--tiles", type=int, default=3) + ap.add_argument("--stack", help="a molecule stack model in OUTDIR: its geometry.plane printed beside") + ap.add_argument("--json") + a = ap.parse_args() + print(f"a tilt of 1.6° is {1000 * math.tan(math.radians(1.6)):.0f} nm per µm, 2.9° is " + f"{1000 * math.tan(math.radians(2.9)):.0f}") + if a.stack: + import torch + from smappy.calibrate.pupil import load_start + m = load_start(os.path.join(a.out, f"{a.stack}_3dcal.h5")) + p = torch.as_tensor(m.shared["geometry.plane"]).double().reshape(-1).tolist() + print(f"stack {a.stack}: focal plane's slope {p[0]:+.1f}, {p[1]:+.1f} nm per µm (x, y), " + f"{angle_deg(*p):.2f}°") + for name in a.names: + d = load(a.out, name) + rows = tile_edges(d["x_px"], d["y_px"], d["h"], d["sd_h"], a.tiles) + if len(rows) < 4: + print(f"{name}: only {len(rows)} tiles with an edge; fewer --tiles") + continue + coef, err, chi2 = plane(rows) + print(f"\n{name}: {len(rows)} tiles of {a.tiles}x{a.tiles}; edge's slope b_x {coef[1]:+.1f} ± {err[1]:.1f}, " + f"b_y {coef[2]:+.1f} ± {err[2]:.1f} nm per µm -> {angle_deg(coef[1], coef[2]):.2f}°; " + f"χ²/dof {chi2:.1f}") + for r in rows: + print(f" tile {r['tile']}: u {r['u_um']:+6.1f} v {r['v_um']:+6.1f} µm edge {r['edge_nm']:+7.1f} ± " + f"{r['sd_nm']:5.1f} nm spike {r['spike']:.2f} n {r['n']}") + if a.json: + with open(a.json, "a") as f: + f.write(json.dumps(dict(kind="tilt_from_edge", model=name, tiles=a.tiles, + b_x_nm_per_um=float(coef[1]), b_y_nm_per_um=float(coef[2]), + b_x_err=float(err[1]), b_y_err=float(err[2]), chi2_dof=chi2, + angle_deg=angle_deg(coef[1], coef[2]), tile_rows=rows)) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docs/uipsf/architecture.md b/docs/uipsf/architecture.md index e1e92fc8..892c75c5 100644 --- a/docs/uipsf/architecture.md +++ b/docs/uipsf/architecture.md @@ -2644,6 +2644,18 @@ this is not yet a clean test of linearity. their printout). - `npc_rings.chain.yaml`, `rings.batch.yaml` and `rings_vs_z.py`: the NPC rings (`m2_pairs_npc_rings.jsonl`). +**Next, scripts written, not yet run** (the open hypotheses above, tested from the molecules' heights alone; each +script's docstring says how to read it): +- `heights.py`: every molecule of the movie (*used*, *held out*, *not drawn*) localised once per model, saved as + `heights_.npz` -- height, CRLB, ½ Σ deviance, chip position, frame. The three below read these files. +- `tilt_from_edge.py`: the coverslip's edge per tile of the field and a plane through it. A real tilt of 1.6° is + ~28 nm per µm, ~700 nm across the field; with `--stack` the molecule stack's own focal-plane slope beside it. +- `edge_vs_time.py`: the edge per block of the 72 000 frames, with one model held: focus drift. +- `score_by_height.py`: two models' scores on the same unseen molecules per height bin: where the stack's pupil + and the in situ pupils each fit better. +- The stack again with h held at 5 nm (`HEIGHT=5`), and in situ started from it, to see whether the stack's + h of 75 nm is part of the gap. + ## 4Pi from beads (October 2026; `docs/4pi.md` D7 and §8) ``` From 7971a3f98f89a9872ee9faa71c9a3f60673bbd8d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 14:24:03 +0000 Subject: [PATCH 5/5] uipsf M2 pairs: z_direction.py, the stage's direction from the metadata against the images Every M2 in situ run's twin test prefers the mirrored pupil, which a ZPositionUm running opposite to the code's convention would give (beads and sweeps on the glass cannot tell the twin). The script prints the files' z tags and direction, the sweeps' and bead stacks' spot elongation at the smallest and largest stage z, and the elongation of the movie's glass molecules, below the focus in its first frames: opposite to the smallest stage z's if the convention holds. Smoke-tested on simulated files; not run on M2's data. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014JegwAEzHHExiXkwvXgA2j --- benchmarks/uipsf/real/m2_pairs/z_direction.py | 155 ++++++++++++++++++ docs/uipsf/architecture.md | 9 + 2 files changed, 164 insertions(+) create mode 100644 benchmarks/uipsf/real/m2_pairs/z_direction.py diff --git a/benchmarks/uipsf/real/m2_pairs/z_direction.py b/benchmarks/uipsf/real/m2_pairs/z_direction.py new file mode 100644 index 00000000..b5dfd9c8 --- /dev/null +++ b/benchmarks/uipsf/real/m2_pairs/z_direction.py @@ -0,0 +1,155 @@ +"""Which way does M2's stage z go? The metadata's direction against what the images show, without any model. + + python z_direction.py [--movie-early 2200] [--movie-late 30000] + +Every SMAPpy path assumes that a larger ``ZPositionUm`` is the objective raised: the focus moves from below +the glass into the buffer. Bead stacks (`calibrate.input.read_bead_stacks`) order their planes by increasing +``ZPositionUm``; molecule sweeps (`molecule_stack.stage_z_um`) take z = −(stage z − focus); in situ's focus is +how far the objective was raised above the coverslip's focus (docs/uipsf/conventions.md, "Positions and the +sign of z"). If on M2 ``ZPositionUm`` were the other way round -- another focus device, or a piezo counting +down -- beads and sweeps would learn the twin of the true pupil (astigmatism and spherical flipped, z mirrored) +and could not tell: molecules and beads on the glass see no depth, so for them the twin is nearly degenerate. +A cell can: its molecules sit above the glass, behind the index mismatch. The twin test preferring the +mirrored pupil by 800-1500 nats in every M2 in situ run (architecture.md, "In situ against molecule stacks on +M2") is what an inverted stage z would give. + +Three checks: + +1. **The metadata.** For every bead stack and molecule sweep file: which frame tags name a z (the key + ``ZPositionUm`` comes from, and any PI or focus device beside it), the first and last value, and whether z + rises along the frames. +2. **The stack's elongation against stage z.** Each spot's elliptical Gaussian (`molecule_stack.first_spots`) + per frame, the median log₂(σx/σy) of each stage z. An astigmatic PSF goes from one elongation to the other + through focus; which one is at the *smallest* ``ZPositionUm`` is the one of molecules *above* the focus if + the convention holds. +3. **The movie's molecules on the glass.** In the nucleus movie the glass molecules appear only in the first + ~2200 frames, and the focus is inside the nucleus, above them: they are *below* the focus, and the most + defocused spots of the early frames. Their elongation is the excess of the early frames' log₂(σx/σy) over + the late frames' at the extremes. + +What to read: if the convention holds, the movie's glass molecules (below focus) are elongated the *opposite* +way to the sweeps' spots at the smallest stage z (above focus). The same way means ``ZPositionUm`` runs +opposite to what the code assumes for these files, and every M2 bead and sweep model is the twin. +""" +import argparse +import glob +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from heights import MOVIE, ROOT # noqa: E402 + +SETS = {"beads": "261112_MR_Intercalib2_Beads_1_700/*/*.ome.tif", + "ab": "261112_MR_Intercalib2_AB_50ms_coverslip/*/*.ome.tif"} +FAR = 0.4 # |log₂(σx/σy)| beyond which a spot counts as clearly elongated + + +def read_with_tags(path, frames=None): + """The file's frames (all, or ``frames`` = (start, stop)) and its frame tags, as `molecule_stack.read_file` + reads them.""" + from smappy.frametags import FrameTags + from smappy.io.tiff import open_stack + source = open_stack(path) + source.tags = FrameTags() + start, stop = (0, None) if frames is None else frames + out = [block for _, block in source.frames(chunk=200, start=start, stop=stop)] + return np.concatenate(out), source + + +def z_keys(source) -> dict: + """Every frame tag whose name says z, focus or PI: {name: (first, last)} where the values are numbers.""" + table = source.tags.table() if source.tags is not None else {} + out = {} + for key, values in table.items(): + k = key.lower() + if "z" in k.split("-")[-1] or "focus" in k or "pi" in k.split("-")[0] or "zposition" in k: + try: + v = np.asarray(values, float) + except (TypeError, ValueError): + continue + if v.size: + out[key] = (float(v[0]), float(v[-1]), float(np.ptp(v))) + return out + + +def elongation(spots) -> np.ndarray: + return np.log2(np.asarray(spots["sigma_x_pix"]) / np.asarray(spots["sigma_y_pix"])) + + +def stack_check(name, pattern): + from smappy.calibrate.molecule_stack import first_spots, stage_z_um + files = sorted(glob.glob(os.path.join(ROOT, pattern))) + print(f"\n== {name}: {len(files)} files") + lows, highs = [], [] + for path in files: + frames, source = read_with_tags(path) + z = stage_z_um(source, path) + print(f" {os.path.basename(os.path.dirname(path)) or os.path.basename(path)}: {len(frames)} frames") + for key, (a, b, span) in z_keys(source).items(): + print(f" tag {key}: first {a:.3f}, last {b:.3f}, range {span:.3f}") + if z is None or len(z) != len(frames) or np.ptp(z) == 0: + print(" no usable stage z") + continue + rising = np.mean(np.diff(z) > 0) + print(f" stage z (ZPositionUm): {z[0]:.3f} -> {z[-1]:.3f} µm, rising in {rising:.0%} of steps") + spots = first_spots(frames) + e = elongation(spots) + zs = z[np.asarray(spots["frame"], int)] + lo, hi = np.quantile(z, [0.1, 0.9]) + bright = np.asarray(spots["photons"]) > np.nanmedian(spots["photons"]) + e_lo, e_hi = e[(zs <= lo) & bright], e[(zs >= hi) & bright] + lows.append(e_lo) + highs.append(e_hi) + print(f" log2(sx/sy), bright spots: at the lowest 10% of stage z {np.nanmedian(e_lo):+.2f} (n {len(e_lo)})," + f" at the highest 10% {np.nanmedian(e_hi):+.2f} (n {len(e_hi)})") + if lows: + lo, hi = np.concatenate(lows), np.concatenate(highs) + print(f" all {name}: smallest stage z {np.nanmedian(lo):+.2f}, largest {np.nanmedian(hi):+.2f}") + return float(np.nanmedian(lo)), float(np.nanmedian(hi)) + return None + + +def movie_check(early_frames: int, late_start: int): + from smappy.calibrate.molecule_stack import first_spots + path = sorted(glob.glob(os.path.join(ROOT, MOVIE)))[0] + print(f"\n== movie {os.path.basename(path)}: frames 0-{early_frames} against {late_start}-" + f"{late_start + early_frames}") + out = {} + for label, span in (("early", (0, early_frames)), ("late", (late_start, late_start + early_frames))): + frames, _ = read_with_tags(path, span) + spots = first_spots(frames) + bright = np.asarray(spots["photons"]) > np.nanmedian(spots["photons"]) + out[label] = elongation(spots)[bright] + e = out[label] + print(f" {label:5s}: {len(e)} bright spots; elongated log2(sx/sy) > +{FAR}: {np.mean(e > FAR):.1%}, " + f"< -{FAR}: {np.mean(e < -FAR):.1%}; median {np.nanmedian(e):+.2f}") + plus = np.mean(out["early"] > FAR) - np.mean(out["late"] > FAR) + minus = np.mean(out["early"] < -FAR) - np.mean(out["late"] < -FAR) + side = "+" if plus > minus else "-" + print(f" the early frames' excess (the glass molecules, below focus): {plus:+.1%} at sx > sy, " + f"{minus:+.1%} at sy > sx -> elongated {side}") + return side + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--movie-early", type=int, default=2200) + ap.add_argument("--movie-late", type=int, default=30000) + a = ap.parse_args() + stacks = {name: stack_check(name, pattern) for name, pattern in SETS.items()} + glass = movie_check(a.movie_early, a.movie_late) + print("\n== verdict") + for name, r in stacks.items(): + if r is None: + continue + low_side = "+" if r[0] > 0 else "-" + agree = low_side != glass + print(f" {name}: spots at the smallest stage z elongated {low_side}, the movie's glass molecules {glass}: " + + ("opposite, as the convention says (smallest z = focus below the glass)" if agree else + "THE SAME: the stage z runs opposite to the code's convention for these files")) + + +if __name__ == "__main__": + main() diff --git a/docs/uipsf/architecture.md b/docs/uipsf/architecture.md index 8378b92f..723bf8d4 100644 --- a/docs/uipsf/architecture.md +++ b/docs/uipsf/architecture.md @@ -2767,6 +2767,15 @@ better): - A tilt held at the edge's value (along y) in situ, to see what the pupil does with it. - Learning the index per height range, or an index gradient. +**The stage's direction** (`z_direction.py`, written after the follow-up, not yet run). The twin test prefers the +mirrored pupil by 800-1500 nats in every M2 in situ run. That is what a stage z running opposite to the code's +convention would give: every path takes a larger ``ZPositionUm`` as the objective raised (beads order their planes +by it, sweeps take z = −(stage z − focus), in situ's focus is the objective's rise above the coverslip), and beads +and molecules on the glass see no depth, so they would learn the twin without noticing. The script prints the +files' z tags and their direction, the sweeps' and bead stacks' spot elongation at the smallest and largest stage +z, and the elongation of the movie's glass molecules (below the focus, in its first ~2200 frames). If the +convention holds, the glass molecules are elongated the opposite way to the spots at the smallest stage z. + ## 4Pi from beads (October 2026; `docs/4pi.md` D7 and §8) ```