Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions benchmarks/uipsf/real/m2_pairs/coverslip_fit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Control: fluorophores at known heights in the sample (AB's model propagated, n_med 1.345, focus 607 nm, no tilt)
localised with the AB coverslip calibration's spline, as a user would (Spline 3D, best of 5 starts).

python coverslip_fit.py OUTDIR

The spline describes a molecule on the glass against defocus; the fluorophores sit 0-1200 nm deep, imaged with
the focus 607 nm into the sample. Prints per height the median fitted z, its spread, the x and y error, and the
slope and the rms about a straight line over 0-1000 nm: what ignoring the depth costs in linearity.
"""
import os
import sys

import numpy as np
import torch

HEIGHTS = np.arange(0.0, 1201.0, 100.0)
REP, PHOTONS, BG = 40, 4570.0, 98.0


def main():
out = sys.argv[1]
from smappy.calibrate.pupil import _best_of, load_start
from smappy.io.calibration import load_spline_calibration
from smappy.psf import SplinePSF

from uipsf.simulate import parameters, simulate, template

t = load_start(os.path.join(out, "insitu_all_n1345_tilt_s315_ab_n1345_3dcal.h5"))
tm = t.model
shared = {k: torch.as_tensor(v).detach().clone() for k, v in t.shared.items()}
shared["geometry.tilt"] = torch.zeros_like(shared["geometry.tilt"])
rng = np.random.default_rng(0)
E = len(HEIGHTS) * REP
z_gt = np.repeat(HEIGHTS, REP)
px, py = tm.pixel_size_nm
xy = rng.uniform(-0.5, 0.5, (E, 2)) * np.array([px, py])
cal = load_spline_calibration(os.path.join(out, "ab_n1345_3dcal.h5"))
box = min(int(cal.coeff.shape[-1]) - 3, 15) | 1
roi = (box, box)
b0 = template(E, torch.zeros(1, dtype=torch.float64), roi, dtype=torch.float32)
values = dict(shared)
values.update({"emitter.xyz": torch.tensor(np.c_[xy, z_gt], dtype=torch.float64),
"emitter.photons": torch.full((E,), PHOTONS), "emitter.background": torch.full((E,), BG)})
batch = simulate(tm, parameters(tm, b0, values), b0, seed=1)
full = batch.data.cpu().numpy().reshape(E, *batch.data.shape[-2:])
c0 = (full.shape[-1] - box) // 2
rois = np.ascontiguousarray(full[:, c0:c0 + box, c0:c0 + box], np.float32)
best = _best_of(lambda z: SplinePSF(cal, z_start_nm=z).fit(rois, iterations=75),
cal.z_index_to_nm(np.linspace(1, cal.shape[0] - 2, 5)))
z = cal.z_index_to_nm(best.theta[:, 4])
h = box // 2
x_nm = (best.theta[:, 0] - h) * px
y_nm = (best.theta[:, 1] - h) * py
print(f"spline z range {cal.z_index_to_nm(0):.0f} .. {cal.z_index_to_nm(cal.shape[0] - 1):.0f} nm, box {box} px")
print(" z_gt z_fit median sd x err y err")
med = []
for hh in HEIGHTS:
k = z_gt == hh
m = float(np.median(z[k]))
med.append(m)
mad = 1.4826 * np.median(np.abs(z[k] - m))
ex, ey = np.median(x_nm[k] - xy[k, 0]), np.median(y_nm[k] - xy[k, 1])
print(f" {hh:5.0f} {m:9.1f} {mad:5.1f} {ex:6.1f} {ey:6.1f}")
med = np.array(med)
inner = HEIGHTS <= 1000
a, b = np.polyfit(HEIGHTS[inner], med[inner], 1)
r = med[inner] - np.polyval([a, b], HEIGHTS[inner])
print(f"slope {a:.3f} (z_fit per nm of height), rms about the line {np.sqrt((r ** 2).mean()):.1f} nm")


if __name__ == "__main__":
main()
179 changes: 179 additions & 0 deletions benchmarks/uipsf/real/m2_pairs/cross_score.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
"""In situ against molecule stacks on M2 (12 November 2026): each model scores the other's held-out molecules.

python cross_score.py OUTDIR STACK_MODEL INSITU_FOR_STACK MODEL [MODEL ...] [--json FILE]

Two held-out sets, neither seen by any model's learning:

* **stack**: the AB coverslip sweep's molecules that pass the selection (significance 5, the runs' setting)
but were not drawn into the 4000 learnt from (`molecule_rois`'s draw is seeded, so the same split as the
runs'). Every model scores them in the stack's geometry (`CoverslipStackGeometry`, STACK_MODEL's): its
pupil phase, γ and σ held, the geometry -- each sweep's focus, the focal plane's slope, the molecules'
height -- learnt again on 600 of the drawn molecules (the same 600 for every model), then the held-out
molecules localised with x, y, photons and background free. So a model is not charged for a focus offset
that only belongs to the other geometry, only for its PSF's shape.
* **in situ**: the nucleus movie's molecules in the *held out* and *not drawn* states (`movie_rois`'s
defaults, deterministic). An in situ model scores them with its own focus; a stack model through
INSITU_FOR_STACK, the in situ run that held every aberration of it and learnt only the focus (and σ). Each
molecule localised with x, y, z, photons and background free from five heights, the best kept.

Each MODEL is a saved `<name>_3dcal.h5` in OUTDIR; a stack model's name is STACK_MODEL itself. Reported: ½ Σ
deviance per molecule on each set (lower is better; a difference is a log-likelihood ratio, nats per
molecule), relative to the first MODEL.
"""
import argparse
import glob
import json
import os

import numpy as np
import torch

ROOT = os.environ.get("M2_DATA", os.path.expanduser("~/Library/Mobile Documents/com~apple~CloudDocs/Vienna/Data/"
"smappy test/261112_new"))
STACK = "261112_MR_Intercalib2_AB_50ms_coverslip/*/*.ome.tif"
MOVIE = "261112_MR_Intercalib2_BGAAF647_50ms_30i_nucleus_1/*_MMStack_Pos0.ome.tif"
STARTS_NM = (-200.0, 200.0, 600.0, 1000.0, 1400.0)
GEOMETRY_MOLECULES = 600


def phase_by_noll(m) -> dict:
"""{Noll index: rad} of a saved model: its learnt modes and the ones it held (in situ's *hold*)."""
pupil = m.model.channels[0].psf.pupil
out = {}
held = getattr(pupil, "held_phase", None)
if isinstance(held, dict):
out.update({int(j): float(v) for j, v in held.items()})
values = torch.as_tensor(m.shared.get("pupil.phase", torch.zeros(0))).reshape(-1)
out.update({int(j): float(v) for j, v in zip(pupil.phase_modes, values, strict=False)})
return out


def gamma_of(m) -> float:
if "pupil.gamma" in m.shared:
return float(torch.as_tensor(m.shared["pupil.gamma"]))
return float(m.model.channels[0].psf.pupil.held_gamma)


def half_deviance(model, noise, fit, problem) -> np.ndarray:
from uipsf.solve import Problem
dev = Problem(model, noise, problem.batch).deviance(fit.params)
return 0.5 * dev.reshape(dev.shape[0], -1).sum(1).double().cpu().numpy()


def stack_scores(out, stack_name, names):
from smappy.calibrate.molecule_stack import MoleculeStackSettings, molecule_rois
from smappy.calibrate.pupil import load_start

from uipsf.params import Params
from uipsf.solve import LevenbergMarquardt, Problem
from uipsf.workflows.scoring import localise

rois = molecule_rois(sorted(glob.glob(os.path.join(ROOT, STACK))),
MoleculeStackSettings(molecules=4000, significance=5.0, height_nm=None))
drawn, held = rois.ids("used"), rois.ids("not drawn")
rng = np.random.default_rng(0)
geo_ids = np.sort(rng.choice(drawn, min(GEOMETRY_MOLECULES, len(drawn)), replace=False))
geo_batch, noise = rois.batch(ids=geo_ids)
held_batch, _ = rois.batch(ids=held)
print(f"stack: {len(drawn)} drawn, {len(held)} held out; geometry from {len(geo_ids)}", flush=True)
template = load_start(os.path.join(out, f"{stack_name}_3dcal.h5"))
model = template.model
modes = list(model.channels[0].psf.pupil.phase_modes)
result = {}
for name in names:
m = template if name == stack_name else load_start(os.path.join(out, f"{name}_3dcal.h5"))
ph = phase_by_noll(m)
shared = {k: torch.as_tensor(v).detach().clone() for k, v in template.shared.items()}
shared["pupil.phase"] = torch.tensor([ph.get(j, 0.0) for j in modes], dtype=shared["pupil.phase"].dtype)
shared["pupil.gamma"] = torch.tensor(gamma_of(m), dtype=torch.float64)
shared["psf.sigma"] = torch.as_tensor(m.shared["psf.sigma"]).reshape(-1)[:1].clone()
# the geometry learnt again with this pupil held: the emitters first, then everything but the PSF
fit0, _ = localise(model, shared, geo_batch, noise)
values = dict(shared)
values.update({k: v.detach() for k, v in fit0.params.physical().items() if k.startswith("emitter.")})
specs = model.specs(geo_batch)
p = Params(specs, geo_batch.n_emitters, dtype=geo_batch.data.dtype, device=geo_batch.data.device,
values=values)
p = p.fix(*[s.name for s in specs if s.scope == "shared" and not s.name.startswith("geometry.")])
fit = LevenbergMarquardt(max_iter=60).solve(Problem(model, noise, geo_batch), p)
learnt = {k: v.detach() for k, v in fit.params.physical().items() if k.startswith("geometry.")}
shared.update(learnt)
hfit, hproblem = localise(model, shared, held_batch, noise)
e = half_deviance(model, noise, hfit, hproblem)
result[name] = e
print(f" {name:44s} stack held-out: {e.mean():9.2f} per molecule; height "
f"{float(learnt['geometry.height']):.0f} nm, plane {learnt['geometry.plane'].tolist()}", flush=True)
return result, len(held)


def insitu_scores(out, stack_name, insitu_for_stack, names):
from smappy.calibrate.insitu_run import movie_rois
from smappy.calibrate.pupil import load_start

from uipsf.workflows.scoring import localise

rois = movie_rois(sorted(glob.glob(os.path.join(ROOT, MOVIE))))
ids = np.sort(np.concatenate([rois.ids("held out"), rois.ids("not drawn")]))
# the *held out* are drawn as the used are (every z bin's share); the *not drawn* are the bins' surplus, so
# they over-represent the heights where molecules are plentiful: reported apart as well
held_only = np.isin(ids, rois.ids("held out"))
batch, noise = rois.batch(ids=ids)
print(f"in situ: {len(ids)} held out ({len(rois.ids('held out'))} held out, "
f"{len(rois.ids('not drawn'))} not drawn)", flush=True)
result = {}
for name in names:
file = insitu_for_stack if name == stack_name else name
m = load_start(os.path.join(out, f"{file}_3dcal.h5"))
shared = {k: torch.as_tensor(v) for k, v in m.shared.items()}
best = np.full(batch.n_emitters, np.inf)
for z0 in STARTS_NM:
xyz = torch.zeros(batch.n_emitters, 3, dtype=torch.float64)
xyz[:, 2] = z0
fit, problem = localise(m.model, shared, batch, noise, start={"emitter.xyz": xyz})
best = np.minimum(best, half_deviance(m.model, noise, fit, problem))
result[name] = best
result[name + "@held out"] = best[held_only]
result[name + "@not drawn"] = best[~held_only]
print(f" {name:44s} in situ held-out: {best.mean():9.2f} per molecule (scored as {file}); held out "
f"{best[held_only].mean():.2f}, not drawn {best[~held_only].mean():.2f}", flush=True)
return result, len(ids)


def main():
ap = argparse.ArgumentParser()
ap.add_argument("out")
ap.add_argument("stack")
ap.add_argument("insitu_for_stack")
ap.add_argument("names", nargs="+")
ap.add_argument("--json")
ap.add_argument("--only", choices=("stack", "insitu"))
a = ap.parse_args()
rows = {n: {} for n in a.names}
for kind, fn in (("stack", lambda: stack_scores(a.out, a.stack, a.names)),
("insitu", lambda: insitu_scores(a.out, a.stack, a.insitu_for_stack, a.names))):
if a.only and a.only != kind:
continue
res, n = fn()
for key, e in res.items():
name, _, part = key.partition("@")
col = kind + ("_" + part.replace(" ", "_") if part else "")
base = res[a.names[0] + ("@" + part if part else "")]
rows[name][f"{col}_molecules"] = len(e)
rows[name][f"{col}_per_molecule"] = float(e.mean())
rows[name][f"{col}_vs_first_per_molecule"] = float((e - base).mean())
rows[name][f"{col}_vs_first_median"] = float(np.median(e - base))
print("\nnats per molecule against", a.names[0], "(negative: fits the held-out molecules better)")
for name, r in rows.items():
nan = float("nan")
print(f" {name:44s} stack {r.get('stack_vs_first_per_molecule', nan):+8.3f} "
f"in situ {r.get('insitu_vs_first_per_molecule', nan):+8.3f} "
f"(held out {r.get('insitu_held_out_vs_first_per_molecule', nan):+.3f}, "
f"not drawn {r.get('insitu_not_drawn_vs_first_per_molecule', nan):+.3f})")
if a.json:
with open(a.json, "a") as f:
for name, r in rows.items():
f.write(json.dumps(dict(model=name, **r)) + "\n")


if __name__ == "__main__":
main()
100 changes: 100 additions & 0 deletions benchmarks/uipsf/real/m2_pairs/fair_compare.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""Learnt molecule-stack models scored on ONE fixed set of molecules: a fair likelihood comparison.

python fair_compare.py OUTDIR NAME [NAME ...] [--soft] [--set ab|nbx4] [--device cpu]

Each learning run rejects molecules between its stages and rounds, so its own objective sums over its own
molecules and cannot be compared with another run's. Here the molecules are detected and selected once
(`molecule_rois`, every one that passes, no draw; ``--soft`` with soft_select.py's selection), and every saved
model ``OUTDIR/<NAME>_3dcal.h5`` -- its forward model with the coverslip geometry and its learnt shared values
(pupil, σ, height, each sweep's focus, the focal plane's slope) -- localises them all again with x, y, photons and
background free (z is the stage's and the geometry's), `uipsf.workflows.scoring.localise`. The score of a model
is ½ Σ deviance over the molecules: lower is better, and differences are log-likelihood ratios (nats) between
models fitted to the same pixels.

Reported per model: the total; the total over the molecules every model fits sensibly (each one's deviance per
pixel below 3 on every model, so that a few blow-ups cannot decide it); and per 200 nm of defocus, the sum and the
difference to the first model -- where in z one model is better.
"""
import argparse
import glob
import json
import os
import sys

import numpy as np
import torch

# the data are not in git: M2_DATA names the folder that holds 261112_new's sets
ROOT = os.environ.get("M2_DATA", os.path.expanduser("~/Library/Mobile Documents/com~apple~CloudDocs/Vienna/Data/"
"smappy test/261112_new"))
SETS = {"ab": "261112_MR_Intercalib2_AB_50ms_coverslip/*/*.ome.tif",
"nbx4": "261112_MR_Intercalib2_NBX4_50ms_coverslip/*/*.ome.tif"}


def main():
ap = argparse.ArgumentParser()
ap.add_argument("out")
ap.add_argument("names", nargs="+")
ap.add_argument("--set", default="ab")
ap.add_argument("--soft", action="store_true")
ap.add_argument("--device", default="cpu")
args = ap.parse_args()
os.environ.setdefault("UIPSF_DEVICE", args.device)
if args.soft:
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import soft_select # noqa: F401
from smappy.calibrate.molecule_stack import MoleculeStackSettings, molecule_rois
from smappy.calibrate.pupil import load_start

from uipsf.solve import Problem
from uipsf.workflows.scoring import localise

files = sorted(glob.glob(os.path.join(ROOT, SETS[args.set])))
rois = molecule_rois(files, MoleculeStackSettings(molecules=10 ** 6, significance=3.5 if args.soft else 5.0))
used = rois.ids("used")
batch, noise = rois.batch()
d = np.asarray(rois.defocus_nm)[used]
print(f"{len(used)} molecules ({'soft' if args.soft else 'standard'} selection), {rois.sweeps} sweeps", flush=True)

per = {}
for name in args.names:
learned = load_start(os.path.join(args.out, f"{name}_3dcal.h5"))
model = learned.model
shared = {k: torch.as_tensor(v) for k, v in learned.shared.items()}
n_focus = shared.get("geometry.focus")
if n_focus is not None and n_focus.numel() != rois.sweeps:
raise ValueError(f"{name}: {n_focus.numel()} sweep foci, the data have {rois.sweeps}")
fit, problem = localise(model, shared, batch, noise)
dev = Problem(model, noise, problem.batch).deviance(fit.params) # [E, C, K, Y, X]
pixels = dev[0].numel()
e = 0.5 * dev.reshape(dev.shape[0], -1).sum(1).double().numpy() # ½ deviance per molecule
per[name] = e
print(f"{name:24s} ½Σdeviance {e.sum():12.1f} per molecule {e.mean():7.2f} "
f"deviance/pixel {2 * e.mean() / pixels:5.3f} converged {fit.converged}", flush=True)

names = args.names
E = np.stack([per[n] for n in names]) # [M, E]
pixels = rois.units.regions.shape[-1] ** 2
sane = (2 * E / pixels < 3).all(0)
base = names[0]
print(f"\nagainst {base} (nats; negative: the model fits better); sane = {sane.sum()} of {len(sane)} molecules")
bins = np.arange(-1000, 1001, 200)
header = "".join(f"{b:+7d}" for b in bins)
print(f"{'model':24s} {'all':>10s} {'sane':>10s} | per 200 nm of defocus, sane: {header}")
rows = []
for k, n in enumerate(names):
diff = E[k] - E[0]
by = [float(diff[sane & (np.abs(d - b) <= 100)].sum()) for b in bins]
print(f"{n:24s} {diff.sum():10.1f} {diff[sane].sum():10.1f} | " + "".join(f"{v:7.0f}" for v in by))
rows.append(dict(model=n, total=float(E[k].sum()), vs_base=float(diff.sum()),
vs_base_sane=float(diff[sane].sum()), by_defocus=dict(zip(bins.tolist(), by, strict=False))))
counts = [int((sane & (np.abs(d - b) <= 100)).sum()) for b in bins]
print(f"{'molecules per bin':24s} {'':21s} | " + "".join(f"{c:7d}" for c in counts))
tag = "soft" if args.soft else "standard"
with open(os.path.join(args.out, f"fair_{args.set}_{tag}.json"), "w") as f:
json.dump(dict(molecules=len(used), sane=int(sane.sum()), selection=tag, rows=rows, bins=bins.tolist(),
counts=counts), f, indent=1)


if __name__ == "__main__":
main()
Loading
Loading