Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
115 commits
Select commit Hold shift + click to select a range
28e744b
Add HITL finetuning test data generation infrastructure
davidackerman Feb 9, 2026
e1157b0
Add LoRA wrapper and CorrectionDataset for HITL finetuning
davidackerman Feb 9, 2026
aba789a
Add LoRA training loop and CLI for HITL finetuning
davidackerman Feb 9, 2026
ffd1a01
Fix LoRA wrapper for Sequential models and add comprehensive document…
davidackerman Feb 9, 2026
f322b22
Complete HITL finetuning pipeline with full component validation
davidackerman Feb 9, 2026
0b3b1e8
Fix zarr structure for Neuroglancer/OME-NGFF compatibility
davidackerman Feb 9, 2026
0cf7b1f
Fix dataset to handle mismatched raw/mask sizes
davidackerman Feb 9, 2026
258ddf9
Disable patching to use full-size corrections for training
davidackerman Feb 9, 2026
8c276fa
Fix CLI to handle None patch_shape
davidackerman Feb 9, 2026
e8c484c
Disable spatial augmentation for mismatched input/output sizes
davidackerman Feb 9, 2026
c2f98cf
Add mito correction generator with fly_organelles model
davidackerman Feb 11, 2026
0bc2078
Add channel selection and logging for LoRA finetuning
davidackerman Feb 11, 2026
5f76102
Add diagnostic tools for analyzing finetuning quality
davidackerman Feb 11, 2026
e15ca7e
Document LoRA finetuning workflow with detailed walkthrough
davidackerman Feb 11, 2026
1d1d29f
Fix normalization pipeline and add sparse annotation support
davidackerman Feb 11, 2026
f662c53
Add utility scripts for sparse annotation workflow
davidackerman Feb 11, 2026
77eec72
Add MinIO hosting scripts for Neuroglancer annotations
davidackerman Feb 12, 2026
f048001
Remove unnecessary HTTP and legacy MinIO scripts
davidackerman Feb 12, 2026
b1a06ea
Add finetune annotation crop viewer integration
davidackerman Feb 12, 2026
0646561
Add bidirectional MinIO annotation syncing and improve finetuning wor…
davidackerman Feb 12, 2026
5dd9078
Update finetuning documentation with dashboard workflow
davidackerman Feb 12, 2026
b238457
Improve finetuning: fix gradient accumulation bug, add live log strea…
davidackerman Feb 13, 2026
4086834
Add finetuning job management system and dashboard integration
davidackerman Feb 13, 2026
4682d58
Add auto-serve inference after finetuning and iterative training on s…
davidackerman Feb 14, 2026
675e3e5
Add auto-serve status display and restart training UI to finetune tab
davidackerman Feb 14, 2026
fc407f3
Fix dark mode styling for modals, form controls, labels, and muted text
davidackerman Feb 14, 2026
68549fd
Fix chunk boundary bug in log marker detection for neuroglancer layer…
davidackerman Feb 14, 2026
9dc1467
Filter noisy debug and werkzeug lines from training log stream
davidackerman Feb 14, 2026
1a4a164
Fix restart layer update: run iteration check every monitor cycle
davidackerman Feb 14, 2026
f3e7c9d
Fix training collapse and add MSE loss with label smoothing
davidackerman Feb 14, 2026
d214fd4
Clamp dataloader batch size to dataset size
davidackerman Feb 14, 2026
3aa86e3
Add margin loss, teacher distillation, and expanded training UI controls
davidackerman Feb 14, 2026
8b084f0
Add class balancing option to prevent foreground overprediction
davidackerman Feb 14, 2026
4f3abdd
Fix duplicate log lines by removing redundant file write
davidackerman Feb 14, 2026
1d168db
Fix MinIO startup failure when console port is already in use
davidackerman Feb 15, 2026
2feef59
Add GPU queue selection to finetuning UI
davidackerman Feb 15, 2026
08d3628
Fix delayed log streaming by disabling pipe buffering
davidackerman Feb 15, 2026
e376cbf
Save only LoRA weights in checkpoints instead of full model
davidackerman Feb 15, 2026
e570e03
Improve restart control path and CLI logging behavior
davidackerman Feb 15, 2026
1008505
Speed up finetune dashboard log streaming and restart UX
davidackerman Feb 15, 2026
795fa43
Add model load timing logs and fs visibility probe tool
davidackerman Feb 15, 2026
f897439
Update finetune progress UI from live SSE epoch/loss logs
davidackerman Feb 15, 2026
083dd4a
Update finetune UI, dataset, and job manager behavior
davidackerman Feb 18, 2026
686ca79
Remove dev scripts, docs artifacts, and output binaries
davidackerman Feb 18, 2026
cae19e8
Revert non-finetune changes to match main
davidackerman Feb 18, 2026
e2ead85
Refactor finetune module and restore blueprint architecture
davidackerman Feb 18, 2026
51ebe33
Fix split_dataset_path for paths with nested .zarr segments
davidackerman Feb 18, 2026
ec43fb3
Deduplicate sync logic, job status, and dataset path extraction
davidackerman Feb 18, 2026
73972a5
Merge branch 'main' into finetuning_refactor
davidackerman Feb 25, 2026
e362282
Add target transform classes for converting annotations to training t…
davidackerman Feb 26, 2026
132cc91
Support target_transform in LoRAFinetuner
davidackerman Feb 26, 2026
80d98d7
Add --output-type, --select-channel, --offsets CLI args and target tr…
davidackerman Feb 26, 2026
83d27eb
Wire output_type, select_channel, and offsets through dashboard and j…
davidackerman Feb 26, 2026
d2630ba
Fix formatting in model_spec_affinities example
davidackerman Feb 26, 2026
878b769
Add finetuning guide documentation
davidackerman Feb 26, 2026
23fa4ae
Remove normalize parameter from CorrectionDataset and create_dataloader
davidackerman Feb 26, 2026
c15ed97
Move dashboard state from state.py into Flow singleton in globals.py
davidackerman Feb 26, 2026
cb471f1
Add finetuning screenshots and image references to guide
davidackerman Feb 26, 2026
363cbd7
Move test_target_transforms.py to tests/finetune/
davidackerman Feb 26, 2026
b7c8932
add my_yamls to gitignore
davidackerman Mar 10, 2026
97ad8c5
Use spawn multiprocessing context in DataLoader to fix tensorstore fo…
davidackerman Mar 10, 2026
8fd6c92
Add FP16 probe, OOM recovery, NaN detection, and margin loss sigmoid …
davidackerman Mar 10, 2026
7b2bb66
Add restart status markers and fresh LoRA reset on training restart
davidackerman Mar 10, 2026
aaf8a1c
Track iteration count to prevent stale restart markers from re-trigge…
davidackerman Mar 10, 2026
04c9713
Show restart sub-status updates in dashboard progress text
davidackerman Mar 10, 2026
926a495
Fix annotation label corruption, persist restart params, and improve …
davidackerman Mar 20, 2026
8d47d45
Merge main into finetuning_refactor and adopt restyle conventions
davidackerman Mar 23, 2026
6d8ab7d
Style modal borders and background for dark theme consistency
davidackerman Apr 1, 2026
52279c1
Load best checkpoint before saving LoRA adapter
davidackerman Apr 6, 2026
fccb8a0
Fix loss plot bouncing by only plotting epoch-level summaries
davidackerman Apr 6, 2026
8d11727
Support extensionless zarr containers in dataset utilities
davidackerman Apr 6, 2026
93953e1
Overhaul pipeline builder: auto-layout, auto-save, and connection fixes
davidackerman Apr 7, 2026
0b8f719
Add FinetuneModelConfig and improve finetuning UI state
davidackerman Apr 7, 2026
bc12346
Add finetuning install instructions and fix non-PyPI dependencies
davidackerman Apr 7, 2026
03efab6
Merge branch 'main' into finetuning_refactor
davidackerman Apr 7, 2026
a0c1665
Support HuggingFace models in LoRA finetuning pipeline
davidackerman Apr 13, 2026
c7d2483
Replace generated model scripts with FinetuneModelConfig YAML
davidackerman Apr 13, 2026
6483aa3
Auto-detect affinity output type from HuggingFace channel names
davidackerman Apr 13, 2026
c41a78c
Add sigmoid probe to prevent double-sigmoid in finetuning loss
davidackerman Apr 13, 2026
88d3d1f
Allow empty models in YAML config for dashboard-only mode
davidackerman Apr 13, 2026
7bf7985
Support remote (HTTP/HTTPS) zarr datasets
davidackerman Apr 14, 2026
11f2fc8
Add ChannelSelector normalizer and SigmoidPostprocessor
davidackerman Apr 14, 2026
4c82424
Remove forced-CPU patching from load_safe_config
davidackerman Apr 14, 2026
2748f60
Use cellmap_model.train() instead of model.pt for LoRA finetuning
davidackerman Apr 21, 2026
5a81dce
Tolerate per-channel masks in DiceLoss; extend OOM fallbacks
davidackerman Apr 21, 2026
80b458a
Override prediction voxel size to match closest raw scale
davidackerman Apr 21, 2026
250225d
Dashboard: resume existing volumes, show annotated regions, persist p…
davidackerman Apr 21, 2026
d69abc0
Stop excluding output/head layers from LoRA wrapping
davidackerman Apr 21, 2026
2507f92
Add stop-early signal to gracefully exit training loop
davidackerman Apr 21, 2026
d237ff8
Snap annotation grid to closest available raw scale
davidackerman Apr 21, 2026
9f8f37d
Simplify finetuning UI: drop restart modal, expose margin & smoothing
davidackerman Apr 21, 2026
e8f37ff
Fix volume-zarr chunk wipe, stale stop button, and OOM mitigation loop
davidackerman Apr 22, 2026
a87ac40
Refactor finetuning dashboard flow
davidackerman Apr 23, 2026
21ce2dc
Split finetune annotation services
davidackerman Apr 23, 2026
aecc287
Update finetuning guide
davidackerman Apr 23, 2026
a03d516
Expand finetuning option docs
davidackerman Apr 23, 2026
dcaf93e
gitignore: ignore browser/, .codex, .vite/ on this branch
davidackerman Apr 27, 2026
f375f88
Bulk-load externally annotated zarr crops via a YAML manifest
davidackerman Apr 27, 2026
c0a31c6
YAML crop loader: modal UI, live progress, BG sampling, source layers
davidackerman Apr 27, 2026
48e6548
Add virtual-patch dataset: lazy random sampling from source zarrs
davidackerman Apr 27, 2026
564c2d3
Unify YAML imports + painted scribbles into one annotation_volume
davidackerman Apr 27, 2026
7ce39eb
Stop hanging on MinIO sync, parallelize copies, surface progress
davidackerman Apr 28, 2026
474ebdb
Fix per-worker RNG cache + add gradient-flow diagnostic
davidackerman Apr 28, 2026
0148d96
Add LoRA gradient-flow regression tests
davidackerman Apr 28, 2026
6dfece5
Re-freeze head/output by default; opt-in via env var
davidackerman Apr 28, 2026
f78ac5d
Revert "Re-freeze head/output by default; opt-in via env var"
davidackerman Apr 28, 2026
5f313b2
Plumb input_norm from dashboard through to trainer + finetuned yaml
davidackerman Apr 28, 2026
0f9689e
Fall back to live g.input_norms when input_norm_config is empty
davidackerman Apr 28, 2026
ab47a2f
Make LambdaNormalizer picklable for DataLoader workers
davidackerman Apr 28, 2026
b5fa2d2
Stop wiping on-disk volume zarr on transient MinIO listing failure
davidackerman Apr 28, 2026
cb91cb7
Stratified sampling, fast remap, and loss-plot epoch/loss pairing fix
davidackerman May 5, 2026
fb39eb7
Stop deleting on-disk chunks based on remote MinIO state
davidackerman May 12, 2026
28765e1
Show painted-chunk regions in annotated_regions overlay
davidackerman May 12, 2026
2157b78
Add Patches per Epoch UI override + prefer log file over bpeek
davidackerman May 12, 2026
1ec987c
Cache sigmoid probe + fix probe batch size + per-epoch start log
davidackerman May 12, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
my_yamls/
browser/
.codex
.vite/

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
Expand Down Expand Up @@ -162,4 +167,13 @@ cython_debug/
#.idea/

# Misc
.vscode/
.vscode/

# Project-specific
ignore/
*.zarr/
.claude/
test_corrections.zarr/
correction_slices/
corrections/
output/
3 changes: 1 addition & 2 deletions cellmap_flow/cli/server_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from cellmap_flow.utils.plugin_manager import load_plugins


logging.basicConfig()
logger = logging.getLogger(__name__)


Expand All @@ -47,7 +46,7 @@ def cli(log_level):
cellmap_flow_server script -s /path/to/script.py -d /path/to/data
cellmap_flow_server cellmap -f /path/to/model -n mymodel -d /path/to/data
"""
logging.basicConfig(level=getattr(logging, log_level.upper()))
logging.basicConfig(level=getattr(logging, log_level.upper()), force=True)


@cli.command(name="list-models")
Expand Down
27 changes: 16 additions & 11 deletions cellmap_flow/cli/yaml_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,17 @@ def _submit_model(model):
)
return model_name

with ThreadPoolExecutor(max_workers=len(models)) as executor:
futures = {executor.submit(_submit_model, model): model for model in models}
for future in as_completed(futures):
try:
name = future.result()
logger.info(f"Job for {name} is ready")
except Exception as e:
model = futures[future]
model_name = getattr(model, "name", None) or type(model).__name__
logger.error(f"Failed to start job for {model_name}: {e}")
if models:
with ThreadPoolExecutor(max_workers=len(models)) as executor:
futures = {executor.submit(_submit_model, model): model for model in models}
for future in as_completed(futures):
try:
name = future.result()
logger.info(f"Job for {name} is ready")
except Exception as e:
model = futures[future]
model_name = getattr(model, "name", None) or type(model).__name__
logger.error(f"Failed to start job for {model_name}: {e}")

generate_neuroglancer_url(dataset_path,wrap_raw=wrap_raw)

Expand Down Expand Up @@ -191,7 +192,11 @@ def main(config_path: str, log_level: str, list_types: bool, validate_only: bool

# Build model configuration objects dynamically
logger.info("Building model configurations...")
g.models_config = build_models(config["models"])
if config["models"]:
g.models_config = build_models(config["models"])
else:
g.models_config = []
logger.info("No models configured — starting dashboard for interactive use")

logger.info(f"Configured {len(g.models_config)} model(s):")
for i, model in enumerate(g.models_config, 1):
Expand Down
9 changes: 5 additions & 4 deletions cellmap_flow/dashboard/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
from flask import Flask
from flask_cors import CORS

from cellmap_flow.dashboard import state
from cellmap_flow.dashboard.state import LogHandler
from cellmap_flow.globals import g, LogHandler
from cellmap_flow.dashboard.routes.logging_routes import logging_bp
from cellmap_flow.dashboard.routes.index_page import index_bp
from cellmap_flow.dashboard.routes.pipeline_builder_page import pipeline_builder_bp
from cellmap_flow.dashboard.routes.models import models_bp
from cellmap_flow.dashboard.routes.pipeline import pipeline_bp
from cellmap_flow.dashboard.routes.blockwise import blockwise_bp
from cellmap_flow.dashboard.routes.bbx_generator import bbx_bp
from cellmap_flow.dashboard.routes.finetune import finetune_bp

logger = logging.getLogger(__name__)

Expand All @@ -37,11 +37,12 @@
app.register_blueprint(pipeline_bp)
app.register_blueprint(blockwise_bp)
app.register_blueprint(bbx_bp)
app.register_blueprint(finetune_bp)


def create_and_run_app(neuroglancer_url=None, inference_servers=None):
state.NEUROGLANCER_URL = neuroglancer_url
state.INFERENCE_SERVER = inference_servers
g.NEUROGLANCER_URL = neuroglancer_url
g.INFERENCE_SERVER = inference_servers
hostname = socket.gethostname()
port = 0
logger.warning(f"Host name: {hostname}")
Expand Down
Loading