From 1026673ac15cfd5141664504be45f4988fbf6686 Mon Sep 17 00:00:00 2001 From: aranega Date: Thu, 28 Aug 2025 05:59:14 -0600 Subject: [PATCH 1/3] CELE-149 Add hook system to plug conversion scripts for synapses coordinate computation --- ingestion/ingestion/ingest.py | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ingestion/ingestion/ingest.py b/ingestion/ingestion/ingest.py index 31fbf7fb..18ab337f 100644 --- a/ingestion/ingestion/ingest.py +++ b/ingestion/ingestion/ingest.py @@ -363,6 +363,20 @@ def upload_3d( ) import re + conversion_script = synapses_dirs[0] / "convert.py" + + conversion_fun = None + if conversion_script.exists(): + logger.info(f"Conversion script detected: {conversion_script}") + import importlib + + spec = importlib.util.spec_from_file_location( # type: ignore + "synapses_converter", conversion_script + ) + module = importlib.util.module_from_spec(spec) # type: ignore + spec.loader.exec_module(module) + conversion_fun = module.convert + synapses_positions_file = synapses_dirs[0] / "synapses_positions.txt" f = synapses_positions_file.open("w") @@ -376,25 +390,15 @@ def extract_last_number(filename: str) -> str | None: mesh = trimesh.load(file) - if file.suffix == ".obj": - # Adapt the mesh position - # Step 1: negate the bbox - bbox_neg = -mesh.bounds - - # Step 2: double x and y - bbox_neg[:, 0] *= 2 # double x - bbox_neg[:, 1] *= 2 # double y - - # Step 3: swap x and y - bbox_swapped = bbox_neg.copy() - bbox_swapped[:, [0, 1]] = bbox_swapped[:, [1, 0]] - else: - bbox_swapped = mesh.bounds - # Gets the center of the bbox - bbox_min, bbox_max = bbox_swapped + bbox = mesh.bounds + bbox_min, bbox_max = bbox center = (bbox_min + bbox_max) / 2 + # We call the conversion function if there is one + if conversion_fun: + center = conversion_fun(file, mesh, center) + # Extract catmaid connector id connector_id = extract_last_number(file.stem) entry = f"{connector_id}: [{center[0]}, {center[1]}, {center[2]}]\n" From f37edc30e83f87e800a4d5bf82d38de5ea39728a Mon Sep 17 00:00:00 2001 From: aranega Date: Thu, 28 Aug 2025 06:14:56 -0600 Subject: [PATCH 2/3] CELE-149 Add warnings if the convert function is not found and add documentation --- ingestion/README.md | 24 +++++++++++++++++++++++- ingestion/ingestion/ingest.py | 7 ++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ingestion/README.md b/ingestion/README.md index abd341f6..ce7f2d3f 100644 --- a/ingestion/README.md +++ b/ingestion/README.md @@ -16,6 +16,7 @@ The C-Elegans Utility CLI Tool supports the C-Elegans application by allowing us - [Ingesting Segmentations](#ingesting-segmentations) - [Extract segmentations from bitmap Files](#extract-segmentations-from-bitmap-files) - [Ingest the Segmentations](#ingest-the-segmentations) + - [Synapse coordinate conversion script](#synapse-coordinate-conversion-script) - [FAQ](#faq) - [What should be the file names and directory structure for the files I want to upload](#what-should-be-the-file-names-and-directory-structure-for-the-files-i-want-to-upload) - [Re-upload new version of the Dataset or related data](#re-upload-new-version-of-the-dataset-or-related-data) @@ -145,7 +146,7 @@ When using the `add-dataset` subcommand, don't forget to specify the dataset ID The following flags help determine which files to upload: - `-seg`/`--segmentation`: Path to the directory or files containing neuron segmentation data. -- `-3`/`--3d`: Path to the directory or files containing 3D neuron models. +- `-3`/`--3d`: Path to the directory or files containing 3D neuron models and 3D synapse models. - `-e`/`--em`: Path to the directory or files containing EM tile images. - `-syn`/`--synapses`: Path to the directory or files containin synapses segmentation data. @@ -157,6 +158,8 @@ For example, to upload 3D neuron models from `/path/to/3d/models` for the datase celegans ingest --data /path/to/data/db-raw-data add-dataset --id witvliet_2020_2 --3d /path/to/3d/models ``` +This command will also try to detect all the 3D synapse models that could be located under `/path/to/3d/models/synapses` and will also check for a conversion script if there is one. If the conversion script exists, the + You can upload multiple datasets by chaining `add-dataset` commands. For example, to upload 3D neuron models for `witvliet_2020_2` and EM images for `witvliet_2020_3`, use this command: ```bash @@ -203,6 +206,24 @@ celegans ingest --data /path/to/data/db-raw-data add-dataset --id witvliet_2020_ 2. Using the correct flag for either neuron segmentations (`-seg` or `--segmentation`) or the synapses segmentations (`-syn` or `--synapses`) 3. Pointing to the segmentation output directory, which is the same as the bitmap files directory. +#### Synapse coordinate conversion script + +The synapse coordinate conversion script is searched in the 3D synapse model folder. It has to be named `convert.py` and has to contain at least one function named `convert` with this signature `convert(f: Path, mesh: trimesh.Geometry, bbox_center: ndarray) -> ndarray`. + +Here is an example of conversion script + +```python +from pathlib import Path +from numpy import ndarray +import trimesh + + +def convert(f: Path, mesh: trimesh.Geometry, bbox_center: ndarray) -> ndarray: + center = -bbox_center + center[0], center[1] = center[1] * 2, center[0] * 2 + return center +``` + ## FAQ ### What should be the file names and directory structure for the files I want to upload @@ -234,6 +255,7 @@ Our suggestion would be to manage and store your files as follows: │   │   ├── ADEL.stl │ │ │ ... | | ├── synapses +| | | ├── convert.py # [optional] coordinate script conversion | | | ├── _0001_ADAL_synapses.ADAL_AVBL,AVBR,RIML_8414886-SEM_adult.stl # or .obj | | | ├── _0002_ADAL_synapses.ADAL_AVBL,AVJL,AVBR_8414874-SEM_adult.stl | | | ├── ... diff --git a/ingestion/ingestion/ingest.py b/ingestion/ingestion/ingest.py index 18ab337f..207c84a2 100644 --- a/ingestion/ingestion/ingest.py +++ b/ingestion/ingestion/ingest.py @@ -375,7 +375,12 @@ def upload_3d( ) module = importlib.util.module_from_spec(spec) # type: ignore spec.loader.exec_module(module) - conversion_fun = module.convert + try: + conversion_fun = getattr(module, "convert") + except AttributeError: + logger.warning( + "Conversion script had been found, but it doesn't have a 'convert' function. Coordinate conversion will not be applied" + ) synapses_positions_file = synapses_dirs[0] / "synapses_positions.txt" f = synapses_positions_file.open("w") From 9f61daa6d781f358a5010deb269b8bf09c12a5cc Mon Sep 17 00:00:00 2001 From: Vincent Aranega Date: Thu, 28 Aug 2025 06:35:22 -0600 Subject: [PATCH 3/3] Update ingestion/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ingestion/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/README.md b/ingestion/README.md index ce7f2d3f..6e276000 100644 --- a/ingestion/README.md +++ b/ingestion/README.md @@ -158,7 +158,7 @@ For example, to upload 3D neuron models from `/path/to/3d/models` for the datase celegans ingest --data /path/to/data/db-raw-data add-dataset --id witvliet_2020_2 --3d /path/to/3d/models ``` -This command will also try to detect all the 3D synapse models that could be located under `/path/to/3d/models/synapses` and will also check for a conversion script if there is one. If the conversion script exists, the +This command will also try to detect all the 3D synapse models that could be located under `/path/to/3d/models/synapses` and will also check for a conversion script if there is one. If the conversion script exists, the tool will automatically run it to convert the coordinates or data as required before uploading. You can upload multiple datasets by chaining `add-dataset` commands. For example, to upload 3D neuron models for `witvliet_2020_2` and EM images for `witvliet_2020_3`, use this command: