diff --git a/ingestion/README.md b/ingestion/README.md index abd341f6..6e276000 100644 --- a/ingestion/README.md +++ b/ingestion/README.md @@ -16,6 +16,7 @@ The C-Elegans Utility CLI Tool supports the C-Elegans application by allowing us - [Ingesting Segmentations](#ingesting-segmentations) - [Extract segmentations from bitmap Files](#extract-segmentations-from-bitmap-files) - [Ingest the Segmentations](#ingest-the-segmentations) + - [Synapse coordinate conversion script](#synapse-coordinate-conversion-script) - [FAQ](#faq) - [What should be the file names and directory structure for the files I want to upload](#what-should-be-the-file-names-and-directory-structure-for-the-files-i-want-to-upload) - [Re-upload new version of the Dataset or related data](#re-upload-new-version-of-the-dataset-or-related-data) @@ -145,7 +146,7 @@ When using the `add-dataset` subcommand, don't forget to specify the dataset ID The following flags help determine which files to upload: - `-seg`/`--segmentation`: Path to the directory or files containing neuron segmentation data. -- `-3`/`--3d`: Path to the directory or files containing 3D neuron models. +- `-3`/`--3d`: Path to the directory or files containing 3D neuron models and 3D synapse models. - `-e`/`--em`: Path to the directory or files containing EM tile images. - `-syn`/`--synapses`: Path to the directory or files containin synapses segmentation data. @@ -157,6 +158,8 @@ For example, to upload 3D neuron models from `/path/to/3d/models` for the datase celegans ingest --data /path/to/data/db-raw-data add-dataset --id witvliet_2020_2 --3d /path/to/3d/models ``` +This command will also try to detect all the 3D synapse models that could be located under `/path/to/3d/models/synapses` and will also check for a conversion script if there is one. If the conversion script exists, the tool will automatically run it to convert the coordinates or data as required before uploading. + You can upload multiple datasets by chaining `add-dataset` commands. For example, to upload 3D neuron models for `witvliet_2020_2` and EM images for `witvliet_2020_3`, use this command: ```bash @@ -203,6 +206,24 @@ celegans ingest --data /path/to/data/db-raw-data add-dataset --id witvliet_2020_ 2. Using the correct flag for either neuron segmentations (`-seg` or `--segmentation`) or the synapses segmentations (`-syn` or `--synapses`) 3. Pointing to the segmentation output directory, which is the same as the bitmap files directory. +#### Synapse coordinate conversion script + +The synapse coordinate conversion script is searched in the 3D synapse model folder. It has to be named `convert.py` and has to contain at least one function named `convert` with this signature `convert(f: Path, mesh: trimesh.Geometry, bbox_center: ndarray) -> ndarray`. + +Here is an example of conversion script + +```python +from pathlib import Path +from numpy import ndarray +import trimesh + + +def convert(f: Path, mesh: trimesh.Geometry, bbox_center: ndarray) -> ndarray: + center = -bbox_center + center[0], center[1] = center[1] * 2, center[0] * 2 + return center +``` + ## FAQ ### What should be the file names and directory structure for the files I want to upload @@ -234,6 +255,7 @@ Our suggestion would be to manage and store your files as follows: │   │   ├── ADEL.stl │ │ │ ... | | ├── synapses +| | | ├── convert.py # [optional] coordinate script conversion | | | ├── _0001_ADAL_synapses.ADAL_AVBL,AVBR,RIML_8414886-SEM_adult.stl # or .obj | | | ├── _0002_ADAL_synapses.ADAL_AVBL,AVJL,AVBR_8414874-SEM_adult.stl | | | ├── ... diff --git a/ingestion/ingestion/ingest.py b/ingestion/ingestion/ingest.py index 31fbf7fb..207c84a2 100644 --- a/ingestion/ingestion/ingest.py +++ b/ingestion/ingestion/ingest.py @@ -363,6 +363,25 @@ def upload_3d( ) import re + conversion_script = synapses_dirs[0] / "convert.py" + + conversion_fun = None + if conversion_script.exists(): + logger.info(f"Conversion script detected: {conversion_script}") + import importlib + + spec = importlib.util.spec_from_file_location( # type: ignore + "synapses_converter", conversion_script + ) + module = importlib.util.module_from_spec(spec) # type: ignore + spec.loader.exec_module(module) + try: + conversion_fun = getattr(module, "convert") + except AttributeError: + logger.warning( + "Conversion script had been found, but it doesn't have a 'convert' function. Coordinate conversion will not be applied" + ) + synapses_positions_file = synapses_dirs[0] / "synapses_positions.txt" f = synapses_positions_file.open("w") @@ -376,25 +395,15 @@ def extract_last_number(filename: str) -> str | None: mesh = trimesh.load(file) - if file.suffix == ".obj": - # Adapt the mesh position - # Step 1: negate the bbox - bbox_neg = -mesh.bounds - - # Step 2: double x and y - bbox_neg[:, 0] *= 2 # double x - bbox_neg[:, 1] *= 2 # double y - - # Step 3: swap x and y - bbox_swapped = bbox_neg.copy() - bbox_swapped[:, [0, 1]] = bbox_swapped[:, [1, 0]] - else: - bbox_swapped = mesh.bounds - # Gets the center of the bbox - bbox_min, bbox_max = bbox_swapped + bbox = mesh.bounds + bbox_min, bbox_max = bbox center = (bbox_min + bbox_max) / 2 + # We call the conversion function if there is one + if conversion_fun: + center = conversion_fun(file, mesh, center) + # Extract catmaid connector id connector_id = extract_last_number(file.stem) entry = f"{connector_id}: [{center[0]}, {center[1]}, {center[2]}]\n"