diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 435ee533..436c5937 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,5 +12,3 @@ "runArgs": ["--gpus", "all", "--shm-size=8g"], "remoteUser": "root" } - - \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 016b68f0..df5d34f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,4 +11,4 @@ htmlcov build/ dist/ *.egg-info -*.egg \ No newline at end of file +*.egg diff --git a/.env_example b/.env_example index 023a0952..47fddbb8 100755 --- a/.env_example +++ b/.env_example @@ -7,4 +7,4 @@ COMET_PROJECT_NAME="project_name" COMET_WORKSPACE="workspace_name" COMET_GIT_DIRECTORY="path/to/dir/" -HYDRA_FULL_ERROR=1 \ No newline at end of file +HYDRA_FULL_ERROR=1 diff --git a/.gitignore b/.gitignore index 38effe58..1c4747d6 100755 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ tmp/ tests/data/toy_dataset.hdf5 +# new trained model and config +trained_model_assets/LargeDataset-LidarHD-100mPatch_7cl_randlanet-Epoch_072.ckpt +trained_model_assets/LargeDataset-LidarHD-100mPatch_7cl_randlanet-Epoch_072-inference-Myria3DV3.10.0.yaml + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b3ba5e8..7cf1e43a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,15 @@ # CHANGELOG +## 3.10.0 +- Add config to handle 100 m x 100 m patches + +- Add Focal and Lovasz Losses and config + +- Add preprocessing steps during hdf5 creation as they do not change at batch selection level (for example,GridSampling will always select the same points as it is deterministic) + +- Add config parameters to set model size, i.e a larger model for a larger dataset -### Not yet released - Check that las/laz output can be read by pdal when running prediction. + - Clean mamba build to reduce Docker image size (14.6GB to 9.2GB) ### 3.9.1 diff --git a/README.md b/README.md index 5f20fc9f..86d5b2f5 100755 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ Myria3D is a deep learning library designed with a focused scope: the multiclass The library implements the training of 3D Segmentation neural networks, with optimized data-processing and evaluation logics at fit time. Inference on unseen, large scale point cloud is also supported. It allows for the evaluation of single-class IoU on the full point cloud, which results in reliable model evaluation. -Myria3D is built upon [PyTorch](https://pytorch.org/). It keeps the standard data format -from [Pytorch-Geometric](https://pytorch-geometric.readthedocs.io/). +Myria3D is built upon [PyTorch](https://pytorch.org/). It keeps the standard data format +from [Pytorch-Geometric](https://pytorch-geometric.readthedocs.io/). Its structure was bootstraped from [this code template](https://github.com/ashleve/lightning-hydra-template), which heavily relies on [Hydra](https://hydra.cc/) and [Pytorch-Lightning](https://github.com/PyTorchLightning/pytorch-lightning) to enable flexible and rapid iterations of deep learning experiments. -Although the library can be extended with new neural network architectures or new data signatures, it makes some opiniated choices in terms of neural network architecture, data processing logics, and inference logic. Indeed, it is initially built with the [French "Lidar HD" project](https://geoservices.ign.fr/lidarhd) in mind, with the ambition to map France in 3D with 10 pulse/m² aerial Lidar by 2025. The data will be openly available, including a semantic segmentation with a minimal number of classes: ground, vegetation, buildings, vehicles, bridges, others. +Although the library can be extended with new neural network architectures or new data signatures, it makes some opiniated choices in terms of neural network architecture, data processing logics, and inference logic. Indeed, it is initially built with the [French "Lidar HD" project](https://geoservices.ign.fr/lidarhd) in mind, with the ambition to map France in 3D with 10 pulse/m² aerial Lidar by 2025. The data will be openly available, including a semantic segmentation with a minimal number of classes: ground, vegetation, buildings, vehicles, bridges, others. > → For installation and usage, please refer to [**Documentation**](https://ignf.github.io/myria3d/). @@ -41,4 +41,4 @@ Please cite Myria3D if it helped your own research. Here is an example BibTex en year={2022}, note={IGN (French Mapping Agency)}, } -``` \ No newline at end of file +``` diff --git a/configs/callbacks/default.yaml b/configs/callbacks/default.yaml index 625f5016..7db8adf9 100755 --- a/configs/callbacks/default.yaml +++ b/configs/callbacks/default.yaml @@ -1,4 +1,4 @@ -# Override with "none" to deactivate. +# Override with "none" to deactivate. log_code: _target_: myria3d.callbacks.comet_callbacks.LogCode @@ -8,7 +8,7 @@ log_logs_dir: _target_: myria3d.callbacks.comet_callbacks.LogLogsPath lr_monitor: - _target_: pytorch_lightning.callbacks.LearningRateMonitor + _target_: pytorch_lightning.callbacks.LearningRateMonitor logging_interval: "step" log_momentum: true @@ -16,7 +16,7 @@ model_checkpoint: _target_: pytorch_lightning.callbacks.ModelCheckpoint monitor: "val/loss_epoch" # name of the logged metric which determines when model is improving mode: "min" # can be "max" or "min" - save_top_k: 1 # save k best models (determined by above metric) + save_top_k: 5 # save k best models (determined by above metric) save_last: True # additionaly always save model from last epoch verbose: True dirpath: "checkpoints/" @@ -29,7 +29,7 @@ early_stopping: mode: "min" # can be "max" or "min" patience: 6 # how many validation epochs of not improving until training stops min_delta: 0 # minimum change in the monitored metric needed to qualify as an improvement - + model_detailed_metrics: _target_: myria3d.callbacks.metric_callbacks.ModelMetrics - num_classes: ${model.num_classes} \ No newline at end of file + num_classes: ${model.num_classes} diff --git a/configs/callbacks/finetuning.yaml b/configs/callbacks/finetuning.yaml index 4bde1b27..1e3c8b98 100644 --- a/configs/callbacks/finetuning.yaml +++ b/configs/callbacks/finetuning.yaml @@ -1,10 +1,10 @@ defaults: - default.yaml -# Override +# Override finetune: _target_: myria3d.callbacks.finetuning_callbacks.FinetuningFreezeUnfreeze d_in: ${model.d_in} num_classes: ${model.num_classes} - unfreeze_fc_end_epoch: ??? # full end fc is unfrozen later than the last MLP. - unfreeze_decoder_train_epoch: ??? # decoder is unfrozen later than last FC layer. \ No newline at end of file + unfreeze_fc_end_epoch: ??? # full end fc is unfrozen later than the last MLP. + unfreeze_decoder_train_epoch: ??? # decoder is unfrozen later than last FC layer. diff --git a/configs/config.yaml b/configs/config.yaml index 13a07eb7..13798b1f 100755 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -23,19 +23,19 @@ ignore_warnings: True # specify here default training configuration defaults: - - _self_ # for hydra legacy reasons + - _self_ # for hydra legacy reasons - trainer: default.yaml - datamodule: hdf5_datamodule.yaml - - dataset_description: 20220607_151_dalles_proto.yaml # describes input features and classes + - dataset_description: 20220607_151_dalles_proto.yaml # describes input features and classes - callbacks: default.yaml # set this to null if you don't want to use callbacks - model: pyg_randla_net_model.yaml - + - logger: comet # set logger here or use command line (e.g. `python run.py logger=wandb`) - task: default.yaml - predict: default.yaml - - experiment: RandLaNetDebug.yaml # default run is for debugging. + - experiment: RandLaNetDebug.yaml # default run is for debugging. - hydra: default.yaml diff --git a/configs/datamodule/hdf5_datamodule.yaml b/configs/datamodule/hdf5_datamodule.yaml index acbd93ed..a3b30260 100755 --- a/configs/datamodule/hdf5_datamodule.yaml +++ b/configs/datamodule/hdf5_datamodule.yaml @@ -1,10 +1,10 @@ _target_: myria3d.pctl.datamodule.hdf5.HDF5LidarDataModule # A standalone precomputed HDF5 dataset can be used without source files by setting data_dir and split_csv_path to null. -data_dir: null # data directory containing LAS files +data_dir: null # data directory containing LAS files epsg: null -split_csv_path: null # csv specifying split, with schema basename, split (where split is one out of train/val/test) -hdf5_file_path: "path/to/dataset_file.hdf5" # where to create a HDF5 dataset file from LAS and CSV sources. +split_csv_path: null # csv specifying split, with schema basename, split (where split is one out of train/val/test) +hdf5_file_path: "path/to/dataset_file.hdf5" # where to create a HDF5 dataset file from LAS and CSV sources. # functions used to load and preprocess LAS data points into a pytorch geometric Data object. points_pre_transform: @@ -14,7 +14,7 @@ points_pre_transform: pre_filter: _target_: functools.partial - _args_: + _args_: - "${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points}" min_num_nodes: 1 @@ -26,6 +26,8 @@ subtile_overlap_predict: "${predict.subtile_overlap}" batch_size: 32 num_workers: 3 prefetch_factor: 3 +pin_memory: true +persistent_workers: true defaults: - transforms: default.yaml diff --git a/configs/datamodule/transforms/augmentations/heavy.yaml b/configs/datamodule/transforms/augmentations/heavy.yaml index 13f0fa7e..03502d24 100644 --- a/configs/datamodule/transforms/augmentations/heavy.yaml +++ b/configs/datamodule/transforms/augmentations/heavy.yaml @@ -2,7 +2,6 @@ defaults: - light.yaml - # Rotation around axis - sample should be centered for this. RandomRotate: _target_: torch_geometric.transforms.RandomRotate diff --git a/configs/datamodule/transforms/augmentations/light.yaml b/configs/datamodule/transforms/augmentations/light.yaml index ac0247f5..9011266e 100644 --- a/configs/datamodule/transforms/augmentations/light.yaml +++ b/configs/datamodule/transforms/augmentations/light.yaml @@ -3,9 +3,9 @@ x_flip: _args_: - 0 p: 0.5 - + y_flip: _target_: torch_geometric.transforms.RandomFlip _args_: - 1 - p: 0.5 \ No newline at end of file + p: 0.5 diff --git a/configs/datamodule/transforms/default.yaml b/configs/datamodule/transforms/default.yaml index 59605c5c..1088d688 100644 --- a/configs/datamodule/transforms/default.yaml +++ b/configs/datamodule/transforms/default.yaml @@ -1,5 +1,5 @@ defaults: - - preparations: points_budget.yaml + - preparations: points_budget_patch_100mx100m.yaml - augmentations: none.yaml - normalizations: default.yaml @@ -13,4 +13,7 @@ preparations_train_list: "${oc.dict.values: datamodule.transforms.preparations.t preparations_eval_list: "${oc.dict.values: datamodule.transforms.preparations.eval}" preparations_predict_list: "${oc.dict.values: datamodule.transforms.preparations.predict}" +# Deterministic transforms applied once, at HDF5-creation time, for the TRAIN split only. +preparations_train_bake_list: "${oc.dict.values: datamodule.transforms.preparations.train_bake}" + normalizations_list: "${oc.dict.values: datamodule.transforms.normalizations}" diff --git a/configs/datamodule/transforms/normalizations/default.yaml b/configs/datamodule/transforms/normalizations/default.yaml index 1dece389..e80c3b2b 100644 --- a/configs/datamodule/transforms/normalizations/default.yaml +++ b/configs/datamodule/transforms/normalizations/default.yaml @@ -8,4 +8,4 @@ NormalizePos: subtile_width: "${datamodule.subtile_width}" StandardizeRGBAndIntensity: - _target_: myria3d.pctl.transforms.transforms.StandardizeRGBAndIntensity \ No newline at end of file + _target_: myria3d.pctl.transforms.transforms.StandardizeRGBAndIntensity diff --git a/configs/datamodule/transforms/preparations/fixed_num_points.yaml b/configs/datamodule/transforms/preparations/fixed_num_points.yaml index a7534fa2..dace64db 100644 --- a/configs/datamodule/transforms/preparations/fixed_num_points.yaml +++ b/configs/datamodule/transforms/preparations/fixed_num_points.yaml @@ -1,7 +1,10 @@ # default preparations with grid sampling and random sampling. -train: - +# Deterministic transforms baked into the HDF5 file once, at dataset-creation time, +# for the TRAIN split only (applied in create_hdf5). They are therefore NOT recomputed +# every epoch, which is a large speed-up on big datasets. +# NB: regenerate the HDF5 file after changing this list. +train_bake: TargetTransform: _target_: myria3d.pctl.transforms.transforms.TargetTransform _args_: @@ -16,6 +19,7 @@ train: _args_: - 0.25 +train: FixedPoints: _target_: torch_geometric.transforms.FixedPoints _args_: @@ -27,7 +31,6 @@ train: _target_: torch_geometric.transforms.Center eval: - CopyFullPos: _target_: myria3d.pctl.transforms.transforms.CopyFullPos @@ -61,9 +64,8 @@ eval: Center: _target_: torch_geometric.transforms.Center - -predict: +predict: CopyFullPos: _target_: myria3d.pctl.transforms.transforms.CopyFullPos @@ -86,4 +88,4 @@ predict: _target_: myria3d.pctl.transforms.transforms.CopySampledPos Center: - _target_: torch_geometric.transforms.Center \ No newline at end of file + _target_: torch_geometric.transforms.Center diff --git a/configs/datamodule/transforms/preparations/points_budget.yaml b/configs/datamodule/transforms/preparations/points_budget.yaml index f34b843d..4786cd46 100644 --- a/configs/datamodule/transforms/preparations/points_budget.yaml +++ b/configs/datamodule/transforms/preparations/points_budget.yaml @@ -1,7 +1,17 @@ # default preparations with grid sampling and random sampling. -train: - +# Maximum number of nodes (points) kept per subtile after sampling. +# This is the ONLY value that differs between points budget variants, so it is +# exposed here as a single parameter. Override it on the CLI with: +# datamodule.transforms.preparations.max_num_nodes=120000 +# or inherit this file and override just this value (see points_budget_patch_100mx100m.yaml). +max_num_nodes: 40000 + +# Deterministic transforms baked into the HDF5 file once, at dataset-creation time, +# for the TRAIN split only (applied in create_hdf5). They are therefore NOT recomputed +# every epoch, which is a large speed-up on big datasets. +# NB: regenerate the HDF5 file after changing this list. +train_bake: TargetTransform: _target_: myria3d.pctl.transforms.transforms.TargetTransform _args_: @@ -15,7 +25,8 @@ train: _target_: torch_geometric.transforms.GridSampling _args_: - 0.25 - + +train: MinimumNumNodes: _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes _args_: @@ -24,13 +35,12 @@ train: MaximumNumNodes: _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes _args_: - - 40000 + - ${datamodule.transforms.preparations.max_num_nodes} Center: _target_: torch_geometric.transforms.Center eval: - TargetTransform: _target_: myria3d.pctl.transforms.transforms.TargetTransform _args_: @@ -59,16 +69,15 @@ eval: MaximumNumNodes: _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes _args_: - - 40000 + - ${datamodule.transforms.preparations.max_num_nodes} CopySampledPos: _target_: myria3d.pctl.transforms.transforms.CopySampledPos Center: _target_: torch_geometric.transforms.Center - -predict: +predict: DropPointsByClass: _target_: myria3d.pctl.transforms.transforms.DropPointsByClass @@ -88,10 +97,10 @@ predict: MaximumNumNodes: _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes _args_: - - 40000 + - ${datamodule.transforms.preparations.max_num_nodes} CopySampledPos: _target_: myria3d.pctl.transforms.transforms.CopySampledPos Center: - _target_: torch_geometric.transforms.Center \ No newline at end of file + _target_: torch_geometric.transforms.Center diff --git a/configs/datamodule/transforms/preparations/points_budget_patch_100mx100m.yaml b/configs/datamodule/transforms/preparations/points_budget_patch_100mx100m.yaml new file mode 100644 index 00000000..9d3dac1e --- /dev/null +++ b/configs/datamodule/transforms/preparations/points_budget_patch_100mx100m.yaml @@ -0,0 +1,8 @@ +# Same preparations as points_budget.yaml, but with a larger points budget suited +# to 100m x 100m patches. It inherits everything from points_budget.yaml and only +# overrides the single value that differs between the two: max_num_nodes. +defaults: + - points_budget + - _self_ + +max_num_nodes: 120000 diff --git a/configs/dataset_description/20220204_BuildingValidation_and_Ground.yaml b/configs/dataset_description/20220204_BuildingValidation_and_Ground.yaml index 8f955119..12df4951 100644 --- a/configs/dataset_description/20220204_BuildingValidation_and_Ground.yaml +++ b/configs/dataset_description/20220204_BuildingValidation_and_Ground.yaml @@ -1,9 +1,9 @@ -_convert_: all # For omegaconf struct to be converted to python dictionnaries +_convert_: all # For omegaconf struct to be converted to python dictionnaries # classification_preprocessing_dict = {source_class_code_int: target_class_code_int}, -classification_preprocessing_dict: {59: 6, 50: 1} +classification_preprocessing_dict: { 59: 6, 50: 1 } # classification_dict = {code_int: name_str, ...} and MUST be sorted (increasing order). -classification_dict: {1: "unclassified", 2: "ground", 6: "building"} +classification_dict: { 1: "unclassified", 2: "ground", 6: "building" } # Input and output dims of neural net are dataset dependant: d_in: 12 -num_classes: 3 \ No newline at end of file +num_classes: 3 diff --git a/configs/dataset_description/20220607_151_dalles_proto.yaml b/configs/dataset_description/20220607_151_dalles_proto.yaml index f6d60956..84f12986 100644 --- a/configs/dataset_description/20220607_151_dalles_proto.yaml +++ b/configs/dataset_description/20220607_151_dalles_proto.yaml @@ -1,4 +1,4 @@ -_convert_: all # For omegaconf struct to be converted to python dictionnaries +_convert_: all # For omegaconf struct to be converted to python dictionnaries # classification_preprocessing_dict = {source_class_code_int: target_class_code_int}, # 3: medium vegetation -> vegetation # 4: high vegetation -> vegetation @@ -6,17 +6,53 @@ _convert_: all # For omegaconf struct to be converted to python dictionnaries # 161: wind_turbines -> lasting_above # 162: pylon -> lasting_above # Note: other codes are also mapped to 1 (unclassified) : they were left in this dataset during its creation, and are a minority. -# We do not drop them (i.e. we do not map them to the reserved noise code (65) during training) +# We do not drop them (i.e. we do not map them to the reserved noise code (65) during training) # in order to avoid unintended conflict in production. -classification_preprocessing_dict: {3: 5, 4: 5, 160: 64, 161: 64, 162: 64, 0: 1, 7: 1, 46: 1, 47: 1, 48: 1, 49: 1, 50: 1, 51: 1, 52: 1, 53: 1, 54: 1, 55: 1, 56: 1, 57: 1, 58: 1, 66: 1, 67: 1, 77: 1, 155: 1, 204: 1} +classification_preprocessing_dict: + { + 3: 5, + 4: 5, + 160: 64, + 161: 64, + 162: 64, + 0: 1, + 7: 1, + 46: 1, + 47: 1, + 48: 1, + 49: 1, + 50: 1, + 51: 1, + 52: 1, + 53: 1, + 54: 1, + 55: 1, + 56: 1, + 57: 1, + 58: 1, + 66: 1, + 67: 1, + 77: 1, + 155: 1, + 204: 1, + } # classification_dict = {code_int: name_str, ...} and MUST be sorted (increasing order). -classification_dict: {1: "unclassified", 2: "ground", 5: vegetation, 6: "building", 9: water, 17: bridge, 64: lasting_above} +classification_dict: + { + 1: "unclassified", + 2: "ground", + 5: vegetation, + 6: "building", + 9: water, + 17: bridge, + 64: lasting_above, + } # class_weights for the CrossEntropyLoss with format "[[w1,w2,w3...,wk]]" with w_i a float e.g. 1.0 # Balanced CE: arbitrary weights based on heuristic. # class_weights: [2.5,1.0,1.0,5.0,20.0,20.0,20.0] normalized so they sum to 7 to preserve scale of CELoss -class_weights: [0.25,0.1,0.1,0.5,2.0,2.0,2.0] +class_weights: [0.25, 0.1, 0.1, 0.5, 2.0, 2.0, 2.0] # Input and output dims of neural net are dataset dependant: d_in: 9 num_classes: 7 diff --git a/configs/dataset_description/20230601_lidarhd_pacasam_dataset.yaml b/configs/dataset_description/20230601_lidarhd_pacasam_dataset.yaml index 98ef0c72..dbdb2887 100644 --- a/configs/dataset_description/20230601_lidarhd_pacasam_dataset.yaml +++ b/configs/dataset_description/20230601_lidarhd_pacasam_dataset.yaml @@ -1,4 +1,4 @@ -_convert_: all # For omegaconf struct to be converted to python dictionnaries +_convert_: all # For omegaconf struct to be converted to python dictionnaries # classification_preprocessing_dict = {source_class_code_int: target_class_code_int}, # 3: medium vegetation -> vegetation # 4: high vegetation -> vegetation @@ -6,11 +6,20 @@ _convert_: all # For omegaconf struct to be converted to python dictionnaries # 66: synthetic points --> noise (synthetic points are useful for specific modelling task on already classified data). # We set them to noise so that they are ignored during training. # Codes that should not have been in the data: 100, 101. -classification_preprocessing_dict: {3: 5, 4: 5, 0: 1, 66: 65, 100: 1, 101: 1} +classification_preprocessing_dict: { 3: 5, 4: 5, 0: 1, 66: 65, 100: 1, 101: 1 } # classification_dict = {code_int: name_str, ...} and MUST be sorted (increasing order). -classification_dict: {1: "unclassified", 2: "ground", 5: vegetation, 6: "building", 9: water, 17: bridge, 64: lasting_above} +classification_dict: + { + 1: "unclassified", + 2: "ground", + 5: vegetation, + 6: "building", + 9: water, + 17: bridge, + 64: lasting_above, + } # Input and output dims of neural net are dataset dependant: d_in: 9 -num_classes: 7 \ No newline at end of file +num_classes: 7 diff --git a/configs/dataset_description/20260703_lidarhd_pacasam_dataset.yaml b/configs/dataset_description/20260703_lidarhd_pacasam_dataset.yaml new file mode 100644 index 00000000..c3787703 --- /dev/null +++ b/configs/dataset_description/20260703_lidarhd_pacasam_dataset.yaml @@ -0,0 +1,33 @@ +_convert_: all # For omegaconf struct to be converted to python dictionnaries +# classification_preprocessing_dict = {source_class_code_int: target_class_code_int}, +# 3: medium vegetation -> vegetation +# 4: high vegetation -> vegetation +# 0: ignored +# 66: synthetic points --> noise (synthetic points are useful for specific modelling task on already classified data). +# We set them to noise so that they are ignored during training. +# 67, 100, 101: ignored + +# Compared to 20230601_lidarhd_pacasam_dataset.yaml, we added 67 to the ignored classes as the new dataset includes points of class 67 +# We also added class_weights for the loss function. + +classification_preprocessing_dict: + { 3: 5, 4: 5, 0: 1, 66: 65, 67: 1, 100: 1, 101: 1 } + +# classification_dict = {code_int: name_str, ...} and MUST be sorted (increasing order). +classification_dict: + { + 1: "unclassified", + 2: "ground", + 5: "vegetation", + 6: "building", + 9: "water", + 17: "bridge", + 64: "lasting_above", + } + +# unclassified is ambiguous anyway, so we set its weight to 1.0. The other weights are normalized so that they sum to 7.0 to preserve the scale of the loss. +class_weights: [1.0, 0.1476, 0.0947, 1.0734, 1.4211, 1.4211, 1.4211] + +# Input and output dims of neural net are dataset dependant: +d_in: 9 +num_classes: 7 diff --git a/configs/experiment/DebugFineTune.yaml b/configs/experiment/DebugFineTune.yaml index d90bc832..ca280eaa 100644 --- a/configs/experiment/DebugFineTune.yaml +++ b/configs/experiment/DebugFineTune.yaml @@ -25,4 +25,4 @@ callbacks: unfreeze_decoder_train_epoch: 3 model: - lr_scheduler: null + lr_scheduler: null diff --git a/configs/experiment/RandLaNet-Overfit.yaml b/configs/experiment/RandLaNet-Overfit.yaml index 384b1667..f74ef36e 100755 --- a/configs/experiment/RandLaNet-Overfit.yaml +++ b/configs/experiment/RandLaNet-Overfit.yaml @@ -5,7 +5,6 @@ defaults: - override /model: pyg_randla_net_model.yaml - override /datamodule/transforms/augmentations: none.yaml - logger: comet: experiment_name: "RandLaNet-Overfit" diff --git a/configs/experiment/RandLaNetDebug.yaml b/configs/experiment/RandLaNetDebug.yaml index c50424f0..36ed5a8b 100755 --- a/configs/experiment/RandLaNetDebug.yaml +++ b/configs/experiment/RandLaNetDebug.yaml @@ -21,4 +21,3 @@ datamodule: num_workers: 1 # runtime.cwd is where application is run from e.g. where run.py is. hdf5_file_path: "${hydra:runtime.cwd}/tests/data/toy_dataset.hdf5" - diff --git a/configs/experiment/RandLaNet_base_run_FR-2x3GPUs.yaml b/configs/experiment/RandLaNet_base_run_FR-2x3GPUs.yaml index 03c8511e..70401c55 100755 --- a/configs/experiment/RandLaNet_base_run_FR-2x3GPUs.yaml +++ b/configs/experiment/RandLaNet_base_run_FR-2x3GPUs.yaml @@ -6,8 +6,7 @@ logger: comet: experiment_name: "RandLaNet_base_run_FR-2x3GPUs" - -# 2 nodes x 3 GPUs - No gradient accumulation. +# 2 nodes x 3 GPUs - No gradient accumulation. # This is equivalent to training with 2 GPUs with gradients accumulated 3 times. # Setting precision=16 did not bring any speed improvement for Lidar HD data and RandLa-Net model. trainer: diff --git a/configs/experiment/RandLaNet_base_run_FR-SQRTOfInverseFreqencyClassWeighting.yaml b/configs/experiment/RandLaNet_base_run_FR-SQRTOfInverseFreqencyClassWeighting.yaml index 131ce2b1..6fe675e2 100755 --- a/configs/experiment/RandLaNet_base_run_FR-SQRTOfInverseFreqencyClassWeighting.yaml +++ b/configs/experiment/RandLaNet_base_run_FR-SQRTOfInverseFreqencyClassWeighting.yaml @@ -9,4 +9,4 @@ logger: dataset_description: # Sqrt(Inverse Frequency) of classes in defaut dataset (a.k.a. `151proto`). - class_weights: [0.19,0.08,0.08,0.36,1.13,3.11,2.05] \ No newline at end of file + class_weights: [0.19, 0.08, 0.08, 0.36, 1.13, 3.11, 2.05] diff --git a/configs/experiment/RandLaNet_base_run_FR.yaml b/configs/experiment/RandLaNet_base_run_FR.yaml index 7949a2aa..7e827a79 100755 --- a/configs/experiment/RandLaNet_base_run_FR.yaml +++ b/configs/experiment/RandLaNet_base_run_FR.yaml @@ -6,7 +6,6 @@ logger: comet: experiment_name: "RandLaNet_base_run_FR-(BatchSize10xBudget(300pts-40000pts))" - # Smaller BS : 10 x 40 000 (max) == 400 000 pts i.e. previous budget of 32 x 12 500pts. datamodule: batch_size: 10 @@ -15,5 +14,5 @@ trainer: num_sanity_val_steps: 2 min_epochs: 100 max_epochs: 150 - accumulate_grad_batches: 3 # b/c larger clouds will not fit in memory with original Batch Size + accumulate_grad_batches: 3 # b/c larger clouds will not fit in memory with original Batch Size # gpus: [1] diff --git a/configs/experiment/predict.yaml b/configs/experiment/predict.yaml index 0c1a6f4c..b159c451 100755 --- a/configs/experiment/predict.yaml +++ b/configs/experiment/predict.yaml @@ -10,9 +10,9 @@ task: task_name: "predict" predict: - src_las: "/path/to/input.las" # Any glob pattern can be used to predict on multiple files. - output_dir: "/path/to/output_dir/" # Predictions are saved in a new file which shares src_las basename. - ckpt_path: "/path/to/lightning_model.ckpt" # Checkpoint of trained model. + src_las: "/path/to/input.las" # Any glob pattern can be used to predict on multiple files. + output_dir: "/path/to/output_dir/" # Predictions are saved in a new file which shares src_las basename. + ckpt_path: "/path/to/lightning_model.ckpt" # Checkpoint of trained model. # + other params in configs/predict/default.yaml hydra: @@ -20,4 +20,4 @@ hydra: datamodule: # larger batch size for speed - batch_size: 50 \ No newline at end of file + batch_size: 50 diff --git a/configs/experiment/test.yaml b/configs/experiment/test.yaml index 078217e3..62676344 100755 --- a/configs/experiment/test.yaml +++ b/configs/experiment/test.yaml @@ -15,12 +15,12 @@ datamodule: # override if needed # test_data_dir: "path/to_dir/with/annotated/files.las" # larger batch size for speed - batch_size: 50 + batch_size: 50 model: - ckpt_path: "path/to/checkpoint" # override + ckpt_path: "path/to/checkpoint" # override trainer: num_sanity_val_steps: 0 min_epochs: 1 - max_epochs: 100000 \ No newline at end of file + max_epochs: 100000 diff --git a/configs/logger/csv.yaml b/configs/logger/csv.yaml index 25b80bf6..4f0358d5 100644 --- a/configs/logger/csv.yaml +++ b/configs/logger/csv.yaml @@ -4,4 +4,4 @@ csv: _target_: pytorch_lightning.loggers.csv_logs.CSVLogger save_dir: "${hydra:run.dir}" name: "csv/" - prefix: "" \ No newline at end of file + prefix: "" diff --git a/configs/model/criterion/CrossEntropyLoss.yaml b/configs/model/criterion/CrossEntropyLoss.yaml index 63f22ffe..6a3c84c0 100644 --- a/configs/model/criterion/CrossEntropyLoss.yaml +++ b/configs/model/criterion/CrossEntropyLoss.yaml @@ -1,3 +1,3 @@ _target_: torch.nn.CrossEntropyLoss label_smoothing: 0.0 -ignore_index: 65 # artefacts are mapped to 65 by convention \ No newline at end of file +ignore_index: 65 # artefacts are mapped to 65 by convention diff --git a/configs/model/criterion/FocalLoss.yaml b/configs/model/criterion/FocalLoss.yaml new file mode 100644 index 00000000..cb7fa864 --- /dev/null +++ b/configs/model/criterion/FocalLoss.yaml @@ -0,0 +1,4 @@ +_target_: myria3d.models.losses.focal.FocalLoss +gamma: 2.0 +ignore_index: 65 # artefacts are mapped to 65 by convention +reduction: mean diff --git a/configs/model/criterion/FocalLossAndLovaszLoss.yaml b/configs/model/criterion/FocalLossAndLovaszLoss.yaml new file mode 100644 index 00000000..6fa9d91f --- /dev/null +++ b/configs/model/criterion/FocalLossAndLovaszLoss.yaml @@ -0,0 +1,17 @@ +_target_: myria3d.models.losses.combined.CombinedLoss +# Relative weights of each loss in the final sum. +weights: + - 1.0 # Focal + - 1.0 # lovasz +losses: + - _target_: myria3d.models.losses.focal.FocalLoss + gamma: 2.0 + ignore_index: 65 # artefacts are mapped to 65 by convention + reduction: mean + alpha: + _target_: torch.FloatTensor + _args_: + - ${dataset_description.class_weights} + - _target_: myria3d.models.losses.lovasz.LovaszLoss + mode: multiclass + ignore_index: 65 # artefacts are mapped to 65 by convention diff --git a/configs/model/criterion/LovazLoss.yaml b/configs/model/criterion/LovazLoss.yaml new file mode 100644 index 00000000..4d11a5b7 --- /dev/null +++ b/configs/model/criterion/LovazLoss.yaml @@ -0,0 +1,3 @@ +_target_: myria3d.models.losses.lovasz.LovaszLoss +mode: multiclass +ignore_index: 65 # artefacts are mapped to 65 by convention diff --git a/configs/model/criterion/WeightedCrossEntropyAndLovaszLoss.yaml b/configs/model/criterion/WeightedCrossEntropyAndLovaszLoss.yaml new file mode 100644 index 00000000..2251c0d1 --- /dev/null +++ b/configs/model/criterion/WeightedCrossEntropyAndLovaszLoss.yaml @@ -0,0 +1,16 @@ +_target_: myria3d.models.losses.combined.CombinedLoss +# Relative weights of each loss in the final sum. +weights: + - 1.0 # weighted cross-entropy + - 1.0 # lovasz +losses: + - _target_: torch.nn.CrossEntropyLoss + label_smoothing: 0.0 + ignore_index: 65 # artefacts are mapped to 65 by convention + weight: + _target_: torch.FloatTensor + _args_: + - ${dataset_description.class_weights} + - _target_: myria3d.models.losses.lovasz.LovaszLoss + mode: multiclass + ignore_index: 65 # artefacts are mapped to 65 by convention diff --git a/configs/model/criterion/WeightedCrossEntropyLoss.yaml b/configs/model/criterion/WeightedCrossEntropyLoss.yaml index 893dc4bf..fafd860a 100644 --- a/configs/model/criterion/WeightedCrossEntropyLoss.yaml +++ b/configs/model/criterion/WeightedCrossEntropyLoss.yaml @@ -1,7 +1,7 @@ _target_: torch.nn.CrossEntropyLoss label_smoothing: 0.0 -ignore_index: 65 # artefacts are mapped to 65 by convention +ignore_index: 65 # artefacts are mapped to 65 by convention weight: _target_: torch.FloatTensor - _args_: - - ${dataset_description.class_weights} \ No newline at end of file + _args_: + - ${dataset_description.class_weights} diff --git a/configs/model/criterion/WeightedFocalLoss.yaml b/configs/model/criterion/WeightedFocalLoss.yaml new file mode 100644 index 00000000..7fe8b1ec --- /dev/null +++ b/configs/model/criterion/WeightedFocalLoss.yaml @@ -0,0 +1,8 @@ +_target_: myria3d.models.losses.focal.FocalLoss +gamma: 2.0 +ignore_index: 65 # artefacts are mapped to 65 by convention +reduction: mean +alpha: + _target_: torch.FloatTensor + _args_: + - ${dataset_description.class_weights} diff --git a/configs/model/default.yaml b/configs/model/default.yaml index 2223d6fd..986622be 100644 --- a/configs/model/default.yaml +++ b/configs/model/default.yaml @@ -1,21 +1,21 @@ _target_: myria3d.models.model.Model ## Inputs and outputs -d_in: ${dataset_description.d_in} # XYZ (3) + Other features (N) +d_in: ${dataset_description.d_in} # XYZ (3) + Other features (N) num_classes: ${dataset_description.num_classes} classification_dict: ${dataset_description.classification_dict} # Architecture defined in sub-configs -ckpt_path: null # str, for resuming training and finetuning. +ckpt_path: null # str, for resuming training and finetuning. neural_net_class_name: ??? neural_net_hparams: ??? # Interpolation params -interpolation_k: ${predict.interpolator.interpolation_k} # interpolation at eval time -num_workers: 4 # for knn_interpolate +interpolation_k: ${predict.interpolator.interpolation_k} # interpolation at eval time +num_workers: 4 # for knn_interpolate ## Optimization -momentum: 0.9 # arbitrary +momentum: 0.9 # arbitrary monitor: "val/loss_epoch" defaults: diff --git a/configs/model/lr_scheduler/OneCycleLR.yaml b/configs/model/lr_scheduler/OneCycleLR.yaml index 6138d61a..ed92cfba 100644 --- a/configs/model/lr_scheduler/OneCycleLR.yaml +++ b/configs/model/lr_scheduler/OneCycleLR.yaml @@ -1,12 +1,7 @@ _target_: functools.partial -_args_: +_args_: - ${get_method:torch.optim.lr_scheduler.OneCycleLR} -div_factor: 10 # from model.lr which whil be used direcly as it is positional -final_div_factor: 1000 # large = anhilihation -epochs: ${trainer.min_epochs} # min_epochs becomes the target and should equate max_epochs -steps_per_epoch: 818 # num_sample / batch_size, adpat to dataset -pct_start: 0.3 -cycle_momentum: true -base_momentum: 0.85 -max_momentum: 0.95 -verbose: true # for debug +max_lr: ${model.lr} # peak LR (required); total_steps is computed automatically in model.py +div_factor: 10 # initial_lr = max_lr / div_factor +final_div_factor: 1000 # min_lr = initial_lr / final_div_factor (large = annihilation) +pct_start: 0.05 # fraction of steps spent warming up diff --git a/configs/model/lr_scheduler/ReduceLROnPlateau.yaml b/configs/model/lr_scheduler/ReduceLROnPlateau.yaml index dbfb3453..59dda5ce 100644 --- a/configs/model/lr_scheduler/ReduceLROnPlateau.yaml +++ b/configs/model/lr_scheduler/ReduceLROnPlateau.yaml @@ -1,8 +1,8 @@ _target_: functools.partial -_args_: +_args_: - ${get_method:torch.optim.lr_scheduler.ReduceLROnPlateau} mode: "min" -factor: 0.5 # halving -patience: 20 # val epochs -cooldown: 5 # val epochs -verbose: true # print when reducing LR \ No newline at end of file +factor: 0.5 # halving +patience: 20 # val epochs +cooldown: 5 # val epochs +verbose: true # print when reducing LR diff --git a/configs/model/optimizer/Adam.yaml b/configs/model/optimizer/Adam.yaml index dcfdf7c4..a3b68778 100644 --- a/configs/model/optimizer/Adam.yaml +++ b/configs/model/optimizer/Adam.yaml @@ -1,4 +1,4 @@ _target_: functools.partial -_args_: +_args_: - "${get_method:torch.optim.Adam}" -lr: ${model.lr} # not needed since we give it in model.py ? \ No newline at end of file +lr: ${model.lr} # not needed since we give it in model.py ? diff --git a/configs/model/optimizer/AdamW.yaml b/configs/model/optimizer/AdamW.yaml index be4de190..d0e23335 100644 --- a/configs/model/optimizer/AdamW.yaml +++ b/configs/model/optimizer/AdamW.yaml @@ -1,4 +1,4 @@ _target_: functools.partial -_args_: +_args_: - "${get_method:torch.optim.AdamW}" -lr: ${model.lr} # not needed since we give it in model.py ? \ No newline at end of file +lr: ${model.lr} # not needed since we give it in model.py ? diff --git a/configs/model/optimizer/SGD.yaml b/configs/model/optimizer/SGD.yaml index 3cf8c31b..64acb33b 100644 --- a/configs/model/optimizer/SGD.yaml +++ b/configs/model/optimizer/SGD.yaml @@ -1,5 +1,5 @@ _target_: functools.partial -_args_: +_args_: - "${get_method:torch.optim.SGD}" lr: ${model.lr} -momentum: ${model.momentum} \ No newline at end of file +momentum: ${model.momentum} diff --git a/configs/model/pyg_randla_net_model.yaml b/configs/model/pyg_randla_net_model.yaml index 46d43953..f793f125 100755 --- a/configs/model/pyg_randla_net_model.yaml +++ b/configs/model/pyg_randla_net_model.yaml @@ -1,12 +1,16 @@ defaults: - default.yaml -lr: 0.003933709606504788 # 200-long LR-test-range betw 10^-4 and 3.0 +lr: 0.002 neural_net_class_name: "PyGRandLANet" neural_net_hparams: - num_features: ${model.d_in} # 3 (xyz) + num of features + num_features: ${model.d_in} # 3 (xyz) + num of features num_classes: ${model.num_classes} num_neighbors: 16 - decimation: 4 # divide by decimation for each of the 4 local encoder. - return_logits: true # to use with crossEntropyLoss directly + decimation: 4 # divide by decimation for each of the 4 local encoder. + # Per-stage encoder output channels (block1..block4). Widen these to increase + # model capacity, e.g. [64, 256, 512, 1024] for a ~4x larger dataset. + # The decoder mirrors these dimensions automatically. + dims: [32, 128, 256, 512] + return_logits: true # to use with crossEntropyLoss directly diff --git a/configs/predict/default.yaml b/configs/predict/default.yaml index 4c078533..b0d0506c 100644 --- a/configs/predict/default.yaml +++ b/configs/predict/default.yaml @@ -1,6 +1,6 @@ -src_las: "/path/to/input.las" # Any glob pattern can be used to predict on multiple files. -output_dir: "/path/to/output_dir/" # Predictions are saved in a new file which shares src_las basename. -ckpt_path: "/path/to/lightning_model.ckpt" # Checkpoint of trained model. +src_las: "/path/to/input.las" # Any glob pattern can be used to predict on multiple files. +output_dir: "/path/to/output_dir/" # Predictions are saved in a new file which shares src_las basename. +ckpt_path: "/path/to/lightning_model.ckpt" # Checkpoint of trained model. gpus: 0 # Probas interpolation parameters @@ -22,8 +22,8 @@ interpolator: # In addition, these dimensions will be created # - Predicted classification based on argmax, with classes specified by dataset_description class dictionary. # Defaut name: `PredictedClassification`. Can be set to Classification to override its values. - # - Shannon entropy of predicted probabilities + # - Shannon entropy of predicted probabilities # Defaut name: `entropy` # Set to `null` to disable saving these values. predicted_classification_channel: PredictedClassification - entropy_channel: entropy \ No newline at end of file + entropy_channel: entropy diff --git a/configs/task/default.yaml b/configs/task/default.yaml index ef0d0e10..31f768bc 100644 --- a/configs/task/default.yaml +++ b/configs/task/default.yaml @@ -1,3 +1,3 @@ # Task at hand. Can be train or predict -task_name: fit # "fit" or "test" or "fit+test", or "predict", or "finetune" -auto_lr_find: false # override with true to run the LR-range test in train.py. +task_name: fit # "fit" or "test" or "fit+test", or "predict", or "finetune" +auto_lr_find: false # override with true to run the LR-range test in train.py. diff --git a/docs/Makefile b/docs/Makefile index f423fb47..b89f3220 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,6 +20,6 @@ help: # If you would regenerate all docs you could use the following, # which WILL ERASE MODIFIFCATIONs. -# sphinx-apidoc -o "$(SOURCEDIR)"/apidoc_new/ ./../ +# sphinx-apidoc -o "$(SOURCEDIR)"/apidoc_new/ ./../ %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/apidoc/myria3d.model.rst b/docs/source/apidoc/myria3d.model.rst index acac18db..895a9df4 100644 --- a/docs/source/apidoc/myria3d.model.rst +++ b/docs/source/apidoc/myria3d.model.rst @@ -13,4 +13,4 @@ Interpolation ------------------------------------- .. automodule:: myria3d.models.interpolation - :members: \ No newline at end of file + :members: diff --git a/docs/source/apidoc/myria3d.models.modules.rst b/docs/source/apidoc/myria3d.models.modules.rst index 5c5890e4..42c9f2c8 100644 --- a/docs/source/apidoc/myria3d.models.modules.rst +++ b/docs/source/apidoc/myria3d.models.modules.rst @@ -7,4 +7,4 @@ myria3d.models.modules Our own implementation of RandLA-Net, to process full point clouds of different sizes, making the best use of the `pytorch-geometric framework `_. .. autoclass:: myria3d.models.modules.pyg_randla_net.PyGRandLANet - :members: \ No newline at end of file + :members: diff --git a/docs/source/apidoc/myria3d.pctl.rst b/docs/source/apidoc/myria3d.pctl.rst index bcb75b66..baec97ca 100644 --- a/docs/source/apidoc/myria3d.pctl.rst +++ b/docs/source/apidoc/myria3d.pctl.rst @@ -57,4 +57,3 @@ myria3d.pctl.transforms.transforms .. automodule:: myria3d.pctl.transforms.transforms :members: - diff --git a/docs/source/apidoc/myria3d.utils.rst b/docs/source/apidoc/myria3d.utils.rst index 0ee642ad..16a91dea 100644 --- a/docs/source/apidoc/myria3d.utils.rst +++ b/docs/source/apidoc/myria3d.utils.rst @@ -8,4 +8,3 @@ myria3d.utils.utils .. automodule:: myria3d.utils.utils :members: :show-inheritance: - diff --git a/docs/source/apidoc/scripts.rst b/docs/source/apidoc/scripts.rst index 35e3c838..f43776de 100644 --- a/docs/source/apidoc/scripts.rst +++ b/docs/source/apidoc/scripts.rst @@ -25,4 +25,4 @@ myria3d.predict .. automodule:: myria3d.predict :members: :undoc-members: - :show-inheritance: \ No newline at end of file + :show-inheritance: diff --git a/docs/source/background/general_design.md b/docs/source/background/general_design.md index c8cbe4bb..b6cb5fdf 100644 --- a/docs/source/background/general_design.md +++ b/docs/source/background/general_design.md @@ -5,7 +5,7 @@ Here are a few challenges relative to training 3D segmentation models for Aerial ## Model should be fast, performant, and practical **Situation**: -- Since the seminal PointNet architecture [[Qi 2016](https://arxiv.org/abs/1612.00593)] and the later [[PointNet++](https://arxiv.org/abs/1706.02413)], there were many attempts to improve these architecture which operate direcly on point clouds in a hierarchical fashion. +- Since the seminal PointNet architecture [[Qi 2016](https://arxiv.org/abs/1612.00593)] and the later [[PointNet++](https://arxiv.org/abs/1706.02413)], there were many attempts to improve these architecture which operate direcly on point clouds in a hierarchical fashion. - Our main requirements are: 1) Speed of inference, in order to operate at a national scale. 2) performances in large-scale outdoor Lidar settings, on e.g. [SemanticKITTI](http://semantic-kitti.org/) and [Semantic 3D](http://semantic3d.net/) benchmarks, by opposition to e.g. the [S3DIS](https://ieeexplore.ieee.org/document/7780539/) benchmark. @@ -13,9 +13,9 @@ Here are a few challenges relative to training 3D segmentation models for Aerial 4) Conceptual simplicity - following a simple hierarchical structure was preferred to more complex architectures. **Strategy**: -- We focused on the RandLA-Net architecture [Hu 2020](https://arxiv.org/abs/1911.11236). - - It was specifically conceived by authors with requirements (1) and (2) in mind. - - It meets requirement (2), by setting SOTA results on both Semantic3D and SemanticKITTI datasets. In particular, it takes the first place from the KPConv architecture by [Thomas et al. 2019](https://arxiv.org/abs/1904.08889) on Semantic3D, and is in particular superior on class `buildings`, meeting requirement (3). +- We focused on the RandLA-Net architecture [Hu 2020](https://arxiv.org/abs/1911.11236). + - It was specifically conceived by authors with requirements (1) and (2) in mind. + - It meets requirement (2), by setting SOTA results on both Semantic3D and SemanticKITTI datasets. In particular, it takes the first place from the KPConv architecture by [Thomas et al. 2019](https://arxiv.org/abs/1904.08889) on Semantic3D, and is in particular superior on class `buildings`, meeting requirement (3). - The official RandLA-Net implementation is ~4 times faster than KPConv on SemanticKITTI, meeting requirement (1). - It is conceptually simple, basically a PointNet++ like encoder-decoder architecture, with random samplings and explicit encoding of local geometric structures. diff --git a/docs/source/background/interpolation.md b/docs/source/background/interpolation.md index 0301d145..89c07bff 100644 --- a/docs/source/background/interpolation.md +++ b/docs/source/background/interpolation.md @@ -4,4 +4,4 @@ -> geometric regularity -> KNN --> test-time overlap \ No newline at end of file +-> test-time overlap diff --git a/docs/source/guides/train_new_model.md b/docs/source/guides/train_new_model.md index ccc22fc5..baca5d01 100644 --- a/docs/source/guides/train_new_model.md +++ b/docs/source/guides/train_new_model.md @@ -7,7 +7,7 @@ Refer to [this other tutorial](../tutorials/prepare_dataset.md) for how to prepa Once your python environment is set up and your dataset ready for training, proceed to the next section. -Some environment variable need to be injected at runtime can be specified in a `.env` file. Rename `.env_example` to `.env` and fill out: +Some environment variable need to be injected at runtime can be specified in a `.env` file. Rename `.env_example` to `.env` and fill out: - `LOGS_DIR`, where hydra logs and config will be saved. - `LOGGER` section, which specifies credentials needed for logging to [comet.ml](https://www.comet.ml/). You will need to create an account first if you choose to use Comet. Alternatively, setting `logger=csv` at runtime will save results in a single, local csv file, and disable comet logging. @@ -35,12 +35,12 @@ After training, you model best checkpoints and hydra config will be saved in a ` ### Optimized learning rate -Pytorch Lightning support au [automated learning rate finder](https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#auto-lr-find), by means of an Learning Rate-range test (see section 3.3 in [this paper](https://arxiv.org/pdf/1506.01186.pdf) for reference). +Pytorch Lightning support au [automated learning rate finder](https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#auto-lr-find), by means of an Learning Rate-range test (see section 3.3 in [this paper](https://arxiv.org/pdf/1506.01186.pdf) for reference). You can perfom this automatically before training by setting `task.auto_lr_find=true` when calling training on your dataset. The best learning rate will be logged and results saved as an image, so that you do not need to perform this test more than once. ### Multi-GPUs -Multi-GPUs training is supported. Refer to e.g. experiment file `RandLaNet_base_run_FR-MultiGPU.yaml` for pytorch lightning flags to activate it. +Multi-GPUs training is supported. Refer to e.g. experiment file `RandLaNet_base_run_FR-MultiGPU.yaml` for pytorch lightning flags to activate it. Multi-GPUs training effectively reduces training time by the number of GPUs used. Batch size might need to be reduced to keep a constant number of steps per epoch in DDP. ## Testing the model @@ -57,7 +57,7 @@ task.task_name="test" \ model.ckpt_path={/path/to/checkpoint.ckpt} \ trainer.gpus={0 for none, [i] to use GPU number i} \ ``` -Arguments `config-path` and `config-name` means you are using the saved configuration from your training, which contains the path to the prepared HDF5 dataset. +Arguments `config-path` and `config-name` means you are using the saved configuration from your training, which contains the path to the prepared HDF5 dataset. If you are using defaut configurations, you can call test using a custom experiment: @@ -67,4 +67,4 @@ python run.py experiment=test ## Inference -To use the checkpointed model to make predictions on new data, refer to section [Performing inference on new data](../tutorials/make_predictions.md). \ No newline at end of file +To use the checkpointed model to make predictions on new data, refer to section [Performing inference on new data](../tutorials/make_predictions.md). diff --git a/docs/source/introduction.md b/docs/source/introduction.md index db4250c2..98fec0f8 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -5,8 +5,8 @@ It allows for the evaluation of single-class IoU on the full point cloud, which Although the library can be easily extended with new neural network architectures or new data signatures, the library makes some opiniated choices in terms of neural network architecture, data processing logics, and inference logic. -Myria3D is built upon [PyTorch](https://pytorch.org/). It keeps the standard data format -from [Pytorch-Geometric](https://pytorch-geometric.readthedocs.io/). +Myria3D is built upon [PyTorch](https://pytorch.org/). It keeps the standard data format +from [Pytorch-Geometric](https://pytorch-geometric.readthedocs.io/). Its structure was bootstraped from [this code template](https://github.com/ashleve/lightning-hydra-template), -which heavily relies on [Hydra](https://hydra.cc/) and [Pytorch-Lightning](https://github.com/PyTorchLightning/pytorch-lightning) to -enable flexible and rapid iterations of deep learning experiments. \ No newline at end of file +which heavily relies on [Hydra](https://hydra.cc/) and [Pytorch-Lightning](https://github.com/PyTorchLightning/pytorch-lightning) to +enable flexible and rapid iterations of deep learning experiments. diff --git a/docs/source/tutorials/install_on_wsl2.md b/docs/source/tutorials/install_on_wsl2.md index b83cd8ed..be3b9265 100644 --- a/docs/source/tutorials/install_on_wsl2.md +++ b/docs/source/tutorials/install_on_wsl2.md @@ -2,7 +2,7 @@ Myria3D is a deep learning library designed with a focused scope: the multiclass semantic segmentation of large scale, high density aerial Lidar points cloud. -For Windows users it is quite tricky to get everything working in Myria3D. Some of the dependencies do not yet support Windows or leads to various version mismatch problems during installation. It's best to use Windows Subsystem for Linux (WSL2) to get the most out of this library if you want to work on Windows. While setting up WSL and installing Myria3D is not that difficult, if you want to utilize GPU, it becomes much more challenging. This guide tries to address the necessary steps that needs to be taken to get a properly working environment with GPU support. +For Windows users it is quite tricky to get everything working in Myria3D. Some of the dependencies do not yet support Windows or leads to various version mismatch problems during installation. It's best to use Windows Subsystem for Linux (WSL2) to get the most out of this library if you want to work on Windows. While setting up WSL and installing Myria3D is not that difficult, if you want to utilize GPU, it becomes much more challenging. This guide tries to address the necessary steps that needs to be taken to get a properly working environment with GPU support. ## [Setting up WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) @@ -11,13 +11,13 @@ Myria3D requires the latest Linux distros to work properly. That's why it's impo 1. You must be running Windows 10 version 2004 and higher (Build 19041 and higher) or Windows 11 to use the commands below. If you are on earlier versions please see the [manual install page](https://learn.microsoft.com/en-us/windows/wsl/install-manual). 2. Open PowerShell or Windows Command Prompt in administrator mode by right-clicking and selecting "Run as administrator", enter the ``wsl --install`` command, then restart your machine. 3. Ensure you have the latest WSL kernel: - + wsl.exe --update 4. This command will enable the features necessary to run WSL and install the Ubuntu distribution (default Ubuntu 22.04) of Linux. 5. If you run ``wsl --install`` and see the WSL help text, that means WSL is already installed. In that case run ``wsl --list --online`` to see a list of available distros and run ``wsl --install -d `` to install a distro. In this case we need Ubuntu 22.04, so type: wsl --install -d Ubuntu -6. You need to update your libraries before moving forward by running +6. You need to update your libraries before moving forward by running sudo apt update && sudo apt upgrade @@ -37,29 +37,29 @@ Myria3D requires the latest Linux distros to work properly. That's why it's impo ## Installing Myria3D 1. Clone the Myria3D main branch in your working directory using the following command - - + + git clone https://github.com/IGNF/myria3d.git - + After it's done, navigate to the `environment.yml` file. You might need to make couple of changes here before installing the libararies. - + 2. Check the cuda version of your Windows system. You can find it easily by running ``nvidia-smi`` in your Windows command prompt. - If the cuda version is 11.3.1 (or the same version stated in the environemnt.yml file), then you can carry on with the installation process without changing anything. - If the cuda version is something else, then you need to cross-check with [this repository](https://data.pyg.org/whl/) if it is supported. At the time of writing this documentation, cuda 9.2, 10.0, 10.1, 10.2, 11.1, 11.3, 11.5, 11.6, 11.7 these versions have corresonding PyG (PyTorch Geometric) libraries to run Myria3D. - If your cuda version is supported, then follow the next steps, otherwise install a driver version that is supported. You can check [here](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html#id4) which driver version is required for the cuda version you need, then also check if that is supported by your GPU model. - After you figured out and installed the compatible driver on your windows machine, you need to change the cuda version in the environment.yml file as well. - - Comment out the following line, as we are using WSL, we should not install the nvidia driver. + - Comment out the following line, as we are using WSL, we should not install the nvidia driver. conda-forge::cudatoolkit=11.3.1 - Change the following links depending on which cuda version and pytorch version you are using - + - --extra-index-url https://download.pytorch.org/whl/cu113 - + Wheels for torch-geometric optionnal dependencies - https://data.pyg.org/whl/torch-1.11.0+cu113/torch_cluster-1.6.0-cp39-cp39-linux_x86_64.whl - https://data.pyg.org/whl/torch-1.11.0+cu113/torch_scatter-2.0.9-cp39-cp39-linux_x86_64.whl - https://data.pyg.org/whl/torch-1.11.0+cu113/torch_sparse-0.6.14-cp39-cp39-linux_x86_64.whl - - Change the following link depending on your cuda version. Check the compatibility tables [here](https://github.com/pyg-team/pytorch_geometric#pip-wheels) and change the git version accordingly. At the time of writing this, PyG (PyTorch Geometric) was in version PyG 2.2.0 and it supports cuda 10.2, 11.3, 11.6 and 11.7 on Linux. If you have any of these cuda, then change the git version to 2.2.0 otherwise keep it at 2.1.0 + - Change the following link depending on your cuda version. Check the compatibility tables [here](https://github.com/pyg-team/pytorch_geometric#pip-wheels) and change the git version accordingly. At the time of writing this, PyG (PyTorch Geometric) was in version PyG 2.2.0 and it supports cuda 10.2, 11.3, 11.6 and 11.7 on Linux. If you have any of these cuda, then change the git version to 2.2.0 otherwise keep it at 2.1.0 - git+https://github.com/pyg-team/pytorch_geometric.git@2.1.0 @@ -75,27 +75,27 @@ Myria3D requires the latest Linux distros to work properly. That's why it's impo 5. Then install from a specific branch from github directly. pip install --upgrade https://github.com/IGNF/myria3d/tarball/main - + Alternatively, navigate your working directory to the cloned myria3d directory and install from sources directly in editable mode with pip install -e . -At this point Myria3D is installed and you can move ahead with inference or testing using the method stated [here](https://ignf.github.io/myria3d/tutorials/make_predictions.html) (CPU-only). CUDA toolkit for WSL is still not installed and because of that GPU will not be accessible to WSL yet. It will be done in the next section. - +At this point Myria3D is installed and you can move ahead with inference or testing using the method stated [here](https://ignf.github.io/myria3d/tutorials/make_predictions.html) (CPU-only). CUDA toolkit for WSL is still not installed and because of that GPU will not be accessible to WSL yet. It will be done in the next section. +

-

- +

+ ## [Install cuda in WSL](https://docs.nvidia.com/cuda/wsl-user-guide/index.html#getting-started-with-cuda-on-wsl-2) **Do not install any driver within the WSL environment. For building a CUDA application, you will need CUDA Toolkit.** The latest NVIDIA Windows GPU Driver will fully support WSL 2. With CUDA support in the driver, existing applications (compiled elsewhere on a Linux system for the same target GPU) can run unmodified within the WSL environment. The CUDA WSL-Ubuntu local installer does not contain the NVIDIA Linux GPU driver, so by following the steps below, you will be able to get just the CUDA toolkit installed on WSL. - + 1. First, remove the old GPG key: sudo apt-key del 7fa2af80 - + 2. Go to this [link](https://developer.nvidia.com/cuda-toolkit-archive) and choose the cuda toolkit version according to your Windows NVIDIA driver's cuda version. For demonstration purposes, [cudatoolkit 11.3.1](https://developer.nvidia.com/cuda-11-3-1-download-archive?target_os=Linux&target_arch=x86_64&Distribution=WSL-Ubuntu&target_version=2.0&target_type=deb_local) is shown here. 3. Run the following commands **one by one**. @@ -107,7 +107,7 @@ At this point Myria3D is installed and you can move ahead with inference or test sudo apt-get update sudo apt-get -y install cuda - If in any step there is an error, just run the commands that are shown on the terminal. For example ``sudo apt-key add ___`` command might be shown on the terminal as the next step. If the links have some placeholder values, it's better to just use commands shown on the terminal to be error-free. + If in any step there is an error, just run the commands that are shown on the terminal. For example ``sudo apt-key add ___`` command might be shown on the terminal as the next step. If the links have some placeholder values, it's better to just use commands shown on the terminal to be error-free. After running these commands cuda toolkit will be installed on WSL and Myria3D GPU functionalities can be used. @@ -115,13 +115,13 @@ At this point Myria3D is installed and you can move ahead with inference or test - + ## Troubleshooting - *OSError(libcusparse.so.11 cannot open shared object file no such file or directory)* ([**](https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625)) - open the .bashrc file from your Ubuntu home directory and at the end of the file, add the following line (replace anaconda3 with miniconda3 if needed) - export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" - *ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: version `GLIBCXX_3.4.30' not found* ([**](https://askubuntu.com/a/582910)) @@ -130,8 +130,8 @@ At this point Myria3D is installed and you can move ahead with inference or test sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get update sudo apt-get install --only-upgrade libstdc++6 -- GPU-related errors: - +- GPU-related errors: + - *RuntimeError: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx* - *Failed to initialize NVML: GPU access blocked by the operating system Failed to properly shut down NVML: GPU access blocked by the operating system* - *Failed to initialize NVML: Driver/library version mismatch* @@ -141,11 +141,7 @@ At this point Myria3D is installed and you can move ahead with inference or test - Make sure to open the command prompt in Admin mode [**](https://forums.developer.nvidia.com/t/failed-to-properly-shut-down-nvml-gpu-access-blocked-by-the-operating-system/234413/5) - In some cases using the Admin mode blocks access to the GPU, use non-elevated command prompt if that occurs - Restart your WSL and try again [**](https://stackoverflow.com/a/43023000/8889660) - + wsl --shutdown - If the error persists then see if you have the correct GPU models. As of now, cuda toolkit in WSL is supported in **NVIDIA GeForce Game Ready or NVIDIA RTX/Quadro card**s only. - - - - diff --git a/docs/source/tutorials/make_predictions.md b/docs/source/tutorials/make_predictions.md index bd3fdb34..f9e4c7b5 100644 --- a/docs/source/tutorials/make_predictions.md +++ b/docs/source/tutorials/make_predictions.md @@ -11,7 +11,7 @@ To run inference, you will need: ## Run inference from source -Then, fill out the {missing parameters} below and run: +Then, fill out the {missing parameters} below and run: ```bash python run.py \ @@ -28,7 +28,7 @@ To show you current inference config, simply add a `--help` flag: python run.py task.task_name=predict --help ``` -Note that `predict.src_las` may be any valid glob pattern (e.g. `/path/to/multiple_files/*.las`), in order to **predict on multiple files successively**. +Note that `predict.src_las` may be any valid glob pattern (e.g. `/path/to/multiple_files/*.las`), in order to **predict on multiple files successively**. If the lidar file doesn't specify an EPSG in its meatadata, it HAS TO BE be specified with `datamodule.epsg=...` ## Run inference from sources @@ -41,9 +41,9 @@ Up to date docker images (named `myria3d`) are created via Github integration ac A docker image encapsulating the virtual environment and application sources can also be built using the provided Dockerfile. At built time, the Dockerfile is not standalone and should be part of the repository - whose content is copied into the image - at the github reference you want to build from. -To run inference: +To run inference: - Mount the needed volumes with the `-v` option. -- Always set `--ipc=host` to allow multithreading (used in pytorch dataloader, as mentionned in [Pytorch's README](https://github.com/pytorch/pytorch#using-pre-built-images)). +- Always set `--ipc=host` to allow multithreading (used in pytorch dataloader, as mentionned in [Pytorch's README](https://github.com/pytorch/pytorch#using-pre-built-images)). - Increase the shared memory with `--shm-size=2gb` (which should be enough for 1km*1km point French "Lidar HD" clouds). - Set `--gpus=all` to make gpus visible to the container if available. diff --git a/docs/source/tutorials/prepare_dataset.md b/docs/source/tutorials/prepare_dataset.md index f02a0278..fe2c34b0 100644 --- a/docs/source/tutorials/prepare_dataset.md +++ b/docs/source/tutorials/prepare_dataset.md @@ -4,14 +4,14 @@ The loading function is dataset dependant, and is `lidar_hd_pre_transform` by default. The function takes points loaded from a LAS file via pdal as input, and returns a `pytorch_geometric.Data` object following the standard naming convention of `pytorch_geometric`, plus a list of features names for later use in transforms. In the loading function, the return number and color information (RGBI) are scaled to 0-1 interval, a NDVI and an average color ((R+G+B)/3) dimension are created, and points that may be occluded (as indicated by higher return number) have their color set to 0. -Customization: You may want to implement your own logic (e.g. with custom, additional features) in directory `points_pre_transform`. It then needs to be referenced similarly to `lidar_hd_pre_transform`. +Customization: You may want to implement your own logic (e.g. with custom, additional features) in directory `points_pre_transform`. It then needs to be referenced similarly to `lidar_hd_pre_transform`. The loading function is designed for the French Lidar HD data provided by IGN (see [the official page](https://geoservices.ign.fr/lidarhd) - link in French). Note that the clouds are shared without color information, and should be colorized (RGB+Infrared) to use myria3d. The [open-source ign-pdal-tools library](https://pypi.org/project/ign-pdal-tools/) is a convenient toolkit that can be used to colorize the raw clouds with IGN aerial imagery (see function 'pdaltools.color.color(...)'). Customization: If you use a different classification (e.g. additional classes), you will need to create a `dataset_description` configuration (similar to `configs/dataset_description/20220607_151_dalles_proto.yaml`). Additionnaly, you can control cloud sampling parameters via two configurations: -- `configs/datamodule/transforms/preparations/points_budget.yaml`: (defaut) allows variable cloud size within lower and higher boundaries. +- `configs/datamodule/transforms/preparations/points_budget.yaml`: (defaut) allows variable cloud size within lower and higher boundaries. - `configs/datamodule/transforms/preparations/fixed_num_points.yaml`: (alternative) samples all clouds to a fixed size, allowing for duplicated points. @@ -25,9 +25,9 @@ Under the hood, the path of each LAS file will be reconstructed like this: '{dat Large input point clouds need to be divided in smaller clouds that can be digested by segmentation models. We found that a receptive field of 50m x 50m was a good balance between context and memory intensity. The division is performed once, to avoid loading large file in memory multiple times during training. -To be able to read the lidar files, an EPSG is needed. If the files don't all specify an EPSG in their metadata, it should be given as a parameter with `datamodule.epsg=...` +To be able to read the lidar files, an EPSG is needed. If the files don't all specify an EPSG in their metadata, it should be given as a parameter with `datamodule.epsg=...` -After division, the smaller clouds are preprocessed (i.e. selection of specific LAS dimensions, on-the-fly creation of dimensions) and regrouped into a single HDF5 file whose path is specified via the `datamodule.hdf5_file_path` parameter. +After division, the smaller clouds are preprocessed (i.e. selection of specific LAS dimensions, on-the-fly creation of dimensions) and regrouped into a single HDF5 file whose path is specified via the `datamodule.hdf5_file_path` parameter. The HDF5 dataset is created at training time. It should only happens once. Once this is done, you do not need sources anymore, and simply specifying the path to the HDF5 dataset is enough (there is no need for data_dir or split_csv_path parameters anymore). @@ -36,11 +36,11 @@ It's also possible to create the hdf5 file without training any model: just fill ## Getting started quickly with a toy dataset -A LAS file is provided as part of the test suite. It can be turned into a small, training-ready dataset to get started with the package. +A LAS file is provided as part of the test suite. It can be turned into a small, training-ready dataset to get started with the package. To create a toy dataset run : ``` python myria3d/pctl/dataset/toy_dataset.py ``` -You will see a new file: `/test/data/toy_dataset.hdf5`. \ No newline at end of file +You will see a new file: `/test/data/toy_dataset.hdf5`. diff --git a/myria3d/_version.py b/myria3d/_version.py index 149e171f..32dd7c93 100644 --- a/myria3d/_version.py +++ b/myria3d/_version.py @@ -1,4 +1,4 @@ -__version__ = "3.9.1.test" +__version__ = "3.10.0" if __name__ == "__main__": diff --git a/myria3d/callbacks/metric_callbacks.py b/myria3d/callbacks/metric_callbacks.py index d0f467c9..754d45ff 100644 --- a/myria3d/callbacks/metric_callbacks.py +++ b/myria3d/callbacks/metric_callbacks.py @@ -1,6 +1,13 @@ -from pytorch_lightning import Callback import torch -from torchmetrics import Accuracy, F1Score, JaccardIndex, Precision, Recall, ConfusionMatrix +from pytorch_lightning import Callback +from torchmetrics import ( + Accuracy, + ConfusionMatrix, + F1Score, + JaccardIndex, + Precision, + Recall, +) from myria3d.callbacks.comet_callbacks import log_comet_cm diff --git a/myria3d/models/interpolation.py b/myria3d/models/interpolation.py index f7bcd4c1..630df792 100644 --- a/myria3d/models/interpolation.py +++ b/myria3d/models/interpolation.py @@ -5,11 +5,10 @@ import numpy as np import pdal import torch +from pdaltools import las_info from torch.distributions import Categorical from torch_scatter import scatter_sum -from pdaltools import las_info - from myria3d.pctl.dataset.utils import get_pdal_info_metadata, get_pdal_reader log = logging.getLogger(__name__) diff --git a/myria3d/models/losses/combined.py b/myria3d/models/losses/combined.py new file mode 100644 index 00000000..0ee64bd4 --- /dev/null +++ b/myria3d/models/losses/combined.py @@ -0,0 +1,32 @@ +from typing import List, Optional + +import torch +from torch import nn + + +class CombinedLoss(nn.Module): + """ + Weighted sum of several loss functions. + + Each sub-loss is called as ``loss(logits, targets)`` and the results are + combined as ``sum(weight_i * loss_i)``. This allows, for instance, to train + with a weighted cross-entropy loss and a Lovász loss at the same time. + + """ + + def __init__(self, losses: List[nn.Module], weights: Optional[List[float]] = None): + super().__init__() + self.losses = nn.ModuleList(losses) + if weights is None: + weights = [1.0] * len(self.losses) + assert len(weights) == len(self.losses), ( + "The number of weights must match the number of losses " + f"({len(weights)} weights for {len(self.losses)} losses)." + ) + self.weights = weights + + def forward(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor: + total_loss = logits.new_zeros(()) + for weight, loss in zip(self.weights, self.losses): + total_loss = total_loss + weight * loss(logits, targets) + return total_loss diff --git a/myria3d/models/losses/focal.py b/myria3d/models/losses/focal.py new file mode 100644 index 00000000..1aaf3251 --- /dev/null +++ b/myria3d/models/losses/focal.py @@ -0,0 +1,83 @@ +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + + +class FocalLoss(nn.Module): + """Multiclass Focal loss for segmentation tasks. + + Focal loss down-weights well-classified examples and focuses training on + hard, misclassified points. See Lin et al., "Focal Loss for Dense Object + Detection" (https://arxiv.org/abs/1708.02002). + + The loss is computed as:: + + FL(p_t) = -alpha_t * (1 - p_t) ** gamma * log(p_t) + + Args: + gamma: Focusing parameter. ``gamma=0`` is equivalent to cross-entropy. + alpha: Optional per-class weighting factor (tensor of shape ``[C]``). + ignore_index: Target value that is ignored and does not contribute to + the loss. + reduction: One of ``"mean"``, ``"sum"`` or ``"none"``. + """ + + def __init__( + self, + gamma: float = 2.0, + alpha: Optional[torch.Tensor] = None, + ignore_index: int = -100, + reduction: str = "mean", + ): + super().__init__() + if reduction not in {"mean", "sum", "none"}: + raise ValueError(f"Unsupported reduction: {reduction}") + self.gamma = gamma + self.ignore_index = ignore_index + self.reduction = reduction + if alpha is not None and not torch.is_tensor(alpha): + alpha = torch.tensor(alpha, dtype=torch.float) + self.register_buffer("alpha", alpha) + + def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor: + """ + Shape: + - y_pred: torch.Tensor of shape (N, C) or (N, C, d1, d2, ...) + - y_true: torch.Tensor of shape (N,) or (N, d1, d2, ...) + """ + C = y_pred.size(1) + if y_pred.dim() > 2: + # (N, C, d1, d2, ...) -> (P, C) + y_pred = torch.movedim(y_pred, 1, -1).contiguous().view(-1, C) + y_true = y_true.view(-1) + + alpha = self.alpha + if alpha is not None: + alpha = alpha.to(dtype=y_pred.dtype, device=y_pred.device) + + log_prob = F.log_softmax(y_pred, dim=-1) + ce_loss = F.nll_loss( + log_prob, + y_true, + weight=alpha, + ignore_index=self.ignore_index, + reduction="none", + ) + + valid = y_true != self.ignore_index + # Probability assigned to the ground-truth class for each point. + safe_true = y_true.clone() + safe_true[~valid] = 0 + pt = log_prob.gather(1, safe_true.unsqueeze(1)).squeeze(1).exp() + focal_factor = (1.0 - pt) ** self.gamma + + loss = focal_factor * ce_loss + loss = loss[valid] + + if self.reduction == "mean": + return loss.mean() if loss.numel() > 0 else y_pred.new_zeros(()) + if self.reduction == "sum": + return loss.sum() + return loss diff --git a/myria3d/models/losses/lovasz.py b/myria3d/models/losses/lovasz.py new file mode 100644 index 00000000..b5eba2bb --- /dev/null +++ b/myria3d/models/losses/lovasz.py @@ -0,0 +1,244 @@ +# File copied from https://github.com/prs-eth/LitePT/blob/main/models/losses/lovasz.py +# and modified for lidar point cloud semantic segmentation. + +# Paper: https://arxiv.org/pdf/1705.08790.pdf + +from itertools import filterfalse +from typing import Optional + +import torch +import torch.nn.functional as F +from torch.nn.modules.loss import _Loss + +BINARY_MODE: str = "binary" +MULTICLASS_MODE: str = "multiclass" +MULTILABEL_MODE: str = "multilabel" + + +def _lovasz_grad(gt_sorted): + """Compute gradient of the Lovasz extension w.r.t sorted errors + See Alg. 1 in paper: https://arxiv.org/pdf/1705.08790.pdf + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1.0 - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def _lovasz_hinge(logits, labels, ignore=None): + """ + Binary Lovasz hinge loss + logits: [B, H, W] Logits at each pixel (between -infinity and +infinity) + labels: [B, H, W] Tensor, binary ground truth masks (0 or 1) + ignore: void class id + """ + return _lovasz_hinge_flat(*_flatten_binary_scores(logits, labels, ignore)) + + +def _lovasz_hinge_flat(logits, labels): + """Binary Lovasz hinge loss + Args: + logits: [P] Logits at each prediction (between -infinity and +infinity) + labels: [P] Tensor, binary ground truth labels (0 or 1) + """ + if len(labels) == 0: + # only void pixels, the gradients should be 0 + return logits.sum() * 0.0 + signs = 2.0 * labels.float() - 1.0 + errors = 1.0 - logits * signs + errors_sorted, perm = torch.sort(errors, dim=0, descending=True) + perm = perm.data + gt_sorted = labels[perm] + grad = _lovasz_grad(gt_sorted) + loss = torch.dot(F.relu(errors_sorted), grad) + return loss + + +def _flatten_binary_scores(scores, labels, ignore=None): + """Flattens predictions in the batch (binary case) + Remove labels equal to 'ignore' + """ + scores = scores.view(-1) + labels = labels.view(-1) + if ignore is None: + return scores, labels + valid = labels != ignore + vscores = scores[valid] + vlabels = labels[valid] + return vscores, vlabels + + +def _lovasz_softmax(probas, labels, classes="present", class_seen=None, ignore=None): + """Computes the multi-class Lovasz-Softmax loss. + + Args: + probas: Class probabilities at each prediction (between 0 and 1). + Point cloud case: `probas` has shape [P, C], where P is the total + number of points in the batch (all clouds flattened together) and + C is the number of classes. + labels: Ground truth labels (between 0 and C - 1). + Point cloud case: `labels` has shape [P] (one class id per point). + classes: 'all' to average over all classes, 'present' to average only + over classes present in `labels`, or a list of classes to average. + ignore: Label value of the void class to ignore. + + Returns: + The scalar Lovasz-Softmax loss. + """ + # Point cloud case: probas [P, C] and labels [P] go straight through + # (_flatten_probas is a no-op reshape for already-flat inputs). + return _lovasz_softmax_flat( + *_flatten_probas(probas, labels, ignore), classes=classes, class_seen=class_seen + ) + + +def _lovasz_softmax_flat(probas, labels, classes="present", class_seen=None): + """Computes the multi-class Lovasz-Softmax loss on flattened predictions. + + Args: + probas: Class probabilities at each prediction (between 0 and 1), with + shape [P, C]. + labels: Ground truth labels (between 0 and C - 1), with shape [P]. + classes: 'all' to average over all classes, 'present' to average only + over classes present in `labels`, or a list of classes to average. + class_seen: Optional list of classes for which the loss should be computed. + If None, all classes are considered. + + Returns: + The scalar Lovasz-Softmax loss. + """ + if probas.numel() == 0: + # only void pixels, the gradients should be 0 + return probas * 0.0 + C = probas.size(1) + losses = [] + for c in labels.unique(): + if class_seen is None: + fg = (labels == c).type_as(probas) # foreground for class c + if classes == "present" and fg.sum() == 0: + continue + if C == 1: + if len(classes) > 1: + raise ValueError("Sigmoid output possible only with 1 class") + class_pred = probas[:, 0] + else: + class_pred = probas[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + losses.append(torch.dot(errors_sorted, _lovasz_grad(fg_sorted))) + else: + if c in class_seen: + fg = (labels == c).type_as(probas) # foreground for class c + if classes == "present" and fg.sum() == 0: + continue + if C == 1: + if len(classes) > 1: + raise ValueError("Sigmoid output possible only with 1 class") + class_pred = probas[:, 0] + else: + class_pred = probas[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + losses.append(torch.dot(errors_sorted, _lovasz_grad(fg_sorted))) + return mean(losses) + + +def _flatten_probas(probas, labels, ignore=None): + """Flattens predictions in the batch. + + Point cloud case: `probas` is already [P, C] (2D) and `labels` is [P], so + the reshapes below are effectively no-ops and the tensors pass through + unchanged (apart from dropping ignored points). + """ + if probas.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probas.size() + probas = probas.view(B, 1, H, W) + + C = probas.size(1) + # Image case: [B, C, Di, Dj, ...] -> [B, Di, Dj, ..., C]. + # Point cloud case: [P, C] -> movedim(1, -1) is a no-op on a 2D tensor. + probas = torch.movedim(probas, 1, -1) # [B, C, Di, Dj, ...] -> [B, Di, Dj, ..., C] + probas = probas.contiguous().view(-1, C) # [P, C] (unchanged for point clouds) + + labels = labels.view(-1) # [P] (already flat for point clouds) + if ignore is None: + return probas, labels + valid = labels != ignore # drop ignored/void points + vprobas = probas[valid] + vlabels = labels[valid] + return vprobas, vlabels + + +def isnan(x): + return x != x + + +def mean(values, ignore_nan=False, empty=0): + """Nan-mean compatible with generators.""" + values = iter(values) + if ignore_nan: + values = filterfalse(isnan, values) + try: + n = 1 + acc = next(values) + except StopIteration: + if empty == "raise": + raise ValueError("Empty mean") + return empty + for n, v in enumerate(values, 2): + acc += v + if n == 1: + return acc + return acc / n + + +class LovaszLoss(_Loss): + def __init__( + self, + mode: str, + class_seen: Optional[int] = None, + ignore_index: Optional[int] = None, + loss_weight: float = 1.0, + ): + """Lovasz loss for segmentation task. + It supports binary, multiclass and multilabel cases + Args: + mode: Loss mode 'binary', 'multiclass' or 'multilabel' + ignore_index: Label that indicates ignored pixels (does not contribute to loss) + Shape + - **y_pred** - torch.Tensor of shape (N, C, H, W) + - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W) + Reference + https://github.com/BloodAxe/pytorch-toolbelt + """ + assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE} + super().__init__() + + self.mode = mode + self.ignore_index = ignore_index + self.class_seen = class_seen + self.loss_weight = loss_weight + + def forward(self, y_pred, y_true): + if self.mode in {BINARY_MODE, MULTILABEL_MODE}: + loss = _lovasz_hinge(y_pred, y_true, ignore=self.ignore_index) + elif self.mode == MULTICLASS_MODE: + y_pred = y_pred.softmax(dim=1) + loss = _lovasz_softmax( + y_pred, + y_true, + class_seen=self.class_seen, + ignore=self.ignore_index, + ) + else: + raise ValueError("Wrong mode {}.".format(self.mode)) + return loss * self.loss_weight diff --git a/myria3d/models/model.py b/myria3d/models/model.py index f1d842d7..98f76954 100755 --- a/myria3d/models/model.py +++ b/myria3d/models/model.py @@ -185,11 +185,22 @@ def configure_optimizers(self): if self.hparams.lr_scheduler is None: return optimizer - return { - "optimizer": optimizer, - "lr_scheduler": self.hparams.lr_scheduler(optimizer), - "monitor": self.hparams.monitor, - } + lr_scheduler_partial = self.hparams.lr_scheduler + if lr_scheduler_partial.func is torch.optim.lr_scheduler.OneCycleLR: + # OneCycleLR needs the total number of optimizer steps and must be + # stepped every batch. Let Lightning compute total_steps so that + # steps_per_epoch/epochs do not have to be set by hand. + scheduler = lr_scheduler_partial( + optimizer, total_steps=self.trainer.estimated_stepping_batches + ) + lr_scheduler_config = {"scheduler": scheduler, "interval": "step"} + else: + lr_scheduler_config = { + "scheduler": lr_scheduler_partial(optimizer), + "monitor": self.hparams.monitor, + } + + return {"optimizer": optimizer, "lr_scheduler": lr_scheduler_config} def _get_batch_tensor_by_enumeration(self, pos_x: torch.Tensor) -> torch.Tensor: """Get batch tensor (e.g. [0,0,1,1,2,2,...,B-1,B-1] ) diff --git a/myria3d/models/modules/pyg_randla_net.py b/myria3d/models/modules/pyg_randla_net.py index 21b18094..4ccbf42f 100644 --- a/myria3d/models/modules/pyg_randla_net.py +++ b/myria3d/models/modules/pyg_randla_net.py @@ -26,6 +26,7 @@ def __init__( num_classes: int, decimation: int = 4, num_neighbors: int = 16, + dims: Tuple[int, int, int, int] = (32, 128, 256, 512), return_logits: bool = False, ): super().__init__() @@ -34,21 +35,27 @@ def __init__( # An option to return logits instead of probas self.return_logits = return_logits + # Per-stage encoder output channels. Increase these to raise the model + # capacity (e.g. (64, 256, 512, 1024)) when training on larger datasets. + if len(dims) != 4: + raise ValueError(f"`dims` must have exactly 4 values, got {len(dims)}: {dims}") + d1, d2, d3, d4 = dims + # Authors use 8, which is a bottleneck # for the final MLP, and also when num_classes>8 # or num_features>8. d_bottleneck = max(32, num_classes, num_features) self.fc0 = Linear(num_features, d_bottleneck) - self.block1 = DilatedResidualBlock(num_neighbors, d_bottleneck, 32) - self.block2 = DilatedResidualBlock(num_neighbors, 32, 128) - self.block3 = DilatedResidualBlock(num_neighbors, 128, 256) - self.block4 = DilatedResidualBlock(num_neighbors, 256, 512) - self.mlp_summit = SharedMLP([512, 512]) - self.fp4 = FPModule(1, SharedMLP([512 + 256, 256])) - self.fp3 = FPModule(1, SharedMLP([256 + 128, 128])) - self.fp2 = FPModule(1, SharedMLP([128 + 32, 32])) - self.fp1 = FPModule(1, SharedMLP([32 + 32, d_bottleneck])) + self.block1 = DilatedResidualBlock(num_neighbors, d_bottleneck, d1) + self.block2 = DilatedResidualBlock(num_neighbors, d1, d2) + self.block3 = DilatedResidualBlock(num_neighbors, d2, d3) + self.block4 = DilatedResidualBlock(num_neighbors, d3, d4) + self.mlp_summit = SharedMLP([d4, d4]) + self.fp4 = FPModule(1, SharedMLP([d4 + d3, d3])) + self.fp3 = FPModule(1, SharedMLP([d3 + d2, d2])) + self.fp2 = FPModule(1, SharedMLP([d2 + d1, d1])) + self.fp1 = FPModule(1, SharedMLP([d1 + d1, d_bottleneck])) self.mlp_classif = SharedMLP([d_bottleneck, 64, 32], dropout=[0.0, 0.5]) self.fc_classif = Linear(32, num_classes) diff --git a/myria3d/pctl/datamodule/hdf5.py b/myria3d/pctl/datamodule/hdf5.py index b1dd36ec..e9f1be54 100644 --- a/myria3d/pctl/datamodule/hdf5.py +++ b/myria3d/pctl/datamodule/hdf5.py @@ -39,6 +39,8 @@ def __init__( batch_size: int = 12, num_workers: int = 1, prefetch_factor: int = 2, + pin_memory: bool = False, + persistent_workers: bool = False, transforms: Optional[Dict[str, TRANSFORMS_LIST]] = None, **kwargs, ): @@ -62,6 +64,8 @@ def __init__( self.batch_size = batch_size self.num_workers = num_workers self.prefetch_factor = prefetch_factor + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers t = transforms self.preparation_train_transform: TRANSFORMS_LIST = t.get("preparations_train_list", []) @@ -72,6 +76,11 @@ def __init__( self.augmentation_transform: TRANSFORMS_LIST = t.get("augmentations_list", []) self.normalization_transform: TRANSFORMS_LIST = t.get("normalizations_list", []) + # Deterministic transforms baked into the HDF5 file at creation time (train split only). + self.preparation_train_bake_transform: TRANSFORMS_LIST = t.get( + "preparations_train_bake_list", [] + ) + @property def train_transform(self) -> CustomCompose: return CustomCompose( @@ -88,6 +97,11 @@ def eval_transform(self) -> CustomCompose: def predict_transform(self) -> CustomCompose: return CustomCompose(self.preparation_predict_transform + self.normalization_transform) + @property + def train_pre_transform(self) -> CustomCompose: + """Deterministic transforms baked into the HDF5 file (train split only).""" + return CustomCompose(self.preparation_train_bake_transform) + def prepare_data(self, stage: Optional[str] = None): """Prepare dataset containing train, val, test data.""" @@ -138,6 +152,7 @@ def dataset(self) -> HDF5Dataset: subtile_width=self.subtile_width, subtile_overlap_train=self.subtile_overlap_train, pre_filter=self.pre_filter, + train_pre_transform=self.train_pre_transform, train_transform=self.train_transform, eval_transform=self.eval_transform, ) @@ -150,6 +165,8 @@ def train_dataloader(self): batch_size=self.batch_size, num_workers=self.num_workers, prefetch_factor=self.prefetch_factor, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, shuffle=True, ) @@ -160,6 +177,8 @@ def val_dataloader(self): batch_size=self.batch_size, num_workers=self.num_workers, prefetch_factor=self.prefetch_factor, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, ) def test_dataloader(self): diff --git a/myria3d/pctl/dataset/hdf5.py b/myria3d/pctl/dataset/hdf5.py index b41438b5..59d7b662 100644 --- a/myria3d/pctl/dataset/hdf5.py +++ b/myria3d/pctl/dataset/hdf5.py @@ -5,6 +5,7 @@ from typing import Callable, List, Optional import h5py +import numpy as np import torch from torch.utils.data import Dataset from torch_geometric.data import Data @@ -17,6 +18,7 @@ split_cloud_into_samples, ) from myria3d.pctl.points_pre_transform.lidar_hd import lidar_hd_pre_transform +from myria3d.pctl.transforms.transforms import ToTensor from myria3d.utils import utils log = utils.get_logger(__name__) @@ -35,6 +37,7 @@ def __init__( subtile_width: Number = 50, subtile_overlap_train: Number = 0, pre_filter=pre_filter_below_n_points, + train_pre_transform: List[Callable] = None, train_transform: List[Callable] = None, eval_transform: List[Callable] = None, ): @@ -49,6 +52,8 @@ def __init__( subtile_width (Number, optional): effective width of a subtile (i.e. receptive field). Defaults to 50. subtile_overlap_train (Number, optional): Overlap for data augmentation of train set. Defaults to 0. pre_filter (_type_, optional): Function to filter out specific subtiles. Defaults to None. + train_pre_transform (List[Callable], optional): Deterministic transforms baked once into the HDF5 file + for the train split only (applied in create_hdf5). Defaults to None. train_transform (List[Callable], optional): Transforms to apply to a sample for training. Defaults to None. eval_transform (List[Callable], optional): Transforms to apply to a sample for evaluation (test/val sets). Defaults to None. @@ -56,6 +61,7 @@ def __init__( self.points_pre_transform = points_pre_transform self.pre_filter = pre_filter + self.train_pre_transform = train_pre_transform self.train_transform = train_transform self.eval_transform = eval_transform @@ -86,6 +92,7 @@ def __init__( pre_filter, subtile_overlap_train, points_pre_transform, + train_pre_transform, ) # Use property once to be sure that samples are all indexed into the hdf5 file. @@ -131,7 +138,9 @@ def _get_data(self, sample_hdf5_path: str) -> Data: return Data( x=torch.from_numpy(grp["x"][...]), pos=torch.from_numpy(grp["pos"][...]), - y=torch.from_numpy(grp["y"][...]), + # Force long targets: nll_loss/cross-entropy do not support Int targets, + # and baked HDF5 files may store y as int32. + y=torch.from_numpy(grp["y"][...]).long(), idx_in_original_cloud=grp["idx_in_original_cloud"][...], x_features_names=grp["x"].attrs["x_features_names"].tolist(), # num_nodes=grp["pos"][...].shape[0], # Not needed - performed under the hood. @@ -203,6 +212,7 @@ def create_hdf5( pre_filter: Optional[Callable[[Data], bool]] = pre_filter_below_n_points, subtile_overlap_train: Number = 0, points_pre_transform: Callable = lidar_hd_pre_transform, + train_pre_transform: Optional[Callable[[Data], Data]] = None, ): """Create a HDF5 dataset file from las. @@ -216,6 +226,9 @@ def create_hdf5( pre_filter: Function to filter out specific subtiles. "pre_filter_below_n_points" by default, subtile_overlap_train (Number, optional): Overlap for data augmentation of train set. 0 by default, points_pre_transform (Callable): Function to turn pdal points into a pyg Data object. + train_pre_transform (Callable, optional): Deterministic transforms baked once into the stored + data for the TRAIN split only (e.g. target mapping, class dropping, grid sampling), so that + they are not recomputed every epoch. None by default (no baking). """ os.makedirs(os.path.dirname(hdf5_file_path), exist_ok=True) @@ -257,6 +270,28 @@ def create_hdf5( if pre_filter is not None and pre_filter(data): # e.g. pre_filter spots situations where num_nodes is too small. continue + + # Bake deterministic transforms (target mapping, class dropping, grid sampling) + # into the stored train data so they are not recomputed every epoch. + if split == "train" and train_pre_transform is not None: + # points_pre_transform returns numpy-backed arrays; GridSampling and + # DropPointsByClass operate on tensors, so convert first (this is normally + # done at load time in _get_data, but the bake runs before storage). + data = ToTensor()(data) + data = train_pre_transform(data) + if data is None or data.num_nodes == 0: + continue + if pre_filter is not None and pre_filter(data): + continue + # Back to numpy-backed arrays for storage in HDF5. + data.x = data.x.numpy() + data.pos = data.pos.numpy() + data.y = data.y.numpy() + # Grid sampling changes the number of points, so the original per-point + # indices no longer apply. They are unused for training (no interpolation + # back to the full cloud), so store a placeholder of the correct length. + sample_idx = np.arange(data.num_nodes) + hdf5_path = os.path.join(split, basename, str(sample_number).zfill(5)) hd5f_path_x = os.path.join(hdf5_path, "x") hdf5_file.create_dataset( @@ -277,7 +312,9 @@ def create_hdf5( hdf5_file.create_dataset( os.path.join(hdf5_path, "y"), data.y.shape, - dtype="i", + # int64 so that baked targets load as a torch.long tensor; + # nll_loss/cross-entropy require Long targets (Int is unsupported). + dtype="i8", data=data.y, ) hdf5_file.create_dataset( diff --git a/myria3d/pctl/dataset/toy_dataset.py b/myria3d/pctl/dataset/toy_dataset.py index bca0a79e..60da719c 100755 --- a/myria3d/pctl/dataset/toy_dataset.py +++ b/myria3d/pctl/dataset/toy_dataset.py @@ -4,14 +4,46 @@ import os.path as osp import sys +from omegaconf import OmegaConf +from torch_geometric.transforms import GridSampling + # to use from CLI. sys.path.append(osp.dirname(osp.dirname(osp.dirname(osp.dirname(__file__))))) from myria3d.pctl.dataset.hdf5 import HDF5Dataset # noqa +from myria3d.pctl.transforms.compose import CustomCompose # noqa +from myria3d.pctl.transforms.transforms import ( # noqa + DropPointsByClass, + TargetTransform, +) TOY_EPSG = "2154" TOY_LAS_DATA = "tests/data/toy_dataset_src/862000_6652000.classified_toy_dataset.100mx100m.las" TOY_DATASET_HDF5_PATH = "tests/data/toy_dataset.hdf5" +# Deterministic transforms baked into the HDF5 file once, at dataset-creation time, +# for the TRAIN split only (applied in create_hdf5). They are therefore NOT recomputed +# every epoch, which is a large speed-up on big datasets. +# Here, we mirror them in the toy dataset creation, so that the toy dataset is consistent with the default config's `train_bake` list. +# creation. + +# Reuse the classification mappings from the dataset_description +# config instead of duplicating them here, so the toy dataset stays consistent with the +# production dataset config. + +_DATASET_DESCRIPTION_PATH = osp.join( + osp.dirname(osp.dirname(osp.dirname(osp.dirname(__file__)))), + "configs", + "dataset_description", + "20260703_lidarhd_pacasam_dataset.yaml", +) +_DATASET_DESCRIPTION = OmegaConf.load(_DATASET_DESCRIPTION_PATH) + +TOY_CLASSIFICATION_PREPROCESSING_DICT = OmegaConf.to_container( + _DATASET_DESCRIPTION.classification_preprocessing_dict +) + +TOY_CLASSIFICATION_DICT = OmegaConf.to_container(_DATASET_DESCRIPTION.classification_dict) + def make_toy_dataset_from_test_file(): """Prepare a toy dataset from a single, small LAS file. @@ -34,7 +66,20 @@ def make_toy_dataset_from_test_file(): if os.path.isfile(TOY_DATASET_HDF5_PATH): os.remove(TOY_DATASET_HDF5_PATH) - # TODO: update transforms ? or use a config ? + # Deterministic transforms baked into the train split, mirroring the default config's + # `train_bake` list. TargetTransform is required so that stored train targets are mapped + # to consecutive integers (0-(n-1)); this mapping is no longer applied at load time. + train_pre_transform = CustomCompose( + [ + TargetTransform( + TOY_CLASSIFICATION_PREPROCESSING_DICT, + TOY_CLASSIFICATION_DICT, + ), + DropPointsByClass(), + GridSampling(0.25), + ] + ) + HDF5Dataset( TOY_DATASET_HDF5_PATH, TOY_EPSG, @@ -45,6 +90,7 @@ def make_toy_dataset_from_test_file(): }, tile_width=110, subtile_width=50, + train_pre_transform=train_pre_transform, train_transform=None, eval_transform=None, pre_filter=None, diff --git a/myria3d/pctl/dataset/utils.py b/myria3d/pctl/dataset/utils.py index 73ede8e9..f724d5a3 100644 --- a/myria3d/pctl/dataset/utils.py +++ b/myria3d/pctl/dataset/utils.py @@ -1,8 +1,8 @@ import glob import json -from pathlib import Path import subprocess as sp from numbers import Number +from pathlib import Path from typing import Dict, List, Literal, Union import numpy as np @@ -62,7 +62,7 @@ def pdal_read_las_array_as_float32(las_path: str, epsg: str): def get_metadata(las_path: str) -> dict: - """ returns metadata contained in a las file + """returns metadata contained in a las file Args: las_path (str): input LAS path to get metadata from. Returns: @@ -83,7 +83,7 @@ def get_pdal_reader(las_path: str, epsg: str) -> pdal.Reader.las: """ - if epsg : + if epsg: # if an epsg in provided, force pdal to read the lidar file with it # epsg can be added as a number like "2154" or as a string like "EPSG:2154" return pdal.Reader.las( @@ -92,8 +92,8 @@ def get_pdal_reader(las_path: str, epsg: str) -> pdal.Reader.las: override_srs=f"EPSG:{epsg}" if str(epsg).isdigit() else epsg, ) - try : - if get_metadata(las_path)['metadata']['readers.las']['srs']['compoundwkt']: + try: + if get_metadata(las_path)["metadata"]["readers.las"]["srs"]["compoundwkt"]: # read the lidar file with pdal default return pdal.Reader.las(filename=las_path) except Exception: diff --git a/myria3d/pctl/transforms/transforms.py b/myria3d/pctl/transforms/transforms.py index da314cad..a260720b 100755 --- a/myria3d/pctl/transforms/transforms.py +++ b/myria3d/pctl/transforms/transforms.py @@ -21,7 +21,7 @@ def __init__(self, keys: List[str] = ["pos", "x", "y"]): self.keys = keys def __call__(self, data: Data): - for key in data.keys: + for key in data.keys(): if key in self.keys: data[key] = torch.from_numpy(data[key]) return data diff --git a/myria3d/predict.py b/myria3d/predict.py index 7c50219e..306dd7ab 100644 --- a/myria3d/predict.py +++ b/myria3d/predict.py @@ -46,7 +46,10 @@ def predict(config: DictConfig) -> str: # Do not require gradient for faster predictions torch.set_grad_enabled(False) - model = Model.load_from_checkpoint(config.predict.ckpt_path) + # strict=False: the criterion is excluded from the checkpoint hyperparameters + # (see Model.__init__), so its buffers (e.g. "criterion.losses.0.weight") are + # not reconstructed at load time and must be ignored here. + model = Model.load_from_checkpoint(config.predict.ckpt_path, strict=False) device = utils.define_device_from_config_param(config.predict.gpus) model.to(device) model.eval() diff --git a/myria3d/train.py b/myria3d/train.py index 1bac4dae..421f926e 100755 --- a/myria3d/train.py +++ b/myria3d/train.py @@ -20,10 +20,10 @@ seed_everything, ) from pytorch_lightning.loggers.logger import Logger +from run import TASK_NAMES from myria3d.models.model import Model from myria3d.utils import utils -from run import TASK_NAMES log = utils.get_logger(__name__) @@ -164,7 +164,16 @@ def train(config: DictConfig) -> Trainer: kwargs_to_override.pop( NEURAL_NET_ARCHITECTURE_CONFIG_GROUP, None ) # removes that key if it's there - model = Model.load_from_checkpoint(config.model.ckpt_path, **kwargs_to_override) + # The criterion is excluded from the checkpoint hyperparameters (see Model.__init__), + # so it is not reconstructed by load_from_checkpoint. Re-provide it from the current + # config and load non-strictly to ignore the criterion buffers stored in the checkpoint + # (e.g. "criterion.losses.0.weight"). + model = Model.load_from_checkpoint( + config.model.ckpt_path, + criterion=model.criterion, + strict=False, + **kwargs_to_override, + ) trainer.fit(model=model, datamodule=datamodule, ckpt_path=None) log.info(f"Best checkpoint:\n{trainer.checkpoint_callback.best_model_path}") log.info("End of training and validating!") diff --git a/run.py b/run.py index 39768867..39eee705 100755 --- a/run.py +++ b/run.py @@ -19,6 +19,7 @@ from myria3d.pctl.dataset.hdf5 import create_hdf5 from myria3d.pctl.dataset.utils import get_las_paths_by_split_dict +from myria3d.pctl.transforms.compose import CustomCompose from myria3d.utils import utils TASK_NAME_DETECTION_STRING = "task.task_name=" @@ -41,7 +42,7 @@ class TASK_NAMES(Enum): log = utils.get_logger(__name__) -@hydra.main(config_path="configs/", config_name="config.yaml", version_base="1.1") +@hydra.main(config_path="configs/", config_name="config.yaml", version_base="1.3") def launch_train( config: DictConfig, ): # pragma: no cover (it's just an initialyzer of a class/method tested elsewhere) @@ -58,7 +59,7 @@ def launch_train( return train(config) -@hydra.main(config_path=DEFAULT_DIRECTORY, config_name=DEFAULT_CONFIG_FILE, version_base="1.1") +@hydra.main(config_path=DEFAULT_DIRECTORY, config_name=DEFAULT_CONFIG_FILE, version_base="1.3") def launch_predict(config: DictConfig): """Infer probabilities and automate semantic segmentation decisions on unseen data.""" # Imports should be nested inside @hydra.main to optimize tab completion @@ -84,7 +85,7 @@ def launch_predict(config: DictConfig): raise RuntimeError(f"Failed to open output file: {output_path}") -@hydra.main(config_path="configs/", config_name="config.yaml", version_base="1.1") +@hydra.main(config_path="configs/", config_name="config.yaml", version_base="1.3") def launch_hdf5(config: DictConfig): """Build an HDF5 file from a directory with las files.""" @@ -106,6 +107,11 @@ def launch_hdf5(config: DictConfig): points_pre_transform=hydra.utils.instantiate( config.datamodule.get("points_pre_transform") ), + train_pre_transform=CustomCompose( + hydra.utils.instantiate( + config.datamodule.transforms.get("preparations_train_bake_list") + ) + ), ) diff --git a/tests/conftest.py b/tests/conftest.py index 322ddc96..2dbd5a1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ -from typing import List import os +from typing import List import pytest import sh @@ -9,7 +9,7 @@ from myria3d.pctl.dataset.toy_dataset import make_toy_dataset_from_test_file SINGLE_POINT_CLOUD = "tests/data/single-point-cloud.laz" -DEFAULT_EPSG = '2154' +DEFAULT_EPSG = "2154" @pytest.fixture(scope="session") diff --git a/tests/data/toy_dataset_src/toy_dataset_split.csv b/tests/data/toy_dataset_src/toy_dataset_split.csv index 36259314..528706e3 100644 --- a/tests/data/toy_dataset_src/toy_dataset_split.csv +++ b/tests/data/toy_dataset_src/toy_dataset_split.csv @@ -1,4 +1,4 @@ basename,split 862000_6652000.classified_toy_dataset.100mx100m.las,train 862000_6652000.classified_toy_dataset.100mx100m.las,val -862000_6652000.classified_toy_dataset.100mx100m.las,test \ No newline at end of file +862000_6652000.classified_toy_dataset.100mx100m.las,test diff --git a/tests/myria3d/models/losses/test_losses.py b/tests/myria3d/models/losses/test_losses.py new file mode 100644 index 00000000..9a625ed3 --- /dev/null +++ b/tests/myria3d/models/losses/test_losses.py @@ -0,0 +1,206 @@ +import pytest +import torch +import torch.nn.functional as F + +from myria3d.models.losses.combined import CombinedLoss +from myria3d.models.losses.focal import FocalLoss +from myria3d.models.losses.lovasz import LovaszLoss + +""" +Unit tests for the custom loss functions (focal, Lovász and their combination). + +These tests work on small, hand-crafted point-cloud-like tensors with the layout used +throughout the project: logits of shape ``(P, C)`` (P points, C classes) and integer +targets of shape ``(P,)``. + +""" + +NUM_CLASSES = 7 +NUM_POINTS = 1024 + + +@pytest.fixture +def logits_and_targets(): + """A reproducible batch of logits ``(P, C)`` and integer targets ``(P,)``.""" + generator = torch.Generator().manual_seed(0) + logits = torch.randn(NUM_POINTS, NUM_CLASSES, generator=generator) + targets = torch.randint(0, NUM_CLASSES, (NUM_POINTS,), generator=generator) + return logits, targets + + +# --------------------------------------------------------------------------- +# FocalLoss +# --------------------------------------------------------------------------- + + +def test_focal_loss_returns_finite_scalar(logits_and_targets): + logits, targets = logits_and_targets + loss = FocalLoss(gamma=2.0)(logits, targets) + assert loss.ndim == 0 + assert torch.isfinite(loss) + assert loss.item() >= 0.0 + + +def test_focal_loss_gamma_zero_matches_cross_entropy(logits_and_targets): + """With ``gamma=0`` the focal factor is 1, so focal loss == cross-entropy.""" + logits, targets = logits_and_targets + focal = FocalLoss(gamma=0.0)(logits, targets) + ce = F.cross_entropy(logits, targets) + assert torch.allclose(focal, ce, atol=1e-6) + + +def test_focal_loss_downweights_easy_examples(logits_and_targets): + """Focal loss (gamma>0) should be smaller than plain cross-entropy here, + since it down-weights well-classified points.""" + logits, targets = logits_and_targets + focal = FocalLoss(gamma=2.0)(logits, targets) + ce = F.cross_entropy(logits, targets) + assert focal.item() < ce.item() + + +def test_focal_loss_ignore_index(): + """Points with the ignore index must not contribute to the loss.""" + logits = torch.randn(6, NUM_CLASSES) + targets = torch.tensor([0, 1, 2, 3, 0, 1]) + ignored = targets.clone() + ignored[:2] = -100 # ignore first two points + + full = FocalLoss(gamma=2.0, ignore_index=-100)(logits, targets) + partial = FocalLoss(gamma=2.0, ignore_index=-100)(logits[2:], targets[2:]) + masked = FocalLoss(gamma=2.0, ignore_index=-100)(logits, ignored) + assert torch.allclose(masked, partial, atol=1e-6) + assert not torch.allclose(masked, full, atol=1e-6) + + +def test_focal_loss_all_ignored_is_zero(): + logits = torch.randn(4, NUM_CLASSES) + targets = torch.full((4,), -100) + loss = FocalLoss(gamma=2.0, ignore_index=-100)(logits, targets) + assert loss.item() == 0.0 + + +def test_focal_loss_with_alpha_weights(logits_and_targets): + logits, targets = logits_and_targets + alpha = torch.tensor([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]) + loss = FocalLoss(gamma=2.0, alpha=alpha)(logits, targets) + assert torch.isfinite(loss) + + +@pytest.mark.parametrize("reduction", ["mean", "sum", "none"]) +def test_focal_loss_reductions(logits_and_targets, reduction): + logits, targets = logits_and_targets + loss = FocalLoss(gamma=2.0, reduction=reduction)(logits, targets) + if reduction == "none": + assert loss.shape == (NUM_POINTS,) + else: + assert loss.ndim == 0 + + +def test_focal_loss_invalid_reduction_raises(): + with pytest.raises(ValueError): + FocalLoss(reduction="invalid") + + +def test_focal_loss_backward(logits_and_targets): + logits, targets = logits_and_targets + logits = logits.clone().requires_grad_(True) + FocalLoss(gamma=2.0)(logits, targets).backward() + assert logits.grad is not None + assert torch.isfinite(logits.grad).all() + + +# --------------------------------------------------------------------------- +# LovaszLoss +# --------------------------------------------------------------------------- + + +def test_lovasz_multiclass_returns_finite_scalar(logits_and_targets): + logits, targets = logits_and_targets + loss = LovaszLoss(mode="multiclass")(logits, targets) + assert loss.ndim == 0 + assert torch.isfinite(loss) + assert loss.item() >= 0.0 + + +def test_lovasz_perfect_prediction_is_low(): + """Confident, correct logits should yield a near-zero Lovász loss.""" + targets = torch.arange(NUM_CLASSES).repeat(4) + logits = F.one_hot(targets, NUM_CLASSES).float() * 20.0 # very confident + loss = LovaszLoss(mode="multiclass")(logits, targets) + assert loss.item() < 1e-2 + + +def test_lovasz_ignore_index(): + targets = torch.arange(NUM_CLASSES).repeat(4) + logits = torch.randn(targets.numel(), NUM_CLASSES) + with_ignore = LovaszLoss(mode="multiclass", ignore_index=0)(logits, targets) + assert torch.isfinite(with_ignore) + + +def test_lovasz_loss_weight_scales(): + targets = torch.arange(NUM_CLASSES).repeat(4) + logits = torch.randn(targets.numel(), NUM_CLASSES) + base = LovaszLoss(mode="multiclass", loss_weight=1.0)(logits, targets) + scaled = LovaszLoss(mode="multiclass", loss_weight=2.5)(logits, targets) + assert torch.allclose(scaled, 2.5 * base, atol=1e-6) + + +def test_lovasz_invalid_mode_raises(): + with pytest.raises(AssertionError): + LovaszLoss(mode="not-a-mode") + + +def test_lovasz_backward(): + targets = torch.arange(NUM_CLASSES).repeat(4) + logits = torch.randn(targets.numel(), NUM_CLASSES, requires_grad=True) + LovaszLoss(mode="multiclass")(logits, targets).backward() + assert logits.grad is not None + assert torch.isfinite(logits.grad).all() + + +# --------------------------------------------------------------------------- +# CombinedLoss +# --------------------------------------------------------------------------- + + +def test_combined_loss_default_weights_sum(logits_and_targets): + logits, targets = logits_and_targets + focal = FocalLoss(gamma=2.0) + lovasz = LovaszLoss(mode="multiclass") + combined = CombinedLoss(losses=[focal, lovasz]) + expected = focal(logits, targets) + lovasz(logits, targets) + assert torch.allclose(combined(logits, targets), expected, atol=1e-6) + + +def test_combined_loss_custom_weights(logits_and_targets): + logits, targets = logits_and_targets + focal = FocalLoss(gamma=2.0) + lovasz = LovaszLoss(mode="multiclass") + weights = [0.7, 0.3] + combined = CombinedLoss(losses=[focal, lovasz], weights=weights) + expected = weights[0] * focal(logits, targets) + weights[1] * lovasz(logits, targets) + assert torch.allclose(combined(logits, targets), expected, atol=1e-6) + + +def test_combined_loss_mismatched_weights_raises(): + with pytest.raises(AssertionError): + CombinedLoss(losses=[FocalLoss(), LovaszLoss(mode="multiclass")], weights=[1.0]) + + +def test_combined_loss_single_loss(logits_and_targets): + logits, targets = logits_and_targets + focal = FocalLoss(gamma=2.0) + combined = CombinedLoss(losses=[focal], weights=[2.0]) + assert torch.allclose(combined(logits, targets), 2.0 * focal(logits, targets), atol=1e-6) + + +def test_combined_loss_backward(logits_and_targets): + logits, targets = logits_and_targets + logits = logits.clone().requires_grad_(True) + combined = CombinedLoss( + losses=[FocalLoss(gamma=2.0), LovaszLoss(mode="multiclass")], + weights=[0.5, 0.5], + ) + combined(logits, targets).backward() + assert logits.grad is not None + assert torch.isfinite(logits.grad).all() diff --git a/tests/myria3d/models/modules/test_randla_nets.py b/tests/myria3d/models/modules/test_randla_nets.py index d128209f..e7b3b6d8 100644 --- a/tests/myria3d/models/modules/test_randla_nets.py +++ b/tests/myria3d/models/modules/test_randla_nets.py @@ -38,3 +38,40 @@ def test_fake_run_pyg_randlanet(num_nodes): ) output = model(data.x, data.pos, data.batch, data.ptr) assert output.shape == torch.Size([sum(num_nodes), num_classes]) + + +# test model with externalized embeddings sizes defined as a list of integers as dims = [48, 192, 384, 768] +@pytest.mark.parametrize("num_nodes", [[12500, 12500], [50, 50], [12500, 10000]]) +def test_fake_run_pyg_randlanet_with_externalized_embeddings(num_nodes): + """Documents expected data format and make a forward pass with PyG RandLa-Net. + + Accepts small clouds even though decimation should lead to empty cloud. + Accepts point clouds of various sizes. + + """ + num_euclidian_dimensions = 3 + num_features = 9 + num_classes = 6 + decimation = 4 + num_neighbors = 16 + + data = Batch.from_data_list( + [ + Data( + x=torch.rand((n, num_features)), + pos=torch.rand((n, num_euclidian_dimensions)), + batch=torch.full((n,), idx), + ) + for idx, n in enumerate(num_nodes) + ] + ) + + model = PyGRandLANet( + num_features, + num_classes, + decimation=decimation, + num_neighbors=num_neighbors, + dims=[48, 192, 384, 768], + ) + output = model(data.x, data.pos, data.batch, data.ptr) + assert output.shape == torch.Size([sum(num_nodes), num_classes]) diff --git a/tests/myria3d/models/test_model.py b/tests/myria3d/models/test_model.py index f4487c4a..122d35c6 100644 --- a/tests/myria3d/models/test_model.py +++ b/tests/myria3d/models/test_model.py @@ -29,6 +29,30 @@ def test_model_get_batch_tensor_by_enumeration(): _ = model._get_batch_tensor_by_enumeration(batch.idx_in_original_cloud) +def test_model_external_embeddings_get_batch_tensor_by_enumeration(): + config = make_default_hydra_cfg( + overrides=[ + "predict.src_las=tests/data/toy_dataset_src/862000_6652000.classified_toy_dataset.100mx100m.las", + "datamodule.epsg=2154", + "work_dir=./../../..", + "datamodule.subtile_width=1", # Extreme case with very few points per subtile + "datamodule.hdf5_file_path=null", + ] + ) + + datamodule: LightningDataModule = hydra.utils.instantiate(config.datamodule) + datamodule._set_predict_data(config.predict.src_las) + + model = Model( + neural_net_class_name="PyGRandLANet", + neural_net_hparams=dict(num_features=2, num_classes=7), + dims=[48, 192, 384, 768], + ) + for batch in datamodule.predict_dataloader(): + # Check that no error is raised ("TypeError: object of type 'numpy.int64' has no len()") + _ = model._get_batch_tensor_by_enumeration(batch.idx_in_original_cloud) + + def test_model_forward(): config = make_default_hydra_cfg( overrides=[ @@ -51,3 +75,28 @@ def test_model_forward(): for batch in datamodule.predict_dataloader(): # Check that no error is raised targets, logits = model.forward(batch) + + +def test_model_externalized_embeddings_forward(): + config = make_default_hydra_cfg( + overrides=[ + "predict.src_las=tests/data/toy_dataset_src/862000_6652000.classified_toy_dataset.100mx100m.las", + "datamodule.epsg=2154", + "model.neural_net_hparams.dims=[48,192,384,768]", + "work_dir=./../../..", + "datamodule.subtile_width=1", # Extreme case with very few points per subtile + "datamodule.hdf5_file_path=null", + ] + ) + + datamodule: LightningDataModule = hydra.utils.instantiate(config.datamodule) + datamodule._set_predict_data(config.predict.src_las) + + model: LightningModule = hydra.utils.instantiate(config.model) + device = utils.define_device_from_config_param(config.predict.gpus) + model.to(device) + model.eval() + print(model.model) + for batch in datamodule.predict_dataloader(): + # Check that no error is raised + targets, logits = model.forward(batch) diff --git a/tests/myria3d/pctl/dataset/test_hdf5.py b/tests/myria3d/pctl/dataset/test_hdf5.py new file mode 100644 index 00000000..ab0fc76a --- /dev/null +++ b/tests/myria3d/pctl/dataset/test_hdf5.py @@ -0,0 +1,82 @@ +"""Tests for myria3d.pctl.dataset.hdf5.create_hdf5. + +The deterministic train transforms (target mapping, class dropping, grid sampling) can be +either baked once into the HDF5 file at creation time (``train_pre_transform``) or applied +on-the-fly at load time (``train_transform``). Both strategies must produce the exact same +train samples. +""" + +import os.path as osp + +import torch +from torch_geometric.transforms import GridSampling + +from myria3d.pctl.dataset.hdf5 import HDF5Dataset +from myria3d.pctl.dataset.toy_dataset import ( + TOY_CLASSIFICATION_DICT, + TOY_CLASSIFICATION_PREPROCESSING_DICT, + TOY_EPSG, + TOY_LAS_DATA, +) +from myria3d.pctl.transforms.compose import CustomCompose +from myria3d.pctl.transforms.transforms import DropPointsByClass, TargetTransform + + +def _deterministic_train_transforms(): + """Transforms that can be baked (train_pre_transform) or applied at load (train_transform).""" + return CustomCompose( + [ + TargetTransform( + TOY_CLASSIFICATION_PREPROCESSING_DICT, + TOY_CLASSIFICATION_DICT, + ), + DropPointsByClass(), + GridSampling(0.25), + ] + ) + + +def _make_hdf5_dataset(hdf5_path: str, bake: bool) -> HDF5Dataset: + """Build a toy train-only HDF5 dataset, baking the transforms or applying them at load time. + + Args: + hdf5_path: where to write the HDF5 file. + bake: if True, pass the transforms as ``train_pre_transform`` (baked into the file); + if False, store raw data and pass the same transforms as ``train_transform``. + + """ + transforms = _deterministic_train_transforms() + return HDF5Dataset( + hdf5_path, + TOY_EPSG, + las_paths_by_split_dict={"train": [TOY_LAS_DATA]}, + tile_width=110, + subtile_width=50, + train_pre_transform=transforms if bake else None, + train_transform=None if bake else transforms, + eval_transform=None, + pre_filter=None, + ) + + +def test_baked_train_pre_transform_matches_unbaked(tmp_path): + """create_hdf5 with a baked train_pre_transform yields the same train samples as storing + raw data and applying the same transforms, unbaked, at load time.""" + baked = _make_hdf5_dataset(osp.join(str(tmp_path), "baked.hdf5"), bake=True) + unbaked = _make_hdf5_dataset(osp.join(str(tmp_path), "unbaked.hdf5"), bake=False) + + baked_train = baked.traindata + unbaked_train = unbaked.traindata + + # Same number of (non-empty) train samples, and the test is not vacuous. + assert len(baked_train) == len(unbaked_train) + assert len(baked_train) > 0 + + for baked_sample, unbaked_sample in zip(baked_train, unbaked_train): + assert baked_sample is not None + assert unbaked_sample is not None + # Same geometry/features/targets after the deterministic transforms. + assert baked_sample.num_nodes == unbaked_sample.num_nodes + assert torch.equal(baked_sample.y, unbaked_sample.y) + assert torch.allclose(baked_sample.pos, unbaked_sample.pos) + assert torch.allclose(baked_sample.x, unbaked_sample.x) diff --git a/tests/myria3d/test_train_and_predict.py b/tests/myria3d/test_train_and_predict.py index 7921708c..66880377 100644 --- a/tests/myria3d/test_train_and_predict.py +++ b/tests/myria3d/test_train_and_predict.py @@ -25,27 +25,70 @@ """ -@pytest.fixture(scope="session") -def one_epoch_trained_RandLaNet_checkpoint(toy_dataset_hdf5_path, tmpdir_factory): +# Embeddings dims variants exercised by the tests. `None` keeps the default RandLaNet +# architecture; each list externalizes the embeddings sizes. This is the only parameter that +# changes between the one-epoch-trained checkpoints below. Add as many variants as needed, +# e.g. [None, [48, 192, 384, 768], [64, 256, 512, 1024], ...]. +EMBEDDINGS_DIMS = [None, [48, 192, 384, 768], [64, 256, 512, 1024]] + + +def _dims_id(dims): + """Readable pytest id for an embeddings-dims variant (scales to any number of variants).""" + if dims is None: + return "default_embeddings" + return "embeddings_" + "_".join(str(d) for d in dims) + + +def _dims_override(dims, prepend_plus: bool = False): + """Build the hydra override for the neural net embeddings dims. + + Args: + dims: embeddings dims (`None` -> default architecture, i.e. no override). + prepend_plus: use the `+` append syntax, needed when the key is not already part + of the composed config (e.g. for predict commands). + + Returns: + list[str]: zero or one hydra override. + """ + if dims is None: + return [] + prefix = "+" if prepend_plus else "" + return [f"{prefix}model.neural_net_hparams.dims={dims}"] + + +@pytest.fixture( + scope="session", + params=EMBEDDINGS_DIMS, + ids=_dims_id, +) +def one_epoch_trained_RandLaNet_checkpoint(request, toy_dataset_hdf5_path, tmpdir_factory): """Train a RandLaNet model for one epoch, in order to run it in different other tests. + Parametrized over ``EMBEDDINGS_DIMS``: ``None`` trains the default architecture while a + list of dims trains a model with externalized embeddings. The matching ``dims`` value is + returned alongside the checkpoint path so tests can rebuild the right model when loading it + (otherwise the externalized-embeddings checkpoint would raise a state_dict size mismatch). + Args: + request (fixture): gives access to the parametrized ``dims`` via ``request.param``. toy_dataset_hdf5_path (str): path to toy dataset as created by fixture. tmpdir_factory (fixture): factory to create a session level tempdir. Returns: - str: path to trained model checkpoint, which persists for the whole pytest session. + Tuple[str, Optional[list]]: (checkpoint path, dims), where dims is None for the default + architecture. The checkpoint persists for the whole pytest session. """ + dims = request.param tmpdir = tmpdir_factory.mktemp("training_logs_dir") tmp_paths_overrides = _make_list_of_necesary_hydra_overrides_with_tmp_paths( toy_dataset_hdf5_path, tmpdir ) cfg_one_epoch = make_default_hydra_cfg( - overrides=["experiment=RandLaNetDebug"] + tmp_paths_overrides + overrides=["experiment=RandLaNetDebug"] + _dims_override(dims) + tmp_paths_overrides ) trainer = train(cfg_one_epoch) - return trainer.checkpoint_callback.best_model_path + return trainer.checkpoint_callback.best_model_path, dims @RunIf(min_gpus=1) @@ -77,19 +120,21 @@ def test_predict_as_command(one_epoch_trained_RandLaNet_checkpoint, tmpdir): """Test running inference by CLI for toy LAS. Args: - one_epoch_trained_RandLaNet_checkpoint (fixture -> str): path to checkpoint of - a RandLa-Net model that was trained for once epoch at start of test session. + one_epoch_trained_RandLaNet_checkpoint (fixture -> (str, dims)): checkpoint of a + RandLa-Net model trained for one epoch at start of test session, and its dims. tmpdir (fixture -> str): temporary directory. """ + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint # Hydra changes CWD, and therefore absolute paths are preferred abs_path_to_toy_LAS = osp.abspath(TOY_LAS_DATA) command = [ "run.py", - f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", + f"predict.ckpt_path={checkpoint}", f"datamodule.epsg={DEFAULT_EPSG}", f"predict.src_las={abs_path_to_toy_LAS}", f"predict.output_dir={tmpdir}", + *_dims_override(dims, prepend_plus=True), "+predict.interpolator.probas_to_save=[building,unclassified]", "task.task_name=predict", ] @@ -104,19 +149,21 @@ def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir): """Test running inference by CLI for toy LAS. Args: - one_epoch_trained_RandLaNet_checkpoint (fixture -> str): path to checkpoint of - a RandLa-Net model that was trained for once epoch at start of test session. + one_epoch_trained_RandLaNet_checkpoint (fixture -> (str, dims)): checkpoint of a + RandLa-Net model trained for one epoch at start of test session, and its dims. tmpdir (fixture -> str): temporary directory. """ + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint # Hydra changes CWD, and therefore absolute paths are preferred abs_path_to_toy_LAS = osp.abspath(TOY_LAS_DATA) command = [ "run.py", - f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", + f"predict.ckpt_path={checkpoint}", f"predict.src_las={abs_path_to_toy_LAS}", f"predict.output_dir={tmpdir}", "datamodule.epsg=null", + *_dims_override(dims, prepend_plus=True), "+predict.interpolator.probas_to_save=[building,unclassified]", "task.task_name=predict", ] @@ -128,14 +175,16 @@ def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir): def test_predict_on_single_point_cloud(one_epoch_trained_RandLaNet_checkpoint, tmpdir): """Test running inference by CLI for cloud with a single point (edge case addressed in V3.4.0)""" + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint # Hydra changes CWD, and therefore absolute paths are preferred abs_path_to_single_point_cloud = osp.abspath(SINGLE_POINT_CLOUD) command = [ "run.py", - f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", + f"predict.ckpt_path={checkpoint}", f"datamodule.epsg={DEFAULT_EPSG}", f"predict.src_las={abs_path_to_single_point_cloud}", f"predict.output_dir={tmpdir}", + *_dims_override(dims, prepend_plus=True), "+predict.interpolator.probas_to_save=[building,unclassified]", "task.task_name=predict", ] @@ -146,11 +195,12 @@ def test_RandLaNet_predict_with_invariance_checks(one_epoch_trained_RandLaNet_ch """Train a model for one epoch, and run test and predict functions using the trained model. Args: - one_epoch_trained_RandLaNet_checkpoint (fixture -> str): path to checkpoint of - a RandLa-Net model that was trained for once epoch at start of test session. + one_epoch_trained_RandLaNet_checkpoint (fixture -> (str, dims)): checkpoint of a + RandLa-Net model trained for one epoch at start of test session, and its dims. tmpdir (fixture -> str): temporary directory. """ + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint tmp_paths_overrides = _make_list_of_necesary_hydra_overrides_with_tmp_paths( "placeholder_because_no_need_for_a_dataset_here", tmpdir ) @@ -158,10 +208,11 @@ def test_RandLaNet_predict_with_invariance_checks(one_epoch_trained_RandLaNet_ch cfg_predict_using_trained_model = make_default_hydra_cfg( overrides=[ "experiment=predict", - f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", + f"predict.ckpt_path={checkpoint}", f"datamodule.epsg={DEFAULT_EPSG}", f"predict.src_las={TOY_LAS_DATA}", f"predict.output_dir={tmpdir}", + *_dims_override(dims, prepend_plus=True), # "+predict.interpolator.interpolation_k=predict.interpolation_k", "predict.interpolator.probas_to_save=[building,unclassified]", ] @@ -189,8 +240,9 @@ def test_RandLaNet_predict_with_invariance_checks(one_epoch_trained_RandLaNet_ch def test_run_test_with_trained_model_on_toy_dataset_on_cpu( one_epoch_trained_RandLaNet_checkpoint, toy_dataset_hdf5_path, tmpdir ): + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint _run_test_right_after_training( - one_epoch_trained_RandLaNet_checkpoint, toy_dataset_hdf5_path, tmpdir, "cpu" + checkpoint, toy_dataset_hdf5_path, tmpdir, "cpu", embeddings_size=dims ) @@ -198,13 +250,47 @@ def test_run_test_with_trained_model_on_toy_dataset_on_cpu( def test_run_test_with_trained_model_on_toy_dataset_on_gpu( one_epoch_trained_RandLaNet_checkpoint, toy_dataset_hdf5_path, tmpdir ): + checkpoint, dims = one_epoch_trained_RandLaNet_checkpoint _run_test_right_after_training( - one_epoch_trained_RandLaNet_checkpoint, toy_dataset_hdf5_path, tmpdir, "gpu" + checkpoint, toy_dataset_hdf5_path, tmpdir, "gpu", embeddings_size=dims ) +# model with externalized embeddings +@RunIf(min_gpus=1) +def test_FrenchLidar_RandLaNetDebug_externalized_embeddings_with_gpu( + toy_dataset_hdf5_path, tmpdir_factory +): + """Train a RandLaNet model for one epoch using GPU. XFail is no GPU available. + + Args: + toy_dataset_hdf5_path (str): path to isolated toy dataset as created by fixture. + tmpdir_factory (fixture): factory to create a session-level tempdir. + + """ + tmpdir = tmpdir_factory.mktemp("training_logs_dir") + tmp_paths_overrides = _make_list_of_necesary_hydra_overrides_with_tmp_paths( + toy_dataset_hdf5_path, tmpdir + ) + gpu_id = find_usable_cuda_devices(1) + cfg_one_epoch = make_default_hydra_cfg( + overrides=[ + "experiment=RandLaNetDebug", + "model.neural_net_hparams.dims=[48,192,384,768]", + "trainer.accelerator=gpu", + f"trainer.devices={gpu_id}", + ] + + tmp_paths_overrides + ) + train(cfg_one_epoch) + + def _run_test_right_after_training( - one_epoch_trained_RandLaNet_checkpoint, toy_dataset_hdf5_path, tmpdir, accelerator + one_epoch_trained_RandLaNet_checkpoint, + toy_dataset_hdf5_path, + tmpdir, + accelerator, + embeddings_size=None, ): """Run test using the model that was just trained for one epoch. @@ -222,6 +308,8 @@ def _run_test_right_after_training( toy_dataset_hdf5_path, tmpdir ) devices = find_usable_cuda_devices(1) if accelerator == "gpu" else 1 + additional_overrides = _dims_override(embeddings_size) + cfg_test_using_trained_model = make_default_hydra_cfg( overrides=[ "experiment=test", # sets task.task_name to "test" @@ -229,6 +317,7 @@ def _run_test_right_after_training( f"trainer.devices={devices}", f"trainer.accelerator={accelerator}", ] + + additional_overrides + tmp_paths_overrides ) train(cfg_test_using_trained_model) diff --git a/tests/test_run.py b/tests/test_run.py index 31dbe5cb..cecb2b57 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -7,22 +7,50 @@ pytestmark = pytest.mark.functional relative_output_dir = "tmp" +# Default config points to previous model checkpoint -@pytest.mark.parametrize( - "test_case, specific_args", - [ +# Specific Hydra config (directory + name, without the ".yaml" extension) and checkpoint +# to use for the externalized-embeddings prediction test. Adjust these to point at your +# externalized-embeddings assets. +EXTERNAL_EMBEDDINGS_CONFIG_DIR = "trained_model_assets" + + +# add a method that matches ckpt with config file and returns a list of tuples with +# (config_file, ckpt_file) for each model in the trained_model_assets directory +def get_model_assets(): + """ + Returns a list of tuples with (config_file, ckpt_file) for each model + in the trained_model_assets directory. + """ + model_assets = [] + for ckpt_file in Path(EXTERNAL_EMBEDDINGS_CONFIG_DIR).glob("*.ckpt"): + # try to find yaml with ckpt prefix and -inference-Myria3DV*.yaml suffix using glob + config_file = list( + Path(EXTERNAL_EMBEDDINGS_CONFIG_DIR).glob(ckpt_file.stem + "-inference-Myria3DV*.yaml") + ) + if config_file: + model_assets.append((config_file[0], ckpt_file)) + else: + raise ValueError(f"No config file found for checkpoint {ckpt_file}") + return model_assets + + +def _predict_test_cases(): + """Shared (test_case, specific_args) parametrization for the predict tests.""" + return [ ("default", []), ( "with_overlap", [ "predict.subtile_overlap=25", - "datamodule.batch_size=10", "predict.interpolator.probas_to_save=[building,ground]", "task.task_name=predict", ], ), - ], -) + ] + + +@pytest.mark.parametrize("test_case, specific_args", _predict_test_cases()) def test_run_predict(monkeypatch, test_case, specific_args): """Test run.py execution when __name__ == "__main__" with mocked command line input.""" test_args = [ @@ -34,3 +62,48 @@ def test_run_predict(monkeypatch, test_case, specific_args): monkeypatch.setattr(sys, "argv", test_args) runpy.run_module("run", run_name="__main__", alter_sys=True) assert Path(f"{relative_output_dir}/{test_case}/792000_6272000_subset_buildings.laz").is_file() + + +EXTERNAL_EMBEDDINGS_AVAILABLE = len(get_model_assets()) > 0 + + +def _all_predict_cases(): + if not EXTERNAL_EMBEDDINGS_AVAILABLE: + return + else: + for config_file, ckpt_file in get_model_assets(): + for test_case, specific_args in _predict_test_cases(): + yield pytest.param( + config_file, + ckpt_file, + test_case, + specific_args, + id=f"{config_file.stem}-{test_case}", + ) + + +@pytest.mark.skipif( + not EXTERNAL_EMBEDDINGS_AVAILABLE, + reason="Externalized-embeddings model assets are not available (not published to the repo).", +) +@pytest.mark.parametrize( + "config_file, ckpt_file, test_case, specific_args", list(_all_predict_cases()) +) +def test_run_predict_externalized_embeddings_with_tile_size( + monkeypatch, config_file, ckpt_file, test_case, specific_args +): + """Test run.py execution when __name__ == "__main__" with mocked command line input.""" + test_args = [ + "run.py", + # Hydra CLI flags selecting a specific config directory + name. These must come + # before any key=value overrides. + f"--config-dir={EXTERNAL_EMBEDDINGS_CONFIG_DIR}", + f"--config-name={config_file.stem}", + "predict.src_las=${hydra:runtime.cwd}/tests/data/las_subset_buildings/792000_6272000_subset_buildings.laz", + "predict.output_dir=${hydra:runtime.cwd}" + f"/{relative_output_dir}/{test_case}", + f"predict.ckpt_path={ckpt_file}", + "task.task_name=predict", + ] + specific_args + monkeypatch.setattr(sys, "argv", test_args) + runpy.run_module("run", run_name="__main__", alter_sys=True) + assert Path(f"{relative_output_dir}/{test_case}/792000_6272000_subset_buildings.laz").is_file() diff --git a/trained_model_assets/placeholder.env b/trained_model_assets/placeholder.env index e1701066..a53f5251 100755 --- a/trained_model_assets/placeholder.env +++ b/trained_model_assets/placeholder.env @@ -10,4 +10,4 @@ COMET_PROJECT_NAME="" COMET_WORKSPACE="" COMET_GIT_DIRECTORY="" -HYDRA_FULL_ERROR=1 \ No newline at end of file +HYDRA_FULL_ERROR=1