diff --git a/configs/data/_spatial_template.yaml b/configs/data/_spatial_template.yaml new file mode 100644 index 00000000..533a6377 --- /dev/null +++ b/configs/data/_spatial_template.yaml @@ -0,0 +1,28 @@ +# @package _global_ +# Coordinate-aware single-scale data shape: identical to _template but the DataModule +# runs over SpatialScaleDataset, which appends each tile's level-0 (x, y) as the last +# two bag columns ((N, D + 2)). Pair with the spatial_transformer aggregator, which +# slices those columns off. Select one artifact card at launch, e.g. +# data/embedded=demo_l3_embed. +dataset: + levels: ??? # filled by the selected data/embedded/* card + data_mapping: ${project_path}/data_mapping.csv + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + train: &dataset + _target_: ml.data.SpatialScaleDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + val: + <<: *dataset + split: val + test: + <<: *dataset + split: test diff --git a/configs/data/_template.yaml b/configs/data/_template.yaml new file mode 100644 index 00000000..854edfc9 --- /dev/null +++ b/configs/data/_template.yaml @@ -0,0 +1,23 @@ +# @package _global_ +dataset: + levels: ??? + data_mapping: ${project_path}/data_mapping.csv + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + train: &dataset + _target_: ml.data.SingleScaleDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + val: + <<: *dataset + split: val + test: + <<: *dataset + split: test diff --git a/configs/data/_template_raw.yaml b/configs/data/_template_raw.yaml new file mode 100644 index 00000000..ebc1afbf --- /dev/null +++ b/configs/data/_template_raw.yaml @@ -0,0 +1,26 @@ +# @package _global_ +dataset: + levels: ??? + data_mapping: ${project_path}/data_mapping.csv + max_tiles: 256 + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + train: &dataset + _target_: ml.data.RawTileSlideDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + max_tiles: ${dataset.max_tiles} + seed: ${seed} + val: + <<: *dataset + split: val + test: + <<: *dataset + split: test diff --git a/configs/data/default.yaml b/configs/data/default.yaml index 84879366..f6b75a2c 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -4,7 +4,8 @@ dataset: name: full_dataset paths: qc_masks: ${project_path}/qc_masks + epithelium_masks: ${project_path}/epithelium_onnx_masks data_mapping: ${project_path}/data_mapping.csv mlflow_uris: - tissue_masks: mlflow-artifacts:/98/aa6301ad8ae341ab935ab5325cde9369/artifacts/tissue_masks - epithelium_masks: null + tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks + cancer_masks: mlflow-artifacts:/2/70b1c915966d43e3b5be46668075bd93/artifacts/heatmaps diff --git a/configs/data/embedded/cancer_mask_5/l2.yaml b/configs/data/embedded/cancer_mask_5/l2.yaml new file mode 100644 index 00000000..6a775896 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l2.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.2 +# Level-2 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/ac2fdbe10e6047a99186e16ac149825c/artifacts/embeddings" + train: "mlflow-artifacts:/3/b88229631c214c8e8d1161ce5f2ef23b/artifacts/l2_cancer_mask_5_embed_train" + val: "mlflow-artifacts:/3/d47d498ad0524ad2b14a140779a66e4f/artifacts/l2_cancer_mask_5_embed_val" + test: "mlflow-artifacts:/3/b1e75bf1510a4dc581d1c60ae4a061c5/artifacts/l2_cancer_mask_5_embed_test" diff --git a/configs/data/embedded/cancer_mask_5/l3.yaml b/configs/data/embedded/cancer_mask_5/l3.yaml new file mode 100644 index 00000000..45d849cf --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l3.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# Level-3 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/1c71b92b718642efb3698e6c5c5e7436/artifacts/embeddings" + train: "mlflow-artifacts:/3/1cbd1a158b6f44c6924608c8b99b8c7b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c6680ed10e66455fa06e61b367f44ae0/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/48ffa80398c2468cba34738a54adc84e/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_5/l4.yaml b/configs/data/embedded/cancer_mask_5/l4.yaml new file mode 100644 index 00000000..e64066a1 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l4.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.4 +# Level-4 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/3492b1dae7be474f80cea02afe37d252/artifacts/embeddings" + train: "mlflow-artifacts:/3/2e018211e7bb4c299a3068678e798872/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/bb56dcfc27f64c62900e0413e881402a/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c6ed24afdc894c8bab90a160b9d96211/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_5/l5.yaml b/configs/data/embedded/cancer_mask_5/l5.yaml new file mode 100644 index 00000000..085d8479 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l5.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.5 +# Level-5 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/d2a3870bbe784ded95a878998513f8fd/artifacts/embeddings" + train: "mlflow-artifacts:/3/4d2342aed93d4686b1a5a464d47fc742/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/9b3c03fdc4844405b9df7877020fad70/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/7c99af8a7d2d4728bfa10c129bcdf862/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l2.yaml b/configs/data/embedded/cancer_mask_9/l2.yaml new file mode 100644 index 00000000..795f76c1 --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l2.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.2 +# Level-2 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/3f9706ef68f6434f89ed97d9031a8a6a/artifacts/embeddings" + train: "mlflow-artifacts:/3/e6e63894393440a49e809b826014606c/artifacts/l2_cancer_mask_9_embed_train" + val: "mlflow-artifacts:/3/5a487183ee21496b95a7d784190d4ee3/artifacts/l2_cancer_mask_9_embed_val" + test: "mlflow-artifacts:/3/e36817eb35de4d46a061ee8a8d8a5b7b/artifacts/l2_cancer_mask_9_embed_test" diff --git a/configs/data/embedded/cancer_mask_9/l3.yaml b/configs/data/embedded/cancer_mask_9/l3.yaml new file mode 100644 index 00000000..010d8669 --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l3.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# Level-3 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/709170ca84d64c21bc37c487942ec48f/artifacts/embeddings" + train: "mlflow-artifacts:/3/a3af7a4b837248b18ca8b387b4262f9a/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/70d7ac7c2d154291857d47ed7def18df/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e5e5ef2e3b1b4a35a9ebb4b04dfdde14/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l4.yaml b/configs/data/embedded/cancer_mask_9/l4.yaml new file mode 100644 index 00000000..6a76261d --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l4.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.4 +# Level-4 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/84dda274e69c4eb8adc3a875eec8e482/artifacts/embeddings" + train: "mlflow-artifacts:/3/a0367e263d6a4bf7a70a5c8b3316c7c1/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d1eda30624384c8eae75edf4ec07a508/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/8798109c8ec640079b2bb778590b0bde/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l5.yaml b/configs/data/embedded/cancer_mask_9/l5.yaml new file mode 100644 index 00000000..9f9859fb --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l5.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.5 +# Level-5 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/3bdfcabbdf884380b3ece1e0f37bf3b7/artifacts/embeddings" + train: "mlflow-artifacts:/3/146e36cdab604071a47e25a2d2720b36/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/b4305a97ff34482b8210b31108c99f31/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/d89478ba8003457080794fe147b4f6f0/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l2.yaml b/configs/data/embedded/epithel/l2.yaml new file mode 100644 index 00000000..12a8cf3f --- /dev/null +++ b/configs/data/embedded/epithel/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings" + train: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l3.yaml b/configs/data/embedded/epithel/l3.yaml new file mode 100644 index 00000000..e7a9553d --- /dev/null +++ b/configs/data/embedded/epithel/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings" + train: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l4.yaml b/configs/data/embedded/epithel/l4.yaml new file mode 100644 index 00000000..c9eec19e --- /dev/null +++ b/configs/data/embedded/epithel/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings" + train: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l5.yaml b/configs/data/embedded/epithel/l5.yaml new file mode 100644 index 00000000..96c79682 --- /dev/null +++ b/configs/data/embedded/epithel/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings" + train: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l2.yaml b/configs/data/embedded/epithel_and_cancer_5/l2.yaml new file mode 100644 index 00000000..3ffacf21 --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings" + train: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l3.yaml b/configs/data/embedded/epithel_and_cancer_5/l3.yaml new file mode 100644 index 00000000..0f97a28f --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings" + train: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l4.yaml b/configs/data/embedded/epithel_and_cancer_5/l4.yaml new file mode 100644 index 00000000..5855b9bd --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings" + train: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l5.yaml b/configs/data/embedded/epithel_and_cancer_5/l5.yaml new file mode 100644 index 00000000..e2ab0691 --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings" + train: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l2.yaml b/configs/data/embedded/epithel_or_cancer_5/l2.yaml new file mode 100644 index 00000000..76081123 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings" + train: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l3.yaml b/configs/data/embedded/epithel_or_cancer_5/l3.yaml new file mode 100644 index 00000000..9dc1dcc4 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings" + train: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l4.yaml b/configs/data/embedded/epithel_or_cancer_5/l4.yaml new file mode 100644 index 00000000..f381cab4 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings" + train: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l5.yaml b/configs/data/embedded/epithel_or_cancer_5/l5.yaml new file mode 100644 index 00000000..302b96e3 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings" + train: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_test" diff --git a/configs/data/embedded/tissue_only/l2.yaml b/configs/data/embedded/tissue_only/l2.yaml new file mode 100644 index 00000000..7bde776e --- /dev/null +++ b/configs/data/embedded/tissue_only/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l2; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l2_tissue_embed +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_tissue_embed +uris: + all: "mlflow-artifacts:/3/d70b72d842604ebe93d3fbd9e1dfd2f5/artifacts/embeddings" + train: "mlflow-artifacts:/3/b11db2e5ec074e1da3c172ea41880c5c/artifacts/l2_tissue_only_mask_embed_train" + val: "mlflow-artifacts:/3/8134ea19af804e8aaf51680157575cf2/artifacts/l2_tissue_only_mask_embed_val" + test: "mlflow-artifacts:/3/d8479367bf224405b3a77bbc7b7ac289/artifacts/l2_tissue_only_mask_embed_test" diff --git a/configs/data/embedded/tissue_only/l3.yaml b/configs/data/embedded/tissue_only/l3.yaml new file mode 100644 index 00000000..58700700 --- /dev/null +++ b/configs/data/embedded/tissue_only/l3.yaml @@ -0,0 +1,11 @@ +# @package dataset.levels.3 +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_tissue_embed +uris: + all: "mlflow-artifacts:/3/c513c87eac3d4792ba9757a2d7300eb6/artifacts/embeddings" + train: "mlflow-artifacts:/3/7bda595fa2b24f618d2476468c564368/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d74838d175b342569af851b7a36f31e9/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/f785f95353f545539b4166e716db9d20/artifacts/embeddings_test" diff --git a/configs/data/embedded/tissue_only/l4.yaml b/configs/data/embedded/tissue_only/l4.yaml new file mode 100644 index 00000000..30be4951 --- /dev/null +++ b/configs/data/embedded/tissue_only/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l4; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l4_tissue_embed +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_tissue_embed +uris: + all: "mlflow-artifacts:/3/ecfc0ecdba5b44d9becc5c9df1103a16/artifacts/embeddings" + train: "mlflow-artifacts:/3/1f67a719b2414301a22144c8a6ca926b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/9be7148759d946e68c64f2a0a498c3ab/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/981c02d861cf4d00b4cd42d5458bd7b3/artifacts/embeddings_test" diff --git a/configs/data/embedded/tissue_only/l5.yaml b/configs/data/embedded/tissue_only/l5.yaml new file mode 100644 index 00000000..d82eaa60 --- /dev/null +++ b/configs/data/embedded/tissue_only/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l5; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l5_tissue_embed +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_tissue_embed +uris: + all: "mlflow-artifacts:/3/e4beda97fdab43f08e8b71fc5bea733c/artifacts/embeddings" + train: "mlflow-artifacts:/3/cac061824af449af90376addd37bbc4b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d0c2242662ed4f81b3d10fa6864a36fd/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/74a96b20a8504705a4295212f2163c07/artifacts/embeddings_test" diff --git a/configs/data/levels/l0.yaml b/configs/data/levels/l0.yaml new file mode 100644 index 00000000..5d83b8a4 --- /dev/null +++ b/configs/data/levels/l0.yaml @@ -0,0 +1,4 @@ +level: 0 +mpp: 0.24 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l1.yaml b/configs/data/levels/l1.yaml new file mode 100644 index 00000000..e87c57d8 --- /dev/null +++ b/configs/data/levels/l1.yaml @@ -0,0 +1,4 @@ +level: 1 +mpp: 0.485 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l2.yaml b/configs/data/levels/l2.yaml new file mode 100644 index 00000000..6f8097f3 --- /dev/null +++ b/configs/data/levels/l2.yaml @@ -0,0 +1,4 @@ +level: 2 +mpp: 0.97 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l3.yaml b/configs/data/levels/l3.yaml new file mode 100644 index 00000000..9c57dc22 --- /dev/null +++ b/configs/data/levels/l3.yaml @@ -0,0 +1,4 @@ +level: 3 +mpp: 1.94 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l4.yaml b/configs/data/levels/l4.yaml new file mode 100644 index 00000000..99a2499f --- /dev/null +++ b/configs/data/levels/l4.yaml @@ -0,0 +1,4 @@ +level: 4 +mpp: 3.88 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l5.yaml b/configs/data/levels/l5.yaml new file mode 100644 index 00000000..61533e3a --- /dev/null +++ b/configs/data/levels/l5.yaml @@ -0,0 +1,4 @@ +level: 5 +mpp: 7.76 +tile_extent: 224 +stride: 112 diff --git a/configs/data/sources/demo_single_l3_embed.yaml b/configs/data/sources/demo_single_l3_embed.yaml new file mode 100644 index 00000000..393474bb --- /dev/null +++ b/configs/data/sources/demo_single_l3_embed.yaml @@ -0,0 +1,5 @@ +# @package _global_ +# DEMO single-scale embeddings source: level-3 embedding card + embeddings template. +defaults: + - /data/_template + - /data/tiled/demo_l3_embed diff --git a/configs/data/sources/demo_single_l3_tiled.yaml b/configs/data/sources/demo_single_l3_tiled.yaml new file mode 100644 index 00000000..91a516ef --- /dev/null +++ b/configs/data/sources/demo_single_l3_tiled.yaml @@ -0,0 +1,5 @@ +# @package _global_ +# DEMO single-scale raw-tile source: level-3 tiled card + raw-tile template. +defaults: + - /data/_template_raw + - /data/tiled/demo_l3_tiled diff --git a/configs/data/tiled/cancer_mask_5/l1.yaml b/configs/data/tiled/cancer_mask_5/l1.yaml new file mode 100644 index 00000000..4cbc190a --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l1.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.1 +# Level-1 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l1; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l1_cancer_mask +defaults: + - /data/levels/l1@_here_ + - _self_ + +name: l1_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/7651a68400f6484db36d3626b1d64e83/artifacts/mou_1_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l2.yaml b/configs/data/tiled/cancer_mask_5/l2.yaml new file mode 100644 index 00000000..81f77b8a --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l2; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l2_cancer_mask +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/698ec2dcda3a4b7aa57b250dbbbfaec0/artifacts/mou_2_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l3.yaml b/configs/data/tiled/cancer_mask_5/l3.yaml new file mode 100644 index 00000000..e82214f2 --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l3.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.3 +# Level-3 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l3; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l3_cancer_mask +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/06de9794199446c79c7edf732cfc3f82/artifacts/mou_3_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l4.yaml b/configs/data/tiled/cancer_mask_5/l4.yaml new file mode 100644 index 00000000..4518c0fd --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_cancer_mask +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/b02182a7b72241b184970623aadc1fbd/artifacts/mou_4_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l5.yaml b/configs/data/tiled/cancer_mask_5/l5.yaml new file mode 100644 index 00000000..8884ed95 --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_cancer_mask +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/e033d2f90707481a983326931159d89f/artifacts/mou_5_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l2.yaml b/configs/data/tiled/cancer_mask_9/l2.yaml new file mode 100644 index 00000000..a33078a0 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l2; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l2_cancer_mask +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/f555b4f833f44c13b60789326ab83a5c/artifacts/mou_2_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l3.yaml b/configs/data/tiled/cancer_mask_9/l3.yaml new file mode 100644 index 00000000..92357ab9 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l3.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.3 +# Level-3 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l3; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l3_cancer_mask +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/182661b3b71c4f2d9a684d700f55dc7b/artifacts/mou_3_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l4.yaml b/configs/data/tiled/cancer_mask_9/l4.yaml new file mode 100644 index 00000000..44eb24c5 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_cancer_mask +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/931879ecd8704c99904a70cd15c0c3fc/artifacts/mou_4_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l5.yaml b/configs/data/tiled/cancer_mask_9/l5.yaml new file mode 100644 index 00000000..23ad67cb --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_cancer_mask +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/a145a861300d46f19c2d3f7fe02ef5a6/artifacts/mou_5_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_test" diff --git a/configs/data/tiled/demo_l3_embed.yaml b/configs/data/tiled/demo_l3_embed.yaml new file mode 100644 index 00000000..fd31952b --- /dev/null +++ b/configs/data/tiled/demo_l3_embed.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# DEMO level-3 EMBEDDINGS card (precomputed Virchow2 features, mpp 1.94). +# Per-split uris produced by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings \ +# --name l3_embed +name: demo_l3_embed +level: 3 +mpp: 1.94 +tile_extent: 224 +uris: + train: "mlflow-artifacts:/3/6f1e6c8f3bd048eebc5addd1c3a9e36c/artifacts/l3_embed_train" + val: "mlflow-artifacts:/3/5b589be048504c169cdb6e81d585f520/artifacts/l3_embed_val" + test: "mlflow-artifacts:/3/4735b43e2fd342bea6961cf4321460bf/artifacts/l3_embed_test" diff --git a/configs/data/tiled/demo_l3_tiled.yaml b/configs/data/tiled/demo_l3_tiled.yaml new file mode 100644 index 00000000..6a22fff6 --- /dev/null +++ b/configs/data/tiled/demo_l3_tiled.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# DEMO level-3 RAW-TILE card (OpenSlide tiled artifact: slides.parquet + tiles.parquet). +# Per-split raw_uris produced by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224 \ +# --name l3_tiled +name: demo_l3_tiled +level: 3 +mpp: 1.94 +tile_extent: 224 +raw_uris: + train: "mlflow-artifacts:/3/59bc4eba00734fd1becac92949065792/artifacts/l3_tiled_train" + val: "mlflow-artifacts:/3/1c3b8348dab84158a95abac79267e71b/artifacts/l3_tiled_val" + test: "mlflow-artifacts:/3/594c785fe3064c9784c2ec50375cc59b/artifacts/l3_tiled_test" diff --git a/configs/data/tiled/tissue_only/l1.yaml b/configs/data/tiled/tissue_only/l1.yaml new file mode 100644 index 00000000..5a26fc40 --- /dev/null +++ b/configs/data/tiled/tissue_only/l1.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.1 +# Level-1 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l1; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l1_tissue +defaults: + - /data/levels/l1@_here_ + - _self_ + +name: l1_tissue +raw_uris: + all: "mlflow-artifacts:/3/43602e9f922d46f18dc06910b66a3be1/artifacts/mou_1_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l1_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l1_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l1_tissue_test" diff --git a/configs/data/tiled/tissue_only/l2.yaml b/configs/data/tiled/tissue_only/l2.yaml new file mode 100644 index 00000000..f29e2ff4 --- /dev/null +++ b/configs/data/tiled/tissue_only/l2.yaml @@ -0,0 +1,11 @@ +# @package dataset.levels.2 +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_tissue +raw_uris: + all: "mlflow-artifacts:/3/c65f9ba257da4595837750ede54d4f25/artifacts/mou_2_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l2_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l2_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l2_tissue_test" diff --git a/configs/data/tiled/tissue_only/l3.yaml b/configs/data/tiled/tissue_only/l3.yaml new file mode 100644 index 00000000..6f85f6e3 --- /dev/null +++ b/configs/data/tiled/tissue_only/l3.yaml @@ -0,0 +1,11 @@ +# @package dataset.levels.3 +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_tissue +raw_uris: + all: "mlflow-artifacts:/3/6948538ec16544ffb0e1ffda2b0a0eec/artifacts/mou_3_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l3_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l3_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l3_tissue_test" diff --git a/configs/data/tiled/tissue_only/l4.yaml b/configs/data/tiled/tissue_only/l4.yaml new file mode 100644 index 00000000..be3dbcd0 --- /dev/null +++ b/configs/data/tiled/tissue_only/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_tissue +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_tissue +raw_uris: + all: "mlflow-artifacts:/3/46b689cdd7a2450ea5c2f997a86e4561/artifacts/mou_4_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l4_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l4_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l4_tissue_test" diff --git a/configs/data/tiled/tissue_only/l5.yaml b/configs/data/tiled/tissue_only/l5.yaml new file mode 100644 index 00000000..f33ed802 --- /dev/null +++ b/configs/data/tiled/tissue_only/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_tissue +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_tissue +raw_uris: + all: "mlflow-artifacts:/3/f939531a6ff849cb8b97b34d030f8725/artifacts/mou_5_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l5_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l5_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l5_tissue_test" diff --git a/configs/experiment/ml/train_mil_embeddings.yaml b/configs/experiment/ml/train_mil_embeddings.yaml new file mode 100644 index 00000000..a3db1268 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings.yaml @@ -0,0 +1,41 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings: identity encoder (frozen +# embeddings) -> mean-pool aggregation -> linear head. Single-scale. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - /trainer: default + - /data: _template + - /data/embedded: tissue_only/l3 + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml new file mode 100644 index 00000000..2f14779f --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -0,0 +1,51 @@ +# @package _global_ +# Multi-task MIL on precomputed level-3 embeddings: identity encoder -> mean-pool +# -> MLP head with out_dim=2. The head emits a class logit (col 0) and a regression +# scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at +# once. Same data as train_mil_embeddings / train_regression, label_mode="both". +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_accuracy + - joint_auroc + - joint_f1 + - joint_precision + - joint_recall + - joint_specificity + - joint_mse + - /trainer: default + - /data: _template + - /data/embedded: tissue_only/l3 + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: mean-pool + 2-output MLP head trained jointly on luminal type + (BCE) and the continuous MammaPrint index (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml new file mode 100644 index 00000000..eeb2ea1a --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -0,0 +1,42 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings, REGRESSION of the MammaPrint index: +# identity encoder (frozen embeddings) -> mean-pool -> linear head (out_dim=1). +# Regression twin of train_mil_embeddings (label_mode=index, MSE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data: _template + - /data/embedded: tissue_only/l3 + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml new file mode 100644 index 00000000..97987f5b --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml @@ -0,0 +1,58 @@ +# @package _global_ +# Spatially-aware transformer MIL on precomputed level-3 embeddings, CLASSIFICATION +# of luminal type: identity encoder (frozen embeddings) -> SpatialTransformerMIL +# (TransMIL self-attention + 2D sinusoidal PE from each tile's (x, y)) -> linear head +# (out_dim=1). Coord-aware twin of train_mil_embeddings_transformer: swaps the data +# template to SpatialScaleDataset (bag carries the x/y columns) and the aggregator to +# spatial_transformer. label_mode=type, BCE. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention with a 2D sinusoidal positional encoding built from each tile's + physical (x, y) coordinate, + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml new file mode 100644 index 00000000..caeb4118 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml @@ -0,0 +1,62 @@ +# @package _global_ +# Multi-task spatially-aware transformer MIL on precomputed level-3 embeddings: +# identity encoder -> SpatialTransformerMIL (TransMIL self-attention + 2D sinusoidal +# PE from each tile's (x, y)) -> MLP head with out_dim=2. The head emits a class logit +# (col 0) and a regression scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * +# MSE(index) trains both at once. Coord-aware twin of +# train_mil_embeddings_transformer_joint (label_mode="both"). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_auroc + - joint_f1 + - joint_mse + - joint_pearson + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: TransMIL-style CLS-token self-attention with a 2D sinusoidal + positional encoding built from each tile's physical (x, y) coordinate, + 2-output + MLP head trained jointly on luminal type (BCE) and the continuous MammaPrint index + (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml new file mode 100644 index 00000000..cefaf4cc --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml @@ -0,0 +1,59 @@ +# @package _global_ +# Spatially-aware transformer MIL on precomputed level-3 embeddings, REGRESSION of +# the MammaPrint index: identity encoder (frozen embeddings) -> SpatialTransformerMIL +# (TransMIL self-attention + 2D sinusoidal PE from each tile's (x, y)) -> linear head +# (out_dim=1). Regression twin of train_mil_embeddings_spatial_transformer +# (label_mode=index, MSE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention with a 2D sinusoidal positional encoding built from each tile's + physical (x, y) coordinate, + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_transformer.yaml b/configs/experiment/ml/train_mil_embeddings_transformer.yaml new file mode 100644 index 00000000..8a2bbf72 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer.yaml @@ -0,0 +1,46 @@ +# @package _global_ +# TransMIL-style transformer MIL on precomputed level-3 embeddings, CLASSIFICATION +# of luminal type: identity encoder (frozen embeddings) -> transformer aggregation +# (CLS-token self-attention over the tile bag) -> linear head (out_dim=1). +# Transformer twin of train_mil_embeddings_att (label_mode=type, BCE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml new file mode 100644 index 00000000..573938cb --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml @@ -0,0 +1,55 @@ +# @package _global_ +# Multi-task transformer MIL on precomputed level-3 embeddings: identity encoder -> +# transformer aggregation (CLS-token self-attention over the tile bag) -> MLP head +# with out_dim=2. The head emits a class logit (col 0) and a regression scalar +# (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at once. +# Transformer twin of train_mil_embeddings_joint (label_mode="both"). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_accuracy + - joint_auroc + - joint_f1 + - joint_precision + - joint_recall + - joint_specificity + - joint_mse + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: TransMIL-style CLS-token self-attention aggregation + 2-output + MLP head trained jointly on luminal type (BCE) and the continuous MammaPrint + index (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml new file mode 100644 index 00000000..e26c8f22 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml @@ -0,0 +1,46 @@ +# @package _global_ +# TransMIL-style transformer MIL on precomputed level-3 embeddings, REGRESSION of +# the MammaPrint index: identity encoder (frozen embeddings) -> transformer +# aggregation (CLS-token self-attention over the tile bag) -> linear head (out_dim=1). +# Regression twin of train_mil_embeddings_transformer (label_mode=index, MSE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml new file mode 100644 index 00000000..b9047c86 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-0 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l0 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l0 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l0_cancer_mask_embed + path: ${dataset.levels.0.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml new file mode 100644 index 00000000..4b9e16e7 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-1 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l1 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask_5/l1 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l1_cancer_mask_embed + path: ${dataset.levels.1.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml new file mode 100644 index 00000000..82803462 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-2 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l2 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask_5/l2 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l2_cancer_mask_embed + path: ${dataset.levels.2.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml new file mode 100644 index 00000000..7e0e29f7 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-3 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l3 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask_5/l3 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l3_cancer_mask_embed + path: ${dataset.levels.3.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml new file mode 100644 index 00000000..bb64ffdb --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-4 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l4 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask_5/l4 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l4_cancer_mask_embed + path: ${dataset.levels.4.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml new file mode 100644 index 00000000..62d4e567 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-5 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l5 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask_5/l5 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l5_cancer_mask_embed + path: ${dataset.levels.5.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml new file mode 100644 index 00000000..a35d4cd5 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-0 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l0 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l0 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l0_tissue_embed + path: ${dataset.levels.0.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml new file mode 100644 index 00000000..b3d00615 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-1 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l1 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l1 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l1_tissue_embed + path: ${dataset.levels.1.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml new file mode 100644 index 00000000..83b04427 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-2 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l2 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l2 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l2_tissue_embed + path: ${dataset.levels.2.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml new file mode 100644 index 00000000..eb37ebae --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-3 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l3 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l3 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l3_tissue_embed + path: ${dataset.levels.3.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml new file mode 100644 index 00000000..94c90b83 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-4 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l4 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l4 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l4_tissue_embed + path: ${dataset.levels.4.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml new file mode 100644 index 00000000..9619a9fe --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-5 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l5 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l5 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l5_tissue_embed + path: ${dataset.levels.5.raw_uris.all} diff --git a/configs/experiment/preprocessing/tiling/_thresholds.yaml b/configs/experiment/preprocessing/tiling/_thresholds.yaml new file mode 100644 index 00000000..77b2b3e7 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/_thresholds.yaml @@ -0,0 +1,22 @@ +# @package _global_ +# Coverage cutoffs shared by every tiling experiment (keep tiles with overlap > +# threshold; higher overlap = cleaner). Defined once here and pulled into each +# tiling experiment via defaults. Chosen from the corrected coverage +# distributions in scripts/notebooks/mask_coverage_threshold.ipynb (mou_3_224 run): +# tissue 0.25 - on-tissue floor (center-50% ROI); raise to 0.5 for stricter purity (drops ~13%). +# blur 0.5 - drops the genuinely out-of-focus tail (~1% of tiles); lenient artifact gate. +# folding 0.1 - kept lenient: folding masks are slide-inconsistent, so a low gate avoids uneven +# filtering while still removing majority-folded tiles. +# residual 0.5 - drops heavily-contaminated tiles (some slides are ~66% residual artifact). +# epithelium null - discovery mode: record epithelium coverage without filtering until a cutoff +# is selected from the threshold-scan dataset. +# cancer 0.5 - keeps tiles with mean cancer probability above 0.5. +# Inherited across all levels; re-verify against a level's own distributions if +# you re-tile it (mask resolution differs by mpp) by overriding these in that +# experiment after `_thresholds` in its defaults list. +tissue_threshold: 0.25 +blur_threshold: 0.5 +folding_threshold: 0.1 +residual_threshold: 0.5 +epithelium_threshold: null +cancer_threshold: 0.5 diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml new file mode 100644 index 00000000..b0ad8795 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-0 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l0; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l0@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_0_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml new file mode 100644 index 00000000..0104c776 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-1 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l1; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l1@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_1_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml new file mode 100644 index 00000000..2ebc3707 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-2 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l2; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml new file mode 100644 index 00000000..9137334e --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-3 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l3; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml new file mode 100644 index 00000000..c8e36285 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-4 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l4; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml new file mode 100644 index 00000000..68a2e6df --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-5 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l5; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/mou_0_224.yaml deleted file mode 100644 index c6afa40a..00000000 --- a/configs/experiment/preprocessing/tiling/mou_0_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.24 # level 0 -tile_extent: 224 -stride: 112 -data_name: mou_0_224 diff --git a/configs/experiment/preprocessing/tiling/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/mou_1_224.yaml deleted file mode 100644 index fe4e9c71..00000000 --- a/configs/experiment/preprocessing/tiling/mou_1_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.485 # level 1 -tile_extent: 224 -stride: 112 -data_name: mou_1_224 diff --git a/configs/experiment/preprocessing/tiling/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/mou_2_224.yaml deleted file mode 100644 index 3fbfbf7c..00000000 --- a/configs/experiment/preprocessing/tiling/mou_2_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.97 # level 2 -tile_extent: 224 -stride: 112 -data_name: mou_2_224 diff --git a/configs/experiment/preprocessing/tiling/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/mou_3_224.yaml deleted file mode 100644 index 639523ae..00000000 --- a/configs/experiment/preprocessing/tiling/mou_3_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 1.94 # level 3 -tile_extent: 224 -stride: 112 -data_name: mou_3_224 diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml new file mode 100644 index 00000000..f9c9cc83 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-2 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml new file mode 100644 index 00000000..f988a0d0 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml @@ -0,0 +1,16 @@ +# @package _global_ +# Full-cohort level-3 threshold-discovery run with every available mask attached. +# +# The established tissue/QC cutoffs remain active so the persisted dataset contains the same +# production-eligible tile population. diagnostics=true additionally logs the tissue/QC drop +# funnel before those filters are applied. Epithelium and cancer filtering stay disabled so their +# complete score distributions on the surviving tiles can be inspected in the threshold notebook. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml new file mode 100644 index 00000000..4ca1ae3c --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-4 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml new file mode 100644 index 00000000..861d1b1f --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-5 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml new file mode 100644 index 00000000..e41023c5 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-0 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l0; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l0@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_0_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml new file mode 100644 index 00000000..ddc378c7 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-1 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l1; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l1@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_1_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml new file mode 100644 index 00000000..b74a380c --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-2 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l2; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml new file mode 100644 index 00000000..64476651 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-3 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l3; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml new file mode 100644 index 00000000..ea1f704e --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-4 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l4; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml new file mode 100644 index 00000000..e93ec836 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-5 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l5; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_tissue diff --git a/configs/ml.yaml b/configs/ml.yaml index 688e709b..f94c364b 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -1,6 +1,25 @@ - defaults: - base - _self_ + # Experiments select the data shape and model components. + +seed: ${random_seed:} + +# Lightning stage + checkpoint. Override for eval/inference, e.g. +# mode=test checkpoint=/path/to/best.ckpt +mode: fit # fit | validate | test | predict +checkpoint: null # path to a .ckpt to resume (fit) or load weights (validate/test/predict) + +# After a `fit` run, evaluate the best checkpoint on the test split so models are +# directly comparable on held-out data right after training. Ignored unless mode=fit. +test_after_fit: true + +# Task shape — set per experiment. Single-output (out_dim=1) for the single-task +# modes: binary classification (label_mode=type, BCE + binary metrics) or +# regression (label_mode=index, MSE). The joint mode (label_mode=both) is +# multi-task: a 2-output head + JointLoss trains classification and regression +# together. No num_classes — the task is binary-or-regression. +label_mode: type # "type" (classification), "index" (regression), "both" (joint) +feature_dim: ??? # tile-feature width = encoder.out_dim; set per experiment -seed: ${random_seed:} \ No newline at end of file +# datamodule / ml / trainer are composed by the +experiment config's defaults. diff --git a/configs/ml/aggregator/attention.yaml b/configs/ml/aggregator/attention.yaml new file mode 100644 index 00000000..1f286462 --- /dev/null +++ b/configs/ml/aggregator/attention.yaml @@ -0,0 +1,3 @@ +_target_: ml.models.aggregators.AttentionMIL +feature_dim: ${feature_dim} +attention_dim: 512 diff --git a/configs/ml/aggregator/max.yaml b/configs/ml/aggregator/max.yaml new file mode 100644 index 00000000..52fcbdc0 --- /dev/null +++ b/configs/ml/aggregator/max.yaml @@ -0,0 +1,2 @@ +_target_: ml.models.aggregators.MaxPool +feature_dim: ${feature_dim} diff --git a/configs/ml/aggregator/mean.yaml b/configs/ml/aggregator/mean.yaml new file mode 100644 index 00000000..757d2cfb --- /dev/null +++ b/configs/ml/aggregator/mean.yaml @@ -0,0 +1,2 @@ +_target_: ml.models.aggregators.MeanPool +feature_dim: ${feature_dim} diff --git a/configs/ml/aggregator/spatial_transformer.yaml b/configs/ml/aggregator/spatial_transformer.yaml new file mode 100644 index 00000000..26640bd7 --- /dev/null +++ b/configs/ml/aggregator/spatial_transformer.yaml @@ -0,0 +1,19 @@ +# TransMIL-style self-attention MIL with an added 2D sinusoidal positional encoding +# built from each tile's level-0 pixel (x, y). Coordinates ride as the last two +# columns of the bag ((N, feature_dim + 2)); the aggregator slices them off, so +# feature_dim (NOT feature_dim + 2) must be divisible by num_heads and by 4. +# +# REQUIRES a coord-aware dataset (ml.data.SpatialScaleDataset) — select the spatial +# data template. tile_extent / mpp come from the chosen level card and set the +# level-0 tile footprint (extent = tile_extent * mpp) used to grid-normalise coords. +_target_: ml.models.aggregators.SpatialTransformerMIL +feature_dim: ${feature_dim} +tile_extent: ${spatial.tile_extent} +mpp: ${spatial.mpp} +num_heads: 8 +num_layers: 2 +mlp_ratio: 4.0 +dropout: 0.1 +pe_scale: 0.1 +pe_base: 10000.0 +pe_span: 64.0 diff --git a/configs/ml/aggregator/transformer.yaml b/configs/ml/aggregator/transformer.yaml new file mode 100644 index 00000000..03942035 --- /dev/null +++ b/configs/ml/aggregator/transformer.yaml @@ -0,0 +1,7 @@ +# TransMIL-style self-attention MIL with a learned CLS token. +_target_: ml.models.aggregators.TransformerMIL +feature_dim: ${feature_dim} +num_heads: 8 +num_layers: 2 +mlp_ratio: 4.0 +dropout: 0.1 diff --git a/configs/ml/encoder/identity.yaml b/configs/ml/encoder/identity.yaml new file mode 100644 index 00000000..458143c7 --- /dev/null +++ b/configs/ml/encoder/identity.yaml @@ -0,0 +1,3 @@ +# Precomputed-embeddings path: features pass through unchanged. +_target_: ml.models.encoders.IdentityEncoder +out_dim: ${feature_dim} diff --git a/configs/ml/encoder/vgg16.yaml b/configs/ml/encoder/vgg16.yaml new file mode 100644 index 00000000..8d953c61 --- /dev/null +++ b/configs/ml/encoder/vgg16.yaml @@ -0,0 +1,4 @@ +# End-to-end path: VGG16 conv features + global max pool (512-dim per tile). +_target_: ml.models.encoders.VGG16Encoder +pretrained: true +freeze: false diff --git a/configs/ml/head/linear.yaml b/configs/ml/head/linear.yaml new file mode 100644 index 00000000..ddc1f8c3 --- /dev/null +++ b/configs/ml/head/linear.yaml @@ -0,0 +1,6 @@ +# Single-output head: one logit for binary classification (with BCE) or the +# regression scalar. out_dim is 1 in both cases. +_target_: ml.models.heads.LinearHead +in_dim: ${feature_dim} +out_dim: 1 +dropout: 0.5 diff --git a/configs/ml/head/mlp.yaml b/configs/ml/head/mlp.yaml new file mode 100644 index 00000000..b52cbd22 --- /dev/null +++ b/configs/ml/head/mlp.yaml @@ -0,0 +1,9 @@ +# Two-layer MLP head: a non-linear readout of the aggregated slide vector. +# Deeper alternative to head=linear; useful for linear probing on frozen +# embeddings where the head carries all task capacity. out_dim is 1 for +# binary classification (with BCE) or the regression scalar. +_target_: ml.models.heads.MLPHead +in_dim: ${feature_dim} +out_dim: 1 +hidden_dim: 256 +dropout: 0.5 diff --git a/configs/ml/loss/bce.yaml b/configs/ml/loss/bce.yaml new file mode 100644 index 00000000..61831546 --- /dev/null +++ b/configs/ml/loss/bce.yaml @@ -0,0 +1,2 @@ +# Binary classification with a single-logit head. Pair with output_activation=identity. +_target_: torch.nn.BCEWithLogitsLoss diff --git a/configs/ml/loss/ce.yaml b/configs/ml/loss/ce.yaml new file mode 100644 index 00000000..127bf997 --- /dev/null +++ b/configs/ml/loss/ce.yaml @@ -0,0 +1,2 @@ +# Multiclass / binary-as-2-class classification. Expects raw logits. +_target_: torch.nn.CrossEntropyLoss diff --git a/configs/ml/loss/joint.yaml b/configs/ml/loss/joint.yaml new file mode 100644 index 00000000..33249b95 --- /dev/null +++ b/configs/ml/loss/joint.yaml @@ -0,0 +1,12 @@ +# Multi-task classification + regression on the `both` label mode. +# Combines BCEWithLogits (luminal type) and MSE (MammaPrint index): +# total = w_cls * BCE(logit, type) + w_reg * MSE(scalar, index) +# +# Requires: head.out_dim=2 (col 0 = class logit, col 1 = regression scalar), +# output_activation=identity, and the dataset's label_mode="both". +# Tune w_cls / w_reg if one task dominates the gradient (the two terms live on +# different scales — BCE ~ O(1), MSE ~ the index variance). +_target_: ml.models.losses.JointLoss +w_cls: 1.0 +w_reg: 1.0 +pos_weight: null diff --git a/configs/ml/loss/mse.yaml b/configs/ml/loss/mse.yaml new file mode 100644 index 00000000..22cd58cc --- /dev/null +++ b/configs/ml/loss/mse.yaml @@ -0,0 +1,2 @@ +# Regression on the MammaPrint index. +_target_: torch.nn.MSELoss diff --git a/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml b/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml new file mode 100644 index 00000000..bb1bc42c --- /dev/null +++ b/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml @@ -0,0 +1,7 @@ +monitor: valid/loss +scheduler: + _target_: torch.optim.lr_scheduler.ReduceLROnPlateau + _partial_: true + mode: min + factor: 0.5 + patience: 3 diff --git a/configs/ml/metrics/binary_accuracy.yaml b/configs/ml/metrics/binary_accuracy.yaml new file mode 100644 index 00000000..960a8f2c --- /dev/null +++ b/configs/ml/metrics/binary_accuracy.yaml @@ -0,0 +1,6 @@ +# @package ml.metrics +accuracy: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryAccuracy diff --git a/configs/ml/metrics/binary_auroc.yaml b/configs/ml/metrics/binary_auroc.yaml new file mode 100644 index 00000000..0379c072 --- /dev/null +++ b/configs/ml/metrics/binary_auroc.yaml @@ -0,0 +1,5 @@ +# @package ml.metrics +auroc: + _target_: ml.metrics.BinaryLogitMetric + metric: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/binary_f1.yaml b/configs/ml/metrics/binary_f1.yaml new file mode 100644 index 00000000..5a30e4e9 --- /dev/null +++ b/configs/ml/metrics/binary_f1.yaml @@ -0,0 +1,6 @@ +# @package ml.metrics +f1: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryF1Score diff --git a/configs/ml/metrics/binary_precision.yaml b/configs/ml/metrics/binary_precision.yaml new file mode 100644 index 00000000..baee3604 --- /dev/null +++ b/configs/ml/metrics/binary_precision.yaml @@ -0,0 +1,6 @@ +# @package ml.metrics +precision: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryPrecision diff --git a/configs/ml/metrics/binary_recall.yaml b/configs/ml/metrics/binary_recall.yaml new file mode 100644 index 00000000..b9127f73 --- /dev/null +++ b/configs/ml/metrics/binary_recall.yaml @@ -0,0 +1,6 @@ +# @package ml.metrics +recall: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryRecall diff --git a/configs/ml/metrics/binary_specificity.yaml b/configs/ml/metrics/binary_specificity.yaml new file mode 100644 index 00000000..21170299 --- /dev/null +++ b/configs/ml/metrics/binary_specificity.yaml @@ -0,0 +1,6 @@ +# @package ml.metrics +specificity: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinarySpecificity diff --git a/configs/ml/metrics/joint_accuracy.yaml b/configs/ml/metrics/joint_accuracy.yaml new file mode 100644 index 00000000..ebf12eee --- /dev/null +++ b/configs/ml/metrics/joint_accuracy.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_accuracy: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryAccuracy diff --git a/configs/ml/metrics/joint_auroc.yaml b/configs/ml/metrics/joint_auroc.yaml new file mode 100644 index 00000000..30a07512 --- /dev/null +++ b/configs/ml/metrics/joint_auroc.yaml @@ -0,0 +1,11 @@ +# @package ml.metrics +# Classification AUROC on the joint head's column 0 (class logit vs type label). +# Sigmoid is applied consistently on every update before BinaryAUROC. This keeps +# ranking correct for the slide dataloader's batch size of one. +joint_auroc: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + metric: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/joint_f1.yaml b/configs/ml/metrics/joint_f1.yaml new file mode 100644 index 00000000..3ef3f577 --- /dev/null +++ b/configs/ml/metrics/joint_f1.yaml @@ -0,0 +1,11 @@ +# @package ml.metrics +# Classification F1 on the joint head's column 0. BinaryLogitMetric thresholds +# logits at zero and leaves the already-binary targets unchanged. +joint_f1: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryF1Score diff --git a/configs/ml/metrics/joint_mse.yaml b/configs/ml/metrics/joint_mse.yaml new file mode 100644 index 00000000..b39bbad8 --- /dev/null +++ b/configs/ml/metrics/joint_mse.yaml @@ -0,0 +1,7 @@ +# @package ml.metrics +# Regression MSE on the joint head's column 1 (index scalar vs MammaPrint index). +joint_mse: + _target_: ml.metrics.ColumnMetric + column: 1 + metric: + _target_: torchmetrics.MeanSquaredError diff --git a/configs/ml/metrics/joint_precision.yaml b/configs/ml/metrics/joint_precision.yaml new file mode 100644 index 00000000..9bb718b5 --- /dev/null +++ b/configs/ml/metrics/joint_precision.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_precision: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryPrecision diff --git a/configs/ml/metrics/joint_recall.yaml b/configs/ml/metrics/joint_recall.yaml new file mode 100644 index 00000000..a59d4dcc --- /dev/null +++ b/configs/ml/metrics/joint_recall.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_recall: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryRecall diff --git a/configs/ml/metrics/joint_specificity.yaml b/configs/ml/metrics/joint_specificity.yaml new file mode 100644 index 00000000..b9ac1734 --- /dev/null +++ b/configs/ml/metrics/joint_specificity.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_specificity: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinarySpecificity diff --git a/configs/ml/metrics/mae.yaml b/configs/ml/metrics/mae.yaml new file mode 100644 index 00000000..9c099bcb --- /dev/null +++ b/configs/ml/metrics/mae.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +mae: + _target_: torchmetrics.MeanAbsoluteError diff --git a/configs/ml/metrics/mse.yaml b/configs/ml/metrics/mse.yaml new file mode 100644 index 00000000..84aea1d0 --- /dev/null +++ b/configs/ml/metrics/mse.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +mse: + _target_: torchmetrics.MeanSquaredError diff --git a/configs/ml/metrics/thresholded_f1.yaml b/configs/ml/metrics/thresholded_f1.yaml new file mode 100644 index 00000000..69851134 --- /dev/null +++ b/configs/ml/metrics/thresholded_f1.yaml @@ -0,0 +1,5 @@ +# @package ml.metrics +# Classification F1 from a regression output: index >= 0 -> class 1 (a luminal). +thresholded_f1: + _target_: ml.metrics.thresholded_f1 + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_precision.yaml b/configs/ml/metrics/thresholded_precision.yaml new file mode 100644 index 00000000..bab03590 --- /dev/null +++ b/configs/ml/metrics/thresholded_precision.yaml @@ -0,0 +1,4 @@ +# @package ml.metrics +thresholded_precision: + _target_: ml.metrics.thresholded_precision + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_recall.yaml b/configs/ml/metrics/thresholded_recall.yaml new file mode 100644 index 00000000..12f3f311 --- /dev/null +++ b/configs/ml/metrics/thresholded_recall.yaml @@ -0,0 +1,5 @@ +# @package ml.metrics +# recall == sensitivity == TP / (TP + FN) +thresholded_recall: + _target_: ml.metrics.thresholded_recall + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_specificity.yaml b/configs/ml/metrics/thresholded_specificity.yaml new file mode 100644 index 00000000..b81742d9 --- /dev/null +++ b/configs/ml/metrics/thresholded_specificity.yaml @@ -0,0 +1,4 @@ +# @package ml.metrics +thresholded_specificity: + _target_: ml.metrics.thresholded_specificity + threshold: 0.0 diff --git a/configs/ml/module.yaml b/configs/ml/module.yaml new file mode 100644 index 00000000..926a6237 --- /dev/null +++ b/configs/ml/module.yaml @@ -0,0 +1,13 @@ +_target_: ml.models.MammaprintModule + +# Swappable pieces — populated by the experiment's defaults list +# (ml/encoder, ml/aggregator, ml/head, ml/loss, ml/optimizer, ml/lr_scheduler, +# ml/output_activation, ml/metrics). +encoder: ??? +aggregator: ??? +head: ??? +output_activation: null +loss: null +optimizer: null +lr_scheduler: null +metrics: null diff --git a/configs/ml/optimizer/adam.yaml b/configs/ml/optimizer/adam.yaml new file mode 100644 index 00000000..35fb45e3 --- /dev/null +++ b/configs/ml/optimizer/adam.yaml @@ -0,0 +1,5 @@ +# Bound to model parameters in configure_optimizers (hence _partial_). +_target_: torch.optim.Adam +_partial_: true +lr: 1e-3 +weight_decay: 1e-3 diff --git a/configs/ml/output_activation/identity.yaml b/configs/ml/output_activation/identity.yaml new file mode 100644 index 00000000..d8bfafa3 --- /dev/null +++ b/configs/ml/output_activation/identity.yaml @@ -0,0 +1 @@ +_target_: torch.nn.Identity diff --git a/configs/ml/output_activation/softmax.yaml b/configs/ml/output_activation/softmax.yaml new file mode 100644 index 00000000..e554c8db --- /dev/null +++ b/configs/ml/output_activation/softmax.yaml @@ -0,0 +1,2 @@ +_target_: torch.nn.Softmax +dim: -1 diff --git a/configs/preprocessing/epithelium_masks.yaml b/configs/preprocessing/epithelium_masks.yaml index c699cf54..46cfc133 100644 --- a/configs/preprocessing/epithelium_masks.yaml +++ b/configs/preprocessing/epithelium_masks.yaml @@ -6,7 +6,7 @@ dataset: mlflow_uris: tissue_masks: ??? -max_concurrent: 4 +max_concurrent: 100 timeout: 3000 model_name: episeg-1 diff --git a/configs/preprocessing/tiling.yaml b/configs/preprocessing/tiling.yaml index 7fc4812d..f70ab851 100644 --- a/configs/preprocessing/tiling.yaml +++ b/configs/preprocessing/tiling.yaml @@ -5,6 +5,17 @@ tile_extent: ??? stride: ??? data_name: ??? +# Tile-filtering thresholds on the *_overlap scores (higher = cleaner). A tile is kept only when +# its overlap exceeds the threshold. See scripts/notebooks/mask_coverage_threshold.ipynb. +tissue_threshold: ??? +blur_threshold: ??? +folding_threshold: ??? +residual_threshold: ??? +epithelium_threshold: ??? +cancer_threshold: ??? + +diagnostics: false + metadata: run_name: Mammaprint Tiling ${dataset.name} description: | @@ -13,10 +24,17 @@ metadata: * reads slide metadata and builds a grid of tiles with the given stride * assigns tissue/epithelium/QC mask paths and computes overlap scores * keeps tiles with tissue in the centered 50% ROI - * computes full-tile overlaps for epithelium, blur, folding, and residual masks + * computes full-tile overlaps for epithelium, cancer, blur, folding, and residual masks + * optionally filters tiles by the per-mask coverage thresholds * saves slide and tile datasets via MLflow hyperparams: mpp: ${mpp} tile_extent: ${tile_extent} stride: ${stride} - data_name: ${data_name} \ No newline at end of file + data_name: ${data_name} + tissue_threshold: ${tissue_threshold} + blur_threshold: ${blur_threshold} + folding_threshold: ${folding_threshold} + residual_threshold: ${residual_threshold} + epithelium_threshold: ${epithelium_threshold} + cancer_threshold: ${cancer_threshold} diff --git a/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv b/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv new file mode 100644 index 00000000..05fae806 --- /dev/null +++ b/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv @@ -0,0 +1,37 @@ +level,head,aggregator,loss,mask_type +2,mlp,mean,bce,epithel +5,linear,attention,bce,epithel_and_cancer_5 +2,linear,max,regression,tissue_only +3,mlp,mean,regression,cancer_mask_9 +5,mlp,max,bce,cancer_mask_5 +2,linear,mean,joint,epithel_or_cancer_5 +3,linear,attention,bce,cancer_mask_9 +3,mlp,max,regression,epithel +4,mlp,attention,regression,epithel +3,linear,transformer,joint,epithel +3,linear,attention,joint,tissue_only +5,linear,max,bce,epithel_or_cancer_5 +2,linear,transformer,bce,cancer_mask_9 +3,linear,mean,joint,epithel_and_cancer_5 +2,linear,max,joint,cancer_mask_5 +4,mlp,attention,joint,tissue_only +4,mlp,mean,bce,epithel_or_cancer_5 +5,mlp,attention,joint,epithel_or_cancer_5 +4,mlp,max,joint,epithel_and_cancer_5 +2,linear,attention,bce,cancer_mask_5 +4,linear,max,joint,cancer_mask_9 +2,mlp,transformer,regression,epithel_or_cancer_5 +5,mlp,transformer,bce,tissue_only +4,linear,transformer,regression,epithel_and_cancer_5 +3,linear,max,regression,epithel_or_cancer_5 +4,linear,mean,regression,cancer_mask_5 +2,mlp,attention,regression,epithel_and_cancer_5 +5,linear,mean,regression,epithel +5,linear,mean,regression,tissue_only +5,mlp,max,regression,cancer_mask_9 +2,linear,max,bce,epithel +3,mlp,max,bce,tissue_only +3,mlp,transformer,regression,cancer_mask_5 +5,mlp,transformer,joint,epithel +4,linear,transformer,bce,epithel_or_cancer_5 +2,mlp,transformer,joint,cancer_mask_9 diff --git a/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv b/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv new file mode 100644 index 00000000..ac67b6bb --- /dev/null +++ b/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv @@ -0,0 +1,49 @@ +level,head,aggregator,loss,mask_type +5,mlp,mean,regression,epithel +3,linear,attention,bce,epithel_and_cancer_5 +3,linear,attention,joint,cancer_mask_9 +4,mlp,mean,joint,cancer_mask_5 +2,mlp,transformer,bce,cancer_mask_9 +2,mlp,attention,joint,epithel_and_cancer_5 +3,mlp,max,joint,cancer_mask_9 +5,linear,mean,joint,epithel_and_cancer_5 +4,linear,attention,bce,epithel_or_cancer_5 +3,mlp,mean,regression,epithel_or_cancer_5 +5,linear,mean,bce,tissue_only +2,mlp,transformer,joint,cancer_mask_5 +5,linear,max,bce,cancer_mask_5 +2,linear,mean,regression,tissue_only +4,mlp,mean,joint,epithel +2,linear,attention,joint,epithel +2,mlp,attention,bce,epithel_or_cancer_5 +4,mlp,max,joint,epithel_and_cancer_5 +2,linear,max,regression,cancer_mask_5 +4,mlp,attention,regression,tissue_only +5,mlp,attention,regression,tissue_only +5,linear,transformer,bce,epithel +5,linear,attention,regression,cancer_mask_9 +3,mlp,mean,bce,tissue_only +3,linear,transformer,regression,epithel_and_cancer_5 +2,linear,transformer,bce,tissue_only +2,linear,max,regression,epithel_or_cancer_5 +2,mlp,mean,bce,epithel_and_cancer_5 +3,mlp,max,joint,tissue_only +2,mlp,max,regression,epithel +4,mlp,transformer,regression,epithel +2,linear,mean,joint,cancer_mask_5 +2,mlp,transformer,joint,cancer_mask_9 +5,linear,max,regression,cancer_mask_9 +4,mlp,transformer,regression,epithel_and_cancer_5 +4,linear,max,joint,tissue_only +4,mlp,attention,bce,cancer_mask_5 +4,linear,transformer,joint,epithel_or_cancer_5 +5,mlp,max,joint,epithel_or_cancer_5 +5,mlp,max,bce,epithel_and_cancer_5 +3,mlp,max,bce,epithel +3,linear,attention,joint,epithel +5,linear,transformer,joint,tissue_only +4,linear,mean,bce,cancer_mask_9 +4,linear,mean,regression,cancer_mask_9 +3,linear,transformer,regression,cancer_mask_5 +5,mlp,transformer,joint,epithel_or_cancer_5 +5,mlp,attention,joint,cancer_mask_5 diff --git a/configs/trainer/default.yaml b/configs/trainer/default.yaml new file mode 100644 index 00000000..9a9cfe28 --- /dev/null +++ b/configs/trainer/default.yaml @@ -0,0 +1,22 @@ +_target_: rationai.mlkit.lightning.Trainer + +max_epochs: 100 +log_every_n_steps: 1 +accelerator: auto +devices: auto + +callbacks: + early_stopping: + _target_: lightning.pytorch.callbacks.EarlyStopping + monitor: valid/loss + mode: min + patience: 10 + model_checkpoint: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + monitor: valid/loss + mode: min + save_top_k: 1 + filename: best + lr_monitor: + _target_: lightning.pytorch.callbacks.LearningRateMonitor + logging_interval: epoch diff --git a/main.py b/main.py new file mode 100644 index 00000000..1de32428 --- /dev/null +++ b/main.py @@ -0,0 +1,118 @@ +import numpy as np +import pandas as pd +from sklearn.preprocessing import OneHotEncoder + +# --------------------------------------------------------- +# 1. DEFINICE VAŠICH FAKTORŮ A JEJICH HLADIN +# --------------------------------------------------------- +factors = { + 'level': [2, 3, 4, 5], # 4 + 'head': ['linear', 'mlp'], # 2 + 'aggregator': ['mean', 'max', 'attention', 'transformer'], # 4 + 'loss': ['bce', 'regression', 'joint'], # 3 + 'mask_type': ['tissue_only', 'cancer_mask_5', 'cancer_mask_9', 'epithel', 'epithel_and_cancer_5', 'epithel_or_cancer_5'] # 5 +} + +# Vytvoření plné mřížky (Full Grid Search = 4 * 2 * 4 * 3 * 3 = 288 kombinací) +grid_tuples = pd.MultiIndex.from_product(factors.values(), names=factors.keys()) +full_grid = grid_tuples.to_frame().reset_index(drop=True) + +print(f"Celkový počet kombinací v plné mřížce (Full Grid): {len(full_grid)}") + +# --------------------------------------------------------- +# 2. PŘEVOD NA KATEGORICKOU MATICI (Dummy / One-Hot Encoding) +# --------------------------------------------------------- +# Jelikož jsou parametry kategorické, převedeme je na 0/1 matici pro výpočet determinantu +encoder = OneHotEncoder(drop='first', sparse_output=False) +X_full = encoder.fit_transform(full_grid) + +# Přidání sloupce jedniček pro absolutní člen (intercept b0) +X_full_intercept = np.hstack([np.ones((X_full.shape[0], 1)), X_full]) +p = X_full_intercept.shape[1] # Počet odhadovaných parametrů v modelu + +print(f"Počet volných parametrů k odhadnutí v modelu (p): {p}") + +# --------------------------------------------------------- +# 3. ALGORITMUS PRO D-OPTIMAL SELECTION (Fedorov's Exchange Idea) +# --------------------------------------------------------- +def compute_d_efficiency(X_sub, X_full_matrix): + """Spočítá D-efektivitu vybraného vzorku vůči celkovému vzorku.""" + N_sub = X_sub.shape[0] + N_full = X_full_matrix.shape[0] + + # Matice informace M = X^T * X + M_sub = np.dot(X_sub.T, X_sub) + M_full = np.dot(X_full_matrix.T, X_full_matrix) + + sign_sub, logdet_sub = np.linalg.slogdet(M_sub) + sign_full, logdet_full = np.linalg.slogdet(M_full) + + if sign_sub <= 0: + return 0.0 # Singulární matice (nedostatek dat pro odhad) + + # Relativní D-efficiency vzorec v logaritmickém tvaru pro numerickou stabilitu + det_ratio_per_param = np.exp((logdet_sub - logdet_full) / p) + d_eff = (det_ratio_per_param) * (N_full / N_sub) + return d_eff * 100 + +def generate_d_optimal_design(X_matrix, full_df, n_samples, n_iterations=100): + """ + Vybere n_samples řádků tak, aby maximizoval determinant X^T * X. + """ + best_det = -np.inf + best_indices = None + + np.random.seed(42) # Pro reprodukovatelnost + + for _ in range(n_iterations): + # Náhodný počáteční výběr + current_indices = np.random.choice(X_matrix.shape[0], size=n_samples, replace=False) + + # Lokální vylepšování determinantu (Exchange algorithm) + for i in range(n_samples): + current_X = X_matrix[current_indices] + sign, current_det = np.linalg.slogdet(np.dot(current_X.T, current_X)) + + if sign <= 0: + current_det = -np.inf + + # Zkusíme zaměnit i-tý prvek za jiný kandidát + candidates = np.setdiff1d(np.arange(X_matrix.shape[0]), current_indices) + for cand in np.random.choice(candidates, size=min(30, len(candidates)), replace=False): + temp_indices = current_indices.copy() + temp_indices[i] = cand + + temp_X = X_matrix[temp_indices] + temp_sign, temp_det = np.linalg.slogdet(np.dot(temp_X.T, temp_X)) + + if temp_sign > 0 and temp_det > current_det: + current_det = temp_det + current_indices = temp_indices.copy() + + if current_det > best_det: + best_det = current_det + best_indices = current_indices + + selected_df = full_df.iloc[best_indices].copy().reset_index(drop=True) + d_eff = compute_d_efficiency(X_matrix[best_indices], X_matrix) + + return selected_df, d_eff + +# --------------------------------------------------------- +# 4. GENEROVÁNÍ DESIGNU PRO 36 A 48 KOMBINACÍ +# --------------------------------------------------------- +print("\nGeneruji D-Optimal Design pro 36 kombinací...") +design_36, d_eff_36 = generate_d_optimal_design(X_full_intercept, full_grid, n_samples=36) +print(f"-> D-Efektivita pro 36 kombinací: {d_eff_36:.2f} %") + +print("\nGeneruji D-Optimal Design pro 48 kombinací...") +design_48, d_eff_48 = generate_d_optimal_design(X_full_intercept, full_grid, n_samples=48) +print(f"-> D-Efektivita pro 48 kombinací: {d_eff_48:.2f} %") + +# --------------------------------------------------------- +# 5. ULOŽENÍ VÝSLEDKŮ DO CSV +# --------------------------------------------------------- +design_36.to_csv("configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv", index=False) +design_48.to_csv("configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv", index=False) + +print("\nSoubory 'configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv' a 'configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv' byly úspěšně vytvořeny.") diff --git a/ml/data/__init__.py b/ml/data/__init__.py index f27993ae..3aa9b514 100644 --- a/ml/data/__init__.py +++ b/ml/data/__init__.py @@ -1 +1,22 @@ """Data utilities for MammaPrint.""" + +from ml.data.datamodule import DataModule, mil_collate + +# ── Multilevel (comment out this line + the __all__ entry to ship single-level) ── +# from ml.data.datasets.pyramid import PyramidSlideDataset +from ml.data.datasets.raw_tiles import RawTileSlideDataset +from ml.data.datasets.single_scale import SingleScaleDataset +from ml.data.datasets.spatial_scale import SpatialScaleDataset + + +# Grouped (single-level first, multilevel last) to make the toggle obvious, so this +# is deliberately not alphabetically sorted. +__all__ = [ + "DataModule", + "RawTileSlideDataset", + "SingleScaleDataset", + "SpatialScaleDataset", + "mil_collate", + # ── Multilevel ── + # "PyramidSlideDataset", +] diff --git a/ml/data/datamodule.py b/ml/data/datamodule.py new file mode 100644 index 00000000..cba27e10 --- /dev/null +++ b/ml/data/datamodule.py @@ -0,0 +1,102 @@ +"""Thin Lightning datamodule that builds its split datasets lazily. + +Each split (``train``/``val``/``test``) is passed as an *unresolved* Hydra config +node and instantiated on demand in :meth:`setup`, only for the running stage — so a +test-only run never downloads the train/val artifacts, and vice versa. The +entrypoint must instantiate this with ``_recursive_=False`` so the split nodes +arrive as ``DictConfig`` rather than pre-built datasets. + +Pattern follows the sibling ``ulcerative_colitis.data.DataModule``. Bags have +variable tile/region counts, so batches stay a list of samples (:func:`mil_collate`). +""" + +from collections.abc import Sequence + +import lightning.pytorch as pl +from hydra.utils import instantiate +from omegaconf import DictConfig +from torch.utils.data import DataLoader, Dataset + +from ml.typing import MILSample + + +type Sample = MILSample + + +def mil_collate[S](batch: Sequence[S]) -> list[S]: + """Collate variable-length bags by keeping the batch as a list of samples.""" + return list(batch) + + +class DataModule(pl.LightningDataModule): + """Builds train/val/test datasets lazily from unresolved config nodes. + + Args: + batch_size: Slides (bags) per batch. + num_workers: Dataloader worker processes. + pin_memory: Pin host memory for faster host->device copies. + persistent_workers: Keep workers alive between epochs. + **datasets: One entry per split (``train``/``val``/``test``), each an + unresolved dataset ``DictConfig`` instantiated in :meth:`setup`. + Prediction reuses the test split. + """ + + def __init__( + self, + batch_size: int = 1, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + **datasets: DictConfig, + ) -> None: + super().__init__() + self.batch_size = batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + self._dataset_cfgs = datasets + self._datasets: dict[str, Dataset[Sample]] = {} + + def _build(self, split: str) -> None: + if split in self._datasets: + return + cfg = self._dataset_cfgs.get(split) + if cfg is None: + raise KeyError(f"No dataset config provided for split '{split}'.") + self._datasets[split] = instantiate(cfg) + + def setup(self, stage: str | None = None) -> None: + needed = { + "fit": ("train", "val"), + "validate": ("val",), + "test": ("test",), + "predict": ("test",), + }.get(stage or "", ("train", "val", "test")) + for split in needed: + self._build(split) + + def _dataloader(self, split: str, shuffle: bool) -> DataLoader[Sample]: + return DataLoader( + self._datasets[split], + batch_size=self.batch_size, + shuffle=shuffle, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, + collate_fn=mil_collate, + ) + + def train_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("train", shuffle=True) + + def val_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("val", shuffle=False) + + def test_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("test", shuffle=False) + + def predict_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("test", shuffle=False) + + +__all__ = ["DataModule", "mil_collate"] diff --git a/ml/data/datasets/__init__.py b/ml/data/datasets/__init__.py index 7108560c..2d6a1d61 100644 --- a/ml/data/datasets/__init__.py +++ b/ml/data/datasets/__init__.py @@ -1,4 +1,23 @@ -from ml.data.datasets.tiles import SlideDataset, TileDataset +"""Datasets for MammaPrint. + +``SlideDataset``/``TileDataset`` are imported lazily: they pull in OpenSlide via +``rationai.mlkit``, which the embeddings/alignment path does not need. Accessing +them (``from ml.data.datasets import SlideDataset``) triggers the import on demand. +""" + +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from ml.data.datasets.tiles import SlideDataset, TileDataset __all__ = ["SlideDataset", "TileDataset"] + + +def __getattr__(name: str) -> Any: + if name in {"SlideDataset", "TileDataset"}: + from ml.data.datasets import tiles + + return getattr(tiles, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py new file mode 100644 index 00000000..1690849c --- /dev/null +++ b/ml/data/datasets/_sources.py @@ -0,0 +1,210 @@ +"""Shared setup helpers for the slide datasets. + +:class:`~ml.data.datasets.pyramid.PyramidSlideDataset` needs a few setup steps: +load + label the slide table, download the per-level artifacts, and find which +slides are actually present across levels. Extracting them here keeps the dataset +free of scaffolding and leaves them independently unit-testable. + +The train/val/test split is NOT applied here — splitting is materialised into +physically separate artifacts upstream by ``scripts/preprocessing/split_dataset.py`` +(the ``split`` column in ``data_mapping.csv`` is the sole split authority). A card's +per-split URI already points at a pure artifact, so no split filter is needed. +""" + +import logging +import tempfile +import time +from collections.abc import Mapping +from pathlib import Path + +import pandas as pd +from mlflow.artifacts import ( + download_artifacts as _download_artifacts, +) +from mlflow.artifacts import ( + list_artifacts as _list_artifacts, +) +from mlflow.entities import FileInfo +from mlflow.exceptions import MlflowException +from mlflow.tracking.artifact_utils import _get_root_uri_and_artifact_path +from mlflow.utils.uri import append_to_uri_path + +from ml.data.datasets.labels import LabelMode, get_target_columns, process_slides + + +logger = logging.getLogger(__name__) + +MLFLOW_DOWNLOAD_MAX_RETRIES = 3 +MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS = 1.0 + + +def _artifact_files(root_uri: str, artifact_path: str) -> list[FileInfo]: + """Return every file below an artifact directory using metadata only.""" + files: list[FileInfo] = [] + pending = [artifact_path] + while pending: + current_path = pending.pop() + current_uri = append_to_uri_path(root_uri, current_path) + for item in _list_artifacts(artifact_uri=current_uri): + if item.is_dir: + pending.append(item.path) + else: + files.append(item) + return files + + +def _is_complete_download(file_info: FileInfo, destination: Path) -> bool: + local_path = destination / file_info.path + if not local_path.is_file(): + return False + return ( + file_info.file_size is None or local_path.stat().st_size == file_info.file_size + ) + + +def _download_incomplete_artifacts( + root_uri: str, + artifact_path: str, + destination: Path, +) -> Path: + remote_files = _artifact_files(root_uri, artifact_path) + if not remote_files: + # The URI may point to a single file rather than a directory. + uri = append_to_uri_path(root_uri, artifact_path) + return Path(_download_artifacts(artifact_uri=uri, dst_path=str(destination))) + + incomplete = [ + file_info + for file_info in remote_files + if not _is_complete_download(file_info, destination) + ] + logger.info( + "Retrying %d incomplete MLflow artifact(s); keeping %d already downloaded.", + len(incomplete), + len(remote_files) - len(incomplete), + ) + for file_info in incomplete: + file_uri = append_to_uri_path(root_uri, file_info.path) + _download_artifacts(artifact_uri=file_uri, dst_path=str(destination)) + + return destination / artifact_path + + +def download_artifacts_with_retries( + artifact_uri: str, + *, + max_retries: int = MLFLOW_DOWNLOAD_MAX_RETRIES, + initial_backoff_seconds: float = MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS, +) -> Path: + """Download an MLflow artifact, resuming transient download failures. + + MLflow raises one aggregate :class:`MlflowException` when any file in a + directory fails. The initial parallel download uses a stable temporary + destination; retries preserve complete files and fetch only missing or + size-mismatched files. + """ + if max_retries < 0: + raise ValueError("max_retries must be non-negative.") + if initial_backoff_seconds < 0: + raise ValueError("initial_backoff_seconds must be non-negative.") + + root_uri, artifact_path = _get_root_uri_and_artifact_path(artifact_uri) + destination = Path(tempfile.mkdtemp(prefix="mammaprint-mlflow-artifacts-")) + + for retry in range(max_retries + 1): + try: + if retry == 0: + return Path( + _download_artifacts( + artifact_uri=artifact_uri, + dst_path=str(destination), + ) + ) + return _download_incomplete_artifacts( + root_uri, + artifact_path, + destination, + ) + except MlflowException: + if retry == max_retries: + raise + + delay = initial_backoff_seconds * 2**retry + logger.warning( + "MLflow artifact download failed for %s; retrying in %.1f seconds " + "(%d/%d).", + artifact_uri, + delay, + retry + 1, + max_retries, + ) + time.sleep(delay) + + raise AssertionError("unreachable") + + +def load_labeled_slides( + data_mapping: str | Path, + label_mode: LabelMode, +) -> pd.DataFrame: + """Load the slide table and drop rows without a label. + + Args: + data_mapping: Path to ``data_mapping.csv`` (``record_num``/``type``/ + ``mammaprint_index``/``path``). + label_mode: Classification (``type``) or regression (``index``) target. + + Returns: + A dataframe with a ``name`` (slide stem) column and the label column, keeping + only rows that carry a label. + """ + slides = pd.read_csv(data_mapping) + raw_target_columns = { + LabelMode.TYPE: ["type"], + LabelMode.INDEX: ["mammaprint_index"], + LabelMode.BOTH: ["type", "mammaprint_index"], + }[label_mode] + slides = slides.dropna(subset=raw_target_columns) + slides = process_slides(slides, mode=label_mode) + + target_columns = get_target_columns(label_mode) + keep = slides[target_columns].notna().all(axis=1) + return slides[keep].reset_index(drop=True) + + +def download_level_sources(sources: Mapping[int, str]) -> dict[int, Path]: + """Download each pyramid level's artifact and return ``level -> local dir``.""" + return { + level: download_artifacts_with_retries(uri) for level, uri in sources.items() + } + + +def available_slides(dirs: Mapping[int, Path]) -> set[str]: + """Slide stems present as ``.parquet`` in *every* level's directory.""" + per_level = ({p.stem for p in d.glob("*.parquet")} for d in dirs.values()) + return set.intersection(*per_level) if dirs else set() + + +def split_uri(card: Mapping[str, object], key: str, split: str, level: int) -> str: + """Read a level card's per-split artifact URI (``card[key][split]``). + + Raises a clear error if the card lacks the URI map or the requested split. + """ + uris = card.get(key) + if not isinstance(uris, Mapping): + raise KeyError(f"Level {level} data card is missing a '{key}' split->URI map.") + uri = uris.get(split) + if not isinstance(uri, str): + raise KeyError( + f"Level {level} data card '{key}' has no URI for split '{split}'." + ) + return uri + + +__all__ = [ + "available_slides", + "download_artifacts_with_retries", + "download_level_sources", + "load_labeled_slides", + "split_uri", +] diff --git a/ml/data/datasets/labels.py b/ml/data/datasets/labels.py index dc6ad689..7465a78c 100644 --- a/ml/data/datasets/labels.py +++ b/ml/data/datasets/labels.py @@ -10,6 +10,7 @@ class LabelMode(Enum): TYPE = "type" INDEX = "index" + BOTH = "both" def _map_luminal_type(value: object) -> int: @@ -27,27 +28,56 @@ def process_slides(slides: pd.DataFrame, mode: LabelMode | None = None) -> pd.Da slides["type_label"] = slides["type"].map(_map_luminal_type) case LabelMode.INDEX: slides = slides.copy() - slides["index"] = slides["index"].astype(float) + slides["mammaprint_index"] = slides["mammaprint_index"].astype(float) + case LabelMode.BOTH: + slides = slides.copy() + slides["type_label"] = slides["type"].map(_map_luminal_type) + slides["mammaprint_index"] = slides["mammaprint_index"].astype(float) slides["name"] = slides["path"].apply(lambda x: Path(x).stem) return slides def get_label(slide_metadata: pd.Series, mode: LabelMode) -> torch.Tensor: + """One slide's target as a float tensor. + + The single-task modes are single-output (``out_dim=1``): binary classification + (luminal a=1 / b=0, for ``BCEWithLogitsLoss`` + binary metrics) and regression + (the MammaPrint index). Shape ``(1,)`` matches the head's per-slide output so + batched preds/targets line up as ``(B, 1)``. + + ``BOTH`` is the multi-task target: ``[type_label, mammaprint_index]`` as a + ``(2,)`` tensor, column 0 the class and column 1 the index. Pair it with a + 2-output head and :class:`~ml.models.losses.joint.JointLoss`, which slices the + columns back apart. + """ match mode: case LabelMode.TYPE: - return torch.tensor(int(slide_metadata["type_label"])).long() + values = [float(slide_metadata["type_label"])] case LabelMode.INDEX: - return torch.tensor(float(slide_metadata["index"])).float() + values = [float(slide_metadata["mammaprint_index"])] + case LabelMode.BOTH: + values = [ + float(slide_metadata["type_label"]), + float(slide_metadata["mammaprint_index"]), + ] + case _: + raise ValueError(f"Unsupported label mode: {mode}") + + return torch.tensor(values, dtype=torch.float32) - raise ValueError(f"Unsupported label mode: {mode}") +def get_target_columns(mode: LabelMode) -> list[str]: + """Label column(s) that must be present for a slide to be usable in ``mode``. -def get_target_column(mode: LabelMode) -> str: + ``BOTH`` requires both columns, so a slide missing either is dropped. + """ match mode: case LabelMode.TYPE: - return "type_label" + return ["type_label"] case LabelMode.INDEX: - return "index" + return ["mammaprint_index"] + case LabelMode.BOTH: + return ["type_label", "mammaprint_index"] raise ValueError(f"Unsupported label mode: {mode}") diff --git a/ml/data/datasets/raw_tiles.py b/ml/data/datasets/raw_tiles.py new file mode 100644 index 00000000..de12ea57 --- /dev/null +++ b/ml/data/datasets/raw_tiles.py @@ -0,0 +1,122 @@ +"""Raw-tile slide dataset for the end-to-end (image backbone) path. + +Each item is one slide's bag of raw tile **images** ``(N, C, H, W)`` — fed to an +image encoder (e.g. ``VGG16Encoder``) that maps each tile to a feature vector, +then to an aggregator + head. Reads pixels via +:class:`ml.data.datasets.SlideDataset` (OpenSlide) from a *tiled* artifact +(``slides.parquet`` + ``tiles.parquet``), so ``rationai.mlkit``/OpenSlide is +imported lazily and only touched on this path. + +Single-scale only (one level card, keyed by ``raw_uris[split]``). Slides can have +thousands of tiles, so ``max_tiles`` caps the bag (random-sampled, seeded) to bound +GPU memory — standard MIL practice. +""" + +import logging +from collections.abc import Mapping +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +import torch +from torch.utils.data import Dataset + +from ml.data.datasets._sources import ( + download_artifacts_with_retries, + load_labeled_slides, + split_uri, +) +from ml.data.datasets.labels import LabelMode, get_label +from ml.typing import LevelSpec, MILSample, SlideMetadata + + +if TYPE_CHECKING: + from ml.data.datasets.tiles import TileDataset + + +logger = logging.getLogger(__name__) + +# ImageNet normalisation, matching preprocessing/embeddings.py. +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +class RawTileSlideDataset(Dataset[MILSample]): + """Per-slide bags of raw tile images from a tiled artifact. + + Args: + split: Which split to load (``train``/``val``/``test``); selects the level + card's ``raw_uris[split]`` (a pure, pre-split artifact). + levels: A one-entry level map (single-scale). The entry's ``raw_uris`` maps + ``split -> tiled artifact URI``. + data_mapping: Label CSV path (type/index labels, joined by slide stem). + label_mode: ``"type"`` (classification) or ``"index"`` (regression). + max_tiles: Cap on tiles per slide; if a slide has more, a seeded random + subset is used. ``None`` keeps all tiles. + seed: Seed for the per-slide tile subsampling. + """ + + def __init__( + self, + split: str, + levels: Mapping[str | int, LevelSpec], + data_mapping: str | Path, + label_mode: str = "type", + max_tiles: int | None = 256, + seed: int = 0, + ) -> None: + if len(levels) != 1: + raise ValueError( + "RawTileSlideDataset is single-scale: pass exactly one level." + ) + + from ml.data.datasets import SlideDataset # lazy: pulls OpenSlide + + self.label_mode = LabelMode(label_mode) + self.max_tiles = max_tiles + self.rng = np.random.default_rng(seed) + + (card,) = levels.values() + uri = split_uri(dict(card), "raw_uris", split, 0) + local_dir = download_artifacts_with_retries(uri) + + transform = _normalize_transform() + slide_dataset = SlideDataset(paths=[local_dir], transforms=transform) + # One TileDataset per slide, keyed by slide stem (matches data_mapping.name). + # generate_datasets() yields TileDataset instances (each has .slide_path). + per_slide = cast("list[TileDataset]", list(slide_dataset.generate_datasets())) + self.tile_datasets: dict[str, TileDataset] = { + Path(td.slide_path).stem: td for td in per_slide + } + + slides = load_labeled_slides(data_mapping, self.label_mode) + self.slides = slides[slides["name"].isin(self.tile_datasets)].reset_index( + drop=True + ) + + def __len__(self) -> int: + return len(self.slides) + + def __getitem__(self, idx: int) -> MILSample: + name = self.slides.iloc[idx]["name"] + tiles = self.tile_datasets[name] + + n = len(tiles) + indices: list[int] = list(range(n)) + if self.max_tiles is not None and n > self.max_tiles: + indices = self.rng.choice(n, self.max_tiles, replace=False).tolist() + logger.debug("Slide %s: sampled %d/%d tiles.", name, self.max_tiles, n) + + images = torch.stack([tiles[i][0] for i in indices]) # (N, C, H, W) + label = get_label(self.slides.iloc[idx], self.label_mode) + metadata: SlideMetadata = {"slide_id": name} + return images, label, metadata + + +def _normalize_transform() -> Any: + import albumentations as A + + return A.Compose([A.Normalize(mean=_IMAGENET_MEAN, std=_IMAGENET_STD)]) + + +__all__ = ["RawTileSlideDataset"] diff --git a/ml/data/datasets/single_scale.py b/ml/data/datasets/single_scale.py new file mode 100644 index 00000000..6183312b --- /dev/null +++ b/ml/data/datasets/single_scale.py @@ -0,0 +1,104 @@ +"""Single-scale slide dataset over precomputed tile embeddings. + +Each item is one slide's flat bag of tile-feature vectors ``(N, D)`` — consumed +directly by a flat aggregator (mean/max/attention) + head. One pyramid level only. + +Self-contained by design: this module imports **no** multi-scale primitives +(``align_regions``/``Region``/``MultiScaleBag``), so the single-level path keeps +working even if every multilevel file is removed or commented out. +""" + +import logging +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +import torch +from torch.utils.data import Dataset + +from ml.data.datasets._sources import ( + available_slides, + download_level_sources, + load_labeled_slides, + split_uri, +) +from ml.data.datasets.labels import LabelMode, get_label +from ml.typing import LevelSpec, MILSample, SlideMetadata + + +logger = logging.getLogger(__name__) + + +def _empty_embedding_slides(directory: Path, slide_ids: set[str]) -> tuple[str, ...]: + """Return slide IDs whose Parquet files contain no tile rows.""" + return tuple( + sorted( + slide_id + for slide_id in slide_ids + if pq.read_metadata(directory / f"{slide_id}.parquet").num_rows == 0 + ) + ) + + +class SingleScaleDataset(Dataset[MILSample]): + """Per-slide flat bags of precomputed tile embeddings (one level). + + Args: + split: Which split to load (``train``/``val``/``test``); selects the level + card's ``uris[split]`` (a pure, pre-split artifact). + levels: A one-entry level map. The entry's ``uris`` maps ``split -> URI``. + data_mapping: Label CSV path (type/index labels, joined by slide stem). + label_mode: ``"type"`` (classification) or ``"index"`` (regression). + """ + + def __init__( + self, + split: str, + levels: Mapping[str | int, LevelSpec], + data_mapping: str | Path, + label_mode: str = "type", + ) -> None: + if len(levels) != 1: + raise ValueError( + "SingleScaleDataset is single-scale: pass exactly one level." + ) + + self.label_mode = LabelMode(label_mode) + (level, card) = next(iter(levels.items())) + self.level = int(level) + + uri = split_uri(dict(card), "uris", split, self.level) + self.embeddings_dir = download_level_sources({self.level: uri})[self.level] + + slides = load_labeled_slides(data_mapping, self.label_mode) + present = available_slides({self.level: self.embeddings_dir}) + self.empty_slides = _empty_embedding_slides(self.embeddings_dir, present) + if self.empty_slides: + logger.warning( + "Excluding %d slide(s) with empty embedding bags from the %s " + "split at level %d: %s", + len(self.empty_slides), + split, + self.level, + ", ".join(self.empty_slides), + ) + present.difference_update(self.empty_slides) + self.slides = slides[slides["name"].isin(present)].reset_index(drop=True) + + def __len__(self) -> int: + return len(self.slides) + + def __getitem__(self, idx: int) -> MILSample: + row = self.slides.iloc[idx] + frame = pd.read_parquet( + (self.embeddings_dir / row["name"]).with_suffix(".parquet") + ) + bag = torch.from_numpy(np.stack(frame["embedding"].to_numpy())).float() + label = get_label(row, self.label_mode) + metadata: SlideMetadata = {"slide_id": row["name"]} + return bag, label, metadata + + +__all__ = ["SingleScaleDataset"] diff --git a/ml/data/datasets/spatial_scale.py b/ml/data/datasets/spatial_scale.py new file mode 100644 index 00000000..08f0890a --- /dev/null +++ b/ml/data/datasets/spatial_scale.py @@ -0,0 +1,54 @@ +"""Coordinate-aware single-scale slide dataset. + +Identical to :class:`~ml.data.datasets.single_scale.SingleScaleDataset` except each +bag is widened from ``(N, D)`` to ``(N, D + 2)`` by appending the tile's level-0 +pixel ``(x, y)`` coordinates (the top-left corner) as the last two columns. Those +columns are already present in every embedding parquet (written by +``preprocessing/embeddings.py`` alongside the ``embedding`` column); this dataset +simply stops discarding them. + +The wide bag is consumed only by +:class:`~ml.models.aggregators.spatial_transformer.SpatialTransformerMIL`, which +slices the coordinate columns back off inside its ``forward``. Pair the two together +(coord-aware dataset + spatial aggregator); no other aggregator understands the extra +columns, and no other dataset produces them, so the flat single-scale path is +unaffected. +""" + +import numpy as np +import pandas as pd +import torch + +from ml.data.datasets.labels import get_label +from ml.data.datasets.single_scale import SingleScaleDataset +from ml.typing import MILSample, SlideMetadata + + +class SpatialScaleDataset(SingleScaleDataset): + """Per-slide bags of embeddings with appended ``(x, y)`` tile coordinates. + + Returns a standard :data:`~ml.typing.MILSample`; the only difference from the + parent is the bag width — ``(N, D + 2)`` instead of ``(N, D)``, with the last two + columns the level-0 pixel coordinates. Construction (split resolution, artifact + download, slide filtering, label mode) is inherited unchanged. + """ + + def __getitem__(self, idx: int) -> MILSample: + row = self.slides.iloc[idx] + frame = pd.read_parquet( + (self.embeddings_dir / row["name"]).with_suffix(".parquet") + ) + features = torch.from_numpy( + np.stack(frame["embedding"].to_numpy()) + ).float() # (N, D) + coords = torch.from_numpy( + frame[["x", "y"]].to_numpy() + ).float() # (N, 2), level-0 pixel top-left corners + bag = torch.cat([features, coords], dim=1) # (N, D + 2) + + label = get_label(row, self.label_mode) + metadata: SlideMetadata = {"slide_id": row["name"]} + return bag, label, metadata + + +__all__ = ["SpatialScaleDataset"] diff --git a/ml/metrics/__init__.py b/ml/metrics/__init__.py new file mode 100644 index 00000000..6e8e1c72 --- /dev/null +++ b/ml/metrics/__init__.py @@ -0,0 +1,25 @@ +"""Custom metrics for MammaPrint.""" + +from ml.metrics.column import ColumnMetric +from ml.metrics.logits import BinaryLogitMetric +from ml.metrics.summary import binary_logit_summary, regression_summary +from ml.metrics.thresholded import ( + ThresholdedBinaryMetric, + thresholded_f1, + thresholded_precision, + thresholded_recall, + thresholded_specificity, +) + + +__all__ = [ + "BinaryLogitMetric", + "ColumnMetric", + "ThresholdedBinaryMetric", + "binary_logit_summary", + "regression_summary", + "thresholded_f1", + "thresholded_precision", + "thresholded_recall", + "thresholded_specificity", +] diff --git a/ml/metrics/column.py b/ml/metrics/column.py new file mode 100644 index 00000000..1ebe5837 --- /dev/null +++ b/ml/metrics/column.py @@ -0,0 +1,34 @@ +"""Select one column of a multi-task prediction/target before metricking. + +In the ``both`` label mode the head emits ``(B, 2)`` and the target is ``(B, 2)`` +— column 0 is the classification channel (logit / type label), column 1 the +regression channel (scalar / MammaPrint index). Standard torchmetrics expect a +single channel, so this wrapper slices out one column and delegates, letting the +existing binary and regression metrics be reused unchanged for the joint model. +""" + +from torch import Tensor +from torchmetrics import Metric +from torchmetrics.wrappers import MetricInputTransformer + + +class ColumnMetric(MetricInputTransformer): + """Wraps a metric, feeding it one column of ``(B, 2)`` preds/targets. + + Args: + metric: The metric to delegate to (already instantiated). + column: Which column to select — ``0`` classification, ``1`` regression. + """ + + def __init__(self, metric: Metric, column: int) -> None: + super().__init__(metric) + self.column = column + + def transform_pred(self, pred: Tensor) -> Tensor: + return pred[:, self.column] + + def transform_target(self, target: Tensor) -> Tensor: + return target[:, self.column] + + +__all__ = ["ColumnMetric"] diff --git a/ml/metrics/logits.py b/ml/metrics/logits.py new file mode 100644 index 00000000..a8bae154 --- /dev/null +++ b/ml/metrics/logits.py @@ -0,0 +1,32 @@ +"""Binary metrics with explicit, batch-independent logit handling.""" + +import torch +from torch import Tensor +from torchmetrics import Metric +from torchmetrics.wrappers import MetricInputTransformer + + +class BinaryLogitMetric(MetricInputTransformer): + """Give a binary metric explicit, batch-independent logit semantics. + + TorchMetrics otherwise guesses whether a floating tensor contains logits or + probabilities independently for every update. With the slide dataloader's + batch size of one, that makes the same logit mean different things depending + on whether it falls inside ``[0, 1]``. This wrapper makes the contract explicit: + model predictions are logits and targets are already binary ``{0, 1}`` labels. + Ranking metrics receive sigmoid probabilities. Confusion-matrix metrics set a + ``decision_threshold`` and receive hard labels, using ``>=`` to match the + prediction CSV exactly. + """ + + def __init__(self, metric: Metric, decision_threshold: float | None = None) -> None: + super().__init__(metric) + self.decision_threshold = decision_threshold + + def transform_pred(self, pred: Tensor) -> Tensor: + if self.decision_threshold is not None: + return (pred >= self.decision_threshold).long() + return torch.sigmoid(pred) + + +__all__ = ["BinaryLogitMetric"] diff --git a/ml/metrics/summary.py b/ml/metrics/summary.py new file mode 100644 index 00000000..8a6c598b --- /dev/null +++ b/ml/metrics/summary.py @@ -0,0 +1,92 @@ +"""Auditable cohort metrics computed from lossless prediction tables.""" + +import math + +import torch +from torch import Tensor +from torchmetrics.classification import BinaryAUROC +from torchmetrics.regression import ( + MeanAbsoluteError, + MeanSquaredError, +) + + +def _divide(numerator: float, denominator: float) -> float: + return numerator / denominator if denominator else math.nan + + +def binary_logit_summary(logits: Tensor, targets: Tensor) -> dict[str, float]: + """Return binary metrics and confusion counts using logit threshold zero.""" + logits = logits.detach().float().flatten().cpu() + targets = targets.detach().long().flatten().cpu() + if logits.shape != targets.shape or logits.numel() == 0: + raise ValueError("Binary logits and targets must be non-empty equal shapes.") + if not torch.all(torch.isfinite(logits)): + raise ValueError("Binary logits must be finite.") + if not torch.all((targets == 0) | (targets == 1)): + raise ValueError("Binary targets must contain only 0 and 1.") + + predictions = (logits >= 0).long() + tp = int(((predictions == 1) & (targets == 1)).sum()) + tn = int(((predictions == 0) & (targets == 0)).sum()) + fp = int(((predictions == 1) & (targets == 0)).sum()) + fn = int(((predictions == 0) & (targets == 1)).sum()) + total = len(targets) + accuracy = (tp + tn) / total + precision = _divide(tp, tp + fp) + recall = _divide(tp, tp + fn) + specificity = _divide(tn, tn + fp) + f1 = _divide(2 * tp, 2 * tp + fp + fn) + + auroc = math.nan + if targets.unique().numel() == 2: + # Sigmoid is monotonic, but makes the input contract explicit and avoids + # TorchMetrics' per-update logit/probability auto-detection. + auroc = float(BinaryAUROC()(torch.sigmoid(logits), targets)) + + return { + "slides": float(total), + "tn": float(tn), + "fp": float(fp), + "fn": float(fn), + "tp": float(tp), + "accuracy": accuracy, + "precision": precision, + "recall": recall, + "specificity": specificity, + "f1": f1, + "auroc": auroc, + } + + +def regression_summary(predictions: Tensor, targets: Tensor) -> dict[str, float]: + """Return continuous and zero-threshold classification metrics.""" + predictions = predictions.detach().float().flatten().cpu() + targets = targets.detach().float().flatten().cpu() + if predictions.shape != targets.shape or predictions.numel() == 0: + raise ValueError( + "Regression predictions and targets must be non-empty equal shapes." + ) + if not torch.all(torch.isfinite(predictions)) or not torch.all( + torch.isfinite(targets) + ): + raise ValueError("Regression predictions and targets must be finite.") + + result = { + "mae": float(MeanAbsoluteError()(predictions, targets)), + "mse": float(MeanSquaredError()(predictions, targets)), + } + + thresholded = binary_logit_summary(predictions, (targets >= 0).long()) + result.update( + { + f"thresholded_{name}": value + for name, value in thresholded.items() + if name != "slides" + } + ) + result["slides"] = float(len(targets)) + return result + + +__all__ = ["binary_logit_summary", "regression_summary"] diff --git a/ml/metrics/thresholded.py b/ml/metrics/thresholded.py new file mode 100644 index 00000000..679fcbc5 --- /dev/null +++ b/ml/metrics/thresholded.py @@ -0,0 +1,88 @@ +"""Classification metrics computed from a regression output by thresholding. + +For the MammaPrint index, the sign is the class boundary: index ``>= 0`` is +"a luminal" (class 1), ``< 0`` is "b luminal" (class 0) — matching +``_map_luminal_type`` and the notebook's LUMINAL_INTERVAL rule. This lets a +*regression* model report classification metrics (F1 / precision / recall / +sensitivity / specificity) during training: binarize both prediction and target at +the threshold, then delegate to a standard binary metric. +""" + +from torch import Tensor +from torchmetrics import Metric +from torchmetrics.classification import ( + BinaryF1Score, + BinaryPrecision, + BinaryRecall, + BinarySpecificity, +) +from torchmetrics.wrappers import MetricInputTransformer + + +class ThresholdedBinaryMetric(MetricInputTransformer): + """Wraps a binary metric, binarizing continuous preds/targets at a threshold. + + Both ``preds`` and ``target`` are mapped to ``{0, 1}`` via ``value >= threshold`` + (positive class = "a luminal", index >= 0), then forwarded to the wrapped metric. + + Args: + metric: The binary metric to delegate to (already instantiated). + threshold: Decision boundary applied to predictions (default 0.0). + target_threshold: Decision boundary applied to continuous targets. Defaults + to ``threshold`` for regression-derived classification. Set this to + ``0.5`` when targets are already binary labels and predictions are + logits thresholded at zero. + """ + + def __init__( + self, + metric: Metric, + threshold: float = 0.0, + target_threshold: float | None = None, + ) -> None: + super().__init__(metric) + self.threshold = threshold + self.target_threshold = ( + threshold if target_threshold is None else target_threshold + ) + + def transform_pred(self, pred: Tensor) -> Tensor: + return (pred >= self.threshold).long() + + def transform_target(self, target: Tensor) -> Tensor: + return (target >= self.target_threshold).long() + + +def thresholded_f1( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryF1Score(), threshold, target_threshold) + + +def thresholded_precision( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryPrecision(), threshold, target_threshold) + + +def thresholded_recall( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + """Recall (a.k.a. sensitivity) == TP / (TP + FN).""" + return ThresholdedBinaryMetric(BinaryRecall(), threshold, target_threshold) + + +def thresholded_specificity( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + """Specificity == TN / (TN + FP).""" + return ThresholdedBinaryMetric(BinarySpecificity(), threshold, target_threshold) + + +__all__ = [ + "ThresholdedBinaryMetric", + "thresholded_f1", + "thresholded_precision", + "thresholded_recall", + "thresholded_specificity", +] diff --git a/ml/models/__init__.py b/ml/models/__init__.py new file mode 100644 index 00000000..867124ac --- /dev/null +++ b/ml/models/__init__.py @@ -0,0 +1,9 @@ +"""MammaPrint training models: generic module + swappable pieces.""" + +from ml.models.aggregators import Aggregator +from ml.models.encoders import Encoder +from ml.models.heads import Head +from ml.models.module import MammaprintModule + + +__all__ = ["Aggregator", "Encoder", "Head", "MammaprintModule"] diff --git a/ml/models/aggregators/__init__.py b/ml/models/aggregators/__init__.py new file mode 100644 index 00000000..dc0ac140 --- /dev/null +++ b/ml/models/aggregators/__init__.py @@ -0,0 +1,25 @@ +"""Bag aggregators (MIL pooling): mean, max, gated attention, transformer.""" + +# ── Single-level aggregators (flat Bag) ────────────────────────────────────── +from ml.models.aggregators.attention import AttentionMIL +from ml.models.aggregators.base import Aggregator +from ml.models.aggregators.max import MaxPool +from ml.models.aggregators.mean import MeanPool +from ml.models.aggregators.spatial_transformer import SpatialTransformerMIL +from ml.models.aggregators.transformer import TransformerMIL + + +# ── Multilevel (comment out to ship single-level only) ─────────────────────── +# from ml.models.aggregators.multiscale import MultiScaleMIL + + +__all__ = [ + "Aggregator", + "AttentionMIL", + "MaxPool", + "MeanPool", + "SpatialTransformerMIL", + "TransformerMIL", + # ── Multilevel ── + # "MultiScaleMIL", +] diff --git a/ml/models/aggregators/attention.py b/ml/models/aggregators/attention.py new file mode 100644 index 00000000..ad931756 --- /dev/null +++ b/ml/models/aggregators/attention.py @@ -0,0 +1,53 @@ +"""Gated-attention MIL aggregator. + +Inspired by "Attention-based Deep Multiple Instance Learning" (Ilse et al., 2018, +arXiv:1802.04712) and adapted from the RationAI ``feature/mil`` reference +implementation to the encoder -> aggregator -> head contract. +""" + +import torch +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class AttentionMIL(Aggregator[Bag]): + """Gated-attention pooling over a bag of tile features. + + Learns a scalar attention weight per tile via the gated mechanism + ``tanh(V h) * sigmoid(U h)``, softmax-normalises the weights across the bag, + and returns the attention-weighted sum of tile features. The per-tile weights + are returned for interpretability (heatmaps). + + Args: + feature_dim: Dimensionality of the incoming tile features. + attention_dim: Width of the attention hidden layer. + """ + + def __init__(self, feature_dim: int, attention_dim: int = 512) -> None: + super().__init__() + self.feature_dim = feature_dim + self.norm = nn.LayerNorm(feature_dim) + self.attention_v = nn.Linear(feature_dim, attention_dim) + self.attention_u = nn.Linear(feature_dim, attention_dim) + self.attention_weights = nn.Linear(attention_dim, 1) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + bag = self.norm(bag) # (N, D) + + gate = torch.tanh(self.attention_v(bag)) * torch.sigmoid( + self.attention_u(bag) + ) # (N, attention_dim) + scores = self.attention_weights(gate).squeeze(-1) # (N,) + attention = torch.softmax(scores, dim=0) # (N,) + + pooled = (attention.unsqueeze(-1) * bag).sum(dim=0) # (D,) + return pooled, attention + + +__all__ = ["AttentionMIL"] diff --git a/ml/models/aggregators/base.py b/ml/models/aggregators/base.py new file mode 100644 index 00000000..a940fd54 --- /dev/null +++ b/ml/models/aggregators/base.py @@ -0,0 +1,55 @@ +"""Aggregator abstract base class. + +An aggregator reduces one slide's tiles to a single slide-level vector ``(D',)``. +This is the multiple-instance-learning (MIL) pooling step; swapping the aggregator +(mean, max, gated attention, transformer, multi-scale) is the primary experimental +axis for the training skeleton. + +The input is either a **flat bag** ``Tensor[N, D]`` (single level) or a +**multi-scale bag** ``list[Region]`` where each region maps ``level -> Tensor[K, D]`` +(tiles across pyramid levels aligned by footprint). Flat aggregators +(mean/max/attention) accept the tensor; :class:`~ml.models.aggregators.multiscale.MultiScaleMIL` +accepts the region list. The output contract is identical, so the module and head +are agnostic to which one is in use. + +Concrete aggregators live in sibling modules, one implementation per file, and +are selected via the ``ml/aggregator`` Hydra config group. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + + +class Aggregator[InputT](nn.Module, ABC): + """Reduces one slide's tiles to a single slide vector. + + Generic (unbounded) over its input type so this base stays single-level-safe + (it references no multilevel type): flat aggregators specialise as + ``Aggregator[Bag]`` (a tile tensor ``(N, D)``), and the optional multi-scale + aggregator as ``Aggregator[MultiScaleBag]`` (a list of aligned regions). + Implementations return the pooled vector and, optionally, weights (``None`` for + parameter-free poolings such as mean/max), surfaced for interpretability. + """ + + @property + @abstractmethod + def out_dim(self) -> int: + """Dimensionality of the aggregated slide vector.""" + + @abstractmethod + def forward(self, bag: InputT) -> tuple[Tensor, Tensor | None]: + """Aggregate one slide's tiles. + + Args: + bag: A flat per-tile feature tensor ``(N, D)`` (``Aggregator[Bag]``) or + a multi-scale bag of regions (``Aggregator[MultiScaleBag]``). + + Returns: + A tuple ``(pooled, attention)`` where ``pooled`` has shape + ``(out_dim,)`` and ``attention`` has shape ``(N,)`` / ``(num_regions,)`` + or is ``None`` if the aggregator assigns no weights. + """ + + +__all__ = ["Aggregator"] diff --git a/ml/models/aggregators/max.py b/ml/models/aggregators/max.py new file mode 100644 index 00000000..f81399f7 --- /dev/null +++ b/ml/models/aggregators/max.py @@ -0,0 +1,30 @@ +"""Max-pooling aggregator.""" + +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class MaxPool(Aggregator[Bag]): + """Takes the feature-wise maximum across the bag. + + Parameter-free; assigns no per-tile weights. + + Args: + feature_dim: Dimensionality of the incoming tile features. + """ + + def __init__(self, feature_dim: int) -> None: + super().__init__() + self.feature_dim = feature_dim + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, None]: + return bag.amax(dim=0), None + + +__all__ = ["MaxPool"] diff --git a/ml/models/aggregators/mean.py b/ml/models/aggregators/mean.py new file mode 100644 index 00000000..ba339f58 --- /dev/null +++ b/ml/models/aggregators/mean.py @@ -0,0 +1,30 @@ +"""Mean-pooling aggregator.""" + +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class MeanPool(Aggregator[Bag]): + """Averages tile features across the bag. + + Parameter-free; assigns no per-tile weights. + + Args: + feature_dim: Dimensionality of the incoming tile features. + """ + + def __init__(self, feature_dim: int) -> None: + super().__init__() + self.feature_dim = feature_dim + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, None]: + return bag.mean(dim=0), None + + +__all__ = ["MeanPool"] diff --git a/ml/models/aggregators/spatial_transformer.py b/ml/models/aggregators/spatial_transformer.py new file mode 100644 index 00000000..09f7a0c9 --- /dev/null +++ b/ml/models/aggregators/spatial_transformer.py @@ -0,0 +1,166 @@ +"""Spatially-aware Transformer MIL aggregator (TransMIL + 2D positional encoding). + +The plain :class:`~ml.models.aggregators.transformer.TransformerMIL` is +permutation-invariant: it treats a slide's tiles as an unordered *set* and has no +notion of where each tile sits on the slide. This variant injects that spatial +signal by adding a **2D sinusoidal positional encoding**, computed from each tile's +physical ``(x, y)`` coordinate, to the tile features before the transformer runs. +Everything downstream (CLS-token self-attention, slide-vector readout, per-tile +attention map) is identical — this module *composes* ``TransformerMIL`` rather than +reimplementing it, so the readout has a single source of truth. + +Why sinusoidal (and not TransMIL's own PPEG / a learned coordinate table): + +* WSI tiles are a **sparse, non-rectangular** subset of the slide (tissue/mask + regions only), variable in count. PPEG-style schemes reshape the token sequence + into a dense ``sqrt(N) x sqrt(N)`` grid and convolve — inventing adjacency across real + tissue gaps and treats *sequence order* as position. That is wrong here. A + sinusoidal PE is a pure per-tile function of that tile's own coordinate, so gaps + and irregular shapes cost nothing and no ordering is assumed. +* It adds **zero learned parameters**, the safest choice against overfitting on the + small (per-slide-labelled) cohorts typical of pathology. + +Coordinates arrive as the **last two columns of the bag** (``(N, feature_dim + 2)``); +the coord-aware dataset (:class:`~ml.data.datasets.spatial_scale.SpatialScaleDataset`) +packs them there. Only this aggregator, paired with that dataset, ever sees the wide +bag — the flat aggregators keep receiving a plain ``(N, feature_dim)`` tensor. + +Positional context for MIL over strong frozen foundation-model features (Virchow2) +is a genuinely marginal signal for bag-of-tiles tasks (subtype / continuous-index), +so treat this as an ablation to compare against the permutation-invariant baselines, +not a default. See the module docstring rationale above for the correctness traps it +deliberately avoids. +""" + +import torch +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.models.aggregators.transformer import TransformerMIL +from ml.typing import Bag + + +class SpatialTransformerMIL(Aggregator[Bag]): + """TransMIL self-attention with an added 2D sinusoidal positional encoding. + + The incoming bag is ``(N, feature_dim + 2)``: the first ``feature_dim`` columns + are tile features, the last two are the tile's level-0 pixel ``(x, y)`` (top-left + corner). Coordinates are converted to per-slide-normalised grid coordinates, a + sinusoidal PE is built from them and added to the features, and the result is fed + to an internal :class:`~ml.models.aggregators.transformer.TransformerMIL`. + + Args: + feature_dim: Dimensionality of the tile **features** ``D`` (not counting the + 2 coordinate columns). Must be divisible by ``num_heads`` and by 4 (the + PE splits ``D`` into two axis-halves, each a sin/cos pair stack). + tile_extent: Tile edge length in tiling pixels (from the level card). With + ``mpp`` this gives the level-0 footprint ``extent = tile_extent * mpp`` + used to turn pixel coordinates into grid indices. + mpp: Microns-per-pixel of the tiling level (from the level card). + num_heads: Attention heads (must divide ``feature_dim``). + num_layers: Transformer encoder layers. + mlp_ratio: Feed-forward width as a multiple of ``feature_dim``. + dropout: Dropout inside the transformer layers. + pe_scale: Multiplier on the PE before it is added to the features. Kept small + (default 0.1) so the encoding does not dominate the feature norm. + pe_base: Sinusoidal frequency base ``θ`` (as in Vaswani et al.). + pe_span: Span constant mapping normalised ``[0, 1]`` coordinates into a + usable phase range (larger = finer positional resolution). + """ + + def __init__( + self, + feature_dim: int, + tile_extent: int, + mpp: float, + num_heads: int = 8, + num_layers: int = 2, + mlp_ratio: float = 4.0, + dropout: float = 0.1, + pe_scale: float = 0.1, + pe_base: float = 10000.0, + pe_span: float = 64.0, + ) -> None: + super().__init__() + if feature_dim % 4 != 0: + raise ValueError( + f"feature_dim ({feature_dim}) must be divisible by 4 for the 2D " + "sin/cos positional encoding (two axis-halves of sin/cos pairs)." + ) + # feature_dim % num_heads is validated inside the composed TransformerMIL. + self.feature_dim = feature_dim + self.extent = tile_extent * mpp + if self.extent <= 0: + raise ValueError( + f"tile_extent * mpp must be positive; got {tile_extent} * {mpp}." + ) + self.pe_scale = pe_scale + self.pe_base = pe_base + self.pe_span = pe_span + + # Compose (don't copy) the plain transformer: it owns the CLS token, the + # encoder stack and the attention readout, so the interpretability map and + # slide-vector logic stay identical to the non-spatial aggregator. + self.transformer = TransformerMIL( + feature_dim=feature_dim, + num_heads=num_heads, + num_layers=num_layers, + mlp_ratio=mlp_ratio, + dropout=dropout, + ) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def _positional_encoding(self, coords: Tensor) -> Tensor: + """Build a 2D sinusoidal PE ``(N, D)`` from level-0 pixel coords ``(N, 2)``. + + Pixels -> grid indices (``/ extent``) -> per-slide bbox min-max to ``[0, 1]`` + (translation-invariant, so absolute tissue position on the glass — a scanner + / sectioning artefact — cannot leak into the encoding). Half the channels + encode the x axis, half the y axis; within each half, sin/cos frequency pairs. + """ + grid = coords / self.extent # (N, 2), float grid indices (kept continuous) + + gmin = grid.min(dim=0).values # (2,) + gmax = grid.max(dim=0).values # (2,) + # clamp_min in *grid units* (1.0 = one tile): a single tile or a bag whose + # tiles share an x- or y-line has zero span on that axis -> normalise to 0 + # (a constant coordinate), which yields a constant PE and cannot divide by 0. + span = (gmax - gmin).clamp_min(1.0) + norm = (grid - gmin) / span # (N, 2) in [0, 1] + + half = self.feature_dim // 2 # channels per axis + num_freqs = half // 2 # sin/cos pairs per axis + freq_idx = torch.arange(num_freqs, device=coords.device, dtype=torch.float32) + # ω_k = 1 / θ^(2k / half); phase = normalised_coord * span * ω_k + inv_freq = 1.0 / (self.pe_base ** (2.0 * freq_idx / half)) # (num_freqs,) + + pe = coords.new_zeros(coords.shape[0], self.feature_dim) # (N, D) + for axis in (0, 1): + phase = norm[:, axis : axis + 1] * self.pe_span * inv_freq # (N, num_freqs) + base = axis * half + pe[:, base : base + half : 2] = torch.sin(phase) + pe[:, base + 1 : base + half : 2] = torch.cos(phase) + return pe + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + # bag: (N, feature_dim + 2) — features then the (x, y) coordinate columns. + expected = self.feature_dim + 2 + if bag.shape[-1] != expected: + raise ValueError( + f"SpatialTransformerMIL expects a bag of width feature_dim + 2 " + f"({expected}); got {bag.shape[-1]}. Is it paired with " + "SpatialScaleDataset (which appends the x/y columns)?" + ) + features = bag[:, : self.feature_dim] # (N, D) + coords = bag[:, self.feature_dim :] # (N, 2) + + pe = self._positional_encoding(coords) # (N, D) + # The CLS token is added *inside* the composed transformer, after this — so it + # receives no positional term (it has no coordinate), as intended. + return self.transformer(features + self.pe_scale * pe) + + +__all__ = ["SpatialTransformerMIL"] diff --git a/ml/models/aggregators/transformer.py b/ml/models/aggregators/transformer.py new file mode 100644 index 00000000..f542c191 --- /dev/null +++ b/ml/models/aggregators/transformer.py @@ -0,0 +1,117 @@ +"""Transformer-based MIL aggregator (TransMIL-style). + +Treats a slide's tiles as a sequence and lets them attend to each other via +multi-head self-attention — unlike mean/gated-attention pooling, which score each +tile independently. A learnable CLS token is prepended; after ``num_layers`` of +self-attention the CLS embedding summarises the whole bag and becomes the slide +vector. The CLS->tiles attention from the final layer is returned as the per-tile +weight (interpretability / heatmaps). + +Reference: Shao et al., "TransMIL: Transformer based Correlated Multiple Instance +Learning for Whole Slide Image Classification" (NeurIPS 2021, arXiv:2106.00908). +""" + +import torch +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class TransformerMIL(Aggregator[Bag]): + """Self-attention over the tile bag with a learned CLS-token readout. + + Args: + feature_dim: Dimensionality of the incoming tile features ``D``. + num_heads: Number of attention heads (must divide ``feature_dim``). + num_layers: Number of transformer encoder layers. + mlp_ratio: Feed-forward hidden width as a multiple of ``feature_dim``. + dropout: Dropout inside the transformer layers. + """ + + def __init__( + self, + feature_dim: int, + num_heads: int = 8, + num_layers: int = 2, + mlp_ratio: float = 4.0, + dropout: float = 0.1, + ) -> None: + super().__init__() + if feature_dim % num_heads != 0: + raise ValueError( + f"feature_dim ({feature_dim}) must be divisible by num_heads " + f"({num_heads})." + ) + if num_layers < 1: + raise ValueError("num_layers must be at least 1.") + self.feature_dim = feature_dim + self.num_heads = num_heads + + # Learnable CLS token prepended to the tile sequence; its final embedding + # is the slide vector. + self.cls_token = nn.Parameter(torch.zeros(1, feature_dim)) + nn.init.trunc_normal_(self.cls_token, std=0.02) + + encoder_layer = nn.TransformerEncoderLayer( + d_model=feature_dim, + nhead=num_heads, + dim_feedforward=int(feature_dim * mlp_ratio), + dropout=dropout, + batch_first=True, + norm_first=True, # pre-norm: more stable for small/deep stacks + ) + # All but the last layer run as a standard stack; the last layer is applied + # manually so we can pull out CLS->tiles attention weights for the readout. + self.blocks = ( + nn.TransformerEncoder( + encoder_layer, num_layers - 1, enable_nested_tensor=False + ) + if num_layers > 1 + else None + ) + self.final_norm1 = nn.LayerNorm(feature_dim) + self.final_attn = nn.MultiheadAttention( + feature_dim, num_heads, dropout=dropout, batch_first=True + ) + self.final_attn_dropout = nn.Dropout(dropout) + self.final_norm2 = nn.LayerNorm(feature_dim) + self.final_ff = nn.Sequential( + nn.Linear(feature_dim, int(feature_dim * mlp_ratio)), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(int(feature_dim * mlp_ratio), feature_dim), + ) + self.final_ff_dropout = nn.Dropout(dropout) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + # bag: (N, D). Prepend CLS -> sequence (1, N+1, D) (batch dim of 1). + sequence = torch.cat([self.cls_token, bag], dim=0).unsqueeze(0) # (1, N+1, D) + + if self.blocks is not None: + sequence = self.blocks(sequence) # (1, N+1, D) + + # Final self-attention layer, keeping the CLS<-all attention map. + normed = self.final_norm1(sequence) + attended, attn_weights = self.final_attn( + normed, normed, normed, need_weights=True, average_attn_weights=True + ) # attended: (1, N+1, D); attn_weights: (1, N+1, N+1) + sequence = sequence + self.final_attn_dropout(attended) + sequence = sequence + self.final_ff_dropout( + self.final_ff(self.final_norm2(sequence)) + ) + + slide_vector = sequence[0, 0] # CLS embedding -> (D,) + + # Per-tile weight = how much the CLS query attends to each tile (drop the + # CLS->CLS self-weight); renormalise so the tile weights sum to 1. + cls_to_tiles = attn_weights[0, 0, 1:] # (N,) + attention = cls_to_tiles / cls_to_tiles.sum().clamp_min(1e-8) + return slide_vector, attention + + +__all__ = ["TransformerMIL"] diff --git a/ml/models/encoders/__init__.py b/ml/models/encoders/__init__.py new file mode 100644 index 00000000..73d65575 --- /dev/null +++ b/ml/models/encoders/__init__.py @@ -0,0 +1,12 @@ +"""Per-tile encoders: identity (embeddings path) and image backbones.""" + +from ml.models.encoders.base import Encoder +from ml.models.encoders.identity import IdentityEncoder +from ml.models.encoders.vgg16 import VGG16Encoder + + +__all__ = [ + "Encoder", + "IdentityEncoder", + "VGG16Encoder", +] diff --git a/ml/models/encoders/base.py b/ml/models/encoders/base.py new file mode 100644 index 00000000..c1c3310c --- /dev/null +++ b/ml/models/encoders/base.py @@ -0,0 +1,45 @@ +"""Encoder abstract base class. + +An encoder maps a bag of raw tiles to a bag of per-tile feature vectors. On the +precomputed-embeddings path the encoder is :class:`~ml.models.encoders.identity.IdentityEncoder` +(the features already exist); on the end-to-end path it is an image backbone such +as :class:`~ml.models.encoders.vgg16.VGG16Encoder` applied to every tile. + +Concrete encoders live in sibling modules, one implementation per file, and are +selected via the ``ml/encoder`` Hydra config group. The generic +:class:`~ml.models.module.MammaprintModule` only depends on this contract, never +on a concrete class. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + + +class Encoder(nn.Module, ABC): + """Maps a bag of tiles ``(N, ...)`` to per-tile features ``(N, out_dim)``. + + Implementations must set :attr:`out_dim` so the downstream aggregator and head + can be wired without hard-coding feature sizes. + """ + + @property + @abstractmethod + def out_dim(self) -> int: + """Dimensionality of a single per-tile feature vector.""" + + @abstractmethod + def forward(self, tiles: Tensor) -> Tensor: + """Encode a bag of tiles. + + Args: + tiles: A bag for one slide. Shape ``(N, C, H, W)`` for image + backbones or ``(N, D)`` for the identity encoder, where ``N`` is + the number of tiles. + + Returns: + Per-tile features of shape ``(N, out_dim)``. + """ + + +__all__ = ["Encoder"] diff --git a/ml/models/encoders/identity.py b/ml/models/encoders/identity.py new file mode 100644 index 00000000..65474abc --- /dev/null +++ b/ml/models/encoders/identity.py @@ -0,0 +1,32 @@ +"""Identity encoder for the precomputed-embeddings path.""" + +from torch import Tensor + +from ml.models.encoders.base import Encoder + + +class IdentityEncoder(Encoder): + """Passes precomputed tile embeddings through unchanged. + + Used when training a head/aggregator on frozen per-slide embedding bags + produced by ``preprocessing.embeddings`` (e.g. Virchow2, 2560-dim). The + feature size is not inferable from a module, so it is provided explicitly. + + Args: + out_dim: Dimensionality of the incoming embeddings (e.g. ``2560`` for + Virchow2, ``512`` for VGG16 global-max-pooled features). + """ + + def __init__(self, out_dim: int) -> None: + super().__init__() + self._out_dim = out_dim + + @property + def out_dim(self) -> int: + return self._out_dim + + def forward(self, tiles: Tensor) -> Tensor: + return tiles + + +__all__ = ["IdentityEncoder"] diff --git a/ml/models/encoders/vgg16.py b/ml/models/encoders/vgg16.py new file mode 100644 index 00000000..74a82131 --- /dev/null +++ b/ml/models/encoders/vgg16.py @@ -0,0 +1,47 @@ +"""VGG16 image-backbone encoder for the end-to-end (raw-tile) path.""" + +import torch +from torch import Tensor, nn +from torchvision.models import VGG16_Weights, vgg16 + +from ml.models.encoders.base import Encoder + + +class VGG16Encoder(Encoder): + """Encodes raw tiles with VGG16 convolutional features + global max pool. + + Applies the ``torchvision`` VGG16 ``features`` stack to every tile in a bag + and reduces each tile's feature map to a single ``512``-dim vector via + adaptive global max pooling. The classifier/fully-connected part of VGG16 is + dropped — the slide-level prediction is produced downstream by the + aggregator + head. + + Args: + pretrained: Load ImageNet-pretrained weights when ``True``. + freeze: Freeze the backbone (no gradient updates) when ``True`` — useful + for training only the aggregator/head on top of a fixed backbone. + """ + + _FEATURE_DIM = 512 + + def __init__(self, pretrained: bool = True, freeze: bool = False) -> None: + super().__init__() + weights = VGG16_Weights.IMAGENET1K_V1 if pretrained else None + self.features = vgg16(weights=weights).features + self.pool = nn.AdaptiveMaxPool2d(output_size=1) + + if freeze: + for param in self.features.parameters(): + param.requires_grad = False + + @property + def out_dim(self) -> int: + return self._FEATURE_DIM + + def forward(self, tiles: Tensor) -> Tensor: + features = self.features(tiles) # (N, 512, H', W') + pooled = self.pool(features) # (N, 512, 1, 1) + return torch.flatten(pooled, start_dim=1) # (N, 512) + + +__all__ = ["VGG16Encoder"] diff --git a/ml/models/heads/__init__.py b/ml/models/heads/__init__.py new file mode 100644 index 00000000..bbf21835 --- /dev/null +++ b/ml/models/heads/__init__.py @@ -0,0 +1,8 @@ +"""Prediction heads ("hats").""" + +from ml.models.heads.base import Head +from ml.models.heads.linear import LinearHead +from ml.models.heads.mlp import MLPHead + + +__all__ = ["Head", "LinearHead", "MLPHead"] diff --git a/ml/models/heads/base.py b/ml/models/heads/base.py new file mode 100644 index 00000000..48ea7b22 --- /dev/null +++ b/ml/models/heads/base.py @@ -0,0 +1,41 @@ +"""Head ("hat") abstract base class. + +A head maps a slide-level vector to the task prediction — logits for +classification or a scalar for regression. Heads are swappable via the +``ml/head`` Hydra config group; the same aggregated representation can feed +different heads. + +Concrete heads live in sibling modules, one implementation per file. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + + +class Head(nn.Module, ABC): + """Maps a slide vector ``(in_dim,)`` to a prediction ``(out_dim,)``. + + ``out_dim`` encodes the task: ``1`` for binary classification (single logit) + or regression (scalar), ``C`` for ``C``-class multiclass classification. + """ + + def __init__(self, in_dim: int, out_dim: int) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + @abstractmethod + def forward(self, features: Tensor) -> Tensor: + """Predict from the slide-level features. + + Args: + features: Aggregated slide vector, shape ``(in_dim,)`` or a batch + ``(B, in_dim)``. + + Returns: + Prediction of shape ``(out_dim,)`` (or ``(B, out_dim)``). + """ + + +__all__ = ["Head"] diff --git a/ml/models/heads/linear.py b/ml/models/heads/linear.py new file mode 100644 index 00000000..d58c6804 --- /dev/null +++ b/ml/models/heads/linear.py @@ -0,0 +1,34 @@ +"""Linear prediction head.""" + +from torch import Tensor, nn + +from ml.models.heads.base import Head + + +class LinearHead(Head): + """Dropout followed by a single linear layer. + + Works for binary classification / regression (``out_dim=1``) and multiclass + classification (``out_dim=C``). The output is raw (logits / unbounded + scalar); any activation is applied by the module's ``output_activation``. + + Args: + in_dim: Dimensionality of the incoming slide vector. + out_dim: Number of outputs (``1`` for binary/regression, ``C`` for + multiclass). + dropout: Dropout probability applied before the linear layer. + """ + + def __init__(self, in_dim: int, out_dim: int = 1, dropout: float = 0.5) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim) + self.dropout = nn.Dropout(p=dropout) + self.linear = nn.Linear(in_dim, out_dim) + + nn.init.xavier_uniform_(self.linear.weight) + nn.init.zeros_(self.linear.bias) + + def forward(self, features: Tensor) -> Tensor: + return self.linear(self.dropout(features)) + + +__all__ = ["LinearHead"] diff --git a/ml/models/heads/mlp.py b/ml/models/heads/mlp.py new file mode 100644 index 00000000..a9d8622b --- /dev/null +++ b/ml/models/heads/mlp.py @@ -0,0 +1,51 @@ +"""Two-layer MLP prediction head.""" + +from torch import Tensor, nn + +from ml.models.heads.base import Head + + +class MLPHead(Head): + """A two-layer MLP: dropout -> linear -> activation -> dropout -> linear. + + A deeper alternative to :class:`~ml.models.heads.LinearHead` for when the + aggregated slide vector benefits from a non-linear readout (e.g. linear + probing on frozen embeddings, where the head carries all task capacity). + The output is raw (logits / unbounded scalar); any activation is applied by + the module's ``output_activation``. + + Args: + in_dim: Dimensionality of the incoming slide vector. + out_dim: Number of outputs (``1`` for binary/regression, ``C`` for + multiclass). + hidden_dim: Width of the hidden layer. + dropout: Dropout probability applied before each linear layer. + """ + + def __init__( + self, + in_dim: int, + out_dim: int = 1, + hidden_dim: int = 256, + dropout: float = 0.5, + ) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim) + self.hidden_dim = hidden_dim + self.net = nn.Sequential( + nn.Dropout(p=dropout), + nn.Linear(in_dim, hidden_dim), + nn.ReLU(inplace=True), + nn.Dropout(p=dropout), + nn.Linear(hidden_dim, out_dim), + ) + + for layer in self.net: + if isinstance(layer, nn.Linear): + nn.init.xavier_uniform_(layer.weight) + nn.init.zeros_(layer.bias) + + def forward(self, features: Tensor) -> Tensor: + return self.net(features) + + +__all__ = ["MLPHead"] diff --git a/ml/models/losses/__init__.py b/ml/models/losses/__init__.py new file mode 100644 index 00000000..29ad7b0f --- /dev/null +++ b/ml/models/losses/__init__.py @@ -0,0 +1,4 @@ +from ml.models.losses.joint import JointLoss + + +__all__ = ["JointLoss"] diff --git a/ml/models/losses/joint.py b/ml/models/losses/joint.py new file mode 100644 index 00000000..167d458a --- /dev/null +++ b/ml/models/losses/joint.py @@ -0,0 +1,71 @@ +"""Joint classification + regression loss for the ``both`` label mode. + +The MammaPrint slide carries two aligned labels: a binary luminal type (a=1 / b=0) +and the continuous MammaPrint index. This loss trains a single model on both at +once — a standard hard-parameter-sharing multi-task setup where the encoder and +aggregator are shared and the head emits two outputs, one per task:: + + total = w_cls * BCEWithLogits(logit, type) + w_reg * MSE(scalar, index) + +Pair it with: + +* a 2-output head (``out_dim=2``): column 0 is the classification **logit**, + column 1 is the regression scalar; +* ``label_mode="both"``, so the target is ``(B, 2)`` = ``[type_label, index]``; +* ``output_activation=identity`` — BCE consumes raw logits, and the regression + column must stay unbounded. + +The two task losses live on different scales (BCE ~ O(1), MSE ~ the index +variance), so ``w_cls``/``w_reg`` trade them off. Defaults weight them equally; +tune them (or normalise the index) if one task dominates the gradient. +""" + +import torch +from torch import Tensor, nn + + +class JointLoss(nn.Module): + """Weighted sum of a binary-classification and a regression loss. + + Expects predictions and targets of shape ``(B, 2)`` where column 0 is the + classification channel (logit / type label) and column 1 is the regression + channel (scalar / MammaPrint index). + + Args: + w_cls: Weight on the classification (BCE-with-logits) term. + w_reg: Weight on the regression (MSE) term. + pos_weight: Optional positive-class weight passed to + :class:`~torch.nn.BCEWithLogitsLoss` to counter class imbalance. + """ + + def __init__( + self, + w_cls: float = 1.0, + w_reg: float = 1.0, + pos_weight: float | None = None, + ) -> None: + super().__init__() + self.w_cls = w_cls + self.w_reg = w_reg + pw: Tensor | None = None if pos_weight is None else torch.tensor([pos_weight]) + self.cls_loss = nn.BCEWithLogitsLoss(pos_weight=pw) + self.reg_loss = nn.MSELoss() + + def forward(self, y_pred: Tensor, y: Tensor) -> Tensor: + """Combine the per-task losses. + + Args: + y_pred: Head output ``(B, 2)`` — ``[:, 0]`` class logits, ``[:, 1]`` + regression scalars. + y: Targets ``(B, 2)`` — ``[:, 0]`` binary type labels, ``[:, 1]`` + MammaPrint indices. + + Returns: + Scalar total loss ``w_cls * BCE + w_reg * MSE``. + """ + cls = self.cls_loss(y_pred[:, 0], y[:, 0]) + reg = self.reg_loss(y_pred[:, 1], y[:, 1]) + return self.w_cls * cls + self.w_reg * reg + + +__all__ = ["JointLoss"] diff --git a/ml/models/module.py b/ml/models/module.py new file mode 100644 index 00000000..922942d5 --- /dev/null +++ b/ml/models/module.py @@ -0,0 +1,180 @@ +"""Generic MammaPrint LightningModule. + +Wires three independently swappable pieces — an :class:`~ml.models.encoders.base.Encoder`, +an :class:`~ml.models.aggregators.base.Aggregator`, and a :class:`~ml.models.heads.base.Head` — +into a multiple-instance-learning pipeline:: + + bag -> encoder (per tile) -> aggregator (bag -> vector) -> head -> prediction + +The module depends only on the abstract contracts, so encoders (identity for +precomputed embeddings, VGG16/Virchow2 for raw tiles), aggregators (mean, max, +gated attention, transformer) and heads are selected purely via Hydra config. +Loss, optimizer, LR scheduler and metrics are likewise config-instantiated, +following the ``feature/mil`` reference module. +""" + +import functools +import logging +from collections.abc import Sequence +from typing import Any + +import lightning.pytorch as pl +import torch +import torchmetrics +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.models.encoders.base import Encoder +from ml.models.heads.base import Head +from ml.typing import Bag, MILSample, SlideMetadata + + +logger = logging.getLogger(__name__) + + +class MammaprintModule(pl.LightningModule): + """MIL module composing encoder -> aggregator -> head. + + Args: + encoder: Per-tile feature extractor. ``IdentityEncoder`` on the + precomputed-embeddings path; an image backbone on the end-to-end path. + aggregator: Bag pooling reducing per-tile features to a slide vector. + head: Prediction head mapping the slide vector to logits / a scalar. + output_activation: Applied to the head output (e.g. ``Sigmoid`` for + binary classification, ``Identity`` for regression / when the loss + expects logits). Defaults to identity. + loss: Loss module. ``None`` disables training (e.g. predict-only). + optimizer: A ``functools.partial`` of an optimizer, bound to the model + parameters in :meth:`configure_optimizers`. + lr_scheduler: Optional dict with a ``scheduler`` partial and an optional + ``monitor`` key, mirroring the ``feature/mil`` convention. + metrics: Torchmetrics keyed by name; cloned per ``train``/``valid``/``test``. + """ + + def __init__( + self, + encoder: Encoder, + aggregator: Aggregator[Any], + head: Head, + output_activation: nn.Module | None = None, + loss: nn.Module | None = None, + optimizer: functools.partial[torch.optim.Optimizer] | None = None, + lr_scheduler: dict[str, Any] | None = None, + metrics: dict[str, torchmetrics.Metric] | None = None, + ) -> None: + super().__init__() + self.encoder = encoder + self.aggregator = aggregator + self.head = head + self.output_activation = output_activation or nn.Identity() + self.loss = loss + self.optimizer = optimizer + self.lr_scheduler = lr_scheduler + + base = torchmetrics.MetricCollection(dict(metrics) if metrics else {}) + self.train_metrics = base.clone(prefix="train/") + self.valid_metrics = base.clone(prefix="valid/") + self.test_metrics = base.clone(prefix="test/") + + def forward(self, bag: Bag) -> tuple[Tensor, Tensor | None]: + """Run one slide through encoder -> aggregator -> head. + + Args: + bag: One slide's flat tile tensor ``(N, D)`` / ``(N, C, H, W)``. The + encoder is applied per tile (identity for precomputed embeddings, + an image backbone for raw tiles). + + Returns: + ``(prediction, attention)`` where ``prediction`` has shape ``(out_dim,)`` + and ``attention`` is the aggregator's weights or ``None``. + + Note: multi-scale handling (a list of aligned regions) lives entirely in + :class:`~ml.models.aggregators.multiscale.MultiScaleMIL`, which holds its own + encoder — so the module stays flat and never imports multilevel code. + """ + encoded = self.encoder(bag) # (N, ...) -> (N, encoder.out_dim) + pooled, attention = self.aggregator(encoded) # (aggregator.out_dim,), attn|None + prediction = self.output_activation(self.head(pooled)) # (out_dim,) + return prediction, attention + + def _forward_batch(self, bags: Sequence[Bag]) -> tuple[Tensor, list[Tensor | None]]: + """Run a batch of variable-length bags, stacking per-slide predictions.""" + predictions: list[Tensor] = [] + attentions: list[Tensor | None] = [] + for bag in bags: + prediction, attention = self(bag) + predictions.append(prediction) + attentions.append(attention) + return torch.stack(predictions), attentions + + def _step( + self, + batch: list[MILSample], + stage: str, + metrics: torchmetrics.MetricCollection, + ) -> Tensor: + bags, labels, _ = _unpack_batch(batch) + y = torch.stack(labels).to(self.device) + y_pred, _ = self._forward_batch(bags) + + if self.loss is None: + raise RuntimeError("A loss is required for training/validation steps.") + + loss = self.loss(y_pred, y) + self.log( + f"{stage}/loss", + loss, + on_step=stage == "train", + on_epoch=True, + batch_size=len(batch), + ) + + metrics.update(y_pred, y) + self.log_dict(metrics, on_step=False, on_epoch=True) + return loss + + def training_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: + return self._step(batch, "train", self.train_metrics) + + def validation_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: + return self._step(batch, "valid", self.valid_metrics) + + def test_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: + return self._step(batch, "test", self.test_metrics) + + def predict_step(self, batch: list[MILSample], batch_idx: int) -> dict[str, Any]: + bags, _, metadata = _unpack_batch(batch) + predictions, attentions = self._forward_batch(bags) + return { + "predictions": predictions, + "attentions": attentions, + "metadata": metadata, + } + + def configure_optimizers(self) -> Any: + if self.optimizer is None: + return None + + optimizer = self.optimizer(self.parameters()) + config: dict[str, Any] = {"optimizer": optimizer} + + if self.lr_scheduler: + scheduler = {"scheduler": self.lr_scheduler["scheduler"](optimizer)} + if monitor := self.lr_scheduler.get("monitor"): + scheduler["monitor"] = monitor + config["lr_scheduler"] = scheduler + + return config + + +def _unpack_batch( + batch: list[MILSample], +) -> tuple[list[Bag], list[Tensor], list[SlideMetadata]]: + """Split a list-of-samples batch into parallel lists of bags/labels/metadata.""" + bags = [sample[0] for sample in batch] + labels = [sample[1] for sample in batch] + metadata = [sample[2] for sample in batch] + return bags, labels, metadata + + +__all__ = ["MammaprintModule"] diff --git a/ml/resume_test.py b/ml/resume_test.py new file mode 100644 index 00000000..f91509bc --- /dev/null +++ b/ml/resume_test.py @@ -0,0 +1,251 @@ +"""Re-run the test-set inference for a run whose post-fit test step died, and append +the results to that **same MLflow run**. + +Background: ``ml/train.py`` does ``fit`` then, if ``test_after_fit``, ``test`` on the +best checkpoint. When a slide download fails during that test step, the run ends up +with a fully trained model (its best checkpoint is logged under the run's +``checkpoints/best`` artifacts) and train/val metrics — but **no test metrics**. There +is no need to retrain; we only need to re-run inference and write it back. + +This entrypoint, given a run id (or run name), does exactly that: + + 1. reads the run's own logged Hydra overrides (``configs/hydra.yaml`` artifact), + so the data shape / model pieces match the original run exactly, + 2. downloads the run's best checkpoint (``checkpoints/best/checkpoint.ckpt``), + 3. re-invokes ``ml.train`` in ``mode=test`` with ``checkpoint=`` and + ``+logger.run_id=`` — Lightning's MLFlowLogger resumes the existing run + when ``run_id`` is set, so the new ``test/*`` metrics append to it. + +Console log: mlkit's StreamCapture re-logs the run's whole console to a single +``console.log`` artifact from an empty buffer, so the resume's ``test`` output would +overwrite the original training log. Before launching, we copy the existing +``console.log`` to ``console.train.log`` to preserve it (disable with +``--no-keep-console-log``). + +Must run in an environment with the data mounts and MLflow reachable — i.e. the same +pod shape the training job used (see ``scripts/ml/submit_resume_test.py``). + +Usage (inside such a pod): + uv run -m ml.resume_test --run-id + uv run -m ml.resume_test --run-name --experiment MammaPrint + uv run -m ml.resume_test --run-id --dry-run # print the command, run nothing +""" + +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +from mlflow import MlflowClient +from omegaconf import OmegaConf + + +DEFAULT_EXPERIMENT = "MammaPrint" + +# Artifact paths written by rationai.mlkit (autolog._log_config / MLFlowLogger). +HYDRA_ARTIFACT = "configs/hydra.yaml" # holds hydra.overrides.task (the CLI args) +BEST_CKPT_ARTIFACT = "checkpoints/best/checkpoint.ckpt" + +# mlkit's StreamCapture re-logs the run's whole console to this single path, from an +# empty buffer — so the resume's `test` output would OVERWRITE the training console +# log. We back the original up under this path before launching the subprocess. +CONSOLE_LOG = "console.log" +CONSOLE_LOG_BACKUP = "console.train.log" + +# Overrides we must drop from the replay: they steer stage/checkpoint/run identity, +# which resume_test sets itself. Anything matching `key=...` (with these keys) is cut. +DROP_OVERRIDE_KEYS = {"mode", "checkpoint", "test_after_fit"} + + +def _client(tracking_uri: str | None) -> MlflowClient: + # tracking_uri=None -> MlflowClient reads MLFLOW_TRACKING_URI from the env. + return MlflowClient(tracking_uri=tracking_uri) + + +def resolve_run_id( + mlflow_client: MlflowClient, run_id: str | None, run_name: str | None, experiment: str +) -> str: + """Return the run id, looking it up by run name in `experiment` if needed.""" + if run_id: + return run_id + exp = mlflow_client.get_experiment_by_name(experiment) + if exp is None: + raise SystemExit(f"MLflow experiment {experiment!r} not found.") + safe = run_name.replace("'", "\\'") + runs = mlflow_client.search_runs( + experiment_ids=[exp.experiment_id], + filter_string=f"attributes.run_name = '{safe}'", + ) + if not runs: + raise SystemExit(f"No MLflow run named {run_name!r} in {experiment!r}.") + if len(runs) > 1: + ids = ", ".join(r.info.run_id for r in runs) + raise SystemExit( + f"{len(runs)} runs named {run_name!r} in {experiment!r} ({ids}); " + f"pass --run-id to disambiguate." + ) + return runs[0].info.run_id + + +def original_overrides(mlflow_client: MlflowClient, run_id: str, dst: Path) -> list[str]: + """Read the run's original Hydra CLI overrides, minus the stage/checkpoint ones. + + ``autolog._log_config`` saves ``hydra.yaml`` via ``OmegaConf.save(HydraConfig.get())``. + ``HydraConfig.get()`` returns the *contents* of the ``hydra`` node, so the CLI args + live at ``overrides.task`` (no ``hydra.`` prefix). We try both keys to be safe. + """ + path = mlflow_client.download_artifacts(run_id, HYDRA_ARTIFACT, str(dst)) + hydra_cfg = OmegaConf.load(path) + task = OmegaConf.select(hydra_cfg, "overrides.task") + if task is None: + task = OmegaConf.select(hydra_cfg, "hydra.overrides.task") + task = list(task or []) + if not task: + raise SystemExit( + f"Run {run_id}: no Hydra overrides found in {HYDRA_ARTIFACT} " + f"(looked at overrides.task and hydra.overrides.task). Without them the " + f"experiment/data config can't be reconstructed. Pass the overrides " + f"manually, or inspect the artifact." + ) + + kept = [] + for ov in task: + key = ov.split("=", 1)[0].lstrip("+~") + if key in DROP_OVERRIDE_KEYS: + continue + kept.append(ov) + return kept + + +def download_checkpoint(mlflow_client: MlflowClient, run_id: str, dst: Path) -> str: + """Download the run's best checkpoint; return its local path.""" + # Fail early with a clear message if the fit never logged a checkpoint. + arts = {a.path for a in mlflow_client.list_artifacts(run_id, "checkpoints/best")} + if BEST_CKPT_ARTIFACT not in arts: + raise SystemExit( + f"Run {run_id} has no {BEST_CKPT_ARTIFACT!r} artifact " + f"(found: {sorted(arts) or 'nothing'}). Cannot resume test without a " + f"trained checkpoint — the fit likely did not complete." + ) + return mlflow_client.download_artifacts(run_id, BEST_CKPT_ARTIFACT, str(dst)) + + +def backup_console_log( + mlflow_client: MlflowClient, run_id: str, dst: Path, dry_run: bool +) -> bool: + """Preserve the original training console log before the resume overwrites it. + + The resume subprocess writes a fresh ``console.log`` (mlkit's StreamCapture starts + from an empty buffer and log_text overwrites), which would clobber the training + console output. We copy the existing ``console.log`` to ``console.train.log`` first. + Idempotent-ish: if a backup already exists we don't re-copy (so re-running the + resume doesn't overwrite the true original with a test-only log). Returns True if a + backup was made (or already present), False if there was no original to save. + """ + top = {a.path for a in mlflow_client.list_artifacts(run_id)} + if CONSOLE_LOG_BACKUP in top: + print(f" {CONSOLE_LOG_BACKUP!r} already exists; leaving it as the original.") + return True + if CONSOLE_LOG not in top: + print(f" No {CONSOLE_LOG!r} to preserve (nothing logged yet).") + return False + + if dry_run: + print(f" [dry-run] would copy {CONSOLE_LOG!r} -> {CONSOLE_LOG_BACKUP!r}.") + return True + + local = mlflow_client.download_artifacts(run_id, CONSOLE_LOG, str(dst)) + text = Path(local).read_text(encoding="utf-8") + mlflow_client.log_text(run_id, text, CONSOLE_LOG_BACKUP) + print(f" Preserved training log as {CONSOLE_LOG_BACKUP!r}.") + return True + + +def build_command(overrides: list[str], checkpoint: str, run_id: str) -> list[str]: + """The `ml.train` test-mode command that appends results to the existing run.""" + return [ + "uv", "run", "-m", "ml.train", + *overrides, + "mode=test", + f"checkpoint={checkpoint}", + "test_after_fit=false", + f"+logger.run_id={run_id}", + ] + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + src = parser.add_mutually_exclusive_group(required=True) + src.add_argument("--run-id", help="MLflow run id to resume test for.") + src.add_argument("--run-name", help="MLflow run name (== job name) to look up.") + parser.add_argument( + "--experiment", + default=DEFAULT_EXPERIMENT, + help=f"Experiment to search when using --run-name (default {DEFAULT_EXPERIMENT}).", + ) + parser.add_argument( + "--tracking-uri", + default=None, + help="MLflow tracking URI (defaults to MLFLOW_TRACKING_URI in the env).", + ) + parser.add_argument( + "--override", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra Hydra override to pass to ml.train, repeatable. When given, these " + "REPLACE the overrides read from the run's logged config (use if the run's " + "hydra.yaml has none, or to fix them).", + ) + parser.add_argument( + "--keep-console-log", + action=argparse.BooleanOptionalAction, + default=True, + help="Before running, copy the run's existing console.log to console.train.log " + "so the resume's test output doesn't clobber the training log (default: on).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Resolve everything and print the command, but don't run test.", + ) + args = parser.parse_args() + + mlflow_client = _client(args.tracking_uri) + run_id = resolve_run_id(mlflow_client, args.run_id, args.run_name, args.experiment) + print(f"Resuming test for run {run_id}.") + + # Keep the downloaded config + checkpoint for the whole subprocess lifetime. + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + if args.override: + overrides = args.override + print("Using overrides passed on the command line (skipping logged config).") + else: + overrides = original_overrides(mlflow_client, run_id, tmp / "cfg") + checkpoint = download_checkpoint(mlflow_client, run_id, tmp / "ckpt") + print(f"Replaying overrides: {overrides}") + print(f"Checkpoint: {checkpoint}") + + if args.keep_console_log: + print("Preserving the original console log:") + backup_console_log(mlflow_client, run_id, tmp / "console", args.dry_run) + + cmd = build_command(overrides, checkpoint, run_id) + printable = " ".join(cmd) + if args.dry_run: + print(f"[dry-run] would run:\n {printable}") + return 0 + + print(f"Running:\n {printable}") + # Inherit stdout/stderr so the test output streams; MLflow also captures it + # into the same run via the logger's stream capture. + result = subprocess.run(cmd) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ml/train.py b/ml/train.py new file mode 100644 index 00000000..4085614d --- /dev/null +++ b/ml/train.py @@ -0,0 +1,65 @@ +"""Entrypoint for MammaPrint MIL models — train / validate / test / predict. + +Mirrors the preprocessing entrypoints: Hydra composes the config, ``@autolog`` +sets up MLflow logging, and every object (datamodule, module, trainer) is built +with :func:`hydra.utils.instantiate`. The Lightning stage is selected by +``config.mode`` (``fit``/``validate``/``test``/``predict``), and ``config.checkpoint`` +optionally resumes / loads weights. + +Run with, e.g.:: + + uv run -m ml.train +experiment=ml/train_mil_embeddings \ + data/embedded=demo_l3_embed + uv run -m ml.train +experiment=ml/train_mil_embeddings \ + data/embedded=demo_l3_embed mode=test checkpoint=/path/best.ckpt +""" + +import logging +import random + +import hydra +from lightning.pytorch import seed_everything +from omegaconf import DictConfig, OmegaConf +from rationai.mlkit import autolog +from rationai.mlkit.lightning.loggers import MLFlowLogger + + +log = logging.getLogger(__name__) + + +# Resolves ${random_seed:} in the config; cached so all consumers share one seed. +OmegaConf.register_new_resolver( + "random_seed", lambda: random.randint(0, 2**31), use_cache=True, replace=True +) + + +@hydra.main(config_path="../configs", config_name="ml", version_base=None) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + seed_everything(int(config.seed), workers=True) + + # _recursive_=False: keep the per-split dataset nodes as DictConfig so the + # DataModule can instantiate them lazily per stage in setup(). + datamodule = hydra.utils.instantiate(config.datamodule, _recursive_=False) + module = hydra.utils.instantiate(config.ml) + trainer = hydra.utils.instantiate(config.trainer, logger=logger) + + # config.mode selects the Lightning stage (fit/validate/test/predict); + # config.checkpoint (or null) resumes training / loads weights for eval. + run_stage = getattr(trainer, config.mode) + run_stage(module, datamodule=datamodule, ckpt_path=config.checkpoint) + + # After training, evaluate the best checkpoint on the test split so runs are + # directly comparable on held-out data. "best" points at the ModelCheckpoint's + # top-k pick; if checkpointing is off it's empty, so fall back to final weights. + if config.mode == "fit" and config.get("test_after_fit", False): + best_ckpt = getattr(trainer.checkpoint_callback, "best_model_path", "") or None + log.info( + "Running test on %s checkpoint after fit.", + "best" if best_ckpt else "final (no checkpoint saved)", + ) + trainer.test(module, datamodule=datamodule, ckpt_path=best_ckpt) + + +if __name__ == "__main__": + main() diff --git a/ml/typing.py b/ml/typing.py index c356da25..3265d868 100644 --- a/ml/typing.py +++ b/ml/typing.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import TypedDict from torch import Tensor @@ -12,3 +13,55 @@ class TileMetadata(TypedDict): type TilesPredictSample = tuple[Tensor, TileMetadata] + + +class SlideMetadata(TypedDict): + """Metadata for a slide-level MIL sample.""" + + slide_id: str + + +class LevelSpec(TypedDict): + """One pyramid level's data card: physical facts + per-split artifact locations. + + Authored once per preprocessing artifact (a ``configs/data/tiled/*.yaml`` card). + ``uris`` maps ``split -> embedding artifact URI`` (``train``/``val``/``test``); + ``raw_uris`` (optional) does the same for the raw-tile artifact. Per-split URIs + because the split is materialised into physically separate artifacts by + ``scripts/preprocessing/split_dataset.py``. + """ + + mpp: float + tile_extent: int + uris: Mapping[str, str] + + +# ── Single-level types (always shipped) ────────────────────────────────────── +type Bag = Tensor +"""A bag of per-slide items: tile features ``(N, D)`` or raw tiles ``(N, C, H, W)``. + +The coordinate-aware single-scale path (``SpatialScaleDataset`` + +``SpatialTransformerMIL``) packs a wider ``(N, D + 2)`` tensor whose last two columns +are each tile's level-0 pixel ``(x, y)``; the spatial aggregator slices them off. No +other dataset or aggregator produces or consumes those extra columns. +""" + +type MILSample = tuple[Bag, Tensor, SlideMetadata] +"""A single-level slide-level training sample: ``(bag, label, metadata)``.""" + + +# ── Multilevel types (comment out this block to ship single-level only) ─────── +# Nothing single-level references these; the multilevel dataset/aggregator are the +# only consumers, so removing/commenting them here + their imports drops multilevel. +# type Region = dict[int, Tensor] +# """One aligned multi-scale region: level -> that level's tiles ``(K_level, D)``. + +# For example ``{3: (1, D), 2: (4, D)}`` is one coarse level-3 tile with the four +# level-2 tiles that zoom into the same footprint. +# """ + +# type MultiScaleBag = list[Region] +# """A slide as a list of aligned multi-scale regions (variable length).""" + +# type MultiScaleSample = tuple[MultiScaleBag, Tensor, SlideMetadata] +# """A multi-scale slide-level training sample: ``(regions, label, metadata)``.""" diff --git a/preprocessing/embeddings.py b/preprocessing/embeddings.py index b0881982..2bd79fdf 100644 --- a/preprocessing/embeddings.py +++ b/preprocessing/embeddings.py @@ -131,7 +131,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: embeddings_path, ) - except Exception as e: + except Exception as e: # noqa: BLE001 - continue with the remaining slides print(f"Error processing slide {slide_name}: {e}") logger.log_artifacts(str(dest), artifact_path="embeddings") diff --git a/preprocessing/epithelium_masks.py b/preprocessing/epithelium_masks.py index 0a9f1464..f78f20ff 100644 --- a/preprocessing/epithelium_masks.py +++ b/preprocessing/epithelium_masks.py @@ -37,7 +37,7 @@ async def _bounded_process(client: AsyncClient, path: str) -> None: output_path=output_path, timeout=timeout, ) - except Exception as e: + except Exception as e: # noqa: BLE001 - isolate per-slide failures print(f"Slide processing failed for {path}: {e}, error: {e!r}") async with AsyncClient(timeout=timeout) as client: @@ -54,7 +54,7 @@ async def _bounded_process(client: AsyncClient, path: str) -> None: @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: slides_df = pd.read_csv(config.dataset.paths.data_mapping) - slides: list[str] = slides_df.apply(lambda x: x["path"] + ".mrxs", axis=1).tolist() + slides: list[str] = slides_df["path"].tolist() with tempfile.TemporaryDirectory( prefix="episeg_", dir=config.project_path diff --git a/preprocessing/filter_embeddings.py b/preprocessing/filter_embeddings.py new file mode 100644 index 00000000..a561498c --- /dev/null +++ b/preprocessing/filter_embeddings.py @@ -0,0 +1,686 @@ +"""Filter existing tissue-only tile embeddings using scored mask overlaps.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +import tempfile +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, TypedDict, cast + +import mlflow +import numpy as np +import pandas as pd +from mlflow.artifacts import download_artifacts +from mlflow.exceptions import MlflowException + + +if TYPE_CHECKING: + from collections.abc import Iterator + + +DEFAULT_DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" +DEFAULT_EXPERIMENT = "MammaPrint" +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +SPLITS = ("train", "val", "test") +STRATEGY_LOGICS = ("epithelium_only", "or", "and") +FILTER_STATE_FILENAME = "filter_state.json" +REQUIRED_SCORE_COLUMNS = ( + "slide_id", + "x", + "y", + "epithelium_overlap", + "cancer_overlap", +) +REQUIRED_EMBEDDING_COLUMNS = ("x", "y", "embedding") + + +@dataclass(frozen=True) +class Strategy: + slug: str + logic: str + + +class FilterState(TypedDict): + schema_version: int + level: int + tiling_uri: str + embeddings_uri: str + strategies: list[str] + counts: dict[str, dict[str, int]] + scored_tiles: int + source_embedding_tiles: int + epithelium_threshold: float + cancer_threshold: float + + +def _threshold_token(value: float) -> str: + return f"{value:g}".replace(".", "") + + +def _strategies( + epithelium_threshold: float, + cancer_threshold: float, + selected_logics: tuple[str, ...] = STRATEGY_LOGICS, +) -> tuple[Strategy, ...]: + epithelium = _threshold_token(epithelium_threshold) + cancer = _threshold_token(cancer_threshold) + definitions = ( + Strategy(f"epithel_{epithelium}", "epithelium_only"), + Strategy(f"epithel_or_cancer_{epithelium}_{cancer}", "or"), + Strategy(f"epithel_and_cancer_{epithelium}_{cancer}", "and"), + ) + selected = set(selected_logics) + return tuple(strategy for strategy in definitions if strategy.logic in selected) + + +def _selection_masks( + scores: pd.DataFrame, + epithelium_threshold: float, + cancer_threshold: float, +) -> dict[str, np.ndarray]: + epithelium = scores["epithelium_overlap"].to_numpy() > epithelium_threshold + cancer = scores["cancer_overlap"].to_numpy() > cancer_threshold + return { + "epithelium_only": epithelium, + "or": epithelium | cancer, + "and": epithelium & cancer, + } + + +def _require_columns( + frame: pd.DataFrame, required: tuple[str, ...], source: Path +) -> None: + missing = sorted(set(required) - set(frame.columns)) + if missing: + raise ValueError(f"{source} is missing required columns: {missing}") + + +def _load_split_map(data_mapping_path: Path) -> dict[str, str]: + mapping = pd.read_csv(data_mapping_path) + _require_columns(mapping, ("path", "split"), data_mapping_path) + mapping = mapping[mapping["path"].notna() & mapping["split"].notna()] + stems = mapping["path"].map(lambda path: Path(str(path)).stem) + if stems.duplicated().any(): + duplicates = sorted(stems[stems.duplicated(keep=False)].unique()) + raise ValueError( + f"Duplicate slide stems in {data_mapping_path}: {duplicates[:10]}" + ) + + split_map = dict(zip(stems, mapping["split"].astype(str), strict=True)) + unexpected = sorted(set(split_map.values()) - set(SPLITS)) + if unexpected: + raise ValueError( + f"Unexpected split values in {data_mapping_path}: {unexpected}" + ) + return split_map + + +def _load_scores( + tiling_dir: Path, expected_level: int +) -> tuple[dict[str, pd.DataFrame], int]: + slides_path = tiling_dir / "slides.parquet" + tiles_path = tiling_dir / "tiles.parquet" + slides = pd.read_parquet(slides_path, columns=["id", "path", "level"]) + tiles = pd.read_parquet(tiles_path, columns=list(REQUIRED_SCORE_COLUMNS)) + _require_columns(slides, ("id", "path", "level"), slides_path) + _require_columns(tiles, REQUIRED_SCORE_COLUMNS, tiles_path) + + actual_levels = sorted(slides["level"].dropna().unique().tolist()) + if actual_levels != [expected_level]: + raise ValueError( + f"Expected a level-{expected_level} tiling artifact, found levels {actual_levels}" + ) + + if slides["id"].duplicated().any(): + raise ValueError(f"Duplicate slide IDs in {slides_path}") + slides = slides.assign(stem=slides["path"].map(lambda path: Path(str(path)).stem)) + if slides["stem"].duplicated().any(): + duplicates = sorted( + slides.loc[slides["stem"].duplicated(keep=False), "stem"].unique() + ) + raise ValueError(f"Duplicate slide stems in {slides_path}: {duplicates[:10]}") + + id_to_stem = dict(zip(slides["id"], slides["stem"], strict=True)) + unknown_ids = sorted(set(tiles["slide_id"].unique()) - set(id_to_stem)) + if unknown_ids: + raise ValueError(f"Tiles reference unknown slide IDs: {unknown_ids[:10]}") + if tiles[["epithelium_overlap", "cancer_overlap"]].isna().any().any(): + null_counts = tiles[["epithelium_overlap", "cancer_overlap"]].isna().sum() + raise ValueError(f"Mask-score columns contain null values:\n{null_counts}") + + empty_scores = tiles.drop(columns="slide_id").iloc[:0].copy() + by_stem = {stem: empty_scores.copy() for stem in slides["stem"]} + for slide_id, frame in tiles.groupby("slide_id", sort=False): + scores = frame.drop(columns="slide_id").reset_index(drop=True) + if scores.duplicated(["x", "y"]).any(): + raise ValueError( + f"Duplicate scored coordinates for slide {id_to_stem[slide_id]}" + ) + by_stem[id_to_stem[slide_id]] = scores + return by_stem, len(tiles) + + +def _prepare_output_dirs( + root: Path, strategies: tuple[Strategy, ...] +) -> dict[str, dict[str, Path]]: + output_dirs: dict[str, dict[str, Path]] = {} + for strategy in strategies: + directories = {"all": root / strategy.slug / "embeddings"} + directories.update( + {split: root / strategy.slug / f"embeddings_{split}" for split in SPLITS} + ) + for directory in directories.values(): + directory.mkdir(parents=True, exist_ok=True) + output_dirs[strategy.logic] = directories + return output_dirs + + +def _link_to_split(source: Path, destination: Path) -> None: + try: + os.link(source, destination) + except OSError: + # A hard link can fail if a caller places temporary output across filesystems. + shutil.copy2(source, destination) + + +def filter_embeddings( + scores_by_stem: dict[str, pd.DataFrame], + embeddings_dir: Path, + output_dirs: dict[str, dict[str, Path]], + split_map: dict[str, str], + epithelium_threshold: float, + cancer_threshold: float, + logger: logging.Logger, +) -> tuple[dict[str, dict[str, int]], int]: + embedding_files = sorted(embeddings_dir.glob("*.parquet")) + if not embedding_files: + raise ValueError(f"No per-slide Parquet files found in {embeddings_dir}") + + embedding_stems = {path.stem for path in embedding_files} + missing_embeddings = sorted(set(scores_by_stem) - embedding_stems) + if missing_embeddings: + raise ValueError( + f"{len(missing_embeddings)} scored slides have no embedding file: " + f"{missing_embeddings[:10]}" + ) + unmapped_splits = sorted(embedding_stems - set(split_map)) + if unmapped_splits: + raise ValueError( + f"{len(unmapped_splits)} embedded slides have no split mapping: {unmapped_splits[:10]}" + ) + + counts = { + logic: {"tiles": 0, "empty_slides": 0, "slides_below_100_tiles": 0} + for logic in output_dirs + } + source_embedding_tiles = 0 + + for index, embeddings_path in enumerate(embedding_files, start=1): + stem = embeddings_path.stem + source_embeddings = pd.read_parquet(embeddings_path) + _require_columns(source_embeddings, REQUIRED_EMBEDDING_COLUMNS, embeddings_path) + if source_embeddings.duplicated(["x", "y"]).any(): + raise ValueError(f"Duplicate embedded coordinates in {embeddings_path}") + + scores = scores_by_stem.get(stem) + if scores is None: + # Extra embedding slides are allowed only when the scored tiling set contains no tiles + # for them. They are preserved as empty bags in every derived dataset. + scores = pd.DataFrame( + columns=["x", "y", "epithelium_overlap", "cancer_overlap"] + ) + + merged = scores.merge( + source_embeddings, + on=["x", "y"], + how="left", + validate="one_to_one", + indicator=True, + ) + unmatched = merged["_merge"].ne("both") + if unmatched.any(): + examples = merged.loc[unmatched, ["x", "y"]].head().to_dict("records") + raise ValueError( + f"{int(unmatched.sum())} scored coordinates for {stem} have no embedding: {examples}" + ) + merged = merged.drop(columns="_merge") + source_embedding_tiles += len(source_embeddings) + masks = _selection_masks(merged, epithelium_threshold, cancer_threshold) + split = split_map[stem] + + for logic in output_dirs: + selected = masks[logic] + output = merged.loc[selected, list(REQUIRED_EMBEDDING_COLUMNS)] + output_path = output_dirs[logic]["all"] / embeddings_path.name + output.to_parquet(output_path, index=False, engine="pyarrow") + _link_to_split( + output_path, output_dirs[logic][split] / embeddings_path.name + ) + counts[logic]["tiles"] += len(output) + counts[logic]["empty_slides"] += int(output.empty) + counts[logic]["slides_below_100_tiles"] += int(len(output) < 100) + + if index % 100 == 0 or index == len(embedding_files): + logger.info("Filtered %d/%d slides", index, len(embedding_files)) + + return counts, source_embedding_tiles + + +def _log_artifacts_with_retries( + local_dir: Path, + artifact_path: str, + logger: logging.Logger, + max_retries: int, +) -> None: + for attempt in range(max_retries + 1): + try: + mlflow.log_artifacts(str(local_dir), artifact_path=artifact_path) + return + except MlflowException: + if attempt == max_retries: + raise + delay = 2**attempt + logger.warning( + "Uploading %s failed; retrying in %d second(s) (%d/%d)", + artifact_path, + delay, + attempt + 1, + max_retries, + ) + time.sleep(delay) + + +def _log_outputs( + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + output_dirs: dict[str, dict[str, Path]], + strategies: tuple[Strategy, ...], + counts: dict[str, dict[str, int]], + scored_tiles: int, + source_embedding_tiles: int, + epithelium_threshold: float, + cancer_threshold: float, + console_log: Path, + logger: logging.Logger, + upload_max_retries: int, +) -> None: + for strategy in strategies: + dataset_name = f"l{level}_{strategy.slug}_embed" + manifest = { + "schema_version": 1, + "dataset_name": dataset_name, + "level": level, + "strategy": strategy.logic, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + "comparison": ">", + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + "scored_tiles": scored_tiles, + "source_embedding_tiles": source_embedding_tiles, + **counts[strategy.logic], + } + + with mlflow.start_run(run_name=f"filter-{dataset_name}") as run: + mlflow.log_params( + { + "dataset_name": dataset_name, + "level": level, + "strategy": strategy.logic, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + } + ) + mlflow.set_tags( + { + "data_variant": strategy.slug, + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + } + ) + mlflow.log_metrics( + {key: float(value) for key, value in counts[strategy.logic].items()} + ) + # Upload the training inputs first. If the optional combined artifact later + # fails, the split datasets are already usable and can be resumed separately. + for split_name in (*SPLITS, "all"): + directory = output_dirs[strategy.logic][split_name] + artifact_path = ( + "embeddings" if split_name == "all" else f"embeddings_{split_name}" + ) + logger.info("Logging %s to %s", dataset_name, artifact_path) + _log_artifacts_with_retries( + directory, + artifact_path, + logger, + max_retries=upload_max_retries, + ) + mlflow.log_dict(manifest, "filter_manifest.json") + for handler in logger.handlers: + handler.flush() + experiment_id = run.info.experiment_id + run_id = run.info.run_id + logger.info("Created %s (run %s)", dataset_name, run_id) + for split_name in ("all", *SPLITS): + artifact_path = ( + "embeddings" if split_name == "all" else f"embeddings_{split_name}" + ) + logger.info( + " %s: mlflow-artifacts:/%s/%s/artifacts/%s", + split_name, + experiment_id, + run_id, + artifact_path, + ) + for handler in logger.handlers: + handler.flush() + mlflow.log_artifact(str(console_log)) + + +@contextmanager +def _working_root( + *, level: int, work_dir: Path, output_dir: Path | None +) -> Iterator[Path]: + if output_dir is not None: + output_dir.mkdir(parents=True, exist_ok=True) + yield output_dir + return + + with tempfile.TemporaryDirectory( + prefix=f"filter-embeddings-l{level}-", dir=work_dir + ) as tmp: + yield Path(tmp) + + +def _state_payload( + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + strategies: tuple[Strategy, ...], + counts: dict[str, dict[str, int]], + scored_tiles: int, + source_embedding_tiles: int, + epithelium_threshold: float, + cancer_threshold: float, +) -> FilterState: + return { + "schema_version": 1, + "level": level, + "tiling_uri": tiling_uri, + "embeddings_uri": embeddings_uri, + "strategies": [strategy.logic for strategy in strategies], + "counts": counts, + "scored_tiles": scored_tiles, + "source_embedding_tiles": source_embedding_tiles, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + } + + +def _load_filter_state( + state_path: Path, + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + strategies: tuple[Strategy, ...], + epithelium_threshold: float, + cancer_threshold: float, +) -> FilterState: + if not state_path.is_file(): + raise FileNotFoundError( + f"Cannot use --upload-only because {state_path} does not exist" + ) + state = cast("FilterState", json.loads(state_path.read_text())) + expected = { + "level": level, + "tiling_uri": tiling_uri, + "embeddings_uri": embeddings_uri, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + } + mismatches = { + key: (state.get(key), expected_value) + for key, expected_value in expected.items() + if state.get(key) != expected_value + } + if mismatches: + raise ValueError(f"Saved filter state does not match this command: {mismatches}") + requested_strategies = {strategy.logic for strategy in strategies} + saved_strategies = set(state.get("strategies", [])) + missing_strategies = sorted(requested_strategies - saved_strategies) + if missing_strategies: + raise ValueError( + f"Saved filter state does not contain requested strategies: {missing_strategies}" + ) + return state + + +def _validate_output_dirs(output_dirs: dict[str, dict[str, Path]]) -> None: + for logic, directories in output_dirs.items(): + all_files = { + path.name: path for path in directories["all"].glob("*.parquet") + } + if not all_files: + raise ValueError(f"No filtered Parquet files found for strategy {logic}") + + split_names: set[str] = set() + for split in SPLITS: + for split_path in directories[split].glob("*.parquet"): + if split_path.name in split_names: + raise ValueError( + f"{split_path.name} appears in more than one split for {logic}" + ) + source_path = all_files.get(split_path.name) + if source_path is None: + raise ValueError( + f"{split_path.name} appears in {split} but not in all for {logic}" + ) + if split_path.stat().st_size != source_path.stat().st_size: + raise ValueError( + f"Size mismatch for {logic}/{split}/{split_path.name}" + ) + split_names.add(split_path.name) + + missing_split_files = sorted(set(all_files) - split_names) + if missing_split_files: + raise ValueError( + f"Filtered files have no split assignment for {logic}: " + f"{missing_split_files[:10]}" + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Filter existing tissue-only embeddings into epithelium-only, epithelium OR cancer, " + "and epithelium AND cancer datasets." + ) + ) + parser.add_argument("--level", type=int, required=True) + parser.add_argument("--tiling-uri", required=True) + parser.add_argument("--embeddings-uri", required=True) + parser.add_argument( + "--embeddings-path", + help="Existing mounted embedding directory; falls back to --embeddings-uri if absent.", + ) + parser.add_argument("--epithelium-threshold", type=float, default=0.25) + parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument( + "--strategies", + nargs="+", + choices=STRATEGY_LOGICS, + default=list(STRATEGY_LOGICS), + help="Strategies to materialise; defaults to all three.", + ) + parser.add_argument("--data-mapping", default=DEFAULT_DATA_MAPPING) + parser.add_argument("--work-dir", default="/mnt/projects/mammaprint") + parser.add_argument( + "--output-dir", + help=( + "Persistent working directory. Filtered files and filter_state.json are kept " + "here so a failed upload can be retried with --upload-only." + ), + ) + parser.add_argument( + "--upload-only", + action="store_true", + help="Upload an existing --output-dir without filtering again.", + ) + parser.add_argument("--upload-max-retries", type=int, default=3) + parser.add_argument( + "--tracking-uri", default=os.getenv("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + ) + parser.add_argument("--experiment-name", default=DEFAULT_EXPERIMENT) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.upload_max_retries < 0: + raise ValueError("--upload-max-retries must be non-negative") + if args.upload_only and args.output_dir is None: + raise ValueError("--upload-only requires --output-dir") + persistent_output = Path(args.output_dir) if args.output_dir else None + if ( + persistent_output is not None + and persistent_output.is_dir() + and any(persistent_output.iterdir()) + and not args.upload_only + ): + raise FileExistsError( + f"Persistent output directory is not empty: {persistent_output}. " + "Use --upload-only to reuse completed filtering or choose a new directory." + ) + + mlflow.set_tracking_uri(args.tracking_uri) + mlflow.set_experiment(args.experiment_name) + selected_logics = tuple(dict.fromkeys(args.strategies)) + strategies = _strategies( + args.epithelium_threshold, + args.cancer_threshold, + selected_logics, + ) + + with _working_root( + level=args.level, + work_dir=Path(args.work_dir), + output_dir=persistent_output, + ) as root: + console_log = root / "console.log" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s", + handlers=[logging.StreamHandler(), logging.FileHandler(console_log)], + force=True, + ) + logger = logging.getLogger(__name__) + output_dirs = _prepare_output_dirs(root / "outputs", strategies) + state_path = root / FILTER_STATE_FILENAME + + if args.upload_only: + state = _load_filter_state( + state_path, + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + strategies=strategies, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + ) + counts = state["counts"] + scored_tiles = int(state["scored_tiles"]) + source_embedding_tiles = int(state["source_embedding_tiles"]) + logger.info("Reusing filtered output recorded in %s", state_path) + else: + logger.info("Downloading scored tiling artifact: %s", args.tiling_uri) + tiling_download_dir = root / "source-tiling" + tiling_download_dir.mkdir(exist_ok=True) + tiling_dir = Path( + download_artifacts(args.tiling_uri, dst_path=str(tiling_download_dir)) + ) + + mounted_embeddings = ( + Path(args.embeddings_path) if args.embeddings_path is not None else None + ) + if mounted_embeddings is not None and mounted_embeddings.is_dir(): + embeddings_dir = mounted_embeddings + logger.info("Using mounted tissue embeddings: %s", embeddings_dir) + else: + if mounted_embeddings is not None: + logger.warning( + "Mounted embedding directory %s is unavailable; downloading %s", + mounted_embeddings, + args.embeddings_uri, + ) + else: + logger.info( + "Downloading tissue embedding artifact: %s", args.embeddings_uri + ) + embeddings_download_dir = root / "source-embeddings" + embeddings_download_dir.mkdir(exist_ok=True) + embeddings_dir = Path( + download_artifacts( + args.embeddings_uri, + dst_path=str(embeddings_download_dir), + ) + ) + + scores_by_stem, scored_tiles = _load_scores(tiling_dir, args.level) + split_map = _load_split_map(Path(args.data_mapping)) + counts, source_embedding_tiles = filter_embeddings( + scores_by_stem=scores_by_stem, + embeddings_dir=embeddings_dir, + output_dirs=output_dirs, + split_map=split_map, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + logger=logger, + ) + state = _state_payload( + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + strategies=strategies, + counts=counts, + scored_tiles=scored_tiles, + source_embedding_tiles=source_embedding_tiles, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + ) + state_path.write_text(json.dumps(state, indent=2)) + logger.info("Saved resumable filter state to %s", state_path) + + _validate_output_dirs(output_dirs) + logger.info("Selection summary:\n%s", json.dumps(counts, indent=2)) + _log_outputs( + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + output_dirs=output_dirs, + strategies=strategies, + counts=counts, + scored_tiles=scored_tiles, + source_embedding_tiles=source_embedding_tiles, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + console_log=console_log, + logger=logger, + upload_max_retries=args.upload_max_retries, + ) + + +if __name__ == "__main__": + main() diff --git a/preprocessing/quality_control.py b/preprocessing/quality_control.py index db043ce1..3b6e2536 100644 --- a/preprocessing/quality_control.py +++ b/preprocessing/quality_control.py @@ -39,7 +39,7 @@ async def put_request( f"Request to {url} timed out after {request_timeout} seconds. Slide: {slide_name}" ) return -1, "Timeout" - except Exception as exc: + except Exception as exc: # noqa: BLE001 - return a failed request result slide_name = Path(data.get("wsi_path", "")).name print(f"Request to {url} failed for {slide_name}: {exc}") return -1, str(exc) @@ -118,7 +118,7 @@ async def generate_report( print( f"Report generation request to {url} timed out after {report_request_timeout} seconds." ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 - report generation is best-effort print(f"Report generation request to {url} failed: {exc}") @@ -171,7 +171,7 @@ async def qc_main( logger.experiment.log_artifacts( run_id=logger.run_id, local_dir=Path(report_path).parent.as_posix() ) - except Exception: + except Exception: # noqa: BLE001 - artifact upload must not abort QC print(f"Could not upload report artifact from {report_path}") @@ -184,7 +184,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: output_path.mkdir(exist_ok=True, parents=True) df = pd.read_csv(config.dataset.paths.data_mapping) - slides = [Path(path + ".mrxs") for path in df["path"]] + slides = [Path(path) for path in df["path"]] semaphore = asyncio.Semaphore(config.request_limit) diff --git a/preprocessing/recover_embedding_splits.py b/preprocessing/recover_embedding_splits.py new file mode 100644 index 00000000..23067a64 --- /dev/null +++ b/preprocessing/recover_embedding_splits.py @@ -0,0 +1,326 @@ +"""Resume split artifact uploads from a complete per-slide embedding artifact.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import tempfile +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING + +import mlflow +from mlflow import MlflowClient +from mlflow.entities import FileInfo +from mlflow.exceptions import MlflowException + +from preprocessing.filter_embeddings import ( + DEFAULT_DATA_MAPPING, + DEFAULT_TRACKING_URI, + SPLITS, + _load_split_map, +) + + +if TYPE_CHECKING: + from collections.abc import Callable + + +@dataclass(frozen=True) +class ArtifactFile: + path: str + size: int + + @property + def name(self) -> str: + return Path(self.path).name + + +@dataclass(frozen=True) +class RecoveryPlan: + expected: dict[str, dict[str, ArtifactFile]] + existing: dict[str, dict[str, ArtifactFile]] + missing: dict[str, dict[str, ArtifactFile]] + + @property + def missing_files(self) -> int: + return sum(len(files) for files in self.missing.values()) + + @property + def missing_bytes(self) -> int: + return sum( + file.size for files in self.missing.values() for file in files.values() + ) + + +def _artifact_files(items: list[FileInfo], artifact_path: str) -> dict[str, ArtifactFile]: + directories = [item.path for item in items if item.is_dir] + if directories: + raise ValueError(f"Unexpected directories under {artifact_path}: {directories[:10]}") + + files: dict[str, ArtifactFile] = {} + for item in items: + name = Path(item.path).name + if Path(name).suffix != ".parquet": + raise ValueError(f"Unexpected non-Parquet artifact: {item.path}") + if name in files: + raise ValueError(f"Duplicate artifact name under {artifact_path}: {name}") + files[name] = ArtifactFile(item.path, int(item.file_size or 0)) + return files + + +def _build_recovery_plan( + source: dict[str, ArtifactFile], + destinations: dict[str, dict[str, ArtifactFile]], + split_map: dict[str, str], +) -> RecoveryPlan: + if not source: + raise ValueError("The source embedding artifact is empty") + + unmapped = sorted(Path(name).stem for name in source if Path(name).stem not in split_map) + if unmapped: + raise ValueError(f"Source slides have no split mapping: {unmapped[:10]}") + + expected: dict[str, dict[str, ArtifactFile]] = {split: {} for split in SPLITS} + for name, file in source.items(): + expected[split_map[Path(name).stem]][name] = file + + for split in SPLITS: + unexpected = sorted(set(destinations[split]) - set(expected[split])) + if unexpected: + raise ValueError( + f"embeddings_{split} contains unexpected files: {unexpected[:10]}" + ) + mismatched = sorted( + name + for name, file in destinations[split].items() + if file.size != expected[split][name].size + ) + if mismatched: + raise ValueError( + f"embeddings_{split} contains size-mismatched files: {mismatched[:10]}" + ) + + missing = { + split: { + name: file + for name, file in expected[split].items() + if name not in destinations[split] + } + for split in SPLITS + } + return RecoveryPlan(expected=expected, existing=destinations, missing=missing) + + +def _retry[T]( + operation: Callable[[], T], *, label: str, max_retries: int, logger: logging.Logger +) -> T: + for attempt in range(max_retries + 1): + try: + return operation() + except MlflowException: + if attempt == max_retries: + raise + delay = 2**attempt + logger.warning( + "%s failed; retrying in %d second(s) (%d/%d)", + label, + delay, + attempt + 1, + max_retries, + ) + time.sleep(delay) + raise AssertionError("unreachable") + + +def _load_plan( + client: MlflowClient, + *, + run_id: str, + source_artifact_path: str, + split_map: dict[str, str], +) -> RecoveryPlan: + source = _artifact_files( + client.list_artifacts(run_id, source_artifact_path), source_artifact_path + ) + destinations = { + split: _artifact_files( + client.list_artifacts(run_id, f"embeddings_{split}"), + f"embeddings_{split}", + ) + for split in SPLITS + } + return _build_recovery_plan(source, destinations, split_map) + + +def _format_bytes(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024 or unit == "TiB": + return f"{value:.1f} {unit}" + value /= 1024 + raise AssertionError("unreachable") + + +def _print_plan(plan: RecoveryPlan) -> None: + for split in SPLITS: + print( + f"{split}: {len(plan.existing[split])}/{len(plan.expected[split])} present; " + f"{len(plan.missing[split])} missing" + ) + print( + f"Total transfer: {plan.missing_files} files, " + f"{_format_bytes(plan.missing_bytes)}" + ) + + +def _recover( + client: MlflowClient, + *, + run_id: str, + plan: RecoveryPlan, + max_retries: int, + logger: logging.Logger, +) -> None: + total = plan.missing_files + completed = 0 + for split in SPLITS: + artifact_path = f"embeddings_{split}" + for name, source in sorted(plan.missing[split].items()): + with tempfile.TemporaryDirectory(prefix="recover-embedding-split-") as tmp: + local_path = Path( + _retry( + partial( + client.download_artifacts, + run_id, + source.path, + dst_path=tmp, + ), + label=f"Download {source.path}", + max_retries=max_retries, + logger=logger, + ) + ) + if local_path.stat().st_size != source.size: + raise ValueError( + f"Downloaded size mismatch for {source.path}: " + f"{local_path.stat().st_size} != {source.size}" + ) + _retry( + partial( + client.log_artifact, + run_id, + str(local_path), + artifact_path=artifact_path, + ), + label=f"Upload {artifact_path}/{name}", + max_retries=max_retries, + logger=logger, + ) + completed += 1 + if completed % 25 == 0 or completed == total: + logger.info("Recovered %d/%d split files", completed, total) + + +def _log_recovery_manifest( + client: MlflowClient, + *, + run_id: str, + source_artifact_path: str, + plan: RecoveryPlan, +) -> None: + manifest = { + "schema_version": 1, + "run_id": run_id, + "source_artifact_path": source_artifact_path, + "recovered_at": datetime.now(UTC).isoformat(), + "file_counts": {split: len(plan.expected[split]) for split in SPLITS}, + } + with tempfile.TemporaryDirectory(prefix="embedding-recovery-manifest-") as tmp: + path = Path(tmp) / "split_recovery_manifest.json" + path.write_text(json.dumps(manifest, indent=2)) + client.log_artifact(run_id, str(path)) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True) + parser.add_argument("--source-artifact-path", default="embeddings") + parser.add_argument("--data-mapping", default=DEFAULT_DATA_MAPPING) + parser.add_argument( + "--tracking-uri", default=os.getenv("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + ) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument( + "--apply", + action="store_true", + help="Perform uploads. Without this flag, only print the recovery plan.", + ) + parser.add_argument( + "--mark-finished", + action="store_true", + help="Mark the recovered MLflow run FINISHED after complete validation.", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.max_retries < 0: + raise ValueError("--max-retries must be non-negative") + if args.mark_finished and not args.apply: + raise ValueError("--mark-finished requires --apply") + + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logger = logging.getLogger(__name__) + mlflow.set_tracking_uri(args.tracking_uri) + client = MlflowClient() + split_map = _load_split_map(Path(args.data_mapping)) + plan = _load_plan( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + split_map=split_map, + ) + _print_plan(plan) + if not args.apply: + print("Dry run only; pass --apply to upload missing files.") + return + + _recover( + client, + run_id=args.run_id, + plan=plan, + max_retries=args.max_retries, + logger=logger, + ) + final_plan = _load_plan( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + split_map=split_map, + ) + if final_plan.missing_files: + raise RuntimeError( + f"Recovery finished with {final_plan.missing_files} files still missing" + ) + + _log_recovery_manifest( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + plan=final_plan, + ) + client.set_tag(run_id=args.run_id, key="split_upload_recovered", value="true") + if args.mark_finished: + client.set_terminated(args.run_id, status="FINISHED") + print("Recovery complete; all split artifact files are present and size-validated.") + + +if __name__ == "__main__": + main() diff --git a/preprocessing/split_dataset.py b/preprocessing/split_dataset.py new file mode 100644 index 00000000..20acbd2f --- /dev/null +++ b/preprocessing/split_dataset.py @@ -0,0 +1,152 @@ +"""Materialise per-split MLflow artifacts from a whole dataset artifact.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import tempfile +from pathlib import Path + +import mlflow +import pandas as pd +import yaml +from mlflow.artifacts import download_artifacts +from rationai.tiling.writers import save_mlflow_dataset + + +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" +MLFLOW_EXPERIMENT = "MammaPrint" +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +SPLITS = ("train", "val", "test") + + +def _load_split_map(data_mapping_path: str) -> dict[str, str]: + """Return ``slide_stem -> split`` from data_mapping.csv, keyed by path stem.""" + df = pd.read_csv(data_mapping_path) + if "split" not in df.columns: + raise ValueError(f"'{data_mapping_path}' has no 'split' column.") + + df = df[df["path"].notna() & df["split"].notna()] + stems = df["path"].apply(lambda path: Path(str(path)).stem) + split_map = dict(zip(stems, df["split"].astype(str), strict=True)) + bad = {split for split in split_map.values() if split not in SPLITS} + if bad: + raise ValueError(f"Unexpected split values in '{data_mapping_path}': {bad}") + return split_map + + +def _assign_or_die(stems: list[str], split_map: dict[str, str]) -> dict[str, str]: + """Map each artifact stem to its split; abort if any stem is unmapped.""" + unmapped = sorted(stem for stem in stems if stem not in split_map) + if unmapped: + raise ValueError( + f"{len(unmapped)} slide(s) in the artifact have no split row in " + f"data_mapping.csv: {unmapped}. Add them (or fix the artifact) first." + ) + return {stem: split_map[stem] for stem in stems} + + +def _configure_mlflow() -> None: + tracking_uri = os.environ.get("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + mlflow.set_tracking_uri(tracking_uri) + mlflow.set_experiment(MLFLOW_EXPERIMENT) + print(f"Using MLflow tracking URI: {tracking_uri} (experiment: {MLFLOW_EXPERIMENT})") + + +def _uri_from_card(card_path: str) -> str: + """Read the whole artifact URI from a tiled or embedded level card.""" + card = yaml.safe_load(Path(card_path).read_text()) + uris = card.get("raw_uris") or card.get("uris") or {} + all_uri = uris.get("all") + if not all_uri: + raise ValueError( + f"Card '{card_path}' has no `raw_uris.all` / `uris.all` URI to split." + ) + return all_uri + + +def _name_from_uri(uri: str) -> str: + name = Path(uri.rstrip("/")).name + if name in {"", ".", "artifacts"}: + raise ValueError(f"Could not infer a dataset name from --uri '{uri}'. Pass --name.") + return name + + +def _split_embedded(local_dir: Path, name: str, split_map: dict[str, str]) -> None: + files = sorted(local_dir.glob("*.parquet")) + if not files: + raise ValueError(f"No .parquet files found under '{local_dir}'.") + assignment = _assign_or_die([file.stem for file in files], split_map) + + with tempfile.TemporaryDirectory(dir=os.getcwd()) as tmp: + split_dirs = {split: Path(tmp) / split for split in SPLITS} + for directory in split_dirs.values(): + directory.mkdir() + for file in files: + shutil.copy2(file, split_dirs[assignment[file.stem]] / file.name) + + for split in SPLITS: + count = sum(1 for value in assignment.values() if value == split) + dataset_name = f"{name}_{split}" + with mlflow.start_run(run_name=f"split-{dataset_name}"): + mlflow.log_artifacts(str(split_dirs[split]), dataset_name) + print(f" {split}: {count} slides -> {dataset_name}") + + +def _split_tiled(local_dir: Path, name: str, split_map: dict[str, str]) -> None: + slides = pd.read_parquet(local_dir / "slides.parquet") + tiles = pd.read_parquet(local_dir / "tiles.parquet") + slides = slides.assign(_stem=slides["path"].apply(lambda path: Path(str(path)).stem)) + assignment = _assign_or_die(slides["_stem"].tolist(), split_map) + slides = slides.assign(split=slides["_stem"].map(assignment)) + + for split in SPLITS: + split_slide_ids = slides.loc[slides["split"] == split, "id"] + split_slides = slides[slides["split"] == split].drop(columns=["_stem", "split"]) + split_tiles = tiles[tiles["slide_id"].isin(split_slide_ids)] + dataset_name = f"{name}_{split}" + with mlflow.start_run(run_name=f"split-{dataset_name}"): + save_mlflow_dataset( + slides=split_slides.reset_index(drop=True), + tiles=split_tiles.reset_index(drop=True), + dataset_name=dataset_name, + ) + print( + f" {split}: {len(split_slides)} slides, {len(split_tiles)} tiles " + f"-> {dataset_name}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Materialise per-split MLflow artifacts from data_mapping.csv." + ) + parser.add_argument("--kind", choices=["embedded", "tiled"], required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--card", help="Level card whose `all` URI is the artifact to split.") + source.add_argument("--uri", help="Whole artifact URI (overrides --card).") + parser.add_argument("--name", help="Output base name (default: inferred from URI).") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + args = parser.parse_args() + + split_map = _load_split_map(args.data_mapping) + _configure_mlflow() + uri = args.uri or _uri_from_card(args.card) + name = args.name or _name_from_uri(uri) + local_dir = Path(download_artifacts(artifact_uri=uri)) + print(f"Splitting {args.kind} artifact '{uri}' as '{name}_{{train,val,test}}'") + + if args.kind == "embedded": + _split_embedded(local_dir, name, split_map) + else: + _split_tiled(local_dir, name, split_map) + + print( + "Done. Point the level card at the three MLflow artifacts named:\n" + f" {name}_train\n {name}_val\n {name}_test" + ) + + +if __name__ == "__main__": + main() diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index cf66793a..c69ebb37 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -1,14 +1,14 @@ """Script for creating slides and tiles datasets for mammaprint prediction using Ray Data.""" +import os from functools import partial from pathlib import Path -from typing import Any +from typing import Any, Literal import hydra import numpy as np import pandas as pd import ray -import ray.data as rd from mlflow.artifacts import download_artifacts from omegaconf import DictConfig from rationai.mlkit import autolog, with_cli_args @@ -19,12 +19,55 @@ from ratiopath.tiling.utils import row_hash from ray.data import Dataset from ray.data.block import DataBatch -from ray.data.expressions import col +from ray.data.expressions import col, lit from shapely.geometry import box from shapely.geometry.polygon import Polygon -BATCH_SIZE = 256 +# Rows per block after the tiling flat_map. Larger blocks amortize Ray's per-block +# scheduling overhead and the per-batch mask file-open in the overlap UDFs (one open per +# (mask, block) instead of one per 256 tiles), at the cost of more object-store memory per +# block. Kept well within the object store sized below. +BATCH_SIZE = 4096 + +# Fraction of the pod's memory to give Ray's object store when RAY_MEMORY_BYTES is set. +# The rest is left for Ray's heap and the Python worker processes. +RAY_OBJECT_STORE_FRACTION = 0.4 + + +def _ray_init_kwargs() -> dict[str, Any]: + """Build ``ray.init`` sizing kwargs from the pod's resources. + + On the K8s cluster ``ray.init()`` auto-detection was unreliable: it over-detected CPUs + (saw the 192-core node instead of the pod's cgroup limit) and under-sized the object + store (~9.5 GiB), which backpressured the repartition queue and collapsed the + overlap+filter stage to a single task. + + To avoid duplicating the pod's resource numbers, the launcher + (``scripts/preprocessing/run_tiling.py``) exports them as environment variables derived + from its own ``submit_job(cpu=..., memory=...)`` args: + + * ``RAY_NUM_CPUS`` -- number of CPUs to give Ray. + * ``RAY_MEMORY_BYTES`` -- the pod's total memory in bytes; the object store is sized as + a fraction of it. + + When unset (e.g. a plain local run), we fall back to Ray's own auto-detection for both. + """ + kwargs: dict[str, Any] = {} + + num_cpus = os.environ.get("RAY_NUM_CPUS") + if num_cpus: + kwargs["num_cpus"] = int(float(num_cpus)) + + memory_bytes = os.environ.get("RAY_MEMORY_BYTES") + if memory_bytes: + # Fraction of pod memory for the object store. The rest is left for Ray's heap and + # the Python worker processes. Capped by /dev/shm at the pod level (see run_tiling.py). + kwargs["object_store_memory"] = int( + float(memory_bytes) * RAY_OBJECT_STORE_FRACTION + ) + + return kwargs def add_tile_overlap( @@ -32,17 +75,32 @@ def add_tile_overlap( roi: Polygon, path_col: str, overlap_col: str, - struct_field: str, + keep: Literal["zero", "nonzero", "mean"], + background: str = "0", ) -> Dataset: - """Compute per-tile overlap score against a mask overlay. + """Compute a per-tile coverage score against a mask overlay. + + This helper uses `tile_overlay_overlap` to obtain, for each tile, the fraction of the ROI + covered by every distinct pixel value in the mask, then reduces it to a single score in + ``[0, 1]`` where **higher = keep**. + + The masks are downsampled to a continuous ``[0, 255]`` range (they are NOT strictly binary): + the background/clean pixel value is ``0`` and *any* non-zero value marks foreground (tissue) or + an artifact, depending on the mask. Three reductions are supported via ``keep``: - This helper uses `tile_overlay_overlap` to compute, for each tile, - how much of the given ROI (tile sub-region) overlaps a binary mask. + * ``keep="nonzero"`` (tissue/epithelium): score = fraction of non-zero pixels = foreground + coverage. Keep tiles with a lot of tissue/epithelium. + * ``keep="zero"`` (QC artifact masks: blur/folding/residual): score = fraction of ``0`` pixels = + clean fraction. Keep tiles that are mostly artifact-free. + * ``keep="mean"`` (cancer probability heatmaps): score = mean pixel intensity divided by 255. + OpenSlide exposes grayscale TIFFs as repeated RGB channels, so the RGB mean remains the + original mean probability. - The mask file path is read from the column ``path_col`` in the Ray Dataset. The - function expects the dataset to contain tile coordinates and microns-per-pixel columns - (``tile_x``, ``tile_y``, ``mpp_x``, ``mpp_y``), as produced by the upstream tiling - pipeline. + In all cases a downstream ``overlap > threshold`` filter keeps the desirable tiles. + + The mask file path is read from the column ``path_col`` in the Ray Dataset. The function expects + the dataset to contain tile coordinates and microns-per-pixel columns (``tile_x``, ``tile_y``, + ``mpp_x``, ``mpp_y``), as produced by the upstream tiling pipeline. Parameters ---------- @@ -54,14 +112,13 @@ def add_tile_overlap( path_col: Column name containing the mask image path for each tile. overlap_col: - Name of the output column to store the computed overlap score in [0, 1]. - struct_field: - The pixel value (as string: "0" or "255") in the binary mask representing the - undesired artifact or background. The mask contains only values 0 and 255. - The overlap is computed as ``1 - (fraction of pixels with this value)``, yielding - the fraction of desired (non-artifact) pixels. - For example, when folding masks have folds as white (255), pass "255" to compute - the amount of non-folded tissue. + Name of the output column to store the computed score in [0, 1]. + keep: + Which pixels count as desirable: ``"zero"`` scores the clean (background) fraction, + ``"nonzero"`` scores the foreground fraction, and ``"mean"`` scores normalized mean + intensity. + background: + Pixel value (as string) of the clean/background class. Defaults to ``"0"``. Returns: ------- @@ -77,34 +134,144 @@ def add_tile_overlap( col("mpp_x"), col("mpp_y"), ) - tiles = tiles.with_column( - overlap_struct_col, overlap_struct, num_cpus=1, memory=BATCH_SIZE * 8 * 512**2 - ) + # num_cpus=1 so Ray schedules one task per core across the pod. Without it the fused + # overlap+filter operator ran with a single task and left 47 of 48 cores idle. + tiles = tiles.with_column(overlap_struct_col, overlap_struct, num_cpus=1) def extract_value(batch: DataBatch) -> DataBatch: - batch[overlap_col] = np.array( - [ - 1.0 - (s.get(struct_field, 1.0) or 1.0) if isinstance(s, dict) else 0.0 - for s in batch[overlap_struct_col] - ] - ) + def score(s: Any) -> float: + # No mask / no readable pixels in the ROI -> score 0 so the tile is dropped. + if not isinstance(s, dict) or not s: + return 0.0 + if keep == "mean": + weighted_sum = 0.0 + total_fraction = 0.0 + for value, fraction in s.items(): + fraction = float(fraction or 0.0) + weighted_sum += float(value) * fraction + total_fraction += fraction + if total_fraction == 0.0: + return 0.0 + return weighted_sum / total_fraction / 255.0 + # Fraction of clean/background (== background value) pixels in the ROI. + zero_fraction = s.get(background, 0.0) or 0.0 + # keep="zero": clean fraction; keep="nonzero": foreground (1 - clean) fraction. + return zero_fraction if keep == "zero" else 1.0 - zero_fraction + + batch[overlap_col] = np.array([score(s) for s in batch[overlap_struct_col]]) return batch tiles = tiles.map_batches( extract_value, batch_format="pandas", num_cpus=1, - memory=BATCH_SIZE * 8 * 512**2, ) return tiles.drop_columns([overlap_struct_col]) +def count_candidate_tiles(slides_df: pd.DataFrame) -> int: + """Number of candidate tiles across all slides, computed analytically (no mask IO). + + Mirrors ``grid_tiles(..., last="keep")``: per axis it yields + ``ceil((extent - tile_extent) / stride) + 1`` tiles, so the per-slide count is the product + over the x and y axes. Summed over slides this is the pre-filter tile count -- computed from + the slide metadata alone, so we never have to materialize the ~250M-row candidate set to count + it. + """ + ex = np.asarray(slides_df["extent_x"], dtype=float) + ey = np.asarray(slides_df["extent_y"], dtype=float) + tx = np.asarray(slides_df["tile_extent_x"], dtype=float) + ty = np.asarray(slides_df["tile_extent_y"], dtype=float) + sx = np.asarray(slides_df["stride_x"], dtype=float) + sy = np.asarray(slides_df["stride_y"], dtype=float) + nx = np.ceil((ex - tx) / sx) + 1 + ny = np.ceil((ey - ty) / sy) + 1 + return int(np.sum(nx * ny)) + + +def log_drop_funnel( + tissue_survivors: Dataset, + n_candidates: int, + thresholds: dict[str, float], + logger: MLFlowLogger, +) -> None: + """Quantify how many tiles each QC threshold drops and log the funnel to MLflow. + + ``tissue_survivors`` must be the dataset AFTER the tissue gate but with ``blur_overlap``, + ``folding_overlap`` and ``residual_overlap`` already computed and NOT yet filtered (so the + dropped tiles' scores are still present). Reports two views: + + * **independent** -- how many tiles each threshold removes on its own. Tissue is measured + against the full candidate grid (``n_candidates``); the QC gates are measured among the + tissue survivors (the only affordable and meaningful denominator). + * **sequential** -- the funnel in pipeline order (tissue -> blur -> folding -> residual), where + each gate is credited only for tiles it is the first to drop. These sum to the total dropped. + + A tile is dropped by a gate when ``overlap <= threshold`` (the strict complement of the + production ``overlap > threshold`` filter; NaN/empty overlaps already score 0.0 and so count as + dropped, matching the run). + """ + df = tissue_survivors.to_pandas() + n_tissue = len(df) + + b = df["blur_overlap"] <= thresholds["blur"] + f = df["folding_overlap"] <= thresholds["folding"] + r = df["residual_overlap"] <= thresholds["residual"] + + independent = { + "tissue": n_candidates - n_tissue, + "blur": int(b.sum()), + "folding": int(f.sum()), + "residual": int(r.sum()), + } + # Sequential attribution in pipeline order: each gate credited only for tiles that survived + # every earlier gate (tissue already applied by construction of this dataset). + sequential = { + "tissue": n_candidates - n_tissue, + "blur": int(b.sum()), + "folding": int((~b & f).sum()), + "residual": int((~b & ~f & r).sum()), + } + kept_final = int((~b & ~f & ~r).sum()) + + def pct(n: int) -> float: + return round(100.0 * n / n_candidates, 3) if n_candidates else 0.0 + + print("\n[DIAGNOSTICS] Tile drop funnel") + print(f" candidate tiles (pre-filter): {n_candidates:,}") + print(f" survived tissue gate: {n_tissue:,}") + print(" independent drops (each gate alone; QC gates among tissue survivors):") + for k in ("tissue", "blur", "folding", "residual"): + print( + f" {k:9} {independent[k]:>12,} ({pct(independent[k]):5.2f}% of candidates)" + ) + print(" sequential drops (pipeline order; mutually exclusive):") + for k in ("tissue", "blur", "folding", "residual"): + print( + f" {k:9} {sequential[k]:>12,} ({pct(sequential[k]):5.2f}% of candidates)" + ) + print( + f" kept (survived all gates): {kept_final:,} ({pct(kept_final):5.2f}%)" + ) + + metrics = { + "diag_candidate_tiles": n_candidates, + "diag_tissue_survivors": n_tissue, + "diag_kept_final": kept_final, + } + for k in ("tissue", "blur", "folding", "residual"): + metrics[f"diag_independent_drop_{k}"] = independent[k] + metrics[f"diag_sequential_drop_{k}"] = sequential[k] + logger.log_metrics(metrics) + + def tiling( row: dict[str, Any], tissue_masks_path: str | None, qc_masks_path: str | None, epithelium_masks_path: str | None, + cancer_masks_path: str | None, ) -> list[dict[str, Any]]: """Generate tile coordinates for a single slide.""" slide_path = Path(row["path"]) @@ -120,6 +287,10 @@ def tiling( else "" ) + cancer_mask = ( + str(Path(cancer_masks_path) / tiff_slide_name) if cancer_masks_path else "" + ) + if qc_masks_path: qc_path = Path(qc_masks_path) blur_mask = str(qc_path / f"Piqe_piqe_median_activity_mask_{tiff_slide_name}") @@ -141,6 +312,7 @@ def tiling( "mpp_y": row["mpp_y"], "tissue_mask_path": tissue_mask, "epithelium_mask_path": epithelium_mask, + "cancer_mask_path": cancer_mask, "blur_mask_path": blur_mask, "folding_mask_path": folding_mask, "residual_mask_path": residual_mask, @@ -165,46 +337,61 @@ def tiling( @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: if not ray.is_initialized(): - ray.init() + ray.init(**_ray_init_kwargs()) tissue_masks_path = ( None if config.dataset.mlflow_uris.tissue_masks is None else download_artifacts(config.dataset.mlflow_uris.tissue_masks) ) - epithelium_masks_path = ( - None - if config.dataset.mlflow_uris.epithelium_masks is None - else download_artifacts(config.dataset.mlflow_uris.epithelium_masks) + # Epithelium masks are generated into persistent project storage, so use them in + # place just like the QC masks rather than trying to download an MLflow artifact. + epithelium_masks_path = config.dataset.paths.get("epithelium_masks") or None + cancer_masks_uri = config.dataset.mlflow_uris.get("cancer_masks") + cancer_masks_path = ( + None if cancer_masks_uri is None else download_artifacts(cancer_masks_uri) ) qc_masks_path = config.dataset.paths.qc_masks or None # Read slide paths from data mapping slide_labels_df = pd.read_csv(config.dataset.paths.data_mapping) - slide_labels_df["slide_path"] = slide_labels_df["path"] + ".mrxs" - slide_paths = slide_labels_df["slide_path"].tolist() - metadata_ds = rd.from_pandas( - slide_labels_df[["slide_path", "type", "mammaprint_index"]] - ) + slide_paths = slide_labels_df["path"].tolist() print(f"\n[INFO] Processing {len(slide_paths)} slides using Ray Data pipeline\n") + # Attach labels (type, mammaprint_index) by path. This is a tiny per-slide lookup + # (~2k rows) that we already hold in the driver, so we broadcast it as a dict and merge + # in a map_batches rather than a Ray hash-shuffle join. The distributed join was pure + # overhead here -- worse, its shuffle produced a wildly inflated per-task memory estimate + # (~1 TiB/task) that no node could satisfy, so the autoscaler rejected it forever and the + # whole pipeline deadlocked at Join: 0/1 before row_hash/tiling ever ran. + labels_by_path = slide_labels_df.set_index("path")[ + ["type", "mammaprint_index"] + ].to_dict("index") + + def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: + labels = batch["path"].map(labels_by_path) + batch["type"] = [lp["type"] if isinstance(lp, dict) else None for lp in labels] + batch["mammaprint_index"] = [ + lp["mammaprint_index"] if isinstance(lp, dict) else None for lp in labels + ] + return batch + # Read slides slides = read_slides( slide_paths, mpp=config.mpp, tile_extent=config.tile_extent, stride=config.stride, - ).join( - metadata_ds, - on=("path",), - right_on=("slide_path",), - num_partitions=16, - join_type="left_outer", - ) + ).map_batches(attach_labels, batch_format="pandas", num_cpus=0.1) - # Create unique slide IDs and save slide metadata - slides = slides.map(row_hash, num_cpus=0.1, memory=128 * 1024**2) + # Create unique slide IDs and save slide metadata. + # Fractional num_cpus so these cheap coordinate-generation stages don't win Ray's + # per-operator CPU reservation. Ray's ReservationOpResourceAllocator prioritizes upstream + # operators, so without a cap row_hash/tiling grabbed ~22 of 48 CPUs and starved the + # expensive downstream overlap+filter stage to a single task while its input queue backed + # up. Keeping these sub-1.0 leaves the CPU budget for the mask-reading UDFs (num_cpus=1). + slides = slides.map(row_hash, num_cpus=0.1) # Expand slides into tile coordinates tiles = slides.flat_map( @@ -213,9 +400,9 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: tissue_masks_path=tissue_masks_path, qc_masks_path=qc_masks_path, epithelium_masks_path=epithelium_masks_path, + cancer_masks_path=cancer_masks_path, ), - num_cpus=0.2, - memory=BATCH_SIZE * 1024**2, + num_cpus=0.25, ).repartition(target_num_rows_per_block=BATCH_SIZE) # Compute masks and filter tissue tiles @@ -226,27 +413,132 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: # Full tile full_roi = box(0, 0, config.tile_extent, config.tile_extent) - # Add tissue overlaps - tiles = add_tile_overlap( - tiles, tissue_roi, "tissue_mask_path", "tissue_overlap", "0" - ) - tiles = tiles.filter(expr=col("tissue_overlap") > 0.0) - tiles = add_tile_overlap(tiles, full_roi, "blur_mask_path", "blur_overlap", "255") - tiles = add_tile_overlap( - tiles, full_roi, "folding_mask_path", "folding_overlap", "255" - ) + # Add coverage scores (all in [0, 1], higher = keep). The masks are downsampled continuous + # [0, 255] where 0 = clean/background and any non-zero = foreground/artifact (per the QC docs). + # tissue/epithelium keep the non-zero (foreground) fraction; the QC artifact masks keep the + # zero (clean) fraction. See scripts/notebooks/mask_coverage_threshold.ipynb for the evidence. + # + # We compute each overlap and immediately filter on it, rather than computing all overlaps up + # front and filtering at the end. The final result is identical (the keep set is the AND of all + # thresholds), but interleaving means a tile discarded by an early gate never pays for the + # remaining mask reads. Only ~2.5% of tiles survive, so this roughly halves total overlap work. + # Thresholds come from the experiment config (see configs/experiment/preprocessing/tiling/*.yaml) + # and the coverage distributions in scripts/notebooks/mask_coverage_threshold.ipynb: a moderate + # tissue gate, and lenient artifact-only QC cutoffs (they only remove damaged tissue). + # + # Order matters for speed (not for the result): the tissue gate runs first because it is both the + # most selective and the cheapest (center-50% ROI = smallest region read). tiles = add_tile_overlap( - tiles, full_roi, "residual_mask_path", "residual_overlap", "0" - ) - tiles = add_tile_overlap( - tiles, full_roi, "epithelium_mask_path", "epithelium_overlap", "0" + tiles, tissue_roi, "tissue_mask_path", "tissue_overlap", keep="nonzero" ) + tiles = tiles.filter(expr=col("tissue_overlap") > config.tissue_threshold) + + if config.get("diagnostics", False): + # Diagnostic path: quantify per-gate drops. Compute ALL three QC overlaps on the tissue + # survivors WITHOUT filtering (so dropped tiles' scores are retained), materialize once so + # the funnel scan and the subsequent production filters both reuse the same pinned blocks + # (no re-reading of masks), log the funnel, then apply the identical QC filters below so the + # written dataset is byte-identical to a normal run. + tiles = add_tile_overlap( + tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" + ) + tiles = add_tile_overlap( + tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" + ) + tiles = add_tile_overlap( + tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" + ) + tiles = tiles.materialize() + + n_candidates = count_candidate_tiles(slides.to_pandas()) + thresholds = { + "tissue": config.tissue_threshold, + "blur": config.blur_threshold, + "folding": config.folding_threshold, + "residual": config.residual_threshold, + } + log_drop_funnel(tiles, n_candidates, thresholds, logger) + + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) + else: + tiles = add_tile_overlap( + tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + + tiles = add_tile_overlap( + tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + + tiles = add_tile_overlap( + tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) + + # Epithelium is computed last so it only runs on tissue/QC survivors. Discovery runs leave the + # threshold unset to preserve the complete score distribution; production runs can set a + # tile-level coverage cutoff once the desired epithelial purity has been selected. + if epithelium_masks_path: + print("[INFO] Computing overlap: epithelium_overlap from epithelium_mask_path") + tiles = add_tile_overlap( + tiles, + full_roi, + "epithelium_mask_path", + "epithelium_overlap", + keep="nonzero", + ) + epithelium_threshold = config.get("epithelium_threshold") + if epithelium_threshold is None: + print( + "[INFO] Epithelium threshold is unset; recording overlap without " + "filtering" + ) + else: + print(f"[INFO] Filtering epithelium_overlap > {epithelium_threshold}") + tiles = tiles.filter(expr=col("epithelium_overlap") > epithelium_threshold) + else: + print( + "[INFO] Skipping epithelium overlap: " + "dataset.paths.epithelium_masks is not set" + ) + tiles = tiles.with_column("epithelium_overlap", lit(float("nan"))) + + # Cancer heatmaps store sigmoid probabilities as uint8 intensity (probability * 255). + # Discovery mode records the mean normalized probability without filtering. + # Once a cutoff has been selected, setting cancer_threshold applies the production gate. + if cancer_masks_path: + print("[INFO] Computing overlap: cancer_overlap from cancer_mask_path") + tiles = add_tile_overlap( + tiles, + full_roi, + "cancer_mask_path", + "cancer_overlap", + keep="mean", + ) + cancer_threshold = config.get("cancer_threshold") + if cancer_threshold is None: + print( + "[INFO] Cancer threshold is unset; recording overlap without filtering" + ) + else: + print(f"[INFO] Filtering cancer_overlap > {cancer_threshold}") + tiles = tiles.filter(expr=col("cancer_overlap") > cancer_threshold) + else: + print( + "[INFO] Skipping cancer overlap: " + "dataset.mlflow_uris.cancer_masks is not set" + ) + tiles = tiles.with_column("cancer_overlap", lit(float("nan"))) # Drop unnecessary columns tiles = tiles.drop_columns( [ "tissue_mask_path", "epithelium_mask_path", + "cancer_mask_path", "blur_mask_path", "folding_mask_path", "residual_mask_path", @@ -255,6 +547,12 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: ] ) + # Rename tile coordinates to the schema mlkit's OpenSlideTilesDataset expects. + # The ratiopath overlap helpers (tile_overlay_overlap) require "tile_x"/"tile_y", so those + # names are kept through the whole pipeline above; mlkit's OpenSlideTilesDataset.__getitem__ + # reads tile["x"]/tile["y"], so we rename only now, right before writing. + tiles = tiles.rename_columns({"tile_x": "x", "tile_y": "y"}) + # Convert Ray Datasets to pandas DataFrames slides_df = slides.to_pandas() tiles_df = tiles.to_pandas() diff --git a/preprocessing/tissue_masks.py b/preprocessing/tissue_masks.py index 59a1aa72..fe45213c 100644 --- a/preprocessing/tissue_masks.py +++ b/preprocessing/tissue_masks.py @@ -35,7 +35,7 @@ def process_slide(slide_path: Path, level: int, output_path: Path) -> None: @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: df = pd.read_csv(config.dataset.paths.data_mapping) - slides = [Path(path + ".mrxs") for path in df["path"]] + slides = [Path(path) for path in df["path"]] with TemporaryDirectory() as output_dir: process_items( diff --git a/scripts/ml/list_pending_gpu_jobs.py b/scripts/ml/list_pending_gpu_jobs.py new file mode 100644 index 00000000..79aadc80 --- /dev/null +++ b/scripts/ml/list_pending_gpu_jobs.py @@ -0,0 +1,138 @@ +"""List this user's Kubernetes jobs that are waiting for a GPU. + +The GPU request is read from the pod spec created by ``kube_jobs.submit_job``. +This script is read-only. + +Usage: + uv run python scripts/ml/list_pending_gpu_jobs.py + uv run python scripts/ml/list_pending_gpu_jobs.py --prefix mammaprint-sweep- + uv run python scripts/ml/list_pending_gpu_jobs.py --user kissmi +""" + +import argparse +from collections import Counter +from collections.abc import Iterator, Mapping + +from kube_jobs.constants import CLUSTER, JOB_NAMESPACE +from kubernetes import client +from kubernetes.config import load_kube_config + + +DEFAULT_USER = "kissmi" +GPU_PRODUCTS = { + "NVIDIA-A100-80GB-PCIe": "A100", + "NVIDIA-A40": "A40", + "NVIDIA-H100-NVL": "H100", + "Tesla-P100-SXM2-16GB": "Tesla P100", +} + + +def balanced_gpu_iterator(waiting_counts: Mapping[str, int]) -> Iterator[str]: + """Yield GPUs forever, first bringing their waiting counts to the same level. + + Ties follow the mapping's insertion order. The input mapping is not modified. + Include every GPU that may be selected, using a count of zero when its queue is + empty. + """ + if not waiting_counts: + raise ValueError("At least one GPU is required.") + if any(count < 0 for count in waiting_counts.values()): + raise ValueError("Waiting counts cannot be negative.") + + counts = dict(waiting_counts) + while True: + lowest_count = min(counts.values()) + for gpu in counts: + if counts[gpu] == lowest_count: + counts[gpu] += 1 + yield gpu + + +def _gpu_name(pod: client.V1Pod) -> str: + limits = { + resource: quantity + for container in pod.spec.containers + for resource, quantity in (container.resources.limits or {}).items() + } + + for resource in sorted(limits): + if resource.startswith("nvidia.com/mig-"): + return resource.removeprefix("nvidia.com/") + + if "nvidia.com/gpu" in limits: + selectors = pod.spec.node_selector or {} + product = selectors.get("nvidia.com/gpu.product", "GPU") + return GPU_PRODUCTS.get(product, product) + + return "none" + + +def _pending_gpu_jobs(user: str, prefix: str | None) -> list[tuple[str, str]]: + load_kube_config(context=CLUSTER) + jobs = ( + client.BatchV1Api() + .list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={user}", + ) + .items + ) + job_names_by_uid = { + job.metadata.uid: job.metadata.name + for job in jobs + if job.metadata is not None + and job.metadata.uid is not None + and (prefix is None or job.metadata.name.startswith(prefix)) + } + + pending = [] + pods = client.CoreV1Api().list_namespaced_pod(namespace=JOB_NAMESPACE).items + for pod in pods: + if pod.status is None or pod.status.phase != "Pending": + continue + owner_uids = { + owner.uid for owner in (pod.metadata.owner_references or []) if owner.uid + } + job_uid = next((uid for uid in owner_uids if uid in job_names_by_uid), None) + if job_uid is not None: + pending.append((_gpu_name(pod), job_names_by_uid[job_uid])) + + return sorted(pending) + + +def pending_gpu_counts(user: str, prefix: str | None = None) -> Counter[str]: + return Counter(gpu for gpu, _ in _pending_gpu_jobs(user, prefix)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--user", + default=DEFAULT_USER, + help=f"Value of the created_by job label (default: {DEFAULT_USER}).", + ) + parser.add_argument( + "--prefix", + help="Only show job names starting with this prefix.", + ) + args = parser.parse_args() + + pending = _pending_gpu_jobs(args.user, args.prefix) + if not pending: + print("No pending jobs found.") + return 0 + + counts = Counter(gpu for gpu, _ in pending) + print( + "Waiting by GPU: " + + ", ".join(f"{gpu}={count}" for gpu, count in sorted(counts.items())) + ) + print(f"\n{'GPU':<16} JOB") + print("-" * 80) + for gpu, job_name in pending: + print(f"{gpu:<16} {job_name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ml/retry_failed_jobs.py b/scripts/ml/retry_failed_jobs.py new file mode 100644 index 00000000..92982ee7 --- /dev/null +++ b/scripts/ml/retry_failed_jobs.py @@ -0,0 +1,237 @@ +"""Log this user's training jobs, retry the most-recently-failed one, wait, repeat. + +The jobs submitted by ``run_train_sweep*.py`` set ``backoff_limit=0`` — one pod +failure marks the whole Job ``Failed`` with no automatic retry. This helper: + + 1. lists your jobs (label ``created_by=``) with their status, + 2. finds the most recent FAILED job, + 3. re-creates it under a bumped name (its own manifest, so image / resources / + command are preserved exactly — works for any job, transformer or spatial), + 4. sleeps ``WAIT_MINUTES``, then loops. + +A failed job whose name ends in ``-`` is retried as ``-``; otherwise ``-2`` +is appended. The old failed Job is deleted first (its name is a unique key, so a +re-create with the same name would 409). Kubernetes' own TTL may already have +removed it — that's fine. + +Usage: + uv run python scripts/ml/retry_failed_jobs.py # loop forever, 30 min + uv run python scripts/ml/retry_failed_jobs.py --once # one pass, no wait + uv run python scripts/ml/retry_failed_jobs.py --wait 30 # 30-minute cadence + uv run python scripts/ml/retry_failed_jobs.py --dry-run # log + show plan, no changes + +Reads/writes only jobs labelled with your username. Requires a working kubectl +context (the same one kube_jobs submits to). +""" + +import argparse +import copy +import re +import time + +from kube_jobs.constants import CLUSTER, JOB_NAMESPACE +from kubernetes import client +from kubernetes.config import load_kube_config + + +USER = "kissmi" # created_by label — the jobs this script is allowed to touch +WAIT_MINUTES = 30 +# Only consider these when picking "the last failed job", so we never resurrect an +# unrelated failure (e.g. a preprocessing job). Narrow this if you want. +NAME_PREFIXES = ( + "mammaprint-train-", + "mammaprint-sweep-", + "mp-doe36-", + "mp-doe48-", +) + + +def _batch_api() -> client.BatchV1Api: + load_kube_config(context=CLUSTER) + return client.BatchV1Api() + + +def _is_failed(job: client.V1Job) -> bool: + return bool(job.status and job.status.failed) + + +def _is_active(job: client.V1Job) -> bool: + return bool(job.status and job.status.active) + + +def _is_succeeded(job: client.V1Job) -> bool: + return bool(job.status and job.status.succeeded) + + +def _status_str(job: client.V1Job) -> str: + if _is_active(job): + return "ACTIVE" + if _is_succeeded(job): + return "SUCCEEDED" + if _is_failed(job): + return "FAILED" + return "UNKNOWN" + + +def list_jobs(batch: client.BatchV1Api) -> list[client.V1Job]: + """This user's jobs, newest first (by start time).""" + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, label_selector=f"created_by={USER}" + ).items + + def start_key(j: client.V1Job) -> str: + st = j.status.start_time if j.status else None + return st.isoformat() if st else "" + + return sorted(jobs, key=start_key, reverse=True) + + +def log_jobs(jobs: list[client.V1Job]) -> None: + print(f"\n{'STATUS':<10} {'START':<22} NAME") + print("-" * 90) + for j in jobs: + st = j.status.start_time if j.status else None + start = st.strftime("%Y-%m-%d %H:%M:%S") if st else "-" + print(f"{_status_str(j):<10} {start:<22} {j.metadata.name}") + + +def _bump_name(name: str) -> str: + """`...-bce-1` -> `...-bce-2`; a name without a trailing `-` gets `-2`.""" + m = re.match(r"^(.*)-(\d+)$", name) + if m: + return f"{m.group(1)}-{int(m.group(2)) + 1}" + return f"{name}-2" + + +def _retry_manifest(job: client.V1Job, new_name: str) -> client.V1Job: + """Copy a Job's spec into a fresh Job named `new_name`, stripping runtime fields. + + Kubernetes injects immutable fields into a running Job's pod template (the + `controller-uid`/`job-name` selector labels and `batch.kubernetes.io/*`). Those + must be removed or the create is rejected — we rebuild a clean metadata/spec and + carry over only what we set at submit time. + """ + src = job.spec + tmpl = copy.deepcopy(src.template) + + # Drop the auto-injected selector labels; keep our own app/created_by labels. + labels = dict(tmpl.metadata.labels or {}) if tmpl.metadata else {} + for k in list(labels): + if k.startswith("batch.kubernetes.io/") or k in {"controller-uid", "job-name"}: + del labels[k] + labels["app"] = new_name + tmpl.metadata = client.V1ObjectMeta(labels=labels) + + # Rename the container so its name matches the new job (cosmetic, matches submit). + for c in tmpl.spec.containers: + c.name = new_name + + new_spec = client.V1JobSpec( + backoff_limit=src.backoff_limit, + ttl_seconds_after_finished=src.ttl_seconds_after_finished, + template=tmpl, + ) + return client.V1Job( + api_version="batch/v1", + kind="Job", + metadata=client.V1ObjectMeta(name=new_name, labels={"created_by": USER}), + spec=new_spec, + ) + + +def last_failed(jobs: list[client.V1Job]) -> client.V1Job | None: + for j in jobs: + if j.metadata.name.startswith(NAME_PREFIXES) and _is_failed(j): + return j + return None + + +def retry_last_failed( + batch: client.BatchV1Api, jobs: list[client.V1Job], dry_run: bool +) -> str | None: + """Delete the newest failed job and re-create it under a bumped name. + + Returns the new job name, or None if there was nothing to retry. + """ + target = last_failed(jobs) + if target is None: + print("\nNo failed job to retry. ✅") + return None + + old = target.metadata.name + new = _bump_name(old) + print(f"\nLast failed job: {old} -> retry as: {new}") + + if dry_run: + print( + "[dry-run] would delete the old job and create the retry; no changes made." + ) + return new + + manifest = _retry_manifest(target, new) + + # Delete the old failed Job so its name is free (may already be TTL-collected). + try: + batch.delete_namespaced_job( + name=old, namespace=JOB_NAMESPACE, propagation_policy="Background" + ) + print(f"Deleted old failed job {old}.") + except client.ApiException as e: + if e.status == 404: + print(f"Old job {old} already gone (TTL); nothing to delete.") + else: + raise + + try: + resp = batch.create_namespaced_job(body=manifest, namespace=JOB_NAMESPACE) + print(f"Retry submitted: {resp.metadata.name} 🚀") + return resp.metadata.name + except client.ApiException as e: + print(f"Retry creation failed: {e.reason} ({e.status})") + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--wait", + type=float, + default=WAIT_MINUTES, + help=f"Minutes to wait between passes (default {WAIT_MINUTES}).", + ) + parser.add_argument( + "--once", + action="store_true", + help="Run a single pass (log + retry) and exit, no waiting.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Log jobs and show the retry plan without changing anything.", + ) + args = parser.parse_args() + + batch = _batch_api() + pass_num = 0 + while True: + pass_num += 1 + print(f"\n{'=' * 90}\nPASS {pass_num}\n{'=' * 90}") + jobs = list_jobs(batch) + log_jobs(jobs) + retry_last_failed(batch, jobs, dry_run=args.dry_run) + + if args.once: + break + + wait_s = args.wait * 60 + print(f"\nSleeping {args.wait:g} min before the next pass (Ctrl-C to stop)...") + try: + time.sleep(wait_s) + except KeyboardInterrupt: + print("\nStopped.") + break + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ml/run_train.py b/scripts/ml/run_train.py new file mode 100644 index 00000000..88c16942 --- /dev/null +++ b/scripts/ml/run_train.py @@ -0,0 +1,22 @@ +from kube_jobs import storage, submit_job + + +submit_job( + job_name="mammaprint-train-mil-embeddings", + username="kissmi", + image="cerit.io/rationai/base:2.0.6", + cpu=24, + memory="64Gi", + gpu="A40", + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + # "git checkout ", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "export HF_TOKEN=", + "uv sync --frozen", + "uv run -m ml.train +experiment=ml/train_mil_embeddings", + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], +) diff --git a/scripts/ml/run_train_d_optimal_sweep.py b/scripts/ml/run_train_d_optimal_sweep.py new file mode 100644 index 00000000..c8939f6a --- /dev/null +++ b/scripts/ml/run_train_d_optimal_sweep.py @@ -0,0 +1,324 @@ +"""Submit the configured D-optimal mask-comparison training sweep. + +Edit the settings block below, then run this file without command-line flags. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shlex +import time +from dataclasses import dataclass +from pathlib import Path + +from kube_jobs import storage, submit_job + + +EXPERIMENTS = { + "bce": "ml/train_mil_embeddings", + "regression": "ml/train_mil_embeddings_regression", + "joint": "ml/train_mil_embeddings_joint", +} +MASK_CONFIGS = { + "tissue_only": ("tissue_only", "tis"), + "cancer_mask_5": ("cancer_mask_5", "c05"), + "cancer_mask_9": ("cancer_mask_9", "c09"), + "epithel_and_cancer_5": ("epithel_and_cancer_5", "eac05"), + "epithel_or_cancer_5": ("epithel_or_cancer_5", "eoc05"), +} +DESIGN_DIR = Path("configs/sweeps/d_optimal_mask_comparison") +GIT_REF = "feat/tiling-values" +USERNAME = "kissmi" +IMAGE = "cerit.io/rationai/base:2.0.6" +POD_MLFLOW_TRACKING_URI = os.getenv( + "POD_MLFLOW_TRACKING_URI", "http://mlflow-s3.rationai-mlflow" +) +MLFLOW_LOOKUP_URI = os.getenv( + "MLFLOW_LOOKUP_URI", + "https://mlflow-s3.rationai.cloud.trusted.e-infra.cz", +) +MLFLOW_EXPERIMENT = "MammaPrint" + +# Sweep settings. Edit these values before running this file. +DESIGN_SIZE = 48 +DATA_MAPPING: str | None = None # e.g. "/mnt/projects/mammaprint/my_mapping.csv" +REPEATS = 20 +SEED_START = 0 +GPUS = ("H100", "A40") +SUBMISSION_INTERVAL_MINUTES = 30.0 +MAX_JOBS: int | None = None +DRY_RUN = False + + +@dataclass(frozen=True) +class SweepJob: + design_row: int + level: int + head: str + aggregator: str + loss: str + mask_type: str + data_config: str + job_token: str + + +@dataclass(frozen=True) +class WorkItem: + job: SweepJob + repeat: int + seed: int + + +def _validate_settings() -> str: + if DESIGN_SIZE not in (36, 48): + raise ValueError(f"DESIGN_SIZE must be 36 or 48, got {DESIGN_SIZE}.") + if DATA_MAPPING is None: + raise ValueError( + "Set DATA_MAPPING in the settings block to a path visible inside the pod." + ) + if not GPUS: + raise ValueError("GPUS must contain at least one GPU type.") + unexpected_gpus = sorted(set(GPUS) - {"H100", "A40"}) + if unexpected_gpus: + raise ValueError(f"Unsupported GPUs: {unexpected_gpus}") + if SUBMISSION_INTERVAL_MINUTES < 0: + raise ValueError("SUBMISSION_INTERVAL_MINUTES cannot be negative.") + if MAX_JOBS is not None and MAX_JOBS <= 0: + raise ValueError("MAX_JOBS must be positive or None.") + return DATA_MAPPING + + +def _load_jobs(design_size: int) -> list[SweepJob]: + path = DESIGN_DIR / f"d_optimal_{design_size}_combinations.csv" + if not path.is_file(): + raise FileNotFoundError( + f"Missing design table {path}; run `uv run python main.py`." + ) + + with path.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + if len(rows) != design_size: + raise RuntimeError(f"Expected {design_size} rows in {path}, found {len(rows)}.") + + jobs = [] + for design_row, row in enumerate(rows, start=1): + loss = row["loss"] + if loss not in EXPERIMENTS: + raise ValueError(f"Unknown loss {loss!r} in {path}.") + mask_type = row["mask_type"] + try: + data_config, job_token = MASK_CONFIGS[mask_type] + except KeyError as error: + raise ValueError(f"Unknown mask type {mask_type!r} in {path}.") from error + jobs.append( + SweepJob( + design_row=design_row, + level=int(row["level"]), + head=row["head"], + aggregator=row["aggregator"], + loss=loss, + mask_type=mask_type, + data_config=data_config, + job_token=job_token, + ) + ) + return jobs + + +def _mapping_id(data_mapping: str) -> str: + return hashlib.sha256(data_mapping.encode()).hexdigest()[:6] + + +def _work_items(jobs: list[SweepJob], repeats: int, seed_start: int) -> list[WorkItem]: + if repeats <= 0: + raise ValueError(f"REPEATS must be positive, got {repeats}.") + return [ + WorkItem(job=job, repeat=repeat, seed=seed_start + repeat) + for repeat in range(repeats) + for job in jobs + ] + + +def _job_name(item: WorkItem, design_size: int, mapping_id: str) -> str: + job = item.job + name = ( + f"mp-doe{design_size}-{job.job_token}-l{job.level}-" + f"{job.head}-{job.aggregator}-{job.loss}-s{item.seed}-m{mapping_id}" + ) + # Leave room for the retry helper's '-2', '-3', ... suffixes. + if len(name) > 60: + raise ValueError(f"Kubernetes job name leaves no retry suffix room: {name}") + return name + + +def _gpu_for(item: WorkItem, gpus: tuple[str, ...]) -> str: + # Rotate each configuration across devices between repetitions instead of + # permanently coupling a model configuration to one GPU type. + gpu_index = (item.job.design_row - 1 + item.repeat) % len(gpus) + return gpus[gpu_index] + + +def _validate_dataset_cards(jobs: list[SweepJob]) -> None: + repo_root = Path(__file__).resolve().parents[2] + checked = set() + for job in jobs: + data_override = f"{job.data_config}/l{job.level}" + if data_override in checked: + continue + checked.add(data_override) + card = repo_root / "configs" / "data" / "embedded" / f"{data_override}.yaml" + if not card.is_file(): + raise FileNotFoundError(f"Missing dataset card: {card}") + text = card.read_text() + missing = [ + split + for split in ("all", "train", "val", "test") + if f" {split}:" not in text + ] + if missing or "" in text: + problem = ", ".join(missing) if missing else "placeholder URI" + raise RuntimeError(f"Dataset card {card} is incomplete: {problem}.") + + +def _existing_mlflow_run_names(expected_names: set[str], prefix: str) -> set[str]: + from mlflow import MlflowClient + + client = MlflowClient(tracking_uri=MLFLOW_LOOKUP_URI) + experiment = client.get_experiment_by_name(MLFLOW_EXPERIMENT) + if experiment is None: + return set() + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + filter_string=f"attributes.run_name LIKE '{prefix}%'", + max_results=50_000, + ) + return {run.info.run_name for run in runs if run.info.run_name in expected_names} + + +def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: + from kube_jobs.constants import CLUSTER, JOB_NAMESPACE + from kubernetes import client + from kubernetes.config import load_kube_config + + load_kube_config(context=CLUSTER) + batch = client.BatchV1Api() + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={USERNAME}", + ).items + existing = set() + for job in jobs: + if job.metadata is None: + continue + name = job.metadata.name + match = re.match(r"^(.*)-(\d+)$", name) + base_name = match.group(1) if match else name + if base_name in expected_names: + existing.add(base_name) + return existing + + +def _submit( + item: WorkItem, + design_size: int, + gpu: str, + mapping_id: str, + data_mapping: str, +) -> None: + job = item.job + name = _job_name(item, design_size, mapping_id) + experiment = EXPERIMENTS[job.loss] + data_override = f"{job.data_config}/l{job.level}" + data_mapping_override = ( + f" {shlex.quote(f'dataset.data_mapping={json.dumps(data_mapping)}')}" + ) + submit_job( + job_name=name, + username=USERNAME, + image=IMAGE, + cpu=16, + memory="48Gi", + gpu=gpu, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_REF}", + f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", + "uv sync --frozen", + f""" + uv run -m ml.train +experiment={experiment} \ + data/embedded={data_override} \ + ml/aggregator={job.aggregator} \ + ml/head={job.head} \ + seed={item.seed}{data_mapping_override} \ + test_after_fit=false \ + metadata.run_name={name} \ + +logger.tags.data_variant={job.mask_type} \ + +logger.tags.d_optimal_design=d_optimal_{design_size} \ + +logger.tags.d_optimal_design_row=row_{job.design_row} \ + +logger.tags.repeat=repeat_{item.repeat + 1} \ + +logger.tags.data_mapping_id=map_{mapping_id} + """, + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + + +def main() -> None: + data_mapping = _validate_settings() + jobs = _load_jobs(DESIGN_SIZE) + _validate_dataset_cards(jobs) + items = _work_items(jobs, REPEATS, SEED_START) + mapping_id = _mapping_id(data_mapping) + expected_names = {_job_name(item, DESIGN_SIZE, mapping_id) for item in items} + prefix = f"mp-doe{DESIGN_SIZE}-" + + if DRY_RUN: + print( + f"D-optimal design: {len(jobs)} configurations x {REPEATS} seeds " + f"= {len(items)} jobs" + ) + print(f"GPUs: {', '.join(GPUS)}") + print(f"Interval: {SUBMISSION_INTERVAL_MINUTES:g} minutes") + print(f"Data mapping: {data_mapping}") + print(f"Data mapping ID: {mapping_id}") + for item in items: + gpu = _gpu_for(item, GPUS) + print(f"{_job_name(item, DESIGN_SIZE, mapping_id)} -> {gpu}") + return + + kubernetes_names = _existing_kubernetes_job_names(expected_names) + mlflow_names = _existing_mlflow_run_names(expected_names, prefix) + existing_names = kubernetes_names | mlflow_names + pending = [ + item + for item in items + if _job_name(item, DESIGN_SIZE, mapping_id) not in existing_names + ] + if MAX_JOBS is not None: + pending = pending[:MAX_JOBS] + + print( + f"Design has {len(items)} jobs; {len(existing_names)} already exist and " + f"{len(pending)} will be submitted now." + ) + interval_seconds = SUBMISSION_INTERVAL_MINUTES * 60 + for pending_index, item in enumerate(pending): + if pending_index > 0: + print( + f"Waiting {SUBMISSION_INTERVAL_MINUTES:g} minutes before the next job." + ) + time.sleep(interval_seconds) + gpu = _gpu_for(item, GPUS) + name = _job_name(item, DESIGN_SIZE, mapping_id) + print(f"Submitting {name} to {gpu} ({pending_index + 1}/{len(pending)}).") + _submit(item, DESIGN_SIZE, gpu, mapping_id, data_mapping) + + +if __name__ == "__main__": + main() diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py new file mode 100644 index 00000000..6853f994 --- /dev/null +++ b/scripts/ml/run_train_sweep.py @@ -0,0 +1,475 @@ +"""Refill GPU waiting queues every five minutes until the sweep is submitted.""" + +import argparse +import itertools +import os +import time +from collections import deque +from collections.abc import Mapping +from heapq import heapify, heappop, heappush +from typing import TYPE_CHECKING, Any + +from kube_jobs import storage, submit_job + + +if TYPE_CHECKING or __package__: + from scripts.ml.list_pending_gpu_jobs import pending_gpu_counts +else: + from list_pending_gpu_jobs import pending_gpu_counts + + +# Each objective points to the matching training experiment. Edit these values, +# together with the dataset/model lists below, to define the Cartesian sweep. +EXPERIMENTS = { + "bce": "ml/train_mil_embeddings", + "regression": "ml/train_mil_embeddings_regression", + "joint": "ml/train_mil_embeddings_joint", +} +DATASETS = { + "tissue": "tissue_only", + "cm5": "cancer_mask_5", + "cm9": "cancer_mask_9", + "e025": "epithel", + "eor5": "epithel_or_cancer_5", + "ean5": "epithel_and_cancer_5", +} +LEVELS = (2, 3, 4, 5) +AGGREGATORS = ("mean", "max", "attention", "transformer") +HEADS = ("mlp", "linear") +SEEDS = tuple(range(20)) +GPUS = ( + "A40", + "H100", + "mig-2g.20gb", + "mig-1g.10gb", +) +POLL_INTERVAL_SECONDS = 5 * 60 +WAITING_JOB_TARGETS = {gpu: 1 if gpu == "H100" else 2 for gpu in GPUS} +BASIC_RESOURCES = {"cpu": 8, "memory": "16Gi"} + +# GPU rules limit which entries from GPUS a matching job may use. Jobs without +# a matching rule may use the complete pool. All assignments share one set of +# projected waiting counts, including assignments restricted to one GPU type. +GPU_RULES: list[dict[str, Any]] = [ + { + "match": { + "level": (2, 3), + "aggregator": ("transformer", "spatial_transformer"), + }, + "only": ("H100",), + }, + { + "match": { + "level": 4, + "aggregator": ("transformer", "spatial_transformer"), + }, + "exclude_gpus": ("mig-1g.10gb",), + }, +] + +# Override host resources independently of GPU placement. Rules are applied in +# order, so later matches override only the fields they specify. +RESOURCE_RULES: list[dict[str, Any]] = [ + { + "match": { + "level": (2, 3), + "aggregator": ("mean", "max"), + }, + "resources": {"memory": "20Gi"}, + }, + { + "match": { + "level": (2, 3), + "aggregator": "attention", + }, + "resources": {"memory": "40Gi"}, + }, + { + "match": { + "level": (2, 3), + "aggregator": ("transformer", "spatial_transformer"), + }, + "resources": {"memory": "56Gi"}, + }, +] +GIT_REF = "feat/tiling-values" +USERNAME = "kissmi" +IMAGE = "cerit.io/rationai/base:2.0.6" +POD_MLFLOW_TRACKING_URI = os.getenv( + "POD_MLFLOW_TRACKING_URI", "http://mlflow-s3.rationai-mlflow" +) +MLFLOW_LOOKUP_URI = os.getenv( + "MLFLOW_LOOKUP_URI", + "https://mlflow-s3.rationai.cloud.trusted.e-infra.cz", +) +MLFLOW_EXPERIMENT = "MammaPrint" +JOB_NAME_PREFIX = "mammaprint-sweep-" + +SweepJob = tuple[int, str, str, int, str, str, str, str, int] +ResolvedJob = tuple[SweepJob, dict[str, Any]] + + +def _job_name( + dataset_key: str, + level: int, + head: str, + aggregator: str, + objective: str, + seed: int, +) -> str: + return f"{JOB_NAME_PREFIX}{dataset_key}-l{level}-{head}-{aggregator}-{objective}-s{seed}" + + +def _sweep_jobs() -> list[SweepJob]: + combinations = itertools.product( + SEEDS, + DATASETS.items(), + LEVELS, + EXPERIMENTS.items(), + AGGREGATORS, + HEADS, + ) + return [ + ( + job_index, + dataset_key, + dataset_config, + level, + objective, + experiment, + aggregator, + head, + seed, + ) + for job_index, ( + seed, + (dataset_key, dataset_config), + level, + (objective, experiment), + aggregator, + head, + ) in enumerate(combinations) + ] + + +def _matches(actual: object, expected: object) -> bool: + if isinstance(expected, (list, tuple, set, frozenset)): + return actual in expected + return actual == expected + + +def _job_matches(job: dict[str, object], conditions: dict[str, object]) -> bool: + return all(_matches(job.get(key), value) for key, value in conditions.items()) + + +def _eligible_gpus(job: dict[str, object]) -> tuple[str, ...]: + eligible = list(GPUS) + for rule in GPU_RULES: + if not _job_matches(job, rule.get("match", {})): + continue + if "only" in rule: + allowed = set(rule["only"]) + eligible = [gpu for gpu in eligible if gpu in allowed] + excluded = set(rule.get("exclude_gpus", ())) + eligible = [gpu for gpu in eligible if gpu not in excluded] + + if not eligible: + raise ValueError(f"No configured GPU can run job {job!r}.") + return tuple(eligible) + + +def _assign_gpu( + job: dict[str, object], + projected_waiting_counts: dict[str, int], + waiting_targets: Mapping[str, int] | None = None, +) -> str | None: + """Choose an eligible GPU with room in its queue; use GPUS order for ties.""" + eligible = _eligible_gpus(job) + if waiting_targets is not None: + eligible = tuple( + gpu + for gpu in eligible + if projected_waiting_counts[gpu] < waiting_targets[gpu] + ) + if not eligible: + return None + gpu = min(eligible, key=lambda candidate: projected_waiting_counts[candidate]) + projected_waiting_counts[gpu] += 1 + return gpu + + +def _resources_for_job( + projected_waiting_counts: dict[str, int], + dataset_key: str, + dataset_config: str, + level: int, + objective: str, + experiment: str, + aggregator: str, + head: str, + seed: int, + *, + waiting_targets: Mapping[str, int] | None = None, +) -> dict[str, Any] | None: + job = { + "dataset_key": dataset_key, + "dataset_config": dataset_config, + "level": level, + "objective": objective, + "experiment": experiment, + "aggregator": aggregator, + "head": head, + "seed": seed, + } + resources: dict[str, Any] = dict(BASIC_RESOURCES) + + for rule in RESOURCE_RULES: + matches = _job_matches(job, rule.get("match", {})) + excluded = "exclude" in rule and _job_matches(job, rule["exclude"]) + if matches and not excluded: + resources.update(rule["resources"]) + + gpu = _assign_gpu(job, projected_waiting_counts, waiting_targets) + if gpu is None: + return None + resources["gpu"] = gpu + return resources + + +def _balanced_job_order( + resolved_jobs: list[ResolvedJob], waiting_counts: Mapping[str, int] +) -> list[ResolvedJob]: + """Interleave GPU queues by their projected waiting-job counts.""" + queues: dict[str, deque[ResolvedJob]] = {} + first_job_index: dict[str, int] = {} + for list_index, resolved_job in enumerate(resolved_jobs): + gpu = resolved_job[1]["gpu"] + if not isinstance(gpu, str): + raise TypeError(f"Resolved GPU must be a string, got {gpu!r}.") + queues.setdefault(gpu, deque()).append(resolved_job) + first_job_index.setdefault(gpu, list_index) + + gpu_heap = [ + (waiting_counts.get(gpu, 0), first_job_index[gpu], gpu) for gpu in queues + ] + heapify(gpu_heap) + + ordered_jobs = [] + while gpu_heap: + projected_count, tie_breaker, gpu = heappop(gpu_heap) + ordered_jobs.append(queues[gpu].popleft()) + if queues[gpu]: + heappush(gpu_heap, (projected_count + 1, tie_breaker, gpu)) + return ordered_jobs + + +def _jobs_to_submit( + pending_jobs: list[SweepJob], + waiting_counts: Mapping[str, int], + max_jobs: int | None, +) -> tuple[list[ResolvedJob], int]: + """Fill only vacant waiting slots, assigning jobs afresh on every poll.""" + projected_counts = {gpu: waiting_counts.get(gpu, 0) for gpu in GPUS} + resolved_jobs: list[ResolvedJob] = [] + deferred_count = 0 + for job in pending_jobs: + try: + resources = _resources_for_job( + projected_counts, + *job[1:], + waiting_targets=WAITING_JOB_TARGETS, + ) + except ValueError as error: + if not str(error).startswith("No configured GPU can run job "): + raise + deferred_count += 1 + continue + if resources is not None: + resolved_jobs.append((job, resources)) + if max_jobs is not None and len(resolved_jobs) >= max_jobs: + break + return _balanced_job_order(resolved_jobs, waiting_counts), deferred_count + + +def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: + """Fetch sweep runs that already exist, including failed/running runs.""" + from mlflow import MlflowClient + + mlflow_client = MlflowClient(tracking_uri=MLFLOW_LOOKUP_URI) + experiment = mlflow_client.get_experiment_by_name(MLFLOW_EXPERIMENT) + if experiment is None: + print(f"MLflow experiment {MLFLOW_EXPERIMENT!r} does not exist yet.") + return set() + + runs = mlflow_client.search_runs( + experiment_ids=[experiment.experiment_id], + filter_string=f"attributes.run_name LIKE '{JOB_NAME_PREFIX}%'", + max_results=50_000, + ) + matching_runs = [run for run in runs if run.info.run_name in expected_names] + existing = {run.info.run_name for run in matching_runs} + print(f"Fetched {len(existing)} existing sweep run(s) from MLflow.") + return existing + + +def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: + """Fetch jobs that were submitted but may not have reached MLflow yet.""" + from kube_jobs.constants import CLUSTER, JOB_NAMESPACE + from kubernetes import client + from kubernetes.config import load_kube_config + + load_kube_config(context=CLUSTER) + batch = client.BatchV1Api() + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={USERNAME}", + ).items + matching_jobs = [ + job + for job in jobs + if job.metadata is not None and job.metadata.name in expected_names + ] + existing = {job.metadata.name for job in matching_jobs} + print(f"Fetched {len(existing)} existing sweep job(s) from Kubernetes.") + return existing + + +def _submit_sweep_job(job: SweepJob, resources: dict[str, Any]) -> None: + ( + _, + dataset_key, + dataset_config, + level, + objective, + experiment, + aggregator, + head, + seed, + ) = job + name = _job_name(dataset_key, level, head, aggregator, objective, seed) + print(f"Submitting {name} with {resources}.") + submit_job( + job_name=name, + username=USERNAME, + image=IMAGE, + cpu=resources["cpu"], + memory=resources["memory"], + gpu=resources["gpu"], + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_REF}", + f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", + # "export HF_TOKEN=", + "uv sync --frozen", + f""" + uv run -m ml.train +experiment={experiment} \ + data/embedded={dataset_config}/l{level} \ + ml/aggregator={aggregator} \ + ml/head={head} \ + seed={seed} \ + metadata.run_name={name} \ + +logger.tags.data_variant={dataset_config} \ + +logger.tags.level=level_{level} \ + +logger.tags.sweep_seed=seed_{seed} \ + """, + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + # kube_jobs catches creation errors internally, so a normal return alone + # does not mean the job was created. + if name not in _existing_kubernetes_job_names({name}): + raise RuntimeError(f"Kubernetes did not confirm submission of {name}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--max-jobs", + type=int, + help="Submit at most this many jobs in total, then exit.", + ) + parser.add_argument( + "--poll-interval-seconds", + type=int, + default=POLL_INTERVAL_SECONDS, + help="Seconds between queue refills (default: 300).", + ) + args = parser.parse_args() + if args.max_jobs is not None and args.max_jobs <= 0: + parser.error("--max-jobs must be a positive integer") + if args.poll_interval_seconds <= 0: + parser.error("--poll-interval-seconds must be a positive integer") + + jobs = _sweep_jobs() + expected_names = { + _job_name(dataset_key, level, head, aggregator, objective, seed) + for _, dataset_key, _, level, objective, _, aggregator, head, seed in jobs + } + submitted_names: set[str] = set() + print(f"GPU waiting targets: {WAITING_JOB_TARGETS}") + try: + while True: + try: + kubernetes_names = _existing_kubernetes_job_names(expected_names) + mlflow_names = _existing_mlflow_run_names(expected_names) + existing_names = mlflow_names | kubernetes_names | submitted_names + pending_jobs = [ + job + for job in jobs + if _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + not in existing_names + ] + print( + f"Sweep contains {len(jobs)} jobs: " + f"{len(existing_names)} already submitted, " + f"{len(pending_jobs)} left to submit." + ) + if not pending_jobs: + print("All sweep jobs have been submitted.") + return + + waiting_counts = pending_gpu_counts(USERNAME) + print( + "GPU waiting counts: " + f"{ {gpu: waiting_counts.get(gpu, 0) for gpu in GPUS} }" + ) + remaining_limit = ( + None + if args.max_jobs is None + else args.max_jobs - len(submitted_names) + ) + jobs_to_submit, deferred_count = _jobs_to_submit( + pending_jobs, waiting_counts, remaining_limit + ) + print(f"Submitting {len(jobs_to_submit)} job(s) to refill the queues.") + if deferred_count: + print( + f"Deferred {deferred_count} job(s) with no eligible " + "configured GPU." + ) + if deferred_count == len(pending_jobs): + print("No remaining jobs can use the configured GPU pool.") + return + + for job, resources in jobs_to_submit: + _submit_sweep_job(job, resources) + submitted_names.add( + _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + ) + if args.max_jobs is not None and len(submitted_names) >= args.max_jobs: + print(f"Reached the submission limit ({args.max_jobs}).") + return + except Exception as error: + # Refresh cluster state before retrying an uncertain submission. + print(f"Queue refill failed: {error}. Will retry after the interval.") + + print(f"Sleeping for {args.poll_interval_seconds} seconds.", flush=True) + time.sleep(args.poll_interval_seconds) + except KeyboardInterrupt: + print("\nSweep queue refiller stopped.") + + +if __name__ == "__main__": + main() diff --git a/scripts/notebooks/check_mask_coverage.ipynb b/scripts/notebooks/check_mask_coverage.ipynb new file mode 100644 index 00000000..3a98f321 --- /dev/null +++ b/scripts/notebooks/check_mask_coverage.ipynb @@ -0,0 +1,1329 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Check mask coverage and tiled datasets against data_mapping\n", + "\n", + "Verifies 1-1 mapping between slides in `data_mapping.csv` and:\n", + "- **MLflow masks** (downloaded from MLflow): tissue, epithelium\n", + "- **QC masks**: blur (`Piqe_piqe_median_activity_mask_`), folding (`FoldingFunction_folding_test_`), residual (`ResidualArtifactsAndCoverage_coverage_mask_`)\n", + "- **Slide files** listed in the `path` column: checks that every slide can be opened without error\n", + "- **Tiled datasets** (MLflow): checks `slides.parquet` ↔ `data_mapping` and `tiles.parquet` ↔ `slides.parquet` consistency" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "155f9f1e-5529-448d-8d5e-7c20208def37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmpxhfqrklg\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas mlflow>3 openslide-python tifffile matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mlflow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from mlflow.artifacts import download_artifacts, list_artifacts\n", + "from openslide import OpenSlide\n", + "from tifffile import TiffFile" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\n", + "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", + "\n", + "PROJECT_PATH = Path(\"/mnt/projects/mammaprint\")\n", + "DATA_MAPPING_CSV = PROJECT_PATH / \"data_mapping.csv\"\n", + "\n", + "# QC masks may live in several directories. They are treated as if everything\n", + "# lived in a single location: all listed dirs are merged together.\n", + "QC_MASKS_PATHS = [\n", + " PROJECT_PATH / \"qc_masks\",\n", + " PROJECT_PATH / \"qc_masks3\",\n", + " PROJECT_PATH / \"qc_masks_big_diff\",\n", + "]\n", + "\n", + "MLFLOW_MASKS = {\n", + " \"tissue\": \"mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\",\n", + " \"epithelium\": \"mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\",\n", + "}\n", + "\n", + "QC_MASK_PREFIXES = {\n", + " \"blur\": \"Piqe_piqe_median_activity_mask_\",\n", + " \"folding\": \"FoldingFunction_folding_test_\",\n", + " \"residual\": \"ResidualArtifactsAndCoverage_coverage_mask_\",\n", + "}\n", + "\n", + "TILED_DATASETS = {\n", + " \"mou_3_224\": \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\",\n", + " # \"mou_2_224\": \"mlflow-artifacts:/3/1ebe489fd3ca4f5682c9a4b3d6f74620/artifacts/mou_2_224\",\n", + "}\n", + "\n", + "EMBEDDINGS = {\n", + " \"embedding3\": \"mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "a55ee407", + "metadata": {}, + "source": [ + "# Functions\n", + "\n", + "All logic is defined below as individually-callable functions. Run every cell in this section once\n", + "to define them, then use the **Calls** section at the bottom to run whichever checks you need.\n", + "\n", + "For every mask source there are two modes:\n", + "- **list-only** (`ls`-equivalent): enumerate filenames without downloading (`list_mlflow_masks`,\n", + " `list_qc_masks`, `list_embeddings`).\n", + "- **download + open**: download artifacts and open each file to verify readability\n", + " (`download_mlflow_masks` + `open_masks`, `plot_sample_masks`)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-3", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B1. Data-mapping loader ----\n", + "def load_data_mapping(csv_path=DATA_MAPPING_CSV):\n", + " \"\"\"Load data_mapping.csv, derive expected .tiff names, return (df, expected_names).\"\"\"\n", + " df = pd.read_csv(csv_path)\n", + " df[\"tiff_name\"] = df[\"path\"].apply(lambda p: Path(p).with_suffix(\".tiff\").name)\n", + " expected_names = set(df[\"tiff_name\"])\n", + " print(f\"Loaded {len(df)} slides from {csv_path}\")\n", + " print(f\"Columns: {df.columns.tolist()}\")\n", + " print(f\"{len(expected_names)} unique slide tiff names expected\")\n", + " return df, expected_names" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-slide-open-check", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B2. Slide-open verification ----\n", + "SLIDE_SUFFIX_CANDIDATES = (\".mrxs\", \".tiff\", \".tif\")\n", + "\n", + "\n", + "def _slide_path_candidates(raw_path: str) -> list[Path]:\n", + " path = Path(raw_path)\n", + " candidates = [path]\n", + " if not path.suffix:\n", + " candidates.extend(Path(f\"{raw_path}{suffix}\") for suffix in SLIDE_SUFFIX_CANDIDATES)\n", + " return candidates\n", + "\n", + "\n", + "def _resolve_slide_path(raw_path: str) -> Path:\n", + " for candidate in _slide_path_candidates(raw_path):\n", + " if candidate.is_file():\n", + " return candidate\n", + " return Path(raw_path)\n", + "\n", + "\n", + "def _try_open_slide(raw_path: str) -> tuple[Path, str | None]:\n", + " slide_path = _resolve_slide_path(raw_path)\n", + " try:\n", + " str_path = str(slide_path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as slide:\n", + " _ = len(slide.pages)\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " _ = slide.dimensions\n", + " return slide_path, None\n", + " except Exception as e:\n", + " return slide_path, f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "def check_slides_open(df: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Try to open every slide file in df. Returns a DataFrame of failures (empty => all OK).\"\"\"\n", + " slide_open_rows = []\n", + " for row in df.itertuples(index=False):\n", + " opened_path, error = _try_open_slide(row.path)\n", + " if error:\n", + " slide_open_rows.append(\n", + " {\n", + " \"record_num\": row.record_num,\n", + " \"path\": row.path,\n", + " \"resolved_path\": str(opened_path),\n", + " \"error\": error,\n", + " }\n", + " )\n", + "\n", + " slide_open_errors = pd.DataFrame(slide_open_rows)\n", + " if slide_open_errors.empty:\n", + " print(f\"All {len(df)} slide files opened successfully.\")\n", + " else:\n", + " print(f\"{len(slide_open_errors)} slide files failed to open.\")\n", + " return slide_open_errors" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-5", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B3. MLflow masks: two modes ----\n", + "def list_mlflow_masks(mlflow_masks=MLFLOW_MASKS) -> dict[str, set[str]]:\n", + " \"\"\"LIST ONLY: enumerate MLflow mask filenames via list_artifacts, without downloading.\"\"\"\n", + " mlflow_files: dict[str, set[str]] = {}\n", + " for mask_name, uri in mlflow_masks.items():\n", + " found = {\n", + " Path(info.path).name\n", + " for info in list_artifacts(artifact_uri=uri)\n", + " if not info.is_dir\n", + " }\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files listed (no download) at {uri}\")\n", + " return mlflow_files\n", + "\n", + "\n", + "def download_mlflow_masks(mlflow_masks=MLFLOW_MASKS) -> tuple[dict[str, Path], dict[str, set[str]]]:\n", + " \"\"\"DOWNLOAD: fetch MLflow mask dirs and enumerate them. Returns (dirs, files).\"\"\"\n", + " mlflow_dirs: dict[str, Path] = {}\n", + " mlflow_files: dict[str, set[str]] = {}\n", + " for mask_name, uri in mlflow_masks.items():\n", + " mask_dir = Path(download_artifacts(uri))\n", + " mlflow_dirs[mask_name] = mask_dir\n", + " found = {p.name for p in mask_dir.iterdir() if p.is_file()}\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files found in {mask_dir}\")\n", + " return mlflow_dirs, mlflow_files" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B4. QC masks (local): list ----\n", + "def list_qc_masks(qc_paths=QC_MASKS_PATHS, prefixes=QC_MASK_PREFIXES) -> dict[str, set[str]]:\n", + " \"\"\"LIST ONLY: enumerate local QC mask filenames (prefix-stripped), without opening them.\n", + "\n", + " Accepts one or more directories; all are listed and merged, as if every mask lived in a\n", + " single location. A single Path is also accepted for convenience.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " qc_files: dict[str, set[str]] = {mask_type: set() for mask_type in prefixes}\n", + " for qc_path in qc_paths:\n", + " for mask_type, prefix in prefixes.items():\n", + " found = {\n", + " p.name.removeprefix(prefix)\n", + " for p in qc_path.iterdir()\n", + " if p.is_file() and p.name.startswith(prefix)\n", + " }\n", + " qc_files[mask_type] |= found\n", + "\n", + " for mask_type, found in qc_files.items():\n", + " prefix = prefixes[mask_type]\n", + " print(f\"{len(found)} {mask_type} mask files found (prefix: {prefix})\")\n", + " return qc_files\n", + "\n", + "\n", + "def _find_qc_mask(filename: str, qc_paths) -> Path | None:\n", + " \"\"\"Return the full path of a QC mask file across the given dirs, or None if not found.\"\"\"\n", + " for qc_path in qc_paths:\n", + " candidate = Path(qc_path) / filename\n", + " if candidate.is_file():\n", + " return candidate\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B6. Coverage report + summary + extras ----\n", + "DATA_MAPPING_COLUMNS = [\"record_num\", \"mammaprint_index\", \"type\", \"path\"]\n", + "\n", + "\n", + "def build_coverage_report(\n", + " expected_names: set[str],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + ") -> pd.DataFrame:\n", + " \"\"\"Per-slide presence table across all mask types, plus an `all_present` column.\"\"\"\n", + " rows = []\n", + " for name in sorted(expected_names):\n", + " row = {\"tiff_name\": name}\n", + " for mask_name in mlflow_files:\n", + " row[mask_name] = name in mlflow_files[mask_name]\n", + " for mask_type in qc_files:\n", + " row[mask_type] = name in qc_files[mask_type]\n", + " rows.append(row)\n", + "\n", + " mask_cols = [*mlflow_files.keys(), *qc_files.keys()]\n", + " report = pd.DataFrame(rows)\n", + " report[\"all_present\"] = report[mask_cols].all(axis=1)\n", + " return report\n", + "\n", + "\n", + "def coverage_summary(report: pd.DataFrame, expected_names: set[str]) -> pd.DataFrame:\n", + " \"\"\"Found/missing/total counts per mask type.\"\"\"\n", + " mask_cols = [c for c in report.columns if c not in (\"tiff_name\", \"all_present\")]\n", + " summary = report[mask_cols].sum().to_frame(\"found\")\n", + " summary[\"missing\"] = len(expected_names) - summary[\"found\"]\n", + " summary[\"total\"] = len(expected_names)\n", + " return summary\n", + "\n", + "\n", + "def slides_missing_masks(report: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Rows for slides missing at least one mask.\"\"\"\n", + " missing = report[~report[\"all_present\"]].drop(columns=[\"all_present\"])\n", + " print(f\"{len(missing)} slides missing at least one mask.\")\n", + " return missing\n", + "\n", + "\n", + "def missing_mask_data_mapping(\n", + " df: pd.DataFrame,\n", + " report: pd.DataFrame,\n", + " mask_types: str | list[str],\n", + " out_csv: str | Path | None = None,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Return a data_mapping.csv-style frame of the slides MISSING the given mask type(s).\n", + "\n", + " ``mask_types`` is a mask column name from the report (e.g. \"epithelium\") or a list of them;\n", + " a row is included if it is missing ANY of the requested masks. The result keeps only the\n", + " original data_mapping columns. If ``out_csv`` is given, it is also written there.\n", + " \"\"\"\n", + " if isinstance(mask_types, str):\n", + " mask_types = [mask_types]\n", + "\n", + " unknown = [m for m in mask_types if m not in report.columns]\n", + " if unknown:\n", + " raise KeyError(f\"Unknown mask type(s) {unknown}. Available: \"\n", + " f\"{[c for c in report.columns if c not in ('tiff_name', 'all_present')]}\")\n", + "\n", + " # tiff_names missing at least one of the requested masks\n", + " missing_mask = ~report[mask_types].all(axis=1)\n", + " missing_names = set(report.loc[missing_mask, \"tiff_name\"])\n", + "\n", + " out = df[df[\"tiff_name\"].isin(missing_names)]\n", + " keep = [c for c in DATA_MAPPING_COLUMNS if c in out.columns]\n", + " out = out[keep].reset_index(drop=True)\n", + "\n", + " print(f\"{len(out)} slides missing {mask_types}.\")\n", + " if out_csv is not None:\n", + " out.to_csv(out_csv, index=False)\n", + " print(f\"Wrote {out_csv}\")\n", + " return out\n", + "\n", + "\n", + "def extra_masks(\n", + " expected_names: set[str],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + ") -> None:\n", + " \"\"\"Print mask files that have no matching slide in data_mapping.\"\"\"\n", + " for mask_name in mlflow_files:\n", + " extra = mlflow_files[mask_name] - expected_names\n", + " print(f\"{len(extra)} extra {mask_name} masks with no matching slide in data_mapping:\")\n", + " if extra:\n", + " print(sorted(extra))\n", + " print()\n", + "\n", + " for mask_type in qc_files:\n", + " extra = qc_files[mask_type] - expected_names\n", + " print(f\"{len(extra)} extra {mask_type} masks with no matching slide:\")\n", + " if extra:\n", + " print(sorted(extra))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "im0bfkx60ei", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B5. Mask readability check (open each existing mask file) ----\n", + "def _try_open_mask(path: Path) -> str | None:\n", + " \"\"\"Try to open a mask file. Returns None on success or an error message.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path):\n", + " pass\n", + " else:\n", + " with OpenSlide(str_path):\n", + " pass\n", + " return None\n", + " except Exception as e:\n", + " return f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "def open_masks(\n", + " expected_names: set[str],\n", + " mlflow_dirs: dict[str, Path],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + " qc_paths=QC_MASKS_PATHS,\n", + " prefixes=QC_MASK_PREFIXES,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Open each existing mask file to verify readability. Returns a DataFrame of failures.\n", + "\n", + " Pass MLflow dirs/files from ``download_mlflow_masks`` (opening implies a prior download).\n", + " Pass QC files from ``list_qc_masks`` (already local; may span multiple ``qc_paths``, which are\n", + " searched in order). Any group may be an empty dict to skip it.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " open_errors: list[dict[str, str]] = []\n", + "\n", + " # MLflow masks (require downloaded dirs)\n", + " for mask_name in mlflow_files:\n", + " mask_dir = mlflow_dirs[mask_name]\n", + " for name in sorted(expected_names & mlflow_files[mask_name]):\n", + " err = _try_open_mask(mask_dir / name)\n", + " if err:\n", + " open_errors.append({\"mask_type\": mask_name, \"tiff_name\": name, \"error\": err})\n", + "\n", + " # QC masks (local; located across qc_paths)\n", + " for mask_type in qc_files:\n", + " prefix = prefixes[mask_type]\n", + " for name in sorted(expected_names & qc_files[mask_type]):\n", + " path = _find_qc_mask(f\"{prefix}{name}\", qc_paths)\n", + " if path is None:\n", + " open_errors.append(\n", + " {\"mask_type\": mask_type, \"tiff_name\": name, \"error\": \"FileNotFound in qc_paths\"}\n", + " )\n", + " continue\n", + " err = _try_open_mask(path)\n", + " if err:\n", + " open_errors.append({\"mask_type\": mask_type, \"tiff_name\": name, \"error\": err})\n", + "\n", + " errors_df = pd.DataFrame(open_errors)\n", + " if errors_df.empty:\n", + " print(\"All checked mask files opened successfully.\")\n", + " else:\n", + " print(f\"{len(errors_df)} mask files failed to open.\")\n", + " return errors_df" + ] + }, + { + "cell_type": "markdown", + "id": "53d0f240", + "metadata": {}, + "source": [ + "## Visual inspection (function)\n", + "\n", + "`plot_sample_masks(...)` displays N sample masks for each mask type (MLflow + QC) in a grid.\n", + "Requires downloaded MLflow dirs (from `download_mlflow_masks`)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f0b337bd", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B7. Visual inspection ----\n", + "THUMB_SIZE = (512, 512)\n", + "\n", + "\n", + "def _read_mask_thumbnail(path: Path, thumb_size=THUMB_SIZE) -> np.ndarray | None:\n", + " \"\"\"Read a mask file and return a thumbnail as a numpy array.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as tif:\n", + " page = tif.pages[0]\n", + " # Read full page then downsample via slicing (nearest-neighbor)\n", + " img = page.asarray()\n", + " step_y = max(1, img.shape[0] // thumb_size[1])\n", + " step_x = max(1, img.shape[1] // thumb_size[0])\n", + " return img[::step_y, ::step_x]\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " thumb = slide.get_thumbnail(thumb_size)\n", + " return np.array(thumb)\n", + " except Exception as e:\n", + " print(f\" Could not read {path.name}: {e}\")\n", + " return None\n", + "\n", + "\n", + "def plot_sample_masks(\n", + " expected_names: set[str],\n", + " mlflow_dirs: dict[str, Path],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + " n_samples: int = 5,\n", + " qc_paths=QC_MASKS_PATHS,\n", + " prefixes=QC_MASK_PREFIXES,\n", + ") -> None:\n", + " \"\"\"Display n_samples sample masks per type (MLflow + QC) in a grid.\n", + "\n", + " QC masks are located across ``qc_paths`` (searched in order), so masks spread over several\n", + " directories are handled as if they lived in a single location.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " # Collect mask paths: list of (mask_type, [paths])\n", + " all_mask_types: list[tuple[str, list[Path]]] = []\n", + " sample_names = sorted(expected_names)[:n_samples]\n", + "\n", + " for mask_name in mlflow_files:\n", + " paths = [mlflow_dirs[mask_name] / name for name in sample_names if name in mlflow_files[mask_name]]\n", + " all_mask_types.append((mask_name, paths[:n_samples]))\n", + "\n", + " for mask_type in qc_files:\n", + " prefix = prefixes[mask_type]\n", + " paths = [\n", + " found\n", + " for name in sample_names\n", + " if name in qc_files[mask_type]\n", + " and (found := _find_qc_mask(f\"{prefix}{name}\", qc_paths)) is not None\n", + " ]\n", + " all_mask_types.append((mask_type, paths[:n_samples]))\n", + "\n", + " # Plot grid: rows = mask types, cols = samples\n", + " n_rows = len(all_mask_types)\n", + " n_cols = n_samples\n", + " fig, axes = plt.subplots(n_rows, n_cols, figsize=(3 * n_cols, 3 * n_rows))\n", + " if n_rows == 1:\n", + " axes = [axes]\n", + "\n", + " for row_idx, (mask_type, paths) in enumerate(all_mask_types):\n", + " for col_idx in range(n_cols):\n", + " ax = axes[row_idx][col_idx]\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " if col_idx == 0:\n", + " ax.set_ylabel(mask_type, fontsize=12, fontweight=\"bold\")\n", + " if col_idx < len(paths):\n", + " thumb = _read_mask_thumbnail(paths[col_idx])\n", + " if thumb is not None:\n", + " ax.imshow(thumb, cmap=\"gray\")\n", + " ax.set_title(paths[col_idx].name, fontsize=7)\n", + " else:\n", + " ax.text(0.5, 0.5, \"read error\", ha=\"center\", va=\"center\", transform=ax.transAxes)\n", + " else:\n", + " ax.text(0.5, 0.5, \"N/A\", ha=\"center\", va=\"center\", transform=ax.transAxes, color=\"gray\")\n", + "\n", + " fig.suptitle(f\"Sample masks ({n_samples} per type)\", fontsize=14, fontweight=\"bold\")\n", + " fig.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e5a23332", + "metadata": {}, + "source": [ + "## Tiled dataset validation (functions)\n", + "\n", + "`download_tiled_datasets(...)` downloads `slides.parquet` / `tiles.parquet` per dataset.\n", + "`validate_tiled_datasets(df, tiled_data)` checks:\n", + "1. Every slide in `data_mapping.csv` has a row in `slides.parquet` (and vice versa)\n", + "2. Every slide in `slides.parquet` has at least one tile in `tiles.parquet`\n", + "3. Every `slide_id` in `tiles.parquet` maps to a slide in `slides.parquet`" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "fe9ffadc", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B8a. Tiled datasets ----\n", + "def download_tiled_datasets(tiled=TILED_DATASETS) -> dict[str, tuple[pd.DataFrame, pd.DataFrame]]:\n", + " \"\"\"Download slides.parquet / tiles.parquet per dataset. Returns {name: (slides, tiles)}.\"\"\"\n", + " tiled_data: dict[str, tuple[pd.DataFrame, pd.DataFrame]] = {}\n", + " for ds_name, uri in tiled.items():\n", + " local_dir = Path(download_artifacts(uri))\n", + " slides = pd.read_parquet(local_dir / \"slides.parquet\")\n", + " tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n", + " tiled_data[ds_name] = (slides, tiles)\n", + " print(f\"{ds_name}: {len(slides)} slides, {len(tiles)} tiles\")\n", + " return tiled_data\n", + "\n", + "\n", + "def validate_tiled_datasets(df: pd.DataFrame, tiled_data: dict) -> None:\n", + " \"\"\"Check slides.parquet <-> data_mapping and tiles.parquet <-> slides.parquet consistency.\"\"\"\n", + " expected_stems = {Path(p).stem for p in df[\"path\"]}\n", + "\n", + " for ds_name, (slides, tiles) in tiled_data.items():\n", + " print(f\"=== {ds_name} ===\\n\")\n", + "\n", + " # 1. slides.parquet path stem <-> data_mapping path stem\n", + " slide_stems = {Path(p).stem for p in slides[\"path\"]}\n", + " missing_from_dataset = expected_stems - slide_stems\n", + " extra_in_dataset = slide_stems - expected_stems\n", + "\n", + " print(f\"Slides in data_mapping: {len(expected_stems)}\")\n", + " print(f\"Slides in dataset: {len(slide_stems)}\")\n", + " print(f\"Missing from dataset: {len(missing_from_dataset)}\")\n", + " print(f\"Extra in dataset: {len(extra_in_dataset)}\")\n", + "\n", + " if missing_from_dataset:\n", + " print(f\"\\nSlides in data_mapping but NOT in {ds_name}:\")\n", + " for p in sorted(missing_from_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " if extra_in_dataset:\n", + " print(f\"\\nSlides in {ds_name} but NOT in data_mapping:\")\n", + " for p in sorted(extra_in_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " # 2. Every slide has at least one tile\n", + " slide_ids = set(slides[\"id\"])\n", + " slide_ids_in_tiles = set(tiles[\"slide_id\"])\n", + "\n", + " slides_without_tiles = slide_ids - slide_ids_in_tiles\n", + " print(f\"\\nSlides without any tiles: {len(slides_without_tiles)}\")\n", + " if slides_without_tiles:\n", + " for s in sorted(slides_without_tiles):\n", + " print(f\" {s}\")\n", + "\n", + " # 3. No orphan tiles (tile references a slide not in slides.parquet)\n", + " orphan_tile_ids = slide_ids_in_tiles - slide_ids\n", + " print(f\"Orphan tile slide_ids: {len(orphan_tile_ids)}\")\n", + " if orphan_tile_ids:\n", + " for sid in sorted(orphan_tile_ids):\n", + " count = int((tiles[\"slide_id\"] == sid).sum())\n", + " print(f\" slide_id={sid}: {count} tiles\")\n", + "\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "ea2eae31", + "metadata": {}, + "source": [ + "## Embeddings validation (functions)\n", + "\n", + "`list_embeddings(df)` checks coverage using `list_artifacts` (no download).\n", + "`validate_embeddings(df)` downloads the artifact dir and checks the same, for parity with the\n", + "download+open mask mode. Both verify every slide in `data_mapping.csv` has a matching `.parquet` file." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cc5ce48a", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B8b. Embeddings: two modes ----\n", + "# Embedding parquet files are named as \"{record_num}.parquet\" (e.g. \"2023/00835.parquet\"),\n", + "# i.e. the stem matches the data_mapping `path` column.\n", + "def _report_embeddings(emb_name: str, emb_files: set[str], expected_record_nums: set[str]) -> None:\n", + " print(f\"=== {emb_name} ===\\n\")\n", + " print(f\"Embedding files found: {len(emb_files)}\")\n", + " print(f\"Slides in data_mapping: {len(expected_record_nums)}\")\n", + "\n", + " missing = expected_record_nums - emb_files\n", + " extra = emb_files - expected_record_nums\n", + "\n", + " print(f\"Missing embeddings: {len(missing)}\")\n", + " if missing:\n", + " for s in sorted(missing):\n", + " print(f\" {s}\")\n", + "\n", + " print(f\"Extra embeddings: {len(extra)}\")\n", + " if extra:\n", + " for s in sorted(extra):\n", + " print(f\" {s}\")\n", + " print()\n", + "\n", + "\n", + "def list_embeddings(df: pd.DataFrame, embeddings=EMBEDDINGS) -> None:\n", + " \"\"\"LIST ONLY: check embedding coverage via list_artifacts, without downloading.\"\"\"\n", + " expected_record_nums = set(df[\"path\"])\n", + " for emb_name, uri in embeddings.items():\n", + " emb_files = {\n", + " Path(info.path).stem\n", + " for info in list_artifacts(artifact_uri=uri)\n", + " if not info.is_dir and info.path.endswith(\".parquet\")\n", + " }\n", + " _report_embeddings(emb_name, emb_files, expected_record_nums)\n", + "\n", + "\n", + "def validate_embeddings(df: pd.DataFrame, embeddings=EMBEDDINGS) -> None:\n", + " \"\"\"DOWNLOAD: fetch the embeddings dir and check coverage against data_mapping.\"\"\"\n", + " expected_record_nums = set(df[\"path\"])\n", + " for emb_name, uri in embeddings.items():\n", + " emb_dir = Path(download_artifacts(uri))\n", + " emb_files = {p.stem for p in emb_dir.iterdir() if p.is_file() and p.suffix == \".parquet\"}\n", + " _report_embeddings(emb_name, emb_files, expected_record_nums)" + ] + }, + { + "cell_type": "markdown", + "id": "fd9a315e", + "metadata": {}, + "source": [ + "# Calls\n", + "\n", + "Run the cells below to execute individual checks. **Comment out whatever you don't need.**\n", + "\n", + "The cheap, network-light path is left uncommented by default (list-only mask enumeration + coverage\n", + "report). The download / open / visual / tiled / embeddings cells are commented out — uncomment them\n", + "when you want to actually fetch and open the files to verify everything works end-to-end." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a9b7a86c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2042 slides from /mnt/projects/mammaprint/data_mapping.csv\n", + "Columns: ['record_num', 'mammaprint_index', 'type', 'path', 'split', 'tiff_name']\n", + "2042 unique slide tiff names expected\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplittiff_name
02023/00835-0.158b luminal/mnt/data/MOU/breast/mammaprint/P2023_0835testP2023_0835.tiff
12023/008360.058a luminal/mnt/data/MOU/breast/mammaprint/P2023_0836trainP2023_0836.tiff
22023/00837-0.039b luminal/mnt/data/MOU/breast/mammaprint/P2023_0837trainP2023_0837.tiff
32023/008380.065a luminal/mnt/data/MOU/breast/mammaprint/P2023_0838testP2023_0838.tiff
42023/00839-0.429b luminal/mnt/data/MOU/breast/mammaprint/P2023_0839testP2023_0839.tiff
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2023/00835 -0.158 b luminal \n", + "1 2023/00836 0.058 a luminal \n", + "2 2023/00837 -0.039 b luminal \n", + "3 2023/00838 0.065 a luminal \n", + "4 2023/00839 -0.429 b luminal \n", + "\n", + " path split tiff_name \n", + "0 /mnt/data/MOU/breast/mammaprint/P2023_0835 test P2023_0835.tiff \n", + "1 /mnt/data/MOU/breast/mammaprint/P2023_0836 train P2023_0836.tiff \n", + "2 /mnt/data/MOU/breast/mammaprint/P2023_0837 train P2023_0837.tiff \n", + "3 /mnt/data/MOU/breast/mammaprint/P2023_0838 test P2023_0838.tiff \n", + "4 /mnt/data/MOU/breast/mammaprint/P2023_0839 test P2023_0839.tiff " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# --- Load data mapping (required by everything below) ---\n", + "df, expected_names = load_data_mapping()\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "533d7fbf", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Slides open check (verifies every slide file in data_mapping opens) ---\n", + "# slide_errors = check_slides_open(df)\n", + "# display(slide_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "23f1e1ad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 tissue mask files listed (no download) at mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\n", + "310 epithelium mask files listed (no download) at mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\n" + ] + } + ], + "source": [ + "# --- MLflow masks: LIST ONLY (no download) ---\n", + "mlflow_files = list_mlflow_masks()\n", + "\n", + "# --- MLflow masks: DOWNLOAD + OPEN (uncomment to actually fetch & verify readability) ---\n", + "# mlflow_dirs, mlflow_files = download_mlflow_masks()\n", + "# mlflow_open_errors = open_masks(expected_names, mlflow_dirs, mlflow_files, {})\n", + "# display(mlflow_open_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "2df7d0f5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1800 blur mask files found (prefix: Piqe_piqe_median_activity_mask_)\n", + "1800 folding mask files found (prefix: FoldingFunction_folding_test_)\n", + "1800 residual mask files found (prefix: ResidualArtifactsAndCoverage_coverage_mask_)\n" + ] + } + ], + "source": [ + "# --- QC masks: LIST ONLY (local ls, no open) ---\n", + "qc_files = list_qc_masks()\n", + "\n", + "# --- QC masks: OPEN (uncomment to verify each local QC mask is readable) ---\n", + "# qc_open_errors = open_masks(expected_names, {}, {}, qc_files)\n", + "# display(qc_open_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "7073481f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tiff_nametissueepitheliumblurfoldingresidualall_present
02024_02851-1.tiffTrueTrueTrueTrueTrueTrue
12024_02865.tiffTrueFalseTrueTrueTrueFalse
22024_02890.tiffTrueFalseTrueTrueTrueFalse
3P2023_01000.tiffTrueFalseTrueTrueTrueFalse
4P2023_01001.tiffTrueFalseTrueTrueTrueFalse
........................
2037P2026_00970.tiffTrueFalseFalseFalseFalseFalse
2038P2026_00973.tiffTrueFalseFalseFalseFalseFalse
2039P2026_00974.tiffTrueFalseFalseFalseFalseFalse
2040P2026_00976.tiffTrueFalseFalseFalseFalseFalse
2041P2026_00977.tiffTrueFalseFalseFalseFalseFalse
\n", + "

2042 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " tiff_name tissue epithelium blur folding residual \\\n", + "0 2024_02851-1.tiff True True True True True \n", + "1 2024_02865.tiff True False True True True \n", + "2 2024_02890.tiff True False True True True \n", + "3 P2023_01000.tiff True False True True True \n", + "4 P2023_01001.tiff True False True True True \n", + "... ... ... ... ... ... ... \n", + "2037 P2026_00970.tiff True False False False False \n", + "2038 P2026_00973.tiff True False False False False \n", + "2039 P2026_00974.tiff True False False False False \n", + "2040 P2026_00976.tiff True False False False False \n", + "2041 P2026_00977.tiff True False False False False \n", + "\n", + " all_present \n", + "0 True \n", + "1 False \n", + "2 False \n", + "3 False \n", + "4 False \n", + "... ... \n", + "2037 False \n", + "2038 False \n", + "2039 False \n", + "2040 False \n", + "2041 False \n", + "\n", + "[2042 rows x 7 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- Coverage report (per-slide presence across all mask types) ---\n", + "report = build_coverage_report(expected_names, mlflow_files, qc_files)\n", + "display(report)\n", + "\n", + "# --- Summary / missing / extras (uncomment as needed) ---\n", + "# display(coverage_summary(report, expected_names))\n", + "# display(slides_missing_masks(report))\n", + "# extra_masks(expected_names, mlflow_files, qc_files)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "47dac634", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "243 slides missing ['blur', 'folding', 'residual'].\n", + "Wrote data_mapping_missing_qc.csv\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepath
02024/04733-0.122b luminal/mnt/data/MOU/breast/mammaprint/P2024_04733
12025/001610.263a luminal/mnt/data/MOU/breast/mammaprint/P2025_00161-2
22025/04084-0.215b luminal/mnt/data/MOU/breast/mammaprint/P2025_04084
32025/04262-0.023b luminal/mnt/data/MOU/breast/mammaprint/P2025_04262
42025/04267-0.309b luminal/mnt/data/MOU/breast/mammaprint/P2025_04267
...............
2382026/00970-0.218b luminal/mnt/data/MOU/breast/mammaprint/P2026_00970
2392026/00973-0.141b luminal/mnt/data/MOU/breast/mammaprint/P2026_00973
2402026/009740.192a luminal/mnt/data/MOU/breast/mammaprint/P2026_00974
2412026/009760.396a luminal/mnt/data/MOU/breast/mammaprint/P2026_00976
2422026/00977-0.065b luminal/mnt/data/MOU/breast/mammaprint/P2026_00977
\n", + "

243 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2024/04733 -0.122 b luminal \n", + "1 2025/00161 0.263 a luminal \n", + "2 2025/04084 -0.215 b luminal \n", + "3 2025/04262 -0.023 b luminal \n", + "4 2025/04267 -0.309 b luminal \n", + ".. ... ... ... \n", + "238 2026/00970 -0.218 b luminal \n", + "239 2026/00973 -0.141 b luminal \n", + "240 2026/00974 0.192 a luminal \n", + "241 2026/00976 0.396 a luminal \n", + "242 2026/00977 -0.065 b luminal \n", + "\n", + " path \n", + "0 /mnt/data/MOU/breast/mammaprint/P2024_04733 \n", + "1 /mnt/data/MOU/breast/mammaprint/P2025_00161-2 \n", + "2 /mnt/data/MOU/breast/mammaprint/P2025_04084 \n", + "3 /mnt/data/MOU/breast/mammaprint/P2025_04262 \n", + "4 /mnt/data/MOU/breast/mammaprint/P2025_04267 \n", + ".. ... \n", + "238 /mnt/data/MOU/breast/mammaprint/P2026_00970 \n", + "239 /mnt/data/MOU/breast/mammaprint/P2026_00973 \n", + "240 /mnt/data/MOU/breast/mammaprint/P2026_00974 \n", + "241 /mnt/data/MOU/breast/mammaprint/P2026_00976 \n", + "242 /mnt/data/MOU/breast/mammaprint/P2026_00977 \n", + "\n", + "[243 rows x 4 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- Export a data_mapping.csv-style file of slides MISSING a chosen mask ---\n", + "# Set mask_types to the mask column you care about, e.g. \"epithelium\", \"tissue\",\n", + "# \"blur\", \"folding\", \"residual\" (or a list of several -> missing ANY of them).\n", + "missing_epithelium = missing_mask_data_mapping(\n", + " df, report, mask_types=[\"blur\", \"folding\", \"residual\"], out_csv=\"data_mapping_missing_qc.csv\"\n", + ")\n", + "display(missing_epithelium)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "b360e3bd", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Visual grid: sample masks per type (needs downloaded MLflow dirs; uncomment to run) ---\n", + "# mlflow_dirs, mlflow_files = download_mlflow_masks()\n", + "# plot_sample_masks(expected_names, mlflow_dirs, mlflow_files, qc_files)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "3e10cb93", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Tiled datasets (downloads slides/tiles parquet; uncomment to run) ---\n", + "# tiled_data = download_tiled_datasets()\n", + "# validate_tiled_datasets(df, tiled_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "d91e48a5", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Embeddings ---\n", + "# list_embeddings(df) # LIST ONLY (no download)\n", + "# validate_embeddings(df) # DOWNLOAD + check coverage" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/notebooks/check_mask_coverage_old.ipynb b/scripts/notebooks/check_mask_coverage_old.ipynb new file mode 100644 index 00000000..4c7579a7 --- /dev/null +++ b/scripts/notebooks/check_mask_coverage_old.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Check mask coverage and tiled datasets against data_mapping\n", + "\n", + "Verifies 1-1 mapping between slides in `data_mapping.csv` and:\n", + "- **MLflow masks** (downloaded from MLflow): tissue, epithelium\n", + "- **QC masks**: blur (`Piqe_piqe_median_activity_mask_`), folding (`FoldingFunction_folding_test_`), residual (`ResidualArtifactsAndCoverage_coverage_mask_`)\n", + "- **Slide files** listed in the `path` column: checks that every slide can be opened without error\n", + "- **Tiled datasets** (MLflow): checks `slides.parquet` ↔ `data_mapping` and `tiles.parquet` ↔ `slides.parquet` consistency" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "155f9f1e-5529-448d-8d5e-7c20208def37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmprwve6ksi\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas mlflow>3 openslide-python tifffile matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import mlflow\n", + "from mlflow.artifacts import download_artifacts\n", + "from openslide import OpenSlide\n", + "import os\n", + "from tifffile import TiffFile" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\n", + "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", + "\n", + "PROJECT_PATH = Path(\"/mnt/projects/mammaprint\")\n", + "DATA_MAPPING_CSV = PROJECT_PATH / \"data_mapping.csv\"\n", + "QC_MASKS_PATH = PROJECT_PATH / \"qc_masks3\"\n", + "\n", + "MLFLOW_MASKS = {\n", + " # \"tissue\": \"mlflow-artifacts:/3/50c597ac29784808bd89d55d9912f699/artifacts/tissue_masks\",\n", + " # \"epithelium\": \"mlflow-artifacts:/3/c5e6c447e4784e5b991bff258195b5df/artifacts/epithelium_masks\",\n", + " \"tissue\": \"mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\",\n", + " \"epithelium\": \"mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\",\n", + "}\n", + "\n", + "QC_MASK_PREFIXES = {\n", + " \"blur\": \"Piqe_piqe_median_activity_mask_\",\n", + " \"folding\": \"FoldingFunction_folding_test_\",\n", + " \"residual\": \"ResidualArtifactsAndCoverage_coverage_mask_\",\n", + "}\n", + "\n", + "TILED_DATASETS = {\n", + " # \"mou_3_224\": \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\",\n", + " # \"mou_2_224\": \"mlflow-artifacts:/3/1ebe489fd3ca4f5682c9a4b3d6f74620/artifacts/mou_2_224\",\n", + "}\n", + "\n", + "EMBEDDINGS = {\n", + " # \"embedding3\": \"mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2042 slides from /mnt/projects/mammaprint/data_mapping.csv\n", + "Columns: ['record_num', 'mammaprint_index', 'type', 'path', 'split']\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplit
02023/00835-0.158b luminal/mnt/data/MOU/breast/mammaprint/P2023_0835test
12023/008360.058a luminal/mnt/data/MOU/breast/mammaprint/P2023_0836train
22023/00837-0.039b luminal/mnt/data/MOU/breast/mammaprint/P2023_0837train
32023/008380.065a luminal/mnt/data/MOU/breast/mammaprint/P2023_0838test
42023/00839-0.429b luminal/mnt/data/MOU/breast/mammaprint/P2023_0839test
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2023/00835 -0.158 b luminal \n", + "1 2023/00836 0.058 a luminal \n", + "2 2023/00837 -0.039 b luminal \n", + "3 2023/00838 0.065 a luminal \n", + "4 2023/00839 -0.429 b luminal \n", + "\n", + " path split \n", + "0 /mnt/data/MOU/breast/mammaprint/P2023_0835 test \n", + "1 /mnt/data/MOU/breast/mammaprint/P2023_0836 train \n", + "2 /mnt/data/MOU/breast/mammaprint/P2023_0837 train \n", + "3 /mnt/data/MOU/breast/mammaprint/P2023_0838 test \n", + "4 /mnt/data/MOU/breast/mammaprint/P2023_0839 test " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Load data mapping\n", + "df = pd.read_csv(DATA_MAPPING_CSV)\n", + "print(f\"Loaded {len(df)} slides from {DATA_MAPPING_CSV}\")\n", + "print(f\"Columns: {df.columns.tolist()}\")\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 unique slide tiff names expected\n" + ] + } + ], + "source": [ + "# Derive the .tiff name that the tiling pipeline expects for each slide\n", + "df[\"tiff_name\"] = df[\"path\"].apply(lambda p: Path(p).with_suffix(\".tiff\").name)\n", + "expected_names = set(df[\"tiff_name\"])\n", + "print(f\"{len(expected_names)} unique slide tiff names expected\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-slide-open-check", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Verify data_mapping slide files can be opened ----\n", + "SLIDE_SUFFIX_CANDIDATES = (\".mrxs\", \".tiff\", \".tif\")\n", + "\n", + "\n", + "def _slide_path_candidates(raw_path: str) -> list[Path]:\n", + " path = Path(raw_path)\n", + " candidates = [path]\n", + " if not path.suffix:\n", + " candidates.extend(Path(f\"{raw_path}{suffix}\") for suffix in SLIDE_SUFFIX_CANDIDATES)\n", + " return candidates\n", + "\n", + "\n", + "def _resolve_slide_path(raw_path: str) -> Path:\n", + " for candidate in _slide_path_candidates(raw_path):\n", + " if candidate.is_file():\n", + " return candidate\n", + " return Path(raw_path)\n", + "\n", + "\n", + "def _try_open_slide(raw_path: str) -> tuple[Path, str | None]:\n", + " slide_path = _resolve_slide_path(raw_path)\n", + " try:\n", + " str_path = str(slide_path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as slide:\n", + " _ = len(slide.pages)\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " _ = slide.dimensions\n", + " return slide_path, None\n", + " except Exception as e:\n", + " return slide_path, f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "# slide_open_rows = []\n", + "# for row in df.itertuples(index=False):\n", + "# opened_path, error = _try_open_slide(row.path)\n", + "# if error:\n", + "# slide_open_rows.append(\n", + "# {\n", + "# \"record_num\": row.record_num,\n", + "# \"path\": row.path,\n", + "# \"resolved_path\": str(opened_path),\n", + "# \"error\": error,\n", + "# }\n", + "# )\n", + "\n", + "# slide_open_errors = pd.DataFrame(slide_open_rows)\n", + "# if slide_open_errors.empty:\n", + "# print(f\"All {len(df)} slide files opened successfully.\")\n", + "# else:\n", + "# print(f\"{len(slide_open_errors)} slide files failed to open:\")\n", + "# display(slide_open_errors)\n", + "# raise AssertionError(f\"{len(slide_open_errors)} slide files failed to open\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cell-5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 tissue mask files found in /tmp/tmpm4swn0wz/tissue_masks\n" + ] + }, + { + "ename": "MlflowException", + "evalue": "The following failures occurred while downloading one or more artifacts from http://mlflow-s3.rationai-mlflow/api/2.0/mlflow-artifacts/artifacts/3/0176fb9485de49699b1e16f362cff5fa/artifacts:\n##### File epithelium_masks/P2023_01448.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2023_03021.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2024_02850.tiff #####\nResponse ended prematurely", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mMlflowException\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 3\u001b[39m mlflow_files: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mset\u001b[39m[\u001b[38;5;28mstr\u001b[39m]] = {}\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m mask_name, uri \u001b[38;5;129;01min\u001b[39;00m MLFLOW_MASKS.items():\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m mask_dir = Path(\u001b[43mdownload_artifacts\u001b[49m\u001b[43m(\u001b[49m\u001b[43muri\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[32m 7\u001b[39m mlflow_dirs[mask_name] = mask_dir\n\u001b[32m 8\u001b[39m found = {p.name \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m mask_dir.iterdir() \u001b[38;5;28;01mif\u001b[39;00m p.is_file()}\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/artifacts/__init__.py:84\u001b[39m, in \u001b[36mdownload_artifacts\u001b[39m\u001b[34m(artifact_uri, run_id, artifact_path, dst_path, tracking_uri, registry_uri)\u001b[39m\n\u001b[32m 81\u001b[39m pathlib.Path(dst_path).mkdir(exist_ok=\u001b[38;5;28;01mTrue\u001b[39;00m, parents=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 83\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m artifact_uri \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m84\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_download_artifact_from_uri\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 85\u001b[39m \u001b[43m \u001b[49m\u001b[43martifact_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdst_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtracking_uri\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtracking_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mregistry_uri\u001b[49m\u001b[43m=\u001b[49m\u001b[43mregistry_uri\u001b[49m\n\u001b[32m 86\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 88\u001b[39m \u001b[38;5;66;03m# Use `runs://` to download both run and model (if exists) artifacts\u001b[39;00m\n\u001b[32m 89\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m run_id \u001b[38;5;129;01mand\u001b[39;00m artifact_path:\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/tracking/artifact_utils.py:128\u001b[39m, in \u001b[36m_download_artifact_from_uri\u001b[39m\u001b[34m(artifact_uri, output_path, lineage_header_info, tracking_uri, registry_uri)\u001b[39m\n\u001b[32m 122\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(repo, ModelsArtifactRepository):\n\u001b[32m 123\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m repo.download_artifacts(\n\u001b[32m 124\u001b[39m artifact_path=artifact_path,\n\u001b[32m 125\u001b[39m dst_path=output_path,\n\u001b[32m 126\u001b[39m lineage_header_info=lineage_header_info,\n\u001b[32m 127\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m128\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrepo\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdownload_artifacts\u001b[49m\u001b[43m(\u001b[49m\u001b[43martifact_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43martifact_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdst_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43moutput_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 129\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 130\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m artifact_uri.startswith(\u001b[33m\"\u001b[39m\u001b[33mm-\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 131\u001b[39m \u001b[38;5;66;03m# When a Model ID like string is passed, suggest using 'models:/{artifact_uri}' instead.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/store/artifact/artifact_repo.py:366\u001b[39m, in \u001b[36mArtifactRepository.download_artifacts\u001b[39m\u001b[34m(self, artifact_path, dst_path)\u001b[39m\n\u001b[32m 358\u001b[39m error_codes = {\n\u001b[32m 359\u001b[39m e.error_code\n\u001b[32m 360\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m e \u001b[38;5;129;01min\u001b[39;00m failed_downloads.values()\n\u001b[32m 361\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, MlflowException) \u001b[38;5;129;01mand\u001b[39;00m e.error_code != \u001b[33m\"\u001b[39m\u001b[33mINTERNAL_ERROR\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 362\u001b[39m }\n\u001b[32m 363\u001b[39m error_code = (\n\u001b[32m 364\u001b[39m ErrorCode.Value(error_codes.pop()) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_codes) == \u001b[32m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m INTERNAL_ERROR\n\u001b[32m 365\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m366\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MlflowException(\n\u001b[32m 367\u001b[39m message=(\n\u001b[32m 368\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mThe following failures occurred while downloading one or more\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 369\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m artifacts from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.artifact_uri\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m_truncate_error(failures)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 370\u001b[39m ),\n\u001b[32m 371\u001b[39m error_code=error_code,\n\u001b[32m 372\u001b[39m )\n\u001b[32m 374\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m os.path.join(dst_path, artifact_path)\n", + "\u001b[31mMlflowException\u001b[39m: The following failures occurred while downloading one or more artifacts from http://mlflow-s3.rationai-mlflow/api/2.0/mlflow-artifacts/artifacts/3/0176fb9485de49699b1e16f362cff5fa/artifacts:\n##### File epithelium_masks/P2023_01448.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2023_03021.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2024_02850.tiff #####\nResponse ended prematurely" + ] + } + ], + "source": [ + "# ---- MLflow masks (download from MLflow) ----\n", + "mlflow_dirs: dict[str, Path] = {}\n", + "mlflow_files: dict[str, set[str]] = {}\n", + "\n", + "for mask_name, uri in MLFLOW_MASKS.items():\n", + " mask_dir = Path(download_artifacts(uri))\n", + " mlflow_dirs[mask_name] = mask_dir\n", + " found = {p.name for p in mask_dir.iterdir() if p.is_file()}\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files found in {mask_dir}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- QC masks (local directory) ----\n", + "qc_files: dict[str, set[str]] = {}\n", + "for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + " found = {\n", + " p.name.removeprefix(prefix)\n", + " for p in QC_MASKS_PATH.iterdir()\n", + " if p.is_file() and p.name.startswith(prefix)\n", + " }\n", + " qc_files[mask_type] = found\n", + " print(f\"{len(found)} {mask_type} mask files found (prefix: {prefix})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Build per-slide report ----\n", + "rows = []\n", + "for name in sorted(expected_names):\n", + " row = {\"tiff_name\": name}\n", + " for mask_name in MLFLOW_MASKS:\n", + " row[mask_name] = name in mlflow_files[mask_name]\n", + " for mask_type in QC_MASK_PREFIXES:\n", + " row[mask_type] = name in qc_files[mask_type]\n", + " rows.append(row)\n", + "\n", + "mask_cols = [*MLFLOW_MASKS.keys(), *QC_MASK_PREFIXES.keys()]\n", + "report = pd.DataFrame(rows)\n", + "report[\"all_present\"] = report[mask_cols].all(axis=1)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "im0bfkx60ei", + "metadata": {}, + "outputs": [], + "source": [ + "# # ---- Try opening each mask file to verify readability ----\n", + "# def _try_open_mask(path: Path) -> str | None:\n", + "# \"\"\"Try to open a mask file. Returns None on success or an error message.\"\"\"\n", + "# try:\n", + "# str_path = str(path)\n", + "# if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + "# with TiffFile(str_path) as slide:\n", + "# pass\n", + "# else:\n", + "# with OpenSlide(str_path) as slide:\n", + "# pass\n", + "# return None\n", + "# except Exception as e:\n", + "# return f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "# open_errors: list[dict[str, str]] = []\n", + "\n", + "# # MLflow masks\n", + "# for mask_name in MLFLOW_MASKS:\n", + "# for name in sorted(expected_names & mlflow_files[mask_name]):\n", + "# path = mlflow_dirs[mask_name] / name\n", + "# err = _try_open_mask(path)\n", + "# if err:\n", + "# open_errors.append({\"mask_type\": mask_name, \"tiff_name\": name, \"error\": err})\n", + "\n", + "# # QC masks\n", + "# for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + "# for name in sorted(expected_names & qc_files[mask_type]):\n", + "# path = QC_MASKS_PATH / f\"{prefix}{name}\"\n", + "# err = _try_open_mask(path)\n", + "# if err:\n", + "# open_errors.append({\"mask_type\": mask_type, \"tiff_name\": name, \"error\": err})\n", + "\n", + "# if open_errors:\n", + "# errors_df = pd.DataFrame(open_errors)\n", + "# print(f\"{len(open_errors)} mask files failed to open:\")\n", + "# display(errors_df)\n", + "# else:\n", + "# print(\"All existing mask files opened successfully.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-8", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Summary ----\n", + "summary = report[mask_cols].sum().to_frame(\"found\")\n", + "summary[\"missing\"] = len(expected_names) - summary[\"found\"]\n", + "summary[\"total\"] = len(expected_names)\n", + "print(\"Coverage summary:\")\n", + "summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-9", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Slides missing at least one mask ----\n", + "missing = report[~report[\"all_present\"]].drop(columns=[\"all_present\"])\n", + "print(f\"{len(missing)} slides missing at least one mask:\")\n", + "missing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Extra masks not in data_mapping ----\n", + "for mask_name in MLFLOW_MASKS:\n", + " extra = mlflow_files[mask_name] - expected_names\n", + " print(f\"{len(extra)} extra {mask_name} masks with no matching slide in data_mapping:\")\n", + " if extra:\n", + " print(sorted(extra))\n", + " print()\n", + "\n", + "for mask_type in QC_MASK_PREFIXES:\n", + " extra = qc_files[mask_type] - expected_names\n", + " print(f\"{len(extra)} extra {mask_type} masks with no matching slide:\")\n", + " if extra:\n", + " print(sorted(extra))" + ] + }, + { + "cell_type": "markdown", + "id": "53d0f240", + "metadata": {}, + "source": [ + "## Visual inspection\n", + "\n", + "Display 5 sample masks for each mask type (MLflow + QC) in a grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0b337bd", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Visual inspection: 5 sample masks per type ----\n", + "N_SAMPLES = 5\n", + "THUMB_SIZE = (512, 512)\n", + "\n", + "\n", + "def _read_mask_thumbnail(path: Path) -> np.ndarray | None:\n", + " \"\"\"Read a mask file and return a thumbnail as a numpy array.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as tif:\n", + " page = tif.pages[0]\n", + " # Read full page then downsample\n", + " img = page.asarray()\n", + " # Resize to thumbnail via slicing (nearest-neighbor)\n", + " step_y = max(1, img.shape[0] // THUMB_SIZE[1])\n", + " step_x = max(1, img.shape[1] // THUMB_SIZE[0])\n", + " return img[::step_y, ::step_x]\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " thumb = slide.get_thumbnail(THUMB_SIZE)\n", + " return np.array(thumb)\n", + " except Exception as e:\n", + " print(f\" Could not read {path.name}: {e}\")\n", + " return None\n", + "\n", + "\n", + "# Collect mask paths: list of (mask_type, [paths])\n", + "all_mask_types: list[tuple[str, list[Path]]] = []\n", + "\n", + "# Pick 5 sample tiff names that exist across all available mask types\n", + "sample_names = sorted(expected_names)[:N_SAMPLES]\n", + "\n", + "for mask_name in MLFLOW_MASKS:\n", + " paths = [mlflow_dirs[mask_name] / name for name in sample_names if name in mlflow_files[mask_name]]\n", + " all_mask_types.append((mask_name, paths[:N_SAMPLES]))\n", + "\n", + "for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + " paths = [QC_MASKS_PATH / f\"{prefix}{name}\" for name in sample_names if name in qc_files[mask_type]]\n", + " all_mask_types.append((mask_type, paths[:N_SAMPLES]))\n", + "\n", + "# Plot grid: rows = mask types, cols = samples\n", + "n_rows = len(all_mask_types)\n", + "n_cols = N_SAMPLES\n", + "fig, axes = plt.subplots(n_rows, n_cols, figsize=(3 * n_cols, 3 * n_rows))\n", + "if n_rows == 1:\n", + " axes = [axes]\n", + "\n", + "for row_idx, (mask_type, paths) in enumerate(all_mask_types):\n", + " for col_idx in range(n_cols):\n", + " ax = axes[row_idx][col_idx]\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " if col_idx == 0:\n", + " ax.set_ylabel(mask_type, fontsize=12, fontweight=\"bold\")\n", + " if col_idx < len(paths):\n", + " thumb = _read_mask_thumbnail(paths[col_idx])\n", + " if thumb is not None:\n", + " ax.imshow(thumb, cmap=\"gray\")\n", + " ax.set_title(paths[col_idx].name, fontsize=7)\n", + " else:\n", + " ax.text(0.5, 0.5, \"read error\", ha=\"center\", va=\"center\", transform=ax.transAxes)\n", + " else:\n", + " ax.text(0.5, 0.5, \"N/A\", ha=\"center\", va=\"center\", transform=ax.transAxes, color=\"gray\")\n", + "\n", + "fig.suptitle(\"Sample masks (5 per type)\", fontsize=14, fontweight=\"bold\")\n", + "fig.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e5a23332", + "metadata": {}, + "source": [ + "## Tiled dataset validation\n", + "\n", + "For each tiled dataset, download `slides.parquet` and `tiles.parquet` from MLflow, then check:\n", + "1. Every slide in `data_mapping.csv` has a row in `slides.parquet` (and vice versa)\n", + "2. Every slide in `slides.parquet` has at least one tile in `tiles.parquet`\n", + "3. Every `slide_id` in `tiles.parquet` maps to a slide in `slides.parquet`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe9ffadc", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Download tiled datasets from MLflow ----\n", + "tiled_data: dict[str, tuple[pd.DataFrame, pd.DataFrame]] = {}\n", + "\n", + "for ds_name, uri in TILED_DATASETS.items():\n", + " local_dir = Path(download_artifacts(uri))\n", + " slides = pd.read_parquet(local_dir / \"slides.parquet\")\n", + " tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n", + " tiled_data[ds_name] = (slides, tiles)\n", + " print(f\"{ds_name}: {len(slides)} slides, {len(tiles)} tiles\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2828c601", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Validate tiled datasets ----\n", + "expected_stems = {Path(p).stem for p in df[\"path\"]}\n", + "\n", + "for ds_name, (slides, tiles) in tiled_data.items():\n", + " print(f\"=== {ds_name} ===\\n\")\n", + "\n", + " # 1. slides.parquet path stem ↔ data_mapping path stem\n", + " slide_stems = {Path(p).stem for p in slides[\"path\"]}\n", + " missing_from_dataset = expected_stems - slide_stems\n", + " extra_in_dataset = slide_stems - expected_stems\n", + "\n", + " print(f\"Slides in data_mapping: {len(expected_stems)}\")\n", + " print(f\"Slides in dataset: {len(slide_stems)}\")\n", + " print(f\"Missing from dataset: {len(missing_from_dataset)}\")\n", + " print(f\"Extra in dataset: {len(extra_in_dataset)}\")\n", + "\n", + " if missing_from_dataset:\n", + " print(f\"\\nSlides in data_mapping but NOT in {ds_name}:\")\n", + " for p in sorted(missing_from_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " if extra_in_dataset:\n", + " print(f\"\\nSlides in {ds_name} but NOT in data_mapping:\")\n", + " for p in sorted(extra_in_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " # 2. Every slide has at least one tile\n", + " slide_ids = set(slides[\"id\"])\n", + " slide_ids_in_tiles = set(tiles[\"slide_id\"])\n", + "\n", + " slides_without_tiles = slide_ids - slide_ids_in_tiles\n", + " print(f\"\\nSlides without any tiles: {len(slides_without_tiles)}\")\n", + " if slides_without_tiles:\n", + " for s in sorted(slides_without_tiles):\n", + " print(f\" {s}\")\n", + "\n", + " # 3. No orphan tiles (tile references a slide not in slides.parquet)\n", + " orphan_tile_ids = slide_ids_in_tiles - slide_ids\n", + " print(f\"Orphan tile slide_ids: {len(orphan_tile_ids)}\")\n", + " if orphan_tile_ids:\n", + " for sid in sorted(orphan_tile_ids):\n", + " count = int((tiles[\"slide_id\"] == sid).sum())\n", + " print(f\" slide_id={sid}: {count} tiles\")\n", + "\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "ea2eae31", + "metadata": {}, + "source": [ + "## Embeddings validation\n", + "\n", + "For each embeddings artifact, check that every slide in `data_mapping.csv` has a corresponding `.parquet` file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc5ce48a", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Download and validate embeddings ----\n", + "# Embedding parquet files are named as \"{record_num}.parquet\" (e.g. \"2023/00835.parquet\")\n", + "expected_record_nums = set(df[\"path\"])\n", + "\n", + "for emb_name, uri in EMBEDDINGS.items():\n", + " print(f\"=== {emb_name} ===\\n\")\n", + " emb_dir = Path(download_artifacts(uri))\n", + " emb_files = {p.stem for p in emb_dir.iterdir() if p.is_file() and p.suffix == \".parquet\"}\n", + " print(f\"Embedding files found: {len(emb_files)}\")\n", + " print(f\"Slides in data_mapping: {len(expected_record_nums)}\")\n", + "\n", + " missing = expected_record_nums - emb_files\n", + " extra = emb_files - expected_record_nums\n", + "\n", + " print(f\"Missing embeddings: {len(missing)}\")\n", + " if missing:\n", + " for s in sorted(missing):\n", + " print(f\" {s}\")\n", + "\n", + " print(f\"Extra embeddings: {len(extra)}\")\n", + " if extra:\n", + " for s in sorted(extra):\n", + " print(f\" {s}\")\n", + "\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/notebooks/data_exploration_script.ipynb b/scripts/notebooks/data_exploration_script.ipynb new file mode 100644 index 00000000..78c79e57 --- /dev/null +++ b/scripts/notebooks/data_exploration_script.ipynb @@ -0,0 +1,3014 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "96a77f75-ac46-4c07-831c-ea759dd0f5f2", + "metadata": {}, + "source": [ + "# 0. Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ed00c175-8e62-4ee0-8944-aa6a5b0a623c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmpoutm06dy\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (2.3.3)\n", + "Requirement already satisfied: numpy in /opt/venv/lib/python3.14/site-packages (2.4.6)\n", + "Requirement already satisfied: openpyxl in /opt/venv/lib/python3.14/site-packages (3.1.5)\n", + "Requirement already satisfied: matplotlib in /opt/venv/lib/python3.14/site-packages (3.11.0)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.14/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/venv/lib/python3.14/site-packages (from pandas) (2026.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/venv/lib/python3.14/site-packages (from pandas) (2025.2)\n", + "Requirement already satisfied: et-xmlfile in /opt/venv/lib/python3.14/site-packages (from openpyxl) (2.0.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (4.63.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.5.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (25.0)\n", + "Requirement already satisfied: pillow>=9 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (12.2.0)\n", + "Requirement already satisfied: pyparsing>=3 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (3.3.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/venv/lib/python3.14/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas numpy openpyxl matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "439e5094-e266-4106-b61b-3d69e71ccb7a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9e671a9f", + "metadata": {}, + "outputs": [], + "source": [ + "def search_df(df, *substrings):\n", + " \"\"\"Display rows containing all requested substrings so source-table issues can be inspected quickly.\"\"\"\n", + " for s in substrings:\n", + " df = df[df.apply(lambda row, s=s: row.astype(str).str.contains(s).any(), axis=1)]\n", + " display(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4c83e452-9159-4b26-b25f-21facd7ebb50", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_PATH = Path(\"/mnt/data/MOU/breast/mammaprint/\")\n", + "\n", + "DATA_FILES = [\n", + " DATA_PATH / \"data_2023-09-25.xlsx\",\n", + " DATA_PATH / \"data_test_set_2024-02-16.xlsx\",\n", + " DATA_PATH / \"data-dodatek-2023-10-19.xlsx\",\n", + " DATA_PATH / \"Mammaprint-surova data.xlsx\",\n", + " DATA_PATH / \"Soubor RNA Mammaprint-1_10_25_pro AI.xlsx\",\n", + " DATA_PATH / \"Mammaprint_anonymizovany.xlsx\"\n", + " \n", + "]\n", + "\n", + "VALID_TYPES = [\"a luminal\", \"b luminal\"]\n", + "INDEX_PATTERN = re.compile(r\"^(\\d{4})/\\s*(\\d{1,5})(?:-(\\d))?$\")\n", + "FILENAME_PATTERN = re.compile(r\"^(P?)(\\d{4})_(\\d{1,5})(?:-(\\d))?$\")\n", + "\n", + "\n", + "COLUMN_MAPPING = {\n", + " \"číslo záznamu\": \"record_num\",\n", + " \"mammaprint index\": \"mammaprint_index\",\n", + " \"typ\": \"type\",\n", + "}\n", + "NON_MAPPING_COLS = [\"source_table\", \"original_index\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9751dfcb-7255-440c-92e5-33a508226e4b", + "metadata": {}, + "outputs": [], + "source": [ + "INDICES_MAPPING = {\n", + " \"2023/3222 a\": \"2023/03222\",\n", + " \"2023/3222 b\": \"2023/03222\",\n", + " \"2024/2324ii\": \"2024/02324-2\",\n", + " \"2023/3206 ii\": \"2023/03206\",\n", + "}\n", + "FILES_MAPPING = {\n", + " \"P2023_03525-35\": \"P2023_03525-1\",\n", + " \"P2023_03525-23\": \"P2023_03525-2\",\n", + " \"P2025_00968\": \"P2025_00968-1\",\n", + "}\n", + "\n", + "# record_num\n", + "DROP_ROWS = [\n", + " # tbd with doctor\n", + "]\n", + "\n", + "RECORD_NUM_CORRECTIONS = [\n", + " {\n", + " \"source_table\": \"Mammaprint-surova data:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint-surova data:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"data_test_set_2024-02-16:List1\",\n", + " \"original_record_num\": \"2023/03525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"data_test_set_2024-02-16:List1\",\n", + " \"original_record_num\": \"2023/03525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Soubor RNA Mammaprint-1_10_25_pro AI:2025\",\n", + " \"original_record_num\": \"2025/968a\",\n", + " \"mammaprint_index\": 0.541,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2025/00968-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Soubor RNA Mammaprint-1_10_25_pro AI:2025\",\n", + " \"original_record_num\": \"2025/968b\",\n", + " \"mammaprint_index\": -0.076,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2025/00968-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2025\",\n", + " \"original_record_num\": \"2025/968a\",\n", + " \"mammaprint_index\": 0.541,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2025/00968-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2025\",\n", + " \"original_record_num\": \"2025/968b\",\n", + " \"mammaprint_index\": -0.076,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2025/00968-2\",\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "7aa885f7-5981-4039-98ec-5c70cf90b470", + "metadata": {}, + "source": [ + "# Part 1: Table Loading & Merging" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5fefc716-eab5-4e46-b235-b636c9790f70", + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_excel_text(value):\n", + " \"\"\"Normalize raw Excel text so header and value comparisons are consistent across files.\"\"\"\n", + " if isinstance(value, str):\n", + " return value.replace(\"\\n\", \" \").strip().lower()\n", + " return value\n", + "\n", + "\n", + "def normalize_excel_frame(df: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Normalize all cell text and column names so downstream mapping sees one canonical format.\"\"\"\n", + " df = df.map(normalize_excel_text)\n", + " df.columns = df.columns.map(normalize_excel_text)\n", + " return df\n", + "\n", + "\n", + "def find_header_row(raw: pd.DataFrame) -> int | None:\n", + " \"\"\"Find the real table header row because source spreadsheets start at different offsets.\"\"\"\n", + " normalized = raw.map(normalize_excel_text)\n", + "\n", + " for i, row in normalized.iterrows():\n", + " row_values = row.tolist()\n", + " if (\n", + " \"číslo záznamu\" in row_values\n", + " and \"mammaprint index\" in row_values\n", + " and \"typ\" in row_values\n", + " ):\n", + " return i\n", + "\n", + " return None\n", + "\n", + "\n", + "def _matches_correction_value(series: pd.Series, expected):\n", + " \"\"\"Compare correction selectors robustly so known fixes match exactly one intended row.\"\"\"\n", + " if isinstance(expected, float):\n", + " values = pd.to_numeric(series, errors=\"coerce\")\n", + " return (values - expected).abs() <= 1e-9\n", + " return series.astype(str).str.strip().str.lower() == str(expected).strip().lower()\n", + "\n", + "\n", + "def apply_record_num_corrections(\n", + " df: pd.DataFrame, path: Path, sheet_name: str\n", + ") -> pd.DataFrame:\n", + " \"\"\"Apply hand-verified record number fixes so ambiguous duplicate entries keep distinct IDs.\"\"\"\n", + " source_table = f\"{Path(path).stem}:{sheet_name}\"\n", + " record_col = \"číslo záznamu\" if \"číslo záznamu\" in df.columns else df.columns[0]\n", + " corrections = [\n", + " correction\n", + " for correction in RECORD_NUM_CORRECTIONS\n", + " if correction[\"source_table\"] == source_table\n", + " ]\n", + "\n", + " for correction in corrections:\n", + " mask = _matches_correction_value(\n", + " df[record_col], correction[\"original_record_num\"]\n", + " )\n", + " mask &= _matches_correction_value(\n", + " df[\"mammaprint index\"], correction[\"mammaprint_index\"]\n", + " )\n", + " mask &= _matches_correction_value(df[\"typ\"], correction[\"type\"])\n", + "\n", + " matches = df.index[mask].tolist()\n", + " if len(matches) != 1:\n", + " raise ValueError(\n", + " f\"Expected exactly one row for record correction {correction}, \"\n", + " f\"found {len(matches)}\"\n", + " )\n", + "\n", + " df.loc[matches[0], record_col] = correction[\"corrected_record_num\"]\n", + "\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "398a3766-6c0a-4a49-b54e-8c2a759ccbc6", + "metadata": {}, + "outputs": [], + "source": [ + "def read_diverse_excel(path: Path, sheet_name: str):\n", + " \"\"\"Load one heterogeneous Excel sheet into a normalized table for merging with other sources.\"\"\"\n", + " # Step 1: read without a header so we can inspect candidate header rows\n", + " raw = pd.read_excel(path, sheet_name=sheet_name, header=None)\n", + "\n", + " # Step 2: find the real header row using normalized candidate rows\n", + " header_row = find_header_row(raw)\n", + "\n", + " # Step 3: load/build the dataframe using the detected header\n", + " if header_row is not None:\n", + " df = pd.read_excel(path, sheet_name=sheet_name, header=header_row)\n", + " else:\n", + " # no header found -> assume first three columns correspond to the expected fields\n", + " df = raw.copy()\n", + " expected_cols = [\"číslo záznamu\", \"mammaprint index\", \"typ\"]\n", + " rest_cols = [f\"col_{i}\" for i in range(df.shape[1] - 3)]\n", + " df.columns = expected_cols + rest_cols\n", + "\n", + " # Step 4: normalize columns and string cells after the table shape is known\n", + " df = normalize_excel_frame(df)\n", + "\n", + " # Step 5: apply known record-number fixes, then normalize inserted values\n", + " df = apply_record_num_corrections(df, path, sheet_name)\n", + " df = normalize_excel_frame(df)\n", + "\n", + " # Step 6: backfill repeated-table IDs and continue with common cleanup\n", + " df.iloc[:, 0] = df.iloc[:, 0].ffill() # use id of previous entry\n", + " df = df.dropna(how=\"all\")\n", + "\n", + " df[\"source_table\"] = f\"{Path(path).stem}:{sheet_name}\"\n", + " df[\"original_index\"] = df.iloc[:, 0].copy()\n", + "\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ce73a8e4-0113-42cd-b2fa-b5e41c1ee350", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading /mnt/data/MOU/breast/mammaprint/data_2023-09-25.xlsx...\n", + " - Sheet 'List1': 263 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/data_test_set_2024-02-16.xlsx...\n", + " - Sheet 'List1': 204 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/data-dodatek-2023-10-19.xlsx...\n", + " - Sheet 'List1': 37 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Mammaprint-surova data.xlsx...\n", + " - Sheet '2024': 602 rows\n", + " - Sheet '2023': 417 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Soubor RNA Mammaprint-1_10_25_pro AI.xlsx...\n", + " - Sheet '2025': 697 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Mammaprint_anonymizovany.xlsx...\n", + " - Sheet '2026': 184 rows\n", + " - Sheet '2025': 871 rows\n", + " - Sheet '2024': 760 rows\n", + " - Sheet '2023': 419 rows\n", + "\n", + "Total rows: 4444\n" + ] + } + ], + "source": [ + "all_dfs = []\n", + "\n", + "for file in DATA_FILES:\n", + " print(f\"Reading {file}...\")\n", + " excel_file = pd.ExcelFile(file)\n", + "\n", + " for sheet_name in excel_file.sheet_names:\n", + " df = read_diverse_excel(file, sheet_name)\n", + " print(f\" - Sheet '{sheet_name}': {len(df)} rows\")\n", + " all_dfs.append(df)\n", + "\n", + "merged_df = pd.concat(all_dfs, ignore_index=True).drop_duplicates()\n", + "print(f\"\\nTotal rows: {len(merged_df)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "405565e5-528a-48a0-86aa-4480f37ce260", + "metadata": {}, + "source": [ + "### Modify columns\n", + "- rename columns\n", + "- join data in unnamed columns into one called `note`" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c2627666-5788-4e53-b9bb-db4c1df02ed5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mapped columns: {'číslo záznamu': 'record_num', 'mammaprint index': 'mammaprint_index', 'typ': 'type'}\n" + ] + } + ], + "source": [ + "mapped_cols = {}\n", + "\n", + "for col in merged_df.columns:\n", + " col_lower = col.lower().strip()\n", + " for source_name, target_name in COLUMN_MAPPING.items():\n", + " if col_lower == source_name.lower():\n", + " mapped_cols[col] = target_name\n", + " break\n", + "\n", + "print(f\"Mapped columns: {mapped_cols}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1899428a-5a0f-4258-b897-256a3401a178", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Result dataframe: 4444 rows, 6 columns\n" + ] + } + ], + "source": [ + "result_df = pd.DataFrame()\n", + "\n", + "for source_col, target_col in mapped_cols.items():\n", + " result_df[target_col] = merged_df[source_col]\n", + "\n", + "result_df[NON_MAPPING_COLS] = merged_df[NON_MAPPING_COLS]\n", + "other_cols = [\n", + " col\n", + " for col in merged_df.columns\n", + " if col not in mapped_cols and col not in NON_MAPPING_COLS\n", + "]\n", + "\n", + "\n", + "def merge_columns_to_note(row):\n", + " \"\"\"Collect non-mapping source columns into a note so useful audit context is not lost.\"\"\"\n", + " notes = []\n", + " for col in other_cols:\n", + " value = row[col]\n", + " if pd.notna(value) and str(value).strip() != \"\":\n", + " notes.append(f\"{col}: {value}\")\n", + " return \"; \".join(notes) if notes else \"\"\n", + "\n", + "\n", + "result_df[\"note\"] = merged_df.apply(merge_columns_to_note, axis=1)\n", + "\n", + "print(f\"\\nResult dataframe: {len(result_df)} rows, {len(result_df.columns)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "c02bbbf2-f880-41c3-97f5-b44705ecd04c", + "metadata": {}, + "source": [ + "# Part 2: Table data quality" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1ce20d87-e629-4b20-8327-091c8454a10b", + "metadata": {}, + "outputs": [], + "source": [ + "def add_entry_quality_flag(df, invalid, flag):\n", + " \"\"\"Append a quality flag to invalid rows so multiple issues can be tracked per entry.\"\"\"\n", + " def append_flag(current):\n", + " if current == \"OK\" or pd.isna(current):\n", + " return flag\n", + " flags = str(current).split(\";\")\n", + " if flag in flags:\n", + " return current\n", + " return \";\".join(flags + [flag])\n", + "\n", + " df.loc[invalid, \"entry_quality\"] = df.loc[invalid, \"entry_quality\"].apply(\n", + " append_flag\n", + " )\n", + "\n", + "\n", + "def has_entry_quality_flag(series, flag):\n", + " \"\"\"Check semicolon-delimited quality flags so later steps can skip known-bad rows.\"\"\"\n", + " return series.astype(str).str.split(\";\").apply(lambda flags: flag in flags)\n", + "\n", + "\n", + "def validate_mammaprint_index(df):\n", + " \"\"\"Flag invalid Mammaprint values so only numeric scores in the expected range pass.\"\"\"\n", + " mammaprint_values = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\")\n", + " invalid = (\n", + " mammaprint_values.isna() | (mammaprint_values < -1) | (mammaprint_values > 1)\n", + " )\n", + " add_entry_quality_flag(df, invalid, \"MAMMAPRINT\")\n", + "\n", + "\n", + "def validate_type(df):\n", + " \"\"\"Flag invalid luminal types so downstream labels use the supported class names only.\"\"\"\n", + " invalid = df[\"type\"].isna() | ~df[\"type\"].astype(str).str.strip().isin(VALID_TYPES)\n", + " add_entry_quality_flag(df, invalid, \"TYPE\")\n", + "\n", + "\n", + "def validate_luminal_interval(df):\n", + " \"\"\"Flag type-score disagreements so A/B luminal labels match the Mammaprint sign.\"\"\"\n", + " mammaprint_values = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\")\n", + " normalized_type = df[\"type\"].astype(str).str.strip().str.lower()\n", + "\n", + " invalid_a = (\n", + " normalized_type.eq(\"a luminal\")\n", + " & mammaprint_values.notna()\n", + " & ~((mammaprint_values > 0) & (mammaprint_values <= 1))\n", + " )\n", + " invalid_b = (\n", + " normalized_type.eq(\"b luminal\")\n", + " & mammaprint_values.notna()\n", + " & ~((mammaprint_values >= -1) & (mammaprint_values < 0))\n", + " )\n", + "\n", + " add_entry_quality_flag(df, invalid_a | invalid_b, \"LUMINAL_INTERVAL\")\n", + "\n", + "\n", + "def validate_record_num(df):\n", + " \"\"\"Flag malformed record numbers before normalization so unexpected IDs are reviewed.\"\"\"\n", + " invalid = df[\"record_num\"].isna() | (\n", + " ~df[\"record_num\"]\n", + " .astype(str)\n", + " .str.fullmatch(INDEX_PATTERN) # doesn't match regex\n", + " & ~df[\"record_num\"].astype(str).isin(INDICES_MAPPING.keys()) # not in dict keys\n", + " )\n", + " add_entry_quality_flag(df, invalid, \"RECORD_NUM\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "81d280de-8b60-40a3-ae1e-ec6b7e0a1330", + "metadata": {}, + "outputs": [], + "source": [ + "result_df[\"entry_quality\"] = \"OK\"\n", + "validate_mammaprint_index(result_df)\n", + "validate_type(result_df)\n", + "validate_luminal_interval(result_df)\n", + "validate_record_num(result_df)" + ] + }, + { + "cell_type": "markdown", + "id": "c122dd0e-f201-4a64-89c0-a334465a9eb4", + "metadata": {}, + "source": [ + "### Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0f74fd0b-6c98-46eb-ade2-314c72477c86", + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_index(idx: str) -> str:\n", + " \"\"\"Canonicalize record numbers so table entries and slide filenames compare reliably.\"\"\"\n", + " if idx in INDICES_MAPPING:\n", + " return INDICES_MAPPING[idx]\n", + "\n", + " match = INDEX_PATTERN.match(idx)\n", + "\n", + " if match:\n", + " year, num, rank = match.groups()\n", + " num_padded = num.zfill(5) # pad middle to 5 digits\n", + " if rank:\n", + " return f\"{year}/{num_padded}-{rank}\"\n", + " else:\n", + " return f\"{year}/{num_padded}\"\n", + " else:\n", + " raise ValueError(f\"Flagged (unexpected format or suffix): '{idx}'\")\n", + "\n", + "\n", + "def normalize_indices(result_df):\n", + " \"\"\"Normalize only valid record numbers so invalid rows stay visible for quality review.\"\"\"\n", + " valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + " result_df.loc[valid_record_num, \"record_num\"] = result_df.loc[\n", + " valid_record_num, \"record_num\"\n", + " ].apply(normalize_index)\n", + "\n", + "\n", + "def flag_conflicting_records(result_df):\n", + " \"\"\"Mark duplicate OK records as conflicts while keeping retry/non-final rows separate.\"\"\"\n", + " conflicting_records = []\n", + " grouped = result_df.groupby(\"record_num\")\n", + "\n", + " for record_num, group in grouped:\n", + " if len(group) == 1: # just one entry, correct\n", + " continue\n", + "\n", + " if (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() == 0: # no OK entry, nothing to flag further\n", + " continue\n", + "\n", + " elif (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() == 1: # one OK entry (retry), others non-needed\n", + " result_df.loc[\n", + " (result_df[\"record_num\"] == record_num)\n", + " & (result_df[\"entry_quality\"] != \"OK\"),\n", + " \"entry_quality\",\n", + " ] = \"NON-FINAL\"\n", + "\n", + " elif (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() >= 2: # multiple OK entries per one id, conflict\n", + " conflicting_records.append(record_num)\n", + " result_df.loc[result_df[\"record_num\"] == record_num, \"entry_quality\"] = (\n", + " \"CONFLICT\"\n", + " )\n", + "\n", + " print(f\"Conflicting record_nums: {len(conflicting_records)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e92839d6-25dc-4986-821a-8a6b213e463d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conflicting record_nums: 0\n" + ] + } + ], + "source": [ + "normalize_indices(result_df)\n", + "result_df = result_df.drop_duplicates(\n", + " subset=[\"record_num\", \"type\", \"mammaprint_index\", \"entry_quality\"]\n", + ")\n", + "flag_conflicting_records(result_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "same-stem-record-debug", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Record stems with multiple record_nums: 4\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_numsrow_countsource_tablesentry_quality
record_num_stem
2023/00946[2023/00946, 2023/00946-2]4[Mammaprint-surova data:2023, data_2023-09-25:...[NON-FINAL, OK]
2023/03525[2023/03525-1, 2023/03525-2]2[data_test_set_2024-02-16:List1][OK]
2024/02324[2024/02324, 2024/02324-2]2[Mammaprint-surova data:2024][MAMMAPRINT;TYPE]
2025/00968[2025/00968-1, 2025/00968-2]2[Soubor RNA Mammaprint-1_10_25_pro AI:2025][OK]
\n", + "
" + ], + "text/plain": [ + " record_nums row_count \\\n", + "record_num_stem \n", + "2023/00946 [2023/00946, 2023/00946-2] 4 \n", + "2023/03525 [2023/03525-1, 2023/03525-2] 2 \n", + "2024/02324 [2024/02324, 2024/02324-2] 2 \n", + "2025/00968 [2025/00968-1, 2025/00968-2] 2 \n", + "\n", + " source_tables \\\n", + "record_num_stem \n", + "2023/00946 [Mammaprint-surova data:2023, data_2023-09-25:... \n", + "2023/03525 [data_test_set_2024-02-16:List1] \n", + "2024/02324 [Mammaprint-surova data:2024] \n", + "2025/00968 [Soubor RNA Mammaprint-1_10_25_pro AI:2025] \n", + "\n", + " entry_quality \n", + "record_num_stem \n", + "2023/00946 [NON-FINAL, OK] \n", + "2023/03525 [OK] \n", + "2024/02324 [MAMMAPRINT;TYPE] \n", + "2025/00968 [OK] " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def record_num_stem(record_num):\n", + " \"\"\"Return the year/number stem without rank so related ranked records can be inspected together.\"\"\"\n", + " match = INDEX_PATTERN.match(str(record_num))\n", + " if not match:\n", + " return pd.NA\n", + "\n", + " year, num, _rank = match.groups()\n", + " return f\"{year}/{num.zfill(5)}\"\n", + "\n", + "\n", + "valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + "record_stem_debug_df = result_df[valid_record_num].copy()\n", + "record_stem_debug_df[\"record_num_stem\"] = record_stem_debug_df[\"record_num\"].apply(\n", + " record_num_stem\n", + ")\n", + "\n", + "same_stem_counts = record_stem_debug_df.groupby(\"record_num_stem\")[\n", + " \"record_num\"\n", + "].nunique()\n", + "same_stem_stems = same_stem_counts[same_stem_counts > 1].index\n", + "same_stem_records_df = record_stem_debug_df[\n", + " record_stem_debug_df[\"record_num_stem\"].isin(same_stem_stems)\n", + "].sort_values([\"record_num_stem\", \"record_num\", \"source_table\", \"original_index\"])\n", + "\n", + "same_stem_summary_df = same_stem_records_df.groupby(\"record_num_stem\").agg(\n", + " record_nums=(\"record_num\", lambda values: sorted(values.unique())),\n", + " row_count=(\"record_num\", \"size\"),\n", + " source_tables=(\"source_table\", lambda values: sorted(values.unique())),\n", + " entry_quality=(\"entry_quality\", lambda values: sorted(values.unique())),\n", + ")\n", + "\n", + "print(f\"Record stems with multiple record_nums: {len(same_stem_summary_df)}\")\n", + "display(same_stem_summary_df)" + ] + }, + { + "cell_type": "markdown", + "id": "aaad3976-5f93-49e9-9dfe-88f1a358f31e", + "metadata": {}, + "source": [ + "### Table quality results" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c3f3172f-29f0-48a4-96d1-80bf4fb5d68c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Quality Summary ===\n", + "OK: 2084 rows (90.5%)\n", + "NON-FINAL: 114 rows (5.0%)\n", + "MAMMAPRINT;TYPE: 89 rows (3.9%)\n", + "TYPE: 14 rows (0.6%)\n", + "MAMMAPRINT;TYPE;RECORD_NUM: 1 rows (0.0%)\n", + "LUMINAL_INTERVAL: 1 rows (0.0%)\n", + "Total: 2303 rows\n", + "\n", + "=== NON-FINAL ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
112023/00946nelzeNaNdata_2023-09-25:List12023/946NON-FINAL
292023/01108nelzeb luminaldata_2023-09-25:List12023/1108NON-FINAL
472023/01237nelzeNaNdata_2023-09-25:List12023/1237NON-FINAL
562023/01294nelzeNaNdata_2023-09-25:List12023/1294NON-FINAL
772023/01462nelzeNaNdata_2023-09-25:List12023/1462NON-FINAL
852023/01486nelzea luminaldata_2023-09-25:List12023/1486NON-FINAL
892023/01591nevyšetřenoNaNdata_2023-09-25:List12023/1591NON-FINAL
912023/01597nelzea luminaldata_2023-09-25:List12023/1597NON-FINAL
932023/01599nelzeb luminaldata_2023-09-25:List12023/1599NON-FINAL
1632023/02298nelzeNaNdata_2023-09-25:List12023/2298NON-FINAL
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "11 2023/00946 nelze NaN data_2023-09-25:List1 \n", + "29 2023/01108 nelze b luminal data_2023-09-25:List1 \n", + "47 2023/01237 nelze NaN data_2023-09-25:List1 \n", + "56 2023/01294 nelze NaN data_2023-09-25:List1 \n", + "77 2023/01462 nelze NaN data_2023-09-25:List1 \n", + "85 2023/01486 nelze a luminal data_2023-09-25:List1 \n", + "89 2023/01591 nevyšetřeno NaN data_2023-09-25:List1 \n", + "91 2023/01597 nelze a luminal data_2023-09-25:List1 \n", + "93 2023/01599 nelze b luminal data_2023-09-25:List1 \n", + "163 2023/02298 nelze NaN data_2023-09-25:List1 \n", + "\n", + " original_index note entry_quality \n", + "11 2023/946 NON-FINAL \n", + "29 2023/1108 NON-FINAL \n", + "47 2023/1237 NON-FINAL \n", + "56 2023/1294 NON-FINAL \n", + "77 2023/1462 NON-FINAL \n", + "85 2023/1486 NON-FINAL \n", + "89 2023/1591 NON-FINAL \n", + "91 2023/1597 NON-FINAL \n", + "93 2023/1599 NON-FINAL \n", + "163 2023/2298 NON-FINAL " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== MAMMAPRINT;TYPE ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
332023/01141nelzeNaNdata_2023-09-25:List12023/1141MAMMAPRINT;TYPE
532023/01268nevyšetřenoNaNdata_2023-09-25:List12023/1268MAMMAPRINT;TYPE
612023/01332nevyšetřenoNaNdata_2023-09-25:List12023/1332MAMMAPRINT;TYPE
822023/01473nevyšetřenoNaNdata_2023-09-25:List12023/1473MAMMAPRINT;TYPE
1442023/02114nelzeNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1452023/02114specialistaNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1462023/02114NaNNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1702023/02353nelzeNaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
1722023/02353specialistaNaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
1732023/023539NaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table original_index \\\n", + "33 2023/01141 nelze NaN data_2023-09-25:List1 2023/1141 \n", + "53 2023/01268 nevyšetřeno NaN data_2023-09-25:List1 2023/1268 \n", + "61 2023/01332 nevyšetřeno NaN data_2023-09-25:List1 2023/1332 \n", + "82 2023/01473 nevyšetřeno NaN data_2023-09-25:List1 2023/1473 \n", + "144 2023/02114 nelze NaN data_2023-09-25:List1 2023/2114 \n", + "145 2023/02114 specialista NaN data_2023-09-25:List1 2023/2114 \n", + "146 2023/02114 NaN NaN data_2023-09-25:List1 2023/2114 \n", + "170 2023/02353 nelze NaN data_2023-09-25:List1 2023/2353 \n", + "172 2023/02353 specialista NaN data_2023-09-25:List1 2023/2353 \n", + "173 2023/02353 9 NaN data_2023-09-25:List1 2023/2353 \n", + "\n", + " note entry_quality \n", + "33 MAMMAPRINT;TYPE \n", + "53 MAMMAPRINT;TYPE \n", + "61 MAMMAPRINT;TYPE \n", + "82 MAMMAPRINT;TYPE \n", + "144 MAMMAPRINT;TYPE \n", + "145 MAMMAPRINT;TYPE \n", + "146 MAMMAPRINT;TYPE \n", + "170 MAMMAPRINT;TYPE \n", + "172 MAMMAPRINT;TYPE \n", + "173 MAMMAPRINT;TYPE " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== TYPE ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
3692023/04116-0.841basal typedata_test_set_2024-02-16:List12023/04116unnamed: 3: výsledek z agendieTYPE
6062024/00783-1bazal typeMammaprint-surova data:20242024/783TYPE
6402024/01000-1bazal typeMammaprint-surova data:20242024/1000TYPE
15832025/00341-0.792basal typeSoubor RNA Mammaprint-1_10_25_pro AI:20252025/341podíl nádorových buněk: 0.7TYPE
16582025/00858-0.115her2Soubor RNA Mammaprint-1_10_25_pro AI:20252025/858podíl nádorových buněk: cca 15%TYPE
17942025/01756-1bazalSoubor RNA Mammaprint-1_10_25_pro AI:20252025/1756podíl nádorových buněk: 0.4TYPE
21742025/04071-0.892bazalSoubor RNA Mammaprint-1_10_25_pro AI:20252025/4071podíl nádorových buněk: 0.5TYPE
23692026/00971-1bazalMammaprint_anonymizovany:20262026/971unnamed: 4: 0.8; číslo biopsie: 2026/1934; čís...TYPE
31932025/04908-1bazalMammaprint_anonymizovany:20252025/4908unnamed: 4: 0.4; číslo biopsie: 2025/1033662; ...TYPE
32452025/051770.182NaNMammaprint_anonymizovany:20252025/5177unnamed: 4: 0.5; číslo biopsie: 2025/9864; čís...TYPE
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "369 2023/04116 -0.841 basal type \n", + "606 2024/00783 -1 bazal type \n", + "640 2024/01000 -1 bazal type \n", + "1583 2025/00341 -0.792 basal type \n", + "1658 2025/00858 -0.115 her2 \n", + "1794 2025/01756 -1 bazal \n", + "2174 2025/04071 -0.892 bazal \n", + "2369 2026/00971 -1 bazal \n", + "3193 2025/04908 -1 bazal \n", + "3245 2025/05177 0.182 NaN \n", + "\n", + " source_table original_index \\\n", + "369 data_test_set_2024-02-16:List1 2023/04116 \n", + "606 Mammaprint-surova data:2024 2024/783 \n", + "640 Mammaprint-surova data:2024 2024/1000 \n", + "1583 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/341 \n", + "1658 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/858 \n", + "1794 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/1756 \n", + "2174 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/4071 \n", + "2369 Mammaprint_anonymizovany:2026 2026/971 \n", + "3193 Mammaprint_anonymizovany:2025 2025/4908 \n", + "3245 Mammaprint_anonymizovany:2025 2025/5177 \n", + "\n", + " note entry_quality \n", + "369 unnamed: 3: výsledek z agendie TYPE \n", + "606 TYPE \n", + "640 TYPE \n", + "1583 podíl nádorových buněk: 0.7 TYPE \n", + "1658 podíl nádorových buněk: cca 15% TYPE \n", + "1794 podíl nádorových buněk: 0.4 TYPE \n", + "2174 podíl nádorových buněk: 0.5 TYPE \n", + "2369 unnamed: 4: 0.8; číslo biopsie: 2026/1934; čís... TYPE \n", + "3193 unnamed: 4: 0.4; číslo biopsie: 2025/1033662; ... TYPE \n", + "3245 unnamed: 4: 0.5; číslo biopsie: 2025/9864; čís... TYPE " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== MAMMAPRINT;TYPE;RECORD_NUM ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
2220NaNNaNNaNMammaprint_anonymizovany:2026NaNdisek.: anoMAMMAPRINT;TYPE;RECORD_NUM
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "2220 NaN NaN NaN Mammaprint_anonymizovany:2026 \n", + "\n", + " original_index note entry_quality \n", + "2220 NaN disek.: ano MAMMAPRINT;TYPE;RECORD_NUM " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== LUMINAL_INTERVAL ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
32012025/049530b luminalMammaprint_anonymizovany:20252025/4953unnamed: 4: 0.6; číslo biopsie: 2025/53612; ma...LUMINAL_INTERVAL
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "3201 2025/04953 0 b luminal Mammaprint_anonymizovany:2025 \n", + "\n", + " original_index note \\\n", + "3201 2025/4953 unnamed: 4: 0.6; číslo biopsie: 2025/53612; ma... \n", + "\n", + " entry_quality \n", + "3201 LUMINAL_INTERVAL " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\n=== Quality Summary ===\")\n", + "counts = result_df[\"entry_quality\"].value_counts()\n", + "\n", + "for label, count in counts.items():\n", + " pct = count / len(result_df) * 100\n", + " print(f\"{label}: {count} rows ({pct:.1f}%)\")\n", + "\n", + "print(f\"Total: {len(result_df)} rows\\n\")\n", + "\n", + "\n", + "for label in counts.keys():\n", + " if label == \"OK\":\n", + " continue\n", + "\n", + " # TODO non final print aj hodnotu co nasleduje\n", + " print(f\"=== {label} ===\")\n", + " display(result_df[result_df[\"entry_quality\"] == label].head(10))\n", + " \n", + " # # Save to CSV\n", + " # temp = result_df[result_df[\"entry_quality\"] == label]\n", + " # safe_label = label.replace(\";\", \"_\").replace(\"/\", \"_\")\n", + " # output_file = DATA_PATH / (\"entry_quality_\" + safe_label + \".csv\")\n", + " # temp.to_csv(output_file, index=False, encoding=\"utf-8\")\n", + " \n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "cf84e23f-a37e-4c60-9ca7-565e5e43e66b", + "metadata": {}, + "source": [ + "# Part 3: File System" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "2e052b41-926d-46b1-871f-1f4747010cb4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading filesystem listing from: /mnt/data/MOU/breast/mammaprint\n", + "Found 2124 folders\n", + "Found 2124 .mrxs files\n" + ] + } + ], + "source": [ + "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\n", + "FILESYSTEM_LISTING = DATA_PATH\n", + "print(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n", + "\n", + "listed_entries = [\n", + " Path(line.strip()).name\n", + " for line in os.listdir(FILESYSTEM_LISTING)\n", + " if line.strip()\n", + "]\n", + "\n", + "\n", + "def is_known_filename(stem):\n", + " \"\"\"Accept a slide stem if it matches the standard pattern OR is a FILES_MAPPING key.\n", + "\n", + " The original filename is kept as-is; the FILES_MAPPING alias is only applied later\n", + " at lookup time (in normalize_filename). This just prevents non-standard names\n", + " (e.g. two-digit ranks like P2023_03525-35) from being dropped by the format filter.\n", + " \"\"\"\n", + " return FILENAME_PATTERN.fullmatch(stem) is not None or stem in FILES_MAPPING\n", + "\n", + "\n", + "all_folders = {\n", + " entry\n", + " for entry in listed_entries\n", + " if Path(entry).suffix == \"\" and entry != \"annotations\" and is_known_filename(entry)\n", + "}\n", + "all_files = {\n", + " Path(entry).stem\n", + " for entry in listed_entries\n", + " if Path(entry).suffix == \".mrxs\" and is_known_filename(entry[:-5])\n", + "}\n", + "\n", + "print(f\"Found {len(all_folders)} folders\")\n", + "print(f\"Found {len(all_files)} .mrxs files\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "00368085-f81c-46db-939d-63b1e25baf45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Perfect 1:1 match between folders and .mrxs files\n" + ] + } + ], + "source": [ + "missing_folders = all_files - all_folders\n", + "missing_files = all_folders - all_files\n", + "\n", + "if not missing_folders and not missing_files:\n", + " print(\"✅ Perfect 1:1 match between folders and .mrxs files\")\n", + "else:\n", + " print(\"⚠️ Mismatch found:\")\n", + " if missing_folders:\n", + " print(\" Files without folders:\", missing_folders)\n", + " if missing_files:\n", + " print(\" Folders without files:\", missing_files)" + ] + }, + { + "cell_type": "markdown", + "id": "7ee9a88a-6fcb-4bbb-944d-b33b563fd4bd", + "metadata": {}, + "source": [ + "### Building mapping between table indices and files" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "bf1297d4-42ae-402c-9c76-51eeadf5b842", + "metadata": {}, + "outputs": [], + "source": [ + "invalid_filename_format_files = set()\n", + "\n", + "\n", + "def normalize_filename(fname):\n", + " \"\"\"Canonicalize slide filename stems so file matching can compare against record numbers.\"\"\"\n", + " if fname in FILES_MAPPING:\n", + " return FILES_MAPPING[fname]\n", + "\n", + " match = FILENAME_PATTERN.match(fname)\n", + " if match:\n", + " p, year, num, rank = match.groups()\n", + " num_padded = num.zfill(5)\n", + " if rank:\n", + " return f\"P{year}_{num_padded}-{rank}\"\n", + " else:\n", + " return f\"P{year}_{num_padded}\"\n", + " else:\n", + " invalid_filename_format_files.add(fname)\n", + " return None\n", + "\n", + "\n", + "def normalize_filename_rankless(fname):\n", + " \"\"\"Canonicalize a filename without rank so sibling slide files can be grouped together.\"\"\"\n", + " fname = normalize_filename(fname)\n", + " if fname is None:\n", + " return None\n", + " if \"-\" in fname:\n", + " return fname.split(\"-\")[0]\n", + " else:\n", + " return fname\n", + "\n", + "\n", + "def build_bi_dict(normalizing_f, data):\n", + " \"\"\"Build forward and reverse lookup dictionaries so normalized IDs can return source files.\"\"\"\n", + " data_to_norm: dict[str, str] = dict()\n", + " norm_to_data: dict[str, list[str]] = dict()\n", + "\n", + " for entry in data:\n", + " norm = normalizing_f(entry)\n", + " if norm is None:\n", + " continue\n", + " data_to_norm[entry] = norm\n", + "\n", + " if norm in norm_to_data:\n", + " norm_to_data[norm].append(entry)\n", + " else:\n", + " norm_to_data[norm] = [entry]\n", + "\n", + " return data_to_norm, norm_to_data\n", + "\n", + "\n", + "def index_to_filename(idx):\n", + " \"\"\"Convert a normalized record number to the matching normalized slide filename stem.\"\"\"\n", + " norm_idx = normalize_index(idx)\n", + " fname = norm_idx.replace(\"/\", \"_\")\n", + " return \"P\" + fname\n", + "\n", + "\n", + "def filename_to_index(fname):\n", + " \"\"\"Convert a normalized slide filename stem back to a record number for missing-entry rows.\"\"\"\n", + " norm_fname = normalize_filename(fname)\n", + " if norm_fname is None:\n", + " return None\n", + " if norm_fname[0] == \"P\":\n", + " norm_fname = norm_fname[1:]\n", + " idx = norm_fname.replace(\"_\", \"/\")\n", + " return idx" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "3dcdd970-4d25-4440-a384-b89322bfdea1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + } + ], + "source": [ + "norm_to_files = build_bi_dict(normalize_filename, all_files)[1]\n", + "norm_to_files_rankless = build_bi_dict(normalize_filename_rankless, all_files)[1]\n", + "if invalid_filename_format_files:\n", + " print(\"Files with unexpected filename format:\")\n", + " for file in sorted(invalid_filename_format_files):\n", + " print(f\" - {file}\")\n", + "\n", + "print([(norm, file) for norm, file in norm_to_files.items() if len(file) > 1])\n", + "\n", + "\n", + "def find_files_by_index(idx, existing_record_nums=None):\n", + " \"\"\"Find slide files while keeping ranked records tied to their exact ranked file.\"\"\"\n", + " existing_record_nums = existing_record_nums or set()\n", + "\n", + " # Step 1: convert table record number, e.g. 2025/00161, to normalized file stem P2025_00161.\n", + " target_file = index_to_filename(idx)\n", + " has_rank = \"-\" in target_file\n", + " rankless_target_file = target_file.split(\"-\")[0]\n", + "\n", + " # Step 2: keep exact normalized filename matches first.\n", + " # Examples: 2025/00161 -> P2025_00161, 2023/00946-2 -> P2023_00946-2.\n", + " exact_matches = norm_to_files.get(target_file, [])\n", + "\n", + " # Step 3: ranked records must not absorb sibling ranks when their exact file exists.\n", + " # Example: 2023/03525-1 should match P2023_03525-35 only, not P2023_03525-23.\n", + " if has_rank and len(exact_matches) > 0:\n", + " return exact_matches\n", + "\n", + " # Step 4: unranked records can collect sibling files that share the same rankless stem.\n", + " # Example: 2025/00161 can match both P2025_00161 and P2025_00161-2.\n", + " sibling_matches = norm_to_files_rankless.get(rankless_target_file, [])\n", + "\n", + " # Step 5: remove files that correspond to a different record already present in the table.\n", + " # Example: 2023/00946 should not also consume P2023_0946-2 when 2023/00946-2 exists.\n", + " res = []\n", + " for file in dict.fromkeys(exact_matches + sibling_matches):\n", + " file_record_num = filename_to_index(file)\n", + " if file_record_num != idx and file_record_num in existing_record_nums:\n", + " continue\n", + " res.append(file)\n", + "\n", + " return res" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "3dc78131-26ae-4261-9413-eefec64b3e98", + "metadata": {}, + "outputs": [], + "source": [ + "matched_files = set()\n", + "file_check_results = []\n", + "\n", + "valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + "existing_record_nums = set(result_df.loc[valid_record_num, \"record_num\"])\n", + "\n", + "for idx, row in result_df[valid_record_num].iterrows():\n", + " record_num = row[\"record_num\"]\n", + " files_list = find_files_by_index(record_num, existing_record_nums)\n", + " matched_files.update(files_list)\n", + "\n", + " issue = \"\"\n", + " if len(files_list) == 0:\n", + " issue = \"File missing\"\n", + " elif len(files_list) == 1:\n", + " issue = \"OK\"\n", + " else:\n", + " issue = \"Multiple files\"\n", + "\n", + " file_check_results.append(\n", + " {\n", + " \"record_num\": record_num,\n", + " \"matched_files\": files_list,\n", + " \"file_count\": len(files_list),\n", + " \"file_issue\": issue,\n", + " }\n", + " )\n", + "\n", + "valid_format_files = all_files - invalid_filename_format_files\n", + "unmatched_files = valid_format_files - matched_files\n", + "\n", + "for file in unmatched_files:\n", + " file_check_results.append(\n", + " {\n", + " \"record_num\": filename_to_index(file) + \"M\",\n", + " \"matched_files\": [file],\n", + " \"file_count\": 1,\n", + " \"file_issue\": \"entry missing\",\n", + " }\n", + " )\n", + "\n", + "file_check_df = pd.DataFrame(file_check_results)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "96b23164-4693-4675-af5b-51bbef6c15d1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Filesystem Summary ===\n", + "OK: 2080 rows (95.3%)\n", + "File missing: 77 rows (3.5%)\n", + "Multiple files: 19 rows (0.9%)\n", + "entry missing: 6 rows (0.3%)\n", + "Total: 2182 rows\n", + "\n", + "=== File missing ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
2052023/02656[]0File missing
2062023/02659[]0File missing
2072023/02660[]0File missing
2082023/02661[]0File missing
2092023/02662[]0File missing
2102023/02674[]0File missing
2112023/02689[]0File missing
2122023/02699[]0File missing
2132023/02701[]0File missing
2142023/02702[]0File missing
2152023/02703[]0File missing
2162023/02704[]0File missing
2192023/02720[]0File missing
5082024/00674[]0File missing
5292024/00816[]0File missing
5522024/00988[]0File missing
6682024/01799[]0File missing
6902024/01928[]0File missing
6992024/01997[]0File missing
7392024/02324-2[]0File missing
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "205 2023/02656 [] 0 File missing\n", + "206 2023/02659 [] 0 File missing\n", + "207 2023/02660 [] 0 File missing\n", + "208 2023/02661 [] 0 File missing\n", + "209 2023/02662 [] 0 File missing\n", + "210 2023/02674 [] 0 File missing\n", + "211 2023/02689 [] 0 File missing\n", + "212 2023/02699 [] 0 File missing\n", + "213 2023/02701 [] 0 File missing\n", + "214 2023/02702 [] 0 File missing\n", + "215 2023/02703 [] 0 File missing\n", + "216 2023/02704 [] 0 File missing\n", + "219 2023/02720 [] 0 File missing\n", + "508 2024/00674 [] 0 File missing\n", + "529 2024/00816 [] 0 File missing\n", + "552 2024/00988 [] 0 File missing\n", + "668 2024/01799 [] 0 File missing\n", + "690 2024/01928 [] 0 File missing\n", + "699 2024/01997 [] 0 File missing\n", + "739 2024/02324-2 [] 0 File missing" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== Multiple files ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
732023/01462[P2023_01462, P2023_01462-1]2Multiple files
792023/01486[P2023_01486-1, P2023_01486-2]2Multiple files
842023/01591[P2023_01591-2, P2023_01591-1]2Multiple files
862023/01597[P2023_01597-1, P2023_01597-2]2Multiple files
882023/01599[P2023_01599-2, P2023_01599-1]2Multiple files
942023/01656[P2023_01656-2, P2023_01656-1]2Multiple files
1752023/02395[P2023_02395-2, P2023_02395-1]2Multiple files
1802023/02444[P2023_02444-2, P2023_02444-1]2Multiple files
2002023/02617[P2023_02617-1, P2023_02617-2]2Multiple files
2172023/02712[P2023_02712-1, P2023_02712-2]2Multiple files
3112023/03714[P2023_03714-3, P2023_03714-4]2Multiple files
6732024/01804[P2024_01804-1, P2024_01804-2]2Multiple files
7972024/02666[P2024_02666-1, P2024_02666-2]2Multiple files
8332024/02851[2024_02851-1, P2024_02851-2]2Multiple files
10812025/00161[P2025_00161, P2025_00161-2]2Multiple files
15232025/02993[P2025_02993-2, P2025_02993-1]2Multiple files
15272025/03031[P2025_03031-2, P2025_03031-1]2Multiple files
15402025/03126[P2025_03126-2, P2025_03126-1]2Multiple files
16642025/03854[P2025_03854-2, P2025_03854-1]2Multiple files
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "73 2023/01462 [P2023_01462, P2023_01462-1] 2 Multiple files\n", + "79 2023/01486 [P2023_01486-1, P2023_01486-2] 2 Multiple files\n", + "84 2023/01591 [P2023_01591-2, P2023_01591-1] 2 Multiple files\n", + "86 2023/01597 [P2023_01597-1, P2023_01597-2] 2 Multiple files\n", + "88 2023/01599 [P2023_01599-2, P2023_01599-1] 2 Multiple files\n", + "94 2023/01656 [P2023_01656-2, P2023_01656-1] 2 Multiple files\n", + "175 2023/02395 [P2023_02395-2, P2023_02395-1] 2 Multiple files\n", + "180 2023/02444 [P2023_02444-2, P2023_02444-1] 2 Multiple files\n", + "200 2023/02617 [P2023_02617-1, P2023_02617-2] 2 Multiple files\n", + "217 2023/02712 [P2023_02712-1, P2023_02712-2] 2 Multiple files\n", + "311 2023/03714 [P2023_03714-3, P2023_03714-4] 2 Multiple files\n", + "673 2024/01804 [P2024_01804-1, P2024_01804-2] 2 Multiple files\n", + "797 2024/02666 [P2024_02666-1, P2024_02666-2] 2 Multiple files\n", + "833 2024/02851 [2024_02851-1, P2024_02851-2] 2 Multiple files\n", + "1081 2025/00161 [P2025_00161, P2025_00161-2] 2 Multiple files\n", + "1523 2025/02993 [P2025_02993-2, P2025_02993-1] 2 Multiple files\n", + "1527 2025/03031 [P2025_03031-2, P2025_03031-1] 2 Multiple files\n", + "1540 2025/03126 [P2025_03126-2, P2025_03126-1] 2 Multiple files\n", + "1664 2025/03854 [P2025_03854-2, P2025_03854-1] 2 Multiple files" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== entry missing ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
23022025/03567M[P2025_03567]1entry missing
23032025/00137M[P2025_00137]1entry missing
23042025/00310M[P2025_00310]1entry missing
23052023/01814M[P2023_01814]1entry missing
23062024/03383M[P2024_03383]1entry missing
23072023/00146-2M[P2023_0146-2]1entry missing
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "2302 2025/03567M [P2025_03567] 1 entry missing\n", + "2303 2025/00137M [P2025_00137] 1 entry missing\n", + "2304 2025/00310M [P2025_00310] 1 entry missing\n", + "2305 2023/01814M [P2023_01814] 1 entry missing\n", + "2306 2024/03383M [P2024_03383] 1 entry missing\n", + "2307 2023/00146-2M [P2023_0146-2] 1 entry missing" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\n=== Filesystem Summary ===\")\n", + "file_check_display = file_check_df.drop_duplicates(subset=[\"record_num\", \"file_issue\"])\n", + "counts = file_check_display[\"file_issue\"].value_counts()\n", + "\n", + "for label, count in counts.items():\n", + " pct = count / len(file_check_display) * 100\n", + " print(f\"{label}: {count} rows ({pct:.1f}%)\")\n", + "\n", + "print(f\"Total: {len(file_check_display)} rows\\n\")\n", + "\n", + "\n", + "for label in counts.keys():\n", + " if label == \"OK\":\n", + " continue\n", + "\n", + " print(f\"=== {label} ===\")\n", + " display(file_check_display[file_check_display[\"file_issue\"] == label].head(20))\n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "7d34c697-97f0-4e88-a16f-938cbdc50543", + "metadata": {}, + "source": [ + "# Part 4: Merge tables and filesystem" + ] + }, + { + "cell_type": "markdown", + "id": "52f77a9c-8d59-4d6f-a670-cd9070605629", + "metadata": {}, + "source": [ + "### Expand rows for multiple files" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bc477030-59f9-427d-a66b-b7ffb93de021", + "metadata": {}, + "outputs": [], + "source": [ + "final_rows = []\n", + "\n", + "for idx, row in file_check_df.iterrows():\n", + " matched_files = row[\"matched_files\"]\n", + "\n", + " if len(matched_files) == 0:\n", + " # No files found - add row with no path\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = None\n", + " final_rows.append(row_dict)\n", + " elif len(matched_files) == 1:\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = str(DATA_PATH / matched_files[0])\n", + " final_rows.append(row_dict)\n", + " else:\n", + " # Create one row per file\n", + " for file_name in matched_files:\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = str(DATA_PATH / file_name)\n", + " final_rows.append(row_dict)\n", + "\n", + "expanded_df = pd.DataFrame(final_rows).drop_duplicates()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "ffaae63c-5135-4c8a-8fd6-8bb57f3662f0", + "metadata": {}, + "outputs": [], + "source": [ + "final_df = result_df.merge(\n", + " expanded_df[[\"record_num\", \"path\", \"file_issue\"]], on=\"record_num\", how=\"outer\"\n", + ")\n", + "\n", + "final_df = final_df[\n", + " (final_df[\"file_issue\"].astype(str).isin([\"OK\", \"Multiple files\"]))\n", + " & (final_df[\"entry_quality\"] == \"OK\")\n", + "]\n", + "final_df = final_df[[\"record_num\", \"mammaprint_index\", \"type\", \"path\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "e66fd8a3", + "metadata": {}, + "outputs": [], + "source": [ + "if DROP_ROWS:\n", + " before = len(final_df)\n", + " mask = final_df[\"record_num\"].apply(\n", + " lambda x: any(sub in str(x) for sub in DROP_ROWS)\n", + " )\n", + " final_df = final_df[~mask]\n", + " print(f\"Dropped {before - len(final_df)} rows matching DROP_ROWS ({len(DROP_ROWS)} entries)\")\n", + " print(f\"Remaining: {len(final_df)} rows\")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a235c852", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Split counts (rows):\n", + "split\n", + "train 1429\n", + "test 308\n", + "val 305\n", + "Name: count, dtype: int64\n", + "\n", + "Split x type (rows):\n", + "type a luminal b luminal\n", + "split \n", + "test 162 146\n", + "train 754 675\n", + "val 161 144\n" + ] + } + ], + "source": [ + "# Assign a train/val/test split.\n", + "#\n", + "# The split is decided here and written into data_mapping.csv as a `split` column,\n", + "# so it propagates through tiling -> embeddings (each stage filters by split) and\n", + "# ends up as three separate embedding artifacts the training datamodule consumes.\n", + "#\n", + "# Split at the record_num (slide) level, not per row: a record can expand to several\n", + "# `path` rows (multi-file slides), and all of them must land in the same split to\n", + "# avoid slide-level leakage. Stratify on `type` and seed for reproducibility.\n", + "import numpy as np\n", + "\n", + "SPLIT_SEED = 42\n", + "SPLIT_RATIOS = {\"train\": 0.7, \"val\": 0.15, \"test\": 0.15}\n", + "\n", + "\n", + "def assign_split(df, ratios=SPLIT_RATIOS, seed=SPLIT_SEED):\n", + " \"\"\"Assign train/val/test per record_num, stratified on type, deterministically.\"\"\"\n", + " assert abs(sum(ratios.values()) - 1.0) < 1e-9, \"ratios must sum to 1\"\n", + " rng = np.random.default_rng(seed)\n", + "\n", + " # One split decision per record_num; use each record's (first) type for strata.\n", + " record_type = df.groupby(\"record_num\")[\"type\"].first()\n", + " split_of_record = {}\n", + "\n", + " for _type, records in record_type.groupby(record_type):\n", + " ids = records.index.to_numpy()\n", + " rng.shuffle(ids) # seeded: reproducible per class\n", + " n = len(ids)\n", + " n_train = int(n * ratios[\"train\"])\n", + " n_val = int(n * ratios[\"val\"])\n", + " for rid in ids[:n_train]:\n", + " split_of_record[rid] = \"train\"\n", + " for rid in ids[n_train : n_train + n_val]:\n", + " split_of_record[rid] = \"val\"\n", + " for rid in ids[n_train + n_val :]:\n", + " split_of_record[rid] = \"test\"\n", + "\n", + " df = df.copy()\n", + " df[\"split\"] = df[\"record_num\"].map(split_of_record)\n", + " return df\n", + "\n", + "\n", + "final_df = assign_split(final_df)\n", + "\n", + "# Sanity: no record_num spans multiple splits, and per-split type balance.\n", + "_per_record_splits = final_df.groupby(\"record_num\")[\"split\"].nunique()\n", + "assert (_per_record_splits == 1).all(), \"a record_num leaked across splits\"\n", + "print(\"Split counts (rows):\")\n", + "print(final_df[\"split\"].value_counts())\n", + "print(\"\\nSplit x type (rows):\")\n", + "print(final_df.groupby([\"split\", \"type\"]).size().unstack(fill_value=0))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "b90a7fac-9575-49f9-884c-28c1875dd435", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplit
14102025/00968-10.541a luminal/mnt/data/MOU/breast/mammaprint/P2025_00968val
14112025/00968-2-0.076b luminal/mnt/data/MOU/breast/mammaprint/P2025_00968-2train
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "1410 2025/00968-1 0.541 a luminal \n", + "1411 2025/00968-2 -0.076 b luminal \n", + "\n", + " path split \n", + "1410 /mnt/data/MOU/breast/mammaprint/P2025_00968 val \n", + "1411 /mnt/data/MOU/breast/mammaprint/P2025_00968-2 train " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplit
3752023/03525-10.235a luminal/mnt/data/MOU/breast/mammaprint/P2023_03525-35train
3762023/03525-2-0.345b luminal/mnt/data/MOU/breast/mammaprint/P2023_03525-23train
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "375 2023/03525-1 0.235 a luminal \n", + "376 2023/03525-2 -0.345 b luminal \n", + "\n", + " path split \n", + "375 /mnt/data/MOU/breast/mammaprint/P2023_03525-35 train \n", + "376 /mnt/data/MOU/breast/mammaprint/P2023_03525-23 train " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "search_df(final_df, \"2025\", \"968\")\n", + "search_df(final_df, \"2023\", \"3525\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "9b0bc82b-0738-4244-83da-5cbee0c2e1fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "✅ Saved final dataset to: data_mapping_final.csv\n", + " 2042 rows × 5 columns\n" + ] + } + ], + "source": [ + "# Save to CSV\n", + "output_file = \"data_mapping_final.csv\"\n", + "final_df.to_csv(output_file, index=False, encoding=\"utf-8\")\n", + "\n", + "print(f\"\\n✅ Saved final dataset to: {output_file}\")\n", + "print(f\" {len(final_df)} rows × {len(final_df.columns)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "8b80e36f-0baa-406d-b142-cbade0998513", + "metadata": {}, + "source": [ + "# Part 5: Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "a98b1bea-95e5-4672-b8c8-3408a22b0a55", + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_year(record_num):\n", + " \"\"\"Extract the record year so dataset coverage can be summarized by calendar year.\"\"\"\n", + " m = re.match(INDEX_PATTERN, record_num)\n", + " try:\n", + " return int(m.group(1))\n", + " except:\n", + " print(record_num)\n", + "\n", + "\n", + "def count_entries_per_year(df, plot=True, ax=None):\n", + " \"\"\"Count records by year so the final mapping can be checked for temporal coverage.\"\"\"\n", + " years = df[\"record_num\"].apply(_extract_year)\n", + " # years = years.dropna().astype(int)\n", + " counts = years.value_counts().sort_index()\n", + " result = {\"counts\": counts, \"total\": int(counts.sum())}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(8, 4))\n", + " created_ax = True\n", + " ax.bar(counts.index.astype(str), counts.values)\n", + " ax.set_title(\"Count of entries per year\")\n", + " ax.set_xlabel(\"Year\")\n", + " ax.set_ylabel(\"Count\")\n", + " ax.set_xticks(counts.index.astype(int))\n", + " ax.set_xticklabels([str(y) for y in counts.index], rotation=45)\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def mammaprint_distribution(df, plot=True, ax=None, bins=20, ddof=1):\n", + " \"\"\"Summarize Mammaprint scores so score range and distribution shifts are visible.\"\"\"\n", + " s = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\").dropna()\n", + " count = int(s.size)\n", + " mean = float(s.mean()) if count > 0 else float(\"nan\")\n", + " variance = float(s.var(ddof=ddof)) if count > 0 else float(\"nan\")\n", + "\n", + " result = {\"series\": s, \"mean\": mean, \"variance\": variance, \"count\": count}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(7, 4))\n", + " created_ax = True\n", + " ax.hist(s, bins=bins)\n", + " ax.axvline(mean, linestyle=\"--\", linewidth=1)\n", + " ax.set_title(\"MammaPrint index distribution\")\n", + " ax.set_xlabel(\"MammaPrint index\")\n", + " ax.set_ylabel(\"Frequency\")\n", + " ax.grid(True, linestyle=\":\", linewidth=0.5)\n", + " ax.text(\n", + " 0.98,\n", + " 0.95,\n", + " f\"mean={mean:.3f}\\nvar={variance:.4f}\\nn={count}\",\n", + " transform=ax.transAxes,\n", + " ha=\"right\",\n", + " va=\"top\",\n", + " fontsize=9,\n", + " bbox=dict(boxstyle=\"round\", alpha=0.1),\n", + " )\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def mammaprint_distribution_per_type(df, plot=True, bins=20, ddof=1):\n", + " \"\"\"Summarize Mammaprint scores per type so class-specific distributions can be compared.\"\"\"\n", + " results = {}\n", + " grouped = df.groupby(\"type\")\n", + "\n", + " for typ, sub in grouped:\n", + " s = pd.to_numeric(sub[\"mammaprint_index\"], errors=\"coerce\").dropna()\n", + " count = int(s.size)\n", + " mean = float(s.mean()) if count > 0 else float(\"nan\")\n", + " variance = float(s.var(ddof=ddof)) if count > 0 else float(\"nan\")\n", + " results[typ] = {\"series\": s, \"mean\": mean, \"variance\": variance, \"count\": count}\n", + "\n", + " if plot:\n", + " fig, ax = plt.subplots(figsize=(8, 4))\n", + " for typ, info in results.items():\n", + " if info[\"count\"] == 0:\n", + " continue\n", + " ax.hist(\n", + " info[\"series\"],\n", + " bins=bins,\n", + " alpha=0.45,\n", + " label=f\"{typ} (n={info['count']})\",\n", + " )\n", + " ax.set_title(\"MammaPrint index distribution by type\")\n", + " ax.set_xlabel(\"MammaPrint index\")\n", + " ax.set_ylabel(\"Frequency\")\n", + " ax.legend()\n", + " ax.grid(True, linestyle=\":\", linewidth=0.5)\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " summary_rows = []\n", + " for typ, info in results.items():\n", + " summary_rows.append(\n", + " {\n", + " \"type\": typ,\n", + " \"count\": info[\"count\"],\n", + " \"mean\": info[\"mean\"],\n", + " \"variance\": info[\"variance\"],\n", + " }\n", + " )\n", + " summary_df = pd.DataFrame(summary_rows).sort_values(\"count\", ascending=False)\n", + " print(\"\\nSummary (by type):\")\n", + " print(summary_df.to_string(index=False))\n", + "\n", + " return results\n", + "\n", + "\n", + "def count_distinct_types(df, plot=True, ax=None):\n", + " \"\"\"Count labels by type so class balance and missing type values can be reviewed.\"\"\"\n", + " counts = df[\"type\"].fillna(\"MISSING\").value_counts()\n", + " result = {\"counts\": counts, \"total_distinct\": int(counts.size)}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(6, 4))\n", + " created_ax = True\n", + " ax.barh(counts.index.astype(str), counts.values)\n", + " ax.set_title(\"Counts by type\")\n", + " ax.set_xlabel(\"Count\")\n", + " ax.set_ylabel(\"Type\")\n", + " for i, v in enumerate(counts.values):\n", + " ax.text(v, i, f\" {v}\", va=\"center\")\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def run_all_stats_and_plot(df, save_fig=None):\n", + " \"\"\"Run all final dataset summaries together so one cell produces the audit plots and metrics.\"\"\"\n", + " results = {}\n", + " results[\"per_year\"] = count_entries_per_year(df, plot=False)\n", + " results[\"mammaprint_overall\"] = mammaprint_distribution(df, plot=False)\n", + " results[\"mammaprint_by_type\"] = mammaprint_distribution_per_type(df, plot=False)\n", + " results[\"counts_by_type\"] = count_distinct_types(df, plot=False)\n", + "\n", + " fig, axs = plt.subplots(2, 2, figsize=(14, 9))\n", + " ax00, ax01, ax10, ax11 = axs.flatten()\n", + "\n", + " counts_year = results[\"per_year\"][\"counts\"]\n", + " if not counts_year.empty:\n", + " ax00.bar(counts_year.index.astype(str), counts_year.values)\n", + " ax00.set_title(\"Entries per year\")\n", + " ax00.set_xlabel(\"Year\")\n", + " ax00.set_ylabel(\"Count\")\n", + " ax00.tick_params(axis=\"x\", rotation=45)\n", + "\n", + " overall_series = results[\"mammaprint_overall\"][\"series\"]\n", + " if overall_series.size > 0:\n", + " ax01.hist(overall_series, bins=20)\n", + " ax01.axvline(results[\"mammaprint_overall\"][\"mean\"], linestyle=\"--\", linewidth=1)\n", + " ax01.set_title(\"MammaPrint index (overall)\")\n", + " ax01.set_xlabel(\"Index\")\n", + " ax01.set_ylabel(\"Frequency\")\n", + "\n", + " counts_type = results[\"counts_by_type\"][\"counts\"]\n", + " if not counts_type.empty:\n", + " ax10.barh(counts_type.index.astype(str), counts_type.values)\n", + " for i, v in enumerate(counts_type.values):\n", + " ax10.text(v, i, f\" {v}\", va=\"center\")\n", + " ax10.set_title(\"Counts by type\")\n", + " ax10.set_xlabel(\"Count\")\n", + "\n", + " for typ, info in results[\"mammaprint_by_type\"].items():\n", + " if info[\"count\"] == 0:\n", + " continue\n", + " ax11.hist(\n", + " info[\"series\"], bins=20, alpha=0.45, label=f\"{typ} (n={info['count']})\"\n", + " )\n", + " ax11.set_title(\"MammaPrint index by type\")\n", + " ax11.set_xlabel(\"Index\")\n", + " ax11.set_ylabel(\"Frequency\")\n", + " if len(results[\"mammaprint_by_type\"]) > 0:\n", + " ax11.legend()\n", + "\n", + " plt.tight_layout()\n", + "\n", + " if save_fig:\n", + " fig.savefig(save_fig, dpi=150)\n", + " print(f\"Saved combined figure to: {save_fig}\")\n", + "\n", + " plt.show()\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "86118b8b-0092-4c58-8758-2e07137b7082", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved combined figure to: stats\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABW0AAAN5CAYAAABzABkEAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAA+NhJREFUeJzs3Xd4FFX7//FPGgmkAaEkhNAk9CogvfeOVEHpgmClKAIqXdEHeMDKgzQFFRVBkCLSpIj0Kr0JhNBbEhJSSM7vD37s1zUJJCFkN8v7dV1zXeyZe87cM0yyk3vPnnEyxhgBAAAAAAAAAOyCs60TAAAAAAAAAAD8H4q2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQDYkXfffVdOTk62TgMAAABIlTfffFMeHh6Zfn/Tpk2Tk5OTLl26lO59P4wxRlWqVNHQoUMzfN+PS4kSJdSxY8cHtiUkJKhMmTJ69913Mzo9wK5RtAXwRJs8ebKcnJySXb766qs09fvmm2/K1dU1fZMFAADAI9u3b5/lXm/s2LFJxrz99tuWmJMnT2Zwhhnvn+fEyclJzs7Oypkzp5o0aaLff/89Q3J49dVX5eXllSH7slfz58/XkSNH9Pbbb9s6lQzl7OyssWPHasqUKTp37pyt0wHsBkVbAJC0detWGWMSLb169crQPCZMmCBjTIbuEwAA4Enk6empr7/+OtG9V3x8vL755ht5enraKDPbGT9+vIwxiouL08aNGxUZGanGjRtrw4YND9128uTJio6OfvxJ2mh/GWHixInq0qWL8uTJY+tUMtyzzz6rnDlzaurUqbZOBbAbFG0BAAAAAE+c9u3b6++//9amTZus2n/77TdduHBBHTp0sFFmtufi4qKyZctq7ty5io+P16effmrrlBzehg0bdPToUXXv3t3WqdiEs7Ozunbtqq+//trhivFAWlG0BYAUuv+VrcjISL344ovKnj27fH199fzzzyssLMwS17FjR02ZMkXx8fFWXzO7/9W6+/2Eh4erT58+8vPzU3BwsKTk57Q9fPiwOnfurNy5cytLliwqUaKEpk6dajUy5MyZM3r++ecVGBiobNmyqXTp0ho3bpwiIyNTdFy3bt3S888/L19fX+XIkUM9e/bUtWvX0pTLg47x3z7//HM5OTlp165didbNnDlTTk5O+vPPP1O1/88++8zq3Pv4+KhOnTpasWJFkseekjwBAIBjefrpp1WuXLlE02HNnTtXtWrVSvKe4OjRo1b3GB4eHipTpowmT56shIQES9ySJUvk5OSkrVu36oMPPlBgYKB8fHz0wgsvKDIyUsYYvf/++8qfP7+yZs2q1q1b68qVK1b7So8+UppvcoKDg+Xi4qKQkBBJ0osvvqhcuXLp5s2b6tGjh3LmzKmyZctKSnqO2fvxERER6tWrl3x9fZU9e3b17NlTt2/ftsS1atVKn3/+uSIjI63yPX/+fLK5Pcr+7lu4cKHKli0rDw8PlSxZUgsXLkx2f3/99Zc6dOigXLlyyd3dXSVLltRnn31mWb9q1Sq5uLho5MiRVtstXbpUTk5OGj9+fLJ9S9KyZcvk7u6uGjVqJFqXkJCgKVOmqFSpUnJ3d5efn586duyoo0ePWmJq166t4ODgJL+116JFCwUFBVn9n//444+qUaOGPD095enpqXr16mnz5s1W2/n7+6tXr17auXOn6tSpo6xZs2r48OGSpHr16ln+n1xcXOTv769u3brpzJkzDzzOB2nQoIFu3ryZKA/gSUXRFgBS6fXXX1e7du107tw5/fTTT1q+fLnVwwJ++uknDR06VC4uLlZTLRQtWtSqnwEDBqhdu3Y6efKkRowYkez+du/erWeeeUZ37tzRhg0bdP36dX3wwQcaN26c3nrrLUtcixYtFBISonXr1unmzZtavHixJOnnn39O0XENHDhQXbt2VUhIiJYsWaLff/9dTZo0UWxsbKpzSc0x9ujRQ97e3vriiy8SrZs+fbpKlChhuXlN6f5fffVVy3mPj4/XwYMHVaVKFbVr1047d+5MU54AAMDx9OrVSz/99JPlQ+4bN27ol19+SXaKrBIlSljd3124cEFvvvmmRo0apf/+97+J4qdNmyZvb28dPHhQa9eu1Zo1a/TGG29o/Pjx8vT01IEDB7R582bt2rVLr7zySpL7fJQ+Upvvv508eVLx8fEKCgqytBlj9NJLL+m5557TqVOnNGTIkAf2YYzRq6++queee07nz5/XggULtHjxYkvxT5KWL1+uV155RZ6enlb55s+f/6E5pmV/0r2iZefOndWoUSP9/fff+u2337Ry5cok5/DdunWrqlatKmOM/vjjD127dk1jx47VyJEjLQ/Patasmd555x19+OGHWr58uSTp1KlT6tmzp5o3b/7Qh2z98ccfKleunLJkyZJo3YABAzRy5Ei98cYbunTpkjZt2qRLly6pWrVqOnHihKR7BeuTJ08mGjkeGhqq1atXq3fv3nJ2vlcCev/999WtWze1adNGJ0+e1OnTp1W1alU1bNgwUcH0/PnzGjt2rL744gudOnVKVatWlXRvZPD9/6eoqCgtX75cZ8+eVatWrdI8UvaZZ56RpETHADyxDAA8wSZNmmQkJbv8/ffflthXXnnFSDLfffedVR+DBw82bm5uJjo62tI2dOhQ4+LikuQ+7/cze/bsROveeecd8+9fzdWrVzdFihSx6t8YY6ZNm2ZcXFxMSEiIuXTpkpFkZsyYkdpTYMln7ty5Vu2rVq0yksycOXNSlcvDjjEpAwcONFmzZjU3btywtG3dutVIMpMmTUr1/pNTtGhRM2DAgETHntI8AQBA5rd3714jyUydOtVcuXLFuLm5ma+++soYY8wnn3xismXLZsLDw8348eONJHPixImH9jlgwABTtGhRy+uff/7ZSDIvvviiVdyoUaNMlixZTP/+/a3ax44da5ydnc3NmzfTtY+U5nv/nIwfP94YY8zdu3fNwYMHTa1atYyzs7NZt26dMcaYvn37Gknm22+/TdTn0KFDjbu7u1Xb/fjFixdbtQ8cONB4eHiY+Ph4S9srr7xiPD09H5p7eu2vcOHCpmrVqlZxd+/eNU899ZSRZC5evGhpr1ixoilRooSJjY21iv/www9NlixZzOXLl40xxsTHx5vGjRub7Nmzm0OHDpny5cubAgUKmGvXrj30ePz9/U2bNm0StR86dMhIMsOGDbNqv3z5ssmWLZvp0qWLMcaYyMhI4+PjY7p3724VN2HCBOPk5GROnz5tjDHmzJkzxtXV1bzxxhuJ9lW7dm1Ts2ZNy+u8efMad3d3y/E9zL59+4wks3btWktb8eLFTYcOHazikmq7z83NLdExAE8qRtoCgJJ/EFmhQoUSxbZs2dLqdZkyZRQXF2f52lhKtWnT5qEx165d09atW9WmTRu5u7tbrWvUqJHi4+O1ZcsW5cqVS/ny5dPEiRP19ddf6/Lly6nKJal8mjZtqqxZs2r9+vWpyiW1xyhJr7zyiu7cuWP19cQvvvhCbm5u6tGjR6r3HxUVpffee08lS5ZU1qxZraaoSOoJ0CnNEwAAOJbcuXOrefPmlnuQuXPnqmPHjvL29k52m6+++ko1atSQj4+P5R7jf//7n06fPp1oyoHmzZtbvS5RooRiY2PVsGFDq/aSJUsqISEhya+WP2ofqcn3vffek5OTk9zc3FSrVi25u7tr1apVatCggVVcau6dnJycEh1DmTJlFB0drQsXLqS4n/Tc399//62///470XG4uLiodevWVm2hoaHau3ev2rVrJzc3N6t1jRo1UmxsrLZu3Srp3rys3377rby8vFSpUiUdOXJECxculJ+f30PzvnXrVpLX3f178fbt21u158mTR7Vr19a6deskSdmyZVPXrl21aNEihYeHS7o36nju3Llq0KCBChcuLOneNA53795Vp06dEu2rYcOG2rZtm9U37apVq5bkg9GOHTumrl27Kl++fHJ1dZWTk5MqVKggSUneb6eUt7e3bt26lebtAUdC0RYAUiFbtmzy8fGxarv/OjU3F1myZFGuXLkeGne/+PrJJ5/I1dVVLi4ucnFxkbOzs8qUKSNJun79ulxcXLR69WqVLVtWAwYMkL+/v0qVKqVx48YpKirqoftxdXVVzpw5E7XnyZPHMq9tSnNJ7TFKUunSpVWnTh3973//kzFG169f18KFC9WqVSvLTWJq9t+jRw99+umn+uCDD3T+/HnFx8fLGKMKFSooLi7Oat+pyRMAADieXr16aePGjVq6dKn27t2b7NQIkvS///1PvXv3VqtWrXTo0CHFxcXJGKM333xTCQkJiYqgAQEBVq/vF+WSa0/qfvJR+khtvuPHj5cxRgkJCbp586bWrl2rxo0bW8X4+PjIy8srUZ7JyZEjR6K5Z9Ny/5ye+7t/z5g3b95E2/+77dKlS5Kk//znP4nuQStXrmzVn3Tvg4A2bdooOjpaDRo0sHzl/2GyZ89uKbb+0/2+/f39E63z9/e32veLL76oqKgoLViwQJK0ceNGnTp1Sn379k10PHXr1rUcj7Ozs5ydnTVmzBjFx8db/b8EBgYmmVPNmjV1/vx5/fLLLwoLC5MxxlKs/ff9dmpEREQoR44cad4ecCQUbQEgFZJ6SFha/PtT+uTcLyaOHDlSd+/eVXx8vOLj45WQkGAZDfzyyy9Lulf4/OWXX3Tr1i1t2bJFLVu21NixYy3rH+Tu3bu6ceNGovYrV65YRgakJpfUHON9r7zyio4fP65169Zpzpw5io6OtrrBTOn+b9++rZ9//lkvv/yynn32Wfn5+Vnm70pq9Epq8wQAAI6lVatW8vPzU+/evVWoUCHVq1cv2dh58+apSpUqGjlypIKCguTq6irp3sjNpCR375iae8pH6SO1+aZEau+d0uv+OT33d//+Nqlvp/277f496Lhx45K9B+3Tp48lfuXKlZo+fbqqVq2qVatW6euvv05R3gULFrQUVP/p/sCK5HL95yjeypUrq3z58pozZ44kac6cOcqRI4eeffbZRMeze/duy/HcL+DfP55/jqxN6v97xYoVun79uj7++GNVrlxZnp6ekh7tupLufbMuLi5OBQsWfKR+AEdB0RYAHgNPT08lJCTo7t27j9RP3rx5VaVKFf38888p/sT6/lNnJ02apPr166d4Iv9ly5ZZvV69erXu3Llj+epdWnJJjWeffVYBAQH6/PPPNWPGDOXLl0/NmjWzrE/p/p2cnGSMSTSFwv2CNgAAwD+5ubnp+eef182bN9WzZ8+HFv3+fY9x7do1rVq16nGm+EgyS76enp5WX8t/nAoXLqxChQoluv9NSEiwPETsvoIFC6ps2bJavHjxQ+/tz549q+7du6t+/frasmWLOnfurJdffll//fXXQ3OqVauWDhw4kOgc3J+a4t8PF7569ao2bdqUaJqMvn37aseOHdqyZYt++uknPf/881Yjj5s3by4XFxf98MMPD83pYf59bc2bN++R+tuxY4ckqXbt2o/UD+AoKNoCwGNQpkwZGWO0fPlyxcfHP1Jf06dP17lz5/Tss89q9+7dioqKsnwVqXHjxrp+/br27t2r9u3ba82aNbpy5Yru3LmjNWvWaPfu3apfv/5D9+Hp6Wl5Ym5ERIQ2bdqk/v37q1y5curWrVuqckkrNzc39evXT0uWLNGpU6fUq1cvubi4pPpceHp6qkGDBpoxY4a2bdum27dva8WKFfrPf/6jcuXKpTk/AADguKZNmyZjjMaMGfPAuDZt2mjLli36+uuvdfv2be3bt0/t27dPVDizF5kp3/vPifj1118f+f45JSZOnKht27bpzTff1KVLlxQSEqJ+/fqpVKlSiWJnzJiho0ePqlOnTtq7d6+ioqIUEhKin3/+WfXr11dkZKRiY2PVqVMneXh4aMGCBXJxcdGsWbMUFBSkjh07KiIi4oH5tG7dWjExMfrzzz+t2kuXLq0+ffpo6tSpmjVrlm7duqXDhw+rQ4cOcnV11dixY63iX3jhBXl4eKhr1666c+eO1TfXJKlIkSIaN26cJk2apHHjxunMmTO6c+eOjh49qk8++US9e/d+6Llr0KCBvLy8NHz4cIWGhury5csaP368YmJiHrrtg6xfv17Zs2enaAv8fxRtAUBS9erVLQ9m+OcyfPjwNPXXoUMH9e/fX/3795ebm5vlIVhpUalSJe3Zs0e5cuVS27ZtlT17dtWsWVNff/21RowYIT8/P5UvX149e/bUf//7X5UtW1a5c+fWoEGD9Oabb+qLL75I0X4+//xzzZs3T4GBgWrTpo1q166ttWvXWn2CnpJcHkX//v0tDzL459fMUrv/b775RnXq1FGrVq0UGBiomTNnasGCBcqaNesj5QcAAJ5sb775pt555x299957ypMnj1566SWNHTtWZcuWtXVqScpM+Xbt2lW9e/dW9+7dLffP58+ff2z7e+655/T9999r1apVKliwoBo1aqTGjRsnOeChevXq2r17t7y8vNSqVStLYfG7777TmDFj5OnpqUGDBmnv3r368ccfLdMLeHt766efftL58+f14osvPjCfunXrqmTJkpo/f36idV9++aXGjx+vKVOmKG/evKpVq5Zy5cqlrVu3qnjx4lax96dDCAkJ0dNPP215ONg/jRw5UosXL9amTZtUoUIF+fn5qUOHDjpz5oxGjx790HOXP39+LVu2TFeuXFGxYsVUqVIlxcTE6D//+c9Dt01OQkKCFixYoF69eiWakxh4UjkZY4ytkwAA2M6rr76qr776Srdv37Z1Krp165YCAgJUrVo1/f7777ZOBwAAAMgw33zzjQYMGKC///5buXPntnU6GWrRokV64YUXdOzYMRUoUMDW6QB2gZG2AAC7sXTpUkVHR6tfv362TgUAAADIUM8//7xKlSqliRMn2jqVDJWQkKDRo0dr6NChFGyBf3C1dQIAAEjSpUuXNHnyZAUHB6tTp062TgcAAADIUE5OTpaHcT1JnJ2ddfDgQVunAdgdRtoCAGyuWrVqCgoKkru7u3788Ue5ubnZOiUAAAAAAGyGOW0BAAAAAAAAwI4w0hYAAAAAAAAA7AhFWwAAAAAAAACwIzyI7DFISEjQhQsX5O3tLScnJ1unAwAA4FCMMYqIiFC+fPnk7MwYhPTGvSwAAMDjk9J7WYq2j8GFCxcUFBRk6zQAAAAcWkhIiPLnz2/rNBwO97IAAACP38PuZSnaPgbe3t6S7p18Hx8fG2cDAADgWMLDwxUUFGS550L64l4WAADg8UnpvSxF28fg/tfIfHx8uNEFAAB4TPjq/uPBvSwApN75m1H6dN1JvdawqPLnyGbrdABkAg+7l2USMAAAAAAAgEdwKypOP+wK0a2oOFunAsBBULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAHkEuL3cNrPeUcnm52zoVAA7C1dYJAAAAAAAAZGb+vh56u1kJW6cBwIEw0hYAAAAAAOAR3I65q62nrut2zF1bpwLAQVC0BQAAAAAAeARnrkWq68xtOnMt0tapAHAQFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAACAR+Dq4iR/Hw+5ujjZOhUADsLV1gmkp3Xr1mnRokW6cuWK8uXLp27duqlatWpWMadPn9Znn32ms2fPKjg4WIMGDZK/v3+qYwAAAAAAACSphL+Pto1saOs0ADgQhxlpO3PmTDVv3lx58+bVc889pyxZsqhmzZr65ZdfLDGnTp1S5cqVdeHCBbVr10779+9X5cqVdfXq1VTFAAAAAAAAAMDj4mSMMbZOIj3Uq1dPBQoU0Lx58yxtjRs3lp+fn77//ntJUs+ePXXo0CHt3LlTTk5Oio2NVdGiRdWtWzd9+OGHKY55mPDwcPn6+iosLEw+Pj7pf7AAYEcKDV9h6xSQTs582NLWKQApwr3W48X5BYDUO3opXL3m7NRXfaqohH/6/e7M6Htt7geBxy+l91oOM9K2RIkSOnHihOLj4yVJUVFROnPmjEqVKmWJWbVqlZ599lk5Od2bYyZLlixq3bq1Vq1alaoYAAAA4HEwxii5MRWxsbG6ffu21RIZGZlsX3FxcY8rTQDAv9yNN7oUHq278Q4xLg6AHXCYou1///tflS9fXkWKFFHjxo0VHBysTp06aeTIkZLuFXGvXLmi/PnzW20XFBSkv//+O8UxSYmJiVF4eLjVAgAAAKTUxo0b1aJFC2XPnl1eXl6qU6eOtm3bZhXzwQcfyNfXV/7+/paldOnSifr67LPPFBAQIA8PDxUuXFg//fRTRh0GAAAA0onDFG3XrFmjH3/8UT169FC/fv3UpUsXffnll9q+fbukeyMTJClbtmxW22XLls2yLiUxSZk4caJ8fX0tS1BQULodFwAAABzfZ599pjfeeEOhoaG6fPmyypUrp6ZNm+r8+fNWcdWrV7caaXvmzBmr9YsWLdLQoUP1v//9T1FRURo2bJiee+457dixIwOPBgAAAI/KYYq2L7/8sl566SWNHz9enTt31n//+1+1atVKgwYNkiR5eXnJxcVFN27csNru+vXryp49e4pjkjJixAiFhYVZlpCQkPQ8NAAAADi4hQsXqmnTpvLy8pKXl5emTJmiyMhIrV+/PlHs3bt3k51CYdq0aWrfvr3atm0rd3d3DRw4UBUrVtSnn376uA8BAAAA6cghirbx8fG6cuWKnnrqKav2okWLKjQ0VJLk6uqq0qVLa//+/VYx+/btU/ny5VMckxR3d3f5+PhYLQAAAEBaXbx4UfHx8fLz87Nq37lzpzw9PeXt7a369etr9+7dlnXx8fHauXOn6tSpY7VN/fr1E021AABIX4VyeWpBv2oqlMvT1qkAcBAOUbR1cXFRpUqV9P3331umMYiMjNSiRYtUtWpVS1yPHj30448/Wr5Gtn//fv3222/q0aNHqmIAAACAxyUhIUGvvvqqihUrpkaNGlnaCxYsqMWLFys8PFynT59WoUKFVL9+fZ09e1aSFBYWppiYGOXOnduqv9y5c+vKlSvJ7o/nMwDAo/Nyd1X1p/zk5e5q61QAOAiHKNpK0qxZsxQaGqoiRYqoYcOGKly4sJydnfXxxx9bYl5//XU1bNhQ5cqVU82aNVWjRg316dNHXbt2TVUMAAAA8Li8/vrr2rp1qxYtWiR3d3dLe+/evdWyZUu5u7srT548+vLLL+Xl5aU5c+ZYbZ+QkJDotZOTU7L74/kMAPDoLoVF66NVR3UpLNrWqQBwEA7zEVCZMmV08OBBHTlyRJcvX1ZgYKCKFy9udYPq5uamH374QceOHdO5c+dUtGhRFS5c2KqflMQAAAAAj8PgwYP13Xffad26dSpTpswDY93c3PTUU0/p9OnTkqTs2bMra9asiUbVXrlyRQEBAcn2M2LECA0ZMsTyOjw8nMItAKTStdsxmr7hlFqWDZC/r4et0wHgABymaCvdmyahTJkyD73BLV68uIoXL/7IMQAAAEB6GTJkiL7++mutWbNGFStWfGj8nTt3dPToUdWuXVuS5OzsrBo1amjdunV69dVXLXFr1qxRrVq1ku3H3d3dakQvAAAAbM9hpkcAAAAAMqthw4Zp1qxZWrJkiYoXL67bt2/r9u3biouLs8Q0b95cv/76qy5evKgDBw6oS5cuiouLU//+/S0xb7/9tpYvX64ZM2YoJCREo0eP1vHjxzVo0CAbHBUAAADSyqFG2gIAAACZ0ZdffqmEhAS1aNHCqn3UqFEaNmyYJOn999/XhAkTtH37dnl6eqpq1aravXu3ChUqZIlv3Lixvv32W33wwQcaMWKEihcvrpUrV6p06dIZeTgAgEyq0PAVGbavMx+2zLB9AZkRRVsAAADAxm7duvXQmKefflqLFy9+aFznzp3VuXPndMgKAJBS2bO5qUvlIGXP5mbrVAA4CIq2AAAAAAAAjyB/jmz6qGM5W6cBwIEwpy0AAAAAAMAjiI6L1/HLEYqOi7d1KgAcBEVbAAAAAACAR3Dyym01mbpJJ6/ctnUqABwERVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAADgEWVxocQCIP242joBAAAAAACAzKxMoK+Ov9/c1mkAcCB8DAQAAAAAAAAAdoSiLQAAAAAAwCM4eSVCLT/ZrJNXImydCgAHQdEWAAAAAADgEUTHJejQhXBFxyXYOhUADoKiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAPAIgnJk0+fdnlZQjmy2TgWAg3C1dQIAAAAAAACZmW82N7UsF2DrNAA4EEbaAgAAAAAAPIKrETGatfm0rkbE2DoVAA6Coi0AAAAAAMAjuBwerQkrjuhyeLStUwHgICjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAI/A28NVjUrmkbeHq61TAeAg+G0CAAAAAADwCAr6eWpWzyq2TgOAA2GkLQAAAAAAwCOIi0/Q9dsxiotPsHUqABwEI22BJ1yh4StsnQLS0ZkPW9o6BQAAAOCJc+xShFp9+oeWv1ZLZQJ9bZ0OAAfASFsAAAAAAAAAsCMOM9L23Llzio2NTdTu6empgIAAq7bo6GhdvXpVefPmVZYsWZLsLyUxAAAAAAAAAJDeHGakbb9+/dSsWTOrJTg4WEOGDLGKGz16tHLmzKkKFSooV65c+vTTTxP1lZIYAAAAAAAAAHgcHGak7W+//Wb1eteuXapSpYqef/55S9v8+fM1adIkrV27VjVq1NAvv/yi9u3bq0SJEmrcuHGKYwAAAAAAAADgcXGYkbb/Nnv2bAUGBqp58+aWtunTp6tDhw6qUaOGJKlNmzaqXbu2pk+fnqoYAAAAAACA+0oG+OivMU1UMsDH1qkAcBAOWbS9c+eOFixYoD59+sjFxUWSlJCQoD179qh69epWsbVq1dKuXbtSHAMAAAAAAPBPLs5O8vZwk4uzk61TAeAgHLJo+9NPPykiIkJ9+/a1tEVERCgmJkZ+fn5Wsbly5dK1a9dSHJOUmJgYhYeHWy0AAAAAAODJ8Pe1SHWfvV1/X4u0dSoAHIRDFm1nz56tJk2aqGDBgpY2Z+d7hxoXF2cVGxsbaxmNm5KYpEycOFG+vr6WJSgoKF2OAwAAAAAA2L/ImLvafOKaImPu2joVAA7C4Yq2J0+e1KZNm9SvXz+rdm9vb/n6+urSpUtW7ZcuXVL+/PlTHJOUESNGKCwszLKEhISk09EAAAAAAAAAeNI4XNF2zpw5ypMnj1q3bp1oXb169fTbb79Ztf3666+qV69eqmL+zd3dXT4+PlYLAAAAAAAAAKSFQxVt4+Pj9fXXX6tXr15yc3NLtP6dd97Rpk2bNGrUKO3cuVOvvPKKQkJCNHTo0FTFAAAAAAAAAMDj4lBF223btilr1qx68cUXk1xfpUoVrV69Wjt37lSvXr0UGhqqjRs3qmjRoqmKAQAAAAAAuC/A10Pj2pZWgK+HrVMB4CBcbZ1AeqpZs6ZOnjz5wJi6deuqbt26jxwDAAAAAACePIWGr0h23ailhzIwEwCOzKFG2gIAAAAAAABAZkfRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsiKutEwAAAAAAAEirQsNX2DoFAEh3jLQFAAAAAAAAADvCSFsAAADADsTGxurQoUOKj49XiRIl5OXllWTcqVOndO7cOQUHByt//vxpjgEAAID9YqQtAAAAYGP/+c9/VLhwYfXt21f9+/dXYGCgZsyYYRVz9+5dPffcc6pQoYKGDx+u4OBgDR06NNUxAAAAsH+MtAUAAABszNnZWX/99Zdy5swpSZo3b5569+6t6tWrq1y5cpKkqVOnau3atfrrr79UqFAh7dq1SzVq1FCNGjXUoUOHFMcAAADA/jHSFgAAALCxN99801KwlaQXXnhBTk5O2rVrl6Xt66+/1nPPPadChQpJkipXrqxGjRrpq6++SlUMAAAA7B9FWwAAAMDObN++XfHx8SpevLike/PdHjlyRBUrVrSKq1ixovbv35/imKTExMQoPDzcagEAAIBtUbQFAAAA7EhYWJj69u2rpk2bqmbNmpKk8PBwJSQkWI3GlSQ/Pz/dvHkzxTFJmThxonx9fS1LUFBQOh8RAAAAUouiLQAAAGAnIiMj1apVK7m7u2vBggWW9ixZskiS7ty5YxUfFRVlWZeSmKSMGDFCYWFhliUkJCRdjgUAAABpx4PIAAAAADsQGRmpli1bKjw8XOvXr1eOHDks63x8fJQzZ85EBdXz589b5q9NSUxS3N3d5e7unm7HAQAAgEfHSFsAAADAxqKiotSyZUvdvHlT69atk5+fX6KYpk2baunSpZbXsbGxWr58uZo1a5aqGAAAANg/RtoCAAAANta2bVvt2bNH06dP165duyztxYoVU5EiRSRJo0eP1jPPPKMePXqoefPmmj9/viRp0KBBlviUxAAAAMD+MdIWAAAAsLEsWbKoRo0amj9/vqZNm2ZZtm/fbokpXry4duzYIR8fHy1YsEClS5fWjh07lDt37lTFAAAAwP4x0hYAAACwsRUrVqQornjx4vrss88eOQYAAAD2jZG2AAAAAAAAAGBHKNoCAAAAAAAAgB1xuOkRtm/frl9//VVOTk7q2LGjSpcubbX+5s2bmj9/vs6ePavg4GD16NFD2bJlS3UMAAAAAAAAADwODjXSdvDgwWrSpIlu374tDw8P9e7dWxs2bLCsv3z5sp5++mktXLhQ2bNn15dffqlq1arp9u3bqYoBAAAAAAAAgMfFYUbaLly4UJ9++qm2bt2qKlWqSJKGDBmiK1euWGImTJggDw8PrV27Vu7u7nr99dcVHBysTz75RCNHjkxxDAAAAAAAAAA8Lg4z0vaLL75Qs2bNLAVbSXJzc1NgYKDl9dKlS9WpUye5u7tLknx9fdW6dWstXbo0VTEAAAAAAAAA8Lg4TNF29+7dqlWrllauXKm33npLkyZN0rFjxyzrY2JiFBISosKFC1ttV6RIEZ04cSLFMUmJiYlReHi41QIAAAAAAAAAaeEQRVtjjCIiIjRv3jxNmjRJuXLl0sGDB1W+fHn9/PPPkqQ7d+5Ikry9va229fHxUVRUVIpjkjJx4kT5+vpalqCgoHQ7NgAAAAAAAABPFoeY09bJyUleXl5ydnbW2rVr5eLiIknKmjWrhg0bpmeffVaenp5ycnLSrVu3rLa9efOmfHx8JClFMUkZMWKEhgwZYnkdHh5O4RYAAAAAAABAmjjESFtJKlOmjMqVK2cp2EpSxYoVdfbsWRlj5ObmpuDgYB05csRquyNHjqhUqVKSlKKYpLi7u8vHx8dqAQAAAAAAAIC0cJiibbdu3bRlyxZFRkZa2tasWaMKFSrIyclJktSlSxf98MMPunHjhiTp7NmzWr58ubp06WLZJiUxAAAAAAAAAPC4OMT0CJI0YMAArV27VmXLllW9evV06NAhhYSEaPny5ZaYt99+W2vXrlWlSpVUo0YN/f7776pbt65efPHFVMUAAAAAAAAAwOPiMEVbNzc3LV26VJs3b9axY8fUoUMH1atXT56enpYYT09Pbd68WWvXrtW5c+fUr18/1a1b1zISN6UxAAAAAAAAAPC4OEzR9r7atWurdu3aya53cXFR06ZNH9hHSmIAAAAAAAAA4HFwmDltAQAAAAAAAMARULQFAAAAAAAAADvicNMjAAAAAAAA2yo0fIWtUwCATI2RtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHbE1dYJAAAAAAAA4MlSaPiKDNvXmQ9bZti+gPTCSFsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjjhM0dYYo7t371ot8fHxtk4LAAAADuzChQu2TgEAAAAOyGGKtu+//77c3Nzk4eFhWfLmzZsobu7cuSpSpIhcXV1VokQJLVmyJE0xAAAAQOXKldWwYUN9++23unPnjq3TAQAAgINwmKKtJFWtWtVqpO21a9es1q9cuVL9+/fXhAkTdPPmTb388svq1KmTdu7cmaoYAAAAQJKWLFmip556Si+//LICAgI0cOBA7dq1y9ZpAQAAIJNzqKLtw0yZMkVt2rRRt27d5O3trddff11PP/20Pv7441TFAAAAAJL0zDPP6Msvv9TFixf1ySef6MiRI3rmmWdUrlw5TZs2LdEgAgAAACAlHKpou2/fPmXNmlU5cuRQ06ZNtW/fPss6Y4y2bdumunXrWm3ToEED/fnnnymOAQAAAP4tW7Zs6tGjhzZs2KB58+bp9OnTGjx4sAIDA9WnTx9dvHjR1ikCAAAgE3GYom2+fPk0f/58XblyRQcOHFCePHlUp04dnT17VpIUERGhqKgo5c6d22q7PHny6PLlyymOSUpMTIzCw8OtFgAAADw5Ll++rEmTJqlkyZLq27evWrZsqTVr1mjp0qU6c+aMGjRoYOsUAQAAkIm42jqB9NKnTx/Lv729vTVnzhwVLlxYs2bN0vjx45PdLiEhQU5OTg/s+2ExEydO1NixY1OfNAAAADK15cuXa+bMmVq5cqWKFSumfv36qUePHsqVK5clplGjRsqZM6euX78uPz8/G2YLAACAzMJhRtr+m5ubm4oVK6aTJ09KulfI9fT01JUrV6zirl69Kn9//xTHJGXEiBEKCwuzLCEhIel8NAAAALBHb7zxhnLmzKkNGzbo0KFDGjJkiFXBVpJcXV01btw4eXh42ChLAAAAZDYOW7SNjY3V0aNHFRgYKElycnJSjRo19Pvvv1vFrVu3TjVq1EhxTFLc3d3l4+NjtQAAAMDxHTx4UHPnzlXNmjUfGDdo0CB5enpmUFYAAADI7BymaNu+fXutX79eN2/e1IkTJ9S9e3eFh4erf//+lpg333xTK1as0Jw5c3T16lVNmjRJ+/fv1+DBg1MVAwAAAEhS1qxZtXXrVhljrNr37t2rqKgoG2UFAACAzM5hirZvv/22pkyZouDgYDVs2FDx8fHasWOHihUrZolp0qSJvv76a02aNEmFChXSt99+qyVLlqhixYqpigEAAAAkafr06Zo/f36i5x/s2LFDQ4cOtVFWAAAAyOyczL+HBeCRhYeHy9fXV2FhYUyVALtXaPgKW6eAdHTmw5YZvk+uIcdhi+sHSAt7uteqVKmSZs2alegD/ujoaAUEBOjatWtycXFJUV/x8fFavny59u7dq44dO6pMmTJW6zds2KANGzZYtWXLlk3Dhg2zaouKitLPP/+sc+fOqVixYmrbtq1cXVP+/GF7Or8AMi/uEWFPuM+FPUnpvZbDjLQFAAAAMlpYWFiiqRGke89KuH37doqnSFi5cqWeeuopzZw5U2PHjtXBgwcTxWzYsEFz5sx5YD83btxQ5cqVNWnSJF26dElvv/22GjZsqJiYmJQdEAAAAOxCyj9yBwAAAGClevXqmjJliubPny9n5/8bDzF58mQVK1ZM3t7eKeonICBAmzdvVkBAgNzc3JKNK1CggMaMGZPs+vHjxysuLk47d+6Up6enRowYoeLFi+vLL7/Ua6+9luLjAgAAgG1RtAUAAADSaPTo0apatarKlCmjBg0ayN3dXdu2bdPOnTu1bNmyFPdzf3qFu3fvPjDu6tWrmjx5srJly6ZnnnlGlStXtlq/aNEi9ejRQ56enpIkf39/tWzZUj/99BNFWwBMWQAAmQjTIwAAAABpVLRoUe3du1eNGzfWrl27tHHjRgUHB2vbtm1q2rRpuu/P1dVVISEh2rZtm2rXrq3+/ftb1kVFRSkkJETBwcFW2wQHB+vYsWPJ9hkTE6Pw8HCrBQAAALZl85G23bt319SpU5UrV65UrQMAAADsQYECBfTxxx8/9v107dpVo0aNskzD8PLLL6tWrVpq0qSJOnbsqMjISEmSr6+v1XbZs2fX7du3k+134sSJGjt27ONLHAAAAKlm85G2v//+u6Kjo5Nct27dOsXFxWVwRgAAAEDqxMbG6vLly7p06ZLVkpCQkG77KF68uNW8udWqVVP58uX1+++/S5JlSoSwsDCr7W7duiUvL69k+x0xYoTCwsIsS0hISLrlDAAAgLSx2Ujb8+fP6+7du4qPj7f8+z5jjI4dO6abN28qd+7ctkoRAAAAeKAzZ86oV69e2rx5c5IF2osXL8rf3/+x7d8Yo6ioKElStmzZFBQUpBMnTljFnDhxQsWLF0+2D3d3d7m7uz+2HAEAAJB6NivaVqtWTaGhoZLuPXX331xcXPTee+/J1dXmMzgAAAAASRo8eLBcXV21Zs0a5cmTJ9H69Jzma+fOnapSpYrl9bZt23TgwAENGjTI0tahQwf98MMPGjlypLJly6ZLly5pxYoVGj9+fLrlAQAAgMfPZhXRP/74Q3fv3lXt2rW1cOFCqxEILi4u8vf3V9asWW2VHgAAAPBQBw4c0PLly1WyZMlH6ufkyZP65ptvLKN1f/rpJx09elSVK1dWq1atJEljx47VnTt3VKFCBV29elULFy5U9+7d9fzzz1v6effdd/Xbb7+pRo0aqlu3rlasWKHy5cvrpZdeeqT8AAAAkLFsVrQtVKiQJOnYsWPy9vaWk5OTrVIBAAAA0qRAgQIPfMhXajk7O2v06NFJrlu+fLn+/PNPbd++XcWLF9fQoUNVvnx5qxg/Pz/t2rVLixcv1rlz5/TRRx+pbdu2fHsNAAAgk7H53ZuPj4/i4uJ0/PhxXbt2TcYYq/XVq1dnji0AAADYpcGDB+utt97S/PnzFRQUlOZ+ihYtqjFjxjw0rkaNGqpRo8YDY7Jly6YXXnghzbkAAADA9mxetN27d6/atWunc+fOJbk+JCRE+fPnz+CsAAAAgId76623dOLECRUoUEBeXl5ycXGxWn/8+PEk57oFAAAAHsTmRdvhw4erXr16mjhxYpI3tHyVCwAAAPZqypQpio6OTna9r69vBmYDAAAAR2HziujZs2f18ccfK1++fLZOBQAAAEiV+w8JAwAAANKTs60TKF68eLJTIwAAAACZwdmzZ7Vu3TrFxsbaOhUAAAA4AJsXbQcOHKg33nhDq1evVmhoqC5dumS1JCQk2DpFAAAAIEmRkZFq27atChUqpEaNGunGjRuSpN69e2vBggU2zg4AAACZlc2nR3jxxRcVGhqqpk2bJrmeB5EBAADAXo0dO1a3b9/WuXPnVKVKFUv7sGHD1LFjR3Xt2tWG2QEAACCzsnnRdt26dYqLi0t2vb+/fwZmAwAAAKTcb7/9pq+++kpBQUFycnKytJcoUUJnzpxReHi4fHx8bJghAAAAMiObF22LFy9u6xQAAACANImIiFC2bNkkyapoe+fOHcXExMjV1ea32wAAAMiEbH4XeezYsQeOtC1RogQ3uwAAALBL1apV04IFCzRmzBhL0dYYo9GjR6tChQqWgi4AAACQGjavhjZs2FChoaHJrmdOWwAAANircePGqXr16tq5c6fCw8P13nvvadeuXTp8+LB+++03W6cHAACATMrmRdu9e/cqPj7equ3s2bMaMWKEGjZsqHz58tkoMwAAAODBihYtqr1792rq1Km6efOm9uzZo3Llymnu3LmqUKGCrdMDAABAJmXzom3u3LkTtfn7+2vhwoUqU6aM3nnnHRtkBQAAAKRM/vz5NWXKFFunAQAAAAdi86Jtcvz8/HTnzh3dunVL2bNnt3U6AAAAQCLx8fEyxiS7nmczAAAAIC1sfheZ1I1uRESEZs6cKRcXF/n6+tooMwAAAODBAgMDdfny5WTXX7x4Uf7+/hmYEQAAAByBzYu2BQsWTPJBZD4+Ppo1a5blKbwAAACAvVm8eLFiY2Ot2s6cOaMPPvhAnTt3Vs6cOW2UGQAAADIzmxdtFy5cqJiYGKs2Hx8fFStWTF5eXjbKCgAAAHi4GjVqJNles2ZNtWrVShMmTMjgjAAAAOAIbF60rV69uq1TAAAAANJVcHCwLl68qMjISHl6eto6HQAAAGQyNi/a3hcaGqpDhw4pJiZGxYoVU/HixW2dEgAAAJAmixcvVkJCgrJly2brVAAAAJAJ2bxoGxsbq9dff12zZs1SfHy8nJycZIxRkyZN9NVXXykgIMDWKQIAAABJql69uq5evWrVFhERoWvXrmny5Mk8nwEAAABpYvOi7fvvv69ff/1VS5cuVb169eTm5qb9+/dryJAh6t27t1atWpXqPi9cuKDvvvtOxYsXV+vWra3WxcXF6ddff9XZs2cVHBysJk2ayNnZOdUxAAAAwMCBAxUVFWXV5uPjo8qVK6tYsWI2ygoAAACZnc2LtkuWLNHMmTPVpEkTS1uVKlW0dOlS+fv7p3oesISEBHXr1k27d+9W06ZNrYq2ERERatCggcLCwlS7dm1NmjRJJUqU0PLly5UlS5YUxwAAAACS1KNHD1unAAAAAAdk86JtVFSU8uTJk6jd19dXbm5uio6OTlXRdvz48fLz81PdunUTrZs4caIuX76sAwcOKHv27AoNDVWpUqX05Zdf6tVXX01xDAAAACBJ8fHxMsakKNbV1ea33gAAAMgkbP6d/6pVq2r8+PFWXytLSEjQhAkTlC9fPvn5+aW4r82bN2vOnDmaOXNmkusXLlyoLl26KHv27JKkwMBAtWrVSj/++GOqYgAAAADp3r2im5tbipZLly7ZOl0AAABkEjb/uH/ChAmqX7++goKCVLlyZWXJkkUHDhzQzZs3tWjRohT3c+PGDb3wwguaPXu2cubMmWh9bGysTp06peLFi1u1lyhRQqtXr05xTFJiYmIUExNjeR0eHp7ivAEAAJB5TZo0SQMHDtTLL7+sWrVqKUuWLNq3b5+mTZumPn36qEWLFpbY1AxGAAAAwJPN5kXbQoUK6ciRI/rqq6+0f/9+xcTE6OWXX1bv3r2TnDYhOb1791bHjh3VqFGjJNdHRkbKGGMZQXtf9uzZFRERkeKYpEycOFFjx45Nca4AAABwDIsXL9b06dPVvXt3S1uzZs3UokULtW3bVh988IENswMAAEBmZfOirSR5eHhowIABad5+zZo1WrVqlUaNGqXJkydLkk6fPi1XV1dNnjxZ/fv3V7Zs2SQlHgUbFhZmmTM3JTFJGTFihIYMGWJ5HR4erqCgoDQfDwAAADKHv/76Sx999FGi9rJly+rKlSu6ffu2vLy8bJAZAAAAMjObzWkbExOj4cOHKz4+Psn1n3zyiU6cOJGivvLly6fXXntNN2/e1KVLl3Tp0iXLlAWXLl1SfHy83N3dVbBgQZ06dcpq21OnTik4OFiSUhSTFHd3d/n4+FgtAAAAcHx58uTRvHnzErX/9NNPcnNzU9asWW2QFQAAADI7m420nTVrlqKjo+Xi4pLk+ty5c2v8+PFJ3gT/W+nSpS0jbO87evSoPDw8rNrbtm2rhQsX6r333pOHh4du3LihZcuW6a233kpVDAAAACDdez5DixYttG7dOtWsWVNubm7av3+/fvvtN33++efJ3usCAAAAD2KzkbbLli1Ty5Ytk13fqlUrLVu2LF33+e677yohIUF169bVO++8o7p166pgwYJ69dVXUxUDAAAASFKDBg10+PBhValSRbt27dKmTZuUN29ebd++/ZGm/wIAAMCTzWYjbU+fPq3ChQsnu97b21suLi66ceOGcubMmer+27dvL1dX68PLnTu39u7dq++++07nzp3T0KFD9dxzz8nDwyNVMQAAAMB9RYoU0SeffGLrNAAAAOBAbFa0dXd3140bN5JdHxsbq9u3b6e5WNqnT58k2729vfXSSy89cNuUxAAAAAD3nT17VidPnlTt2rWVJUsWW6cDAACATM5m0yM888wz+uabb5Jdv3DhQgUHBytbtmwZmBUAAACQcpGRkWrbtq0KFSqkRo0aWQYl9O7dWwsWLLBxdgAAAMisbFa0ff311zVjxgwNGzZMN2/etLTHxcXpiy++0MCBA/Xmm2/aKj0AAADgocaOHavbt2/r3Llzyps3r6V92LBhmjBhgg0zAwAAQGZms+kRypcvr7lz56pv376aPHmy8uXLJ3d3d507d07x8fF6++231bNnT1ulBwAAADzUb7/9pq+++kpBQUFycnKytJcoUUJnzpxReHi4fHx8bJghAHtWaPgKW6cAALBTNivaSlK3bt1Uv359LViwQIcPH9bdu3dVuHBhdejQQWXKlLFlapkKb/SO5cyHLW2dAgAASKGIiAjLdF7/LNreuXNHMTExiR6MCwAAAKSEze8iAwICNGTIEFunAQAAAKRatWrVtGDBAo0ZM8ZStDXGaPTo0apQoQLPZwAAAECa2LxoCwAAAGRW48aNU/Xq1bVz506Fh4frvffe065du3T48GH99ttvtk4PAAAAmRRFWwAAACCNihYtqr1792rq1Km6efOm9uzZo3Llymnu3LmqUKGCrdMDAADK+GklmfYQ6YGiLQAAAJBGn332mXr37q0pU6bYOhUAAAA4EGdbJwAAAABkVpMmTdLNmzdtnQYAAAAcDEVbAAAAII3Kly+vrVu32joNAAAAOBimRwAAAADSqGPHjho4cKBOnTqlChUqyMPDw2p9jRo1lCVLFhtlBwAAgMyKoi0AAACQRsOGDdP169c1YsSIJNdfvHhR/v7+GZwVAAAAMjuKtgAAAEAahYaGyhiT7HpXV263AQAAkHrMaQsAAACkUqFChXT58mW5uLjI1dVVH330kaKjo+Xq6mq1AAAAAGlB0RYAAABIpejoaKsRtp9++qlu375tw4wAAADgSCjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEibYAAACANJg2bZq8vLwkSZGRkVav7xs8eLA8PT1tkR4AAAAyMYq2AAAAQCrly5dP33zzjeW1r6+v1ev7XnrpJYq2AAAASDWKtgAAAEAq7dmzx9YpAAAAwIExpy0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAA2IGTJ0/qrbfeUqNGjbRhw4YkY44cOaKBAweqZcuWGjx4sM6fP5+mGAAAANg3irYAAACAjc2YMUPNmzdXzpw5tW7dOl26dClRzJEjR1S1alXFxMSoV69eOnnypKpWraorV66kKgYAAAD2j6ItAAAAYGMdOnTQ8ePH9dZbbyUbM3bsWJUtW1Zz5sxRp06dtGjRIjk7O2vq1KmpigEAAID9o2gLAAAA2FiuXLnk5OT0wJjVq1erbdu2ltdZsmRRy5YttXr16lTFAAAAwP5RtAUAAADsXHh4uG7evKn8+fNbtefPn19nzpxJcUxSYmJiFB4ebrUAAADAtijaAgAAAHYuNjZWkpQ1a1ar9mzZslnWpSQmKRMnTpSvr69lCQoKSs/UAQAAkAYUbQEAAAA75+PjI2dnZ924ccOq/fr168qRI0eKY5IyYsQIhYWFWZaQkJD0PwAAAACkisMUbaOiojRp0iTVqlVLJUqUUJs2bbR+/fpEcWvWrFGDBg301FNPqVmzZtq+fXuaYgAAAICMkiVLFpUqVUp79uyxat+zZ48qVKiQ4pikuLu7y8fHx2oBAACAbTlM0fbdd99VVFSUpkyZop9//lkVK1ZU06ZNtWXLFkvM1q1b1bJlSzVq1EhLlixR6dKl1aBBAx07dixVMQAAAEBG69Wrl77//nv9/fffkqTt27dr3bp16tWrV6piAAAAYP+cjDHG1kmkh4SEBDk7W9egCxQooBdffFGjRo2SJLVp00axsbFatWqVJaZMmTKqUaOGvvzyyxTHPEx4eLh8fX0VFhaWISMVCg1f8dj3gYxz5sOWGbo/rh/HktHXj8Q15Ehscf0AaZHR91oZYdeuXRo+fLiMMVq/fr3KlCmjvHnzqnXr1nrjjTckSXfv3lWvXr30888/q0SJEjp8+LBee+01/ec//7H0k5KYh3HE8wvYK+6jAMfEfTUeJKX3Wq4ZmNNj9e+C7bp163T58mXVqVPH0rZx40ZLAfe+Zs2aafny5amKAQAAANJT4cKFNXz4cEn35pi9L3/+/JZ/u7q66ptvvtGZM2d07tw5BQcHKyAgwKqflMQAAADA/jlM0VaSjh07poYNGyoyMlLR0dGaM2eO6tWrJ0mKiIhQeHi4/P39rbbJmzevQkNDUxyTlJiYGMXExFheh4eHp9MRAQAA4Eng5+enRo0apSi2UKFCKlSo0CPHAAAAwH45zJy2kvTUU09p27Zt2rx5s9566y3169dPmzdvlnRv+gTp3uiDf3Jzc1N8fHyKY5IyceJE+fr6WpagoKB0OyYAAAAAAAAATxaHKtq6uroqf/78KlOmjMaNG6c6depoypQpkiRvb2+5u7vr2rVrVttcu3ZNuXPnTnFMUkaMGKGwsDDLEhISks5HBgAAAAAAAOBJ4VBF23/Lli2boqKiJN2b87Zy5crasmWLVczmzZv1zDPPpDgmKe7u7vLx8bFaAAAAAAAAACAtHKZo+8Ybb+jMmTOSJGOMFi5cqOXLl6tTp06WmFdeeUU///yz1q9fL0lauHCh/vjjD7388supigEAAAAAAACAx8VhHkRWo0YNtWzZUqGhoYqJiVFgYKA++eQT9evXzxLTtWtXnTlzRu3atVNCQoLc3d01ffp01a9fP1UxAAAAAAAAAPC4OEzRtkuXLurSpYsiIiKUJUsWubu7Jxk3YsQIvfXWW7p586Zy5swpFxeXNMUAAAAAAAAAwOPgMEXb+7y9vR8a4+rq+sAHi6U0BgAAAAAAAADSm8PMaQsAAAAAAAAAjoCiLQAAAAAAAADYEYebHgEAAAAAAACwlULDV2TYvs582DLD9oWMxUhbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO+Jq6wQAAAAAAEhOoeErMnR/Zz5smaH7AwAgKYy0BQAAAAAAAAA7QtEWAAAAAAAAAOwI0yMAAAAAAPD/ZfR0DAAAJIWRtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHbEoea0jYyM1N69exUXF6eyZcsqV65cScYdOHBAZ8+eVXBwsEqUKJHmGAAAAAAAAABIbw4z0nbcuHEqVqyYhg8frjFjxqhgwYKaNGmSVUxsbKzatWun+vXra9q0aXrmmWfUt29fGWNSFQMAAAAAAAAAj4vDjLTNkyePjh49Km9vb0nSkiVL9Oyzz6pevXqqUqWKJGnatGnasmWL9u/fr/z58+vQoUOqXLmy6tWrp+7du6c4BgAAAAAAAAAeF4cZaTtgwABLwVaS2rVrpyxZsmjv3r2Wtvnz56tLly7Knz+/JKl06dJq3ry55s+fn6oYAAAAAAAAAHhcHKZo+29//PGHYmNjVbp0aUnS3bt3deTIEZUrV84qrly5cjpw4ECKY5ISExOj8PBwqwUAAAAAAAAA0sIhi7Y3b95Unz591LJlS9WsWVOSdPv2bcXHxytHjhxWsX5+frp161aKY5IyceJE+fr6WpagoKB0PR4AAAAAAAAATw6HK9qGh4erefPmyp49u7777jtLu7u7uyQpKirKKv727dvy8PBIcUxSRowYobCwMMsSEhKSLscCAAAAAAAA4MnjMA8ik6SIiAg1a9ZMd+/e1dq1a+Xj42NZlzVrVvn7++vcuXNW25w7d05FihRJcUxS3N3dLQVfAAAAAAAAAHgUDjPS9vbt22rWrJliY2O1Zs0aZc+ePVFM8+bNtXjxYiUkJEiSoqOjtWzZMjVv3jxVMQAAAAAAAADwuDjMSNvWrVvrwIEDmjp1qtatW2dpL1WqlEqVKiVJGjVqlKpUqaIOHTqoRYsW+v777+Xq6qohQ4ZY4lMSAwAAAAAAAACPi8OMtA0ICFDTpk21atUqff/995bl4MGDlphChQppz549KlGihDZs2KDatWtr586d8vPzS1UMAAAAAAAAADwuDjPS9p8PHXuQoKAgTZw48ZFjAAAAAAAAAOBxcJiRtgAAAAAAAADgCCjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAZAITJkyQh4eH1ZI/f/5EcQsWLFDZsmXl6+urKlWqaO3atTbIFgAAAI+Coi0AAACQCdy9e1eVKlXSrVu3LMupU6esYlatWqUePXpo0KBBOnr0qNq0aaNWrVrp4MGDNsoaAAAAaUHRFgAAAMgknJycrEbauru7W62fNGmS2rRpo759+yogIEDvvfeeSpQooalTp9ooYwAAAKQFRVsAAAAgk9i7d69y5cqlAgUKqFOnTjpx4oRlXUJCgrZu3ar69etbbdOwYUNt2bIlo1MFAADAI6BoCwAAAGQCuXPn1hdffKG//vpLq1at0t27d1W9enVdunRJknTr1i3duXNHefLksdouT548lpikxMTEKDw83GoBAACAbVG0BQAAADKBV155RT179lRAQIBKlSql7777Ts7Ozpo5c6ZVnLOzc6LXxphk+504caJ8fX0tS1BQ0GPJHwAAAClH0RYAAADIhLJmzaoSJUro+PHjkiRfX1+5u7vr6tWrVnFXrlxR3rx5k+1nxIgRCgsLsywhISGPNW8AAAA8HEVbAAAAIBOKi4vTyZMn5e/vL0lycXHRM888o40bN1rF/f7776pevXqy/bi7u8vHx8dqAQAAgG1RtAUAAAAygeeff147d+5UbGysLly4oL59++rWrVvq27evJWbw4MFavHixFi1apKioKH322Wfav3+/XnvtNRtmDgAAgNSiaAsAAABkAr1799bQoUOVPXt2lSlTRlevXtWWLVtUokQJS8yzzz6rjz/+WG+88Ya8vLz08ccf64cfflDlypVtmDkAAABSy9XWCQAAAAB4uEaNGqlRo0YyxsjJySnZuIEDB2rgwIFKSEhI9FAyAAAAZA7cxQEAAACZyIMKtv9EwRYAACDzYqQtAACwmULDV9g6BaSjMx+2tHUKAAAAgEPg43cAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCOutk4AAAAAAAAAQOoVGr4iQ/d35sOWGbq/JxlFWwAAAABAqmR0kQAAgCeNQ02PcPjwYb3++uuqVauWVq9enWTMX3/9pT59+qhhw4YaMGCATp8+naYYAAAAAAAAAHgcHKZoO2PGDHXs2FFFihTRli1bdOXKlUQxhw8fVo0aNeTq6qpBgwbpxo0bqlatmkJDQ1MVAwAAAAAAAACPi8MUbbt06aLDhw9r0KBBycaMHz9e5cqV05dffqnWrVtrwYIF8vLy0pQpU1IVAwAAAAAAAACPi8MUbbNnz/7QmLVr16p169aW1y4uLmrVqpXWrFmTqhgAAAAAAAAAeFyemAeRRUZG6tq1awoMDLRqDwwM1Llz51Ick5SYmBjFxMRYXoeHh6dj5gAAAAAAAACeJA4z0vZh4uLiJEkeHh5W7VmzZlVsbGyKY5IyceJE+fr6WpagoKD0TB0AAAAAAADAE+SJKdp6e3vLzc1N169ft2q/fv26cubMmeKYpIwYMUJhYWGWJSQkJP0PAAAAAAAAAMAT4YmZHsHFxUVly5bV7t27rdp37NihihUrpjgmKe7u7nJ3d0//pAEAAAAAAAA8cZ6YkbaS1KdPHy1cuFBHjx6VJG3dulXr1q1Tnz59UhUDAAAAAAAAAI+Lw4y03bVrlwYNGmR5PX78eP3vf/9T69at9fbbb0uSBg4cqAMHDqhChQp66qmndOrUKb355ptq3769ZbuUxAAAAAAAAADA4+IwRdvg4GB9+OGHidr9/f0t/3Z2dtaMGTM0duxYhYSEqHDhwsqVK5dVfEpiAAAAAAAAAOBxcZiira+vr2rVqpWiWH9/f6tiblpjAAAAAAAAACC9PVFz2gIAAAAAAACAvaNoCwAAAAAAAAB2hKItAAAAAAAAANgRh5nTFgAAAACeVIWGr7B1CgAAIB0x0hYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOuNo6AQAAAABwRIWGr7B1CgAAIJNipC0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdcbV1AgAAAAAAAADsX6HhKzJsX2c+bJlh+7JHjLQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwI8xpCwAAAOCJkZFz8QEAAKQVI20BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCHPaAgAAAAAAALArGT0P/ZkPW2bo/h6GkbYAAAAAAAAAYEcYaZuMGzduKDQ0VAULFpSPj4+t0wEAAABS7P69bKFCheTt7W3rdB4oo0fRAAAAZAaMtP0XY4zeeOMNBQQEqF27dsqTJ4/Gjx9v67QAAACAhzLG6NVXX7W6l504caKt0wIAAEAqMdL2X2bOnKm5c+dq165dKlu2rDZu3KhGjRqpfPnyatOmja3TAwAAAJI1ffp0zZ8/X3v27FHp0qW1fv16NWnSROXLl1eLFi1snR4AAABSiJG2/zJz5kx17NhRZcuWlSTVrVtXDRo00KxZs2ycGQAAAPBgM2fOVJcuXVS6dGlJUoMGDVS3bl3uZQEAADIZirb/EB8fr/3796tKlSpW7VWrVtWePXtslBUAAADwcHFxcfrrr7+4lwUAAHAATI/wDxEREYqLi5Ofn59Ve65cuXT9+vVkt4uJiVFMTIzldVhYmCQpPDz88ST6LwkxURmyH2SMjLpu7uP6cSwZff1IXEOOhOsHjyqjrqH7+zHGZMj+MouwsDDFx8dzLwsAAJAG9nYvS9H2H9zc3CTJ6qZVku7cuWNZl5SJEydq7NixidqDgoLSN0E8EXyn2ToDZGZcP3gUXD94VBl9DUVERMjX1zdjd2rHuJcFAABIO3u7l6Vo+w+enp7KmTOnLly4YNV+4cIFFShQINntRowYoSFDhlheJyQk6MaNG/Lz85OTk9Njy/dJEh4erqCgIIWEhMjHx8fW6SCT4frBo+IawqPg+kl/xhhFREQoX758tk7Frvj6+srX1zdT3cvy85E2nLfU45ylDectbThvacN5Sz3OWdrY+ryl9F6Wou2/NGzYUCtWrNDbb78t6d5N68qVKx/4tF13d3e5u7tbtWXPnv1xpvnE8vHx4RcR0ozrB4+KawiPgusnfTHCNmkNGjTQihUrNHToUEn3ntmwcuVKtW/fPtlt7OFelp+PtOG8pR7nLG04b2nDeUsbzlvqcc7SxpbnLSX3sjyI7F/effdd7dq1S6+//rrWrFmjnj176saNG5YbXwAAAMBejRo1Slu3btXgwYO1Zs0ade/eXREREVYjaQEAAGD/KNr+S7ly5bR582ZdvXpVY8aMkYuLi/78888HfqUMAAAAsAcVKlTQpk2bdPHiRY0ePVru7u76888/FRgYaOvUAAAAkApMj5CESpUqacGCBbZOA//g7u5u+cMDSC2uHzwqriE8Cq4fZLQqVaro+++/t3UaKcLPR9pw3lKPc5Y2nLe04bylDect9ThnaZNZzpuTMcbYOgkAAAAAAAAAwD1MjwAAAAAAAAAAdoSiLQAAAAAAAADYEea0BQAAAPDYnT59Wjdv3lSlSpVStU1sbKyKFSsmZ+ekx5ukJCazio2N1YkTJ5Q9e/aHPkwuNDRUFy9eTHJdpUqV5OTkpOjoaB08eDDR+uLFi8vb2ztdcrYHN27cUGhoqAoWLCgfH58Hxl68eFGhoaFWba6urqpQocIj9ZsZnTlzRnfu3FGxYsXk4uLy0Pi7d+/q1KlTypkzp3Lnzp1o/eHDhxUVFWXVljt3bhUsWDDdcs4ocXFxOnHihLy9vRUUFJRu26Sl38zk1q1bCgkJUVBQkLJnz56ibS5cuKA7d+6oYMGCcnW1LllduXJF586ds2pzcnJK1ftKZhASEqKIiAgFBwfLzc3tgbHHjh1TRESEVVvOnDlVpEiRR+o3s4mPj9fx48eVNWtWFSpU6IGxV69e1dmzZ5NcV7ZsWbm7uys+Pl579+5NtL5IkSLKmTNneqScMgYAAAAAHpPFixebunXrGm9vb+Pi4pKibf7++29TsWJF4+vra/LmzWsKFixotm/fnuqYzGzx4sUmZ86cJigoyGTNmtU0bdrUhIWFJRv/6aefmkqVKlkt2bNnN76+vubu3bvGGGP++usvI8mUL1/eKm7Xrl0ZdViPVUJCgnn99ddNlixZTJEiRYy7u7sZN27cA7cZP3688fT0tDof9erVe+R+M5MLFy6YqlWrGm9vbxMQEGACAwPNpk2bko2/deuWGTRokPH19TXFihUzvr6+pnr16ubYsWNWceXLlzf58+e3Orfjx49/3IeT7n799VeTO3dukz9/fuPp6Wnq1atnrl+//sjbpKXfzGT48OHG3d3d8jMzfPjwB8Z/++23pnjx4iYgIMAULlzY+Pn5mTlz5ljFTJ061Xh4eFhdU1WrVn2ch5Ghrl27ZurVq2c8PT1N/vz5Te7cuc2vv/76wG3q1q1rAgICrM7JsGHDHrnfzGTTpk0mMDDQBAQEGG9vb1O1alVz4cKFZON/+OGHRO+XefLkMS4uLubatWvGGGNu3rxpJJmSJUtaxa1cuTKjDssYYwxFW2R6oaGhZvfu3SY+Pt7WqSATmjFjhjl37pyt00AmtWfPHjNw4EBbp4FMbN26dWbu3LkmISHB1qkAj83w4cPN+vXrzdy5c1NctK1evbpp3LixiY2NNQkJCaZfv34mMDDQREVFpSomszp79qzx8PAwU6dONcYYc/36dVOsWDHTu3fvFPdx9+5dExAQYF5++WVL2/2i7cWLF9M7ZbswY8YM4+3tbQ4cOGCMMWbDhg3G1dXVLF26NNltxo8f/9CiT1r6zUwaN25satWqZe7cuWOMMWbw4MEmV65cyX5I8Ndff5mpU6eayMhIY4wxkZGRpnnz5qZs2bJWceXLlzeTJk16vMk/ZpcuXTJeXl6WYnN4eLgpV66c6dSp0yNtk5Z+M5Nvv/3WZM2a1ezYscMYY8z27duNu7u7+fbbb5PdZtSoUVaF/7lz5xpnZ2dLH8bcK9qWLl368SVuY506dTIVKlQw4eHhxhhjxo4da7y8vMzly5eT3aZu3brmnXfeSfd+M4uwsDCTK1cu8+abbxpjjLlz547l/iA1KlasaNq0aWN5fb9ou3PnznTNN7Uo2iLTiomJMQMGDDAuLi5GkqlRo4blRgNIicWLFxtJpkiRIhRukWp79uwxefLkMQsXLrR1KsiEYmNjTcuWLU316tXNvHnzHjh6DnAUKS3aHjx40EgyGzdutLSFhoYaJycns2jRohTHZGYTJkwwuXLlsoyQNcaYL774wnh4eFgKZQ/zyy+/GElm7969lrb7Rdtdu3aZI0eOmJiYmPRO3aYqV66cqLDdpEkT07p162S3GT9+vKlSpYo5evSoOXPmTJIfoqWl38zizJkzRpJZsWKFpe369evG1dXVfP311ynu5+effzaSzI0bNyxt5cuXN6NHjzYHDhzItCNIp02bZry9vU10dLSlbd68ecbFxSXZY0rJNmnpNzOpV69eogJ0+/btTf369VPVj6+vr/nvf/9reT116lRTsmRJc+zYMXP69GmHGrh1/fp14+LiYlXYjoqKMl5eXubjjz9Odru6deuawYMHmwMHDpirV6+mW7+ZxVdffWXc3NzMzZs3LW333//OnDmToj727NljJJlly5ZZ2u4XbVesWGEOHjxosw+EHWvSJzxR+vTpo9DQUJ06dUpbtmzR/v37tXbtWlunhUymXr16cnFxUb169RQSEmLrdJBJ7N27V82aNdPnn3+ujh072jodZEKffvqpwsPDtXnzZnXv3t0h50YE0mrPnj2SpCpVqlja8uXLp/z581vWpSQmM9uzZ4+efvppq3lFq1atqujoaB0+fDhFfcyePVuVK1dOcm7WZs2aqUWLFvL29tbQoUMVFxeXXqnbTHx8vPbv3291TUj3ztvDromdO3eqRYsWqly5sgICAvTjjz+mS7+ZQVI/Szlz5lRwcHCqjm/fvn3Knj17onlLJ06cqE6dOikwMFC1a9fW8ePH0yXvjLJnzx7LHJf3Va1aVfHx8Tpw4ECat0lLv5nJnj17Hvln5vTp04qIiEg0P+mRI0fUrFkzVatWTXnz5tW8efPSI2Wb279/v+Lj463OW9asWVW2bNmHnrfPPvtMnTp1UsGCBVWlShWra+hR+s0M9uzZo+DgYKvfPVWrVpWkJOekTcrs2bMVGBio5s2bJ1rXrVs3tWvXTr6+vurTp0+i+YMfNx5Ehkxpx44d2rNnj/bu3St3d3cVLFhQZcqUUVxcnJYtW6bq1asrV65ctk4Tdq5EiRK6efOmNmzYoHr16qlevXrasGGDXFxcFBkZqeDgYFunCDu0b98+NWzYUFOnTlXHjh0VGRmpcePG6aeffpKLi4u6deumd955x+Em90f6Wrx4sXr06GEpyFy6dEnTp0/XhQsX1Lp1a7Vp08bGGQLJO3DggGJjY5Nd7+7urrJly6a5/+vXrytr1qzKmjWrVXuuXLl0/fr1FMfYk6ioqIcWW/PmzWt5ENH169eVL18+q/X3721TcnyXL1/WihUr9Pnnn1u1e3l56eeff1a7du0kSdu2bVOTJk3k7e2tMWPGpPBoMs6FCxd04cKFB8bcf4haRESE4uLi5OfnZ7X+YddExYoVdezYMRUrVkzGGH300Ufq1q2bihQposqVK6e5X1s6dOiQ7ty5k+x6Nzc3lS9fXtL/XU//frBOao5v3759mjRpkkaPHi0nJydL+yuvvKJu3brJ09NTN27cUOfOnfXss89q7969ypIlS2oPyyauX7+e5P/9/XVp3SYt/WYWd+/eVXh4eJLHFxYWpvj4+Ic+6C4uLk59+vRRmTJl1KpVK0t7qVKldPDgQZUuXVrSvQ/Be/XqpUKFCqlOnTrpfzAZ6P7/e2p/1/Ts2VNLly6Vr6+vIiIi1L17d7Vp00YHDx6Ul5dXmvvNLJL6Wbr/OiXHFx0dre+++06vvvqq1XXp6uqq2bNnq1evXnJ2dtbhw4fVpEkTDR48WLNmzUrfg3gAirbIlH777Te99dZblk8mt2zZoj179uiVV17R7du35erqqhUrVqh69eo2zhT2rGjRojp16pTy5s1rVbh1dXXVoEGDKNoiSbly5VLOnDk1e/ZsNW/eXK1atVL27Nk1bNgwHT58WO+//77279+vn3/+2dapwo5FR0frypUrku4VwBo1aqSKFSsqJiZGbdu21fjx4/Xuu+/aOEsgaSNGjNDly5eTXZ8vXz798ssvae7fzc1NcXFxMsZYFYDu3Llj+UAsJTH25OzZsxowYMADY7p166YhQ4ZIund8MTExVuvvF+FScnxff/213N3d1bVrV6v2QoUKWY1aq1atmgYMGKBvvvnGLou2S5cu1ezZsx8YM2PGDFWqVMlyXpI6bw86Zy1btrT828nJScOHD9fcuXP1/fffq3Llymnu15ZGjx6tM2fOJLs+e/bslm8o3j+G2NhYqw9BUnp8J06cUIsWLdShQwe99dZbVuv69etn+XfOnDn13//+V+XLl9eePXtUrVq11BySzbi5uSkqKsqq7WE/iynZJi39ZhYuLi5ycnJK8mfG2dlZzs4P/sJ3fHy8unfvrlOnTmnTpk1W56NJkyZWsa+99prmzp2r7777LtMXbR/0u8bb2zvZ7Xr37m35t7e3tz755BMVLFhQf/zxh5o1a5bmfjOLpN4vo6OjLeseZvHixQoLC1OfPn2s2r28vKzaSpUqpWHDhmnEiBH68ssvH3odpxeKtsiUXn/9dcXHx0uSDh8+rPbt2+vTTz9V//79FRERoVatWmngwIHat2+fbROFXXNzc1O+fPl04sQJlShRQt9++62eeeYZ5cqVy+oTXeCf8ufPbynyFy9eXM2aNdOCBQss6+vUqaOOHTtq5cqVatGihQ0zhT2rXr26vvzySw0ePFj9+/fXhx9+aLkxnDBhgsaOHav+/fsrT548Ns4USGzFihWPtf8CBQro7t27unLlivLmzStJMsbo0qVLKlCgQIpj7EnJkiW1a9euFMcXKFAg0cjc+yNOU3J8c+bM0XPPPZeiP8jz5cun8+fPpzi3jDRw4EANHDgwRbGenp7KmTNnopG5Fy5cSPU1ERAQYDkn6dlvRvnpp59SHHv/GC5cuKCnnnrK0n7x4kWrgnZSTp48qfr166t+/fqaO3eu1QcoSbk/etxer7ekFChQQOvXr7dqe9jPYkq2SUu/mYWTk5OCgoKS/JkJCgp64HUSHx+vHj166I8//tCGDRtUuHDhh+7Pnn+HpcY/fxYDAgIs7RcuXFDjxo1T3E9AQICcnJws5yS9+rVXBQoUSDRNZmp+lmbPnq3GjRsnmoYjKfny5VNUVJRu3LiRYd/sZk5bZEq+vr6Wr/AUKlRICxYs0EsvvSQnJyf5+Pho+PDhKZ7vC0+2EiVK6NChQ7pw4YJeeOEFvfPOO8qRIwdz3OKB7hdu/fz89M4771it69Chg4oWLZqqP87x5Bk8eLCuXbuml19+WadOnbL6JP+1117T3bt3H/qVYMCR/PXXX5b33Vq1ailLlixWxeHNmzfr1q1batiwYYpjMrOGDRtq165dlhH5krRs2TIVLFjQUli7c+eOdu3alWh+vT/++EPHjh2zGuV43/3RR/+0bt06lSxZMp2PwDYaNmxodU0kJCRo5cqVVtfEhQsXrAZ2/Puc3LhxQ/v377c6JynpN7OqWrWqPD09rY5vz549Cg0NtTq+Q4cO6ezZs5bXp06dUv369VWnTh3Nmzcv0dfdk7vWJGWq661hw4aJjn3ZsmXKnTu3ZRqYmJgY7dq1S2FhYSneJiUxmVnDhg21cuVKy2tjjJYvX251TV2+fFm7d++2vE5ISFDPnj21ceNG/f777ypatGiifv99XUVERGjnzp2Z6ppKTrly5ZQ7d26rn8XTp0/r8OHDVuft2LFjOn36tKR7154xxqqf9evXyxhjOScp7Tezatiwoc6fP281j++yZcvk5eWlZ555RtK9KTt27dqlGzduWG37999/6/fff0/V+2WePHkSTcfwWNnk8WdAKty9e9fMnDnTdO7c2QwaNMhcvHjxodtMmzbN1KxZMwOyQ2awZcsW06dPH/PCCy+YTZs2Wa0bNmyY6d+/vylWrJiZNGmSMebe06eDg4PNa6+9Zot0YWcuXbpkRo4caTp06GA+/vhjq6fUhoSEmLi4OKv4hIQEky9fPvP9999ndKqwQw96D/v222+Nk5OTcXFxsXq67dq1a03u3LmtnigNZGanT582O3fuNKNHjzYuLi5m586dZufOnSYiIsISU7BgQfPGG29YXr/zzjsmR44cZt68eWbx4sWmSJEipmPHjlb9piQms4qNjTUVKlQwNWvWNCtXrjRTp041rq6uZv78+ZaYvXv3Gknm999/t9q2Z8+epmzZskn2O3ToUPP666+bX375xfz222+md+/extXV1axYseJxHk6G2b9/v8maNat57bXXzOrVq80LL7xgcubMac6ePWuJGT16tPH19bW8rlq1qpkyZYpZv369+fHHH83TTz9tChYsaK5cuZKqfjOziRMnGm9vbzN79myzdOlSU7JkSdOsWTOrmNKlS5u+ffsaY+7dKxcoUMBUrlzZbNu2zfIzvXPnTnPnzh1jjDHbtm0zdevWNfPmzTMbN240U6dONdmzZzc9evTI8ON7FPHx8aZGjRqmcuXKZvny5ebzzz837u7u5n//+58l5sSJE1ZPnk/JNimJycyOHz9uvL29Tb9+/czq1avNiy++aLy9vc3x48ctMZMmTTIuLi6W13369DEeHh5mwYIFVtdUSEiIJaZu3brmo48+MuvWrTOLFi0y1apVMwEBAeb8+fMZenyPy/Tp042Hh4f54osvzPLly83TTz9tatSoYfX3R82aNU2HDh2MMcYcOXLEVKtWzcydO9ds3LjRfP755yZ37tymTZs2qe43M2vWrJkpXbq0Wbp0qZk9e7bx9PQ0H374oWX91atXjSSr91Bj7t1H5MmTx8TGxibqc9KkSaZXr15m0aJFZt26dWbw4MHGxcXFzJo167Efzz9RtIVdi46ONo0bNzZlypQxffv2NXny5DHFixe33Awk5dChQyZXrlyJinN4Mn388cfGz8/P9OrVy1SuXNm4uLiYrVu3WtYvWrTISLIUbO+7fPlykr+88WT566+/jL+/v2ndurXp2LGjcXZ2NkOGDHngNh9++KEpU6aMiYmJyaAsYa9S8h72/fffG09PT1OsWDHzww8/mJkzZ5qAgACzePFiG2YOpK/33nvPVKpUKdGyb98+S0zr1q3NlClTLK8TEhLM9OnTTaNGjUydOnXMhAkTEt3/pSQmM7t+/boZMmSIqVmzpmnZsmWi3wvHjh0zlSpVMrt27bK0xcbGmjp16pivvvoqyT7j4uLMzJkzTdu2bU2DBg3MwIEDzZEjRx7rcWS0Xbt2meeee87UqFHD9OzZ0xw9etRq/YwZM0y9evUsr69cuWJGjhxpGjZsaFq3bm3Gjx9vwsPDU91vZjdnzhzTpEkTU7t2bTNq1CgTGRlptb5Lly5m/PjxxhhjNm/enOTPdKVKlczJkyct22zfvt306dPH1K5d2zz33HPmhx9+yNBjSi9hYWHm7bffNrVq1TLNmzc3CxYssFofEhJiKlWqZDZv3pzibVIak5kdOHDAPP/886Z69erm+eefNwcOHLBaP3/+fFO1alXL64YNGyZ5TU2bNs0Sc+PGDTNq1CjTqFEj07JlSzNq1Chz8+bNjDqkDLFgwQLTvHlzU6tWLfP222+bsLAwq/V9+/Y1w4YNs7zev3+/6d+/v6lTp47p3Lmz+eqrr5Isxj6s38wsMjLSjBo1ytSuXds0adLEzJkzx2r9rVu3TKVKlczKlSut2tu1a2f+85//JNlnQkKC+f77703Hjh1NvXr1TN++fc2OHTse2zEkx8mYf42lBuzImDFjtH37di1btkyurq46evSoypYtq+nTp+vFF1+0ij106JB++eUXffrpp5o2bZo6d+5so6xhLw4cOKAGDRpo586dKly4sBISEixz1q5atcoSt23btkzzMARkHGOMypcvr5dfftny8JiPPvpII0eO1JUrV6y+FhMXF6f169dr7ty5OnbsmH7++ecUzYsEx5bS97DTp09r0qRJ+vPPP+Xv768RI0aoXr16tkscAAAAgM1RtIXdSkhIUIECBbRjxw7LpPWS1KBBA5UtW1Yff/yxVfzKlSt1+vRpdezYUf7+/hmdLuzQgAEDVKFCBaunNX/xxRd6//33FRoaasPMkBls2LBBY8aM0YYNGyxtN27ckJ+fnzZt2qTatWtb2o0xmjZtmgoWLKjWrVtn+qf+4tGl9j0MAAAAAP7J1dYJAMm5ffu2nn32Was/dqV7Dx67c+dOonie0o5/c3d313PPPWfVltz1A/zb3bt3E43oz5kzp7y9vRNdQ05OTho8eHBGpgc7l9r3MAAAAAD4J2dbJwAkx8fHR59++mmidg8PDyUkJFheHz9+XIcPH87I1JBJfPzxx8qePbtV27+vn9jYWKsnmwL3NWrUSC+88EKi9n9fQ7/++qtiYmIyMjVkAryHAQAAAHgUFG1hV9avX69Zs2Zp//79ycY4Oztb/uA9fvy4GjZs+MB4PDlOnDihOXPm6Ndff1VcXFySMf+8fmJjY9WpUyd99dVXGZgl7FVERIQWLFig7777TleuXEk27p/X0Ny5czVgwABdunQpo9KEnTLGaN26dZo9e7YOHDiQbBzvYQAAAABSgukRYBdiYmL07LPP6uDBg/L09NTRo0fVsmVLzZo1K9H8tE5OTjLGWP7Yff/999W1a1cbZQ578emnn+rdd99VcHCw/vrrL+XLl09z5sxR/fr1reLuXz/3C7ZZsmTRd999Z6OsYS/279+vFi1aKHfu3AoNDdXt27f17rvvasSIEXJ2tv588/41NHfuXI0ZM0br1q1TwYIFbZQ57EF0dLTatm2ro0ePKmvWrDp27Jhat26tWbNmKU+ePFaxvIcBAAAASAlG2sIujBs3Tk5OTjp16pSOHDmi9evXa//+/apWrZpOnTplFevk5KTTp09b/tjt0aOHjbKGvdizZ4/GjRun/fv3a9euXTp9+rRKlSqlJk2a6JtvvrGKdXJyUkJCgqVgu2DBArm68vnVk+z+9TBhwgTt27dPFy5c0HvvvafRo0erW7duio+Pt4p3cnLSvHnzLAXbokWL2ihz2IvRo0fLw8NDp06d0tGjR7V27Vrt2bNH1apV099//20Vy3sYAAAAgJSgaAu7sHz5cnXr1s3yxPX69etr+/btypo1q5o0aaKbN29aYl1dXfXHH3/wxy4sVqxYoVq1aqlQoUKSpMDAQC1btky9evVSr169tHr1akusq6uroqKiKNjC4tixYzpx4oS6d+8uSXJzc9PIkSO1aNEiLV68WK+99ppVvKurq/78808KtrC4/x52//dJw4YNtW3bNrm5ualJkyYKCwuzxPIeBgAAACAlKNrCLuTIkUNbtmyxasuXL59Wr16t27dv680337S0DxgwQN9++y1/7MIiR44c2rlzp2JjYy1tzs7OmjFjhlq1aqWePXsqKipKklS5cmVNmjSJgi0s7j+s7s8//7Rqb9u2rWbNmqXp06drzZo1lvYPP/xQv//+OwVbWCT1HpY/f36tXr1at27d0rBhwyztvIcBAJD+du7c+cA55QEgM3IyxhhbJwHMmjVLL7/8sv744w8988wzVuu+/fZb9erVS9euXZOvr6+NMoQ9u3jxop566ikNHjxY77//vtW6a9euqXDhwvr8888pkiBZjRo10rVr17Rt2zZ5eHhYrWvdurWcnJz0yy+/2Cg72Lv//e9/euONN/Tnn3+qUqVKVuu+/vpr9e/fX9evX5eXl5eNMgQAwD7t2rVLcXFxql69+iP107FjR/n7++uzzz5Lp8wAwPYYaQu70KdPH9WoUUNt2rTRsWPHrNZ16tRJ8fHxOn/+vI2yg70LCAjQxIkTNXHiRM2cOdNqXa5cudSgQYNE80oC//T555/r5MmT6tKli9WIbUnq3Lkz1w8eqF+/fqpSpYpat26tEydOWK3r3LmzYmNjFRoaaqPsAACwX5MnT9bYsWNtnQYA2CWKtshQsbGxevfddxUUFKTAwEBNmjRJ0r2vsi9atEh58uRR7dq19fvvv1u2OXDggHLkyKGnnnrKVmnDjixZskRPP/20cuTIoc6dO1vmO37jjTf02muv6aWXXtK4ceMsD4+KiYnR0aNHVaVKFVumDTtx+vRptWvXTjly5NDTTz+trVu3SpKKFy+uxYsXa/Xq1WrevLmuXLli2Wb//v1cP5B07/fJO++8o/z58yswMFD//e9/JUkuLi5asmSJcuTIodq1a2vjxo2Wbfbv369cuXKpcOHCtkobAIBMY/PmzTp69KgiIyO1detWbd26VTExMYnioqKitG7dOu3atUvR0dHJ9nfgwAGtWrUq0YeqR44c0fLly/XPLx6HhYVpyZIlunz5cvodEAA8AiZ0RIZJSEhQu3btFBMTo48++kjbt2/XsGHDVLBgQXXu3Fl+fn7auHGjunfvroYNG6pFixYqXLiwFi5cqC+//DLRV5bx5Jk5c6bGjBmjd999V/Hx8Ro9erReeukl/fjjj5Kkjz/+WIGBgXr33Xf1ww8/qEmTJtqwYYOqVq2qFi1a2Dh72NrJkydVq1YtvfDCC+rSpYs++eQTtW3bVkeOHJGfn5+aNGmidevW6bnnnlPx4sXVqVMn3bp1S3v27NEff/xh6/RhYwkJCWrdurWMMfrPf/6jP//8U0OHDlWhQoXUvn175cqVS5s3b9YLL7yg+vXrq2XLlipYsKAWLlyomTNnKkuWLLY+BAAA7N6IESOUJUsWnT17VoULF9aJEyfk5eWlLVu2WJ5DcOjQITVp0kTZsmVT3rx5deXKFWXLlk3+/v6Wfi5fvqxnn31WFy5cUPHixXXgwAFVq1ZN33//vdzd3eXl5aUePXpo1KhRGjRokCTppZde0vHjx7Vt2zYbHDkAJMactsgw3333naZNm6Y///zT8gCo5s2bKy4uTmvXrrWKXb16tZYtWyZnZ2f16tVLFStWtEXKsCM3btxQsWLFtHHjRpUuXVqSNHv2bPXr10/Xrl1Tzpw5LbGnTp3S119/rYsXL6p+/frq2rWrnJycbJU67ESbNm1Us2ZNvf3225KkS5cuqUiRIvrkk0/04osvWuIiIyM1b9487d69W0WKFNHAgQOVI0cOW6UNOzFv3jxNnz5df/zxh1xcXCRJjRs3louLi1atWmUVu2rVKq1YsUIuLi7q3bu3ypcvb4uUAQCwe88995xu3bpleS+tVauWTp48qT179ihfvny6c+eOSpUqpQEDBlju4erWravcuXPrxx9/lLOzsxYvXqwOHTrolVdescxp27RpUwUFBWnGjBlycXFRZGSkateurfbt2+vdd9+VJC1cuFDdu3fXjh07tGfPHr3yyivavXu3SpQoYZuTAQD/wkhbZJi5c+dq0qRJloKtdO8BP5MnT04U26RJEzVp0iQj04OdW7hwobp27Wop2EqyjHo7c+aMVdH2qaee0rhx42yRJuzU1atXtW/fPi1ZssTS5u/vr8qVK+v06dNWsZ6enho4cGAGZwh7N3fuXE2ePNlSsJXu/Q5K6oEnzZo1U7NmzTIyPQAAHEanTp2UL18+SVLWrFlVs2ZNHT16VJIUGhqqTZs2aceOHXJ2vjfbY/v27a0KrefPn9fq1av1ySef6Ndff5UxRsYYlSxZUmvXrrUUbTt16qSVK1eqU6dOunjxoqZMmULBFoBdoWiLDNO9e3fVqVPHqs3f319xcXE2ygiZSbVq1fTMM89YteXJk0fOzs5cQ3goLy8vTZ482XJzfx+/g5BSvXr1Uo0aNazauH4AAEh/fn5+Vq89PDx0+/ZtSdLZs2clSUWKFLGK+efzT+5/IL9q1Sq5ublZxZUpU8bq9cSJE5U/f37LaF4AsCc8iAwZpkePHom+ou7m5mY1+fvJkyc1cuTIjE4NmUD58uWTnCbD1dXVcg3FxcVp8ODBVg+RAqR7ozQ6d+6cqP3fv4Pmzp2rX3/9NSNTQybRs2fPh76HHT9+XO+8805GpwYAwBPj/rfrwsLCrNpv3bpl+be3t7ckacKECVqyZInV8u9vyIwcOVIFChTQkSNHtGLFisebPACkEkVb2JSTk5MSEhIk3SvYNmzYUMHBwTbOCpnJ/WsoLi5OXbp00dmzZ5l/FCn2z99Bc+fO1ZgxY/gdhBT75/Vz/PhxNWzYUMWLF7dxVgAAOK4iRYoob968+uWXXyxtFy5c0M6dOy2vy5YtK39/f82aNSvR9v8c3LFo0SJ99913WrJkid5991316dOHwR8A7ArTI8CmnJycZIyxFGzHjBmj3r172zotZCJOTk6KjY1Vly5dJEk//PBDoq9BAcm5/zvofsF23bp1Klq0qK3TQiZx//q5X7B9//331aNHD1unBQCAw8qSJYvGjRun119/XeHh4QoMDNS0adOUNWtWS4yrq6tmzpypjh076tatW2ratKlu3LihZcuWqVmzZnrrrbcUGhqq/v3764MPPlC5cuVUunRprV69Wn379tWyZctseIQA8H8o2sKmnJ2ddfv2bQq2SDNnZ2cNHDhQJUuWpGCLVHN2dtbKlSu1ePFiCrZINWdnZ4WHh1OwBQAgjapUqaLIyEjL6zp16iR6GFjFihUVHR1ted2/f3/lzp1bixYt0uXLlzV58mTt37/f6tt2rVq10v79+/XVV19pxYoVyp8/v0aNGqW6detKkn766Sd17dpVgwcPliS5uLjom2++0ZAhQ7Rz505VqVLlcR42AKSIk/nnZGxABvvjjz9Up04dzZ49m4It0sTf3181atSgYIs0efXVV7V06VL9/vvvFGyRahs2bFDDhg01d+5cCrYAAAAA0hVFW9jcvn37VKFCBVungUzq4MGDKl68OAVbpElYWJhu3LihwoUL2zoVZFK8hwEAAAB4HCjaAgAAAAAAAIAdcbZ1AgAAAAAAAACA/0PRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BwAFERUXp22+/1enTpxOtu3r1qr755htdvnzZBpkBAAAAAIDUcjLGGFsnAQB4dM8//7wOHTqkHTt2KEuWLJb2Nm3a6Pr169q0aZNcXFxsmCEAAAAAAEgJirYA4CBu3bqlsmXL6oUXXtDEiRMlSXPmzNHrr7+u/fv366mnnlJ8fLy2bdumS5cu6amnnlKFChWs+vjzzz91+vRpOTk5KVeuXKpYsaLy5MljFfPbb78pMDBQuXPn1rZt2+Tr66t69epl0FECAAAAAOD4KNoCgANZu3atmjdvro0bNyowMFDlypXTpEmT1L9/f507d06tWrVSfHy8ihcvrt27d6tkyZJasmSJPDw8JEnTp0/Xli1bJEkXLlzQzp079eWXX6pr166WfVSrVk1ubm46f/68KlSooFq1amno0KE2OV4AAAAAABwRRVsAcDCvvfaafv31VwUGBsrLy0srVqyQJNWuXVuVKlXStGnTJEnR0dGqVauW2rZtq/feey/JvhYtWqR+/fopNDRUWbNmlXSvaPv333/rwIEDyps3b4YcEwAAAAAATxKKtgDgYO7cuaMKFSro6tWrOnTokAICAnTq1CkVLVpUkydPVmBgoIwxMsZo0aJFunnzptavX2/Z/uLFizpw4ICuXbum2NhY9enTR/v371e5cuUk3Sva/j/27js8iur9+/gnZRMSSIGEEEoIhCJSRAkd6VKjINLBAiIqTVFsWEBFvoAKqIiACKjYEEGQohQFBVEEaQLSS0LoLQkhhSTn+YMn+2NJApu6G3y/rmsvnTP3nLlnNuXk5uyZ8PBwTZ061VGXCAAAAADALc3d0QkAAPKWl5eXGjVqpAMHDqh06dKSpCNHjkiS/vrrL1ksFpvYqlWrWrcnTJigMWPGKDw8XMHBwdbYM2fO2JwjODg4n68CAAAAAID/Loq2APAf4OfnJ0l6+eWXVbt27UxjLly4oJEjR2rVqlVq3bq1pKvF2i+//FLXfyjDxcUlfxMGAAAAAOA/zNXRCQAA8t+dd96p0qVL66OPPsqw79ixY5Kks2fPyhijypUrW/fNnz+/wHIEAAAAAABXMdMWAP4D3N3dNWfOHHXp0kWnT59W27ZtdfbsWS1dulSPPPKIBg8erLCwMN1xxx3q06ePHnnkEe3Zs0dfffWVo1MHAAAAAOA/h5m2AHALaty4sdq1a2fT1q5dO+3evVvh4eHavHmzUlJSNG3aNA0ePFiS5ObmpjVr1qhDhw7auHGjfHx8tHHjRvXt29e6Nq4ktW/fXrVq1SrQ6wEAAAAA4L/ExVy/UCEAAAAAAAAAwGGYaQsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAsEpJSVFSUpKj0wAAAEAhl5iYqNTU1EJ/vqSkJKWkpOR5vzeSmpqqxMREGWMK9LwAnAtFWwC4gbS0NEenkKX8KLAOHz5cAQEBedpnZigOAwAAZ3HlyhUlJiYqOTk5y5jk5GQlJibqypUrBZiZ46Tfk/RXdsfEly5dkpeXl955550c55CdImxenC8rpUqV0tChQ/O83xuZOXOmvLy8dPTo0Xw9D8VhwLlRtAWA62zevFl9+/ZV2bJl5enpqWLFiql69ep66qmntHv3bkenZ/XSSy+paNGijk4jRwpz7gAA4NbSvXt3eXl5qXjx4oqLi8uw//z58/Lx8ZGXl5f69+/vgAwLXvo98ff3l7+/vywWi0JCQvTMM88oJibmpse7uLjI09NT7u7uOTr/2bNn5eXlpcmTJ9sVn9vz/VfNnTtXXl5e2rt3r6NTAZAJirYAcI33339fDRo0kKurq5YvX67ExESdOXNG7733njZt2qQePXo4OkUAAADkMQ8PD125ckXz58/PsO+rr76Su7u7XFxcHJCZ47i5uVln2sbFxWnMmDGaMmWKOnXqdNNjixYtqsTERD333HMFkGnBnw8ACgJFWwD4/37++Wc988wzGjJkiObOnavatWvLzc1NXl5eatu2rdavX6+HH344y+Oz+tjYlStXMv24XfrHka51/bIBWX1U6cqVK9aPi1370bXM4nPzcaesjk1JScmQ+7XH3OjjbDfKPTv92nuvMuPMy14AAICC5+XlpYiICH366acZ9s2ZM0fdunWTq2vGP5+vXUbgZssrXLu0QlbjlhuNUfKiD3vzvZ63t7f69eun3r1767ffftP27dslZVzCICUlxXr+zMaD17dllqsxxjrGSx8bJiYm3nRd2ZyeLzO5HVemj1szW04j/W+A7I5HM8vpZssb3Oi+paSkWPNLX/4jPa/s9JtZrL33ryDXPAYKI4q2APD/vfXWW/L29tZbb72V6X43Nze98MILNm0rVqxQ48aN5eHhIU9PT91xxx2aO3euTUz37t11xx13ZOhv2rRp8vLy0rFjx6xt6WvKxsbG6sEHH1SxYsVUpEgRde3aVefPn7fGPfLII5oyZYpSU1OtH1vz9/fX4cOHJUkHDhxQ165dFRAQIE9PT1WqVEkvv/yyYmNj7boX586dU9euXVW0aFF5e3urZ8+eOnXqlHX/J598Ii8vL23cuDHDsTNmzJCXl5f+/vvvTPu+Ue7Z6Tf9Xt0s13Tbt29Xp06d5OPjI4vFogoVKmjcuHEUcAEAgCSpX79+WrdunQ4ePGht27Fjh7Zs2ZLlsggDBgywjmW8vb3l4+OjDh06WIua6erXr6/77rtPO3bsUKNGjeTp6akKFSro+++/lyRt3bpVTZo0UZEiRRQUFKTp06dnOFde9GFvvlmpUqWKJCk6OlrHjh2Tl5eXpk2bpi+//FJVqlSRp6enFi5cmOkas9fGz58/X5UrV5bFYlGlSpW0YMECa9zWrVtVqVIlSdJrr71mzXfYsGFZ5pWb86U7deqUevbsKW9vbxUtWlRdunTR2bNnMz3flStX9NZbb6ly5cry8PCQl5eX7r33Xu3Zs0fS1aJlRESEgoODbdalTUpKUv369VWlShWbsX1WkpOTNWzYMPn5+cnDw0OtW7fWrl27rPv/+OMPeXl5afbs2RmOXb9+vby8vPT5559n2vf48eM1ePBgSVK9evWs93nZsmXZ6jd9/d39+/ffMNd0x44dU79+/RQYGCiLxaKgoCA99dRTio+Pv+n9AP5zDADAxMfHG3d3d9O6dWu7j1m8eLFxdXU1AwYMMCdOnDDnz583b7zxhpFkJk2aZI3r3Lmzue222zIcP2XKFCPJREVFWduGDBliihYtavr162dWrVplEhMTzW+//Wb8/f3NI488YnP8iBEjjJubW6a5ValSxbRq1cocPnzYpKWlmSNHjpgJEyaYr7/++obXlH7+Hj16WM//559/mooVK5qaNWuahIQEY4wxcXFxxtfX1zz88MMZ+rjjjjtMzZo1b3ierHLPTr/25mqMMX/++afx8vIyDzzwgDlw4IBJSkoyy5YtMyVKlDDDhg27Ya4AAODW1rlzZ+Pn52euXLligoKCzGuvvWbdN3z4cFOhQgWTlpZm3NzcTN++fbPsJzU11ezbt8907tzZBAcHm7Nnz1r31a5d29SrV8888MADZt++fSY+Pt4899xzxmKxmDVr1piOHTuaPXv2mMuXL5uXXnrJSDLbtm2z6T8v+rA3386dO2c6VuvZs6eRZP755x8TFRVlJJmIiAgzePBgc/LkSbN//37z+++/m7i4OCPJjBs3znpsevz9999vnn76aXPq1Clz4cIF06dPH2OxWMzRo0etsWfOnDGSzDvvvJNl/tfK7fkSExNNzZo1TcWKFc0ff/xhEhMTzerVq03Xrl2Nr6+veeKJJ6yxaWlpJiIiwgQGBpoFCxaY+Ph4Ex0dbbp162ZKlChhIiMjjTHGnDp1ypQpU8bUq1fPJCUlGWOMGThwoLFYLOaPP/644fVMmzbNSDJdu3Y1n3/+uYmPjzf79u0zjRs3NgEBAebYsWPW2Nq1a5s6depk6KN3796mWLFiJi4uLsvzzJkzx0gy//77b4Z99vabnVyPHTtmSpcubRo0aGC2bNlikpOTzV9//WX92yUtLe2G9wX4r6FoCwDGmAMHDhhJpl+/fnYfc9ttt5lq1aqZ1NRUm/aIiAjj4+NjHchkt2gryXz77bc2selFzsTExAxt1zt58qSRZGbMmGH3tVx//rlz59q0r1692kgyn3zyibVt6NChpkiRIjYD/HXr1hlJZvLkyTc8z40Kzvb2m51c69WrZ6pUqWIdMKf78MMPjaurq82gHQAA/LekF22NMeaZZ54xoaGhJi0tzSQnJ5uSJUua119/3Rhjblq0TUlJMQkJCSYyMjLDGKV27drGYrFYC3rGXJ004O3tbXx9fc3hw4et7QkJCaZYsWLm6aeftuk/L/qwN9/0om1CQoJJSEgwJ0+eNB988IFxdXU1bdu2Ncb8X1H09ttvz1Bsu1ER9fpC4KlTp4yrq6sZM2aMtS0vi7b2nG/27NlGklm1apVN7BdffGEk2RRtFyxYYCRlmAwRHx9vgoKCzNChQ61t69atM+7u7ubJJ580n376qZFk3n///ZteT3oh9MUXX7Rpj4yMNBaLxQwfPtzaNn36dCPJ/PnnnzbX6OHhYQYMGHDD89yoaGtvv9nJtX///qZo0aLmxIkTNrFr1641ksxPP/10w3yB/xqWRwAAyfpgCWPn+kvHjx/X3r17dd9992VY3+yBBx5QXFycNm/enON82rdvb7Ndo0YNpaamKjIy8qbHlixZUiEhIRozZoxmzJhh1zHXi4iIsNlu3bq1vL299csvv1jbBg8erMTERM2ZM8faNm3aNHl4eOjBBx/M9jlz2u/Ncj1z5ow2bdqkjh07ytXVVVeuXLGuM9ykSROlpaVpw4YNOc4XAADcOvr376+jR49qzZo1Wrp0qc6ePatHHnkky/jDhw+rb9++CgoKksVikZ+fn3UJgWuXWZCk22+/XSEhIdZtb29vhYSEqFy5cqpQoYK1vUiRIgoNDbUue5WXfWQn32uXsqpSpYpmzJihl19+WQsXLrSJi4iIyNZD2q4f5wYFBSkwMDDT680L9pxvzZo18vT0VOvWrW1iM3vo2rJly+Tm5qaOHTta14VNTk6Wu7u76tWrp3Xr1llj7777bo0fP17Tp0/XwIED1aNHDz311FN25379ODckJER33nmnzZj8wQcflJ+fn6ZNm2ZtmzVrlpKTkzVgwAC7z3W97PZrT67Lli1Tw4YNFRgYaDMmr1evniwWi829A8CatgAgSSpdurQsFovNmlM3cuHCBUlSqVKlMuwLCgqSpJuuU5XVWqrp64tdK307Jibmprm5urpq9erVatSokZ5//nmFhoYqLCxML7/8suLi4m56vLu7u4oXL56hvWTJkjbXdPvtt6tVq1aaPn26jDE6c+aMvvvuO3Xu3FmBgYE3PU9WstOvPbmePn1akjR16lQVK1ZMPj4+8vHxka+vrxo2bChPT0+77isAALj11apVS3fddZc+/fRTzZkzRy1atLAphl7r8uXLat68ufbs2aMff/xRSUlJSkpKsq6Dev1DqIKDgzP0UaxYsSzbMxuf5KaP7Obr5uZmfThVbGysdu7cqTFjxqho0aI2cWXKlMlw7hvJLFcfH598G4/Zc77z58+rZMmSGYrPPj4+KlKkiE3b6dOnlZqaqqCgIOvY0tfXV76+vlq9enWGtVm7d+8uDw8PXblyRcOHD89W7ul/V1zfdu2YvGjRonrkkUc0b948nT9/Xmlpafr44491++23q1GjRtk637Wy2689uZ45c0Zr167NMCYvUaKEXF1d7X7+BvBfQdEWAHT1icEtW7bUn3/+aS3I3kh68fDkyZMZ9p04ccImxt/fP9NiaXR0dKZ9Z2emQlaqVq2qb7/9VhcuXNDff/+tvn376u2339agQYNuemxKSkqGgrMxRqdOnVLJkiVt2gcPHqyDBw9q5cqVefIv+tnt155c0//7wgsvWP/wuP71xBNP5DpnAABwa+jXr58WLFigH3/8McsHkEnSunXrFBUVpbfeekvh4eGyWCySlOWM0azGeNkZ++Wmj+zma6/0fuyVF2PdvD5fYGCgzpw5k+FTdxcvXlRiYqJNW8mSJVWkSBHFxcVlOq7cv3+/NTY5OVk9evRQiRIlVKVKFfXv31+XLl2yO/fMHq578uTJTMfk6Z9UW758uY4cOZJnY3J7+7Un18DAQEVERGQ5Jv/ggw9ynTNwK6FoCwD/36hRo3TlyhW98MILme5PTk7WqFGjJF2dYVuzZk0tWrRIqampNnHz58+Xv7+/6tWrJ0mqXLmyTp48aTOQSUtL09KlS3OVb7FixZSamqqUlJQsY9zc3FSnTh2NGTNGLVu2tHsZgEWLFtls//jjj0pMTNQ999xj0965c2eVLVtWH374oWbMmKGQkBC1adMm17lnp9+b5RoUFKSGDRtq4cKFSkpKumluAADgv61v375yc3OTn5+funbtmmWcm5ubpKuf/LnWp59+mp/p5Vhhyjd9Nm9Bjd3uueceJSUl6ccff7Rpv34pCOnqODUxMVHff//9TfsdMWKENm/erHnz5mnBggWKjIzUY489Znde149zDx8+rG3btmUYk992221q3bq1pk+frqlTp8piseihhx66af/FihWTlPV9zk6/9uTauXNn/frrr9ZJLgBujKItAPx/TZo00cyZM/XZZ5/p/vvv12+//aYLFy7o1KlTmj9/vurUqWMzOJs4caKOHDmiBx98UHv37tXRo0f13HPPadWqVRo/fry8vLwkXZ2tYbFYNHToUEVGRmrfvn0aMGCA7rzzzlzle8cdd0iSvvvuO8XHxysxMVHGGG3ZskUdO3bUkiVLFBkZqQsXLmjJkiX666+/7CqoFi1aVL/88osWLlyoM2fOaOXKlXr88ccVHh6u3r1728S6u7vr8ccf19KlS3XkyBH169cvwxq/2ck9u/3am+v06dN18uRJ3Xvvvfr99991/vx5HThwQN9++62aNm2qc+fO3TRnAADw3xAQEKDY2FidPXtW3t7eWcY1aNBApUuX1muvvabdu3crOjpa77zzjs6cOVOA2dqvMOXr5eWlKlWqaPXq1Tp58qQSExNvOFEht3r27Knw8HA9/vjjWrFihc6cOaPvv/9eK1eulJ+fn01s586d1bt3bz3xxBP66KOPdPjwYZ09e1Z//vmnnnvuOb355puSpHnz5unDDz/UuHHj1KxZM9WqVUsfffSRtd0ep06d0tSpU3Xy5Elt3rxZ3bp1U8mSJfXcc89liB08eLAOHDign376Sffee2+myxVcr1atWnJxcdGiRYsUGxurxMTEDEu42duvPbmOHTtWgYGBateunZYvX67Tp08rMjJSP/74o7p06cKatsB1KNoCwDX69++vHTt2qEyZMnriiSdUsWJF1alTR5MnT1bXrl21fPlya2zbtm21Zs0aXbx4UU2aNNGdd96pjRs36vvvv7f5uH25cuX0/fff6+DBg6pVq5Z69+6tnj17qmnTpvL09LT5yJbFYsmwbpZ0dWaEp6enTeGyS5cueuqpp/TCCy+oZMmS8vf31+HDh1WnTh09/fTTmj17tpo2baoqVaro9ddf16hRo246QEw//0cffaQffvhBNWrU0MMPP6wOHTpo5cqVmX787fHHH5fFYpGLi8sNP0J4raxyz0m/9uRau3Ztbdu2TZUqVdIjjzyiChUqKCIiQosXL9aECRMUEBBgV94AAODW4+Hhken463pFihSRh4eHddvHx0c//fST/Pz81KxZMzVu3FhnzpzR9OnT5enpaTMW8fT0tDk2J+257SM7+dpzT1xcXOTp6Zlh5m5W+24Un9k1zJo1S5cvX9Ztt90mf39/DRs2LFu5ZOd8FotFK1euVIcOHfTII4+oRo0a+v777zVjxgwVKVIkwxj4yy+/1Lvvvquvv/5adevWVY0aNfTcc8+pXLlyevbZZ3XgwAENHTpUDzzwgJ5//nnrcf369dPAgQP1yiuvaMuWLVlej7u7uzw9PTVmzBhFR0erXr16ateuncqXL6/169dnWjjt3LmzypUrJ0l2L41w2223aeLEifriiy9UunRp+fv7a9myZTnq980337xprkFBQdq8ebMiIiL0wgsvqEqVKmrevLk++ugj9e/fX3fffbddeQP/FS7G3kelAwCQifPnz6tMmTK6++67tXr16gLrd+jQofr000+ztS4YAAAAcKsKCwtTUlKSIiMjrcth5He/06dP16BBg3T48OEsH9oHIGeYaQsAyJVFixYpKSlJAwcOLBT9AgAAALeav/76S4cPH9ajjz6apwXb/OoXwM1l/JwAAAB2SE5O1vHjx/Xuu++qWrVqN3xQhzP0CwAAANxqUlJSdPHiRY0ePVq+vr43XEbCGfoFYD9m2gIAcqRNmzaqVauWSpQoofnz52e6Xlh+9pvV+r8AAADAf8WwYcMUGhqq6OhozZ8/364HkOVlv+nr7177nA4AeYM1bQEAAAAAAADAiTDTFgAAAAAAAACcCEVbAAAAAAAAAHAiPIgsH6Slpen48ePy8fFhXRcAAIA8ZoxRXFycypQpI1dX5iDkNcayAAAA+cfesSxF23xw/PhxhYSEODoNAACAW1pUVJTKlSvn6DRuOYxlAQAA8t/NxrIUbfOBj4+PpKs339fX18HZAAAA3FpiY2MVEhJiHXMhbzGWBQAAyD/2jmUp2uaD9I+R+fr6MtAFAADIJ3x0P38wlgUAAMh/NxvLsggYAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE2FNWwAAAAAAABSo1NRUXblyxdFpAHnOYrHIzc0t1/1QtAUAAAAAAECBMMbo5MmTunjxoqNTAfKNv7+/goODc/XgXIq2AAAAAAAAKBDpBdugoCB5e3vnqqgFOBtjjC5fvqzTp09LkkqXLp3jvijaAgAAAAAAIN+lpqZaC7YBAQGOTgfIF15eXpKk06dPKygoKMdLJfAgMgAAAAAAAOS79DVsvb29HZwJkL/Sv8Zzs24zRVsAAAAAAAAUGJZEwK0uL77GKdoCAAAAAAAAeSghIUGbN29WSkpKoT2Xvf1evHhR//77b56e25ldvHhRe/bsyffzsKYtAAAAAAAAkIf27t2revXq6cyZMwoMDCyU57K338cff1zh4eG6/fbb8+zcWdmzZ49SUlJUs2bNTPcbY7Rv3z65u7urUqVKNvsuXLiggwcPZnrc7bffrqJFi9oVY4xRs2bNtH79elWtWjV3F3QDFG0BAAAAAADgUDN/O1Rg5xrYLKzAzlUQvL29FR4eLovFUuDn3rhxo1avXq3PPvssX88za9YsffTRR9q3b59KlSqlAwcOZIjZunWrunXrpkuXLik5OVkhISFauHChKleuLEnatGmTXn75ZZtjTpw4oePHj2v//v2qXLmyXTHFixfXwIEDNXLkSC1YsCDfrpnlEQAAAAAAAIAspKSkaPPmzdq8ebO2b9+u2NjYbPdx8eJFbd261aYtMTFRmzdvtj6s6tKlS9q8ebOMMUpISNDu3bsVFxdnjT937px2796tpKQkm35CQkI0ffp0FS1aNEM/iYmJ2rNnj+Lj4/P8miTp/fffV+/eveXl5WX3uXNiy5Yt+vjjjzVixIhM9ycnJ+uBBx5Qs2bNdPLkSZ05c0blypVTz549rTFt27a1XnP6q0aNGmrcuLG1sGtPjCT169dPixcvVlRUVK6vLSvMtM1HNUevkKsnT0QEAOC/6Mj4CEenAAAAgDwQExOjJ598UtLV4uCBAwfUpUsXzZ49W56ennb1sXbtWj344IO6dOmSte3IkSOqV6+eoqKiVK5cOW3btk1NmzZV//79tXz5cnl7e+vUqVOaPXu2fvnlFy1ZskQeHh66fPmyli9frrp160rKuIxBej/Dhw/XV199JT8/Px07dkwzZ85U37598+yaUlJStHTpUs2dO9faZs+5JWnz5s037NvHx0e33XabdXvq1KmSpCVLlmQav2rVKh05ckSvv/66XFxc5O7urtdee02NGzfWli1bVKdOnQzHHD16VD///LNmzZqVZR5ZxVSpUkUhISH64YcfNGTIkBteS05RtAUAAAAAALeeDVNyfmzjYXmXBwq9gIAAmyLjmTNn1LJlS3344YdZzvzMDVdXV0VHR8vNzU1PPPGEevfurUGDBunYsWNydXVV3759NXLkSK1ateqG/Vy8eFHR0dFyd3fXxIkTNWTIEHXv3l0eHh55ck3//vuv4uLidMcdd2Tr3KmpqdaCcVbCw8M1Y8YMu/KQrs7EDQoKUmhoqLWtfv36cnFxybJoO2fOHBUrVkw9evTIst8bxdx11136448/8q1oy/IIAAAAAAAAwE3ExsZq165dOnr0qJo0aaJffvklX87z6quvys3NTZIUEREhY4xee+01ubq6Wtt27Nhx035Gjx4td/er8zW7d++umJgYHT161CYmN9d06tQpScr0IWU3Orebm1uGJQiuf2WnYCtdXToiICDAps3NzU3+/v46d+5chvi0tDTNmTNHffv2lbd35p+Sv1lMYGCgTp48ma08s4OZtgAAAAAAAEAWEhMT9fDDD+uHH35Q+fLl5evrq1OnTqlkyZL5cr7g4GDr/6cXC69vs2ed2Mz6ST8uL64pfRmFxMRE+fj42H1uKfvLI9yMxWLJsNZvem6ZPaBt9erVioyM1GOPPZZlnzeLSUxMVJEiRezOMbso2gIAAABOwBijlStXauvWrercubNuv/32DDEHDhzQunXrlJqaqnr16ql27doZYpKSkrR06VJFRkaqatWq6tChg3VmDgAAyL733ntP27dv17Fjx6yzSl977bUs11fNjIuLS4a29AeQOUJeXFNYWJgk6dixY9kq9ubH8gjly5fXqVOnlJqaap2lfOHCBSUkJKh8+fIZ4mfNmqXw8PBMl02wN+bYsWOqWbOm3TlmF0VbAAAAwMFWrlypIUOGKDQ0VD///LMqVKiQoWj70EMPadOmTWrcuLEk6ZlnntHDDz9sfTCHdHX9uBYtWig5OVlNmzbVe++9p6pVq2r58uWZzjIBAAA3d+DAAdWtW9dmGYCbrSd7vaCgIMXHx+vChQsqXry4pJvPNs1PeXFNZcuWVaVKlfTHH3/orrvusvu49OUR8lKrVq0UHx+vX3/9Va1atZJ09aFl7u7uatasmU3suXPntHjxYr3//vtZ9nezmNTUVG3evFlPPfVU3l3EdSjaAgAAAA5WvHhx/fTTTwoNDc2yuNq3b199/vnn1pk6AwYM0N13362ePXta/xgZO3asYmNjtX37dvn4+OjYsWOqXr26Zs6cqcGDBxfY9QAAcCtp3ry5Bg8erM8++0zlypXTnDlztGXLFlWvXt3uPu666y6FhIToySef1ODBg7Vv3z698cYb+Zj1jeXFNUlSv3799N133+X7OGPv3r2Ki4vT8ePHlZSUZC361q5dWxaLRTVq1FDfvn01YMAAvfvuu0pMTNSzzz6r4cOHKygoyKavuXPnyt3dXb17987yfDeLWbFihby8vNSxY8e8u8jrULQFAAAAHKxevXqSpJSUlCxj2rdvb7PdqFEjubm56eDBg9ai7fz589WnTx/runLlypVTx44dNX/+fIq2AADk0EMPPaS4uDh98cUXunLlipo2baoPPvhA69aty/IYb29vhYeHW/8xtkiRIlq1apXGjBmj0aNHq0aNGpo3b56efvppeXh4SLq6jmt4eLjNska+vr4KDw+36bt48eI2H9m//lyZ9WOxWBQeHm5dX9aea7q+38wMGjRIkydP1u7du1W9enW7zp0T77zzjrZt2yZJKlWqlHV5heXLl1uLsrNnz9akSZM0ZcoUubu76/XXX890/LNt2zY9/fTT8vX1zfJ8N4uZNm2annnmGet7lx9cjDEm33r/j4qNjZWfn59Chn8rV8+cf0ECAIDC68j4CEencMtKH2vFxMTccLBdGKWkpMhisejrr79Wr169bhi7YMECdevWTf/8849q1qyphIQEeXt769NPP9UjjzxijRs9erRmzJiR5dONk5KSbB7cERsbq5CQkFvy/gL4j9kwJefHNh6Wd3nAKjExUYcPH1bFihXz9QFOKFizZs1SVFSUXn/9dUenUiAOHTqkJ598UkuWLLE+jO16N/pat3csy0xbAAAAoJA5ePCgnnzySQ0aNMj6AIxLly5Jkvz8/Gxi/f39rfsyM27cOId+PBMAABRuAwYMcHQKBSosLEwrV67M9/PwGFkAAACgEImMjNQ999yjZs2a6YMPPrC2p3/kMDY21iY+JiZGRYsWzbK/kSNHKiYmxvqKiorKn8QBAABgN2baAgAAAIVEVFSUWrRooTvvvFPffPON3N3/bzhftGhRlS1bVgcOHLA55sCBA6patWqWfXp6emb50T4AAAA4BjNtAQAAgELg2LFjatGihWrXrq1vv/0204eCdOnSRd9++60SExMlSWfOnNGyZcv0wAMPFHS6AAAAyAVm2gIAAAAOdvjwYc2bN09paWmSpB9++EFHjhzRXXfdpXbt2kmS7rnnHp05c0Z16tTRxIkTrcc2a9ZMjRs3liSNGjVKP/30k5o1a6aWLVvqhx9+0G233aZBgwYV/EUBAAAgxyjaAgAAAA525coVXbx4UZL04osvSpIuXryohIQEa0zXrl2Vmpqq+Ph4m2OTkpKs/1+yZElt2bJF8+bNU2RkpF599VV1795dHh4e+X8RAAAAyDMUbQEAAAAHq1q1qsaPH3/DmLFjx9rVl4+Pjx577LG8SAsA/rs2TMnd8Y2H5U0eAP6zWNMWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAIAsbNy4UYGBgTbryN/Mzp07FRgYqPPnz+djZvl7Lnv7HTt2rEaMGJGn53aUQ4cO6fbbb1dsbKyjU2FNWwAAAAAAADhYbtcRzo5srjl85coVnTt3TsYYu49JSUnRuXPnlJaWlt3ssi2/zmVPv8eOHdPbb7+tnTt35um5r7dp0ya9+eab2rlzp7y9vdW3b1+99NJLcnW1nY+6bNkyTZo0Sfv27VPNmjU1fvx41a5d27o/ODhYKSkpNseMHTtWTzzxhCQpLCxMderU0dixYzVhwoR8vaabYaYtAAAAAAAAUEjVqlVLZ86cUUBAQIGf+8MPP1TLli0VEhKSb+fYvXu3mjdvrmrVqunnn3/WjBkzNHfuXL300ks2cbNnz1bXrl3VrVs3bdiwQS+//LLGjRtnE3P27Fl99NFH2rNnj/X1yCOP2MQ8/vjjmjFjhuLj4/PtmuxB0RYAAAAAAAC4iRUrVqhFixYqX768OnTooH379tl97I8//qgKFSrYtB04cECBgYE6ceKEpP9bhuHrr79Wq1atVL58ebVt21Z79+7VggUL1KBBA5UvX149e/bU2bNnrf38+++/qlatmi5cuGDTz48//qjWrVurQoUKatu2rXbt2mU95vTp0woMDFRgYKBKly6txo0ba+7cudm+J19++aW6dOli3bbn3Nn11VdfqVSpUnrnnXcUFhamu+++W+PGjdP777+vixcvSpJiY2M1fPhwjRo1SoMGDVJISIiaNm2qr7/+OkN/vr6+1msPDAxUkSJFbPY3a9ZMbm5uWrJkSY5zzgsUbQEAAAAAAICbeOaZZ/T6669r5cqV8vHxUdu2be1e5zYpKcmm0Cr93/IDqampkv5vGYZJkyZp/PjxWr16teLj49WyZUtNmjRJU6dO1YoVK3TgwAE9//zzGfpJX8YgvZ9XX31Vb731ln755Rf5+/ura9eu1iUeSpYsaZ1p+vfff+vZZ5/VsGHD9MMPP9h9Pw4fPqxjx46pXr161jZ7zp2ammpTNM3s1blzZ2ufiYmJ8vLysjm3t7e3kpOT9ccff0i6WlCPi4vLMGvWxcUlQ96DBg1SuXLl1LRpU82ePTvDshcuLi6qV6+e1q5da/e9yA+saQsAAAAAAADcxMSJE9WiRQtJVz+KX65cOc2bN08PP/xwnp5n+vTpCg8PlyQ99dRT6tWrl1atWqUaNWpIkgYPHqwxY8bctJ8ZM2aobt26kqQ33nhD1atXV3R0tMqVKycXFxcFBgZaY7t166bNmzdr7ty56tSpk115Hj16VNLVdWKzc243Nzft2bPnhn1bLBbr/0dERGjSpEmaM2eO+vXrp/Pnz1uXPTh+/Lgkad++fQoICNCff/6pMWPGKCYmRnfccYdef/113XXXXda+OnbsqBEjRqh8+fL6+eefNXz4cB09elRvvPGGzflLly6tyMhIu+5DfmGmLQAAAAAAAHATjRo1sv5/sWLFVLt2bf3zzz95fp7bb7/d+v/p69Re33bu3Lmb9lO9enXr/5csWVKSbI6bPXu27r77boWEhCgwMFBTpkyxFmLtkT5D2M3NLdvnvtlMWz8/P2tsy5YtNXPmTI0ePVre3t6qVKmS7r//frm6utrM3o2JidHEiRM1a9YsrVq1SiVLllSLFi1siq8//PCDmjdvrooVK+qxxx7TqFGjNH78eF25csUmfzc3twwPLCtoFG0BAAAAAACAm3B3d8+wnZycnOP+rv9YfjpX14zluuvbsjr2Zv2kH/fNN99Yl0T49ddftWfPHg0fPjxb11OmTBlJ0qlTp7J17uwujyBJAwYMUGRkpM6dO6dz586pbdu2SktLU2hoqCQpKChIKSkpmjx5ssLDw1W5cmVNnz5daWlpWrx4cZbXUK9ePSUnJ+vw4cM27adPn1bZsmXtvhf5geURAAAAAAAAgJvYvn27WrduLenq2q27du1S165d7TrWz89PCQkJunLlivWj/0eOHMmvVG/ql19+UUREhHr27Glty86D1SSpWrVqKlGihP7++29VrVrV7uOyuzzCtby9vSVJ8+fPl7+/v5o0aSLp/2ZBX7v2rbu7uywWS4ZZtNfav3+/XFxcrDOa0/39998ZlkwoaMy0BQAAAAAAAG7ipZde0okTJ5ScnKxXX31VSUlJ6tOnj13H1qpVS56enpo+fbokKSoqSqNGjcrPdG8oNDRUf/75p06cOKHU1FR98cUXWrhwYbb6cHFxUdeuXbV8+fJsnz87yyNI0rPPPqtz587JGKPFixdrwoQJmjBhgrWIW7t2bbVq1UpvvPGG4uLilJqaqvHjx+vy5cvq2LGjpKuF3k8//VSxsbEyxmj9+vUaNWqUunXrZlO03b59u86dO2f32r75hZm2AAAAAAAAhcjM3w5lK35gs7B8yuS/pUePHqpTp47Onz+vMmXKaMGCBfL397fr2MDAQH3yySd6+umn9eKLL6p06dJ67LHHtHnz5vxNOgtPP/20Nm7cqNDQULm5ualmzZrq27evduzYke1+GjZsqPPnz6tEiRL5lO3VNXJr1aqlixcvKjg4WFOnTlW/fv1sYubNm6fHHntMgYGBcnNzU1hYmBYtWqRq1apJuro27uuvv64RI0bo8uXL8vf3V79+/fTaa6/Z9DN79mz16dNHQUFB+XY99nAx9iyCgWyJjY2Vn5+fQoZ/K1dPb0enAwAAHODI+AhHp3DLSh9rxcTEyNfX19Hp3HK4vwBuGRumOO7cjYfla/eFtWibmJiow4cPq2LFiipSpIij07FbSkqKLl68qMDAQElSfHy8ihYtesNjUlNTdeHCBQUEBMjFxcVmX0JCgry8vKwxJUqUkKura4bzSFeXYYiJibFpS05OVlxcnHV26PXnyqwfY4zOnTun4sWL2zw4LDk5WampqfLy8lJCQoKSkpKshegbXcO1Bg0aJF9fX02YMCFb584Je+59cnKyjDHy9PTMMib9PbhedHS07rrrLm3bts26Zm9O3Ohr3d6xFjNtAQAAAAAAgCy4u7vbFCFvVjSUrq7beu0x10ovFl4fc/15pKtru17f5uHhYfNxfnv6cXFxyTQfDw8Pm7yuLWTe6Bqu9cEHHyg+Pj7b584Je+79tdeUlcwKttLVGdEHDhxwin+4pmgLAAAAAAAAIEcsFovdy0Q4O09PzxvO0C1IPIgMAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAQIExxjg6BSBf5cXXOEVbAAAAAAAA5DuLxSJJunz5soMzAfJX+td4+td8TrjnVTIAAAAAAABAVtzc3OTv76/Tp09Lkry9veXi4uLgrIC8Y4zR5cuXdfr0afn7+8vNzS3HfVG0BQAAAAAAQIEIDg6WJGvhFrgV+fv7W7/Wc4qiLQAAAAAAAAqEi4uLSpcuraCgIF25csXR6QB5zmKx5GqGbTqKtgAAAAAAAChQbm5ueVLYAm5VPIgMAAAAAAAAAJwIM20BAAAAAIDz2TDF0RncMmb+dihb8QObheVTJgDsxUxbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCA8iAwAAAAAAgFV2H1yWHTzkDLAPM20BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIm4OzoBAAAAAABwC9owxdEZAEChxUxbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCA8iAwAAAAAAcKCZvx1ydAoAnAwzbQEAAAAHS0lJ0cKFC9WmTRsFBgbq+++/zzRu8eLFatSokcqWLauWLVvq999/z1EMAAAAnBtFWwAAAMDBJk+erC+++ELPPPOMzp07p6SkpAwxa9asUbdu3dSrVy+tXbtW9evXV5s2bbR3795sxQAAAMD5UbQFAAAAHOy5557TwoUL1bZt2yxjxo8fr44dO+rpp59WlSpVNGHCBIWFhWnSpEnZigEAAIDzo2gLAAAAOJiLi8sN9xtjtH79et1zzz027W3atNG6devsjgEAAEDhwIPIAAAAACd38eJFXb58WaVKlbJpL1WqlE6cOGF3TGaSkpJslmOIjY3Nw8wBAACQE8y0BQAAAJycMUaS5ObmZtPu7u6utLQ0u2MyM27cOPn5+VlfISEheZk6AAAAciBbRdsrV64oJSUlWyfIyTE5lV/nKshrAAAAt64tW7bovvvuU/ny5VWlShW9+uqrunLlSoa4n376Sa1atVKZMmXUsmVL/fnnnzb7Q0ND5e/vb/N69913C+oy4AB+fn7y8PDQmTNnbNrPnDmjkiVL2h2TmZEjRyomJsb6ioqKyvsLAAAAQLZkq2jboUMHvfrqq9k6Qffu3TV8+PBsHZNT+XWugrwGAABwazpw4ICaN2+usLAwrVu3Tt98842WL1+uoUOH2sR99dVX6ty5s7p06aLNmzfr7bff1sSJE21iYmJi9MEHH+jIkSPW17BhwwryclDA3NzcFB4ervXr19u0//rrr2rQoIHdMZnx9PSUr6+vzQsAAACOdUstj+Dh4SGLxeLoNAAAADKYP3++vL29NXnyZIWGhio8PFzvvPOOPvnkE+t6owkJCRo2bJief/55DRs2TGXKlFG9evU0f/78DP15e3vbzLT19PQs6EtCAXvqqaf03XffacWKFUpNTdWnn36qzZs3a8iQIdmKAQAAgPMr8AeRJScny83NzWatrStXrsjFxUXu7u6ZbhtjbJ6oe/12ui+//NKm/fp+MpOSkmJd+qBIkSK5uzgAAIAsJCUlycPDQ66u//dv5kWKFFFaWprWr1+v7t27a82aNTp//rwefvjhm/b3zDPPaOjQoQoNDdWDDz6owYMHZ1jLFIXHL7/8oh49eli3H3/8cQ0dOlT9+vWzLn3Rq1cvHTt2TL1799alS5cUEBCgTz/9VI0bN7YeZ08MAAAAnF+2i7YnT55U586d9euvvyolJUUDBgzQpEmT7P4joX79+nrwwQf13HPPWdseeughFStWTJ988omkq8swBAUFKSYmRhs2bFBSUpIeffRRjRgxQoMGDdKaNWtUrFgxjRw50qaf7t27q1y5cvrwww+t/YSEhOjSpUv6+eeflZKSol69eumjjz6yFnKfeeYZzZw58+rNcHdXgwYNNGXKFFWvXj27twYAACBLERERGjNmjN577z0NGzZMFy9e1JtvvilJio6OliTt379fRYoU0b///qvu3bvr3LlzqlGjhkaPHm1TdGvbtq2GDBmisLAw/fbbb3rqqad06NAhTZ482SHXhtxr2rSp9uzZk6Hdy8vLZvu5557TiBEjFBcXl+UyBvbEAADy18zfDjk6BQCFXLaXR/jss8/UunVrnThxQj///LPmzZuXYZ21vDB//nz1799fp0+f1pIlSzRt2jTVr19fgwYN0qVLl/TZZ5/phRde0M6dO2/Yz9y5c9W1a1edPHlS69ev1/z58zV79mzr/ilTpigxMVGJiYmKjo5WjRo11KVLl0wfCpKVpKQkxcbG2rwAAACuVa9ePX311VeaOnWqvL29FRYWpvbt28vb29sak5qaquTkZI0dO1Zz5szR77//rurVq+uee+7R3r17rXHffvutmjdvrpCQEPXt21fjxo3TBx98oLi4OEdcGvKAxWJRYGBghlfRokUzxLq4uNy0GGtPDAAAAJxXtou2DRs21FNPPSUvLy81aNBAL774Yr7M6rj//vvVrVs3WSwWtW7dWjVq1FCHDh3UuXNnWSwW3XvvvapQoUKGpylfr0uXLurVq5c8PDx0xx13KCIiQhs2bMg01svLS2+99ZYOHjyo3bt3253ruHHj5OfnZ32FhIRk61oBAMB/Q8+ePbV//37FxcXp4sWL6tq1qy5fvqwKFSpIkkqVKqW0tDSNHz9ederUUWhoqCZNmiQfHx8tWLAgy37r1q2rtLQ07d+/v4CuBAAAAEB+ynbRtk6dOjbb4eHhOnnypGJiYvIsKUkKCwuz2fb19VXFihUztF28eDFb/fj7+9scs337drVt21a+vr7y9vZWUFCQUlNTFRUVZXeuI0eOVExMjPWVnWMBAMB/j4eHh1xcXPTtt9+qaNGiatmypSSpUaNGkmTzUDEXFxd5eHhY1+DPTPrH6kuWLJmPWQMAAAAoKNku2hpjbLbT0tKuduRqX1eZPUAsvY+bxWXWlpPzpTPGKCIiQrfffrv279+vK1eu6PLly7JYLDf8w+h6np6e8vX1tXkBAABc79lnn9Xx48dljNGyZcv05ptvauzYsfLz85N09R+b77//fr355ps6d+6cUlNTNXnyZJ06dUqdOnWSJC1evFhTp07VuXPnlJaWpg0bNmjkyJG69957+bQPAAAAcIvIdtF248aNGbZDQkLk4+Nj1/HFixfXuXPnbNquXaOtIEVHRys6OlrDhg1TqVKl5OLios2bN2drPVsAAAB71a9fX40bN5aXl5eGDRumiRMn6umnn7aJ+fTTTxUYGKiyZcuqWLFimjVrlhYuXKg777xTktSiRQsdOXJENWrUkJeXl7p3766ePXvq66+/dsAVAQAAAMgP2S7abtmyRW+++aZOnTqlFStW6O2339aIESPsPr558+b68ssvtW3bNp06dUqjRo3Sjh07sptGnggODlZAQICmTZumc+fOadOmTRowYIBDcgEAALe+Xr166ciRI4qJidGhQ4c0cODADDF+fn768ssvdfnyZcXGxmrnzp269957bfa/8847OnnypOLi4hQdHa23335bxYoVK8hLAQAAAJCPslW09fDw0ODBg3XgwAHVq1dP/fv31+DBgzVs2LAbHmOxWKzbzz//vNq1a6cOHTqoSZMmSkhIUPfu3W1irj9GuroEgbu7+w3brj8us34sFos8PDwkSe7u7lqwYIHWrl2rsLAwPfTQQxo8eLACAgLk5uZ2w34AAABy6to1a7Pi6up60/FH+pgGAAAAwK3FxVy/SC1yLTY2Vn5+fgoZ/q1cPb0dnQ4AAHCAI+MjHJ3CLSt9rBUTE8OzBPIB9xdAntkwxdEZOMzMFMYBWRnYLOzmQcAtzN6xVraXRwAAAAAAAAAA5B+KtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIATcXd0AgAAAAAAwEltmOLoDADgP4mZtgAAAAAAAADgRJhpCwAAAAAAgAIx87dD2Yof2CwsnzIBnBszbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAibg7OgEAAAAA9rl8+bLWr1+v06dPq0yZMmratKksFotNTFpamtatW6fIyEhVrVpVDRo0cFC2AAAAyCmKtgAAAEAh8NdffykiIkLly5fX7bffrk2bNik5OVm//PKLKlasKEmKj49Xx44ddejQIdWvX1+//vqrWrdura+++kpubm4OvgIAAADYi+URAAAAgEJg1KhRuvPOO/X333/riy++0M6dO1WkSBFNmDDBGjNhwgQdPHhQW7du1YIFC/THH39oyZIl+vzzzx2YOQAAALKLoi0AAABQCBhjVKpUKeu2xWJRiRIlbGK+/vpr9e7dW4GBgZKkKlWqqH379vr6668LNFcAAADkDssjAAAAAIXAO++8o969e2vYsGGqVq2a/vjjD6WmpurVV1+VJCUmJurgwYOqXr26zXE1atTQrFmzsuw3KSlJSUlJ1u3Y2Nj8uQAAAADYjaItAAAAUAj4+vqqXLlyWrdunc6fP6+///5bd955p7y9vSVJcXFxMsaoePHiNseVKFHihoXYcePG6Y033sjX3AE40IYpjs4AAJADLI8AAAAAFALdu3eXr6+vtm7dqi+//FI7d+7U0aNHNWjQIEmSl5eXpKvF22vFxcVZ92Vm5MiRiomJsb6ioqLy7yIAAABgF4q2AAAAgJNLTk7W5s2bde+998rFxUWS5O7urvbt2+v333+XJBUrVkylSpXSkSNHbI49fPiwKleunGXfnp6e8vX1tXkBAADAsSjaAgAAAE7Ow8NDpUuX1o4dO2zat2/frtDQUOv2fffdp++++04pKSmSrq5Pu3TpUt13330Fmi8AAAByhzVtAQAAgEJg7NixeuKJJ3Tp0iXVrFlTGzZs0NKlS7Vs2TJrzOjRo1W/fn117NhRbdq00fz58xUUFKRhw4Y5MHMAAABkFzNtAQAAgBy6fimC/NS/f39t2rRJ5cqV0759+1S7dm3t2bNHbdq0scaUK1dO27ZtU6tWrXT48GH17dtXf/75p3x8fAosTwAAAOQeM20BAACAHGrUqJHCwsL06KOPqmfPnipWrFi+nq927dqqXbv2DWOCgoL00ksv5WseAAAAyF/MtAUAAABy6KefflJ4eLhefPFFBQcHq3///lq/fr2j0wIAAEAhR9EWAAAAyKHatWvrgw8+0PHjxzV79mydOHFCzZs312233abx48frxIkTjk4RAAAAhRBFWwAAACCXPDw81KNHD/3444/69NNPFR0drZEjRyokJER9+vRRVFSUo1MEAABAIULRFgAAAMilqKgojRkzRpUrV9YTTzyhrl27at26dVq9erUuXLigVq1aOTpFAAAAFCI8iAwAAADIoQULFujjjz/W6tWrVbt2bY0YMUJ9+/aVn5+fNaZp06YqXry4zp49q8DAQAdmCwAAgMKCoi0AAACQQy+//LJatWqlv/76S+Hh4ZnGuLm5acKECfL29i7g7AAAAFBYUbQFAAAAcuiff/6Rh4fHTeMGDRpUANkAAADgVsGatgAAAEAOWSwWrV27VsYYm/aNGzfq0qVLDsoKAAAAhR1FWwAAACCHpkyZou+//14uLi427bt379bw4cMdkxQAAAAKPYq2AAAAQA7NmTNHjz76aIb2Pn36aMGCBUpJSXFAVgAAACjsKNoCAAAAOXTp0iVduXIlQ7sxRnFxcUpISHBAVgAAACjsKNoCAAAAOdSkSRNNmDAhw4za//3vf6pevbp8fHwclBkAAAAKM3dHJwAAAAAUVqNHj1bDhg1VrVo1tWjRQp6entq4caN27typ5cuXOzo9AAAKvZm/HcpW/MBmYfmUCVCwmGkLAAAA5FDFihW1detWde3aVQcOHND27dtVt25dbd68Wa1atXJ0egAAACikmGkLAAAA5EKZMmU0YcIER6cBAACAWwhFWwAAACCX4uPjdfbsWRljbNpDQkLk5ubmoKwAAABQWFG0BQAAAHLo4MGD6tu3rzZu3Jjp/hMnTig4OLiAswIAAEBhR9EWAAAAyKFnnnlGJUqU0O+//66goKAM+0uWLOmArAAAAFDYUbTNRzvfaCdfX19HpwEAAIB8smvXLi1btkzVqlVzdCoAAAC4hbg6OgEAAACgsKpQoYJiYmIcnQYAAABuMRRtAQAAgBx67rnnNGLECO3fv9/RqQAAAOAWwvIIAAAAQA4NGjRIR48eVdWqVWWxWOTqajsn4ujRoypVqpSDsgMAAEBhRdEWAAAAyKFPPvlEiYmJWe739/cvuGQAAABwy6BoCwAAAOTQPffc4+gUAAAAcAtiTVsAAAAgl/bu3atly5YpKSlJkpSamurgjAAAAFCYUbQFAAAAciguLk7t2rVTtWrVdO+99+rChQuSpMcee0xz5851cHYAAAAorCjaAgAAADk0evRoubi46PTp0zYPHBs5cqTGjh3rwMwAAABQmLGmLQAAAJBDq1ev1ty5c1WyZEm5uLhY26tUqaKoqCjFxMTIz8/PgRkCAACgMGKmLQAAAJBD8fHx8vT0lCSbom18fLySkpLk4eHhqNQAAABQiFG0BQAAAHKocePG+vzzzyX9X9E2LS1Nr7zyisLDw+Xl5eXI9AAAAFBIsTwCAAAAkENvvvmmGjVqpI0bNyomJkYvvPCCNm/erCNHjmjVqlWOTg8AAACFFDNtAQAAgByqWLGitm/froYNG6p+/fo6ePCgmjZtqs2bN6tJkyaOTg8AAACFFDNtAQAAgFwoVaqUxo4d6+g0AAAAcAuhaAsAAADk0OXLl5WWlpbl/qJFi9o8oAwAAACwB0VbAAAAIIfCwsJ06tSpLPefOHFCwcHBBZgRgFvShimOzgAAUMAo2gIAAAA5tHz5ciUnJ9u0HTlyRGPGjFG3bt0UEBDgoMwAAABQmFG0BQAAAHKoTp06GdoaNmyoBg0aqE2bNnrjjTcckBUAAAAKO1dHJwAAAADcaipWrKjTp08rPj7e0akAAACgEKJoCwAAAOSxr776SpLk7e3t4EwAAABQGLE8AgAAAJBDderU0enTp23a4uLidPnyZb3//vtycXFxUGYAAAAozCjaAgAAADn04osvKiEhwabN19dXderUUYUKFRyTFAAAAAo9irYAAABADvXs2dPRKQAAAOAWRNEWAAAAyKHLly8rLS3NrtiiRYuyXAIAAADswoPIAAAAgBwKCwuTj4+PXa9Tp07l+nzx8fF666231KRJEzVt2lQff/yxjDE2Mb/99pvuu+8+1a5dW927d9c///yT6/MCAACgYFG0BQAAAHJo0qRJ8vHx0WuvvaYVK1ZozZo1mjx5ssqWLatXX31VmzZtsr4CAwNzda7ExES1aNFCixcv1qhRo/Tee+9px44d+v77760xf/31l9q0aaM777xT06dPl7+/v5o2baqjR4/m9lIBAABQgFzM9f80j1yLjY2Vn5+fYmJi5Ovr6+h0AAAAbinONNbq1KmTevfurd69e9u07969Wx06dMjTYulbb72lSZMm6cCBAypRooS1PSUlRe7u7tZ8kpKStGLFCkmSMUZVq1ZVx44d9f7779t1Hme6vwD+vw1THJ0BsmlmSoSjU/jPGtgszNEpADdk71iLmbYAAABADu3atUt16tTJ0H777bfr7NmziouLy7Nzff3113rggQdsCraSrAVbY4zWrl2r9u3bW/e5uLioQ4cOWrNmTZ7lAQAAgPxH0RYAAADIoTJlyuiTTz7J0P7VV1/Jw8ND3t7eeXauvXv3qmrVqnr66ad11113qV27dpo1a5b1QWgxMTGKi4tT6dKlbY4LDg7WsWPHsuw3KSlJsbGxNi8AAAA4lrujEwAAAAAKq//9739q166dVq9erSZNmshisWj79u369ddfNXPmTLm5ueXJeVJTU5Wamqo333xTL730kj755BPt3LlTTz31lI4dO6bRo0crNTVVkuTh4WFzrKenp1JSUrLse9y4cXrjjTfyJE8AAADkDWbaAgAAADnUtGlT7du3T23atNGBAwe0Y8cO3Xbbbdq6daseffTRPDuPm5ubihcvriZNmujVV19VeHi4HnnkET399NP6+OOPJUl+fn5yd3fX2bNnbY49e/asAgICsux75MiRiomJsb6ioqLyLG8AAADkDDNtAQAAgFwoV66c3n777Xw/T7169TKsZxsQEKBLly5Jurq27Z133qk///xTjz/+uDVmw4YNqlu3bpb9enp6ytPTM3+SBgAAQI5QtAUAAAByae/evTpw4IDuueceeXp6KjU1Nc+WRkg3ZMgQPfjgg9qxY4fuuOMOnTlzRp988ok6duxojXniiSf0zDPPaPDgwapbt66WLl2q9evXa8WKFXmaCwAAzmrmb4eyFT+wWVg+ZQLkDkVbAAAAIIfi4uLUrVs3rVy5UpJ04sQJBQcH67HHHlOrVq300EMP5dm5OnXqpNdff13NmjWTr6+vzpw5o4iICE2dOtUa89hjj+nAgQO6++675efnp8uXL2vy5Mm655578iwPAAAA5D8XY4xxdBK3mtjYWPn5+Slk+Ldy9cy7JwYDAAA4oyPjIwr0fOljrZiYGPn6+hboua/37LPPavfu3Zo7d65q1aqlbdu2KTg4WPv27VOnTp20Z8+ePD9nUlKSjh8/rtKlS6tIkSKZxsTHx+vUqVMqU6ZMljFZcab7C9wyNkxxdAYoYDNTCvZ3I3KOmbYoaPaOtZhpCwAAAOTQ6tWrNXfuXJUsWVIuLi7W9ipVqigqKkoxMTHy8/PL03N6enqqYsWKN4wpWrSowsL4IxQAAKCwcnV0AgAAAEBhFR8fb32I17VF2/j4eCUlJcnDw8NRqQEAAKAQo2gLAAAA5FDjxo31+eefS/q/om1aWppeeeUVhYeHy8vLy5HpAQAAoJBieQQAAAAgh9588001atRIGzduVExMjF544QVt3rxZR44c0apVqxydHgAAAAopZtoCAAAAOVSxYkVt375dDRs2VP369XXw4EE1bdpUmzdvVpMmTRydHgAAAAopZtoCAAAAOfTuu+/qySef1NixYx2dCgAAAG4hzLQFAAAAcujDDz/UhQsXHJ0GAAAAbjEUbQEAAIAcuuuuu7R+/XpHpwEAAIBbDMsjAAAAADnUqVMnDR48WHv37tWdd96pIkWK2Oxv1aqVPDw8HJQdAAAACiuKtgAAAEAOvfbaa4qLi9Nbb72V6f7o6GiVKlWqgLMCAABAYUfRFgAAAMihY8eOOToFAAAA3IJY0xYAAADIpuDgYJ08edK6PWrUKMXFxTkwIwAAANxKKNoCAAAAufTxxx8rPj7e0WkAAADgFsHyCAAAAAAAAPhPmvnboWzFD2wWlk+ZALaYaQsAAAAAAAAAToSZtgAAAEAOTJgwQUWLFpUkXbp0yWY73UsvvaRixYo5Ij0AAAAUYhRtAQAAgGyqWLGilixZYt0ODg622U43fPhwirYAAADINoq2AAAAQDb98ccfjk4BAAAAtzDWtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ+Lu6AQAAAAAAHB6G6Y4OgMAwH8IM20BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiPIgMAAAAAAAAsMPM3w5lK35gs7B8ygS3OmbaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIATcXd0AgAAAAAA2GXDlJwf23hY3uUBAEA+Y6YtAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAUMjExMRo0aJF2rx5c6b7d+3apR9//FEHDx4s4MwAAACQFyjaAgAAAIXMgAED1KNHD7377rs27cnJybr//vt19913a/z48apdu7aGDBkiY4yDMgUAAEBOuDs6AQAAAAD2mzZtms6ePasWLVpk2Ddx4kRt2LBB//zzj8qVK6dt27apQYMGatasmXr27FnwyQIAACBHmGkLAAAAFBI7d+7UmDFj9Pnnn8vVNeNQfu7cuerdu7fKlSsnSbrzzjvVpk0bff755wWdKgAAAHKBoi0AAABQCFy+fFk9e/bUpEmTVL58+Qz7k5KStHfvXtWuXdum/c4779SOHTuy7DcpKUmxsbE2LwAAADgWRVsAAACgEHjqqacUHh6uXr16Zbo/Li5OaWlpKl68uE17QECALl68mGW/48aNk5+fn/UVEhKSl2kDAAAgByjaAgAAAE7ut99+0xdffKF77rlHixYt0qJFi3T69GlFR0dr0aJFSkxMlIeHh6SrM3KvdenSJXl6embZ98iRIxUTE2N9RUVF5eu1AAAA4OZ4EBkAAADg5CwWi9q3b6+FCxda26Kjo2WxWPTpp5+qadOmCggIUEBAQIaia1RUlCpWrJhl356enjcs6gIAAKDgUbQFAAAAnFyjRo20aNEim7b27dvL399f33zzjbWtQ4cO+v777/Xiiy/KxcVFiYmJWrp0qR577LECzhgAAAC5wfIIAAAAwC1i9OjR2rdvn3r16qU5c+bo3nvvlYeHh4YPH+7o1AAAAJANFG0BAACAQqhx48aqV6+eTVvlypW1ZcsWhYSE6Mcff1SjRo30119/qUSJEg7KEgAAADnB8ggAAABAITRq1KhM2ytWrKh33323gLMBAABAXmKmLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBNxd3QCAAAAAAAAwK1o5m+HshU/sFlYPmWCwoaiLQAAAPJVQkKCXF1d5enpmWVMSkqK3N0zDk2Tk5OVnJxs03bp0qWbxqQrVqxYDjIGAAAAHCvfl0eIjo7WqVOn8vs0+XqugrwGAACAW0FqaqoWLlyoNm3aqGjRonrkkUcyjfvrr79Up04dFSlSRH5+fnr22WeVmppq3f/GG28oODjY5hUaGipJSktLyzKmePHiCgoKssYAAAAAhUm+F22HDBmiMWPG5Pdp8vVcBXkNAAAAt4L9+/dr7ty5eu6559ShQ4dMY86ePav27durWbNmiomJ0S+//KIvvvhCb7zxhjVm7NixunTpkvUVFxenkJAQSZKrq2uWMaGhoerevbs1BgAAAChMbqlRbLly5RQcHOzoNAAAAP7zqlWrpu+//17t2rWTi4tLpjGfffaZ0tLS9Pbbb6to0aIKDw/X008/rQ8//FApKSmZHrN27VodPnz4hudeu3atDh48qIEDB+b6OgAAAABHyFbR9tSpU9qzZ4/27NmjM2fO5OiER44c0fnz523ajh8/rpMnT1q3o6KirP1fuHAhw9IEx44dU2xsbIa+R44caTM4v76fc+fO5cs1AQAAIPv++OMPNWjQQB4eHta2li1b6sKFC9q7d2+mx8yaNUtVq1a9Yb+zZs1StWrVdPfdd+dpvgAAAEBBydaDyKZNm6ZvvvlGknT69GkVL15cc+bMUbNmzezu4/7779eDDz6o5557ztr27LPPqlixYvrkk08kSf3795enp6eOHj2qixcv6vTp02rbtq1ee+019e/fX7GxsTp9+rSefPJJffDBB9Z+hgwZonLlyunDDz+09uPr66tjx47p+PHjOnfunJo2bapFixbJ29s7z64JAAAA2Xf69GmVL1/epq1kyZLWfTVq1LDZd/HiRS1YsECvvfaaXnnllUz7TI8ZO3Zs/iQNAACQj2b+dsju2IHNwvIxEzhatmbavv7669ZZqefOndOgQYPUu3dvJSQk5Hlia9eu1Zw5c3Ts2DHt2LFDK1euVIcOHfTVV1/p2LFj2rhxo6ZOnaqNGzfesJ+ffvpJ77//vo4dO6YjR45o165dmj59ep5eU1JSkmJjY21eAAAAuLnrHxSWvp3Zkgpffvml0tLS1Lt37yz7S495+OGH8zZRAAAAoABla6ZtuvPnz1sfHPHSSy9p9+7dCg8Pz9PEunbtqnr16km6uiZajRo1VK9ePd15552SpLvuukthYWHaunWrGjRocMN+GjVqJEkqVaqU2rRpo61bt+bpNY0bN87mgRkAAAC4uTJlyuj06dM2benbpUuXzhA/a9YsdenSRQEBAVn2mR4TGBiYt8kCKPw2THF0BgAA2C1bM21Xr16tqlWrKjQ0VB06dFC3bt2Ulpam6OjoPE+sTJkyNttFixbNtO3SpUs37Kds2bI228WKFVNcXJx1Oy+uaeTIkYqJibG+oqKi7D4WAADgv+ruu+/WH3/8YfMJp1WrVikoKEhVqlSxid26dau2bt16w4eL2RMDAAAAFAZ2F21TU1PVvXt3DRgwQBcvXtTBgwf1zz//yNXVNcPH2m4ks4+6ZfV04PyWV9fk6ekpX19fmxcAAMB/XXx8vC5duqTU1FSlpKTo0qVLunz5snX/ww8/LH9/fw0cOFCHDx/W8uXL9d577+mFF16Qq6vtMHXWrFkKCwtTq1atsjyfPTEAAABAYWD38ggnTpzQxYsX1bVrV7m5uUmSfv31V6WmpmbrhCVLltSJEyes22lpadqxY4dDHvyVV9cEAACAjEJDQ5WYmGjdDg4OVqlSpXTw4EFJkq+vr37++Wc988wzqlevnooXL65XX31VI0aMsOknOTlZP/zwg4YOHZrpBAB7YwAAAIDCwu6ibZkyZRQaGqpRo0ZpxIgROnTokEaMGJHtQXHHjh01ZswYNW3aVKVLl9b06dN1+PBhhxRt8+qaAAAAkNHZs2dvGlOtWjX9+OOPN4zx8PBQZGRkrmMAAACAwsLuoq2rq6uWL1+u1157TY8++qhKly6tadOmadSoUfLx8cnyuHLlyik4ONi6PXToUMXHx2vChAkqWrSounTpIn9/fxUrVswaU758eQUFBdn0ExoamuGBEhUrVrR5EMX158qsn2v323tN1/cLAAAAAAAAAPnFxRhjHJ3ErSY2NlZ+fn4KGf6tXD29HZ0OAABAvjoyPqJAz5c+1oqJieFZAvmA+wuntmGKozMA7DIzpWB/N+K/aWCzMEengBywd6xl94PIAAAAAAAAAAD5j6ItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4ETcHZ0AAAAAAOA/YsMUR2cAAEChwExbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAi7o5OAAAAAMDNxcfHa9asWfr111+Vmpqq+vXra+jQofL19bWJ2759u9577z1FRkaqatWqevHFF1WhQgXHJA0AAIAcYaYtAAAAUAg0btxYhw8fVt++ffXwww/r+++/V5MmTXT58mVrzD///KPGjRurSJEiGj58uE6fPq2GDRvq5MmTDswcAAAA2cVMWwAAAKAQWLdunc2s2saNG6t06dJasWKFunTpIkl68803FR4ermnTpkmSOnTooEqVKmnSpEl6++23HZI3AADIHzN/O5St+IHNwvIpE+QHZtoCAAAAhcD1yyAUK1ZMrq6uSkxMtLatXr1anTp1sm67u7srIiJCq1atKrA8AQAAkHvMtAUAAAAKobffflve3t5q1aqVJCkmJkYXL15U2bJlbeLKli2ryMjILPtJSkpSUlKSdTs2NjZ/EgYAAIDdmGkLAAAAFDLfffed/ve//2nGjBkqVaqUJOnKlSuSpCJFitjEenl5KTk5Ocu+xo0bJz8/P+srJCQk/xIHAACAXSjaAgAAAIXIDz/8oL59++rDDz9Unz59rO1+fn5yc3PTuXPnbOLPnTunEiVKZNnfyJEjFRMTY31FRUXlW+4AAACwD0VbAAAAoJBYsmSJevToocmTJ+vJJ5+02WexWFSjRg39/fffNu1//fWX7rrrriz79PT0lK+vr80LAAAAjkXRFgAAACgEli1bpu7du2vSpEkaPHhwpjGPPvqovvnmG+3bt0+StH79eq1du1aPPvpoQaYKAACAXOJBZAAAAEAh0KNHD1ksFn311Vf66quvrO2PP/64Hn74YUnS0KFD9c8//6h27doKCwvTwYMHNXLkSHXq1MlRaQMAACAHKNoCAAAAhcDKlStljMnQHhoaav1/Nzc3ffLJJxozZoyioqJUqVIlBQQEFGSa+C/YMMXRGQAAcMujaAsAAAAUAk2aNLE7tnTp0ipdunQ+ZgMAAID8xJq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4ER5EBgAAAAAAANziZv52KFvxA5uF5VMmsAczbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCI8iAwAAAAACpsNU3J+bONheZcHAADIF8y0BQAAAAAAAAAnQtEWAAAAAAAAAJwIyyMAAAAAwH9JbpZWAAAABYKZtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBEeRAYAAAAABY2HgQEAgBtgpi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE7E3dEJAAAAAEChtGGKozMAAAC3KGbaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRHgQWT7a+UY7+fr6OjoNAAAAAAAAIF/N/O1QtuIHNgvLp0xuDcy0BQAAAAAAAAAnQtEWAAAAAAAAAJwIyyMAAAAA+G/aMMXRGQAA4LSyu9wB8hYzbQEAAAAAAADAiVC0BQAAAAAAAAAnwvIIAAAAAAAAAApUfi+/MLBZWL72n9+YaQsAAAAAAAAAToSZtgAAAAAKLx4mBgAAbkHMtAUAAAAAAAAAJ0LRFgAAAAAAAACcCMsjAAAAALeYkydPKioqSmFhYQoICHB0OgAAAAUuuw86c7YHlzHTFgAAALhFpKam6rHHHlPFihXVr18/lS1bVqNHj3Z0WgAAAMgmZtoCAAAAt4gPP/xQCxYs0Pbt21W1alWtX79eLVu2VHh4uDp16uTo9AAAAGAnirYAAADALWL27Nnq1auXqlatKkm6++671aJFC82ePdt5i7Ybpjg6AwAAAKdD0TYfGGMkSbGxsQ7OBAAA4NaTPsZKH3PhqitXrmjXrl0aNmyYTXv9+vX1xRdfZHlcUlKSkpKSrNsxMTGSCnAsG59QMOcBgAKUkBLn6BQAZFNBjX3sHctStM0H586dkySFhIQ4OBMAAIBbV1xcnPz8/BydhtOIiYlRampqhgePBQQE6Pz581keN27cOL3xxhsZ2hnLAkBuvOjoBABk09MFfL6bjWUp2uaDEiVKSJIiIyP5Q8KJxcbGKiQkRFFRUfL19XV0OrgB3qvCg/eq8OC9Khx4nzJnjFFcXJzKlCnj6FScisVikSQlJibatCckJMjDwyPL40aOHKlnn33Wup2Wlqbz588rICBALi4u+ZPs/8fXeM5w37KPe5Yz3Lec4b7lDPct+7hnOePo+2bvWJaibT5wdXWVJPn5+fFNUwj4+vryPhUSvFeFB+9V4cF7VTjwPmXEP4xn5OfnJ39/f0VHR9u0R0dHq3z58lke5+npKU9PT5s2f3///EgxS3yN5wz3Lfu4ZznDfcsZ7lvOcN+yj3uWM468b/aMZV0LIA8AAAAABeCee+7RkiVLrNspKSlatmyZ2rRp48CsAAAAkF0UbQEAAIBbxKhRo7R582YNGjRIS5YsUa9evZSYmGiz/AEAAACcH0XbfODp6anRo0dn+JgZnAvvU+HBe1V48F4VHrxXhQPvE7KrVq1a2rBhgxISEjR58mSVLFlSf/75p4KDgx2dWqb4Gs8Z7lv2cc9yhvuWM9y3nOG+ZR/3LGcKy31zMcYYRycBAAAAAAAAALiKmbYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOxN3RCdxqYmNjtW/fPgUFBal8+fKOTuc/KS0tTfv375eLi4sqVqwoi8WSadyxY8d08uRJVa5cWf7+/jmOQe4dPXpUUVFRqlGjhooXL26zzxijf//9V8nJyapZs6bc3TP+2LInBrl3/PhxnTp1StWrV890wfbLly/r33//lb+/vypVqpRpH/bEIHeioqJ06tQplS5dWmXLls005tSpU4qMjFTFihUVGBiY4xjYLykpSX///beCg4MVFhaWaYw9Y4i8igEc4d9//9X58+fVpEkTu+IZA2Tv92b6eCozTZo0kYuLiy5fvqwtW7Zk2F+rVi35+fnlSc7O4OTJk4qKilJYWJgCAgJuGBsVFaWjR4/atLm7u6thw4a56rcw2rNnjxISElSjRg15eHjcND4xMVF79+5ViRIlVK5cObm4uNjs37Ztmy5dumTTFhwcrMqVK+dp3gUhMTFRu3fvlo+Pj6pUqZJnx+Sk38Lk9OnTOnr0qEJDQxUUFHTTeGOMDh8+rISEBFWqVElFihSx2X/8+HEdOnTIps3FxcXu3yuFxf79+xUXF6fq1atnuAfX++effxQTE2PTFhgYqGrVquWq38ImOTlZu3btkpeXV6bXfq0TJ07o4MGDme6rW7euihQpotTUVP3xxx8Z9t92220qWbJknuRsF4M8M3XqVOPl5WWqVatmvL29TefOnc3ly5cdndZ/yvjx403p0qVNtWrVTMWKFU3p0qXNwoULbWKSkpJMz549jZeXl7n99ttNkSJFzDvvvJPtGOSNM2fOmJCQECPJLFmyxGbf/v37TfXq1U3JkiVN+fLlTenSpc369euzHYPcOXXqlGnXrp3x8fExdevWNRUqVDCLFi2yifnmm2+Mr6+vqVKlivHx8THNmzc358+fz3YMci4qKsrUr1/fFC9e3ISHhxtfX1/TvHlzc+bMGWtMWlqaGTx4sPH09DTVq1c3np6e5sUXX7Tpx54Y2O/cuXPm+eefN2XKlDHe3t5myJAhmcbZM4bIqxigoH3zzTemQYMGpnjx4sbNzc2uYxgDZP/35rRp00yTJk1sXiVKlDD+/v4mJSXFGGPMP//8YySZ+vXr28Rt2bKloC4rX6WmpprHH3/c5nfYyy+/fMNjxowZY3x8fGzuR/v27XPdb2ESGRlpateubQICAkzFihVNyZIlzerVq7OMP3/+vHnyySeNv7+/ueOOO0xgYKCpU6eO2bVrl01c7dq1TYUKFWzu7fjx4/P7cvLc4sWLTfHixU3lypWNv7+/adiwoTl16lSuj8lJv4XJ8OHDbb5nhg8ffsP42bNnm4oVK5qKFSua22+/3fj7+5tp06bZxEyePNl4e3vbfE01b948H6+iYJ06dco0bNjQ+Pv7m8qVK5vixYubxYsX3/CY5s2bm5CQEJt78uqrr+a638Jk9erVpmTJkqZixYomICDA1K5d20RGRmYZv2DBggy/L0uXLm3c3d3NuXPnjDHGXLhwwUgytWvXtolbsWJFQV2WMcYYirZ5ZNOmTcbFxcVaIDxx4oQpV66cef755x2c2X/LK6+8Yk6fPm3dHjdunPH09DRHjx61tr3++usmODjY+k28fPly4+LiYn799ddsxSD30tLSTEREhHnxxRczLdrWq1fPdOzY0fqHxpAhQ0xwcLCJj4/PVgxyLiUlxdStW9e0aNHCxMTEGGOu/gL74osvrDGHDh0yHh4e5qOPPjLGGBMTE2OqV69uHnrooWzFIHf69OljatWqZf3av3DhgqlYsaJNkXD69OnGx8fH7Ny50xhjzMaNG42Hh4eZN29etmJgvx07dpjx48eb06dPmwYNGmRatLVnDJFXMYAjjB492mzYsMHMmTPH7qLtf30MkBe/N69cuWJKlSplhg0bZm1LL9qeOHEiz3N2Bh9++KHx8/Mz//77rzHGmA0bNhiLxWIWLFiQ5TFjxowxDRo0yPN+C5MWLVqYFi1amKSkJGOMMS+++KIpXry4uXDhQqbxu3btMtOmTbPGJyYmmk6dOpnbb7/dJq527dqFfuLL8ePHjbe3t3n77beNMcbEx8ebOnXqmC5duuTqmJz0W5h8+umnxtvb22zbts0YY8yWLVuMl5eX+eyzz7I8ZuzYsebIkSPW7a+//tq4uLiYP/74w9o2efJkU6NGjfxL3MHuv/9+U7duXevvsXHjxhlvb+8b/sxu3ry5eeWVV/K838LiwoULpnjx4tZ/SEtKSjJNmzY1LVu2zFY/d9xxh833X3rRdtOmTXmab3ZRtM0jgwcPNjVr1rRpe/31101gYKBJS0tzUFa4ePGikWQzoCpfvrx56aWXbOLq1q1rHnnkkWzFIPcmTpxoWrZsaU6fPp2haLtjxw4jyWbGzPHjx42rq6uZP3++3THInQULFhhJZu/evVnGvPnmmyYoKMikpqZa26ZPn248PT3NpUuX7I5B7rRp08Y8+OCDNm0RERGmR48e1u369eubfv362cTce++9pl27dtmKQc5kVbS1ZwyRVzGAI9lbtGUMkDe/N7///nsjyezYscPall60/f33383WrVtNXFxcnufuSHXq1DGPPfaYTVv79u1NRERElseMGTPGhIeHm23btpk9e/aYK1eu5Em/hcWhQ4eMJPPTTz9Z2y5cuGAsFouZM2eO3f0sWrTISLLOUjPmatH2lVdeMX/99ZeJjo7Oy7QLzKRJk4yvr6+1QG2MMV988YVxc3MzZ8+ezfExOem3MGnWrJnp1auXTVu3bt2yPSvW39/fTJw40bo9efJkU61aNbN9+3aze/duk5ycnBfpOoUzZ84YV1dX880331jbEhISjI+Pj5k8eXKWxzVv3twMGzbM/PXXXyYqKirP+i0sZs+ebTw8PKwTjIwxZunSpUaSOXz4sF19bNq0yUgyy5cvt7alF20XLlxoNm/enOU/YuU3HkSWR7Zu3arw8HCbtvr16+vs2bM6duyYg7LCpk2bJMm6btL58+cVGRmZ6Xu1detWu2OQe3///bfeeecdff755xnWv5JkvdfXvg+lS5dWuXLlrPvsiUHu/Pzzz7rttttUtWpV7d6927pu4LW2bt2qu+66S66u//crpX79+kpKStLu3bvtjkHuvPzyy1q1apXef/99/fzzzxo/frz+/vtvvfjii5KurhG2ffv2G/5ssycGec+eMURexQCFAWOAvPm9OWvWLDVo0EC1atXKsK9Hjx7q06ePSpQooSFDhmT43V4Ypaam6p9//snR77CtW7eqT58+at26tYKDgzV37tw86bcwyOx7yd/fX1WqVMnW9W3atEklSpTI8HyK9957TwMHDlS1atVUv379Qjfu27p1q2rVqmWzxm/9+vWVmpqqHTt25PiYnPRbmGQ1JsnO19T+/fsVExOTYQ3kvXv3qlevXmrXrp1KliypmTNn5knOjrZjxw6lpaXZ3LciRYqoVq1aN71vM2fO1MCBA1WzZk3dcccdNmuX56bfwmDr1q2qUqWKfH19rW3169e37rPHrFmzFBISonbt2mXY98QTT6hfv34KCgpSnz59MqwfnN9urZX6Hej8+fMZFqNP3z5//rxCQkIckdZ/2oULFzRo0CB16tRJd9xxh6Sr74WkTN+r9H32xCB34uLi1KtXL33wwQcqV66czp49myHm/Pnz8vb2zrBA+vXv1c1ikDvHjx+Xr6+vmjVrprNnzyoxMVHx8fGaMWOG7r//fklX34frH3h17c8/e2OQO/Xq1VPnzp31+uuvq2LFijp06JAeffRR1ahRQ5IUHx+vpKSkG/5ssycGec+eMURexQB5YdOmTUpKSspyf5EiRVS3bt0c938rjgEuXbqkbdu23TCmbNmyqlixoqTc/948fvy4fvzxR82YMcOm3dfXVz/++KPat28v6eoftK1atZK/v7/Gjh1r7+UUmMjISEVGRt4wJv0hanFxcbpy5Uq2f4fVq1dPhw4dUmhoqKSrRcZ+/fqpSpUqatiwYY77daQtW7bo8uXLWe63WCxq0KCBpP/7eipRooRNTHaub/PmzZo4caL+97//2UzGeOaZZ9SzZ08VKVJEsbGx6tmzp7p06aIdO3Zk+lBbZ3Sz3605PSYn/RYWKSkpiouLy/T6YmNjlZqaKjc3txv2kZycrH79+umuu+5Sx44dre133HGH9u3bZy3kzpw5U0888YQqVaqkVq1a5f3FFKCc1iEGDhyoZcuWqWjRorp8+bIeeeQRde7c2fqAu1u9vpHZ91L6zzN7ri8hIUFff/21nnnmGZt/KLVYLPryyy/Vp08fSdKBAwfUunVrPfXUU/rss8/y8ApujKJtHrFYLEpMTLRpS0hIkCS7nryJvHXp0iVFRETIz89Pn3/+ubXdYrFIUqbvVfr7ZE8McueVV15RyZIlVbp0aa1fv976r1W7d+9WaGioatWqJYvFoqSkJBljbAZ/179XN4tB7lgsFm3atEmzZ89W//79JUmjR4/Wgw8+qEOHDikoKMiun3/8jMx/Dz74oE6ePKnIyEj5+PjowoULatasmWJjY/XJJ5/w88+J5dX3EN9nKCjjxo3T6dOns9xfunRpzZ8/P8f934pjgOjoaL300ks3jOnRo4eeeuopSbn/fv7ss8/k7e2tnj172rSXL19e5cuXt27fddddeuKJJ/TNN984ZdF2xYoVN/3jeMqUKbrrrrty/Dvs+plVw4cP1/Tp0zV//nw1bNiwUP5unDhxoo4ePZrlfn9/fy1dulTS//3uT0pKkpeXlzXG3uv7999/FRERoQcffFDDhw+32ffII49Y/9/X11fvvPOOatWqpS1btqhRo0bZuSSHsVgsGQrgN/tetOeYnPRbWLi5ucnV1TXT7xlXV9ebFmxTUlLUq1cvHT9+XL/99pvc3f+vbHV9YXbgwIGaMWOG5s2bV+iLtjf6WXPtLNLr9e3b1/r/3t7emjx5skJCQrR+/Xp16NAhx/0WFpn9vkzftud76bvvvlNcXJweffRRm/aiRYtaC7bS1U9vP//883rhhRc0Z84cmwJvfqJom0dCQ0MVHR1t0xYdHS0XFxdmthSwS5cuqUOHDkpMTNTq1avl5+dn3VemTBlZLJZM36v0Aaw9McidYsWKydXV1frHS0pKiqSrf2BERUVpypQpCg0NVWpqqk6dOqXg4GBJUlpamk6ePGl9H+yJQe5UqFBBRYoUUb9+/axtTz75pN58801t2bJF7du3V2hoqHbu3GlzXPr3z7Xv1c1ikHPGGC1btkzvv/++fHx8JEnFixdXnz59NHnyZEmSp6enSpUqdcOfbfbEIO/ZM4bIqxggLyxcuDBf+78VxwC33Xab1q9fb3d8bn9vzp49W71791axYsVuGpvZz31nMXDgQA0cONCu2KJFiyogICBPfodde0/yst+C8uWXX9odmz7DODo62uZj6MePH9e99957w2P37NmjVq1aKSIiQh9//HGmS55dq1SpUtZzFRahoaFavXq1TdvNvhftOSYn/RYW6eOOnHzPpKSkqHfv3tqyZYvWrl1r1/jFmX+GZce134ulS5e2tkdHR6tmzZp29xMUFCQXFxfrPcmrfp1VaGioVq1aZdOWne+lWbNmqX379nZ/rSUkJOj8+fMKDAzMWcLZxJq2eaRNmzb65ZdfFB8fb21bvHixGjZsaNdgCXkjPj5eHTt2VHx8vFavXp3hYz4Wi0XNmzfXDz/8YG1LTEzUihUr1KZNG7tjkDv/+9//tH79eusr/V/6J0yYoClTpkiSmjZtKk9PT5v34ddff9XFixet74M9Mciddu3aKSkpSWfOnLG2pa+NWbJkSUlXf/5t3rxZJ06csMYsXrxYFStWVKVKleyOQc65uLgoMDAww7qlUVFR1vdJuvo+LFmyxLqdmpqqpUuX2ny/2BODvGXPGCKvYgBntXnzZh0+fFgSYwDJvt+b8fHxNp9YSrd27VodOHAg02LntT8f0q1ateqW+MNdyvx32LJly2y+JiIjI/Xnn39at6+/J2fOnNG2bdts7ok9/RZWDRo0kI+Pj8330qZNm3T8+HGb69u6dasOHDhg3d67d69atmyp9u3b65NPPslQsL18+bKMMTZtK1eulCTr0k2FQZs2bbRr1y4dPHjQ2rZ48WIFBwdb14tOTEzU+vXrdeHCBbuPsSemMGvTpo2WLl1q/RowxmjJkiU2X1PR0dH6/fffrdupqanq06ePNm3apLVr16pChQoZ+r3++zUmJkZ//fXXLfEzrFatWipVqpTN9+K+ffv077//2ty3f/75R3v27JF09WsvNTXVpp9Vq1bJGGO9J/b2W1i1adNG0dHRNuv4Ll68WL6+vtZlYFJSUrR+/Xqbv2mlq0se/Pbbb3b/vly5cqWCg4MzLMeQrxzy+LNb0KVLl0yVKlVM69atzaJFi8zIkSONu7u7+eWXXxyd2n9GSkqKadGihQkICDCLFy8269ats76OHz9ujfvjjz+Mh4eHGTFihFm8eLHp0KGDKV++vM3TAO2JQd45c+aMkWSWLFli0/7GG28YX19f8/HHH5uvvvrKlC9f3vTp0yfbMcidjh07miZNmpjFixebefPmmerVq5s2bdpYn0afkpJi6tWrZ+rVq2cWLlxoxo0bZ9zd3c28efOsfdgTg9wZN26c8fLyMm+//bZZuXKleeutt4zFYjHTpk2zxuzZs8f4+PiYAQMGmB9++MH06tXLBAQEmMjIyGzFwH6pqanW30XVq1c3DzzwgFm3bp3ZsmWLNcaeMURexQCOsHfvXrNu3Trz8ssvGzc3N+v3xLVPeg4NDTVPP/20dfu/Pgaw5/fm1q1bjSSzZs0am2MffPBBc+edd2ba73PPPWcef/xx8+2335rFixeb3r17Gw8PD7Ny5cr8vJwCs2vXLlO0aFHz+OOPmx9++MF0797dlCxZ0hw7dswaM3r0aOPn52fdrlevnhk7dqxZvny5+fzzz03NmjVN5cqVzblz57LVb2E2ceJE4+3tbaZOnWrmzZtnqlSpYjp16mQTU6NGDTNgwABjjDGRkZGmTJkypm7duubXX3+1+bsrPj7eGGPMX3/9ZRo1amQ+/vhjs2LFCjNu3Djj4+NjBg4cWODXlxtpaWmmefPmpnbt2ua7774zEydONBaLxcyaNcsas3//fpu/Zew5xp6YwuzgwYPG39/fPPzww+aHH34wDz30kPH39zcHDx60xrzzzjvGzc3Nuv3QQw+ZIkWKmLlz59p8TR0+fNga06xZM/PGG2+YZcuWmS+//NLUqVPHlC9f3pw4caIgLy/fzJo1y3h4eJhJkyaZ7777ztSqVcs0b97c+neXMcY0adLEdO3a1RhzddweHh5upk2bZlasWGEmTpxoSpQoYbp3757tfguz++67z1StWtXMmzfPTJ061Xh5eZlJkyZZ96fXG+bOnWtz3EsvvWSCg4PNlStXMvQ5ceJE06tXL/Pll1+aZcuWmUGDBhl3d3fz+eef5/v1XMvFmOv++Qs5dvr0aY0fP17bt29XUFCQBg8erKZNmzo6rf+MhISELP+l6Nlnn9UDDzxg3d64caOmTJmiEydOqEaNGnrxxRczPOzBnhjkjZiYGEVEROjtt99W48aNbfZ9+umnWrBggZKTk9W2bVsNGzYsw9o09sQg5xITE/X+++/rl19+kbe3t5o1a6bBgwfbPEAiJiZGb7/9tjZu3Ch/f38NGDBAHTp0sOnHnhjkzvfff68FCxbo1KlTKlOmjHr16pXhHu/atcu61l2lSpX0wgsvZHgqrz0xsE9Wv5vCwsJs1ly3ZwyRVzFAQRszZoxWrFiRoX3atGnWGWXdu3dX06ZNrWu6SowBbvZ788CBA+rXr591TVdJunLliiIiItSvXz+btfjSpaWl6auvvtLSpUsVFxen2267TUOHDlVYWFiBXVd+++effzRp0iRFRkaqSpUqeuGFF2yub/bs2Zo/f75+/PFHSVcfXvzhhx9q48aN8vb2Vv369TV48GB5e3tnq9/C7uuvv9Y333yjhIQEtWzZUs8884zNg/4efvhh699EGzZs0AsvvJBpP3PnzrU+UG/79u36+OOPtX//fpUtW1ZdunRRp06dCuR68lJ8fLzeeecd/f777/Lx8dHDDz9sfSCvdHXGaM+ePW3+lrnZMfbGFGZ79uzRu+++q0OHDiksLEzPPfecqlWrZt3/zTffaPr06Vq7dq0k6d5779XFixcz9NO7d28NGTJEkhQbG6upU6dqw4YN8vT0VHh4uIYOHWpdHuxWsGjRIn3++eeKi4tTkyZN9Pzzz6to0aLW/YMHD1ZAQIDGjBkj6ep9njZtmvbs2aPg4GB17Ngxw3rm9vRbmCUmJmry5Mlas2aNvLy81KtXL/Xu3du6P73eMGrUKLVt29ba3qNHDzVu3DjDetzpFi9erO+++05nz55VpUqV9Pjjj1sfcl9QKNoCAAAAAAAAgBNhTVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAUWr///ru2bNni6DQAIE9RtAWAQmzXrl169tln1aZNG7Vr105Dhw7Vxo0bHZLLmjVr1KZNG4ecGwAAAIXPH3/8oV9//TXX/UyePFmzZ8/Og4wAwHlQtAWAQmr69OmqU6eOEhMTNXz4cD3//POqUaOGRowYoXnz5hV4PufOndOmTZsK/LwAAAAonN5//32NGzfO0WkAgFOiaAsAhdCWLVs0ZMgQvfvuu/roo48UERGhe+65R4MGDdL69esVERFhjT148KCeeOIJNW/eXN26ddPy5ctt+po3b5769u1r0/bnn3/q7rvvtm6vWLFCERER2rhxo/r376977rlHL7zwguLi4qz5vPzyy7p06ZIaNmyohg0b6qOPPsrHOwAAAIBbzc8//6ydO3fq4sWLWrNmjdasWaOEhIQMcbGxsVq2bJnWr1+v+Pj4TPtKS0vTX3/9pUWLFumff/6x2bdz50599913MsZY2y5cuKBvvvlGJ0+ezNuLAoAccnd0AgCA7Js+fbpKly6tQYMGZbq/WLFikqRTp06pfv36atu2rV5++WVt3bpV999/v7744gv16NFDknTixAlt377d5viLFy/qzz//tG6fOXNGK1eu1IkTJ/Tyyy+rSJEievHFFxUZGalvvvlGlStXVv/+/fW///1P7733niSpbNmy+XDlAAAAuFWNHj1arq6uOn78uKpVq6a9e/dKkjZu3KgSJUpIkrZv3642bdooKChIwcHBOnLkiLy8vBQcHGztJzo6Wp06ddKlS5dUrVo1bd26VTVq1NDChQvl5eWlEiVK6Mknn9Thw4f1/PPPS5Iee+wxHTt2TN26dSv4CweATFC0BYBCaNu2bapZs6bc3W/8Y3zChAkqXbq0vvrqK7m4uKhdu3a6ePGiXnrpJWvR1l4pKSn67rvvFBYWJkm6cuWK+vbtK2OMfH19VaVKFbm5ualhw4Y5vi4AAAD8tx08eFBbt25VUFCQkpOTVb16dc2YMUMjR46UJA0ePFitW7e2jm+XLl2q++67T82bN7f28fDDD6tBgwaaOnWqXFxclJiYqKZNm2rChAl6/fXXVaZMGX3yySfq2bOnWrdurb///lsrV67U1q1bbzq+BoCCwvIIAFAIJSYmytfX96ZxmzZtUrt27eTi4mJt69ixow4fPqzz589n65wlSpSwFmwlqVy5ckpISFBsbGy2+gEAAACy0rVrVwUFBUmSPDw81KhRI+uM22PHjmnDhg167rnnrOPbe++9V9WrV7ceHxkZqV9++UWVKlXSggULNH/+fP3www8KCwvTmjVrrHH333+/+vXrp+7du2v48OF6//33Vbly5QK8UgC4Mf4JCQAKoZCQEB0+fPimcXFxcdalEtKlb8fFxVk/ZmYPi8Vis50+UL52LTAAAAAgN64fn3p6eurSpUuSrhZkJalChQo2MRUrVrT+/5EjRyRJv//+u81Dcl1cXFS3bt3/1979gyTXxmEcv56kPxDhEBGVERgKZSekSCXagrYkqDWwIqKGlsD+PA01BEJNghGCSzRESDg1ubdF4GBEKSVUgwlBiVO9QyRE7/NCD8V7gu9nOufmd27vs8jh4ty/8+66tbU1xWIxuVwuTU5OftUtAMCXILQFgB/I7/drbm5OqVRKhmH8sa69vV3pdPrdWDqdVlVVlWw2mySptrZWxWLxXc3ffIChoqKCABcAAADfpr6+XtLrR8Pejt/O34Lct91oq6ur6unp+c/5gsGg7Ha70um0EomERkZGvmXdAPA3aI8AAD9QIBBQb2+vxsfHdX5+Xh5/fn5WPB5XMpmUJE1NTSmRSOj4+FiSdH9/r1AopEAgIIvFIkkyDENXV1c6OTmRJBUKBYXD4U+vqbGxUY+Pj7RLAAAAwLew2+1qampSIpEoj+VyuXdv1BqGoZaWFu3s7Hy4/ubmpny8v7+veDyuRCKh9fV1TU9P6/b29lvXDwCfQWgLAD9QdXW1ksmkPB6P3G63nE6nDMOQ1WpVPB6Xw+GQ9Nq/dnl5WYODg+ro6FBbW5uampoUCoXKc/l8Ps3MzKi/v1/d3d0yDENut/vTa/J6vfJ4PHI6nfL5fNre3v6q2wUAAABUWVmpjY0N/f79WysrK4pEIhoaGlJdXV25xmKxKBaLaW9vT6Ojo4pGowqFQhoYGNDBwYGk1zYLs7Oz2tzcVGdnpxYXF+VyuRQIBNg5BsA0fr3wjwQAP1qpVNLFxYUqKipkt9tVU1Pzoebh4UGXl5dqaGhQa2vrv85zd3enQqEgh8OhYrGos7Mzeb1eSVI+n9f19fW7LWZPT09KpVLq6+srv7X78vKiTCajfD6v5ubmP/4WAAAAEA6HVSwWtbS0JOm1x2xXV5fGxsbKNdFoVKVSSfPz8+Wxo6MjHR4eymq1anh4WKenp7JarZqYmCjXZLNZ7e7uKpPJyGazye/3l59tI5GIstmstra2yvW5XE7BYFALCwsfet8CwP+B0BYAAAAAAAAATIT2CAAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCL/AKOrK3g1aPnlAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "{'per_year': {'counts': record_num\n", + " 2023 376\n", + " 2024 718\n", + " 2025 807\n", + " 2026 141\n", + " Name: count, dtype: int64,\n", + " 'total': 2042},\n", + " 'mammaprint_overall': {'series': 1 -0.158\n", + " 2 0.058\n", + " 3 -0.039\n", + " 4 0.065\n", + " 5 -0.429\n", + " ... \n", + " 2304 -0.218\n", + " 2306 -0.141\n", + " 2307 0.192\n", + " 2308 0.396\n", + " 2309 -0.065\n", + " Name: mammaprint_index, Length: 2042, dtype: float64,\n", + " 'mean': -0.008001126346718901,\n", + " 'variance': 0.09011556584390465,\n", + " 'count': 2042},\n", + " 'mammaprint_by_type': {'a luminal': {'series': 2 0.058\n", + " 4 0.065\n", + " 6 0.451\n", + " 7 0.073\n", + " 11 0.090\n", + " ... \n", + " 2300 0.169\n", + " 2302 0.091\n", + " 2303 0.235\n", + " 2307 0.192\n", + " 2308 0.396\n", + " Name: mammaprint_index, Length: 1077, dtype: float64,\n", + " 'mean': 0.22133119777158775,\n", + " 'variance': 0.022705340289735015,\n", + " 'count': 1077},\n", + " 'b luminal': {'series': 1 -0.158\n", + " 3 -0.039\n", + " 5 -0.429\n", + " 8 -0.052\n", + " 9 -0.259\n", + " ... \n", + " 2299 -0.241\n", + " 2301 -0.391\n", + " 2304 -0.218\n", + " 2306 -0.141\n", + " 2309 -0.065\n", + " Name: mammaprint_index, Length: 965, dtype: float64,\n", + " 'mean': -0.26395025906735753,\n", + " 'variance': 0.041114951880119543,\n", + " 'count': 965}},\n", + " 'counts_by_type': {'counts': type\n", + " a luminal 1077\n", + " b luminal 965\n", + " Name: count, dtype: int64,\n", + " 'total_distinct': 2}}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "run_all_stats_and_plot(final_df, \"stats\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/notebooks/diff_script.ipynb b/scripts/notebooks/diff_script.ipynb new file mode 100644 index 00000000..f58472a2 --- /dev/null +++ b/scripts/notebooks/diff_script.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "d06d8e25-a2e4-47d0-8322-040882d4bd58", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmp63gnbhm0\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (3.0.1)\n", + "Requirement already satisfied: numpy in /opt/venv/lib/python3.14/site-packages (2.4.3)\n", + "Requirement already satisfied: openpyxl in /opt/venv/lib/python3.14/site-packages (3.1.5)\n", + "Requirement already satisfied: matplotlib in /opt/venv/lib/python3.14/site-packages (3.10.8)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.14/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: et-xmlfile in /opt/venv/lib/python3.14/site-packages (from openpyxl) (2.0.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (4.62.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.5.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (25.0)\n", + "Requirement already satisfied: pillow>=8 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (12.1.1)\n", + "Requirement already satisfied: pyparsing>=3 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (3.3.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/venv/lib/python3.14/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas numpy openpyxl matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0b50df43-0b03-497c-99ce-227d656fd14c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved 160 full rows (B minus A on 'record_num') to /mnt/projects/mammaprint/data_mapping_diff2.csv\n", + "SUCCESS: A is a subset of B based on key 'record_num'.\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import sys\n", + "\n", + "# ---- Hard-coded file paths ----\n", + "FILE_A = \"/mnt/projects/mammaprint/data_mapping.csv\"\n", + "FILE_B = \"/mnt/projects/mammaprint/data_mapping_new.csv\"\n", + "\n", + "# Output for rows in B but not in A (full rows, not just keys)\n", + "ONLY_IN_B = \"/mnt/projects/mammaprint/data_mapping_diff2.csv\"\n", + "\n", + "KEY = \"record_num\"\n", + "\n", + "\n", + "def main():\n", + " # Load CSVs\n", + " df_a = pd.read_csv(FILE_A)\n", + " df_b = pd.read_csv(FILE_B)\n", + "\n", + " # Ensure key column exists\n", + " if KEY not in df_a.columns or KEY not in df_b.columns:\n", + " print(f\"ERROR: '{KEY}' column is missing in one of the files.\")\n", + " sys.exit(1)\n", + "\n", + " # ---- Compare keys ----\n", + " keys_a = df_a[KEY].drop_duplicates()\n", + " keys_b = df_b[KEY].drop_duplicates()\n", + "\n", + " # Keys in A not found in B\n", + " a_not_in_b = keys_a[~keys_a.isin(keys_b)]\n", + "\n", + " # ---- FULL ROWS from B that do not exist in A (keep all columns) ----\n", + " b_not_in_a_full = df_b[~df_b[KEY].isin(keys_a)]\n", + " b_not_in_a_full.to_csv(ONLY_IN_B, index=False)\n", + "\n", + " print(f\"Saved {len(b_not_in_a_full)} full rows (B minus A on '{KEY}') to {ONLY_IN_B}\")\n", + "\n", + " # Check subset condition\n", + " if not a_not_in_b.empty:\n", + " print(f\"ERROR: A is NOT a subset of B based on key '{KEY}'.\")\n", + " print(f\"Keys in A but missing in B: {len(a_not_in_b)}\")\n", + " print(a_not_in_b)\n", + " sys.exit(1)\n", + "\n", + " print(f\"SUCCESS: A is a subset of B based on key '{KEY}'.\")\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/notebooks/mask_coverage_threshold.ipynb b/scripts/notebooks/mask_coverage_threshold.ipynb new file mode 100644 index 00000000..34d434a6 --- /dev/null +++ b/scripts/notebooks/mask_coverage_threshold.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6eb94fd9", + "metadata": {}, + "source": [ + "# Mask coverage → pick a filtering threshold\n", + "\n", + "The tiling pipeline (`preprocessing/tiling.py`) already computes a **per-tile coverage score** for\n", + "each mask type and stores it in the tiled dataset's `tiles.parquet` as these columns:\n", + "`tissue_overlap`, `epithelium_overlap`, `cancer_overlap`, `blur_overlap`, `folding_overlap`,\n", + "`residual_overlap`. Each is in `[0, 1]`: coverage columns are fractions of desired pixels and\n", + "`cancer_overlap` is the mean normalized cancer probability. Tiles can be kept/dropped by\n", + "thresholding these scores.\n", + "\n", + "This notebook loads those columns straight from a tiled-dataset MLflow artifact and shows their\n", + "**distribution + CDF** so you can pick sensible thresholds.\n", + "\n", + "> **Note on \"AUC\".** A true ROC/AUC needs a per-tile keep/discard ground truth, which we don't have.\n", + "> Instead we plot the **coverage distribution** and its **CDF** — the CDF axis directly answers\n", + "> *\"if I set the threshold at T, what fraction of tiles do I keep vs drop?\"*, which is exactly the\n", + "> decision you want to make. An optional luminal A/B overlay lets you check the two classes behave\n", + "> similarly under a given cutoff.\n", + "\n", + "No masks or OpenSlide access needed — the overlaps are read as-is from the parquet, so the numbers\n", + "are exactly what the pipeline produced.\n", + "\n", + "Edit the **Config** cell (tiled-dataset URI + which overlaps to look at), then run the **Calls** cells." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73e2c71a", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install \"pandas\" \"mlflow>3\" pyarrow matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6472a2b0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mlflow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from mlflow.artifacts import download_artifacts" + ] + }, + { + "cell_type": "markdown", + "id": "4a898180", + "metadata": {}, + "source": [ + "# Config\n", + "\n", + "This is the one cell you edit. Point `TILED_DATASET_URI` at the tiling run's MLflow artifact dir\n", + "(the one containing `slides.parquet` / `tiles.parquet`), and pick which overlaps to plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37f50707", + "metadata": {}, + "outputs": [], + "source": "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\nos.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\nmlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n\n# Tiled-dataset artifact dir (contains slides.parquet + tiles.parquet). Same style as the\n# TILED_DATASETS entries in check_mask_coverage.ipynb. Point this at your dataset.\nTILED_DATASET_URI = \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\"\n\n# The overlap columns to analyze (as stored in tiles.parquet). Drop any you don't care about.\nOVERLAP_COLUMNS = [\n \"tissue_overlap\",\n \"epithelium_overlap\",\n \"cancer_overlap\",\n \"blur_overlap\",\n \"folding_overlap\",\n \"residual_overlap\",\n]\n\n# All current overlap columns already use the notebook convention: higher means a larger\n# fraction of the desired/keep signal. Only list legacy columns here if an older artifact stored\n# them in the opposite direction.\nINVERT_COLUMNS = []\n\n# Column holding the luminal type on each tile row (for the optional A/B overlay). Set to None\n# to disable the per-class split. tiling.py copies `type` onto every tile.\nTYPE_COLUMN = \"type\"" + }, + { + "cell_type": "markdown", + "id": "992db56f", + "metadata": {}, + "source": [ + "# Functions\n", + "\n", + "Run every cell in this section once, then use the **Calls** section at the bottom." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e128468", + "metadata": {}, + "outputs": [], + "source": "# ---- Load tiles.parquet from a tiled-dataset MLflow artifact ----\n# type_label mirrors ml/data/datasets/labels.py:_map_luminal_type (a luminal -> 1, b luminal -> 0)\n_LUMINAL_MAP = {\"a luminal\": 1, \"b luminal\": 0}\n\n\ndef load_tiles(uri=TILED_DATASET_URI, overlap_columns=OVERLAP_COLUMNS,\n invert_columns=INVERT_COLUMNS, type_column=TYPE_COLUMN):\n \"\"\"Download the tiled dataset's tiles.parquet and return it as a DataFrame.\n\n Any column in `invert_columns` is replaced with `1 - value` (fixes overlaps stored with the\n opposite convention, so higher = cleaner/keep everywhere). Adds `type_label` (luminal A=1/B=0)\n if `type_column` is present. Warns about requested overlap columns missing from the parquet.\n \"\"\"\n local_dir = Path(download_artifacts(uri))\n tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n print(f\"Loaded {len(tiles):,} tiles from {local_dir / 'tiles.parquet'}\")\n print(f\"Columns: {tiles.columns.tolist()}\")\n\n missing = [c for c in overlap_columns if c not in tiles.columns]\n if missing:\n print(f\"WARNING: requested overlap columns not in parquet: {missing}\")\n\n for col_name in invert_columns:\n if col_name in tiles.columns:\n tiles[col_name] = 1.0 - tiles[col_name]\n print(f\"Inverted '{col_name}' -> plotting 1 - {col_name}\")\n\n if type_column and type_column in tiles.columns:\n tiles[\"type_label\"] = tiles[type_column].str.strip().str.lower().map(_LUMINAL_MAP)\n print(tiles[type_column].value_counts(dropna=False).to_string())\n return tiles" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8b0baa0", + "metadata": {}, + "outputs": [], + "source": "# ---- Per-column quantile table (read cutoffs straight off it) ----\ndef summary_table(tiles, overlap_columns=OVERLAP_COLUMNS):\n \"\"\"Count + quantiles of each overlap column.\n\n Includes fine LOW-end quantiles (1/2/5/10%) because for right-skewed overlaps the useful\n threshold sits in the bottom tail: e.g. the 5% row tells you the cutoff that drops ~5% of tiles.\n \"\"\"\n qs = [0.0, 0.01, 0.02, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0]\n cols = [c for c in overlap_columns if c in tiles.columns]\n out = (\n tiles[cols]\n .describe(percentiles=qs)\n .drop(index=[\"mean\", \"std\"], errors=\"ignore\")\n .T\n )\n return out" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "428367e0", + "metadata": {}, + "outputs": [], + "source": "# ---- Plot: coverage histogram + CDF, with candidate thresholds ----\n_CLASS_NAMES = {1: \"a luminal\", 0: \"b luminal\"}\n\n\ndef auto_xlim(values, tail=0.02, pad=0.01):\n \"\"\"Zoom window that frames the informative tail of a skewed distribution.\n\n Returns (lo, hi) spanning the `tail` quantile (e.g. 2%) up to the max, padded a little. When\n almost every tile has coverage ~1.0, this zooms in near 1.0 where the threshold decision lives.\n Falls back to the full (0, 1) range for empty/all-NaN or degenerate (all-identical) input.\n \"\"\"\n values = np.asarray(pd.Series(values).dropna())\n if values.size == 0:\n return (0.0, 1.0)\n lo = float(np.quantile(values, tail))\n hi = float(values.max())\n lo = max(0.0, lo - pad)\n hi = min(1.0, hi + pad)\n if hi - lo < 1e-6: # degenerate (all identical) -> fall back to full range\n return (0.0, 1.0)\n return (lo, hi)\n\n\ndef plot_overlap(tiles, column, thresholds=(0.1, 0.25, 0.5), by_class=True,\n bins=50, xlim=(0.0, 1.0)):\n \"\"\"Histogram (left) + CDF (right) of one overlap column.\n\n The CDF reads directly as 'fraction of tiles dropped if threshold = x'; keep-fraction is\n printed for each candidate threshold. Optional luminal A/B overlay.\n\n `xlim` sets the x-axis window (bins + histogram range track it), so for heavily skewed\n columns you can zoom toward 1.0. Pass `xlim=auto_xlim(tiles[column].dropna())` to auto-frame\n the tail. The keep/drop stats below always use the FULL data, regardless of the zoom window.\n \"\"\"\n if column not in tiles.columns:\n print(f\"Column '{column}' not in tiles.\")\n return\n values_all = tiles[column].dropna()\n if values_all.empty:\n print(f\"No values for '{column}' (empty or all-NaN) -- skipping.\")\n return\n\n lo, hi = xlim\n fig, (ax_hist, ax_cdf) = plt.subplots(1, 2, figsize=(13, 4.5))\n\n def _plot_series(values, label):\n values = np.asarray(values)\n ax_hist.hist(values, bins=bins, range=(lo, hi), alpha=0.5, density=True, label=label)\n xs = np.sort(values)\n ys = np.arange(1, len(xs) + 1) / len(xs) # P(overlap <= x) = fraction dropped\n ax_cdf.plot(xs, ys, label=label)\n\n if by_class and \"type_label\" in tiles.columns and tiles[\"type_label\"].notna().any():\n for lbl, group in tiles.dropna(subset=[column]).groupby(\"type_label\", dropna=True):\n _plot_series(group[column], _CLASS_NAMES.get(lbl, str(lbl)))\n else:\n _plot_series(values_all, \"all\")\n\n for t in thresholds:\n if lo <= t <= hi:\n ax_hist.axvline(t, color=\"k\", ls=\"--\", lw=0.8)\n ax_cdf.axvline(t, color=\"k\", ls=\"--\", lw=0.8)\n\n zoom_note = \"\" if (lo, hi) == (0.0, 1.0) else f\" [zoom {lo:.3f}–{hi:.3f}]\"\n ax_hist.set(title=f\"{column}: distribution{zoom_note}\", xlabel=\"coverage\", ylabel=\"density\",\n xlim=(lo, hi))\n ax_hist.legend()\n ax_cdf.set(title=f\"{column}: CDF (= fraction dropped at threshold){zoom_note}\",\n xlabel=\"threshold\", ylabel=\"fraction of tiles with coverage ≤ threshold\",\n xlim=(lo, hi))\n ax_cdf.legend()\n fig.tight_layout()\n plt.show()\n\n total = len(values_all)\n print(f\"{column}: {total:,} tiles\")\n for t in thresholds:\n kept = int((values_all > t).sum())\n print(f\" threshold > {t:>5}: keep {kept:>9,} ({100 * kept / total:5.1f}%), \"\n f\"drop {total - kept:>9,} ({100 * (total - kept) / total:5.1f}%)\")" + }, + { + "cell_type": "markdown", + "id": "5c7488ad", + "metadata": {}, + "source": [ + "# Calls\n", + "\n", + "Run these to load the dataset and inspect each overlap. Adjust `thresholds=` to your candidate\n", + "cutoffs and re-run cell 3 as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6020e97", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Load tiles.parquet from the tiled-dataset artifact.\n", + "tiles = load_tiles()\n", + "tiles.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b681e8af", + "metadata": {}, + "outputs": [], + "source": [ + "# 2. Quantile table — read candidate thresholds straight off it.\n", + "summary_table(tiles)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a37d329b", + "metadata": {}, + "outputs": [], + "source": "# 3. Distribution + CDF per overlap column.\n# - `thresholds=` : candidate cutoffs to annotate + print keep/drop for.\n# - `xlim=` : x-axis zoom window. Only folding/residual are zoomed here (their mass sits\n# near 1.0); the rest are shown full-range. auto_xlim frames the tail --\n# lower `tail=` to zoom tighter, or pass a manual window like (0.9, 1.0).\nZOOM_COLUMNS = [\"folding_overlap\", \"residual_overlap\"]\n\nfor column in OVERLAP_COLUMNS:\n if column not in tiles.columns:\n continue\n if column in ZOOM_COLUMNS:\n xlim = auto_xlim(tiles[column].dropna()) # or e.g. (0.9, 1.0)\n else:\n xlim = (0.0, 1.0)\n plot_overlap(tiles, column, thresholds=(0.1, 0.25, 0.5), xlim=xlim)" + }, + { + "cell_type": "markdown", + "id": "29607684", + "source": "## How to pick a threshold\n\nThere's no single \"correct\" cutoff — you're trading off **tile quality** against **how much data you keep**. Use the plots + quantile table together:\n\n1. **Look for a gap / elbow in the distribution.** If the histogram is bimodal — a big clean lump near 1.0 and a smaller junk lump near 0 — put the threshold *in the valley between them*. That's the most defensible cutoff: it separates two populations rather than cutting into one.\n2. **If there's no clear gap (just a smooth skew toward 1.0),** there's no natural boundary, so pick by **data-loss budget** instead. Decide how many tiles you're willing to drop (e.g. \"≤5%\") and read the cutoff off the matching low-end quantile in `summary_table` (the 5% row) or off the CDF (find where y = 0.05, take the x). A skewed column usually means the mask rarely fires, so a low cutoff (drop only the worst 1–5%) is typically enough.\n3. **Match the cutoff to what the mask is *for*.**\n - `tissue_overlap` — the gate that decides a tile is on tissue at all; a moderate cutoff (e.g. 0.25–0.5) is normal. (The pipeline already drops `tissue_overlap == 0`.)\n - `epithelium_overlap` — how much tumor epithelium a tile must contain; set by how epithelium-pure you want tiles, not by data loss.\n - `blur` / `folding` / `residual` (QC) — these only remove artifacts, so keep them **lenient**: drop just the clearly-bad tail (often 0.01–0.1) so you don't throw away good tissue.\n4. **Sanity-check the A/B overlay.** A QC/tissue threshold should affect luminal A and B tiles *about equally*. If one class loses far more tiles than the other at your cutoff, you're introducing class bias — loosen it or reconsider.\n5. **Confirm with the printed keep/drop line.** Once a candidate looks right, add it to `thresholds=` and check the exact keep/drop counts are acceptable before committing.\n\nIn short: **cut at a gap if one exists; otherwise cut by an acceptable data-loss budget — lenient for pure-artifact masks, and balanced across A/B.**", + "metadata": {} + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/notebooks/mask_coverage_threshold.py b/scripts/notebooks/mask_coverage_threshold.py new file mode 100644 index 00000000..d4f49fce --- /dev/null +++ b/scripts/notebooks/mask_coverage_threshold.py @@ -0,0 +1,316 @@ +"""Analyze cancer-mask tile scores and choose a production filtering threshold. + +This is the non-interactive cluster version of mask_coverage_threshold.ipynb. +It is preconfigured for the corrected level-3 cancer-probability tiling run. +The script prints quantiles and exact keep/drop counts, then logs CSV tables and +a histogram/CDF figure to a small MLflow analysis run. +""" + +from __future__ import annotations + +import argparse +import os +import tempfile +from pathlib import Path + +import matplotlib + + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt +import mlflow +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +from mlflow.artifacts import download_artifacts + + +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DEFAULT_DATASET_URI = ( + "mlflow-artifacts:/3/41a0cfd625574ed9a5c547a8c254bc8a/artifacts/" + "mou_3_224_cancer_probability_threshold_scan" +) +DEFAULT_COLUMN = "cancer_overlap" +DEFAULT_THRESHOLDS = ( + 0.05, + 0.10, + 0.15, + 0.20, + 0.25, + 0.30, + 0.40, + 0.50, + 0.60, + 0.70, + 0.75, + 0.80, + 0.85, + 0.90, + 0.95, +) +QUANTILES = ( + 0.0, + 0.001, + 0.01, + 0.02, + 0.05, + 0.10, + 0.25, + 0.50, + 0.75, + 0.90, + 0.95, + 0.99, + 0.999, + 1.0, +) +LUMINAL_MAP = {"a luminal": "A", "b luminal": "B"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Analyze cancer_overlap from a tiled MLflow dataset.", + ) + parser.add_argument("--dataset-uri", default=DEFAULT_DATASET_URI) + parser.add_argument("--column", default=DEFAULT_COLUMN) + parser.add_argument("--tracking-uri", default=DEFAULT_TRACKING_URI) + parser.add_argument( + "--thresholds", + type=float, + nargs="+", + default=list(DEFAULT_THRESHOLDS), + ) + parser.add_argument("--experiment", default="MammaPrint") + parser.add_argument( + "--run-name", + default="Cancer probability threshold analysis l3", + ) + return parser.parse_args() + + +def resolve_tiles_path(dataset_uri: str) -> Path: + local_path = Path(download_artifacts(dataset_uri)) + if local_path.is_file(): + return local_path + tiles_path = local_path / "tiles.parquet" + if not tiles_path.is_file(): + raise FileNotFoundError( + f"Downloaded '{dataset_uri}', but '{tiles_path}' does not exist." + ) + return tiles_path + + +def load_tiles(tiles_path: Path, column: str) -> pd.DataFrame: + available_columns = set(pq.read_schema(tiles_path).names) + if column not in available_columns: + raise ValueError( + f"Column '{column}' is missing from {tiles_path}. " + f"Available columns: {sorted(available_columns)}" + ) + + columns = [column] + if "type" in available_columns: + columns.append("type") + tiles = pd.read_parquet(tiles_path, columns=columns, engine="pyarrow") + tiles[column] = pd.to_numeric(tiles[column], errors="coerce") + if "type" in tiles: + tiles["luminal_class"] = ( + tiles["type"].astype("string").str.strip().str.lower().map(LUMINAL_MAP) + ) + return tiles + + +def quantile_table(tiles: pd.DataFrame, column: str) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + groups.extend( + (f"luminal_{label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ) + + for group_name, values in groups: + clean = values.dropna() + quantiles = clean.quantile(QUANTILES) + for quantile, value in quantiles.items(): + rows.append( + { + "group": group_name, + "quantile": float(quantile), + "cancer_probability": float(value), + "tile_count": len(clean), + } + ) + return pd.DataFrame(rows) + + +def threshold_table( + tiles: pd.DataFrame, + column: str, + thresholds: list[float], +) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + groups.extend( + (f"luminal_{label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ) + + for group_name, values in groups: + clean = values.dropna() + total = len(clean) + for threshold in thresholds: + kept = int((clean > threshold).sum()) + dropped = total - kept + rows.append( + { + "group": group_name, + "threshold": threshold, + "total_tiles": total, + "kept_tiles": kept, + "dropped_tiles": dropped, + "kept_percent": 100.0 * kept / total if total else 0.0, + "dropped_percent": 100.0 * dropped / total if total else 0.0, + } + ) + return pd.DataFrame(rows) + + +def class_bias_table(thresholds: pd.DataFrame) -> pd.DataFrame: + class_rows = thresholds[thresholds["group"].isin(["luminal_A", "luminal_B"])] + if class_rows["group"].nunique() != 2: + return pd.DataFrame() + pivot = class_rows.pivot( + index="threshold", + columns="group", + values="kept_percent", + ).reset_index() + pivot["absolute_keep_rate_gap_pp"] = ( + pivot["luminal_A"] - pivot["luminal_B"] + ).abs() + return pivot + + +def plot_distribution( + tiles: pd.DataFrame, + column: str, + thresholds: list[float], + output_path: Path, +) -> None: + bins = np.linspace(0.0, 1.0, 201) + figure, (histogram_axis, cdf_axis) = plt.subplots(1, 2, figsize=(14, 5)) + + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + class_groups = [ + (f"luminal {label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ] + if class_groups: + groups = class_groups + + for label, values in groups: + clean = values.dropna().to_numpy() + counts, edges = np.histogram(clean, bins=bins) + centers = (edges[:-1] + edges[1:]) / 2 + density = counts / counts.sum() / np.diff(edges) if counts.sum() else counts + cdf = np.cumsum(counts) / counts.sum() if counts.sum() else counts + histogram_axis.plot(centers, density, label=label) + cdf_axis.plot(edges[1:], cdf, label=label) + + for threshold in thresholds: + histogram_axis.axvline(threshold, color="black", alpha=0.18, linewidth=0.7) + cdf_axis.axvline(threshold, color="black", alpha=0.18, linewidth=0.7) + + histogram_axis.set( + title=f"{column}: probability distribution", + xlabel="mean cancer probability", + ylabel="density", + xlim=(0.0, 1.0), + ) + cdf_axis.set( + title="CDF: fraction dropped at threshold", + xlabel="threshold", + ylabel="fraction with score ≤ threshold", + xlim=(0.0, 1.0), + ylim=(0.0, 1.0), + ) + histogram_axis.legend() + cdf_axis.legend() + figure.tight_layout() + figure.savefig(output_path, dpi=180) + plt.close(figure) + + +def main() -> None: + args = parse_args() + os.environ["MLFLOW_TRACKING_URI"] = args.tracking_uri + mlflow.set_tracking_uri(args.tracking_uri) + + print(f"Downloading tiled dataset: {args.dataset_uri}") + tiles_path = resolve_tiles_path(args.dataset_uri) + tiles = load_tiles(tiles_path, args.column) + values = tiles[args.column].dropna() + + print(f"Loaded {len(tiles):,} tile rows from {tiles_path}") + print(f"Valid {args.column} values: {len(values):,}") + print(f"Unique {args.column} values: {values.nunique():,}") + print("\nCancer probability summary:") + print( + values.describe( + percentiles=[0.001, 0.01, 0.02, 0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99, 0.999] + ).to_string() + ) + + quantiles = quantile_table(tiles, args.column) + thresholds = threshold_table(tiles, args.column, args.thresholds) + bias = class_bias_table(thresholds) + + print("\nThreshold keep/drop table (all tiles):") + print( + thresholds[thresholds["group"] == "all"].to_string( + index=False, + formatters={ + "kept_percent": "{:.2f}".format, + "dropped_percent": "{:.2f}".format, + }, + ) + ) + if not bias.empty: + print("\nLuminal A/B keep-rate comparison:") + print(bias.to_string(index=False, float_format=lambda value: f"{value:.2f}")) + + mlflow.set_experiment(args.experiment) + with tempfile.TemporaryDirectory() as directory: + output_dir = Path(directory) + quantiles.to_csv(output_dir / "cancer_probability_quantiles.csv", index=False) + thresholds.to_csv(output_dir / "cancer_threshold_keep_drop.csv", index=False) + if not bias.empty: + bias.to_csv(output_dir / "cancer_threshold_class_bias.csv", index=False) + plot_distribution( + tiles, + args.column, + args.thresholds, + output_dir / "cancer_probability_distribution_cdf.png", + ) + + with mlflow.start_run(run_name=args.run_name) as run: + mlflow.log_param("source_dataset_uri", args.dataset_uri) + mlflow.log_param("score_column", args.column) + mlflow.log_param("candidate_thresholds", ",".join(map(str, args.thresholds))) + mlflow.log_metric("tile_count", len(values)) + mlflow.log_metric("cancer_probability_mean", float(values.mean())) + mlflow.log_metric("cancer_probability_median", float(values.median())) + mlflow.log_artifacts(str(output_dir), artifact_path="threshold_analysis") + print(f"\nLogged analysis to MLflow run: {run.info.run_id}") + print( + "Analysis artifacts: " + f"mlflow-artifacts:/{run.info.experiment_id}/{run.info.run_id}/" + "artifacts/threshold_analysis" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_epithelium_masks.py b/scripts/preprocessing/run_epithelium_masks.py index 6d77dab9..7f7aca47 100644 --- a/scripts/preprocessing/run_epithelium_masks.py +++ b/scripts/preprocessing/run_epithelium_masks.py @@ -9,9 +9,10 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", "uv run -m preprocessing.epithelium_masks", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], diff --git a/scripts/preprocessing/run_filter_embeddings.py b/scripts/preprocessing/run_filter_embeddings.py new file mode 100644 index 00000000..d02447b9 --- /dev/null +++ b/scripts/preprocessing/run_filter_embeddings.py @@ -0,0 +1,124 @@ +"""Submit one CPU-only embedding-filtering job.""" + +from __future__ import annotations + +import argparse +import shlex + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +TISSUE_EMBEDDING_URIS = { + 2: "mlflow-artifacts:/3/d70b72d842604ebe93d3fbd9e1dfd2f5/artifacts/embeddings", + 3: "mlflow-artifacts:/3/c513c87eac3d4792ba9757a2d7300eb6/artifacts/embeddings", + 4: "mlflow-artifacts:/3/ecfc0ecdba5b44d9becc5c9df1103a16/artifacts/embeddings", + 5: "mlflow-artifacts:/3/e4beda97fdab43f08e8b71fc5bea733c/artifacts/embeddings", +} +TISSUE_EMBEDDING_PATHS = { + level: f"/mnt/projects/mammaprint/embeddings/l{level}_tissue_embed" + for level in TISSUE_EMBEDDING_URIS +} +STRATEGY_CHOICES = ("epithelium_only", "or", "and") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--level", type=int, choices=sorted(TISSUE_EMBEDDING_URIS), required=True + ) + parser.add_argument("--tiling-uri", required=True) + parser.add_argument("--embeddings-uri") + parser.add_argument("--embeddings-path") + parser.add_argument("--epithelium-threshold", type=float, default=0.25) + parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument( + "--strategies", + nargs="+", + choices=STRATEGY_CHOICES, + default=list(STRATEGY_CHOICES), + ) + parser.add_argument("--output-dir") + parser.add_argument("--upload-only", action="store_true") + parser.add_argument("--upload-max-retries", type=int, default=3) + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def _token(value: float) -> str: + return f"{value:g}".replace(".", "") + + +def main() -> None: + args = _parse_args() + if args.upload_only and not args.output_dir: + raise ValueError("--upload-only requires --output-dir") + embeddings_uri = args.embeddings_uri or TISSUE_EMBEDDING_URIS[args.level] + embeddings_path = args.embeddings_path or TISSUE_EMBEDDING_PATHS[args.level] + strategies = tuple(dict.fromkeys(args.strategies)) + strategy_token = "-".join( + {"epithelium_only": "epi", "or": "or", "and": "and"}[strategy] + for strategy in strategies + ) + job_name = ( + f"mammaprint-filter-embeddings-l{args.level}-" + f"{strategy_token}-epi{_token(args.epithelium_threshold)}-" + f"c{_token(args.cancer_threshold)}" + ) + command = [ + "uv", + "run", + "-m", + "preprocessing.filter_embeddings", + "--level", + str(args.level), + "--tiling-uri", + args.tiling_uri, + "--embeddings-uri", + embeddings_uri, + "--embeddings-path", + embeddings_path, + "--epithelium-threshold", + f"{args.epithelium_threshold:g}", + "--cancer-threshold", + f"{args.cancer_threshold:g}", + "--strategies", + *strategies, + "--upload-max-retries", + str(args.upload_max_retries), + ] + if args.output_dir: + command.extend(["--output-dir", args.output_dir]) + if args.upload_only: + command.append("--upload-only") + filter_command = shlex.join(command) + + if args.dry_run: + print(f"Job: {job_name}") + print(f"Command: {filter_command}") + return + + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=16, + memory="64Gi", + shm="16Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + filter_command, + ], + storage=[storage.secure.PROJECTS], + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_mask_coverage_threshold.py b/scripts/preprocessing/run_mask_coverage_threshold.py new file mode 100644 index 00000000..09f03479 --- /dev/null +++ b/scripts/preprocessing/run_mask_coverage_threshold.py @@ -0,0 +1,22 @@ +"""Submit the prefilled cancer-threshold analysis to the cluster.""" + +from kube_jobs import submit_job + + +submit_job( + job_name="mammaprint-cancer-probability-threshold-analysis-l3", + username="kissmi", + image="cerit.io/rationai/base:2.0.6", + cpu=4, + memory="16Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + "git checkout feat/tiling-values", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + "uv run python scripts/notebooks/mask_coverage_threshold.py", + ], +) diff --git a/scripts/preprocessing/run_qc.py b/scripts/preprocessing/run_qc.py index 9842c9ba..2797ca95 100644 --- a/scripts/preprocessing/run_qc.py +++ b/scripts/preprocessing/run_qc.py @@ -10,9 +10,10 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", "uv run -m preprocessing.quality_control", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], diff --git a/scripts/preprocessing/run_recover_embedding_splits.py b/scripts/preprocessing/run_recover_embedding_splits.py new file mode 100644 index 00000000..14ea2c22 --- /dev/null +++ b/scripts/preprocessing/run_recover_embedding_splits.py @@ -0,0 +1,84 @@ +"""Submit a resumable job that completes split uploads from an MLflow run's all artifact.""" + +from __future__ import annotations + +import argparse +import re +import shlex + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" + + +def _job_name(run_id: str) -> str: + token = re.sub(r"[^a-z0-9]+", "-", run_id.lower()).strip("-") + return f"mammaprint-recover-embedding-splits-{token[:12]}" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True) + parser.add_argument("--source-artifact-path", default="embeddings") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument("--mark-finished", action="store_true") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + command = [ + "uv", + "run", + "-m", + "preprocessing.recover_embedding_splits", + "--run-id", + args.run_id, + "--source-artifact-path", + args.source_artifact_path, + "--data-mapping", + args.data_mapping, + "--max-retries", + str(args.max_retries), + "--apply", + ] + if args.mark_finished: + command.append("--mark-finished") + worker_command = shlex.join(command) + job_name = _job_name(args.run_id) + + if args.dry_run: + print(f"Job: {job_name}") + print(f"Command: {worker_command}") + return + + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=4, + memory="16Gi", + shm="4Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + f"export MLFLOW_TRACKING_URI={TRACKING_URI}", + "uv sync --frozen", + worker_command, + ], + storage=[storage.secure.PROJECTS], + ) + print(f"Submitted {job_name}.") + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_split_dataset.py b/scripts/preprocessing/run_split_dataset.py new file mode 100644 index 00000000..d6b56a21 --- /dev/null +++ b/scripts/preprocessing/run_split_dataset.py @@ -0,0 +1,118 @@ +"""Submit a Kubernetes job that materialises train/val/test MLflow artifacts. + +Examples:: + + uv run python scripts/preprocessing/run_split_dataset.py --kind tiled \ + --card configs/data/tiled/tissue_only/l5.yaml --name l5_tissue + + uv run python scripts/preprocessing/run_split_dataset.py --kind embedded \ + --uri mlflow-artifacts:/3//artifacts/embeddings \ + --name l5_tissue_embed + +The submitted pod runs ``preprocessing.split_dataset``. The projects volume is +mounted so the worker can read ``/mnt/projects/mammaprint/data_mapping.csv``. +""" + +from __future__ import annotations + +import argparse +import re +import shlex +from pathlib import Path + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +IMAGE = "cerit.io/rationai/base:2.0.6" +CPU = 8 +MEMORY = "64Gi" +SHM = "16Gi" +TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" + + +def _slug(value: str) -> str: + """Convert a dataset identifier to a Kubernetes-compatible name fragment.""" + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + + +def _default_job_name( + kind: str, card: str | None, uri: str | None, name: str | None +) -> str: + if name: + identifier = name + elif card: + path = Path(card) + identifier = f"{path.parent.name}-{path.stem}" + else: + identifier = Path((uri or "dataset").rstrip("/")).name + + job_name = f"mammaprint-split-{kind}-{_slug(identifier)}" + return job_name[:63].rstrip("-") + + +def _worker_command(args: argparse.Namespace) -> str: + command = [ + "uv", + "run", + "-m", + "preprocessing.split_dataset", + "--kind", + args.kind, + "--data-mapping", + args.data_mapping, + ] + if args.card: + command.extend(["--card", args.card]) + else: + command.extend(["--uri", args.uri]) + if args.name: + command.extend(["--name", args.name]) + return shlex.join(command) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Submit a Kubernetes job that splits one MLflow dataset artifact." + ) + parser.add_argument("--kind", choices=["embedded", "tiled"], required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--card", help="Repository-relative level card containing an `all` URI." + ) + source.add_argument("--uri", help="Whole MLflow artifact URI to split.") + parser.add_argument("--name", help="Output base name (default: inferred by the worker).") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + parser.add_argument("--job-name", help="Override the generated Kubernetes job name.") + args = parser.parse_args() + + job_name = args.job_name or _default_job_name( + args.kind, args.card, args.uri, args.name + ) + print(f"Submitting {job_name} with {CPU} CPUs and {MEMORY} RAM.") + submit_job( + job_name=job_name, + username=USERNAME, + image=IMAGE, + cpu=CPU, + memory=MEMORY, + shm=SHM, + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {shlex.quote(GIT_BRANCH)}", + f"export MLFLOW_TRACKING_URI={shlex.quote(TRACKING_URI)}", + "uv sync --frozen", + _worker_command(args), + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + print(f"Submitted {job_name}.") + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_tiling.py b/scripts/preprocessing/run_tiling.py index 134a55d6..0042e7fb 100644 --- a/scripts/preprocessing/run_tiling.py +++ b/scripts/preprocessing/run_tiling.py @@ -1,19 +1,42 @@ +from humanfriendly import parse_size from kube_jobs import storage, submit_job +# Pod resources. These are the single source of truth: they size the K8s request/limit AND +# are forwarded to Ray via environment variables so tiling.py does not hard-code them. +CPU = 48 +MEMORY = "64Gi" +SHM = "32Gi" + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +EXPERIMENT = "preprocessing/tiling/threshold_scan/mou_3_224" + +# Ray's auto-detection is unreliable on this cluster (it over-detects CPUs from the node and +# under-sizes the object store), so we hand it the pod's real numbers. tiling.py reads these +# and derives ray.init(num_cpus=..., object_store_memory=fraction * RAY_MEMORY_BYTES). +ray_env = [ + f"export RAY_NUM_CPUS={CPU}", + f"export RAY_MEMORY_BYTES={parse_size(MEMORY, binary=True)}", +] + submit_job( - job_name="mammaprint-tiling-...", - username=..., + job_name="mammaprint-tiling-all-masks-l3-threshold-scan", + username=USERNAME, image="cerit.io/rationai/base:2.0.6", - cpu=8, - memory="32Gi", + cpu=CPU, + memory=MEMORY, + shm=SHM, gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", + f"git checkout {GIT_BRANCH}", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + *ray_env, "uv sync --frozen", - "uv run -m preprocessing.tiling +experiment=preprocessing/tiling/...", + f"uv run -m preprocessing.tiling +experiment={EXPERIMENT}", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], ) diff --git a/scripts/preprocessing/run_tiling_threshold_scans.py b/scripts/preprocessing/run_tiling_threshold_scans.py new file mode 100644 index 00000000..9878d3ad --- /dev/null +++ b/scripts/preprocessing/run_tiling_threshold_scans.py @@ -0,0 +1,83 @@ +import argparse +import time + +from humanfriendly import parse_size +from kube_jobs import storage, submit_job + + +CPU = 48 +MEMORY = "64Gi" +SHM = "32Gi" + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +DEFAULT_LEVELS = (2, 4, 5) +DEFAULT_INTERVAL_MINUTES = 30 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Submit full-cohort all-mask tiling scans at a fixed interval." + ) + parser.add_argument( + "--levels", + nargs="+", + type=int, + choices=range(6), + default=DEFAULT_LEVELS, + ) + parser.add_argument( + "--interval-minutes", + type=float, + default=DEFAULT_INTERVAL_MINUTES, + ) + return parser.parse_args() + + +def submit_level(level: int) -> None: + experiment = f"preprocessing/tiling/threshold_scan/mou_{level}_224" + job_name = f"mammaprint-tiling-all-masks-l{level}-threshold-scan" + ray_env = [ + f"export RAY_NUM_CPUS={CPU}", + f"export RAY_MEMORY_BYTES={parse_size(MEMORY, binary=True)}", + ] + + print(f"Submitting {job_name}.", flush=True) + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=CPU, + memory=MEMORY, + shm=SHM, + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + *ray_env, + "uv sync --frozen", + f"uv run -m preprocessing.tiling +experiment={experiment}", + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + + +def main() -> None: + args = parse_args() + interval_seconds = args.interval_minutes * 60 + + for index, level in enumerate(args.levels): + if index: + print( + f"Waiting {args.interval_minutes:g} minutes before the next submission.", + flush=True, + ) + time.sleep(interval_seconds) + submit_level(level) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_tissue_masks.py b/scripts/preprocessing/run_tissue_masks.py index ad149d49..7c803de1 100644 --- a/scripts/preprocessing/run_tissue_masks.py +++ b/scripts/preprocessing/run_tissue_masks.py @@ -10,10 +10,11 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", - "uv run -m preprocessing.tissue_masks", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + "uv run python -m preprocessing.tissue_masks", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], ) diff --git a/tests/preprocessing/__init__.py b/tests/preprocessing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/preprocessing/test_filter_embeddings.py b/tests/preprocessing/test_filter_embeddings.py new file mode 100644 index 00000000..d9bb4318 --- /dev/null +++ b/tests/preprocessing/test_filter_embeddings.py @@ -0,0 +1,83 @@ +import json +import logging + +import pandas as pd + +from preprocessing.filter_embeddings import ( + _load_filter_state, + _prepare_output_dirs, + _strategies, + _validate_output_dirs, + filter_embeddings, +) + + +def test_filter_embeddings_materialises_only_selected_strategies(tmp_path): + embeddings_dir = tmp_path / "source" + embeddings_dir.mkdir() + pd.DataFrame( + { + "x": [0, 1, 2], + "y": [10, 11, 12], + "embedding": [[1.0], [2.0], [3.0]], + } + ).to_parquet(embeddings_dir / "slide.parquet", index=False) + scores = pd.DataFrame( + { + "x": [0, 1, 2], + "y": [10, 11, 12], + "epithelium_overlap": [0.3, 0.1, 0.4], + "cancer_overlap": [0.1, 0.8, 0.9], + } + ) + strategies = _strategies(0.25, 0.5, ("or", "and")) + output_dirs = _prepare_output_dirs(tmp_path / "outputs", strategies) + + counts, source_tiles = filter_embeddings( + scores_by_stem={"slide": scores}, + embeddings_dir=embeddings_dir, + output_dirs=output_dirs, + split_map={"slide": "train"}, + epithelium_threshold=0.25, + cancer_threshold=0.5, + logger=logging.getLogger(__name__), + ) + + assert source_tiles == 3 + assert counts["or"]["tiles"] == 3 + assert counts["and"]["tiles"] == 1 + assert set(output_dirs) == {"or", "and"} + assert len(pd.read_parquet(output_dirs["or"]["all"] / "slide.parquet")) == 3 + assert len(pd.read_parquet(output_dirs["and"]["all"] / "slide.parquet")) == 1 + _validate_output_dirs(output_dirs) + + +def test_upload_only_accepts_subset_of_saved_strategies(tmp_path): + state_path = tmp_path / "filter_state.json" + state_path.write_text( + json.dumps( + { + "level": 2, + "tiling_uri": "tiling", + "embeddings_uri": "embeddings", + "strategies": ["or", "and"], + "epithelium_threshold": 0.25, + "cancer_threshold": 0.5, + "counts": {}, + "scored_tiles": 3, + "source_embedding_tiles": 3, + } + ) + ) + + state = _load_filter_state( + state_path, + level=2, + tiling_uri="tiling", + embeddings_uri="embeddings", + strategies=_strategies(0.25, 0.5, ("and",)), + epithelium_threshold=0.25, + cancer_threshold=0.5, + ) + + assert state["strategies"] == ["or", "and"] diff --git a/tests/preprocessing/test_recover_embedding_splits.py b/tests/preprocessing/test_recover_embedding_splits.py new file mode 100644 index 00000000..c72a5522 --- /dev/null +++ b/tests/preprocessing/test_recover_embedding_splits.py @@ -0,0 +1,50 @@ +import pytest + +from preprocessing.recover_embedding_splits import ( + ArtifactFile, + _build_recovery_plan, +) + + +def _file(path: str, size: int = 10) -> ArtifactFile: + return ArtifactFile(path=path, size=size) + + +def test_recovery_plan_reuses_valid_files_and_finds_missing_files(): + source = { + "train.parquet": _file("embeddings/train.parquet", 10), + "val.parquet": _file("embeddings/val.parquet", 20), + "test.parquet": _file("embeddings/test.parquet", 30), + } + destinations = { + "train": { + "train.parquet": _file("embeddings_train/train.parquet", 10), + }, + "val": {}, + "test": {}, + } + + plan = _build_recovery_plan( + source, + destinations, + {"train": "train", "val": "val", "test": "test"}, + ) + + assert plan.missing_files == 2 + assert plan.missing_bytes == 50 + assert set(plan.missing["val"]) == {"val.parquet"} + assert set(plan.missing["test"]) == {"test.parquet"} + + +def test_recovery_plan_rejects_size_mismatch(): + source = {"slide.parquet": _file("embeddings/slide.parquet", 10)} + destinations = { + "train": { + "slide.parquet": _file("embeddings_train/slide.parquet", 9), + }, + "val": {}, + "test": {}, + } + + with pytest.raises(ValueError, match="size-mismatched"): + _build_recovery_plan(source, destinations, {"slide": "train"}) diff --git a/tests/test_run_train_sweep.py b/tests/test_run_train_sweep.py new file mode 100644 index 00000000..05f25fbf --- /dev/null +++ b/tests/test_run_train_sweep.py @@ -0,0 +1,194 @@ +import io +import unittest +from collections import Counter +from contextlib import redirect_stdout +from unittest.mock import patch + +from scripts.ml import run_train_sweep as sweep + + +def job_name(job: sweep.SweepJob) -> str: + return sweep._job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + + +class QueueSelectionTests(unittest.TestCase): + def setUp(self) -> None: + self.jobs = sweep._sweep_jobs() + + def test_empty_queues_fill_to_each_target(self) -> None: + selected, deferred = sweep._jobs_to_submit(self.jobs, {}, None) + self.assertEqual(deferred, 0) + self.assertEqual( + Counter(resources["gpu"] for _, resources in selected), + {"A40": 2, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + ) + self.assertEqual(len({job_name(job) for job, _ in selected}), 7) + + def test_existing_queues_only_fill_deficits(self) -> None: + waiting = {"A40": 1, "H100": 3, "mig-2g.20gb": 2, "mig-1g.10gb": 0} + selected, _ = sweep._jobs_to_submit(self.jobs, waiting, None) + self.assertEqual( + Counter(resources["gpu"] for _, resources in selected), + {"A40": 1, "mig-1g.10gb": 2}, + ) + self.assertEqual(waiting["A40"], 1) + + def test_full_queues_submit_nothing(self) -> None: + selected, _ = sweep._jobs_to_submit(self.jobs, sweep.WAITING_JOB_TARGETS, None) + self.assertEqual(selected, []) + + def test_restricted_jobs_wait_for_h100_capacity(self) -> None: + restricted = [ + job for job in self.jobs if job[3] in (2, 3) and job[6] == "transformer" + ] + selected, _ = sweep._jobs_to_submit(restricted, {"H100": 1}, None) + self.assertEqual(selected, []) + selected, _ = sweep._jobs_to_submit(restricted, {}, None) + self.assertEqual(len(selected), 1) + self.assertEqual(selected[0][1]["gpu"], "H100") + self.assertEqual(selected[0][1]["memory"], "56Gi") + + def test_level_four_transformers_do_not_use_mig10(self) -> None: + restricted = [ + job for job in self.jobs if job[3] == 4 and job[6] == "transformer" + ] + selected, _ = sweep._jobs_to_submit( + restricted, {"A40": 2, "H100": 1, "mig-2g.20gb": 2}, None + ) + self.assertEqual(selected, []) + + def test_unconfigured_h100_jobs_are_deferred(self) -> None: + restricted = [ + job for job in self.jobs if job[3] in (2, 3) and job[6] == "transformer" + ] + with patch.object(sweep, "GPUS", ("A40",)): + selected, deferred = sweep._jobs_to_submit(restricted, {}, None) + self.assertEqual(selected, []) + self.assertEqual(deferred, len(restricted)) + + def test_submission_limit_caps_the_refill(self) -> None: + selected, _ = sweep._jobs_to_submit(self.jobs, {}, 3) + self.assertEqual(len(selected), 3) + + def test_resource_and_gpu_policies_for_entire_grid(self) -> None: + self.assertEqual(len(self.jobs), 11_520) + counts = dict.fromkeys(sweep.GPUS, 0) + for job in self.jobs: + resources = sweep._resources_for_job(counts, *job[1:]) + self.assertIsNotNone(resources) + assert resources is not None + self.assertEqual(resources["cpu"], 8) + if job[3] in (2, 3): + if job[6] == "transformer": + self.assertEqual(resources["gpu"], "H100") + self.assertEqual(resources["memory"], "56Gi") + elif job[6] == "attention": + self.assertEqual(resources["memory"], "40Gi") + else: + self.assertEqual(resources["memory"], "20Gi") + else: + self.assertEqual(resources["memory"], "16Gi") + if job[3] == 4 and job[6] == "transformer": + self.assertNotEqual(resources["gpu"], "mig-1g.10gb") + + +class RefillLoopTests(unittest.TestCase): + def setUp(self) -> None: + self.jobs = sweep._sweep_jobs()[:3] + self.stdout = io.StringIO() + self.enterContext(redirect_stdout(self.stdout)) + self.jobs_mock = self.enterContext( + patch.object(sweep, "_sweep_jobs", return_value=self.jobs) + ) + self.kubernetes = self.enterContext( + patch.object(sweep, "_existing_kubernetes_job_names", return_value=set()) + ) + self.mlflow = self.enterContext( + patch.object(sweep, "_existing_mlflow_run_names", return_value=set()) + ) + self.waiting = self.enterContext(patch.object(sweep, "pending_gpu_counts")) + self.submit = self.enterContext(patch.object(sweep, "_submit_sweep_job")) + self.sleep = self.enterContext(patch.object(sweep.time, "sleep")) + + def run_main(self, *arguments: str) -> None: + with patch("sys.argv", ["run_train_sweep.py", *arguments]): + sweep.main() + + def test_refills_again_and_avoids_duplicates_before_mlflow_appears(self) -> None: + self.waiting.side_effect = [ + {"A40": 1, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + {"A40": 0, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + ] + self.run_main("--max-jobs", "3") + submitted = [call.args[0] for call in self.submit.call_args_list] + self.assertEqual(submitted, self.jobs) + self.assertEqual(self.kubernetes.call_count, 2) + self.assertEqual(self.mlflow.call_count, 2) + self.sleep.assert_called_once_with(300) + + def test_refreshes_external_submissions_and_stops_when_complete(self) -> None: + self.waiting.return_value = sweep.WAITING_JOB_TARGETS.copy() + self.kubernetes.side_effect = [set(), {job_name(job) for job in self.jobs}] + self.run_main("--poll-interval-seconds", "17") + self.submit.assert_not_called() + self.sleep.assert_called_once_with(17) + + def test_lookup_failure_waits_before_retrying(self) -> None: + self.mlflow.side_effect = [RuntimeError("MLflow unavailable"), set()] + self.waiting.return_value = {} + self.run_main("--max-jobs", "1") + self.assertEqual(self.submit.call_count, 1) + self.sleep.assert_called_once_with(300) + self.assertIn("Queue refill failed", self.stdout.getvalue()) + + def test_partial_submission_failure_refreshes_before_retrying(self) -> None: + self.waiting.return_value = {} + self.submit.side_effect = [None, RuntimeError("connection lost"), None] + # The failed call actually created the job; the next poll must skip it. + self.kubernetes.side_effect = [set(), {job_name(self.jobs[1])}] + self.run_main("--max-jobs", "2") + self.assertEqual( + [call.args[0] for call in self.submit.call_args_list], self.jobs + ) + self.sleep.assert_called_once_with(300) + + def test_interrupt_stops_cleanly(self) -> None: + self.waiting.return_value = sweep.WAITING_JOB_TARGETS.copy() + self.sleep.side_effect = KeyboardInterrupt + self.run_main() + self.submit.assert_not_called() + self.assertIn("Sweep queue refiller stopped", self.stdout.getvalue()) + + +class SubmissionConfirmationTests(unittest.TestCase): + def test_unconfirmed_submission_raises(self) -> None: + job = sweep._sweep_jobs()[0] + with ( + patch.object(sweep, "submit_job") as submit, + patch.object(sweep, "_existing_kubernetes_job_names", return_value=set()), + redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "did not confirm"), + ): + sweep._submit_sweep_job(job, {"gpu": "A40", "cpu": 8, "memory": "20Gi"}) + submit.assert_called_once() + + def test_confirmed_submission_preserves_training_parameters(self) -> None: + job = sweep._sweep_jobs()[0] + with ( + patch.object(sweep, "submit_job") as submit, + patch.object( + sweep, "_existing_kubernetes_job_names", return_value={job_name(job)} + ) as lookup, + redirect_stdout(io.StringIO()), + ): + sweep._submit_sweep_job(job, {"gpu": "A40", "cpu": 8, "memory": "20Gi"}) + lookup.assert_called_once_with({job_name(job)}) + self.assertEqual(submit.call_args.kwargs["job_name"], job_name(job)) + self.assertEqual(submit.call_args.kwargs["gpu"], "A40") + self.assertIn( + "data/embedded=tissue_only/l2", submit.call_args.kwargs["script"][-1] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 2c6c4a7a..baa11a1b 100644 --- a/uv.lock +++ b/uv.lock @@ -711,7 +711,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -2019,8 +2018,8 @@ dependencies = [ [[package]] name = "rationai-mlkit" -version = "0.4.0" -source = { git = "https://github.com/RationAI/mlkit.git#55314049191dbfd43b21074ba2a3b6e7f6612647" } +version = "0.4.1" +source = { git = "https://github.com/RationAI/mlkit.git#db38501bb363cd3d76c6a3d31bec702f16002dd1" } dependencies = [ { name = "datasets" }, { name = "gitpython" },