Feat/tiling values

configs/data/_spatial_template.yaml

@@ -0,0 +1,28 @@
+# @package _global_
+# Coordinate-aware single-scale data shape: identical to _template but the DataModule
+# runs over SpatialScaleDataset, which appends each tile's level-0 (x, y) as the last
+# two bag columns ((N, D + 2)). Pair with the spatial_transformer aggregator, which
+# slices those columns off. Select one artifact card at launch, e.g.
+# data/embedded=demo_l3_embed.
+dataset:
+  levels: ???            # filled by the selected data/embedded/* card
+  data_mapping: ${project_path}/data_mapping.csv
+
+datamodule:
+  _target_: ml.data.DataModule
+  batch_size: 1
+  num_workers: 0
+  pin_memory: false
+  persistent_workers: false
+  train: &dataset
+    _target_: ml.data.SpatialScaleDataset
+    split: train
+    levels: ${dataset.levels}
+    data_mapping: ${dataset.data_mapping}
+    label_mode: ${label_mode}
+  val:
+    <<: *dataset
+    split: val
+  test:
+    <<: *dataset
+    split: test

configs/data/_template.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+dataset:
+  levels: ???
+  data_mapping: ${project_path}/data_mapping.csv
+
+datamodule:
+  _target_: ml.data.DataModule
+  batch_size: 1
+  num_workers: 0
+  pin_memory: false
+  persistent_workers: false
+  train: &dataset
+    _target_: ml.data.SingleScaleDataset
+    split: train
+    levels: ${dataset.levels}
+    data_mapping: ${dataset.data_mapping}
+    label_mode: ${label_mode}
+  val:
+    <<: *dataset
+    split: val
+  test:
+    <<: *dataset
+    split: test

configs/data/_template_raw.yaml

@@ -0,0 +1,26 @@
+# @package _global_
+dataset:
+  levels: ???
+  data_mapping: ${project_path}/data_mapping.csv
+  max_tiles: 256
+
+datamodule:
+  _target_: ml.data.DataModule
+  batch_size: 1
+  num_workers: 0
+  pin_memory: false
+  persistent_workers: false
+  train: &dataset
+    _target_: ml.data.RawTileSlideDataset
+    split: train
+    levels: ${dataset.levels}
+    data_mapping: ${dataset.data_mapping}
+    label_mode: ${label_mode}
+    max_tiles: ${dataset.max_tiles}
+    seed: ${seed}
+  val:
+    <<: *dataset
+    split: val
+  test:
+    <<: *dataset
+    split: test

configs/data/default.yaml

@@ -4,7 +4,8 @@ dataset:
   name: full_dataset
   paths:
     qc_masks: ${project_path}/qc_masks
+    epithelium_masks: ${project_path}/epithelium_onnx_masks
     data_mapping: ${project_path}/data_mapping.csv
   mlflow_uris:
-    tissue_masks: mlflow-artifacts:/98/aa6301ad8ae341ab935ab5325cde9369/artifacts/tissue_masks
-    epithelium_masks: null
+    tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks
+    cancer_masks: mlflow-artifacts:/2/70b1c915966d43e3b5be46668075bd93/artifacts/heatmaps

configs/data/embedded/cancer_mask_5/l2.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.2
+# Level-2 embeddings selected with a cancer-probability threshold of 0.5.
+# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the
+# split placeholders after running scripts/preprocessing/split_dataset.py.
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_cancer_mask_5_embed
+uris:
+  all: "mlflow-artifacts:/3/ac2fdbe10e6047a99186e16ac149825c/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/b88229631c214c8e8d1161ce5f2ef23b/artifacts/l2_cancer_mask_5_embed_train"
+  val: "mlflow-artifacts:/3/d47d498ad0524ad2b14a140779a66e4f/artifacts/l2_cancer_mask_5_embed_val"
+  test: "mlflow-artifacts:/3/b1e75bf1510a4dc581d1c60ae4a061c5/artifacts/l2_cancer_mask_5_embed_test"

configs/data/embedded/cancer_mask_5/l3.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.3
+# Level-3 embeddings selected with a cancer-probability threshold of 0.5.
+# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the
+# split placeholders after running scripts/preprocessing/split_dataset.py.
+defaults:
+  - /data/levels/l3@_here_
+  - _self_
+
+name: l3_cancer_mask_5_embed
+uris:
+  all: "mlflow-artifacts:/3/1c71b92b718642efb3698e6c5c5e7436/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/1cbd1a158b6f44c6924608c8b99b8c7b/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/c6680ed10e66455fa06e61b367f44ae0/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/48ffa80398c2468cba34738a54adc84e/artifacts/embeddings_test"

configs/data/embedded/cancer_mask_5/l4.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.4
+# Level-4 embeddings selected with a cancer-probability threshold of 0.5.
+# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the
+# split placeholders after running scripts/preprocessing/split_dataset.py.
+defaults:
+  - /data/levels/l4@_here_
+  - _self_
+
+name: l4_cancer_mask_5_embed
+uris:
+  all: "mlflow-artifacts:/3/3492b1dae7be474f80cea02afe37d252/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/2e018211e7bb4c299a3068678e798872/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/bb56dcfc27f64c62900e0413e881402a/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/c6ed24afdc894c8bab90a160b9d96211/artifacts/embeddings_test"

configs/data/embedded/cancer_mask_5/l5.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.5
+# Level-5 embeddings selected with a cancer-probability threshold of 0.5.
+# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the
+# split artifacts with scripts/preprocessing/split_dataset.py before training.
+defaults:
+  - /data/levels/l5@_here_
+  - _self_
+
+name: l5_cancer_mask_5_embed
+uris:
+  all: "mlflow-artifacts:/3/d2a3870bbe784ded95a878998513f8fd/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/4d2342aed93d4686b1a5a464d47fc742/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/9b3c03fdc4844405b9df7877020fad70/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/7c99af8a7d2d4728bfa10c129bcdf862/artifacts/embeddings_test"

configs/data/embedded/cancer_mask_9/l2.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.2
+# Level-2 embeddings selected with a cancer-probability threshold of 0.9.
+# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the
+# split placeholders after running scripts/preprocessing/split_dataset.py.
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_cancer_mask_9_embed
+uris:
+  all: "mlflow-artifacts:/3/3f9706ef68f6434f89ed97d9031a8a6a/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/e6e63894393440a49e809b826014606c/artifacts/l2_cancer_mask_9_embed_train"
+  val: "mlflow-artifacts:/3/5a487183ee21496b95a7d784190d4ee3/artifacts/l2_cancer_mask_9_embed_val"
+  test: "mlflow-artifacts:/3/e36817eb35de4d46a061ee8a8d8a5b7b/artifacts/l2_cancer_mask_9_embed_test"

configs/data/embedded/cancer_mask_9/l3.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.3
+# Level-3 embeddings selected with a cancer-probability threshold of 0.9.
+# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the
+# split placeholders after running scripts/preprocessing/split_dataset.py.
+defaults:
+  - /data/levels/l3@_here_
+  - _self_
+
+name: l3_cancer_mask_9_embed
+uris:
+  all: "mlflow-artifacts:/3/709170ca84d64c21bc37c487942ec48f/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/a3af7a4b837248b18ca8b387b4262f9a/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/70d7ac7c2d154291857d47ed7def18df/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/e5e5ef2e3b1b4a35a9ebb4b04dfdde14/artifacts/embeddings_test"

configs/data/embedded/cancer_mask_9/l4.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.4
+# Level-4 embeddings selected with a cancer-probability threshold of 0.9.
+# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the
+# split artifacts with scripts/preprocessing/split_dataset.py before training.
+defaults:
+  - /data/levels/l4@_here_
+  - _self_
+
+name: l4_cancer_mask_9_embed
+uris:
+  all: "mlflow-artifacts:/3/84dda274e69c4eb8adc3a875eec8e482/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/a0367e263d6a4bf7a70a5c8b3316c7c1/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/d1eda30624384c8eae75edf4ec07a508/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/8798109c8ec640079b2bb778590b0bde/artifacts/embeddings_test"

configs/data/embedded/cancer_mask_9/l5.yaml

@@ -0,0 +1,14 @@
+# @package dataset.levels.5
+# Level-5 embeddings selected with a cancer-probability threshold of 0.9.
+# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the
+# split artifacts with scripts/preprocessing/split_dataset.py before training.
+defaults:
+  - /data/levels/l5@_here_
+  - _self_
+
+name: l5_cancer_mask_9_embed
+uris:
+  all: "mlflow-artifacts:/3/3bdfcabbdf884380b3ece1e0f37bf3b7/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/146e36cdab604071a47e25a2d2720b36/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/b4305a97ff34482b8210b31108c99f31/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/d89478ba8003457080794fe147b4f6f0/artifacts/embeddings_test"

configs/data/embedded/epithel/l2.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.2
+# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25.
+# The whole dataset and all three predefined splits are stored in one MLflow run.
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_epithel_025_embed
+uris:
+  all: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_test"

configs/data/embedded/epithel/l3.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.3
+# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25.
+# The whole dataset and all three predefined splits are stored in one MLflow run.
+defaults:
+  - /data/levels/l3@_here_
+  - _self_
+
+name: l3_epithel_025_embed
+uris:
+  all: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_test"

configs/data/embedded/epithel/l4.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.4
+# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25.
+# The whole dataset and all three predefined splits are stored in one MLflow run.
+defaults:
+  - /data/levels/l4@_here_
+  - _self_
+
+name: l4_epithel_025_embed
+uris:
+  all: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_test"

configs/data/embedded/epithel/l5.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.5
+# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25.
+# The whole dataset and all three predefined splits are stored in one MLflow run.
+defaults:
+  - /data/levels/l5@_here_
+  - _self_
+
+name: l5_epithel_025_embed
+uris:
+  all: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_test"

configs/data/embedded/epithel_and_cancer_5/l2.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.2
+# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# AND cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_epithel_and_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_test"

configs/data/embedded/epithel_and_cancer_5/l3.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.3
+# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# AND cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l3@_here_
+  - _self_
+
+name: l3_epithel_and_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_test"

configs/data/embedded/epithel_and_cancer_5/l4.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.4
+# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# AND cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l4@_here_
+  - _self_
+
+name: l4_epithel_and_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_test"

configs/data/embedded/epithel_and_cancer_5/l5.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.5
+# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# AND cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l5@_here_
+  - _self_
+
+name: l5_epithel_and_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_test"

configs/data/embedded/epithel_or_cancer_5/l2.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.2
+# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# OR cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_epithel_or_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_test"

configs/data/embedded/epithel_or_cancer_5/l3.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.3
+# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# OR cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l3@_here_
+  - _self_
+
+name: l3_epithel_or_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_test"

configs/data/embedded/epithel_or_cancer_5/l4.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.4
+# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# OR cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l4@_here_
+  - _self_
+
+name: l4_epithel_or_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_test"

configs/data/embedded/epithel_or_cancer_5/l5.yaml

@@ -0,0 +1,13 @@
+# @package dataset.levels.5
+# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25
+# OR cancer probability is greater than 0.5.
+defaults:
+  - /data/levels/l5@_here_
+  - _self_
+
+name: l5_epithel_or_cancer_025_05_embed
+uris:
+  all: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_train"
+  val: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_val"
+  test: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_test"

configs/data/embedded/tissue_only/l2.yaml

@@ -0,0 +1,17 @@
+# @package dataset.levels.2
+# Level-2 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features.
+# Physical facts come from /data/levels/l2; only name + URIs live here.
+# `uris.all` is the whole (un-split) embeddings artifact written by the
+# embeddings step. Per-split URIs are produced from it by:
+#   python scripts/preprocessing/split_dataset.py --kind embedded \
+#     --uri <the `all` URI below> --name l2_tissue_embed
+defaults:
+  - /data/levels/l2@_here_
+  - _self_
+
+name: l2_tissue_embed
+uris:
+  all: "mlflow-artifacts:/3/d70b72d842604ebe93d3fbd9e1dfd2f5/artifacts/embeddings"
+  train: "mlflow-artifacts:/3/b11db2e5ec074e1da3c172ea41880c5c/artifacts/l2_tissue_only_mask_embed_train"
+  val: "mlflow-artifacts:/3/8134ea19af804e8aaf51680157575cf2/artifacts/l2_tissue_only_mask_embed_val"
+  test: "mlflow-artifacts:/3/d8479367bf224405b3a77bbc7b7ac289/artifacts/l2_tissue_only_mask_embed_test"