From 4453a23ddc417f34af4a573714a8028cf99491ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 12:49:13 +0200 Subject: [PATCH 01/79] feat: aggregators --- ml/models/aggregators/__init__.py | 18 ++++++++++ ml/models/aggregators/base.py | 57 +++++++++++++++++++++++++++++++ ml/models/aggregators/mean.py | 30 ++++++++++++++++ ml/typing.py | 39 +++++++++++++++++++++ 4 files changed, 144 insertions(+) create mode 100644 ml/models/aggregators/__init__.py create mode 100644 ml/models/aggregators/base.py create mode 100644 ml/models/aggregators/mean.py diff --git a/ml/models/aggregators/__init__.py b/ml/models/aggregators/__init__.py new file mode 100644 index 00000000..0c9b77d1 --- /dev/null +++ b/ml/models/aggregators/__init__.py @@ -0,0 +1,18 @@ +"""Bag aggregators (MIL pooling): mean, max, gated attention, transformer.""" + +# from ml.models.aggregators.attention import AttentionMIL +from ml.models.aggregators.base import Aggregator +# from ml.models.aggregators.max import MaxPool +from ml.models.aggregators.mean import MeanPool +# from ml.models.aggregators.multiscale import MultiScaleMIL +# from ml.models.aggregators.transformer import TransformerMIL + + +__all__ = [ + "Aggregator", + # "AttentionMIL", + # "MaxPool", + "MeanPool", + # "MultiScaleMIL", + # "TransformerMIL", +] diff --git a/ml/models/aggregators/base.py b/ml/models/aggregators/base.py new file mode 100644 index 00000000..54ae9fa7 --- /dev/null +++ b/ml/models/aggregators/base.py @@ -0,0 +1,57 @@ +"""Aggregator abstract base class. + +An aggregator reduces one slide's tiles to a single slide-level vector ``(D',)``. +This is the multiple-instance-learning (MIL) pooling step; swapping the aggregator +(mean, max, gated attention, transformer, multi-scale) is the primary experimental +axis for the training skeleton. + +The input is either a **flat bag** ``Tensor[N, D]`` (single level) or a +**multi-scale bag** ``list[Region]`` where each region maps ``level -> Tensor[K, D]`` +(tiles across pyramid levels aligned by footprint). Flat aggregators +(mean/max/attention) accept the tensor; :class:`~ml.models.aggregators.multiscale.MultiScaleMIL` +accepts the region list. The output contract is identical, so the module and head +are agnostic to which one is in use. + +Concrete aggregators live in sibling modules, one implementation per file, and +are selected via the ``ml/aggregator`` Hydra config group. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + +from ml.typing import Bag, MultiScaleBag + + +class Aggregator[InputT: (Bag, MultiScaleBag)](nn.Module, ABC): + """Reduces one slide's tiles to a single slide vector. + + Generic over its input type: flat aggregators specialise as ``Aggregator[Bag]`` + (a tile tensor ``(N, D)``), and multi-scale aggregators as + ``Aggregator[MultiScaleBag]`` (a list of aligned regions). Implementations + return the pooled vector and, optionally, weights (``None`` for parameter-free + poolings such as mean/max), surfaced for interpretability (e.g. heatmaps) and + not required by the training loop. + """ + + @property + @abstractmethod + def out_dim(self) -> int: + """Dimensionality of the aggregated slide vector.""" + + @abstractmethod + def forward(self, bag: InputT) -> tuple[Tensor, Tensor | None]: + """Aggregate one slide's tiles. + + Args: + bag: A flat per-tile feature tensor ``(N, D)`` (``Aggregator[Bag]``) or + a multi-scale bag of regions (``Aggregator[MultiScaleBag]``). + + Returns: + A tuple ``(pooled, attention)`` where ``pooled`` has shape + ``(out_dim,)`` and ``attention`` has shape ``(N,)`` / ``(num_regions,)`` + or is ``None`` if the aggregator assigns no weights. + """ + + +__all__ = ["Aggregator"] diff --git a/ml/models/aggregators/mean.py b/ml/models/aggregators/mean.py new file mode 100644 index 00000000..ba339f58 --- /dev/null +++ b/ml/models/aggregators/mean.py @@ -0,0 +1,30 @@ +"""Mean-pooling aggregator.""" + +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class MeanPool(Aggregator[Bag]): + """Averages tile features across the bag. + + Parameter-free; assigns no per-tile weights. + + Args: + feature_dim: Dimensionality of the incoming tile features. + """ + + def __init__(self, feature_dim: int) -> None: + super().__init__() + self.feature_dim = feature_dim + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, None]: + return bag.mean(dim=0), None + + +__all__ = ["MeanPool"] diff --git a/ml/typing.py b/ml/typing.py index c356da25..c984b7e9 100644 --- a/ml/typing.py +++ b/ml/typing.py @@ -12,3 +12,42 @@ class TileMetadata(TypedDict): type TilesPredictSample = tuple[Tensor, TileMetadata] + + +class SlideMetadata(TypedDict): + """Metadata for a slide-level MIL sample.""" + + slide_id: str + + +class LevelSpec(TypedDict): + """One pyramid level's data card: physical facts + artifact location. + + Authored once per preprocessing artifact (a ``configs/data/tiled/*.yaml`` card). + ``uris`` points at the embedding artifact; ``raw_uris`` (optional) at the + raw-tile artifact for the OpenSlide path. + """ + + mpp: float + tile_extent: int + uris: str + + +type Bag = Tensor +"""A bag of per-slide items: tile features ``(N, D)`` or raw tiles ``(N, C, H, W)``.""" + +type Region = dict[int, Tensor] +"""One aligned multi-scale region: level -> that level's tiles ``(K_level, D)``. + +For example ``{3: (1, D), 2: (4, D)}`` is one coarse level-3 tile with the four +level-2 tiles that zoom into the same footprint. +""" + +type MultiScaleBag = list[Region] +"""A slide as a list of aligned multi-scale regions (variable length).""" + +type MILSample = tuple[Bag, Tensor, SlideMetadata] +"""A single-level slide-level training sample: ``(bag, label, metadata)``.""" + +type MultiScaleSample = tuple[MultiScaleBag, Tensor, SlideMetadata] +"""A multi-scale slide-level training sample: ``(regions, label, metadata)``.""" From d11e858d02cf5de68ba53293cc0bd8d14b94bbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 12:51:00 +0200 Subject: [PATCH 02/79] feat: head --- configs/ml/head/linear.yaml | 4 ++++ ml/models/heads/__init__.py | 7 +++++++ ml/models/heads/base.py | 41 +++++++++++++++++++++++++++++++++++++ ml/models/heads/linear.py | 34 ++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 configs/ml/head/linear.yaml create mode 100644 ml/models/heads/__init__.py create mode 100644 ml/models/heads/base.py create mode 100644 ml/models/heads/linear.py diff --git a/configs/ml/head/linear.yaml b/configs/ml/head/linear.yaml new file mode 100644 index 00000000..0597b20f --- /dev/null +++ b/configs/ml/head/linear.yaml @@ -0,0 +1,4 @@ +_target_: ml.models.heads.LinearHead +in_dim: ${feature_dim} +out_dim: ${num_classes} +dropout: 0.5 diff --git a/ml/models/heads/__init__.py b/ml/models/heads/__init__.py new file mode 100644 index 00000000..1cab67ef --- /dev/null +++ b/ml/models/heads/__init__.py @@ -0,0 +1,7 @@ +"""Prediction heads ("hats").""" + +from ml.models.heads.base import Head +from ml.models.heads.linear import LinearHead + + +__all__ = ["Head", "LinearHead"] diff --git a/ml/models/heads/base.py b/ml/models/heads/base.py new file mode 100644 index 00000000..48ea7b22 --- /dev/null +++ b/ml/models/heads/base.py @@ -0,0 +1,41 @@ +"""Head ("hat") abstract base class. + +A head maps a slide-level vector to the task prediction — logits for +classification or a scalar for regression. Heads are swappable via the +``ml/head`` Hydra config group; the same aggregated representation can feed +different heads. + +Concrete heads live in sibling modules, one implementation per file. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + + +class Head(nn.Module, ABC): + """Maps a slide vector ``(in_dim,)`` to a prediction ``(out_dim,)``. + + ``out_dim`` encodes the task: ``1`` for binary classification (single logit) + or regression (scalar), ``C`` for ``C``-class multiclass classification. + """ + + def __init__(self, in_dim: int, out_dim: int) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + @abstractmethod + def forward(self, features: Tensor) -> Tensor: + """Predict from the slide-level features. + + Args: + features: Aggregated slide vector, shape ``(in_dim,)`` or a batch + ``(B, in_dim)``. + + Returns: + Prediction of shape ``(out_dim,)`` (or ``(B, out_dim)``). + """ + + +__all__ = ["Head"] diff --git a/ml/models/heads/linear.py b/ml/models/heads/linear.py new file mode 100644 index 00000000..d58c6804 --- /dev/null +++ b/ml/models/heads/linear.py @@ -0,0 +1,34 @@ +"""Linear prediction head.""" + +from torch import Tensor, nn + +from ml.models.heads.base import Head + + +class LinearHead(Head): + """Dropout followed by a single linear layer. + + Works for binary classification / regression (``out_dim=1``) and multiclass + classification (``out_dim=C``). The output is raw (logits / unbounded + scalar); any activation is applied by the module's ``output_activation``. + + Args: + in_dim: Dimensionality of the incoming slide vector. + out_dim: Number of outputs (``1`` for binary/regression, ``C`` for + multiclass). + dropout: Dropout probability applied before the linear layer. + """ + + def __init__(self, in_dim: int, out_dim: int = 1, dropout: float = 0.5) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim) + self.dropout = nn.Dropout(p=dropout) + self.linear = nn.Linear(in_dim, out_dim) + + nn.init.xavier_uniform_(self.linear.weight) + nn.init.zeros_(self.linear.bias) + + def forward(self, features: Tensor) -> Tensor: + return self.linear(self.dropout(features)) + + +__all__ = ["LinearHead"] From cb2ae216d60aa38df796c8e85e89a4c27b22e68e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 12:52:14 +0200 Subject: [PATCH 03/79] feat: encoder --- configs/ml/encoder/identity.yaml | 3 +++ ml/models/encoders/__init__.py | 14 ++++++++++ ml/models/encoders/base.py | 45 ++++++++++++++++++++++++++++++++ ml/models/encoders/identity.py | 32 +++++++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 configs/ml/encoder/identity.yaml create mode 100644 ml/models/encoders/__init__.py create mode 100644 ml/models/encoders/base.py create mode 100644 ml/models/encoders/identity.py diff --git a/configs/ml/encoder/identity.yaml b/configs/ml/encoder/identity.yaml new file mode 100644 index 00000000..570e3fd2 --- /dev/null +++ b/configs/ml/encoder/identity.yaml @@ -0,0 +1,3 @@ +# Precomputed-embeddings path: features pass through unchanged. +_target_: ml.models.encoders.IdentityEncoder +out_dim: 2560 # Virchow2 embedding size; 512 for VGG16 global-max-pooled features diff --git a/ml/models/encoders/__init__.py b/ml/models/encoders/__init__.py new file mode 100644 index 00000000..44f2e042 --- /dev/null +++ b/ml/models/encoders/__init__.py @@ -0,0 +1,14 @@ +"""Per-tile encoders: identity (embeddings path) and image backbones.""" + +from ml.models.encoders.base import Encoder +from ml.models.encoders.identity import IdentityEncoder +# from ml.models.encoders.vgg16 import VGG16Encoder +# from ml.models.encoders.virchow2 import Virchow2Encoder + + +__all__ = [ + "Encoder", + "IdentityEncoder", + # "VGG16Encoder", + # "Virchow2Encoder" + ] diff --git a/ml/models/encoders/base.py b/ml/models/encoders/base.py new file mode 100644 index 00000000..c1c3310c --- /dev/null +++ b/ml/models/encoders/base.py @@ -0,0 +1,45 @@ +"""Encoder abstract base class. + +An encoder maps a bag of raw tiles to a bag of per-tile feature vectors. On the +precomputed-embeddings path the encoder is :class:`~ml.models.encoders.identity.IdentityEncoder` +(the features already exist); on the end-to-end path it is an image backbone such +as :class:`~ml.models.encoders.vgg16.VGG16Encoder` applied to every tile. + +Concrete encoders live in sibling modules, one implementation per file, and are +selected via the ``ml/encoder`` Hydra config group. The generic +:class:`~ml.models.module.MammaprintModule` only depends on this contract, never +on a concrete class. +""" + +from abc import ABC, abstractmethod + +from torch import Tensor, nn + + +class Encoder(nn.Module, ABC): + """Maps a bag of tiles ``(N, ...)`` to per-tile features ``(N, out_dim)``. + + Implementations must set :attr:`out_dim` so the downstream aggregator and head + can be wired without hard-coding feature sizes. + """ + + @property + @abstractmethod + def out_dim(self) -> int: + """Dimensionality of a single per-tile feature vector.""" + + @abstractmethod + def forward(self, tiles: Tensor) -> Tensor: + """Encode a bag of tiles. + + Args: + tiles: A bag for one slide. Shape ``(N, C, H, W)`` for image + backbones or ``(N, D)`` for the identity encoder, where ``N`` is + the number of tiles. + + Returns: + Per-tile features of shape ``(N, out_dim)``. + """ + + +__all__ = ["Encoder"] diff --git a/ml/models/encoders/identity.py b/ml/models/encoders/identity.py new file mode 100644 index 00000000..65474abc --- /dev/null +++ b/ml/models/encoders/identity.py @@ -0,0 +1,32 @@ +"""Identity encoder for the precomputed-embeddings path.""" + +from torch import Tensor + +from ml.models.encoders.base import Encoder + + +class IdentityEncoder(Encoder): + """Passes precomputed tile embeddings through unchanged. + + Used when training a head/aggregator on frozen per-slide embedding bags + produced by ``preprocessing.embeddings`` (e.g. Virchow2, 2560-dim). The + feature size is not inferable from a module, so it is provided explicitly. + + Args: + out_dim: Dimensionality of the incoming embeddings (e.g. ``2560`` for + Virchow2, ``512`` for VGG16 global-max-pooled features). + """ + + def __init__(self, out_dim: int) -> None: + super().__init__() + self._out_dim = out_dim + + @property + def out_dim(self) -> int: + return self._out_dim + + def forward(self, tiles: Tensor) -> Tensor: + return tiles + + +__all__ = ["IdentityEncoder"] From 909eb86bcad82501137acb0f82ceb5b1566580a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 12:54:04 +0200 Subject: [PATCH 04/79] feat: config related --- configs/ml/loss/ce.yaml | 2 ++ configs/ml/loss/mse.yaml | 2 ++ configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml | 7 +++++++ configs/ml/metrics/mse.yaml | 3 +++ configs/ml/metrics/multiclass_auroc.yaml | 4 ++++ configs/ml/optimizer/adam.yaml | 5 +++++ configs/ml/output_activation/identity.yaml | 1 + configs/ml/output_activation/softmax.yaml | 2 ++ 8 files changed, 26 insertions(+) create mode 100644 configs/ml/loss/ce.yaml create mode 100644 configs/ml/loss/mse.yaml create mode 100644 configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml create mode 100644 configs/ml/metrics/mse.yaml create mode 100644 configs/ml/metrics/multiclass_auroc.yaml create mode 100644 configs/ml/optimizer/adam.yaml create mode 100644 configs/ml/output_activation/identity.yaml create mode 100644 configs/ml/output_activation/softmax.yaml diff --git a/configs/ml/loss/ce.yaml b/configs/ml/loss/ce.yaml new file mode 100644 index 00000000..127bf997 --- /dev/null +++ b/configs/ml/loss/ce.yaml @@ -0,0 +1,2 @@ +# Multiclass / binary-as-2-class classification. Expects raw logits. +_target_: torch.nn.CrossEntropyLoss diff --git a/configs/ml/loss/mse.yaml b/configs/ml/loss/mse.yaml new file mode 100644 index 00000000..22cd58cc --- /dev/null +++ b/configs/ml/loss/mse.yaml @@ -0,0 +1,2 @@ +# Regression on the MammaPrint index. +_target_: torch.nn.MSELoss diff --git a/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml b/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml new file mode 100644 index 00000000..bb1bc42c --- /dev/null +++ b/configs/ml/lr_scheduler/reduce_lr_on_plateau.yaml @@ -0,0 +1,7 @@ +monitor: valid/loss +scheduler: + _target_: torch.optim.lr_scheduler.ReduceLROnPlateau + _partial_: true + mode: min + factor: 0.5 + patience: 3 diff --git a/configs/ml/metrics/mse.yaml b/configs/ml/metrics/mse.yaml new file mode 100644 index 00000000..84aea1d0 --- /dev/null +++ b/configs/ml/metrics/mse.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +mse: + _target_: torchmetrics.MeanSquaredError diff --git a/configs/ml/metrics/multiclass_auroc.yaml b/configs/ml/metrics/multiclass_auroc.yaml new file mode 100644 index 00000000..02c94bbe --- /dev/null +++ b/configs/ml/metrics/multiclass_auroc.yaml @@ -0,0 +1,4 @@ +# @package ml.metrics +auroc: + _target_: torchmetrics.classification.MulticlassAUROC + num_classes: ${num_classes} diff --git a/configs/ml/optimizer/adam.yaml b/configs/ml/optimizer/adam.yaml new file mode 100644 index 00000000..35fb45e3 --- /dev/null +++ b/configs/ml/optimizer/adam.yaml @@ -0,0 +1,5 @@ +# Bound to model parameters in configure_optimizers (hence _partial_). +_target_: torch.optim.Adam +_partial_: true +lr: 1e-3 +weight_decay: 1e-3 diff --git a/configs/ml/output_activation/identity.yaml b/configs/ml/output_activation/identity.yaml new file mode 100644 index 00000000..d8bfafa3 --- /dev/null +++ b/configs/ml/output_activation/identity.yaml @@ -0,0 +1 @@ +_target_: torch.nn.Identity diff --git a/configs/ml/output_activation/softmax.yaml b/configs/ml/output_activation/softmax.yaml new file mode 100644 index 00000000..e554c8db --- /dev/null +++ b/configs/ml/output_activation/softmax.yaml @@ -0,0 +1,2 @@ +_target_: torch.nn.Softmax +dim: -1 From 343453580c0b1815d4f160d575fa27b3e01f5fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 13:05:27 +0200 Subject: [PATCH 05/79] !!! squash typing --- ml/typing.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ml/typing.py b/ml/typing.py index c984b7e9..a060dcb3 100644 --- a/ml/typing.py +++ b/ml/typing.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import TypedDict from torch import Tensor @@ -21,16 +22,18 @@ class SlideMetadata(TypedDict): class LevelSpec(TypedDict): - """One pyramid level's data card: physical facts + artifact location. + """One pyramid level's data card: physical facts + per-split artifact locations. Authored once per preprocessing artifact (a ``configs/data/tiled/*.yaml`` card). - ``uris`` points at the embedding artifact; ``raw_uris`` (optional) at the - raw-tile artifact for the OpenSlide path. + ``uris`` maps ``split -> embedding artifact URI`` (``train``/``val``/``test``); + ``raw_uris`` (optional) does the same for the raw-tile artifact. Per-split URIs + because the split is materialised into physically separate artifacts by + ``scripts/preprocessing/split_dataset.py``. """ mpp: float tile_extent: int - uris: str + uris: Mapping[str, str] type Bag = Tensor @@ -46,8 +49,14 @@ class LevelSpec(TypedDict): type MultiScaleBag = list[Region] """A slide as a list of aligned multi-scale regions (variable length).""" +type AnyBag = Bag #| MultiScaleBag +"""Either a flat single-level bag or a multi-scale bag.""" + type MILSample = tuple[Bag, Tensor, SlideMetadata] """A single-level slide-level training sample: ``(bag, label, metadata)``.""" type MultiScaleSample = tuple[MultiScaleBag, Tensor, SlideMetadata] """A multi-scale slide-level training sample: ``(regions, label, metadata)``.""" + +type AnySample = MILSample #| MultiScaleSample +"""Either a single-level or a multi-scale training sample.""" From ec6468bd9dc93ed460a772bea3d5ab644c8cb8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 15:47:41 +0200 Subject: [PATCH 06/79] feat: main config --- configs/ml.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/configs/ml.yaml b/configs/ml.yaml index 688e709b..4dcaf0fe 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -1,6 +1,13 @@ - defaults: - base - _self_ + # A +experiment config selects the data source (data/sources/*) and model pieces. + +seed: ${random_seed:} + +# Task shape — set per experiment; consumed by the ml/head + ml/metrics configs. +label_mode: type # "type" (classification) or "index" (regression) +num_classes: 2 +feature_dim: ??? # tile-feature width = encoder.out_dim; set per experiment -seed: ${random_seed:} \ No newline at end of file +# datamodule / ml / trainer are composed by the +experiment config's defaults. \ No newline at end of file From f6add9e5a0600e81ef9dd52f7553f4b7b8ba55a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 15:49:30 +0200 Subject: [PATCH 07/79] fix: rename mammaprint index --- ml/data/datasets/labels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ml/data/datasets/labels.py b/ml/data/datasets/labels.py index dc6ad689..99930f0d 100644 --- a/ml/data/datasets/labels.py +++ b/ml/data/datasets/labels.py @@ -27,7 +27,7 @@ def process_slides(slides: pd.DataFrame, mode: LabelMode | None = None) -> pd.Da slides["type_label"] = slides["type"].map(_map_luminal_type) case LabelMode.INDEX: slides = slides.copy() - slides["index"] = slides["index"].astype(float) + slides["mammaprint_index"] = slides["mammaprint_index"].astype(float) slides["name"] = slides["path"].apply(lambda x: Path(x).stem) return slides @@ -38,7 +38,7 @@ def get_label(slide_metadata: pd.Series, mode: LabelMode) -> torch.Tensor: case LabelMode.TYPE: return torch.tensor(int(slide_metadata["type_label"])).long() case LabelMode.INDEX: - return torch.tensor(float(slide_metadata["index"])).float() + return torch.tensor(float(slide_metadata["mammaprint_index"])).float() raise ValueError(f"Unsupported label mode: {mode}") @@ -48,6 +48,6 @@ def get_target_column(mode: LabelMode) -> str: case LabelMode.TYPE: return "type_label" case LabelMode.INDEX: - return "index" + return "mammaprint_index" raise ValueError(f"Unsupported label mode: {mode}") From 5916e5a2f8585412f3ac87e674cb64fb7bbaf3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 15:56:14 +0200 Subject: [PATCH 08/79] feat: trainer and main models --- configs/ml/module.yaml | 13 +++ configs/trainer/default.yaml | 22 +++++ ml/models/__init__.py | 9 ++ ml/models/module.py | 184 +++++++++++++++++++++++++++++++++++ ml/train.py | 37 +++++++ 5 files changed, 265 insertions(+) create mode 100644 configs/ml/module.yaml create mode 100644 configs/trainer/default.yaml create mode 100644 ml/models/__init__.py create mode 100644 ml/models/module.py create mode 100644 ml/train.py diff --git a/configs/ml/module.yaml b/configs/ml/module.yaml new file mode 100644 index 00000000..926a6237 --- /dev/null +++ b/configs/ml/module.yaml @@ -0,0 +1,13 @@ +_target_: ml.models.MammaprintModule + +# Swappable pieces — populated by the experiment's defaults list +# (ml/encoder, ml/aggregator, ml/head, ml/loss, ml/optimizer, ml/lr_scheduler, +# ml/output_activation, ml/metrics). +encoder: ??? +aggregator: ??? +head: ??? +output_activation: null +loss: null +optimizer: null +lr_scheduler: null +metrics: null diff --git a/configs/trainer/default.yaml b/configs/trainer/default.yaml new file mode 100644 index 00000000..9a9cfe28 --- /dev/null +++ b/configs/trainer/default.yaml @@ -0,0 +1,22 @@ +_target_: rationai.mlkit.lightning.Trainer + +max_epochs: 100 +log_every_n_steps: 1 +accelerator: auto +devices: auto + +callbacks: + early_stopping: + _target_: lightning.pytorch.callbacks.EarlyStopping + monitor: valid/loss + mode: min + patience: 10 + model_checkpoint: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + monitor: valid/loss + mode: min + save_top_k: 1 + filename: best + lr_monitor: + _target_: lightning.pytorch.callbacks.LearningRateMonitor + logging_interval: epoch diff --git a/ml/models/__init__.py b/ml/models/__init__.py new file mode 100644 index 00000000..867124ac --- /dev/null +++ b/ml/models/__init__.py @@ -0,0 +1,9 @@ +"""MammaPrint training models: generic module + swappable pieces.""" + +from ml.models.aggregators import Aggregator +from ml.models.encoders import Encoder +from ml.models.heads import Head +from ml.models.module import MammaprintModule + + +__all__ = ["Aggregator", "Encoder", "Head", "MammaprintModule"] diff --git a/ml/models/module.py b/ml/models/module.py new file mode 100644 index 00000000..481e13b9 --- /dev/null +++ b/ml/models/module.py @@ -0,0 +1,184 @@ +"""Generic MammaPrint LightningModule. + +Wires three independently swappable pieces — an :class:`~ml.models.encoders.base.Encoder`, +an :class:`~ml.models.aggregators.base.Aggregator`, and a :class:`~ml.models.heads.base.Head` — +into a multiple-instance-learning pipeline:: + + bag -> encoder (per tile) -> aggregator (bag -> vector) -> head -> prediction + +The module depends only on the abstract contracts, so encoders (identity for +precomputed embeddings, VGG16/Virchow2 for raw tiles), aggregators (mean, max, +gated attention, transformer) and heads are selected purely via Hydra config. +Loss, optimizer, LR scheduler and metrics are likewise config-instantiated, +following the ``feature/mil`` reference module. +""" + +import functools +import logging +from collections.abc import Sequence +from typing import Any + +import lightning.pytorch as pl +import torch +import torchmetrics +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.models.encoders.base import Encoder +from ml.models.heads.base import Head +from ml.typing import AnyBag, AnySample, SlideMetadata + + +logger = logging.getLogger(__name__) + + +class MammaprintModule(pl.LightningModule): + """MIL module composing encoder -> aggregator -> head. + + Args: + encoder: Per-tile feature extractor. ``IdentityEncoder`` on the + precomputed-embeddings path; an image backbone on the end-to-end path. + aggregator: Bag pooling reducing per-tile features to a slide vector. + head: Prediction head mapping the slide vector to logits / a scalar. + output_activation: Applied to the head output (e.g. ``Sigmoid`` for + binary classification, ``Identity`` for regression / when the loss + expects logits). Defaults to identity. + loss: Loss module. ``None`` disables training (e.g. predict-only). + optimizer: A ``functools.partial`` of an optimizer, bound to the model + parameters in :meth:`configure_optimizers`. + lr_scheduler: Optional dict with a ``scheduler`` partial and an optional + ``monitor`` key, mirroring the ``feature/mil`` convention. + metrics: Torchmetrics keyed by name; cloned per ``train``/``valid``/``test``. + """ + + def __init__( + self, + encoder: Encoder, + aggregator: Aggregator[Any], + head: Head, + output_activation: nn.Module | None = None, + loss: nn.Module | None = None, + optimizer: functools.partial[torch.optim.Optimizer] | None = None, + lr_scheduler: dict[str, Any] | None = None, + metrics: dict[str, torchmetrics.Metric] | None = None, + ) -> None: + super().__init__() + self.encoder = encoder + self.aggregator = aggregator + self.head = head + self.output_activation = output_activation or nn.Identity() + self.loss = loss + self.optimizer = optimizer + self.lr_scheduler = lr_scheduler + + base = torchmetrics.MetricCollection(dict(metrics) if metrics else {}) + self.train_metrics = base.clone(prefix="train/") + self.valid_metrics = base.clone(prefix="valid/") + self.test_metrics = base.clone(prefix="test/") + + def forward(self, bag: AnyBag) -> tuple[Tensor, Tensor | None]: + """Run one slide through encoder -> aggregator -> head. + + Args: + bag: One slide, either a flat tile tensor ``(N, D)``/``(N, C, H, W)`` + (single level) or a multi-scale bag (list of regions, each mapping + ``level -> (K, D)``/``(K, C, H, W)``). The encoder is applied per + tile in both cases (identity for precomputed embeddings). + + Returns: + ``(prediction, attention)`` where ``prediction`` has shape ``(out_dim,)`` + and ``attention`` is the aggregator's weights or ``None``. + """ + encoded = self._encode(bag) + pooled, attention = self.aggregator(encoded) # (aggregator.out_dim,), attn|None + prediction = self.output_activation(self.head(pooled)) # (out_dim,) + return prediction, attention + + def _encode(self, bag: AnyBag) -> AnyBag: + """Apply the encoder per tile, preserving the bag's single/multi structure.""" + if isinstance(bag, list): # multi-scale: encode each level within each region + return [ + {level: self.encoder(tiles) for level, tiles in region.items()} + for region in bag + ] + return self.encoder(bag) # flat bag: (N, ...) -> (N, encoder.out_dim) + + def _forward_batch( + self, bags: Sequence[AnyBag] + ) -> tuple[Tensor, list[Tensor | None]]: + """Run a batch of variable-length bags, stacking per-slide predictions.""" + predictions: list[Tensor] = [] + attentions: list[Tensor | None] = [] + for bag in bags: + prediction, attention = self(bag) + predictions.append(prediction) + attentions.append(attention) + return torch.stack(predictions), attentions + + def _step( + self, + batch: list[AnySample], + stage: str, + metrics: torchmetrics.MetricCollection, + ) -> Tensor: + bags, labels, _ = _unpack_batch(batch) + y = torch.stack(labels).to(self.device) + y_pred, _ = self._forward_batch(bags) + + if self.loss is None: + raise RuntimeError("A loss is required for training/validation steps.") + + loss = self.loss(y_pred, y) + self.log(f"{stage}/loss", loss, on_step=stage == "train", on_epoch=True) + + metrics.update(y_pred, y) + self.log_dict(metrics, on_step=False, on_epoch=True) + return loss + + def training_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + return self._step(batch, "train", self.train_metrics) + + def validation_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + return self._step(batch, "valid", self.valid_metrics) + + def test_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + return self._step(batch, "test", self.test_metrics) + + def predict_step( + self, batch: list[AnySample], batch_idx: int + ) -> dict[str, Any]: + bags, _, metadata = _unpack_batch(batch) + predictions, attentions = self._forward_batch(bags) + return { + "predictions": predictions, + "attentions": attentions, + "metadata": metadata, + } + + def configure_optimizers(self) -> Any: + if self.optimizer is None: + return None + + optimizer = self.optimizer(self.parameters()) + config: dict[str, Any] = {"optimizer": optimizer} + + if self.lr_scheduler: + scheduler = {"scheduler": self.lr_scheduler["scheduler"](optimizer)} + if monitor := self.lr_scheduler.get("monitor"): + scheduler["monitor"] = monitor + config["lr_scheduler"] = scheduler + + return config + + +def _unpack_batch( + batch: list[AnySample], +) -> tuple[list[AnyBag], list[Tensor], list[SlideMetadata]]: + """Split a list-of-samples batch into parallel lists of bags/labels/metadata.""" + bags = [sample[0] for sample in batch] + labels = [sample[1] for sample in batch] + metadata = [sample[2] for sample in batch] + return bags, labels, metadata + + +__all__ = ["MammaprintModule"] diff --git a/ml/train.py b/ml/train.py new file mode 100644 index 00000000..80585e8b --- /dev/null +++ b/ml/train.py @@ -0,0 +1,37 @@ +"""Training entrypoint for MammaPrint MIL models. + +Mirrors the preprocessing entrypoints: Hydra composes the config, ``@autolog`` +sets up MLflow logging, and every object (datamodule, module, trainer) is built +with :func:`hydra.utils.instantiate`. + +Run with, e.g.:: + + uv run -m ml.train +experiment=ml/train_mil_embeddings +""" + +import random + +import hydra +from omegaconf import DictConfig, OmegaConf +from rationai.mlkit import autolog +from rationai.mlkit.lightning.loggers import MLFlowLogger + + +# Resolves ${random_seed:} in the config; cached so all consumers share one seed. +OmegaConf.register_new_resolver( + "random_seed", lambda: random.randint(0, 2**31), use_cache=True, replace=True +) + + +@hydra.main(config_path="../configs", config_name="ml", version_base=None) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + datamodule = hydra.utils.instantiate(config.datamodule) + module = hydra.utils.instantiate(config.ml) + trainer = hydra.utils.instantiate(config.trainer, logger=logger) + + trainer.fit(module, datamodule=datamodule) + + +if __name__ == "__main__": + main() From b9e4fa27274c7d6d9cd118f43605787d20e25a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 15:57:30 +0200 Subject: [PATCH 09/79] feat: datasets --- ml/data/datasets/__init__.py | 21 +++++++++++- ml/data/datasets/_sources.py | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 ml/data/datasets/_sources.py diff --git a/ml/data/datasets/__init__.py b/ml/data/datasets/__init__.py index 7108560c..2d6a1d61 100644 --- a/ml/data/datasets/__init__.py +++ b/ml/data/datasets/__init__.py @@ -1,4 +1,23 @@ -from ml.data.datasets.tiles import SlideDataset, TileDataset +"""Datasets for MammaPrint. + +``SlideDataset``/``TileDataset`` are imported lazily: they pull in OpenSlide via +``rationai.mlkit``, which the embeddings/alignment path does not need. Accessing +them (``from ml.data.datasets import SlideDataset``) triggers the import on demand. +""" + +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from ml.data.datasets.tiles import SlideDataset, TileDataset __all__ = ["SlideDataset", "TileDataset"] + + +def __getattr__(name: str) -> Any: + if name in {"SlideDataset", "TileDataset"}: + from ml.data.datasets import tiles + + return getattr(tiles, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py new file mode 100644 index 00000000..faf41f0a --- /dev/null +++ b/ml/data/datasets/_sources.py @@ -0,0 +1,64 @@ +"""Shared setup helpers for the slide datasets. + +:class:`~ml.data.datasets.pyramid.PyramidSlideDataset` needs a few setup steps: +load + label the slide table, download the per-level artifacts, and find which +slides are actually present across levels. Extracting them here keeps the dataset +free of scaffolding and leaves them independently unit-testable. + +The train/val/test split is NOT applied here — splitting is materialised into +physically separate artifacts upstream by ``scripts/preprocessing/split_dataset.py`` +(the ``split`` column in ``data_mapping.csv`` is the sole split authority). A card's +per-split URI already points at a pure artifact, so no split filter is needed. +""" + +import logging +from collections.abc import Mapping +from pathlib import Path + +import pandas as pd +from mlflow.artifacts import download_artifacts + +from ml.data.datasets.labels import LabelMode, get_target_column, process_slides + + +logger = logging.getLogger(__name__) + + +def load_labeled_slides( + data_mapping: str | Path, + label_mode: LabelMode, +) -> pd.DataFrame: + """Load the slide table and drop rows without a label. + + Args: + data_mapping: Path to ``data_mapping.csv`` (``record_num``/``type``/ + ``mammaprint_index``/``path``). + label_mode: Classification (``type``) or regression (``index``) target. + + Returns: + A dataframe with a ``name`` (slide stem) column and the label column, keeping + only rows that carry a label. + """ + slides = pd.read_csv(data_mapping) + slides = process_slides(slides, mode=label_mode) + + target_column = get_target_column(label_mode) + keep = slides[target_column].notna() + return slides[keep].reset_index(drop=True) + + +def download_level_sources(sources: Mapping[int, str]) -> dict[int, Path]: + """Download each pyramid level's artifact and return ``level -> local dir``.""" + return { + level: Path(download_artifacts(artifact_uri=uri)) + for level, uri in sources.items() + } + + +def available_slides(dirs: Mapping[int, Path]) -> set[str]: + """Slide stems present as ``.parquet`` in *every* level's directory.""" + per_level = ({p.stem for p in d.glob("*.parquet")} for d in dirs.values()) + return set.intersection(*per_level) if dirs else set() + + +__all__ = ["available_slides", "download_level_sources", "load_labeled_slides"] From b33e13b0a886b12339c224dad049f72f51f105a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 8 Jul 2026 15:58:18 +0200 Subject: [PATCH 10/79] feat: data --- configs/data/_template.yaml | 35 ++++++++++++++++ ml/data/__init__.py | 10 +++++ ml/data/datamodule.py | 79 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 configs/data/_template.yaml create mode 100644 ml/data/datamodule.py diff --git a/configs/data/_template.yaml b/configs/data/_template.yaml new file mode 100644 index 00000000..5189f308 --- /dev/null +++ b/configs/data/_template.yaml @@ -0,0 +1,35 @@ +# @package _global_ +# Parent "shape" for the data layer: DataModule wiring + mandatory fields the +# selected cards must fill. A data/sources/* selector pulls this in plus one or +# more data/tiled/* cards (which populate dataset.levels). The datamodule reads the +# whole dataset.levels map; a one-entry map is the single-scale case. +dataset: + levels: ??? # filled by the selected data/tiled/* card(s) + data_mapping: ${project_path}/data_mapping.csv + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + # data_mapping/levels/label_mode are defined once above (dataset.*) and referenced + # per split; only `split` differs. Each split is a separate dataset construction. + train: + _target_: ml.data.PyramidSlideDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + val: + _target_: ml.data.PyramidSlideDataset + split: val + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + test: + _target_: ml.data.PyramidSlideDataset + split: test + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} diff --git a/ml/data/__init__.py b/ml/data/__init__.py index f27993ae..8ebc2ac2 100644 --- a/ml/data/__init__.py +++ b/ml/data/__init__.py @@ -1 +1,11 @@ """Data utilities for MammaPrint.""" + +from ml.data.datamodule import DataModule, mil_collate +# from ml.data.datasets.pyramid import PyramidSlideDataset + + +__all__ = [ + "DataModule", + # "PyramidSlideDataset", + "mil_collate", +] diff --git a/ml/data/datamodule.py b/ml/data/datamodule.py new file mode 100644 index 00000000..fdbe1a6e --- /dev/null +++ b/ml/data/datamodule.py @@ -0,0 +1,79 @@ +"""Thin Lightning datamodule wiring three slide datasets. + +Splits are owned by the datasets: each of ``train``/``val``/``test`` is a fully +constructed :class:`~ml.data.datasets.pyramid.PyramidSlideDataset` (built by Hydra +with its own ``split=...``). This module only wraps them in dataloaders — no +artifact download or split logic lives here. + +Bags have variable tile/region counts, so batches are collated as a list of +samples (:func:`mil_collate`) rather than a stacked tensor. +""" + +from collections.abc import Sequence + +import lightning.pytorch as pl +from torch.utils.data import DataLoader, Dataset + +from ml.typing import MILSample #, MultiScaleSample + + +type Sample = MILSample # | MultiScaleSample + + +def mil_collate[S](batch: Sequence[S]) -> list[S]: + """Collate variable-length bags by keeping the batch as a list of samples.""" + return list(batch) + + +class DataModule(pl.LightningDataModule): + """Wraps pre-built train/val/test datasets in dataloaders. + + Args: + train: Training dataset (a slide dataset yielding MIL samples). + val: Validation dataset. + test: Test dataset. + batch_size: Slides (bags) per batch. + num_workers: Dataloader worker processes. + pin_memory: Pin host memory for faster host->device copies. + persistent_workers: Keep workers alive between epochs. + """ + + def __init__( + self, + train: Dataset[Sample], + val: Dataset[Sample], + test: Dataset[Sample], + batch_size: int = 1, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + ) -> None: + super().__init__() + self.datasets = {"train": train, "val": val, "test": test} + self.batch_size = batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + + def _dataloader(self, split: str, shuffle: bool) -> DataLoader[Sample]: + return DataLoader( + self.datasets[split], + batch_size=self.batch_size, + shuffle=shuffle, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers and self.num_workers > 0, + collate_fn=mil_collate, + ) + + def train_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("train", shuffle=True) + + def val_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("val", shuffle=False) + + def test_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("test", shuffle=False) + + +__all__ = ["DataModule", "mil_collate"] From 221eb94acd9273ff08f30c263d1ea20cf71fdefc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 13:52:17 +0200 Subject: [PATCH 11/79] feat: data and datasets 2 --- ml/data/__init__.py | 13 +++- ml/data/datasets/_sources.py | 25 ++++++- ml/data/datasets/raw_tiles.py | 122 +++++++++++++++++++++++++++++++ ml/data/datasets/single_scale.py | 81 ++++++++++++++++++++ ml/typing.py | 31 ++++---- 5 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 ml/data/datasets/raw_tiles.py create mode 100644 ml/data/datasets/single_scale.py diff --git a/ml/data/__init__.py b/ml/data/__init__.py index 8ebc2ac2..c8f16106 100644 --- a/ml/data/__init__.py +++ b/ml/data/__init__.py @@ -1,11 +1,20 @@ """Data utilities for MammaPrint.""" from ml.data.datamodule import DataModule, mil_collate + +# ── Multilevel (comment out this line + the __all__ entry to ship single-level) ── # from ml.data.datasets.pyramid import PyramidSlideDataset +from ml.data.datasets.raw_tiles import RawTileSlideDataset +from ml.data.datasets.single_scale import SingleScaleDataset -__all__ = [ +# Grouped (single-level first, multilevel last) to make the toggle obvious, so this +# is deliberately not alphabetically sorted. +__all__ = [ # noqa: RUF022 "DataModule", - # "PyramidSlideDataset", + "RawTileSlideDataset", + "SingleScaleDataset", "mil_collate", + # ── Multilevel ── + # "PyramidSlideDataset", ] diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py index faf41f0a..d97ce3d0 100644 --- a/ml/data/datasets/_sources.py +++ b/ml/data/datasets/_sources.py @@ -61,4 +61,27 @@ def available_slides(dirs: Mapping[int, Path]) -> set[str]: return set.intersection(*per_level) if dirs else set() -__all__ = ["available_slides", "download_level_sources", "load_labeled_slides"] +def split_uri(card: Mapping[str, object], key: str, split: str, level: int) -> str: + """Read a level card's per-split artifact URI (``card[key][split]``). + + Raises a clear error if the card lacks the URI map or the requested split. + """ + uris = card.get(key) + if not isinstance(uris, Mapping): + raise KeyError( + f"Level {level} data card is missing a '{key}' split->URI map." + ) + uri = uris.get(split) + if not isinstance(uri, str): + raise KeyError( + f"Level {level} data card '{key}' has no URI for split '{split}'." + ) + return uri + + +__all__ = [ + "available_slides", + "download_level_sources", + "load_labeled_slides", + "split_uri", +] diff --git a/ml/data/datasets/raw_tiles.py b/ml/data/datasets/raw_tiles.py new file mode 100644 index 00000000..1ed6cb65 --- /dev/null +++ b/ml/data/datasets/raw_tiles.py @@ -0,0 +1,122 @@ +"""Raw-tile slide dataset for the end-to-end (image backbone) path. + +Each item is one slide's bag of raw tile **images** ``(N, C, H, W)`` — fed to an +image encoder (e.g. ``VGG16Encoder``) that maps each tile to a feature vector, +then to an aggregator + head. Reads pixels via +:class:`ml.data.datasets.SlideDataset` (OpenSlide) from a *tiled* artifact +(``slides.parquet`` + ``tiles.parquet``), so ``rationai.mlkit``/OpenSlide is +imported lazily and only touched on this path. + +Single-scale only (one level card, keyed by ``raw_uris[split]``). Slides can have +thousands of tiles, so ``max_tiles`` caps the bag (random-sampled, seeded) to bound +GPU memory — standard MIL practice. +""" + +import logging +from collections.abc import Mapping +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +import torch +from torch.utils.data import Dataset + +from ml.data.datasets._sources import load_labeled_slides, split_uri +from ml.data.datasets.labels import LabelMode, get_label +from ml.typing import LevelSpec, MILSample, SlideMetadata + + +if TYPE_CHECKING: + from ml.data.datasets.tiles import TileDataset + + +logger = logging.getLogger(__name__) + +# ImageNet normalisation, matching preprocessing/embeddings.py. +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +class RawTileSlideDataset(Dataset[MILSample]): + """Per-slide bags of raw tile images from a tiled artifact. + + Args: + split: Which split to load (``train``/``val``/``test``); selects the level + card's ``raw_uris[split]`` (a pure, pre-split artifact). + levels: A one-entry level map (single-scale). The entry's ``raw_uris`` maps + ``split -> tiled artifact URI``. + data_mapping: Label CSV path (type/index labels, joined by slide stem). + label_mode: ``"type"`` (classification) or ``"index"`` (regression). + max_tiles: Cap on tiles per slide; if a slide has more, a seeded random + subset is used. ``None`` keeps all tiles. + seed: Seed for the per-slide tile subsampling. + """ + + def __init__( + self, + split: str, + levels: Mapping[str | int, LevelSpec], + data_mapping: str | Path, + label_mode: str = "type", + max_tiles: int | None = 256, + seed: int = 0, + ) -> None: + if len(levels) != 1: + raise ValueError( + "RawTileSlideDataset is single-scale: pass exactly one level." + ) + + from mlflow.artifacts import download_artifacts + + from ml.data.datasets import SlideDataset # lazy: pulls OpenSlide + + self.label_mode = LabelMode(label_mode) + self.max_tiles = max_tiles + self.rng = np.random.default_rng(seed) + + (card,) = levels.values() + uri = split_uri(dict(card), "raw_uris", split, 0) + local_dir = Path(download_artifacts(artifact_uri=uri)) + + transform = _normalize_transform() + slide_dataset = SlideDataset(paths=[local_dir], transforms=transform) + # One TileDataset per slide, keyed by slide stem (matches data_mapping.name). + # generate_datasets() yields TileDataset instances (each has .slide_path). + per_slide = cast( + "list[TileDataset]", list(slide_dataset.generate_datasets()) + ) + self.tile_datasets: dict[str, TileDataset] = { + Path(td.slide_path).stem: td for td in per_slide + } + + slides = load_labeled_slides(data_mapping, self.label_mode) + self.slides = slides[ + slides["name"].isin(self.tile_datasets) + ].reset_index(drop=True) + + def __len__(self) -> int: + return len(self.slides) + + def __getitem__(self, idx: int) -> MILSample: + name = self.slides.iloc[idx]["name"] + tiles = self.tile_datasets[name] + + n = len(tiles) + indices: list[int] = list(range(n)) + if self.max_tiles is not None and n > self.max_tiles: + indices = self.rng.choice(n, self.max_tiles, replace=False).tolist() + logger.debug("Slide %s: sampled %d/%d tiles.", name, self.max_tiles, n) + + images = torch.stack([tiles[i][0] for i in indices]) # (N, C, H, W) + label = get_label(self.slides.iloc[idx], self.label_mode) + metadata: SlideMetadata = {"slide_id": name} + return images, label, metadata + + +def _normalize_transform() -> Any: + import albumentations as A + + return A.Compose([A.Normalize(mean=_IMAGENET_MEAN, std=_IMAGENET_STD)]) + + +__all__ = ["RawTileSlideDataset"] diff --git a/ml/data/datasets/single_scale.py b/ml/data/datasets/single_scale.py new file mode 100644 index 00000000..bf38b41f --- /dev/null +++ b/ml/data/datasets/single_scale.py @@ -0,0 +1,81 @@ +"""Single-scale slide dataset over precomputed tile embeddings. + +Each item is one slide's flat bag of tile-feature vectors ``(N, D)`` — consumed +directly by a flat aggregator (mean/max/attention) + head. One pyramid level only. + +Self-contained by design: this module imports **no** multi-scale primitives +(``align_regions``/``Region``/``MultiScaleBag``), so the single-level path keeps +working even if every multilevel file is removed or commented out. +""" + +import logging +from collections.abc import Mapping +from pathlib import Path + +import numpy as np +import pandas as pd +import torch +from torch.utils.data import Dataset + +from ml.data.datasets._sources import ( + available_slides, + download_level_sources, + load_labeled_slides, + split_uri, +) +from ml.data.datasets.labels import LabelMode, get_label +from ml.typing import LevelSpec, MILSample, SlideMetadata + + +logger = logging.getLogger(__name__) + + +class SingleScaleDataset(Dataset[MILSample]): + """Per-slide flat bags of precomputed tile embeddings (one level). + + Args: + split: Which split to load (``train``/``val``/``test``); selects the level + card's ``uris[split]`` (a pure, pre-split artifact). + levels: A one-entry level map. The entry's ``uris`` maps ``split -> URI``. + data_mapping: Label CSV path (type/index labels, joined by slide stem). + label_mode: ``"type"`` (classification) or ``"index"`` (regression). + """ + + def __init__( + self, + split: str, + levels: Mapping[str | int, LevelSpec], + data_mapping: str | Path, + label_mode: str = "type", + ) -> None: + if len(levels) != 1: + raise ValueError( + "SingleScaleDataset is single-scale: pass exactly one level." + ) + + self.label_mode = LabelMode(label_mode) + (level, card) = next(iter(levels.items())) + self.level = int(level) + + uri = split_uri(dict(card), "uris", split, self.level) + self.embeddings_dir = download_level_sources({self.level: uri})[self.level] + + slides = load_labeled_slides(data_mapping, self.label_mode) + present = available_slides({self.level: self.embeddings_dir}) + self.slides = slides[slides["name"].isin(present)].reset_index(drop=True) + + def __len__(self) -> int: + return len(self.slides) + + def __getitem__(self, idx: int) -> MILSample: + row = self.slides.iloc[idx] + frame = pd.read_parquet( + (self.embeddings_dir / row["name"]).with_suffix(".parquet") + ) + bag = torch.from_numpy(np.stack(frame["embedding"].to_numpy())).float() + label = get_label(row, self.label_mode) + metadata: SlideMetadata = {"slide_id": row["name"]} + return bag, label, metadata + + +__all__ = ["SingleScaleDataset"] diff --git a/ml/typing.py b/ml/typing.py index a060dcb3..9d4298ba 100644 --- a/ml/typing.py +++ b/ml/typing.py @@ -36,27 +36,26 @@ class LevelSpec(TypedDict): uris: Mapping[str, str] +# ── Single-level types (always shipped) ────────────────────────────────────── type Bag = Tensor """A bag of per-slide items: tile features ``(N, D)`` or raw tiles ``(N, C, H, W)``.""" -type Region = dict[int, Tensor] -"""One aligned multi-scale region: level -> that level's tiles ``(K_level, D)``. - -For example ``{3: (1, D), 2: (4, D)}`` is one coarse level-3 tile with the four -level-2 tiles that zoom into the same footprint. -""" +type MILSample = tuple[Bag, Tensor, SlideMetadata] +"""A single-level slide-level training sample: ``(bag, label, metadata)``.""" -type MultiScaleBag = list[Region] -"""A slide as a list of aligned multi-scale regions (variable length).""" -type AnyBag = Bag #| MultiScaleBag -"""Either a flat single-level bag or a multi-scale bag.""" +# ── Multilevel types (comment out this block to ship single-level only) ─────── +# Nothing single-level references these; the multilevel dataset/aggregator are the +# only consumers, so removing/commenting them here + their imports drops multilevel. +# type Region = dict[int, Tensor] +# """One aligned multi-scale region: level -> that level's tiles ``(K_level, D)``. -type MILSample = tuple[Bag, Tensor, SlideMetadata] -"""A single-level slide-level training sample: ``(bag, label, metadata)``.""" +# For example ``{3: (1, D), 2: (4, D)}`` is one coarse level-3 tile with the four +# level-2 tiles that zoom into the same footprint. +# """ -type MultiScaleSample = tuple[MultiScaleBag, Tensor, SlideMetadata] -"""A multi-scale slide-level training sample: ``(regions, label, metadata)``.""" +# type MultiScaleBag = list[Region] +# """A slide as a list of aligned multi-scale regions (variable length).""" -type AnySample = MILSample #| MultiScaleSample -"""Either a single-level or a multi-scale training sample.""" +# type MultiScaleSample = tuple[MultiScaleBag, Tensor, SlideMetadata] +# """A multi-scale slide-level training sample: ``(regions, label, metadata)``.""" From 3d78d8a4e002ec33a7ea3a2a4591bcecb32557ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 13:57:54 +0200 Subject: [PATCH 12/79] fix: models --- ml/models/aggregators/__init__.py | 17 +++-- ml/models/aggregators/attention.py | 53 ++++++++++++++ ml/models/aggregators/base.py | 16 ++--- ml/models/aggregators/max.py | 30 ++++++++ ml/models/aggregators/transformer.py | 103 +++++++++++++++++++++++++++ ml/models/encoders/__init__.py | 12 ++-- ml/models/encoders/vgg16.py | 47 ++++++++++++ ml/models/module.py | 42 +++++------ 8 files changed, 274 insertions(+), 46 deletions(-) create mode 100644 ml/models/aggregators/attention.py create mode 100644 ml/models/aggregators/max.py create mode 100644 ml/models/aggregators/transformer.py create mode 100644 ml/models/encoders/vgg16.py diff --git a/ml/models/aggregators/__init__.py b/ml/models/aggregators/__init__.py index 0c9b77d1..6fff616b 100644 --- a/ml/models/aggregators/__init__.py +++ b/ml/models/aggregators/__init__.py @@ -1,18 +1,23 @@ """Bag aggregators (MIL pooling): mean, max, gated attention, transformer.""" -# from ml.models.aggregators.attention import AttentionMIL +# ── Single-level aggregators (flat Bag) ────────────────────────────────────── +from ml.models.aggregators.attention import AttentionMIL from ml.models.aggregators.base import Aggregator -# from ml.models.aggregators.max import MaxPool +from ml.models.aggregators.max import MaxPool from ml.models.aggregators.mean import MeanPool +from ml.models.aggregators.transformer import TransformerMIL + + +# ── Multilevel (comment out to ship single-level only) ─────────────────────── # from ml.models.aggregators.multiscale import MultiScaleMIL -# from ml.models.aggregators.transformer import TransformerMIL __all__ = [ "Aggregator", - # "AttentionMIL", - # "MaxPool", + "AttentionMIL", + "MaxPool", "MeanPool", + "TransformerMIL", + # ── Multilevel ── # "MultiScaleMIL", - # "TransformerMIL", ] diff --git a/ml/models/aggregators/attention.py b/ml/models/aggregators/attention.py new file mode 100644 index 00000000..ad931756 --- /dev/null +++ b/ml/models/aggregators/attention.py @@ -0,0 +1,53 @@ +"""Gated-attention MIL aggregator. + +Inspired by "Attention-based Deep Multiple Instance Learning" (Ilse et al., 2018, +arXiv:1802.04712) and adapted from the RationAI ``feature/mil`` reference +implementation to the encoder -> aggregator -> head contract. +""" + +import torch +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class AttentionMIL(Aggregator[Bag]): + """Gated-attention pooling over a bag of tile features. + + Learns a scalar attention weight per tile via the gated mechanism + ``tanh(V h) * sigmoid(U h)``, softmax-normalises the weights across the bag, + and returns the attention-weighted sum of tile features. The per-tile weights + are returned for interpretability (heatmaps). + + Args: + feature_dim: Dimensionality of the incoming tile features. + attention_dim: Width of the attention hidden layer. + """ + + def __init__(self, feature_dim: int, attention_dim: int = 512) -> None: + super().__init__() + self.feature_dim = feature_dim + self.norm = nn.LayerNorm(feature_dim) + self.attention_v = nn.Linear(feature_dim, attention_dim) + self.attention_u = nn.Linear(feature_dim, attention_dim) + self.attention_weights = nn.Linear(attention_dim, 1) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + bag = self.norm(bag) # (N, D) + + gate = torch.tanh(self.attention_v(bag)) * torch.sigmoid( + self.attention_u(bag) + ) # (N, attention_dim) + scores = self.attention_weights(gate).squeeze(-1) # (N,) + attention = torch.softmax(scores, dim=0) # (N,) + + pooled = (attention.unsqueeze(-1) * bag).sum(dim=0) # (D,) + return pooled, attention + + +__all__ = ["AttentionMIL"] diff --git a/ml/models/aggregators/base.py b/ml/models/aggregators/base.py index 54ae9fa7..a940fd54 100644 --- a/ml/models/aggregators/base.py +++ b/ml/models/aggregators/base.py @@ -20,18 +20,16 @@ from torch import Tensor, nn -from ml.typing import Bag, MultiScaleBag - -class Aggregator[InputT: (Bag, MultiScaleBag)](nn.Module, ABC): +class Aggregator[InputT](nn.Module, ABC): """Reduces one slide's tiles to a single slide vector. - Generic over its input type: flat aggregators specialise as ``Aggregator[Bag]`` - (a tile tensor ``(N, D)``), and multi-scale aggregators as - ``Aggregator[MultiScaleBag]`` (a list of aligned regions). Implementations - return the pooled vector and, optionally, weights (``None`` for parameter-free - poolings such as mean/max), surfaced for interpretability (e.g. heatmaps) and - not required by the training loop. + Generic (unbounded) over its input type so this base stays single-level-safe + (it references no multilevel type): flat aggregators specialise as + ``Aggregator[Bag]`` (a tile tensor ``(N, D)``), and the optional multi-scale + aggregator as ``Aggregator[MultiScaleBag]`` (a list of aligned regions). + Implementations return the pooled vector and, optionally, weights (``None`` for + parameter-free poolings such as mean/max), surfaced for interpretability. """ @property diff --git a/ml/models/aggregators/max.py b/ml/models/aggregators/max.py new file mode 100644 index 00000000..f81399f7 --- /dev/null +++ b/ml/models/aggregators/max.py @@ -0,0 +1,30 @@ +"""Max-pooling aggregator.""" + +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class MaxPool(Aggregator[Bag]): + """Takes the feature-wise maximum across the bag. + + Parameter-free; assigns no per-tile weights. + + Args: + feature_dim: Dimensionality of the incoming tile features. + """ + + def __init__(self, feature_dim: int) -> None: + super().__init__() + self.feature_dim = feature_dim + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, None]: + return bag.amax(dim=0), None + + +__all__ = ["MaxPool"] diff --git a/ml/models/aggregators/transformer.py b/ml/models/aggregators/transformer.py new file mode 100644 index 00000000..69bd89d0 --- /dev/null +++ b/ml/models/aggregators/transformer.py @@ -0,0 +1,103 @@ +"""Transformer-based MIL aggregator (TransMIL-style). + +Treats a slide's tiles as a sequence and lets them attend to each other via +multi-head self-attention — unlike mean/gated-attention pooling, which score each +tile independently. A learnable CLS token is prepended; after ``num_layers`` of +self-attention the CLS embedding summarises the whole bag and becomes the slide +vector. The CLS->tiles attention from the final layer is returned as the per-tile +weight (interpretability / heatmaps). + +Reference: Shao et al., "TransMIL: Transformer based Correlated Multiple Instance +Learning for Whole Slide Image Classification" (NeurIPS 2021, arXiv:2106.00908). +""" + +import torch +from torch import Tensor, nn + +from ml.models.aggregators.base import Aggregator +from ml.typing import Bag + + +class TransformerMIL(Aggregator[Bag]): + """Self-attention over the tile bag with a learned CLS-token readout. + + Args: + feature_dim: Dimensionality of the incoming tile features ``D``. + num_heads: Number of attention heads (must divide ``feature_dim``). + num_layers: Number of transformer encoder layers. + mlp_ratio: Feed-forward hidden width as a multiple of ``feature_dim``. + dropout: Dropout inside the transformer layers. + """ + + def __init__( + self, + feature_dim: int, + num_heads: int = 8, + num_layers: int = 2, + mlp_ratio: float = 4.0, + dropout: float = 0.1, + ) -> None: + super().__init__() + if feature_dim % num_heads != 0: + raise ValueError( + f"feature_dim ({feature_dim}) must be divisible by num_heads " + f"({num_heads})." + ) + self.feature_dim = feature_dim + self.num_heads = num_heads + + # Learnable CLS token prepended to the tile sequence; its final embedding + # is the slide vector. + self.cls_token = nn.Parameter(torch.zeros(1, feature_dim)) + nn.init.trunc_normal_(self.cls_token, std=0.02) + + encoder_layer = nn.TransformerEncoderLayer( + d_model=feature_dim, + nhead=num_heads, + dim_feedforward=int(feature_dim * mlp_ratio), + dropout=dropout, + batch_first=True, + norm_first=True, # pre-norm: more stable for small/deep stacks + ) + # All but the last layer run as a standard stack; the last layer is applied + # manually so we can pull out CLS->tiles attention weights for the readout. + self.blocks = ( + nn.TransformerEncoder( + encoder_layer, num_layers - 1, enable_nested_tensor=False + ) + if num_layers > 1 + else None + ) + self.final_attn = nn.MultiheadAttention( + feature_dim, num_heads, dropout=dropout, batch_first=True + ) + self.final_norm = nn.LayerNorm(feature_dim) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + # bag: (N, D). Prepend CLS -> sequence (1, N+1, D) (batch dim of 1). + sequence = torch.cat([self.cls_token, bag], dim=0).unsqueeze(0) # (1, N+1, D) + + if self.blocks is not None: + sequence = self.blocks(sequence) # (1, N+1, D) + + # Final self-attention layer, keeping the CLS<-all attention map. + normed = self.final_norm(sequence) + attended, attn_weights = self.final_attn( + normed, normed, normed, need_weights=True, average_attn_weights=True + ) # attended: (1, N+1, D); attn_weights: (1, N+1, N+1) + sequence = sequence + attended # residual + + slide_vector = sequence[0, 0] # CLS embedding -> (D,) + + # Per-tile weight = how much the CLS query attends to each tile (drop the + # CLS->CLS self-weight); renormalise so the tile weights sum to 1. + cls_to_tiles = attn_weights[0, 0, 1:] # (N,) + attention = cls_to_tiles / cls_to_tiles.sum().clamp_min(1e-8) + return slide_vector, attention + + +__all__ = ["TransformerMIL"] diff --git a/ml/models/encoders/__init__.py b/ml/models/encoders/__init__.py index 44f2e042..73d65575 100644 --- a/ml/models/encoders/__init__.py +++ b/ml/models/encoders/__init__.py @@ -2,13 +2,11 @@ from ml.models.encoders.base import Encoder from ml.models.encoders.identity import IdentityEncoder -# from ml.models.encoders.vgg16 import VGG16Encoder -# from ml.models.encoders.virchow2 import Virchow2Encoder +from ml.models.encoders.vgg16 import VGG16Encoder __all__ = [ - "Encoder", - "IdentityEncoder", - # "VGG16Encoder", - # "Virchow2Encoder" - ] + "Encoder", + "IdentityEncoder", + "VGG16Encoder", +] diff --git a/ml/models/encoders/vgg16.py b/ml/models/encoders/vgg16.py new file mode 100644 index 00000000..74a82131 --- /dev/null +++ b/ml/models/encoders/vgg16.py @@ -0,0 +1,47 @@ +"""VGG16 image-backbone encoder for the end-to-end (raw-tile) path.""" + +import torch +from torch import Tensor, nn +from torchvision.models import VGG16_Weights, vgg16 + +from ml.models.encoders.base import Encoder + + +class VGG16Encoder(Encoder): + """Encodes raw tiles with VGG16 convolutional features + global max pool. + + Applies the ``torchvision`` VGG16 ``features`` stack to every tile in a bag + and reduces each tile's feature map to a single ``512``-dim vector via + adaptive global max pooling. The classifier/fully-connected part of VGG16 is + dropped — the slide-level prediction is produced downstream by the + aggregator + head. + + Args: + pretrained: Load ImageNet-pretrained weights when ``True``. + freeze: Freeze the backbone (no gradient updates) when ``True`` — useful + for training only the aggregator/head on top of a fixed backbone. + """ + + _FEATURE_DIM = 512 + + def __init__(self, pretrained: bool = True, freeze: bool = False) -> None: + super().__init__() + weights = VGG16_Weights.IMAGENET1K_V1 if pretrained else None + self.features = vgg16(weights=weights).features + self.pool = nn.AdaptiveMaxPool2d(output_size=1) + + if freeze: + for param in self.features.parameters(): + param.requires_grad = False + + @property + def out_dim(self) -> int: + return self._FEATURE_DIM + + def forward(self, tiles: Tensor) -> Tensor: + features = self.features(tiles) # (N, 512, H', W') + pooled = self.pool(features) # (N, 512, 1, 1) + return torch.flatten(pooled, start_dim=1) # (N, 512) + + +__all__ = ["VGG16Encoder"] diff --git a/ml/models/module.py b/ml/models/module.py index 481e13b9..c5d9944b 100644 --- a/ml/models/module.py +++ b/ml/models/module.py @@ -26,7 +26,7 @@ from ml.models.aggregators.base import Aggregator from ml.models.encoders.base import Encoder from ml.models.heads.base import Head -from ml.typing import AnyBag, AnySample, SlideMetadata +from ml.typing import Bag, MILSample, SlideMetadata logger = logging.getLogger(__name__) @@ -76,35 +76,29 @@ def __init__( self.valid_metrics = base.clone(prefix="valid/") self.test_metrics = base.clone(prefix="test/") - def forward(self, bag: AnyBag) -> tuple[Tensor, Tensor | None]: + def forward(self, bag: Bag) -> tuple[Tensor, Tensor | None]: """Run one slide through encoder -> aggregator -> head. Args: - bag: One slide, either a flat tile tensor ``(N, D)``/``(N, C, H, W)`` - (single level) or a multi-scale bag (list of regions, each mapping - ``level -> (K, D)``/``(K, C, H, W)``). The encoder is applied per - tile in both cases (identity for precomputed embeddings). + bag: One slide's flat tile tensor ``(N, D)`` / ``(N, C, H, W)``. The + encoder is applied per tile (identity for precomputed embeddings, + an image backbone for raw tiles). Returns: ``(prediction, attention)`` where ``prediction`` has shape ``(out_dim,)`` and ``attention`` is the aggregator's weights or ``None``. + + Note: multi-scale handling (a list of aligned regions) lives entirely in + :class:`~ml.models.aggregators.multiscale.MultiScaleMIL`, which holds its own + encoder — so the module stays flat and never imports multilevel code. """ - encoded = self._encode(bag) + encoded = self.encoder(bag) # (N, ...) -> (N, encoder.out_dim) pooled, attention = self.aggregator(encoded) # (aggregator.out_dim,), attn|None prediction = self.output_activation(self.head(pooled)) # (out_dim,) return prediction, attention - def _encode(self, bag: AnyBag) -> AnyBag: - """Apply the encoder per tile, preserving the bag's single/multi structure.""" - if isinstance(bag, list): # multi-scale: encode each level within each region - return [ - {level: self.encoder(tiles) for level, tiles in region.items()} - for region in bag - ] - return self.encoder(bag) # flat bag: (N, ...) -> (N, encoder.out_dim) - def _forward_batch( - self, bags: Sequence[AnyBag] + self, bags: Sequence[Bag] ) -> tuple[Tensor, list[Tensor | None]]: """Run a batch of variable-length bags, stacking per-slide predictions.""" predictions: list[Tensor] = [] @@ -117,7 +111,7 @@ def _forward_batch( def _step( self, - batch: list[AnySample], + batch: list[MILSample], stage: str, metrics: torchmetrics.MetricCollection, ) -> Tensor: @@ -135,17 +129,17 @@ def _step( self.log_dict(metrics, on_step=False, on_epoch=True) return loss - def training_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + def training_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: return self._step(batch, "train", self.train_metrics) - def validation_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + def validation_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: return self._step(batch, "valid", self.valid_metrics) - def test_step(self, batch: list[AnySample], batch_idx: int) -> Tensor: + def test_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: return self._step(batch, "test", self.test_metrics) def predict_step( - self, batch: list[AnySample], batch_idx: int + self, batch: list[MILSample], batch_idx: int ) -> dict[str, Any]: bags, _, metadata = _unpack_batch(batch) predictions, attentions = self._forward_batch(bags) @@ -172,8 +166,8 @@ def configure_optimizers(self) -> Any: def _unpack_batch( - batch: list[AnySample], -) -> tuple[list[AnyBag], list[Tensor], list[SlideMetadata]]: + batch: list[MILSample], +) -> tuple[list[Bag], list[Tensor], list[SlideMetadata]]: """Split a list-of-samples batch into parallel lists of bags/labels/metadata.""" bags = [sample[0] for sample in batch] labels = [sample[1] for sample in batch] From 99779d8dd6c1b3fff6b680a838f5fc179573f40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 15:17:18 +0200 Subject: [PATCH 13/79] feat: take logic from UC --- configs/data/_template.yaml | 26 ++++------ configs/data/_template_raw.yaml | 29 +++++++++++ configs/experiment/ml/train_regression.yaml | 37 ++++++++++++++ configs/ml.yaml | 12 +++-- configs/ml/aggregator/attention.yaml | 3 ++ configs/ml/aggregator/max.yaml | 2 + configs/ml/aggregator/mean.yaml | 2 + configs/ml/aggregator/transformer.yaml | 5 ++ configs/ml/encoder/vgg16.yaml | 4 ++ configs/ml/head/linear.yaml | 4 +- configs/ml/loss/bce.yaml | 2 + configs/ml/metrics/binary_accuracy.yaml | 3 ++ configs/ml/metrics/binary_auroc.yaml | 3 ++ configs/ml/metrics/binary_cohen_kappa.yaml | 3 ++ configs/ml/metrics/binary_f1.yaml | 3 ++ configs/ml/metrics/binary_precision.yaml | 3 ++ configs/ml/metrics/binary_recall.yaml | 3 ++ configs/ml/metrics/binary_specificity.yaml | 3 ++ configs/ml/metrics/mae.yaml | 3 ++ configs/ml/metrics/multiclass_auroc.yaml | 4 -- configs/ml/metrics/pearson.yaml | 3 ++ configs/ml/metrics/r2.yaml | 3 ++ configs/ml/metrics/spearman.yaml | 3 ++ ml/data/__init__.py | 2 +- ml/data/datamodule.py | 54 ++++++++++++++------- ml/data/datasets/labels.py | 15 ++++-- ml/train.py | 18 +++++-- scripts/ml/run_train.py | 22 +++++++++ 28 files changed, 222 insertions(+), 52 deletions(-) create mode 100644 configs/data/_template_raw.yaml create mode 100644 configs/experiment/ml/train_regression.yaml create mode 100644 configs/ml/aggregator/attention.yaml create mode 100644 configs/ml/aggregator/max.yaml create mode 100644 configs/ml/aggregator/mean.yaml create mode 100644 configs/ml/aggregator/transformer.yaml create mode 100644 configs/ml/encoder/vgg16.yaml create mode 100644 configs/ml/loss/bce.yaml create mode 100644 configs/ml/metrics/binary_accuracy.yaml create mode 100644 configs/ml/metrics/binary_auroc.yaml create mode 100644 configs/ml/metrics/binary_cohen_kappa.yaml create mode 100644 configs/ml/metrics/binary_f1.yaml create mode 100644 configs/ml/metrics/binary_precision.yaml create mode 100644 configs/ml/metrics/binary_recall.yaml create mode 100644 configs/ml/metrics/binary_specificity.yaml create mode 100644 configs/ml/metrics/mae.yaml delete mode 100644 configs/ml/metrics/multiclass_auroc.yaml create mode 100644 configs/ml/metrics/pearson.yaml create mode 100644 configs/ml/metrics/r2.yaml create mode 100644 configs/ml/metrics/spearman.yaml create mode 100644 scripts/ml/run_train.py diff --git a/configs/data/_template.yaml b/configs/data/_template.yaml index 5189f308..6055314d 100644 --- a/configs/data/_template.yaml +++ b/configs/data/_template.yaml @@ -1,10 +1,10 @@ # @package _global_ -# Parent "shape" for the data layer: DataModule wiring + mandatory fields the -# selected cards must fill. A data/sources/* selector pulls this in plus one or -# more data/tiled/* cards (which populate dataset.levels). The datamodule reads the -# whole dataset.levels map; a one-entry map is the single-scale case. +# Single-scale data shape: DataModule over SingleScaleDataset (flat embedding bags). +# A data/sources/* selector pulls this in plus ONE data/tiled/* card (one level). +# The dataset spec is written ONCE (the &dataset anchor on `train`); val/test merge +# it and override only `split`. Splits are built lazily per stage in DataModule.setup. dataset: - levels: ??? # filled by the selected data/tiled/* card(s) + levels: ??? # filled by the selected data/tiled/* card data_mapping: ${project_path}/data_mapping.csv datamodule: @@ -13,23 +13,15 @@ datamodule: num_workers: 0 pin_memory: false persistent_workers: false - # data_mapping/levels/label_mode are defined once above (dataset.*) and referenced - # per split; only `split` differs. Each split is a separate dataset construction. - train: - _target_: ml.data.PyramidSlideDataset + train: &dataset + _target_: ml.data.SingleScaleDataset split: train levels: ${dataset.levels} data_mapping: ${dataset.data_mapping} label_mode: ${label_mode} val: - _target_: ml.data.PyramidSlideDataset + <<: *dataset split: val - levels: ${dataset.levels} - data_mapping: ${dataset.data_mapping} - label_mode: ${label_mode} test: - _target_: ml.data.PyramidSlideDataset + <<: *dataset split: test - levels: ${dataset.levels} - data_mapping: ${dataset.data_mapping} - label_mode: ${label_mode} diff --git a/configs/data/_template_raw.yaml b/configs/data/_template_raw.yaml new file mode 100644 index 00000000..813e9920 --- /dev/null +++ b/configs/data/_template_raw.yaml @@ -0,0 +1,29 @@ +# @package _global_ +# Raw-tile data shape: DataModule over RawTileSlideDataset (image bags). +# A data/sources/* selector pulls this in plus ONE data/tiled/* card (with raw_uris). +# Dataset spec written once (&dataset on `train`); val/test override only `split`. +dataset: + levels: ??? # filled by the selected data/tiled/* card + data_mapping: ${project_path}/data_mapping.csv + max_tiles: 256 # per-slide tile cap (random-sampled if exceeded) + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + train: &dataset + _target_: ml.data.RawTileSlideDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + max_tiles: ${dataset.max_tiles} + seed: ${seed} + val: + <<: *dataset + split: val + test: + <<: *dataset + split: test diff --git a/configs/experiment/ml/train_regression.yaml b/configs/experiment/ml/train_regression.yaml new file mode 100644 index 00000000..5e0db81e --- /dev/null +++ b/configs/experiment/ml/train_regression.yaml @@ -0,0 +1,37 @@ +# @package _global_ +# Regression of the MammaPrint index on precomputed level-3 embeddings: identity +# encoder -> mean-pool -> linear head (out_dim=1). Same data as train_mil_embeddings. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Regression — MammaPrint index (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings, mean-pool + linear head. + hyperparameters: + aggregator: ${ml.aggregator._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/ml.yaml b/configs/ml.yaml index 4dcaf0fe..fcfacb4a 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -5,9 +5,15 @@ defaults: seed: ${random_seed:} -# Task shape — set per experiment; consumed by the ml/head + ml/metrics configs. -label_mode: type # "type" (classification) or "index" (regression) -num_classes: 2 +# Lightning stage + checkpoint. Override for eval/inference, e.g. +# mode=test checkpoint=/path/to/best.ckpt +mode: fit # fit | validate | test | predict +checkpoint: null # path to a .ckpt to resume (fit) or load weights (validate/test/predict) + +# Task shape — set per experiment. The head is single-output (out_dim=1) either way: +# binary classification (label_mode=type, BCE + binary metrics) or regression +# (label_mode=index, MSE). No num_classes — the task is binary-or-regression. +label_mode: type # "type" (binary classification) or "index" (regression) feature_dim: ??? # tile-feature width = encoder.out_dim; set per experiment # datamodule / ml / trainer are composed by the +experiment config's defaults. \ No newline at end of file diff --git a/configs/ml/aggregator/attention.yaml b/configs/ml/aggregator/attention.yaml new file mode 100644 index 00000000..1f286462 --- /dev/null +++ b/configs/ml/aggregator/attention.yaml @@ -0,0 +1,3 @@ +_target_: ml.models.aggregators.AttentionMIL +feature_dim: ${feature_dim} +attention_dim: 512 diff --git a/configs/ml/aggregator/max.yaml b/configs/ml/aggregator/max.yaml new file mode 100644 index 00000000..52fcbdc0 --- /dev/null +++ b/configs/ml/aggregator/max.yaml @@ -0,0 +1,2 @@ +_target_: ml.models.aggregators.MaxPool +feature_dim: ${feature_dim} diff --git a/configs/ml/aggregator/mean.yaml b/configs/ml/aggregator/mean.yaml new file mode 100644 index 00000000..757d2cfb --- /dev/null +++ b/configs/ml/aggregator/mean.yaml @@ -0,0 +1,2 @@ +_target_: ml.models.aggregators.MeanPool +feature_dim: ${feature_dim} diff --git a/configs/ml/aggregator/transformer.yaml b/configs/ml/aggregator/transformer.yaml new file mode 100644 index 00000000..2c72746d --- /dev/null +++ b/configs/ml/aggregator/transformer.yaml @@ -0,0 +1,5 @@ +# Extension point — not implemented in the skeleton (raises NotImplementedError). +_target_: ml.models.aggregators.TransformerMIL +feature_dim: ${feature_dim} +num_heads: 8 +num_layers: 2 diff --git a/configs/ml/encoder/vgg16.yaml b/configs/ml/encoder/vgg16.yaml new file mode 100644 index 00000000..8d953c61 --- /dev/null +++ b/configs/ml/encoder/vgg16.yaml @@ -0,0 +1,4 @@ +# End-to-end path: VGG16 conv features + global max pool (512-dim per tile). +_target_: ml.models.encoders.VGG16Encoder +pretrained: true +freeze: false diff --git a/configs/ml/head/linear.yaml b/configs/ml/head/linear.yaml index 0597b20f..ddc1f8c3 100644 --- a/configs/ml/head/linear.yaml +++ b/configs/ml/head/linear.yaml @@ -1,4 +1,6 @@ +# Single-output head: one logit for binary classification (with BCE) or the +# regression scalar. out_dim is 1 in both cases. _target_: ml.models.heads.LinearHead in_dim: ${feature_dim} -out_dim: ${num_classes} +out_dim: 1 dropout: 0.5 diff --git a/configs/ml/loss/bce.yaml b/configs/ml/loss/bce.yaml new file mode 100644 index 00000000..61831546 --- /dev/null +++ b/configs/ml/loss/bce.yaml @@ -0,0 +1,2 @@ +# Binary classification with a single-logit head. Pair with output_activation=identity. +_target_: torch.nn.BCEWithLogitsLoss diff --git a/configs/ml/metrics/binary_accuracy.yaml b/configs/ml/metrics/binary_accuracy.yaml new file mode 100644 index 00000000..ee74925f --- /dev/null +++ b/configs/ml/metrics/binary_accuracy.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +accuracy: + _target_: torchmetrics.classification.BinaryAccuracy diff --git a/configs/ml/metrics/binary_auroc.yaml b/configs/ml/metrics/binary_auroc.yaml new file mode 100644 index 00000000..907fc01d --- /dev/null +++ b/configs/ml/metrics/binary_auroc.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +auroc: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/binary_cohen_kappa.yaml b/configs/ml/metrics/binary_cohen_kappa.yaml new file mode 100644 index 00000000..9be6978f --- /dev/null +++ b/configs/ml/metrics/binary_cohen_kappa.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +cohen_kappa: + _target_: torchmetrics.classification.BinaryCohenKappa diff --git a/configs/ml/metrics/binary_f1.yaml b/configs/ml/metrics/binary_f1.yaml new file mode 100644 index 00000000..5ea30fb4 --- /dev/null +++ b/configs/ml/metrics/binary_f1.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +f1: + _target_: torchmetrics.classification.BinaryF1Score diff --git a/configs/ml/metrics/binary_precision.yaml b/configs/ml/metrics/binary_precision.yaml new file mode 100644 index 00000000..0adbc194 --- /dev/null +++ b/configs/ml/metrics/binary_precision.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +precision: + _target_: torchmetrics.classification.BinaryPrecision diff --git a/configs/ml/metrics/binary_recall.yaml b/configs/ml/metrics/binary_recall.yaml new file mode 100644 index 00000000..3401fede --- /dev/null +++ b/configs/ml/metrics/binary_recall.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +recall: + _target_: torchmetrics.classification.BinaryRecall diff --git a/configs/ml/metrics/binary_specificity.yaml b/configs/ml/metrics/binary_specificity.yaml new file mode 100644 index 00000000..61568078 --- /dev/null +++ b/configs/ml/metrics/binary_specificity.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +specificity: + _target_: torchmetrics.classification.BinarySpecificity diff --git a/configs/ml/metrics/mae.yaml b/configs/ml/metrics/mae.yaml new file mode 100644 index 00000000..9c099bcb --- /dev/null +++ b/configs/ml/metrics/mae.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +mae: + _target_: torchmetrics.MeanAbsoluteError diff --git a/configs/ml/metrics/multiclass_auroc.yaml b/configs/ml/metrics/multiclass_auroc.yaml deleted file mode 100644 index 02c94bbe..00000000 --- a/configs/ml/metrics/multiclass_auroc.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# @package ml.metrics -auroc: - _target_: torchmetrics.classification.MulticlassAUROC - num_classes: ${num_classes} diff --git a/configs/ml/metrics/pearson.yaml b/configs/ml/metrics/pearson.yaml new file mode 100644 index 00000000..00ddc961 --- /dev/null +++ b/configs/ml/metrics/pearson.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +pearson: + _target_: torchmetrics.PearsonCorrCoef diff --git a/configs/ml/metrics/r2.yaml b/configs/ml/metrics/r2.yaml new file mode 100644 index 00000000..c1969749 --- /dev/null +++ b/configs/ml/metrics/r2.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +r2: + _target_: torchmetrics.R2Score diff --git a/configs/ml/metrics/spearman.yaml b/configs/ml/metrics/spearman.yaml new file mode 100644 index 00000000..1fa4508f --- /dev/null +++ b/configs/ml/metrics/spearman.yaml @@ -0,0 +1,3 @@ +# @package ml.metrics +spearman: + _target_: torchmetrics.SpearmanCorrCoef diff --git a/ml/data/__init__.py b/ml/data/__init__.py index c8f16106..6b1864f3 100644 --- a/ml/data/__init__.py +++ b/ml/data/__init__.py @@ -10,7 +10,7 @@ # Grouped (single-level first, multilevel last) to make the toggle obvious, so this # is deliberately not alphabetically sorted. -__all__ = [ # noqa: RUF022 +__all__ = [ "DataModule", "RawTileSlideDataset", "SingleScaleDataset", diff --git a/ml/data/datamodule.py b/ml/data/datamodule.py index fdbe1a6e..6d5b5909 100644 --- a/ml/data/datamodule.py +++ b/ml/data/datamodule.py @@ -1,23 +1,26 @@ -"""Thin Lightning datamodule wiring three slide datasets. +"""Thin Lightning datamodule that builds its split datasets lazily. -Splits are owned by the datasets: each of ``train``/``val``/``test`` is a fully -constructed :class:`~ml.data.datasets.pyramid.PyramidSlideDataset` (built by Hydra -with its own ``split=...``). This module only wraps them in dataloaders — no -artifact download or split logic lives here. +Each split (``train``/``val``/``test``) is passed as an *unresolved* Hydra config +node and instantiated on demand in :meth:`setup`, only for the running stage — so a +test-only run never downloads the train/val artifacts, and vice versa. The +entrypoint must instantiate this with ``_recursive_=False`` so the split nodes +arrive as ``DictConfig`` rather than pre-built datasets. -Bags have variable tile/region counts, so batches are collated as a list of -samples (:func:`mil_collate`) rather than a stacked tensor. +Pattern follows the sibling ``ulcerative_colitis.data.DataModule``. Bags have +variable tile/region counts, so batches stay a list of samples (:func:`mil_collate`). """ from collections.abc import Sequence import lightning.pytorch as pl +from hydra.utils import instantiate +from omegaconf import DictConfig from torch.utils.data import DataLoader, Dataset -from ml.typing import MILSample #, MultiScaleSample +from ml.typing import MILSample -type Sample = MILSample # | MultiScaleSample +type Sample = MILSample def mil_collate[S](batch: Sequence[S]) -> list[S]: @@ -26,38 +29,53 @@ def mil_collate[S](batch: Sequence[S]) -> list[S]: class DataModule(pl.LightningDataModule): - """Wraps pre-built train/val/test datasets in dataloaders. + """Builds train/val/test datasets lazily from unresolved config nodes. Args: - train: Training dataset (a slide dataset yielding MIL samples). - val: Validation dataset. - test: Test dataset. batch_size: Slides (bags) per batch. num_workers: Dataloader worker processes. pin_memory: Pin host memory for faster host->device copies. persistent_workers: Keep workers alive between epochs. + **datasets: One entry per split (``train``/``val``/``test``), each an + unresolved dataset ``DictConfig`` instantiated in :meth:`setup`. """ def __init__( self, - train: Dataset[Sample], - val: Dataset[Sample], - test: Dataset[Sample], batch_size: int = 1, num_workers: int = 0, pin_memory: bool = False, persistent_workers: bool = False, + **datasets: DictConfig, ) -> None: super().__init__() - self.datasets = {"train": train, "val": val, "test": test} self.batch_size = batch_size self.num_workers = num_workers self.pin_memory = pin_memory self.persistent_workers = persistent_workers + self._dataset_cfgs = datasets + self._datasets: dict[str, Dataset[Sample]] = {} + + def _build(self, split: str) -> None: + if split in self._datasets: + return + cfg = self._dataset_cfgs.get(split) + if cfg is None: + raise KeyError(f"No dataset config provided for split '{split}'.") + self._datasets[split] = instantiate(cfg) + + def setup(self, stage: str | None = None) -> None: + needed = { + "fit": ("train", "val"), + "validate": ("val",), + "test": ("test",), + }.get(stage or "", ("train", "val", "test")) + for split in needed: + self._build(split) def _dataloader(self, split: str, shuffle: bool) -> DataLoader[Sample]: return DataLoader( - self.datasets[split], + self._datasets[split], batch_size=self.batch_size, shuffle=shuffle, num_workers=self.num_workers, diff --git a/ml/data/datasets/labels.py b/ml/data/datasets/labels.py index 99930f0d..b2e6efcc 100644 --- a/ml/data/datasets/labels.py +++ b/ml/data/datasets/labels.py @@ -34,13 +34,22 @@ def process_slides(slides: pd.DataFrame, mode: LabelMode | None = None) -> pd.Da def get_label(slide_metadata: pd.Series, mode: LabelMode) -> torch.Tensor: + """One slide's target as a ``(1,)`` float tensor. + + Both modes are single-output (``out_dim=1``): binary classification (luminal + a=1 / b=0, for ``BCEWithLogitsLoss`` + binary metrics) and regression (the + MammaPrint index). Shape ``(1,)`` matches the head's per-slide output so batched + preds/targets line up as ``(B, 1)``. + """ match mode: case LabelMode.TYPE: - return torch.tensor(int(slide_metadata["type_label"])).long() + value = float(slide_metadata["type_label"]) case LabelMode.INDEX: - return torch.tensor(float(slide_metadata["mammaprint_index"])).float() + value = float(slide_metadata["mammaprint_index"]) + case _: + raise ValueError(f"Unsupported label mode: {mode}") - raise ValueError(f"Unsupported label mode: {mode}") + return torch.tensor([value], dtype=torch.float32) def get_target_column(mode: LabelMode) -> str: diff --git a/ml/train.py b/ml/train.py index 80585e8b..da8e4c86 100644 --- a/ml/train.py +++ b/ml/train.py @@ -1,12 +1,15 @@ -"""Training entrypoint for MammaPrint MIL models. +"""Entrypoint for MammaPrint MIL models — train / validate / test / predict. Mirrors the preprocessing entrypoints: Hydra composes the config, ``@autolog`` sets up MLflow logging, and every object (datamodule, module, trainer) is built -with :func:`hydra.utils.instantiate`. +with :func:`hydra.utils.instantiate`. The Lightning stage is selected by +``config.mode`` (``fit``/``validate``/``test``/``predict``), and ``config.checkpoint`` +optionally resumes / loads weights. Run with, e.g.:: - uv run -m ml.train +experiment=ml/train_mil_embeddings + uv run -m ml.train +experiment=ml/train_mil_embeddings # fit (default) + uv run -m ml.train +experiment=ml/train_mil_embeddings mode=test checkpoint=/path/best.ckpt """ import random @@ -26,11 +29,16 @@ @hydra.main(config_path="../configs", config_name="ml", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: - datamodule = hydra.utils.instantiate(config.datamodule) + # _recursive_=False: keep the per-split dataset nodes as DictConfig so the + # DataModule can instantiate them lazily per stage in setup(). + datamodule = hydra.utils.instantiate(config.datamodule, _recursive_=False) module = hydra.utils.instantiate(config.ml) trainer = hydra.utils.instantiate(config.trainer, logger=logger) - trainer.fit(module, datamodule=datamodule) + # config.mode selects the Lightning stage (fit/validate/test/predict); + # config.checkpoint (or null) resumes training / loads weights for eval. + run_stage = getattr(trainer, config.mode) + run_stage(module, datamodule=datamodule, ckpt_path=config.checkpoint) if __name__ == "__main__": diff --git a/scripts/ml/run_train.py b/scripts/ml/run_train.py new file mode 100644 index 00000000..88c16942 --- /dev/null +++ b/scripts/ml/run_train.py @@ -0,0 +1,22 @@ +from kube_jobs import storage, submit_job + + +submit_job( + job_name="mammaprint-train-mil-embeddings", + username="kissmi", + image="cerit.io/rationai/base:2.0.6", + cpu=24, + memory="64Gi", + gpu="A40", + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + # "git checkout ", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "export HF_TOKEN=", + "uv sync --frozen", + "uv run -m ml.train +experiment=ml/train_mil_embeddings", + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], +) From 9145175b1146100795c2840191d8abc928339d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 15:42:54 +0200 Subject: [PATCH 14/79] feat: training configs --- configs/experiment/ml/train_e2e_vgg16.yaml | 39 ++++++++++++++++++ .../experiment/ml/train_mil_embeddings.yaml | 41 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 configs/experiment/ml/train_e2e_vgg16.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings.yaml diff --git a/configs/experiment/ml/train_e2e_vgg16.yaml b/configs/experiment/ml/train_e2e_vgg16.yaml new file mode 100644 index 00000000..85b969b8 --- /dev/null +++ b/configs/experiment/ml/train_e2e_vgg16.yaml @@ -0,0 +1,39 @@ +# @package _global_ +# End-to-end classification: VGG16 backbone over raw level-3 tiles -> mean-pool +# -> linear head. Reads tile images via RawTileSlideDataset (OpenSlide). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: vgg16 + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data/sources: single_l3_tiled + - _self_ + +label_mode: type +feature_dim: 512 # VGG16 global-max-pooled feature size + +metadata: + experiment_name: MammaPrint + run_name: End-to-end VGG16 — luminal classification (raw tiles) + description: | + End-to-end luminal classification with a VGG16 backbone over raw tiles and + mean-pool aggregation. Backbone/aggregator/head are all Hydra-swappable. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings.yaml b/configs/experiment/ml/train_mil_embeddings.yaml new file mode 100644 index 00000000..edbab26a --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings.yaml @@ -0,0 +1,41 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings: identity encoder (frozen +# embeddings) -> mean-pool aggregation -> linear head. Single-scale. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} From 44b47205a94ccf3d8b371a57c5fd258332d43976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 16:20:42 +0200 Subject: [PATCH 15/79] feat: regression running configs --- .../ml/train_e2e_vgg16_regression.yaml | 40 +++++++++++++++++++ .../ml/train_mil_embeddings_regression.yaml | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 configs/experiment/ml/train_e2e_vgg16_regression.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_regression.yaml diff --git a/configs/experiment/ml/train_e2e_vgg16_regression.yaml b/configs/experiment/ml/train_e2e_vgg16_regression.yaml new file mode 100644 index 00000000..ab486886 --- /dev/null +++ b/configs/experiment/ml/train_e2e_vgg16_regression.yaml @@ -0,0 +1,40 @@ +# @package _global_ +# End-to-end regression: VGG16 backbone over raw level-3 tiles -> mean-pool +# -> linear head (out_dim=1), regressing the MammaPrint index. Reads tile images +# via RawTileSlideDataset (OpenSlide). Same data/model as train_e2e_vgg16, but the +# regression task (label_mode=index, MSE, correlation metrics). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: vgg16 + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - /trainer: default + - /data/sources: single_l3_tiled + - _self_ + +label_mode: index +feature_dim: 512 # VGG16 global-max-pooled feature size + +metadata: + experiment_name: MammaPrint + run_name: End-to-end VGG16 — MammaPrint index regression (raw tiles) + description: | + End-to-end regression of the continuous MammaPrint index with a VGG16 backbone + over raw level-3 tiles and mean-pool aggregation. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml new file mode 100644 index 00000000..5b2ba509 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -0,0 +1,40 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings, REGRESSION of the MammaPrint index: +# identity encoder (frozen embeddings) -> mean-pool -> linear head (out_dim=1). +# Regression twin of train_mil_embeddings (label_mode=index, MSE, correlation metrics). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} From f149e5429e08cf09e78ff3c32eac960dd6304c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 20:54:01 +0200 Subject: [PATCH 16/79] feat: binary metrics (but for regression) --- .../ml/train_e2e_vgg16_regression.yaml | 4 + .../ml/train_mil_embeddings_regression.yaml | 4 + configs/experiment/ml/train_regression.yaml | 4 + configs/ml/metrics/thresholded_f1.yaml | 5 ++ configs/ml/metrics/thresholded_precision.yaml | 4 + configs/ml/metrics/thresholded_recall.yaml | 5 ++ .../ml/metrics/thresholded_specificity.yaml | 4 + ml/metrics/__init__.py | 18 +++++ ml/metrics/thresholded.py | 74 +++++++++++++++++++ 9 files changed, 122 insertions(+) create mode 100644 configs/ml/metrics/thresholded_f1.yaml create mode 100644 configs/ml/metrics/thresholded_precision.yaml create mode 100644 configs/ml/metrics/thresholded_recall.yaml create mode 100644 configs/ml/metrics/thresholded_specificity.yaml create mode 100644 ml/metrics/__init__.py create mode 100644 ml/metrics/thresholded.py diff --git a/configs/experiment/ml/train_e2e_vgg16_regression.yaml b/configs/experiment/ml/train_e2e_vgg16_regression.yaml index ab486886..9e43ea84 100644 --- a/configs/experiment/ml/train_e2e_vgg16_regression.yaml +++ b/configs/experiment/ml/train_e2e_vgg16_regression.yaml @@ -18,6 +18,10 @@ defaults: - pearson - spearman - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity - /trainer: default - /data/sources: single_l3_tiled - _self_ diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml index 5b2ba509..a7dc014f 100644 --- a/configs/experiment/ml/train_mil_embeddings_regression.yaml +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -17,6 +17,10 @@ defaults: - pearson - spearman - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity - /trainer: default - /data/sources: single_l3_embed - _self_ diff --git a/configs/experiment/ml/train_regression.yaml b/configs/experiment/ml/train_regression.yaml index 5e0db81e..17ebe568 100644 --- a/configs/experiment/ml/train_regression.yaml +++ b/configs/experiment/ml/train_regression.yaml @@ -16,6 +16,10 @@ defaults: - pearson - spearman - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity - /trainer: default - /data/sources: single_l3_embed - _self_ diff --git a/configs/ml/metrics/thresholded_f1.yaml b/configs/ml/metrics/thresholded_f1.yaml new file mode 100644 index 00000000..69851134 --- /dev/null +++ b/configs/ml/metrics/thresholded_f1.yaml @@ -0,0 +1,5 @@ +# @package ml.metrics +# Classification F1 from a regression output: index >= 0 -> class 1 (a luminal). +thresholded_f1: + _target_: ml.metrics.thresholded_f1 + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_precision.yaml b/configs/ml/metrics/thresholded_precision.yaml new file mode 100644 index 00000000..bab03590 --- /dev/null +++ b/configs/ml/metrics/thresholded_precision.yaml @@ -0,0 +1,4 @@ +# @package ml.metrics +thresholded_precision: + _target_: ml.metrics.thresholded_precision + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_recall.yaml b/configs/ml/metrics/thresholded_recall.yaml new file mode 100644 index 00000000..12f3f311 --- /dev/null +++ b/configs/ml/metrics/thresholded_recall.yaml @@ -0,0 +1,5 @@ +# @package ml.metrics +# recall == sensitivity == TP / (TP + FN) +thresholded_recall: + _target_: ml.metrics.thresholded_recall + threshold: 0.0 diff --git a/configs/ml/metrics/thresholded_specificity.yaml b/configs/ml/metrics/thresholded_specificity.yaml new file mode 100644 index 00000000..b81742d9 --- /dev/null +++ b/configs/ml/metrics/thresholded_specificity.yaml @@ -0,0 +1,4 @@ +# @package ml.metrics +thresholded_specificity: + _target_: ml.metrics.thresholded_specificity + threshold: 0.0 diff --git a/ml/metrics/__init__.py b/ml/metrics/__init__.py new file mode 100644 index 00000000..cbf9fbad --- /dev/null +++ b/ml/metrics/__init__.py @@ -0,0 +1,18 @@ +"""Custom metrics for MammaPrint.""" + +from ml.metrics.thresholded import ( + ThresholdedBinaryMetric, + thresholded_f1, + thresholded_precision, + thresholded_recall, + thresholded_specificity, +) + + +__all__ = [ + "ThresholdedBinaryMetric", + "thresholded_f1", + "thresholded_precision", + "thresholded_recall", + "thresholded_specificity", +] diff --git a/ml/metrics/thresholded.py b/ml/metrics/thresholded.py new file mode 100644 index 00000000..6a58345b --- /dev/null +++ b/ml/metrics/thresholded.py @@ -0,0 +1,74 @@ +"""Classification metrics computed from a regression output by thresholding. + +For the MammaPrint index, the sign is the class boundary: index ``>= 0`` is +"a luminal" (class 1), ``< 0`` is "b luminal" (class 0) — matching +``_map_luminal_type`` and the notebook's LUMINAL_INTERVAL rule. This lets a +*regression* model report classification metrics (F1 / precision / recall / +sensitivity / specificity) during training: binarize both prediction and target at +the threshold, then delegate to a standard binary metric. +""" + +from torch import Tensor +from torchmetrics import Metric +from torchmetrics.classification import ( + BinaryF1Score, + BinaryPrecision, + BinaryRecall, + BinarySpecificity, +) + + +class ThresholdedBinaryMetric(Metric): + """Wraps a binary metric, binarizing continuous preds/targets at a threshold. + + Both ``preds`` and ``target`` are mapped to ``{0, 1}`` via ``value >= threshold`` + (positive class = "a luminal", index >= 0), then forwarded to the wrapped metric. + + Args: + metric: The binary metric to delegate to (already instantiated). + threshold: Decision boundary applied to both preds and targets (default 0.0). + """ + + def __init__(self, metric: Metric, threshold: float = 0.0) -> None: + super().__init__() + self.metric = metric + self.threshold = threshold + + def update(self, preds: Tensor, target: Tensor) -> None: + pred_cls = (preds >= self.threshold).long() + target_cls = (target >= self.threshold).long() + self.metric.update(pred_cls, target_cls) + + def compute(self) -> Tensor: + return self.metric.compute() + + def reset(self) -> None: + super().reset() + self.metric.reset() + + +def thresholded_f1(threshold: float = 0.0) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryF1Score(), threshold) + + +def thresholded_precision(threshold: float = 0.0) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryPrecision(), threshold) + + +def thresholded_recall(threshold: float = 0.0) -> ThresholdedBinaryMetric: + """Recall (a.k.a. sensitivity) == TP / (TP + FN).""" + return ThresholdedBinaryMetric(BinaryRecall(), threshold) + + +def thresholded_specificity(threshold: float = 0.0) -> ThresholdedBinaryMetric: + """Specificity == TN / (TN + FP).""" + return ThresholdedBinaryMetric(BinarySpecificity(), threshold) + + +__all__ = [ + "ThresholdedBinaryMetric", + "thresholded_f1", + "thresholded_precision", + "thresholded_recall", + "thresholded_specificity", +] From 36942e86528c9d2bc8f24386c155f74ffe20b90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 21 Jul 2026 14:31:26 +0200 Subject: [PATCH 17/79] feat: more training configs --- .../ml/train_mil_embeddings_att.yaml | 41 +++++++++++++++++ .../train_mil_embeddings_att_regression.yaml | 44 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 configs/experiment/ml/train_mil_embeddings_att.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_att_regression.yaml diff --git a/configs/experiment/ml/train_mil_embeddings_att.yaml b/configs/experiment/ml/train_mil_embeddings_att.yaml new file mode 100644 index 00000000..e56ae060 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_att.yaml @@ -0,0 +1,41 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings: identity encoder (frozen +# embeddings) -> mean-pool aggregation -> linear head. Single-scale. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: attention + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_att_regression.yaml b/configs/experiment/ml/train_mil_embeddings_att_regression.yaml new file mode 100644 index 00000000..8f928fdd --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_att_regression.yaml @@ -0,0 +1,44 @@ +# @package _global_ +# Linear head on precomputed level-3 embeddings, REGRESSION of the MammaPrint index: +# identity encoder (frozen embeddings) -> mean-pool -> linear head (out_dim=1). +# Regression twin of train_mil_embeddings (label_mode=index, MSE, correlation metrics). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: attention + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size + +metadata: + experiment_name: MammaPrint + run_name: Linear head — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} From 8a8a0ef1147ddbd197ba35a0a5872a615204d80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 21 Jul 2026 19:29:22 +0200 Subject: [PATCH 18/79] feat: joint loss --- .../ml/train_mil_embeddings_joint.yaml | 47 ++++++++++++ .../ml/train_mil_embeddings_joint_att.yaml | 47 ++++++++++++ configs/ml.yaml | 10 ++- configs/ml/loss/joint.yaml | 12 +++ configs/ml/metrics/joint_auroc.yaml | 8 ++ configs/ml/metrics/joint_f1.yaml | 10 +++ configs/ml/metrics/joint_mse.yaml | 7 ++ configs/ml/metrics/joint_pearson.yaml | 7 ++ ml/data/datasets/_sources.py | 6 +- ml/data/datasets/labels.py | 43 ++++++++--- ml/metrics/__init__.py | 2 + ml/metrics/column.py | 38 ++++++++++ ml/models/losses/__init__.py | 4 + ml/models/losses/joint.py | 73 +++++++++++++++++++ 14 files changed, 296 insertions(+), 18 deletions(-) create mode 100644 configs/experiment/ml/train_mil_embeddings_joint.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_joint_att.yaml create mode 100644 configs/ml/loss/joint.yaml create mode 100644 configs/ml/metrics/joint_auroc.yaml create mode 100644 configs/ml/metrics/joint_f1.yaml create mode 100644 configs/ml/metrics/joint_mse.yaml create mode 100644 configs/ml/metrics/joint_pearson.yaml create mode 100644 ml/metrics/column.py create mode 100644 ml/models/losses/__init__.py create mode 100644 ml/models/losses/joint.py diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml new file mode 100644 index 00000000..8917b60f --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# Multi-task MIL on precomputed level-3 embeddings: identity encoder -> mean-pool +# -> MLP head with out_dim=2. The head emits a class logit (col 0) and a regression +# scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at +# once. Same data as train_mil_embeddings / train_regression, label_mode="both". +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: mean + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_auroc + - joint_f1 + - joint_mse + - joint_pearson + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: mean-pool + 2-output MLP head trained jointly on luminal type + (BCE) and the continuous MammaPrint index (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_joint_att.yaml b/configs/experiment/ml/train_mil_embeddings_joint_att.yaml new file mode 100644 index 00000000..85a3137d --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_joint_att.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# Multi-task MIL on precomputed level-3 embeddings: identity encoder -> mean-pool +# -> MLP head with out_dim=2. The head emits a class logit (col 0) and a regression +# scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at +# once. Same data as train_mil_embeddings / train_regression, label_mode="both". +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: attention + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_auroc + - joint_f1 + - joint_mse + - joint_pearson + - /trainer: default + - /data/sources: single_l3_embed + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: mean-pool + 2-output MLP head trained jointly on luminal type + (BCE) and the continuous MammaPrint index (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/ml.yaml b/configs/ml.yaml index fcfacb4a..0af14e22 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -10,10 +10,12 @@ seed: ${random_seed:} mode: fit # fit | validate | test | predict checkpoint: null # path to a .ckpt to resume (fit) or load weights (validate/test/predict) -# Task shape — set per experiment. The head is single-output (out_dim=1) either way: -# binary classification (label_mode=type, BCE + binary metrics) or regression -# (label_mode=index, MSE). No num_classes — the task is binary-or-regression. -label_mode: type # "type" (binary classification) or "index" (regression) +# Task shape — set per experiment. Single-output (out_dim=1) for the single-task +# modes: binary classification (label_mode=type, BCE + binary metrics) or +# regression (label_mode=index, MSE). The joint mode (label_mode=both) is +# multi-task: a 2-output head + JointLoss trains classification and regression +# together. No num_classes — the task is binary-or-regression. +label_mode: type # "type" (classification), "index" (regression), "both" (joint) feature_dim: ??? # tile-feature width = encoder.out_dim; set per experiment # datamodule / ml / trainer are composed by the +experiment config's defaults. \ No newline at end of file diff --git a/configs/ml/loss/joint.yaml b/configs/ml/loss/joint.yaml new file mode 100644 index 00000000..33249b95 --- /dev/null +++ b/configs/ml/loss/joint.yaml @@ -0,0 +1,12 @@ +# Multi-task classification + regression on the `both` label mode. +# Combines BCEWithLogits (luminal type) and MSE (MammaPrint index): +# total = w_cls * BCE(logit, type) + w_reg * MSE(scalar, index) +# +# Requires: head.out_dim=2 (col 0 = class logit, col 1 = regression scalar), +# output_activation=identity, and the dataset's label_mode="both". +# Tune w_cls / w_reg if one task dominates the gradient (the two terms live on +# different scales — BCE ~ O(1), MSE ~ the index variance). +_target_: ml.models.losses.JointLoss +w_cls: 1.0 +w_reg: 1.0 +pos_weight: null diff --git a/configs/ml/metrics/joint_auroc.yaml b/configs/ml/metrics/joint_auroc.yaml new file mode 100644 index 00000000..8d06e094 --- /dev/null +++ b/configs/ml/metrics/joint_auroc.yaml @@ -0,0 +1,8 @@ +# @package ml.metrics +# Classification AUROC on the joint head's column 0 (class logit vs type label). +# BinaryAUROC is threshold-free and consumes logits directly. +joint_auroc: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/joint_f1.yaml b/configs/ml/metrics/joint_f1.yaml new file mode 100644 index 00000000..d23e1345 --- /dev/null +++ b/configs/ml/metrics/joint_f1.yaml @@ -0,0 +1,10 @@ +# @package ml.metrics +# Classification F1 on the joint head's column 0. The class logit is thresholded +# at 0 (logit >= 0 <=> prob >= 0.5 <=> class 1), so BinaryF1Score is fed a +# hard {0,1} prediction against the {0,1} type label. +joint_f1: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: torchmetrics.classification.BinaryF1Score + threshold: 0.0 diff --git a/configs/ml/metrics/joint_mse.yaml b/configs/ml/metrics/joint_mse.yaml new file mode 100644 index 00000000..b39bbad8 --- /dev/null +++ b/configs/ml/metrics/joint_mse.yaml @@ -0,0 +1,7 @@ +# @package ml.metrics +# Regression MSE on the joint head's column 1 (index scalar vs MammaPrint index). +joint_mse: + _target_: ml.metrics.ColumnMetric + column: 1 + metric: + _target_: torchmetrics.MeanSquaredError diff --git a/configs/ml/metrics/joint_pearson.yaml b/configs/ml/metrics/joint_pearson.yaml new file mode 100644 index 00000000..6e7d71ba --- /dev/null +++ b/configs/ml/metrics/joint_pearson.yaml @@ -0,0 +1,7 @@ +# @package ml.metrics +# Regression Pearson correlation on the joint head's column 1. +joint_pearson: + _target_: ml.metrics.ColumnMetric + column: 1 + metric: + _target_: torchmetrics.PearsonCorrCoef diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py index d97ce3d0..244518b3 100644 --- a/ml/data/datasets/_sources.py +++ b/ml/data/datasets/_sources.py @@ -18,7 +18,7 @@ import pandas as pd from mlflow.artifacts import download_artifacts -from ml.data.datasets.labels import LabelMode, get_target_column, process_slides +from ml.data.datasets.labels import LabelMode, get_target_columns, process_slides logger = logging.getLogger(__name__) @@ -42,8 +42,8 @@ def load_labeled_slides( slides = pd.read_csv(data_mapping) slides = process_slides(slides, mode=label_mode) - target_column = get_target_column(label_mode) - keep = slides[target_column].notna() + target_columns = get_target_columns(label_mode) + keep = slides[target_columns].notna().all(axis=1) return slides[keep].reset_index(drop=True) diff --git a/ml/data/datasets/labels.py b/ml/data/datasets/labels.py index b2e6efcc..7465a78c 100644 --- a/ml/data/datasets/labels.py +++ b/ml/data/datasets/labels.py @@ -10,6 +10,7 @@ class LabelMode(Enum): TYPE = "type" INDEX = "index" + BOTH = "both" def _map_luminal_type(value: object) -> int: @@ -28,35 +29,55 @@ def process_slides(slides: pd.DataFrame, mode: LabelMode | None = None) -> pd.Da case LabelMode.INDEX: slides = slides.copy() slides["mammaprint_index"] = slides["mammaprint_index"].astype(float) + case LabelMode.BOTH: + slides = slides.copy() + slides["type_label"] = slides["type"].map(_map_luminal_type) + slides["mammaprint_index"] = slides["mammaprint_index"].astype(float) slides["name"] = slides["path"].apply(lambda x: Path(x).stem) return slides def get_label(slide_metadata: pd.Series, mode: LabelMode) -> torch.Tensor: - """One slide's target as a ``(1,)`` float tensor. + """One slide's target as a float tensor. - Both modes are single-output (``out_dim=1``): binary classification (luminal - a=1 / b=0, for ``BCEWithLogitsLoss`` + binary metrics) and regression (the - MammaPrint index). Shape ``(1,)`` matches the head's per-slide output so batched - preds/targets line up as ``(B, 1)``. + The single-task modes are single-output (``out_dim=1``): binary classification + (luminal a=1 / b=0, for ``BCEWithLogitsLoss`` + binary metrics) and regression + (the MammaPrint index). Shape ``(1,)`` matches the head's per-slide output so + batched preds/targets line up as ``(B, 1)``. + + ``BOTH`` is the multi-task target: ``[type_label, mammaprint_index]`` as a + ``(2,)`` tensor, column 0 the class and column 1 the index. Pair it with a + 2-output head and :class:`~ml.models.losses.joint.JointLoss`, which slices the + columns back apart. """ match mode: case LabelMode.TYPE: - value = float(slide_metadata["type_label"]) + values = [float(slide_metadata["type_label"])] case LabelMode.INDEX: - value = float(slide_metadata["mammaprint_index"]) + values = [float(slide_metadata["mammaprint_index"])] + case LabelMode.BOTH: + values = [ + float(slide_metadata["type_label"]), + float(slide_metadata["mammaprint_index"]), + ] case _: raise ValueError(f"Unsupported label mode: {mode}") - return torch.tensor([value], dtype=torch.float32) + return torch.tensor(values, dtype=torch.float32) + +def get_target_columns(mode: LabelMode) -> list[str]: + """Label column(s) that must be present for a slide to be usable in ``mode``. -def get_target_column(mode: LabelMode) -> str: + ``BOTH`` requires both columns, so a slide missing either is dropped. + """ match mode: case LabelMode.TYPE: - return "type_label" + return ["type_label"] case LabelMode.INDEX: - return "mammaprint_index" + return ["mammaprint_index"] + case LabelMode.BOTH: + return ["type_label", "mammaprint_index"] raise ValueError(f"Unsupported label mode: {mode}") diff --git a/ml/metrics/__init__.py b/ml/metrics/__init__.py index cbf9fbad..b9c78dfd 100644 --- a/ml/metrics/__init__.py +++ b/ml/metrics/__init__.py @@ -1,5 +1,6 @@ """Custom metrics for MammaPrint.""" +from ml.metrics.column import ColumnMetric from ml.metrics.thresholded import ( ThresholdedBinaryMetric, thresholded_f1, @@ -10,6 +11,7 @@ __all__ = [ + "ColumnMetric", "ThresholdedBinaryMetric", "thresholded_f1", "thresholded_precision", diff --git a/ml/metrics/column.py b/ml/metrics/column.py new file mode 100644 index 00000000..1f8f8abe --- /dev/null +++ b/ml/metrics/column.py @@ -0,0 +1,38 @@ +"""Select one column of a multi-task prediction/target before metricking. + +In the ``both`` label mode the head emits ``(B, 2)`` and the target is ``(B, 2)`` +— column 0 is the classification channel (logit / type label), column 1 the +regression channel (scalar / MammaPrint index). Standard torchmetrics expect a +single channel, so this wrapper slices out one column and delegates, letting the +existing binary and regression metrics be reused unchanged for the joint model. +""" + +from torch import Tensor +from torchmetrics import Metric + + +class ColumnMetric(Metric): + """Wraps a metric, feeding it one column of ``(B, 2)`` preds/targets. + + Args: + metric: The metric to delegate to (already instantiated). + column: Which column to select — ``0`` classification, ``1`` regression. + """ + + def __init__(self, metric: Metric, column: int) -> None: + super().__init__() + self.metric = metric + self.column = column + + def update(self, preds: Tensor, target: Tensor) -> None: + self.metric.update(preds[:, self.column], target[:, self.column]) + + def compute(self) -> Tensor: + return self.metric.compute() + + def reset(self) -> None: + super().reset() + self.metric.reset() + + +__all__ = ["ColumnMetric"] diff --git a/ml/models/losses/__init__.py b/ml/models/losses/__init__.py new file mode 100644 index 00000000..29ad7b0f --- /dev/null +++ b/ml/models/losses/__init__.py @@ -0,0 +1,4 @@ +from ml.models.losses.joint import JointLoss + + +__all__ = ["JointLoss"] diff --git a/ml/models/losses/joint.py b/ml/models/losses/joint.py new file mode 100644 index 00000000..6c1f01f6 --- /dev/null +++ b/ml/models/losses/joint.py @@ -0,0 +1,73 @@ +"""Joint classification + regression loss for the ``both`` label mode. + +The MammaPrint slide carries two aligned labels: a binary luminal type (a=1 / b=0) +and the continuous MammaPrint index. This loss trains a single model on both at +once — a standard hard-parameter-sharing multi-task setup where the encoder and +aggregator are shared and the head emits two outputs, one per task:: + + total = w_cls * BCEWithLogits(logit, type) + w_reg * MSE(scalar, index) + +Pair it with: + +* a 2-output head (``out_dim=2``): column 0 is the classification **logit**, + column 1 is the regression scalar; +* ``label_mode="both"``, so the target is ``(B, 2)`` = ``[type_label, index]``; +* ``output_activation=identity`` — BCE consumes raw logits, and the regression + column must stay unbounded. + +The two task losses live on different scales (BCE ~ O(1), MSE ~ the index +variance), so ``w_cls``/``w_reg`` trade them off. Defaults weight them equally; +tune them (or normalise the index) if one task dominates the gradient. +""" + +import torch +from torch import Tensor, nn + + +class JointLoss(nn.Module): + """Weighted sum of a binary-classification and a regression loss. + + Expects predictions and targets of shape ``(B, 2)`` where column 0 is the + classification channel (logit / type label) and column 1 is the regression + channel (scalar / MammaPrint index). + + Args: + w_cls: Weight on the classification (BCE-with-logits) term. + w_reg: Weight on the regression (MSE) term. + pos_weight: Optional positive-class weight passed to + :class:`~torch.nn.BCEWithLogitsLoss` to counter class imbalance. + """ + + def __init__( + self, + w_cls: float = 1.0, + w_reg: float = 1.0, + pos_weight: float | None = None, + ) -> None: + super().__init__() + self.w_cls = w_cls + self.w_reg = w_reg + # Registered as a buffer so BCE's pos_weight follows the module's device. + pw = None if pos_weight is None else torch.tensor([pos_weight]) + self.register_buffer("_pos_weight", pw) + self.cls_loss = nn.BCEWithLogitsLoss(pos_weight=self._pos_weight) + self.reg_loss = nn.MSELoss() + + def forward(self, y_pred: Tensor, y: Tensor) -> Tensor: + """Combine the per-task losses. + + Args: + y_pred: Head output ``(B, 2)`` — ``[:, 0]`` class logits, ``[:, 1]`` + regression scalars. + y: Targets ``(B, 2)`` — ``[:, 0]`` binary type labels, ``[:, 1]`` + MammaPrint indices. + + Returns: + Scalar total loss ``w_cls * BCE + w_reg * MSE``. + """ + cls = self.cls_loss(y_pred[:, 0], y[:, 0]) + reg = self.reg_loss(y_pred[:, 1], y[:, 1]) + return self.w_cls * cls + self.w_reg * reg + + +__all__ = ["JointLoss"] From f4b615fca7e5b3feea31c47f76651a0488a82a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 21 Jul 2026 20:12:02 +0200 Subject: [PATCH 19/79] feat: 2 layer mlp --- configs/ml/head/mlp.yaml | 9 +++++++ ml/models/heads/__init__.py | 3 ++- ml/models/heads/mlp.py | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 configs/ml/head/mlp.yaml create mode 100644 ml/models/heads/mlp.py diff --git a/configs/ml/head/mlp.yaml b/configs/ml/head/mlp.yaml new file mode 100644 index 00000000..b52cbd22 --- /dev/null +++ b/configs/ml/head/mlp.yaml @@ -0,0 +1,9 @@ +# Two-layer MLP head: a non-linear readout of the aggregated slide vector. +# Deeper alternative to head=linear; useful for linear probing on frozen +# embeddings where the head carries all task capacity. out_dim is 1 for +# binary classification (with BCE) or the regression scalar. +_target_: ml.models.heads.MLPHead +in_dim: ${feature_dim} +out_dim: 1 +hidden_dim: 256 +dropout: 0.5 diff --git a/ml/models/heads/__init__.py b/ml/models/heads/__init__.py index 1cab67ef..bbf21835 100644 --- a/ml/models/heads/__init__.py +++ b/ml/models/heads/__init__.py @@ -2,6 +2,7 @@ from ml.models.heads.base import Head from ml.models.heads.linear import LinearHead +from ml.models.heads.mlp import MLPHead -__all__ = ["Head", "LinearHead"] +__all__ = ["Head", "LinearHead", "MLPHead"] diff --git a/ml/models/heads/mlp.py b/ml/models/heads/mlp.py new file mode 100644 index 00000000..a9d8622b --- /dev/null +++ b/ml/models/heads/mlp.py @@ -0,0 +1,51 @@ +"""Two-layer MLP prediction head.""" + +from torch import Tensor, nn + +from ml.models.heads.base import Head + + +class MLPHead(Head): + """A two-layer MLP: dropout -> linear -> activation -> dropout -> linear. + + A deeper alternative to :class:`~ml.models.heads.LinearHead` for when the + aggregated slide vector benefits from a non-linear readout (e.g. linear + probing on frozen embeddings, where the head carries all task capacity). + The output is raw (logits / unbounded scalar); any activation is applied by + the module's ``output_activation``. + + Args: + in_dim: Dimensionality of the incoming slide vector. + out_dim: Number of outputs (``1`` for binary/regression, ``C`` for + multiclass). + hidden_dim: Width of the hidden layer. + dropout: Dropout probability applied before each linear layer. + """ + + def __init__( + self, + in_dim: int, + out_dim: int = 1, + hidden_dim: int = 256, + dropout: float = 0.5, + ) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim) + self.hidden_dim = hidden_dim + self.net = nn.Sequential( + nn.Dropout(p=dropout), + nn.Linear(in_dim, hidden_dim), + nn.ReLU(inplace=True), + nn.Dropout(p=dropout), + nn.Linear(hidden_dim, out_dim), + ) + + for layer in self.net: + if isinstance(layer, nn.Linear): + nn.init.xavier_uniform_(layer.weight) + nn.init.zeros_(layer.bias) + + def forward(self, features: Tensor) -> Tensor: + return self.net(features) + + +__all__ = ["MLPHead"] From 7495f4c801f617a26b5f245f13ad2708e24d8a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 24 Jul 2026 14:54:44 +0200 Subject: [PATCH 20/79] feat: new config structure --- configs/data/levels/l0.yaml | 9 +++++++++ configs/data/levels/l1.yaml | 9 +++++++++ configs/data/levels/l2.yaml | 9 +++++++++ configs/data/levels/l3.yaml | 9 +++++++++ configs/data/levels/l4.yaml | 9 +++++++++ configs/data/levels/l5.yaml | 9 +++++++++ configs/data/tiled/tissue_only/l2.yaml | 17 ++++++++++++++++ configs/data/tiled/tissue_only/l3.yaml | 17 ++++++++++++++++ .../embeddings/tissue_only/l2.yaml | 20 +++++++++++++++++++ .../embeddings/tissue_only/l3.yaml | 20 +++++++++++++++++++ .../preprocessing/tiling/_thresholds.yaml | 17 ++++++++++++++++ .../preprocessing/tiling/mou_0_224.yaml | 6 ------ .../preprocessing/tiling/mou_1_224.yaml | 6 ------ .../preprocessing/tiling/mou_2_224.yaml | 6 ------ .../preprocessing/tiling/mou_3_224.yaml | 6 ------ .../tiling/tissue_only/mou_0_224.yaml | 11 ++++++++++ .../tiling/tissue_only/mou_1_224.yaml | 11 ++++++++++ .../tiling/tissue_only/mou_2_224.yaml | 11 ++++++++++ .../tiling/tissue_only/mou_3_224.yaml | 11 ++++++++++ .../tiling/tissue_only/mou_4_224.yaml | 11 ++++++++++ .../tiling/tissue_only/mou_5_224.yaml | 11 ++++++++++ 21 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 configs/data/levels/l0.yaml create mode 100644 configs/data/levels/l1.yaml create mode 100644 configs/data/levels/l2.yaml create mode 100644 configs/data/levels/l3.yaml create mode 100644 configs/data/levels/l4.yaml create mode 100644 configs/data/levels/l5.yaml create mode 100644 configs/data/tiled/tissue_only/l2.yaml create mode 100644 configs/data/tiled/tissue_only/l3.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml create mode 100644 configs/experiment/preprocessing/tiling/_thresholds.yaml delete mode 100644 configs/experiment/preprocessing/tiling/mou_0_224.yaml delete mode 100644 configs/experiment/preprocessing/tiling/mou_1_224.yaml delete mode 100644 configs/experiment/preprocessing/tiling/mou_2_224.yaml delete mode 100644 configs/experiment/preprocessing/tiling/mou_3_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml diff --git a/configs/data/levels/l0.yaml b/configs/data/levels/l0.yaml new file mode 100644 index 00000000..30ec0f7a --- /dev/null +++ b/configs/data/levels/l0.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 0, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.0): - /data/levels/l0@_here_ +# tiling experiments (package _global_): - /data/levels/l0@_global_ +level: 0 +mpp: 0.24 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l1.yaml b/configs/data/levels/l1.yaml new file mode 100644 index 00000000..8c4f72d8 --- /dev/null +++ b/configs/data/levels/l1.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 1, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.1): - /data/levels/l1@_here_ +# tiling experiments (package _global_): - /data/levels/l1@_global_ +level: 1 +mpp: 0.485 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l2.yaml b/configs/data/levels/l2.yaml new file mode 100644 index 00000000..25dd689a --- /dev/null +++ b/configs/data/levels/l2.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 2, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.2): - /data/levels/l2@_here_ +# tiling experiments (package _global_): - /data/levels/l2@_global_ +level: 2 +mpp: 0.97 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l3.yaml b/configs/data/levels/l3.yaml new file mode 100644 index 00000000..75097836 --- /dev/null +++ b/configs/data/levels/l3.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 3, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.3): - /data/levels/l3@_here_ +# tiling experiments (package _global_): - /data/levels/l3@_global_ +level: 3 +mpp: 1.94 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l4.yaml b/configs/data/levels/l4.yaml new file mode 100644 index 00000000..e86ed2cf --- /dev/null +++ b/configs/data/levels/l4.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 4, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.4): - /data/levels/l4@_here_ +# tiling experiments (package _global_): - /data/levels/l4@_global_ +level: 4 +mpp: 3.88 +tile_extent: 224 +stride: 112 diff --git a/configs/data/levels/l5.yaml b/configs/data/levels/l5.yaml new file mode 100644 index 00000000..5ecffd88 --- /dev/null +++ b/configs/data/levels/l5.yaml @@ -0,0 +1,9 @@ +# Physical facts for pyramid level 5, shared across every mask variant and +# across the tiled card / embedded card / tiling experiment that use this level. +# Pulled in via a defaults @-redirect: +# variant cards (package dataset.levels.5): - /data/levels/l5@_here_ +# tiling experiments (package _global_): - /data/levels/l5@_global_ +level: 5 +mpp: 7.76 +tile_extent: 224 +stride: 112 diff --git a/configs/data/tiled/tissue_only/l2.yaml b/configs/data/tiled/tissue_only/l2.yaml new file mode 100644 index 00000000..a72ce3f1 --- /dev/null +++ b/configs/data/tiled/tissue_only/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l2; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l2_tissue +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_tissue +raw_uris: + all: "mlflow-artifacts:/3/0d8e51f080df435bbaa9b189b7973aac/artifacts/mou_2_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l2_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l2_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l2_tissue_test" diff --git a/configs/data/tiled/tissue_only/l3.yaml b/configs/data/tiled/tissue_only/l3.yaml new file mode 100644 index 00000000..3fcfc88b --- /dev/null +++ b/configs/data/tiled/tissue_only/l3.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.3 +# Level-3 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l3; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l3_tissue +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_tissue +raw_uris: + all: "mlflow-artifacts:/3/6b1ba82c3d29423f87004e5223e4cf99/artifacts/mou_3_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l3_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l3_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l3_tissue_test" diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml new file mode 100644 index 00000000..af5d6ae1 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml @@ -0,0 +1,20 @@ +# @package _global_ +# Level-2 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l2 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l2 + - /preprocessing: embeddings + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l2_tissue_embed + path: ${dataset.levels.2.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml new file mode 100644 index 00000000..96630313 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml @@ -0,0 +1,20 @@ +# @package _global_ +# Level-3 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l3 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l3 + - /preprocessing: embeddings + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l3_tissue_embed + path: ${dataset.levels.3.raw_uris.all} diff --git a/configs/experiment/preprocessing/tiling/_thresholds.yaml b/configs/experiment/preprocessing/tiling/_thresholds.yaml new file mode 100644 index 00000000..8cc2d41b --- /dev/null +++ b/configs/experiment/preprocessing/tiling/_thresholds.yaml @@ -0,0 +1,17 @@ +# @package _global_ +# Coverage cutoffs shared by every tiling experiment (keep tiles with overlap > +# threshold; higher overlap = cleaner). Defined once here and pulled into each +# tiling experiment via defaults. Chosen from the corrected coverage +# distributions in scripts/notebooks/mask_coverage_threshold.ipynb (mou_3_224 run): +# tissue 0.25 - on-tissue floor (center-50% ROI); raise to 0.5 for stricter purity (drops ~13%). +# blur 0.5 - drops the genuinely out-of-focus tail (~1% of tiles); lenient artifact gate. +# folding 0.1 - kept lenient: folding masks are slide-inconsistent, so a low gate avoids uneven +# filtering while still removing majority-folded tiles. +# residual 0.5 - drops heavily-contaminated tiles (some slides are ~66% residual artifact). +# Inherited across all levels; re-verify against a level's own distributions if +# you re-tile it (mask resolution differs by mpp) by overriding these in that +# experiment after `_thresholds` in its defaults list. +tissue_threshold: 0.25 +blur_threshold: 0.5 +folding_threshold: 0.1 +residual_threshold: 0.5 diff --git a/configs/experiment/preprocessing/tiling/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/mou_0_224.yaml deleted file mode 100644 index c6afa40a..00000000 --- a/configs/experiment/preprocessing/tiling/mou_0_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.24 # level 0 -tile_extent: 224 -stride: 112 -data_name: mou_0_224 diff --git a/configs/experiment/preprocessing/tiling/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/mou_1_224.yaml deleted file mode 100644 index fe4e9c71..00000000 --- a/configs/experiment/preprocessing/tiling/mou_1_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.485 # level 1 -tile_extent: 224 -stride: 112 -data_name: mou_1_224 diff --git a/configs/experiment/preprocessing/tiling/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/mou_2_224.yaml deleted file mode 100644 index 3fbfbf7c..00000000 --- a/configs/experiment/preprocessing/tiling/mou_2_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 0.97 # level 2 -tile_extent: 224 -stride: 112 -data_name: mou_2_224 diff --git a/configs/experiment/preprocessing/tiling/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/mou_3_224.yaml deleted file mode 100644 index 639523ae..00000000 --- a/configs/experiment/preprocessing/tiling/mou_3_224.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package _global_ - -mpp: 1.94 # level 3 -tile_extent: 224 -stride: 112 -data_name: mou_3_224 diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml new file mode 100644 index 00000000..e41023c5 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_0_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-0 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l0; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l0@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_0_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml new file mode 100644 index 00000000..ddc378c7 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_1_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-1 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l1; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l1@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_1_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml new file mode 100644 index 00000000..b74a380c --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_2_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-2 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l2; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml new file mode 100644 index 00000000..64476651 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_3_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-3 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l3; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml new file mode 100644 index 00000000..ea1f704e --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_4_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-4 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l4; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_tissue diff --git a/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml new file mode 100644 index 00000000..e93ec836 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/tissue_only/mou_5_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-5 tiling (tissue masks). mpp/tile_extent/stride come from +# /data/levels/l5; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_tissue From 0015a6f2f1211d95a53bfb3719b0674378c5d1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 27 Jul 2026 15:24:21 +0200 Subject: [PATCH 21/79] fix: configs, squash --- configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml | 1 - configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml index af5d6ae1..83b04427 100644 --- a/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l2.yaml @@ -11,7 +11,6 @@ # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - /data/tiled/tissue_only/l2 - - /preprocessing: embeddings - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml index 96630313..eb37ebae 100644 --- a/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l3.yaml @@ -11,7 +11,6 @@ # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - /data/tiled/tissue_only/l3 - - /preprocessing: embeddings - /preprocessing/tile_encoder: virchow2 - _self_ From ac10e372991f5913347c77149abb75e4a82649a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 24 Jul 2026 18:22:19 +0200 Subject: [PATCH 22/79] fix: joint f1 --- configs/ml/metrics/joint_f1.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/ml/metrics/joint_f1.yaml b/configs/ml/metrics/joint_f1.yaml index d23e1345..d4a0885d 100644 --- a/configs/ml/metrics/joint_f1.yaml +++ b/configs/ml/metrics/joint_f1.yaml @@ -1,10 +1,12 @@ # @package ml.metrics # Classification F1 on the joint head's column 0. The class logit is thresholded -# at 0 (logit >= 0 <=> prob >= 0.5 <=> class 1), so BinaryF1Score is fed a -# hard {0,1} prediction against the {0,1} type label. +# at 0 (logit >= 0 <=> prob >= 0.5 <=> class 1) by ThresholdedBinaryMetric, which +# binarizes the logit itself and feeds a hard {0,1} prediction to BinaryF1Score. +# A bare BinaryF1Score(threshold=0.0) does NOT do this — it treats preds as +# probabilities, ignores the logit values, and returns a constant F1. joint_f1: _target_: ml.metrics.ColumnMetric column: 0 metric: - _target_: torchmetrics.classification.BinaryF1Score + _target_: ml.metrics.thresholded_f1 threshold: 0.0 From 3b7d77fc681dbd39e63f5466aa179c3faef59893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 25 Jul 2026 15:24:25 +0200 Subject: [PATCH 23/79] feat: eval test set after fit --- configs/ml.yaml | 4 ++++ ml/train.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/configs/ml.yaml b/configs/ml.yaml index 0af14e22..8641b938 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -10,6 +10,10 @@ seed: ${random_seed:} mode: fit # fit | validate | test | predict checkpoint: null # path to a .ckpt to resume (fit) or load weights (validate/test/predict) +# After a `fit` run, evaluate the best checkpoint on the test split so models are +# directly comparable on held-out data right after training. Ignored unless mode=fit. +test_after_fit: true + # Task shape — set per experiment. Single-output (out_dim=1) for the single-task # modes: binary classification (label_mode=type, BCE + binary metrics) or # regression (label_mode=index, MSE). The joint mode (label_mode=both) is diff --git a/ml/train.py b/ml/train.py index da8e4c86..1f9c523c 100644 --- a/ml/train.py +++ b/ml/train.py @@ -12,6 +12,7 @@ uv run -m ml.train +experiment=ml/train_mil_embeddings mode=test checkpoint=/path/best.ckpt """ +import logging import random import hydra @@ -20,6 +21,9 @@ from rationai.mlkit.lightning.loggers import MLFlowLogger +log = logging.getLogger(__name__) + + # Resolves ${random_seed:} in the config; cached so all consumers share one seed. OmegaConf.register_new_resolver( "random_seed", lambda: random.randint(0, 2**31), use_cache=True, replace=True @@ -40,6 +44,17 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: run_stage = getattr(trainer, config.mode) run_stage(module, datamodule=datamodule, ckpt_path=config.checkpoint) + # After training, evaluate the best checkpoint on the test split so runs are + # directly comparable on held-out data. "best" points at the ModelCheckpoint's + # top-k pick; if checkpointing is off it's empty, so fall back to final weights. + if config.mode == "fit" and config.get("test_after_fit", False): + best_ckpt = getattr(trainer.checkpoint_callback, "best_model_path", "") or None + log.info( + "Running test on %s checkpoint after fit.", + "best" if best_ckpt else "final (no checkpoint saved)", + ) + trainer.test(module, datamodule=datamodule, ckpt_path=best_ckpt) + if __name__ == "__main__": main() From 9d0f7c937eb1a1eafe1f03da8397bb73ba543949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 14 Aug 2026 10:28:38 +0200 Subject: [PATCH 24/79] fix: calculate classification metrics from logits correctly --- .../ml/train_mil_embeddings_joint.yaml | 5 + .../ml/train_mil_embeddings_joint_att.yaml | 5 + configs/ml/metrics/binary_accuracy.yaml | 5 +- configs/ml/metrics/binary_auroc.yaml | 4 +- configs/ml/metrics/binary_cohen_kappa.yaml | 5 +- configs/ml/metrics/binary_f1.yaml | 5 +- configs/ml/metrics/binary_precision.yaml | 5 +- configs/ml/metrics/binary_recall.yaml | 5 +- configs/ml/metrics/binary_specificity.yaml | 5 +- configs/ml/metrics/joint_accuracy.yaml | 9 + configs/ml/metrics/joint_auroc.yaml | 7 +- configs/ml/metrics/joint_cohen_kappa.yaml | 9 + configs/ml/metrics/joint_f1.yaml | 13 +- configs/ml/metrics/joint_precision.yaml | 9 + configs/ml/metrics/joint_recall.yaml | 9 + configs/ml/metrics/joint_specificity.yaml | 9 + ml/metrics/__init__.py | 5 + ml/metrics/column.py | 18 +- ml/metrics/logits.py | 34 ++++ ml/metrics/summary.py | 112 ++++++++++ ml/metrics/thresholded.py | 60 +++--- tests/ml/test_metrics.py | 192 ++++++++++++++++++ 22 files changed, 480 insertions(+), 50 deletions(-) create mode 100644 configs/ml/metrics/joint_accuracy.yaml create mode 100644 configs/ml/metrics/joint_cohen_kappa.yaml create mode 100644 configs/ml/metrics/joint_precision.yaml create mode 100644 configs/ml/metrics/joint_recall.yaml create mode 100644 configs/ml/metrics/joint_specificity.yaml create mode 100644 ml/metrics/logits.py create mode 100644 ml/metrics/summary.py create mode 100644 tests/ml/test_metrics.py diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml index 8917b60f..2d188f69 100644 --- a/configs/experiment/ml/train_mil_embeddings_joint.yaml +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -13,8 +13,13 @@ defaults: - /ml/optimizer@ml.optimizer: adam - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - /ml/metrics: + - joint_accuracy - joint_auroc - joint_f1 + - joint_precision + - joint_recall + - joint_specificity + - joint_cohen_kappa - joint_mse - joint_pearson - /trainer: default diff --git a/configs/experiment/ml/train_mil_embeddings_joint_att.yaml b/configs/experiment/ml/train_mil_embeddings_joint_att.yaml index 85a3137d..74423a32 100644 --- a/configs/experiment/ml/train_mil_embeddings_joint_att.yaml +++ b/configs/experiment/ml/train_mil_embeddings_joint_att.yaml @@ -13,8 +13,13 @@ defaults: - /ml/optimizer@ml.optimizer: adam - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - /ml/metrics: + - joint_accuracy - joint_auroc - joint_f1 + - joint_precision + - joint_recall + - joint_specificity + - joint_cohen_kappa - joint_mse - joint_pearson - /trainer: default diff --git a/configs/ml/metrics/binary_accuracy.yaml b/configs/ml/metrics/binary_accuracy.yaml index ee74925f..960a8f2c 100644 --- a/configs/ml/metrics/binary_accuracy.yaml +++ b/configs/ml/metrics/binary_accuracy.yaml @@ -1,3 +1,6 @@ # @package ml.metrics accuracy: - _target_: torchmetrics.classification.BinaryAccuracy + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryAccuracy diff --git a/configs/ml/metrics/binary_auroc.yaml b/configs/ml/metrics/binary_auroc.yaml index 907fc01d..0379c072 100644 --- a/configs/ml/metrics/binary_auroc.yaml +++ b/configs/ml/metrics/binary_auroc.yaml @@ -1,3 +1,5 @@ # @package ml.metrics auroc: - _target_: torchmetrics.classification.BinaryAUROC + _target_: ml.metrics.BinaryLogitMetric + metric: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/binary_cohen_kappa.yaml b/configs/ml/metrics/binary_cohen_kappa.yaml index 9be6978f..ee0d72cd 100644 --- a/configs/ml/metrics/binary_cohen_kappa.yaml +++ b/configs/ml/metrics/binary_cohen_kappa.yaml @@ -1,3 +1,6 @@ # @package ml.metrics cohen_kappa: - _target_: torchmetrics.classification.BinaryCohenKappa + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryCohenKappa diff --git a/configs/ml/metrics/binary_f1.yaml b/configs/ml/metrics/binary_f1.yaml index 5ea30fb4..5a30e4e9 100644 --- a/configs/ml/metrics/binary_f1.yaml +++ b/configs/ml/metrics/binary_f1.yaml @@ -1,3 +1,6 @@ # @package ml.metrics f1: - _target_: torchmetrics.classification.BinaryF1Score + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryF1Score diff --git a/configs/ml/metrics/binary_precision.yaml b/configs/ml/metrics/binary_precision.yaml index 0adbc194..baee3604 100644 --- a/configs/ml/metrics/binary_precision.yaml +++ b/configs/ml/metrics/binary_precision.yaml @@ -1,3 +1,6 @@ # @package ml.metrics precision: - _target_: torchmetrics.classification.BinaryPrecision + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryPrecision diff --git a/configs/ml/metrics/binary_recall.yaml b/configs/ml/metrics/binary_recall.yaml index 3401fede..b9127f73 100644 --- a/configs/ml/metrics/binary_recall.yaml +++ b/configs/ml/metrics/binary_recall.yaml @@ -1,3 +1,6 @@ # @package ml.metrics recall: - _target_: torchmetrics.classification.BinaryRecall + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryRecall diff --git a/configs/ml/metrics/binary_specificity.yaml b/configs/ml/metrics/binary_specificity.yaml index 61568078..21170299 100644 --- a/configs/ml/metrics/binary_specificity.yaml +++ b/configs/ml/metrics/binary_specificity.yaml @@ -1,3 +1,6 @@ # @package ml.metrics specificity: - _target_: torchmetrics.classification.BinarySpecificity + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinarySpecificity diff --git a/configs/ml/metrics/joint_accuracy.yaml b/configs/ml/metrics/joint_accuracy.yaml new file mode 100644 index 00000000..ebf12eee --- /dev/null +++ b/configs/ml/metrics/joint_accuracy.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_accuracy: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryAccuracy diff --git a/configs/ml/metrics/joint_auroc.yaml b/configs/ml/metrics/joint_auroc.yaml index 8d06e094..30a07512 100644 --- a/configs/ml/metrics/joint_auroc.yaml +++ b/configs/ml/metrics/joint_auroc.yaml @@ -1,8 +1,11 @@ # @package ml.metrics # Classification AUROC on the joint head's column 0 (class logit vs type label). -# BinaryAUROC is threshold-free and consumes logits directly. +# Sigmoid is applied consistently on every update before BinaryAUROC. This keeps +# ranking correct for the slide dataloader's batch size of one. joint_auroc: _target_: ml.metrics.ColumnMetric column: 0 metric: - _target_: torchmetrics.classification.BinaryAUROC + _target_: ml.metrics.BinaryLogitMetric + metric: + _target_: torchmetrics.classification.BinaryAUROC diff --git a/configs/ml/metrics/joint_cohen_kappa.yaml b/configs/ml/metrics/joint_cohen_kappa.yaml new file mode 100644 index 00000000..e2088f54 --- /dev/null +++ b/configs/ml/metrics/joint_cohen_kappa.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_cohen_kappa: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryCohenKappa diff --git a/configs/ml/metrics/joint_f1.yaml b/configs/ml/metrics/joint_f1.yaml index d4a0885d..3ef3f577 100644 --- a/configs/ml/metrics/joint_f1.yaml +++ b/configs/ml/metrics/joint_f1.yaml @@ -1,12 +1,11 @@ # @package ml.metrics -# Classification F1 on the joint head's column 0. The class logit is thresholded -# at 0 (logit >= 0 <=> prob >= 0.5 <=> class 1) by ThresholdedBinaryMetric, which -# binarizes the logit itself and feeds a hard {0,1} prediction to BinaryF1Score. -# A bare BinaryF1Score(threshold=0.0) does NOT do this — it treats preds as -# probabilities, ignores the logit values, and returns a constant F1. +# Classification F1 on the joint head's column 0. BinaryLogitMetric thresholds +# logits at zero and leaves the already-binary targets unchanged. joint_f1: _target_: ml.metrics.ColumnMetric column: 0 metric: - _target_: ml.metrics.thresholded_f1 - threshold: 0.0 + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryF1Score diff --git a/configs/ml/metrics/joint_precision.yaml b/configs/ml/metrics/joint_precision.yaml new file mode 100644 index 00000000..9bb718b5 --- /dev/null +++ b/configs/ml/metrics/joint_precision.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_precision: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryPrecision diff --git a/configs/ml/metrics/joint_recall.yaml b/configs/ml/metrics/joint_recall.yaml new file mode 100644 index 00000000..a59d4dcc --- /dev/null +++ b/configs/ml/metrics/joint_recall.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_recall: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinaryRecall diff --git a/configs/ml/metrics/joint_specificity.yaml b/configs/ml/metrics/joint_specificity.yaml new file mode 100644 index 00000000..b9ac1734 --- /dev/null +++ b/configs/ml/metrics/joint_specificity.yaml @@ -0,0 +1,9 @@ +# @package ml.metrics +joint_specificity: + _target_: ml.metrics.ColumnMetric + column: 0 + metric: + _target_: ml.metrics.BinaryLogitMetric + decision_threshold: 0.0 + metric: + _target_: torchmetrics.classification.BinarySpecificity diff --git a/ml/metrics/__init__.py b/ml/metrics/__init__.py index b9c78dfd..6e8e1c72 100644 --- a/ml/metrics/__init__.py +++ b/ml/metrics/__init__.py @@ -1,6 +1,8 @@ """Custom metrics for MammaPrint.""" from ml.metrics.column import ColumnMetric +from ml.metrics.logits import BinaryLogitMetric +from ml.metrics.summary import binary_logit_summary, regression_summary from ml.metrics.thresholded import ( ThresholdedBinaryMetric, thresholded_f1, @@ -11,8 +13,11 @@ __all__ = [ + "BinaryLogitMetric", "ColumnMetric", "ThresholdedBinaryMetric", + "binary_logit_summary", + "regression_summary", "thresholded_f1", "thresholded_precision", "thresholded_recall", diff --git a/ml/metrics/column.py b/ml/metrics/column.py index 1f8f8abe..1ebe5837 100644 --- a/ml/metrics/column.py +++ b/ml/metrics/column.py @@ -9,9 +9,10 @@ from torch import Tensor from torchmetrics import Metric +from torchmetrics.wrappers import MetricInputTransformer -class ColumnMetric(Metric): +class ColumnMetric(MetricInputTransformer): """Wraps a metric, feeding it one column of ``(B, 2)`` preds/targets. Args: @@ -20,19 +21,14 @@ class ColumnMetric(Metric): """ def __init__(self, metric: Metric, column: int) -> None: - super().__init__() - self.metric = metric + super().__init__(metric) self.column = column - def update(self, preds: Tensor, target: Tensor) -> None: - self.metric.update(preds[:, self.column], target[:, self.column]) + def transform_pred(self, pred: Tensor) -> Tensor: + return pred[:, self.column] - def compute(self) -> Tensor: - return self.metric.compute() - - def reset(self) -> None: - super().reset() - self.metric.reset() + def transform_target(self, target: Tensor) -> Tensor: + return target[:, self.column] __all__ = ["ColumnMetric"] diff --git a/ml/metrics/logits.py b/ml/metrics/logits.py new file mode 100644 index 00000000..e146189c --- /dev/null +++ b/ml/metrics/logits.py @@ -0,0 +1,34 @@ +"""Binary metrics with explicit, batch-independent logit handling.""" + +import torch +from torch import Tensor +from torchmetrics import Metric +from torchmetrics.wrappers import MetricInputTransformer + + +class BinaryLogitMetric(MetricInputTransformer): + """Give a binary metric explicit, batch-independent logit semantics. + + TorchMetrics otherwise guesses whether a floating tensor contains logits or + probabilities independently for every update. With the slide dataloader's + batch size of one, that makes the same logit mean different things depending + on whether it falls inside ``[0, 1]``. This wrapper makes the contract explicit: + model predictions are logits and targets are already binary ``{0, 1}`` labels. + Ranking metrics receive sigmoid probabilities. Confusion-matrix metrics set a + ``decision_threshold`` and receive hard labels, using ``>=`` to match the + prediction CSV exactly. + """ + + def __init__( + self, metric: Metric, decision_threshold: float | None = None + ) -> None: + super().__init__(metric) + self.decision_threshold = decision_threshold + + def transform_pred(self, pred: Tensor) -> Tensor: + if self.decision_threshold is not None: + return (pred >= self.decision_threshold).long() + return torch.sigmoid(pred) + + +__all__ = ["BinaryLogitMetric"] diff --git a/ml/metrics/summary.py b/ml/metrics/summary.py new file mode 100644 index 00000000..01ec3480 --- /dev/null +++ b/ml/metrics/summary.py @@ -0,0 +1,112 @@ +"""Auditable cohort metrics computed from lossless prediction tables.""" + +import math + +import torch +from torch import Tensor +from torchmetrics.classification import BinaryAUROC +from torchmetrics.regression import ( + MeanAbsoluteError, + MeanSquaredError, + PearsonCorrCoef, + R2Score, + SpearmanCorrCoef, +) + + +def _divide(numerator: int | float, denominator: int | float) -> float: + return numerator / denominator if denominator else math.nan + + +def binary_logit_summary(logits: Tensor, targets: Tensor) -> dict[str, float]: + """Return binary metrics and confusion counts using logit threshold zero.""" + logits = logits.detach().float().flatten().cpu() + targets = targets.detach().long().flatten().cpu() + if logits.shape != targets.shape or logits.numel() == 0: + raise ValueError("Binary logits and targets must be non-empty equal shapes.") + if not torch.all(torch.isfinite(logits)): + raise ValueError("Binary logits must be finite.") + if not torch.all((targets == 0) | (targets == 1)): + raise ValueError("Binary targets must contain only 0 and 1.") + + predictions = (logits >= 0).long() + tp = int(((predictions == 1) & (targets == 1)).sum()) + tn = int(((predictions == 0) & (targets == 0)).sum()) + fp = int(((predictions == 1) & (targets == 0)).sum()) + fn = int(((predictions == 0) & (targets == 1)).sum()) + total = len(targets) + accuracy = (tp + tn) / total + precision = _divide(tp, tp + fp) + recall = _divide(tp, tp + fn) + specificity = _divide(tn, tn + fp) + f1 = _divide(2 * tp, 2 * tp + fp + fn) + + true_positive_rate = (tp + fn) / total + predicted_positive_rate = (tp + fp) / total + expected_agreement = ( + true_positive_rate * predicted_positive_rate + + (1 - true_positive_rate) * (1 - predicted_positive_rate) + ) + kappa = _divide(accuracy - expected_agreement, 1 - expected_agreement) + auroc = math.nan + if targets.unique().numel() == 2: + # Sigmoid is monotonic, but makes the input contract explicit and avoids + # TorchMetrics' per-update logit/probability auto-detection. + auroc = float(BinaryAUROC()(torch.sigmoid(logits), targets)) + + return { + "slides": float(total), + "tn": float(tn), + "fp": float(fp), + "fn": float(fn), + "tp": float(tp), + "accuracy": accuracy, + "precision": precision, + "recall": recall, + "specificity": specificity, + "f1": f1, + "cohen_kappa": kappa, + "auroc": auroc, + } + + +def regression_summary(predictions: Tensor, targets: Tensor) -> dict[str, float]: + """Return continuous and zero-threshold classification metrics.""" + predictions = predictions.detach().float().flatten().cpu() + targets = targets.detach().float().flatten().cpu() + if predictions.shape != targets.shape or predictions.numel() == 0: + raise ValueError( + "Regression predictions and targets must be non-empty equal shapes." + ) + if not torch.all(torch.isfinite(predictions)) or not torch.all( + torch.isfinite(targets) + ): + raise ValueError("Regression predictions and targets must be finite.") + + result = { + "mae": float(MeanAbsoluteError()(predictions, targets)), + "mse": float(MeanSquaredError()(predictions, targets)), + "pearson": math.nan, + "spearman": math.nan, + "r2": math.nan, + } + if predictions.numel() >= 2: + if predictions.std() > 0 and targets.std() > 0: + result["pearson"] = float(PearsonCorrCoef()(predictions, targets)) + result["spearman"] = float(SpearmanCorrCoef()(predictions, targets)) + if targets.var() > 0: + result["r2"] = float(R2Score()(predictions, targets)) + + thresholded = binary_logit_summary(predictions, (targets >= 0).long()) + result.update( + { + f"thresholded_{name}": value + for name, value in thresholded.items() + if name != "slides" + } + ) + result["slides"] = float(len(targets)) + return result + + +__all__ = ["binary_logit_summary", "regression_summary"] diff --git a/ml/metrics/thresholded.py b/ml/metrics/thresholded.py index 6a58345b..679fcbc5 100644 --- a/ml/metrics/thresholded.py +++ b/ml/metrics/thresholded.py @@ -16,9 +16,10 @@ BinaryRecall, BinarySpecificity, ) +from torchmetrics.wrappers import MetricInputTransformer -class ThresholdedBinaryMetric(Metric): +class ThresholdedBinaryMetric(MetricInputTransformer): """Wraps a binary metric, binarizing continuous preds/targets at a threshold. Both ``preds`` and ``target`` are mapped to ``{0, 1}`` via ``value >= threshold`` @@ -26,43 +27,56 @@ class ThresholdedBinaryMetric(Metric): Args: metric: The binary metric to delegate to (already instantiated). - threshold: Decision boundary applied to both preds and targets (default 0.0). + threshold: Decision boundary applied to predictions (default 0.0). + target_threshold: Decision boundary applied to continuous targets. Defaults + to ``threshold`` for regression-derived classification. Set this to + ``0.5`` when targets are already binary labels and predictions are + logits thresholded at zero. """ - def __init__(self, metric: Metric, threshold: float = 0.0) -> None: - super().__init__() - self.metric = metric + def __init__( + self, + metric: Metric, + threshold: float = 0.0, + target_threshold: float | None = None, + ) -> None: + super().__init__(metric) self.threshold = threshold + self.target_threshold = ( + threshold if target_threshold is None else target_threshold + ) - def update(self, preds: Tensor, target: Tensor) -> None: - pred_cls = (preds >= self.threshold).long() - target_cls = (target >= self.threshold).long() - self.metric.update(pred_cls, target_cls) + def transform_pred(self, pred: Tensor) -> Tensor: + return (pred >= self.threshold).long() - def compute(self) -> Tensor: - return self.metric.compute() + def transform_target(self, target: Tensor) -> Tensor: + return (target >= self.target_threshold).long() - def reset(self) -> None: - super().reset() - self.metric.reset() +def thresholded_f1( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryF1Score(), threshold, target_threshold) -def thresholded_f1(threshold: float = 0.0) -> ThresholdedBinaryMetric: - return ThresholdedBinaryMetric(BinaryF1Score(), threshold) +def thresholded_precision( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: + return ThresholdedBinaryMetric(BinaryPrecision(), threshold, target_threshold) -def thresholded_precision(threshold: float = 0.0) -> ThresholdedBinaryMetric: - return ThresholdedBinaryMetric(BinaryPrecision(), threshold) - -def thresholded_recall(threshold: float = 0.0) -> ThresholdedBinaryMetric: +def thresholded_recall( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: """Recall (a.k.a. sensitivity) == TP / (TP + FN).""" - return ThresholdedBinaryMetric(BinaryRecall(), threshold) + return ThresholdedBinaryMetric(BinaryRecall(), threshold, target_threshold) -def thresholded_specificity(threshold: float = 0.0) -> ThresholdedBinaryMetric: +def thresholded_specificity( + threshold: float = 0.0, target_threshold: float | None = None +) -> ThresholdedBinaryMetric: """Specificity == TN / (TN + FP).""" - return ThresholdedBinaryMetric(BinarySpecificity(), threshold) + return ThresholdedBinaryMetric(BinarySpecificity(), threshold, target_threshold) __all__ = [ diff --git a/tests/ml/test_metrics.py b/tests/ml/test_metrics.py new file mode 100644 index 00000000..36ddf693 --- /dev/null +++ b/tests/ml/test_metrics.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +import torch +from hydra.utils import instantiate +from omegaconf import OmegaConf +from torchmetrics import Metric + +from ml.metrics import ( + ColumnMetric, + binary_logit_summary, + regression_summary, + thresholded_f1, +) + + +METRICS_DIR = Path(__file__).resolve().parents[2] / "configs" / "ml" / "metrics" + + +def _configured_metric(name: str) -> Metric: + config = OmegaConf.load(METRICS_DIR / f"{name}.yaml") + metric = instantiate(next(iter(config.values()))) + if not isinstance(metric, Metric): + raise TypeError(f"{name} did not instantiate a TorchMetric") + return metric + + +def _stream(metric: Metric, predictions: torch.Tensor, targets: torch.Tensor) -> float: + for prediction, target in zip(predictions, targets, strict=True): + metric.update(prediction.unsqueeze(0), target.unsqueeze(0)) + return float(metric.compute()) + + +class BinaryLogitMetricsTest(unittest.TestCase): + def setUp(self) -> None: + # At logit threshold zero this gives TN=FP=FN=TP=1. Several logits are + # inside [0, 1], which exercises the former per-batch auto-detection bug. + self.logits = torch.tensor([-2.0, 0.2, 0.8, -0.1]) + self.targets = torch.tensor([0, 0, 1, 1]) + + def test_all_configured_binary_metrics_match_oracle_one_slide_at_a_time( + self, + ) -> None: + expected = { + "binary_accuracy": 0.5, + "binary_auroc": 0.75, + "binary_cohen_kappa": 0.0, + "binary_f1": 0.5, + "binary_precision": 0.5, + "binary_recall": 0.5, + "binary_specificity": 0.5, + } + for name, oracle in expected.items(): + with self.subTest(metric=name): + actual = _stream(_configured_metric(name), self.logits, self.targets) + self.assertAlmostEqual(actual, oracle) + + def test_one_slide_updates_equal_one_full_batch(self) -> None: + for name in ( + "binary_accuracy", + "binary_auroc", + "binary_cohen_kappa", + "binary_f1", + "binary_precision", + "binary_recall", + "binary_specificity", + ): + with self.subTest(metric=name): + streamed = _stream(_configured_metric(name), self.logits, self.targets) + batched_metric = _configured_metric(name) + batched_metric.update(self.logits, self.targets) + self.assertAlmostEqual(streamed, float(batched_metric.compute())) + + def test_auditable_summary_contains_the_same_confusion_matrix(self) -> None: + summary = binary_logit_summary(self.logits, self.targets) + self.assertEqual( + {key: summary[key] for key in ("tn", "fp", "fn", "tp")}, + {"tn": 1.0, "fp": 1.0, "fn": 1.0, "tp": 1.0}, + ) + self.assertEqual(summary["f1"], 0.5) + self.assertEqual(summary["auroc"], 0.75) + + def test_zero_logit_uses_the_same_positive_boundary_as_prediction_csv(self) -> None: + for name in ( + "binary_accuracy", + "binary_f1", + "binary_precision", + "binary_recall", + ): + with self.subTest(metric=name): + metric = _configured_metric(name) + metric.update(torch.tensor([0.0]), torch.tensor([1])) + self.assertEqual(float(metric.compute()), 1.0) + + +class JointMetricsTest(unittest.TestCase): + def setUp(self) -> None: + self.predictions = torch.tensor( + [[-2.0, -0.5], [0.2, 0.2], [0.8, 0.8], [-0.1, 1.1]] + ) + self.targets = torch.tensor( + [[0.0, -0.3], [0.0, 0.1], [1.0, 0.7], [1.0, 1.0]] + ) + + def test_joint_f1_preserves_binary_zero_targets(self) -> None: + self.assertAlmostEqual( + _stream( + _configured_metric("joint_f1"), self.predictions, self.targets + ), + 0.5, + ) + + def test_joint_auroc_is_batch_independent(self) -> None: + metric = _configured_metric("joint_auroc") + streamed = _stream(metric, self.predictions, self.targets) + metric.reset() + metric.update(self.predictions, self.targets) + self.assertAlmostEqual(streamed, 0.75) + self.assertAlmostEqual(float(metric.compute()), 0.75) + + def test_all_joint_classification_metrics_match_oracle(self) -> None: + expected = { + "joint_accuracy": 0.5, + "joint_auroc": 0.75, + "joint_cohen_kappa": 0.0, + "joint_f1": 0.5, + "joint_precision": 0.5, + "joint_recall": 0.5, + "joint_specificity": 0.5, + } + for name, oracle in expected.items(): + with self.subTest(metric=name): + actual = _stream( + _configured_metric(name), self.predictions, self.targets + ) + self.assertAlmostEqual(actual, oracle) + + def test_regression_columns_remain_untransformed(self) -> None: + mse = _configured_metric("joint_mse") + mse.update(self.predictions, self.targets) + expected = torch.mean((self.predictions[:, 1] - self.targets[:, 1]) ** 2) + torch.testing.assert_close(mse.compute(), expected) + + def test_wrappers_clone_with_independent_state(self) -> None: + metric = ColumnMetric( + thresholded_f1(threshold=0.0, target_threshold=0.5), column=0 + ) + cloned = metric.clone() + metric.update(self.predictions, self.targets) + cloned.update(self.predictions[:1], self.targets[:1]) + self.assertAlmostEqual(float(metric.compute()), 0.5) + self.assertEqual(float(cloned.compute()), 0.0) + + +class RegressionThresholdMetricsTest(unittest.TestCase): + def setUp(self) -> None: + self.predictions = torch.tensor([-0.8, -0.2, 0.9, -0.1]) + self.targets = torch.tensor([-1.0, -0.5, 0.3, 0.0]) + + def test_zero_is_positive_for_continuous_index_predictions_and_targets(self) -> None: + expected = { + "thresholded_f1": 2 / 3, + "thresholded_precision": 1.0, + "thresholded_recall": 0.5, + "thresholded_specificity": 1.0, + } + for name, oracle in expected.items(): + with self.subTest(metric=name): + actual = _stream( + _configured_metric(name), self.predictions, self.targets + ) + self.assertAlmostEqual(actual, oracle) + + def test_all_continuous_regression_metrics_match_auditable_summary(self) -> None: + summary = regression_summary(self.predictions, self.targets) + for name in ("mae", "mse", "pearson", "spearman", "r2"): + with self.subTest(metric=name): + metric = _configured_metric(name) + metric.update(self.predictions, self.targets) + self.assertAlmostEqual(float(metric.compute()), summary[name]) + + def test_auditable_regression_summary(self) -> None: + summary = regression_summary(self.predictions, self.targets) + self.assertAlmostEqual(summary["mae"], 0.3) + self.assertAlmostEqual(summary["mse"], 0.125) + self.assertAlmostEqual(summary["thresholded_f1"], 2 / 3) + + +if __name__ == "__main__": + unittest.main() From fba6d754944b75d9825e852234f377a092220ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 18:53:36 +0200 Subject: [PATCH 25/79] chore: trim modeling configs and metrics --- configs/data/_template.yaml | 6 +- configs/data/_template_raw.yaml | 7 +- configs/data/levels/l0.yaml | 5 - configs/data/levels/l1.yaml | 5 - configs/data/levels/l2.yaml | 5 - configs/data/levels/l3.yaml | 5 - configs/data/levels/l4.yaml | 5 - configs/data/levels/l5.yaml | 5 - configs/data/tiled/tissue_only/l2.yaml | 6 - configs/data/tiled/tissue_only/l3.yaml | 6 - configs/experiment/ml/train_e2e_vgg16.yaml | 39 ---- .../ml/train_e2e_vgg16_regression.yaml | 44 ---- .../experiment/ml/train_mil_embeddings.yaml | 1 - .../ml/train_mil_embeddings_att.yaml | 41 ---- .../train_mil_embeddings_att_regression.yaml | 44 ---- .../ml/train_mil_embeddings_joint.yaml | 2 - .../ml/train_mil_embeddings_joint_att.yaml | 52 ----- .../ml/train_mil_embeddings_regression.yaml | 5 +- configs/experiment/ml/train_regression.yaml | 41 ---- configs/ml/metrics/binary_cohen_kappa.yaml | 6 - configs/ml/metrics/joint_cohen_kappa.yaml | 9 - configs/ml/metrics/joint_pearson.yaml | 7 - configs/ml/metrics/pearson.yaml | 3 - configs/ml/metrics/r2.yaml | 3 - configs/ml/metrics/spearman.yaml | 3 - ml/metrics/summary.py | 20 -- tests/ml/test_metrics.py | 192 ------------------ 27 files changed, 4 insertions(+), 563 deletions(-) delete mode 100644 configs/experiment/ml/train_e2e_vgg16.yaml delete mode 100644 configs/experiment/ml/train_e2e_vgg16_regression.yaml delete mode 100644 configs/experiment/ml/train_mil_embeddings_att.yaml delete mode 100644 configs/experiment/ml/train_mil_embeddings_att_regression.yaml delete mode 100644 configs/experiment/ml/train_mil_embeddings_joint_att.yaml delete mode 100644 configs/experiment/ml/train_regression.yaml delete mode 100644 configs/ml/metrics/binary_cohen_kappa.yaml delete mode 100644 configs/ml/metrics/joint_cohen_kappa.yaml delete mode 100644 configs/ml/metrics/joint_pearson.yaml delete mode 100644 configs/ml/metrics/pearson.yaml delete mode 100644 configs/ml/metrics/r2.yaml delete mode 100644 configs/ml/metrics/spearman.yaml delete mode 100644 tests/ml/test_metrics.py diff --git a/configs/data/_template.yaml b/configs/data/_template.yaml index 6055314d..854edfc9 100644 --- a/configs/data/_template.yaml +++ b/configs/data/_template.yaml @@ -1,10 +1,6 @@ # @package _global_ -# Single-scale data shape: DataModule over SingleScaleDataset (flat embedding bags). -# A data/sources/* selector pulls this in plus ONE data/tiled/* card (one level). -# The dataset spec is written ONCE (the &dataset anchor on `train`); val/test merge -# it and override only `split`. Splits are built lazily per stage in DataModule.setup. dataset: - levels: ??? # filled by the selected data/tiled/* card + levels: ??? data_mapping: ${project_path}/data_mapping.csv datamodule: diff --git a/configs/data/_template_raw.yaml b/configs/data/_template_raw.yaml index 813e9920..ebc1afbf 100644 --- a/configs/data/_template_raw.yaml +++ b/configs/data/_template_raw.yaml @@ -1,11 +1,8 @@ # @package _global_ -# Raw-tile data shape: DataModule over RawTileSlideDataset (image bags). -# A data/sources/* selector pulls this in plus ONE data/tiled/* card (with raw_uris). -# Dataset spec written once (&dataset on `train`); val/test override only `split`. dataset: - levels: ??? # filled by the selected data/tiled/* card + levels: ??? data_mapping: ${project_path}/data_mapping.csv - max_tiles: 256 # per-slide tile cap (random-sampled if exceeded) + max_tiles: 256 datamodule: _target_: ml.data.DataModule diff --git a/configs/data/levels/l0.yaml b/configs/data/levels/l0.yaml index 30ec0f7a..5d83b8a4 100644 --- a/configs/data/levels/l0.yaml +++ b/configs/data/levels/l0.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 0, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.0): - /data/levels/l0@_here_ -# tiling experiments (package _global_): - /data/levels/l0@_global_ level: 0 mpp: 0.24 tile_extent: 224 diff --git a/configs/data/levels/l1.yaml b/configs/data/levels/l1.yaml index 8c4f72d8..e87c57d8 100644 --- a/configs/data/levels/l1.yaml +++ b/configs/data/levels/l1.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 1, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.1): - /data/levels/l1@_here_ -# tiling experiments (package _global_): - /data/levels/l1@_global_ level: 1 mpp: 0.485 tile_extent: 224 diff --git a/configs/data/levels/l2.yaml b/configs/data/levels/l2.yaml index 25dd689a..6f8097f3 100644 --- a/configs/data/levels/l2.yaml +++ b/configs/data/levels/l2.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 2, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.2): - /data/levels/l2@_here_ -# tiling experiments (package _global_): - /data/levels/l2@_global_ level: 2 mpp: 0.97 tile_extent: 224 diff --git a/configs/data/levels/l3.yaml b/configs/data/levels/l3.yaml index 75097836..9c57dc22 100644 --- a/configs/data/levels/l3.yaml +++ b/configs/data/levels/l3.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 3, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.3): - /data/levels/l3@_here_ -# tiling experiments (package _global_): - /data/levels/l3@_global_ level: 3 mpp: 1.94 tile_extent: 224 diff --git a/configs/data/levels/l4.yaml b/configs/data/levels/l4.yaml index e86ed2cf..99a2499f 100644 --- a/configs/data/levels/l4.yaml +++ b/configs/data/levels/l4.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 4, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.4): - /data/levels/l4@_here_ -# tiling experiments (package _global_): - /data/levels/l4@_global_ level: 4 mpp: 3.88 tile_extent: 224 diff --git a/configs/data/levels/l5.yaml b/configs/data/levels/l5.yaml index 5ecffd88..61533e3a 100644 --- a/configs/data/levels/l5.yaml +++ b/configs/data/levels/l5.yaml @@ -1,8 +1,3 @@ -# Physical facts for pyramid level 5, shared across every mask variant and -# across the tiled card / embedded card / tiling experiment that use this level. -# Pulled in via a defaults @-redirect: -# variant cards (package dataset.levels.5): - /data/levels/l5@_here_ -# tiling experiments (package _global_): - /data/levels/l5@_global_ level: 5 mpp: 7.76 tile_extent: 224 diff --git a/configs/data/tiled/tissue_only/l2.yaml b/configs/data/tiled/tissue_only/l2.yaml index a72ce3f1..92849f62 100644 --- a/configs/data/tiled/tissue_only/l2.yaml +++ b/configs/data/tiled/tissue_only/l2.yaml @@ -1,10 +1,4 @@ # @package dataset.levels.2 -# Level-2 RAW-TILE card (tissue masks). Physical facts come from -# /data/levels/l2; only the variant name + artifact URIs live here. -# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings -# step. Per-split URIs are produced from it by: -# python scripts/preprocessing/split_dataset.py --kind tiled \ -# --uri --name l2_tissue defaults: - /data/levels/l2@_here_ - _self_ diff --git a/configs/data/tiled/tissue_only/l3.yaml b/configs/data/tiled/tissue_only/l3.yaml index 3fcfc88b..631badef 100644 --- a/configs/data/tiled/tissue_only/l3.yaml +++ b/configs/data/tiled/tissue_only/l3.yaml @@ -1,10 +1,4 @@ # @package dataset.levels.3 -# Level-3 RAW-TILE card (tissue masks). Physical facts come from -# /data/levels/l3; only the variant name + artifact URIs live here. -# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings -# step. Per-split URIs are produced from it by: -# python scripts/preprocessing/split_dataset.py --kind tiled \ -# --uri --name l3_tissue defaults: - /data/levels/l3@_here_ - _self_ diff --git a/configs/experiment/ml/train_e2e_vgg16.yaml b/configs/experiment/ml/train_e2e_vgg16.yaml deleted file mode 100644 index 85b969b8..00000000 --- a/configs/experiment/ml/train_e2e_vgg16.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# @package _global_ -# End-to-end classification: VGG16 backbone over raw level-3 tiles -> mean-pool -# -> linear head. Reads tile images via RawTileSlideDataset (OpenSlide). -defaults: - - /ml: module - - /ml/encoder@ml.encoder: vgg16 - - /ml/aggregator@ml.aggregator: mean - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: bce - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - binary_accuracy - - binary_auroc - - binary_f1 - - binary_precision - - binary_recall - - binary_specificity - - binary_cohen_kappa - - /trainer: default - - /data/sources: single_l3_tiled - - _self_ - -label_mode: type -feature_dim: 512 # VGG16 global-max-pooled feature size - -metadata: - experiment_name: MammaPrint - run_name: End-to-end VGG16 — luminal classification (raw tiles) - description: | - End-to-end luminal classification with a VGG16 backbone over raw tiles and - mean-pool aggregation. Backbone/aggregator/head are all Hydra-swappable. - hyperparameters: - encoder: ${ml.encoder._target_} - aggregator: ${ml.aggregator._target_} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_e2e_vgg16_regression.yaml b/configs/experiment/ml/train_e2e_vgg16_regression.yaml deleted file mode 100644 index 9e43ea84..00000000 --- a/configs/experiment/ml/train_e2e_vgg16_regression.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# @package _global_ -# End-to-end regression: VGG16 backbone over raw level-3 tiles -> mean-pool -# -> linear head (out_dim=1), regressing the MammaPrint index. Reads tile images -# via RawTileSlideDataset (OpenSlide). Same data/model as train_e2e_vgg16, but the -# regression task (label_mode=index, MSE, correlation metrics). -defaults: - - /ml: module - - /ml/encoder@ml.encoder: vgg16 - - /ml/aggregator@ml.aggregator: mean - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: mse - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - mae - - mse - - pearson - - spearman - - r2 - - thresholded_f1 - - thresholded_precision - - thresholded_recall - - thresholded_specificity - - /trainer: default - - /data/sources: single_l3_tiled - - _self_ - -label_mode: index -feature_dim: 512 # VGG16 global-max-pooled feature size - -metadata: - experiment_name: MammaPrint - run_name: End-to-end VGG16 — MammaPrint index regression (raw tiles) - description: | - End-to-end regression of the continuous MammaPrint index with a VGG16 backbone - over raw level-3 tiles and mean-pool aggregation. - hyperparameters: - encoder: ${ml.encoder._target_} - aggregator: ${ml.aggregator._target_} - loss: ${ml.loss._target_} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings.yaml b/configs/experiment/ml/train_mil_embeddings.yaml index edbab26a..9e2a6335 100644 --- a/configs/experiment/ml/train_mil_embeddings.yaml +++ b/configs/experiment/ml/train_mil_embeddings.yaml @@ -17,7 +17,6 @@ defaults: - binary_precision - binary_recall - binary_specificity - - binary_cohen_kappa - /trainer: default - /data/sources: single_l3_embed - _self_ diff --git a/configs/experiment/ml/train_mil_embeddings_att.yaml b/configs/experiment/ml/train_mil_embeddings_att.yaml deleted file mode 100644 index e56ae060..00000000 --- a/configs/experiment/ml/train_mil_embeddings_att.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# @package _global_ -# Linear head on precomputed level-3 embeddings: identity encoder (frozen -# embeddings) -> mean-pool aggregation -> linear head. Single-scale. -defaults: - - /ml: module - - /ml/encoder@ml.encoder: identity - - /ml/aggregator@ml.aggregator: attention - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: bce - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - binary_accuracy - - binary_auroc - - binary_f1 - - binary_precision - - binary_recall - - binary_specificity - - binary_cohen_kappa - - /trainer: default - - /data/sources: single_l3_embed - - _self_ - -label_mode: type -feature_dim: 2560 # Virchow2 embedding size - -metadata: - experiment_name: MammaPrint - run_name: Linear head — luminal classification (level-3 Virchow2 embeddings) - description: | - Multiple-instance-learning classification of luminal type (A/B) over frozen - per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. - hyperparameters: - encoder: ${ml.encoder._target_} - aggregator: ${ml.aggregator._target_} - head: ${ml.head._target_} - loss: ${ml.loss._target_} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_att_regression.yaml b/configs/experiment/ml/train_mil_embeddings_att_regression.yaml deleted file mode 100644 index 8f928fdd..00000000 --- a/configs/experiment/ml/train_mil_embeddings_att_regression.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# @package _global_ -# Linear head on precomputed level-3 embeddings, REGRESSION of the MammaPrint index: -# identity encoder (frozen embeddings) -> mean-pool -> linear head (out_dim=1). -# Regression twin of train_mil_embeddings (label_mode=index, MSE, correlation metrics). -defaults: - - /ml: module - - /ml/encoder@ml.encoder: identity - - /ml/aggregator@ml.aggregator: attention - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: mse - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - mae - - mse - - pearson - - spearman - - r2 - - thresholded_f1 - - thresholded_precision - - thresholded_recall - - thresholded_specificity - - /trainer: default - - /data/sources: single_l3_embed - - _self_ - -label_mode: index -feature_dim: 2560 # Virchow2 embedding size - -metadata: - experiment_name: MammaPrint - run_name: Linear head — MammaPrint index regression (level-3 Virchow2 embeddings) - description: | - Multiple-instance-learning regression of the continuous MammaPrint index over - frozen per-slide level-3 Virchow2 tile embeddings, mean-pool aggregation + linear head. - hyperparameters: - encoder: ${ml.encoder._target_} - aggregator: ${ml.aggregator._target_} - head: ${ml.head._target_} - loss: ${ml.loss._target_} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml index 2d188f69..400c7aee 100644 --- a/configs/experiment/ml/train_mil_embeddings_joint.yaml +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -19,9 +19,7 @@ defaults: - joint_precision - joint_recall - joint_specificity - - joint_cohen_kappa - joint_mse - - joint_pearson - /trainer: default - /data/sources: single_l3_embed - _self_ diff --git a/configs/experiment/ml/train_mil_embeddings_joint_att.yaml b/configs/experiment/ml/train_mil_embeddings_joint_att.yaml deleted file mode 100644 index 74423a32..00000000 --- a/configs/experiment/ml/train_mil_embeddings_joint_att.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# @package _global_ -# Multi-task MIL on precomputed level-3 embeddings: identity encoder -> mean-pool -# -> MLP head with out_dim=2. The head emits a class logit (col 0) and a regression -# scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at -# once. Same data as train_mil_embeddings / train_regression, label_mode="both". -defaults: - - /ml: module - - /ml/encoder@ml.encoder: identity - - /ml/aggregator@ml.aggregator: attention - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: joint - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - joint_accuracy - - joint_auroc - - joint_f1 - - joint_precision - - joint_recall - - joint_specificity - - joint_cohen_kappa - - joint_mse - - joint_pearson - - /trainer: default - - /data/sources: single_l3_embed - - _self_ - -label_mode: both -feature_dim: 2560 # Virchow2 embedding size - -# Two outputs: [class logit, regression scalar]. -ml: - head: - out_dim: 2 - -metadata: - experiment_name: MammaPrint - run_name: Joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) - description: | - Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 - tile embeddings: mean-pool + 2-output MLP head trained jointly on luminal type - (BCE) and the continuous MammaPrint index (MSE) via JointLoss. - hyperparameters: - aggregator: ${ml.aggregator._target_} - head: ${ml.head._target_} - loss: ${ml.loss._target_} - w_cls: ${ml.loss.w_cls} - w_reg: ${ml.loss.w_reg} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml index a7dc014f..bf007d35 100644 --- a/configs/experiment/ml/train_mil_embeddings_regression.yaml +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -1,7 +1,7 @@ # @package _global_ # Linear head on precomputed level-3 embeddings, REGRESSION of the MammaPrint index: # identity encoder (frozen embeddings) -> mean-pool -> linear head (out_dim=1). -# Regression twin of train_mil_embeddings (label_mode=index, MSE, correlation metrics). +# Regression twin of train_mil_embeddings (label_mode=index, MSE). defaults: - /ml: module - /ml/encoder@ml.encoder: identity @@ -14,9 +14,6 @@ defaults: - /ml/metrics: - mae - mse - - pearson - - spearman - - r2 - thresholded_f1 - thresholded_precision - thresholded_recall diff --git a/configs/experiment/ml/train_regression.yaml b/configs/experiment/ml/train_regression.yaml deleted file mode 100644 index 17ebe568..00000000 --- a/configs/experiment/ml/train_regression.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# @package _global_ -# Regression of the MammaPrint index on precomputed level-3 embeddings: identity -# encoder -> mean-pool -> linear head (out_dim=1). Same data as train_mil_embeddings. -defaults: - - /ml: module - - /ml/encoder@ml.encoder: identity - - /ml/aggregator@ml.aggregator: mean - - /ml/head@ml.head: linear - - /ml/output_activation@ml.output_activation: identity - - /ml/loss@ml.loss: mse - - /ml/optimizer@ml.optimizer: adam - - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - - /ml/metrics: - - mae - - mse - - pearson - - spearman - - r2 - - thresholded_f1 - - thresholded_precision - - thresholded_recall - - thresholded_specificity - - /trainer: default - - /data/sources: single_l3_embed - - _self_ - -label_mode: index -feature_dim: 2560 # Virchow2 embedding size - -metadata: - experiment_name: MammaPrint - run_name: Regression — MammaPrint index (level-3 Virchow2 embeddings) - description: | - Multiple-instance-learning regression of the continuous MammaPrint index over - frozen per-slide level-3 Virchow2 tile embeddings, mean-pool + linear head. - hyperparameters: - aggregator: ${ml.aggregator._target_} - loss: ${ml.loss._target_} - lr: ${ml.optimizer.lr} - batch_size: ${datamodule.batch_size} - max_epochs: ${trainer.max_epochs} diff --git a/configs/ml/metrics/binary_cohen_kappa.yaml b/configs/ml/metrics/binary_cohen_kappa.yaml deleted file mode 100644 index ee0d72cd..00000000 --- a/configs/ml/metrics/binary_cohen_kappa.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# @package ml.metrics -cohen_kappa: - _target_: ml.metrics.BinaryLogitMetric - decision_threshold: 0.0 - metric: - _target_: torchmetrics.classification.BinaryCohenKappa diff --git a/configs/ml/metrics/joint_cohen_kappa.yaml b/configs/ml/metrics/joint_cohen_kappa.yaml deleted file mode 100644 index e2088f54..00000000 --- a/configs/ml/metrics/joint_cohen_kappa.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# @package ml.metrics -joint_cohen_kappa: - _target_: ml.metrics.ColumnMetric - column: 0 - metric: - _target_: ml.metrics.BinaryLogitMetric - decision_threshold: 0.0 - metric: - _target_: torchmetrics.classification.BinaryCohenKappa diff --git a/configs/ml/metrics/joint_pearson.yaml b/configs/ml/metrics/joint_pearson.yaml deleted file mode 100644 index 6e7d71ba..00000000 --- a/configs/ml/metrics/joint_pearson.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# @package ml.metrics -# Regression Pearson correlation on the joint head's column 1. -joint_pearson: - _target_: ml.metrics.ColumnMetric - column: 1 - metric: - _target_: torchmetrics.PearsonCorrCoef diff --git a/configs/ml/metrics/pearson.yaml b/configs/ml/metrics/pearson.yaml deleted file mode 100644 index 00ddc961..00000000 --- a/configs/ml/metrics/pearson.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# @package ml.metrics -pearson: - _target_: torchmetrics.PearsonCorrCoef diff --git a/configs/ml/metrics/r2.yaml b/configs/ml/metrics/r2.yaml deleted file mode 100644 index c1969749..00000000 --- a/configs/ml/metrics/r2.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# @package ml.metrics -r2: - _target_: torchmetrics.R2Score diff --git a/configs/ml/metrics/spearman.yaml b/configs/ml/metrics/spearman.yaml deleted file mode 100644 index 1fa4508f..00000000 --- a/configs/ml/metrics/spearman.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# @package ml.metrics -spearman: - _target_: torchmetrics.SpearmanCorrCoef diff --git a/ml/metrics/summary.py b/ml/metrics/summary.py index 01ec3480..269c2997 100644 --- a/ml/metrics/summary.py +++ b/ml/metrics/summary.py @@ -8,9 +8,6 @@ from torchmetrics.regression import ( MeanAbsoluteError, MeanSquaredError, - PearsonCorrCoef, - R2Score, - SpearmanCorrCoef, ) @@ -41,13 +38,6 @@ def binary_logit_summary(logits: Tensor, targets: Tensor) -> dict[str, float]: specificity = _divide(tn, tn + fp) f1 = _divide(2 * tp, 2 * tp + fp + fn) - true_positive_rate = (tp + fn) / total - predicted_positive_rate = (tp + fp) / total - expected_agreement = ( - true_positive_rate * predicted_positive_rate - + (1 - true_positive_rate) * (1 - predicted_positive_rate) - ) - kappa = _divide(accuracy - expected_agreement, 1 - expected_agreement) auroc = math.nan if targets.unique().numel() == 2: # Sigmoid is monotonic, but makes the input contract explicit and avoids @@ -65,7 +55,6 @@ def binary_logit_summary(logits: Tensor, targets: Tensor) -> dict[str, float]: "recall": recall, "specificity": specificity, "f1": f1, - "cohen_kappa": kappa, "auroc": auroc, } @@ -86,16 +75,7 @@ def regression_summary(predictions: Tensor, targets: Tensor) -> dict[str, float] result = { "mae": float(MeanAbsoluteError()(predictions, targets)), "mse": float(MeanSquaredError()(predictions, targets)), - "pearson": math.nan, - "spearman": math.nan, - "r2": math.nan, } - if predictions.numel() >= 2: - if predictions.std() > 0 and targets.std() > 0: - result["pearson"] = float(PearsonCorrCoef()(predictions, targets)) - result["spearman"] = float(SpearmanCorrCoef()(predictions, targets)) - if targets.var() > 0: - result["r2"] = float(R2Score()(predictions, targets)) thresholded = binary_logit_summary(predictions, (targets >= 0).long()) result.update( diff --git a/tests/ml/test_metrics.py b/tests/ml/test_metrics.py deleted file mode 100644 index 36ddf693..00000000 --- a/tests/ml/test_metrics.py +++ /dev/null @@ -1,192 +0,0 @@ -from __future__ import annotations - -import unittest -from pathlib import Path - -import torch -from hydra.utils import instantiate -from omegaconf import OmegaConf -from torchmetrics import Metric - -from ml.metrics import ( - ColumnMetric, - binary_logit_summary, - regression_summary, - thresholded_f1, -) - - -METRICS_DIR = Path(__file__).resolve().parents[2] / "configs" / "ml" / "metrics" - - -def _configured_metric(name: str) -> Metric: - config = OmegaConf.load(METRICS_DIR / f"{name}.yaml") - metric = instantiate(next(iter(config.values()))) - if not isinstance(metric, Metric): - raise TypeError(f"{name} did not instantiate a TorchMetric") - return metric - - -def _stream(metric: Metric, predictions: torch.Tensor, targets: torch.Tensor) -> float: - for prediction, target in zip(predictions, targets, strict=True): - metric.update(prediction.unsqueeze(0), target.unsqueeze(0)) - return float(metric.compute()) - - -class BinaryLogitMetricsTest(unittest.TestCase): - def setUp(self) -> None: - # At logit threshold zero this gives TN=FP=FN=TP=1. Several logits are - # inside [0, 1], which exercises the former per-batch auto-detection bug. - self.logits = torch.tensor([-2.0, 0.2, 0.8, -0.1]) - self.targets = torch.tensor([0, 0, 1, 1]) - - def test_all_configured_binary_metrics_match_oracle_one_slide_at_a_time( - self, - ) -> None: - expected = { - "binary_accuracy": 0.5, - "binary_auroc": 0.75, - "binary_cohen_kappa": 0.0, - "binary_f1": 0.5, - "binary_precision": 0.5, - "binary_recall": 0.5, - "binary_specificity": 0.5, - } - for name, oracle in expected.items(): - with self.subTest(metric=name): - actual = _stream(_configured_metric(name), self.logits, self.targets) - self.assertAlmostEqual(actual, oracle) - - def test_one_slide_updates_equal_one_full_batch(self) -> None: - for name in ( - "binary_accuracy", - "binary_auroc", - "binary_cohen_kappa", - "binary_f1", - "binary_precision", - "binary_recall", - "binary_specificity", - ): - with self.subTest(metric=name): - streamed = _stream(_configured_metric(name), self.logits, self.targets) - batched_metric = _configured_metric(name) - batched_metric.update(self.logits, self.targets) - self.assertAlmostEqual(streamed, float(batched_metric.compute())) - - def test_auditable_summary_contains_the_same_confusion_matrix(self) -> None: - summary = binary_logit_summary(self.logits, self.targets) - self.assertEqual( - {key: summary[key] for key in ("tn", "fp", "fn", "tp")}, - {"tn": 1.0, "fp": 1.0, "fn": 1.0, "tp": 1.0}, - ) - self.assertEqual(summary["f1"], 0.5) - self.assertEqual(summary["auroc"], 0.75) - - def test_zero_logit_uses_the_same_positive_boundary_as_prediction_csv(self) -> None: - for name in ( - "binary_accuracy", - "binary_f1", - "binary_precision", - "binary_recall", - ): - with self.subTest(metric=name): - metric = _configured_metric(name) - metric.update(torch.tensor([0.0]), torch.tensor([1])) - self.assertEqual(float(metric.compute()), 1.0) - - -class JointMetricsTest(unittest.TestCase): - def setUp(self) -> None: - self.predictions = torch.tensor( - [[-2.0, -0.5], [0.2, 0.2], [0.8, 0.8], [-0.1, 1.1]] - ) - self.targets = torch.tensor( - [[0.0, -0.3], [0.0, 0.1], [1.0, 0.7], [1.0, 1.0]] - ) - - def test_joint_f1_preserves_binary_zero_targets(self) -> None: - self.assertAlmostEqual( - _stream( - _configured_metric("joint_f1"), self.predictions, self.targets - ), - 0.5, - ) - - def test_joint_auroc_is_batch_independent(self) -> None: - metric = _configured_metric("joint_auroc") - streamed = _stream(metric, self.predictions, self.targets) - metric.reset() - metric.update(self.predictions, self.targets) - self.assertAlmostEqual(streamed, 0.75) - self.assertAlmostEqual(float(metric.compute()), 0.75) - - def test_all_joint_classification_metrics_match_oracle(self) -> None: - expected = { - "joint_accuracy": 0.5, - "joint_auroc": 0.75, - "joint_cohen_kappa": 0.0, - "joint_f1": 0.5, - "joint_precision": 0.5, - "joint_recall": 0.5, - "joint_specificity": 0.5, - } - for name, oracle in expected.items(): - with self.subTest(metric=name): - actual = _stream( - _configured_metric(name), self.predictions, self.targets - ) - self.assertAlmostEqual(actual, oracle) - - def test_regression_columns_remain_untransformed(self) -> None: - mse = _configured_metric("joint_mse") - mse.update(self.predictions, self.targets) - expected = torch.mean((self.predictions[:, 1] - self.targets[:, 1]) ** 2) - torch.testing.assert_close(mse.compute(), expected) - - def test_wrappers_clone_with_independent_state(self) -> None: - metric = ColumnMetric( - thresholded_f1(threshold=0.0, target_threshold=0.5), column=0 - ) - cloned = metric.clone() - metric.update(self.predictions, self.targets) - cloned.update(self.predictions[:1], self.targets[:1]) - self.assertAlmostEqual(float(metric.compute()), 0.5) - self.assertEqual(float(cloned.compute()), 0.0) - - -class RegressionThresholdMetricsTest(unittest.TestCase): - def setUp(self) -> None: - self.predictions = torch.tensor([-0.8, -0.2, 0.9, -0.1]) - self.targets = torch.tensor([-1.0, -0.5, 0.3, 0.0]) - - def test_zero_is_positive_for_continuous_index_predictions_and_targets(self) -> None: - expected = { - "thresholded_f1": 2 / 3, - "thresholded_precision": 1.0, - "thresholded_recall": 0.5, - "thresholded_specificity": 1.0, - } - for name, oracle in expected.items(): - with self.subTest(metric=name): - actual = _stream( - _configured_metric(name), self.predictions, self.targets - ) - self.assertAlmostEqual(actual, oracle) - - def test_all_continuous_regression_metrics_match_auditable_summary(self) -> None: - summary = regression_summary(self.predictions, self.targets) - for name in ("mae", "mse", "pearson", "spearman", "r2"): - with self.subTest(metric=name): - metric = _configured_metric(name) - metric.update(self.predictions, self.targets) - self.assertAlmostEqual(float(metric.compute()), summary[name]) - - def test_auditable_regression_summary(self) -> None: - summary = regression_summary(self.predictions, self.targets) - self.assertAlmostEqual(summary["mae"], 0.3) - self.assertAlmostEqual(summary["mse"], 0.125) - self.assertAlmostEqual(summary["thresholded_f1"], 2 / 3) - - -if __name__ == "__main__": - unittest.main() From 2147c6702fa4d47d2a00cb6e9a095c49f85e82f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 18:59:50 +0200 Subject: [PATCH 26/79] refactor: adopt data card schema in train configs --- configs/experiment/ml/train_mil_embeddings.yaml | 3 ++- configs/experiment/ml/train_mil_embeddings_joint.yaml | 3 ++- configs/experiment/ml/train_mil_embeddings_regression.yaml | 3 ++- configs/ml.yaml | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/configs/experiment/ml/train_mil_embeddings.yaml b/configs/experiment/ml/train_mil_embeddings.yaml index 9e2a6335..3a8bd559 100644 --- a/configs/experiment/ml/train_mil_embeddings.yaml +++ b/configs/experiment/ml/train_mil_embeddings.yaml @@ -18,7 +18,8 @@ defaults: - binary_recall - binary_specificity - /trainer: default - - /data/sources: single_l3_embed + - /data: _template + - optional /data/embedded: null - _self_ label_mode: type diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml index 400c7aee..f407abd7 100644 --- a/configs/experiment/ml/train_mil_embeddings_joint.yaml +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -21,7 +21,8 @@ defaults: - joint_specificity - joint_mse - /trainer: default - - /data/sources: single_l3_embed + - /data: _template + - optional /data/embedded: null - _self_ label_mode: both diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml index bf007d35..912a0ca5 100644 --- a/configs/experiment/ml/train_mil_embeddings_regression.yaml +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -19,7 +19,8 @@ defaults: - thresholded_recall - thresholded_specificity - /trainer: default - - /data/sources: single_l3_embed + - /data: _template + - optional /data/embedded: null - _self_ label_mode: index diff --git a/configs/ml.yaml b/configs/ml.yaml index 8641b938..f94c364b 100644 --- a/configs/ml.yaml +++ b/configs/ml.yaml @@ -1,7 +1,7 @@ defaults: - base - _self_ - # A +experiment config selects the data source (data/sources/*) and model pieces. + # Experiments select the data shape and model components. seed: ${random_seed:} @@ -22,4 +22,4 @@ test_after_fit: true label_mode: type # "type" (classification), "index" (regression), "both" (joint) feature_dim: ??? # tile-feature width = encoder.out_dim; set per experiment -# datamodule / ml / trainer are composed by the +experiment config's defaults. \ No newline at end of file +# datamodule / ml / trainer are composed by the +experiment config's defaults. From b254c47430134b301ff79665baab13b49c05ae56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 19:03:36 +0200 Subject: [PATCH 27/79] fix: require l3 embedded data card --- configs/data/embedded/tissue_only/l3.yaml | 11 +++++++++++ configs/experiment/ml/train_mil_embeddings.yaml | 2 +- configs/experiment/ml/train_mil_embeddings_joint.yaml | 2 +- .../ml/train_mil_embeddings_regression.yaml | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 configs/data/embedded/tissue_only/l3.yaml diff --git a/configs/data/embedded/tissue_only/l3.yaml b/configs/data/embedded/tissue_only/l3.yaml new file mode 100644 index 00000000..58700700 --- /dev/null +++ b/configs/data/embedded/tissue_only/l3.yaml @@ -0,0 +1,11 @@ +# @package dataset.levels.3 +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_tissue_embed +uris: + all: "mlflow-artifacts:/3/c513c87eac3d4792ba9757a2d7300eb6/artifacts/embeddings" + train: "mlflow-artifacts:/3/7bda595fa2b24f618d2476468c564368/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d74838d175b342569af851b7a36f31e9/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/f785f95353f545539b4166e716db9d20/artifacts/embeddings_test" diff --git a/configs/experiment/ml/train_mil_embeddings.yaml b/configs/experiment/ml/train_mil_embeddings.yaml index 3a8bd559..a3db1268 100644 --- a/configs/experiment/ml/train_mil_embeddings.yaml +++ b/configs/experiment/ml/train_mil_embeddings.yaml @@ -19,7 +19,7 @@ defaults: - binary_specificity - /trainer: default - /data: _template - - optional /data/embedded: null + - /data/embedded: tissue_only/l3 - _self_ label_mode: type diff --git a/configs/experiment/ml/train_mil_embeddings_joint.yaml b/configs/experiment/ml/train_mil_embeddings_joint.yaml index f407abd7..2f14779f 100644 --- a/configs/experiment/ml/train_mil_embeddings_joint.yaml +++ b/configs/experiment/ml/train_mil_embeddings_joint.yaml @@ -22,7 +22,7 @@ defaults: - joint_mse - /trainer: default - /data: _template - - optional /data/embedded: null + - /data/embedded: tissue_only/l3 - _self_ label_mode: both diff --git a/configs/experiment/ml/train_mil_embeddings_regression.yaml b/configs/experiment/ml/train_mil_embeddings_regression.yaml index 912a0ca5..eeb2ea1a 100644 --- a/configs/experiment/ml/train_mil_embeddings_regression.yaml +++ b/configs/experiment/ml/train_mil_embeddings_regression.yaml @@ -20,7 +20,7 @@ defaults: - thresholded_specificity - /trainer: default - /data: _template - - optional /data/embedded: null + - /data/embedded: tissue_only/l3 - _self_ label_mode: index From d596efa97441493dc9176c040998411d1c2627ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 19:20:02 +0200 Subject: [PATCH 28/79] fix: address modeling runtime issues --- ml/data/datamodule.py | 5 +++++ ml/data/datasets/_sources.py | 10 +++++++--- ml/models/losses/joint.py | 6 ++---- ml/train.py | 3 +++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ml/data/datamodule.py b/ml/data/datamodule.py index 6d5b5909..cba27e10 100644 --- a/ml/data/datamodule.py +++ b/ml/data/datamodule.py @@ -38,6 +38,7 @@ class DataModule(pl.LightningDataModule): persistent_workers: Keep workers alive between epochs. **datasets: One entry per split (``train``/``val``/``test``), each an unresolved dataset ``DictConfig`` instantiated in :meth:`setup`. + Prediction reuses the test split. """ def __init__( @@ -69,6 +70,7 @@ def setup(self, stage: str | None = None) -> None: "fit": ("train", "val"), "validate": ("val",), "test": ("test",), + "predict": ("test",), }.get(stage or "", ("train", "val", "test")) for split in needed: self._build(split) @@ -93,5 +95,8 @@ def val_dataloader(self) -> DataLoader[Sample]: def test_dataloader(self) -> DataLoader[Sample]: return self._dataloader("test", shuffle=False) + def predict_dataloader(self) -> DataLoader[Sample]: + return self._dataloader("test", shuffle=False) + __all__ = ["DataModule", "mil_collate"] diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py index 244518b3..80272c0d 100644 --- a/ml/data/datasets/_sources.py +++ b/ml/data/datasets/_sources.py @@ -40,6 +40,12 @@ def load_labeled_slides( only rows that carry a label. """ slides = pd.read_csv(data_mapping) + raw_target_columns = { + LabelMode.TYPE: ["type"], + LabelMode.INDEX: ["mammaprint_index"], + LabelMode.BOTH: ["type", "mammaprint_index"], + }[label_mode] + slides = slides.dropna(subset=raw_target_columns) slides = process_slides(slides, mode=label_mode) target_columns = get_target_columns(label_mode) @@ -68,9 +74,7 @@ def split_uri(card: Mapping[str, object], key: str, split: str, level: int) -> s """ uris = card.get(key) if not isinstance(uris, Mapping): - raise KeyError( - f"Level {level} data card is missing a '{key}' split->URI map." - ) + raise KeyError(f"Level {level} data card is missing a '{key}' split->URI map.") uri = uris.get(split) if not isinstance(uri, str): raise KeyError( diff --git a/ml/models/losses/joint.py b/ml/models/losses/joint.py index 6c1f01f6..167d458a 100644 --- a/ml/models/losses/joint.py +++ b/ml/models/losses/joint.py @@ -47,10 +47,8 @@ def __init__( super().__init__() self.w_cls = w_cls self.w_reg = w_reg - # Registered as a buffer so BCE's pos_weight follows the module's device. - pw = None if pos_weight is None else torch.tensor([pos_weight]) - self.register_buffer("_pos_weight", pw) - self.cls_loss = nn.BCEWithLogitsLoss(pos_weight=self._pos_weight) + pw: Tensor | None = None if pos_weight is None else torch.tensor([pos_weight]) + self.cls_loss = nn.BCEWithLogitsLoss(pos_weight=pw) self.reg_loss = nn.MSELoss() def forward(self, y_pred: Tensor, y: Tensor) -> Tensor: diff --git a/ml/train.py b/ml/train.py index 1f9c523c..630964f8 100644 --- a/ml/train.py +++ b/ml/train.py @@ -16,6 +16,7 @@ import random import hydra +from lightning.pytorch import seed_everything from omegaconf import DictConfig, OmegaConf from rationai.mlkit import autolog from rationai.mlkit.lightning.loggers import MLFlowLogger @@ -33,6 +34,8 @@ @hydra.main(config_path="../configs", config_name="ml", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: + seed_everything(int(config.seed), workers=True) + # _recursive_=False: keep the per-split dataset nodes as DictConfig so the # DataModule can instantiate them lazily per stage in setup(). datamodule = hydra.utils.instantiate(config.datamodule, _recursive_=False) From 2686cfd7dfa2c14e15029c42a5c3c83e3bad3e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 20:39:15 +0200 Subject: [PATCH 29/79] fix: correct metric weighting and transformer layers --- configs/ml/aggregator/transformer.yaml | 4 +++- configs/ml/encoder/identity.yaml | 2 +- ml/models/aggregators/transformer.py | 20 +++++++++++++++++--- ml/models/module.py | 16 +++++++++------- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/configs/ml/aggregator/transformer.yaml b/configs/ml/aggregator/transformer.yaml index 2c72746d..03942035 100644 --- a/configs/ml/aggregator/transformer.yaml +++ b/configs/ml/aggregator/transformer.yaml @@ -1,5 +1,7 @@ -# Extension point — not implemented in the skeleton (raises NotImplementedError). +# TransMIL-style self-attention MIL with a learned CLS token. _target_: ml.models.aggregators.TransformerMIL feature_dim: ${feature_dim} num_heads: 8 num_layers: 2 +mlp_ratio: 4.0 +dropout: 0.1 diff --git a/configs/ml/encoder/identity.yaml b/configs/ml/encoder/identity.yaml index 570e3fd2..458143c7 100644 --- a/configs/ml/encoder/identity.yaml +++ b/configs/ml/encoder/identity.yaml @@ -1,3 +1,3 @@ # Precomputed-embeddings path: features pass through unchanged. _target_: ml.models.encoders.IdentityEncoder -out_dim: 2560 # Virchow2 embedding size; 512 for VGG16 global-max-pooled features +out_dim: ${feature_dim} diff --git a/ml/models/aggregators/transformer.py b/ml/models/aggregators/transformer.py index 69bd89d0..f542c191 100644 --- a/ml/models/aggregators/transformer.py +++ b/ml/models/aggregators/transformer.py @@ -43,6 +43,8 @@ def __init__( f"feature_dim ({feature_dim}) must be divisible by num_heads " f"({num_heads})." ) + if num_layers < 1: + raise ValueError("num_layers must be at least 1.") self.feature_dim = feature_dim self.num_heads = num_heads @@ -68,10 +70,19 @@ def __init__( if num_layers > 1 else None ) + self.final_norm1 = nn.LayerNorm(feature_dim) self.final_attn = nn.MultiheadAttention( feature_dim, num_heads, dropout=dropout, batch_first=True ) - self.final_norm = nn.LayerNorm(feature_dim) + self.final_attn_dropout = nn.Dropout(dropout) + self.final_norm2 = nn.LayerNorm(feature_dim) + self.final_ff = nn.Sequential( + nn.Linear(feature_dim, int(feature_dim * mlp_ratio)), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(int(feature_dim * mlp_ratio), feature_dim), + ) + self.final_ff_dropout = nn.Dropout(dropout) @property def out_dim(self) -> int: @@ -85,11 +96,14 @@ def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: sequence = self.blocks(sequence) # (1, N+1, D) # Final self-attention layer, keeping the CLS<-all attention map. - normed = self.final_norm(sequence) + normed = self.final_norm1(sequence) attended, attn_weights = self.final_attn( normed, normed, normed, need_weights=True, average_attn_weights=True ) # attended: (1, N+1, D); attn_weights: (1, N+1, N+1) - sequence = sequence + attended # residual + sequence = sequence + self.final_attn_dropout(attended) + sequence = sequence + self.final_ff_dropout( + self.final_ff(self.final_norm2(sequence)) + ) slide_vector = sequence[0, 0] # CLS embedding -> (D,) diff --git a/ml/models/module.py b/ml/models/module.py index c5d9944b..922942d5 100644 --- a/ml/models/module.py +++ b/ml/models/module.py @@ -97,9 +97,7 @@ def forward(self, bag: Bag) -> tuple[Tensor, Tensor | None]: prediction = self.output_activation(self.head(pooled)) # (out_dim,) return prediction, attention - def _forward_batch( - self, bags: Sequence[Bag] - ) -> tuple[Tensor, list[Tensor | None]]: + def _forward_batch(self, bags: Sequence[Bag]) -> tuple[Tensor, list[Tensor | None]]: """Run a batch of variable-length bags, stacking per-slide predictions.""" predictions: list[Tensor] = [] attentions: list[Tensor | None] = [] @@ -123,7 +121,13 @@ def _step( raise RuntimeError("A loss is required for training/validation steps.") loss = self.loss(y_pred, y) - self.log(f"{stage}/loss", loss, on_step=stage == "train", on_epoch=True) + self.log( + f"{stage}/loss", + loss, + on_step=stage == "train", + on_epoch=True, + batch_size=len(batch), + ) metrics.update(y_pred, y) self.log_dict(metrics, on_step=False, on_epoch=True) @@ -138,9 +142,7 @@ def validation_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: def test_step(self, batch: list[MILSample], batch_idx: int) -> Tensor: return self._step(batch, "test", self.test_metrics) - def predict_step( - self, batch: list[MILSample], batch_idx: int - ) -> dict[str, Any]: + def predict_step(self, batch: list[MILSample], batch_idx: int) -> dict[str, Any]: bags, _, metadata = _unpack_batch(batch) predictions, attentions = self._forward_batch(bags) return { From 6205f118a38df28e7089a3e1ee9a59dc91ffeaec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 20:47:12 +0200 Subject: [PATCH 30/79] chore: satisfy Python lint checks --- ml/data/datasets/raw_tiles.py | 10 ++++------ ml/metrics/logits.py | 4 +--- ml/metrics/summary.py | 2 +- preprocessing/embeddings.py | 2 +- preprocessing/epithelium_masks.py | 2 +- preprocessing/quality_control.py | 6 +++--- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/ml/data/datasets/raw_tiles.py b/ml/data/datasets/raw_tiles.py index 1ed6cb65..8994532c 100644 --- a/ml/data/datasets/raw_tiles.py +++ b/ml/data/datasets/raw_tiles.py @@ -82,17 +82,15 @@ def __init__( slide_dataset = SlideDataset(paths=[local_dir], transforms=transform) # One TileDataset per slide, keyed by slide stem (matches data_mapping.name). # generate_datasets() yields TileDataset instances (each has .slide_path). - per_slide = cast( - "list[TileDataset]", list(slide_dataset.generate_datasets()) - ) + per_slide = cast("list[TileDataset]", list(slide_dataset.generate_datasets())) self.tile_datasets: dict[str, TileDataset] = { Path(td.slide_path).stem: td for td in per_slide } slides = load_labeled_slides(data_mapping, self.label_mode) - self.slides = slides[ - slides["name"].isin(self.tile_datasets) - ].reset_index(drop=True) + self.slides = slides[slides["name"].isin(self.tile_datasets)].reset_index( + drop=True + ) def __len__(self) -> int: return len(self.slides) diff --git a/ml/metrics/logits.py b/ml/metrics/logits.py index e146189c..a8bae154 100644 --- a/ml/metrics/logits.py +++ b/ml/metrics/logits.py @@ -19,9 +19,7 @@ class BinaryLogitMetric(MetricInputTransformer): prediction CSV exactly. """ - def __init__( - self, metric: Metric, decision_threshold: float | None = None - ) -> None: + def __init__(self, metric: Metric, decision_threshold: float | None = None) -> None: super().__init__(metric) self.decision_threshold = decision_threshold diff --git a/ml/metrics/summary.py b/ml/metrics/summary.py index 269c2997..8a6c598b 100644 --- a/ml/metrics/summary.py +++ b/ml/metrics/summary.py @@ -11,7 +11,7 @@ ) -def _divide(numerator: int | float, denominator: int | float) -> float: +def _divide(numerator: float, denominator: float) -> float: return numerator / denominator if denominator else math.nan diff --git a/preprocessing/embeddings.py b/preprocessing/embeddings.py index b0881982..2bd79fdf 100644 --- a/preprocessing/embeddings.py +++ b/preprocessing/embeddings.py @@ -131,7 +131,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: embeddings_path, ) - except Exception as e: + except Exception as e: # noqa: BLE001 - continue with the remaining slides print(f"Error processing slide {slide_name}: {e}") logger.log_artifacts(str(dest), artifact_path="embeddings") diff --git a/preprocessing/epithelium_masks.py b/preprocessing/epithelium_masks.py index 0a9f1464..60258397 100644 --- a/preprocessing/epithelium_masks.py +++ b/preprocessing/epithelium_masks.py @@ -37,7 +37,7 @@ async def _bounded_process(client: AsyncClient, path: str) -> None: output_path=output_path, timeout=timeout, ) - except Exception as e: + except Exception as e: # noqa: BLE001 - isolate per-slide failures print(f"Slide processing failed for {path}: {e}, error: {e!r}") async with AsyncClient(timeout=timeout) as client: diff --git a/preprocessing/quality_control.py b/preprocessing/quality_control.py index db043ce1..3081dee6 100644 --- a/preprocessing/quality_control.py +++ b/preprocessing/quality_control.py @@ -39,7 +39,7 @@ async def put_request( f"Request to {url} timed out after {request_timeout} seconds. Slide: {slide_name}" ) return -1, "Timeout" - except Exception as exc: + except Exception as exc: # noqa: BLE001 - return a failed request result slide_name = Path(data.get("wsi_path", "")).name print(f"Request to {url} failed for {slide_name}: {exc}") return -1, str(exc) @@ -118,7 +118,7 @@ async def generate_report( print( f"Report generation request to {url} timed out after {report_request_timeout} seconds." ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 - report generation is best-effort print(f"Report generation request to {url} failed: {exc}") @@ -171,7 +171,7 @@ async def qc_main( logger.experiment.log_artifacts( run_id=logger.run_id, local_dir=Path(report_path).parent.as_posix() ) - except Exception: + except Exception: # noqa: BLE001 - artifact upload must not abort QC print(f"Could not upload report artifact from {report_path}") From ff58aa5b02467c6424d84df23b218dbe1a63bc84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 22 Apr 2026 14:13:20 +0200 Subject: [PATCH 31/79] feat: data exploration --- .../notebooks/data_exploration_script.ipynb | 2863 +++++++++++++++++ 1 file changed, 2863 insertions(+) create mode 100644 scripts/notebooks/data_exploration_script.ipynb diff --git a/scripts/notebooks/data_exploration_script.ipynb b/scripts/notebooks/data_exploration_script.ipynb new file mode 100644 index 00000000..663f99c3 --- /dev/null +++ b/scripts/notebooks/data_exploration_script.ipynb @@ -0,0 +1,2863 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "96a77f75-ac46-4c07-831c-ea759dd0f5f2", + "metadata": {}, + "source": [ + "# 0. Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ed00c175-8e62-4ee0-8944-aa6a5b0a623c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmp2ekpckmj\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (3.0.3)\n", + "Requirement already satisfied: numpy in /opt/venv/lib/python3.14/site-packages (2.4.6)\n", + "Requirement already satisfied: openpyxl in /opt/venv/lib/python3.14/site-packages (3.1.5)\n", + "Requirement already satisfied: matplotlib in /opt/venv/lib/python3.14/site-packages (3.11.0)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.14/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: et-xmlfile in /opt/venv/lib/python3.14/site-packages (from openpyxl) (2.0.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (4.63.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.5.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (25.0)\n", + "Requirement already satisfied: pillow>=9 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (12.2.0)\n", + "Requirement already satisfied: pyparsing>=3 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (3.3.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/venv/lib/python3.14/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas numpy openpyxl matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "439e5094-e266-4106-b61b-3d69e71ccb7a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9e671a9f", + "metadata": {}, + "outputs": [], + "source": [ + "def search_df(df, *substrings):\n", + " \"\"\"Display rows containing all requested substrings so source-table issues can be inspected quickly.\"\"\"\n", + " for s in substrings:\n", + " df = df[df.apply(lambda row, s=s: row.astype(str).str.contains(s).any(), axis=1)]\n", + " display(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4c83e452-9159-4b26-b25f-21facd7ebb50", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_PATH = Path(\"/mnt/data/MOU/breast/mammaprint/\")\n", + "\n", + "DATA_FILES = [\n", + " DATA_PATH / \"data_2023-09-25.xlsx\",\n", + " DATA_PATH / \"data_test_set_2024-02-16.xlsx\",\n", + " DATA_PATH / \"data-dodatek-2023-10-19.xlsx\",\n", + " DATA_PATH / \"Mammaprint-surova data.xlsx\",\n", + " DATA_PATH / \"Soubor RNA Mammaprint-1_10_25_pro AI.xlsx\",\n", + " DATA_PATH / \"Mammaprint_anonymizovany.xlsx\"\n", + " \n", + "]\n", + "\n", + "VALID_TYPES = [\"a luminal\", \"b luminal\"]\n", + "INDEX_PATTERN = re.compile(r\"^(\\d{4})/\\s*(\\d{1,5})(?:-(\\d))?$\")\n", + "FILENAME_PATTERN = re.compile(r\"^(P?)(\\d{4})_(\\d{1,5})(?:-(\\d))?$\")\n", + "\n", + "\n", + "COLUMN_MAPPING = {\n", + " \"číslo záznamu\": \"record_num\",\n", + " \"mammaprint index\": \"mammaprint_index\",\n", + " \"typ\": \"type\",\n", + "}\n", + "NON_MAPPING_COLS = [\"source_table\", \"original_index\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9751dfcb-7255-440c-92e5-33a508226e4b", + "metadata": {}, + "outputs": [], + "source": [ + "INDICES_MAPPING = {\n", + " \"2023/3222 a\": \"2023/03222\",\n", + " \"2023/3222 b\": \"2023/03222\",\n", + " \"2024/2324ii\": \"2024/02324-2\",\n", + " \"2023/3206 ii\": \"2023/03206\",\n", + "}\n", + "FILES_MAPPING = {\n", + " \"P2023_03525-35\": \"P2023_03525-1\",\n", + " \"P2023_03525-23\": \"P2023_03525-2\",\n", + " \"P2025_00968\": \"P2025_00968-1\",\n", + "}\n", + "\n", + "# record_num\n", + "DROP_ROWS = [\n", + " # tbd with doctor\n", + "]\n", + "\n", + "RECORD_NUM_CORRECTIONS = [\n", + " {\n", + " \"source_table\": \"Mammaprint-surova data:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint-surova data:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"data_test_set_2024-02-16:List1\",\n", + " \"original_record_num\": \"2023/03525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"data_test_set_2024-02-16:List1\",\n", + " \"original_record_num\": \"2023/03525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": 0.235,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2023/03525-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2023\",\n", + " \"original_record_num\": \"2023/3525\",\n", + " \"mammaprint_index\": -0.345,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2023/03525-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Soubor RNA Mammaprint-1_10_25_pro AI:2025\",\n", + " \"original_record_num\": \"2025/968a\",\n", + " \"mammaprint_index\": 0.541,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2025/00968-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Soubor RNA Mammaprint-1_10_25_pro AI:2025\",\n", + " \"original_record_num\": \"2025/968b\",\n", + " \"mammaprint_index\": -0.076,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2025/00968-2\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2025\",\n", + " \"original_record_num\": \"2025/968a\",\n", + " \"mammaprint_index\": 0.541,\n", + " \"type\": \"a luminal\",\n", + " \"corrected_record_num\": \"2025/00968-1\",\n", + " },\n", + " {\n", + " \"source_table\": \"Mammaprint_anonymizovany:2025\",\n", + " \"original_record_num\": \"2025/968b\",\n", + " \"mammaprint_index\": -0.076,\n", + " \"type\": \"b luminal\",\n", + " \"corrected_record_num\": \"2025/00968-2\",\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "7aa885f7-5981-4039-98ec-5c70cf90b470", + "metadata": {}, + "source": [ + "# Part 1: Table Loading & Merging" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5fefc716-eab5-4e46-b235-b636c9790f70", + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_excel_text(value):\n", + " \"\"\"Normalize raw Excel text so header and value comparisons are consistent across files.\"\"\"\n", + " if isinstance(value, str):\n", + " return value.replace(\"\\n\", \" \").strip().lower()\n", + " return value\n", + "\n", + "\n", + "def normalize_excel_frame(df: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Normalize all cell text and column names so downstream mapping sees one canonical format.\"\"\"\n", + " df = df.map(normalize_excel_text)\n", + " df.columns = df.columns.map(normalize_excel_text)\n", + " return df\n", + "\n", + "\n", + "def find_header_row(raw: pd.DataFrame) -> int | None:\n", + " \"\"\"Find the real table header row because source spreadsheets start at different offsets.\"\"\"\n", + " normalized = raw.map(normalize_excel_text)\n", + "\n", + " for i, row in normalized.iterrows():\n", + " row_values = row.tolist()\n", + " if (\n", + " \"číslo záznamu\" in row_values\n", + " and \"mammaprint index\" in row_values\n", + " and \"typ\" in row_values\n", + " ):\n", + " return i\n", + "\n", + " return None\n", + "\n", + "\n", + "def _matches_correction_value(series: pd.Series, expected):\n", + " \"\"\"Compare correction selectors robustly so known fixes match exactly one intended row.\"\"\"\n", + " if isinstance(expected, float):\n", + " values = pd.to_numeric(series, errors=\"coerce\")\n", + " return (values - expected).abs() <= 1e-9\n", + " return series.astype(str).str.strip().str.lower() == str(expected).strip().lower()\n", + "\n", + "\n", + "def apply_record_num_corrections(\n", + " df: pd.DataFrame, path: Path, sheet_name: str\n", + ") -> pd.DataFrame:\n", + " \"\"\"Apply hand-verified record number fixes so ambiguous duplicate entries keep distinct IDs.\"\"\"\n", + " source_table = f\"{Path(path).stem}:{sheet_name}\"\n", + " record_col = \"číslo záznamu\" if \"číslo záznamu\" in df.columns else df.columns[0]\n", + " corrections = [\n", + " correction\n", + " for correction in RECORD_NUM_CORRECTIONS\n", + " if correction[\"source_table\"] == source_table\n", + " ]\n", + "\n", + " for correction in corrections:\n", + " mask = _matches_correction_value(\n", + " df[record_col], correction[\"original_record_num\"]\n", + " )\n", + " mask &= _matches_correction_value(\n", + " df[\"mammaprint index\"], correction[\"mammaprint_index\"]\n", + " )\n", + " mask &= _matches_correction_value(df[\"typ\"], correction[\"type\"])\n", + "\n", + " matches = df.index[mask].tolist()\n", + " if len(matches) != 1:\n", + " raise ValueError(\n", + " f\"Expected exactly one row for record correction {correction}, \"\n", + " f\"found {len(matches)}\"\n", + " )\n", + "\n", + " df.loc[matches[0], record_col] = correction[\"corrected_record_num\"]\n", + "\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "398a3766-6c0a-4a49-b54e-8c2a759ccbc6", + "metadata": {}, + "outputs": [], + "source": [ + "def read_diverse_excel(path: Path, sheet_name: str):\n", + " \"\"\"Load one heterogeneous Excel sheet into a normalized table for merging with other sources.\"\"\"\n", + " # Step 1: read without a header so we can inspect candidate header rows\n", + " raw = pd.read_excel(path, sheet_name=sheet_name, header=None)\n", + "\n", + " # Step 2: find the real header row using normalized candidate rows\n", + " header_row = find_header_row(raw)\n", + "\n", + " # Step 3: load/build the dataframe using the detected header\n", + " if header_row is not None:\n", + " df = pd.read_excel(path, sheet_name=sheet_name, header=header_row)\n", + " else:\n", + " # no header found -> assume first three columns correspond to the expected fields\n", + " df = raw.copy()\n", + " expected_cols = [\"číslo záznamu\", \"mammaprint index\", \"typ\"]\n", + " rest_cols = [f\"col_{i}\" for i in range(df.shape[1] - 3)]\n", + " df.columns = expected_cols + rest_cols\n", + "\n", + " # Step 4: normalize columns and string cells after the table shape is known\n", + " df = normalize_excel_frame(df)\n", + "\n", + " # Step 5: apply known record-number fixes, then normalize inserted values\n", + " df = apply_record_num_corrections(df, path, sheet_name)\n", + " df = normalize_excel_frame(df)\n", + "\n", + " # Step 6: backfill repeated-table IDs and continue with common cleanup\n", + " df.iloc[:, 0] = df.iloc[:, 0].ffill() # use id of previous entry\n", + " df = df.dropna(how=\"all\")\n", + "\n", + " df[\"source_table\"] = f\"{Path(path).stem}:{sheet_name}\"\n", + " df[\"original_index\"] = df.iloc[:, 0].copy()\n", + "\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ce73a8e4-0113-42cd-b2fa-b5e41c1ee350", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading /mnt/data/MOU/breast/mammaprint/data_2023-09-25.xlsx...\n", + " - Sheet 'List1': 263 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/data_test_set_2024-02-16.xlsx...\n", + " - Sheet 'List1': 204 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/data-dodatek-2023-10-19.xlsx...\n", + " - Sheet 'List1': 37 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Mammaprint-surova data.xlsx...\n", + " - Sheet '2024': 602 rows\n", + " - Sheet '2023': 417 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Soubor RNA Mammaprint-1_10_25_pro AI.xlsx...\n", + " - Sheet '2025': 697 rows\n", + "Reading /mnt/data/MOU/breast/mammaprint/Mammaprint_anonymizovany.xlsx...\n", + " - Sheet '2026': 184 rows\n", + " - Sheet '2025': 871 rows\n", + " - Sheet '2024': 760 rows\n", + " - Sheet '2023': 419 rows\n", + "\n", + "Total rows: 4444\n" + ] + } + ], + "source": [ + "all_dfs = []\n", + "\n", + "for file in DATA_FILES:\n", + " print(f\"Reading {file}...\")\n", + " excel_file = pd.ExcelFile(file)\n", + "\n", + " for sheet_name in excel_file.sheet_names:\n", + " df = read_diverse_excel(file, sheet_name)\n", + " print(f\" - Sheet '{sheet_name}': {len(df)} rows\")\n", + " all_dfs.append(df)\n", + "\n", + "merged_df = pd.concat(all_dfs, ignore_index=True).drop_duplicates()\n", + "print(f\"\\nTotal rows: {len(merged_df)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "405565e5-528a-48a0-86aa-4480f37ce260", + "metadata": {}, + "source": [ + "### Modify columns\n", + "- rename columns\n", + "- join data in unnamed columns into one called `note`" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c2627666-5788-4e53-b9bb-db4c1df02ed5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mapped columns: {'číslo záznamu': 'record_num', 'mammaprint index': 'mammaprint_index', 'typ': 'type'}\n" + ] + } + ], + "source": [ + "mapped_cols = {}\n", + "\n", + "for col in merged_df.columns:\n", + " col_lower = col.lower().strip()\n", + " for source_name, target_name in COLUMN_MAPPING.items():\n", + " if col_lower == source_name.lower():\n", + " mapped_cols[col] = target_name\n", + " break\n", + "\n", + "print(f\"Mapped columns: {mapped_cols}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1899428a-5a0f-4258-b897-256a3401a178", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Result dataframe: 4444 rows, 6 columns\n" + ] + } + ], + "source": [ + "result_df = pd.DataFrame()\n", + "\n", + "for source_col, target_col in mapped_cols.items():\n", + " result_df[target_col] = merged_df[source_col]\n", + "\n", + "result_df[NON_MAPPING_COLS] = merged_df[NON_MAPPING_COLS]\n", + "other_cols = [\n", + " col\n", + " for col in merged_df.columns\n", + " if col not in mapped_cols and col not in NON_MAPPING_COLS\n", + "]\n", + "\n", + "\n", + "def merge_columns_to_note(row):\n", + " \"\"\"Collect non-mapping source columns into a note so useful audit context is not lost.\"\"\"\n", + " notes = []\n", + " for col in other_cols:\n", + " value = row[col]\n", + " if pd.notna(value) and str(value).strip() != \"\":\n", + " notes.append(f\"{col}: {value}\")\n", + " return \"; \".join(notes) if notes else \"\"\n", + "\n", + "\n", + "result_df[\"note\"] = merged_df.apply(merge_columns_to_note, axis=1)\n", + "\n", + "print(f\"\\nResult dataframe: {len(result_df)} rows, {len(result_df.columns)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "c02bbbf2-f880-41c3-97f5-b44705ecd04c", + "metadata": {}, + "source": [ + "# Part 2: Table data quality" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1ce20d87-e629-4b20-8327-091c8454a10b", + "metadata": {}, + "outputs": [], + "source": [ + "def add_entry_quality_flag(df, invalid, flag):\n", + " \"\"\"Append a quality flag to invalid rows so multiple issues can be tracked per entry.\"\"\"\n", + " def append_flag(current):\n", + " if current == \"OK\" or pd.isna(current):\n", + " return flag\n", + " flags = str(current).split(\";\")\n", + " if flag in flags:\n", + " return current\n", + " return \";\".join(flags + [flag])\n", + "\n", + " df.loc[invalid, \"entry_quality\"] = df.loc[invalid, \"entry_quality\"].apply(\n", + " append_flag\n", + " )\n", + "\n", + "\n", + "def has_entry_quality_flag(series, flag):\n", + " \"\"\"Check semicolon-delimited quality flags so later steps can skip known-bad rows.\"\"\"\n", + " return series.astype(str).str.split(\";\").apply(lambda flags: flag in flags)\n", + "\n", + "\n", + "def validate_mammaprint_index(df):\n", + " \"\"\"Flag invalid Mammaprint values so only numeric scores in the expected range pass.\"\"\"\n", + " mammaprint_values = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\")\n", + " invalid = (\n", + " mammaprint_values.isna() | (mammaprint_values < -1) | (mammaprint_values > 1)\n", + " )\n", + " add_entry_quality_flag(df, invalid, \"MAMMAPRINT\")\n", + "\n", + "\n", + "def validate_type(df):\n", + " \"\"\"Flag invalid luminal types so downstream labels use the supported class names only.\"\"\"\n", + " invalid = df[\"type\"].isna() | ~df[\"type\"].astype(str).str.strip().isin(VALID_TYPES)\n", + " add_entry_quality_flag(df, invalid, \"TYPE\")\n", + "\n", + "\n", + "def validate_luminal_interval(df):\n", + " \"\"\"Flag type-score disagreements so A/B luminal labels match the Mammaprint sign.\"\"\"\n", + " mammaprint_values = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\")\n", + " normalized_type = df[\"type\"].astype(str).str.strip().str.lower()\n", + "\n", + " invalid_a = (\n", + " normalized_type.eq(\"a luminal\")\n", + " & mammaprint_values.notna()\n", + " & ~((mammaprint_values > 0) & (mammaprint_values <= 1))\n", + " )\n", + " invalid_b = (\n", + " normalized_type.eq(\"b luminal\")\n", + " & mammaprint_values.notna()\n", + " & ~((mammaprint_values >= -1) & (mammaprint_values < 0))\n", + " )\n", + "\n", + " add_entry_quality_flag(df, invalid_a | invalid_b, \"LUMINAL_INTERVAL\")\n", + "\n", + "\n", + "def validate_record_num(df):\n", + " \"\"\"Flag malformed record numbers before normalization so unexpected IDs are reviewed.\"\"\"\n", + " invalid = df[\"record_num\"].isna() | (\n", + " ~df[\"record_num\"]\n", + " .astype(str)\n", + " .str.fullmatch(INDEX_PATTERN) # doesn't match regex\n", + " & ~df[\"record_num\"].astype(str).isin(INDICES_MAPPING.keys()) # not in dict keys\n", + " )\n", + " add_entry_quality_flag(df, invalid, \"RECORD_NUM\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "81d280de-8b60-40a3-ae1e-ec6b7e0a1330", + "metadata": {}, + "outputs": [], + "source": [ + "result_df[\"entry_quality\"] = \"OK\"\n", + "validate_mammaprint_index(result_df)\n", + "validate_type(result_df)\n", + "validate_luminal_interval(result_df)\n", + "validate_record_num(result_df)" + ] + }, + { + "cell_type": "markdown", + "id": "c122dd0e-f201-4a64-89c0-a334465a9eb4", + "metadata": {}, + "source": [ + "### Preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0f74fd0b-6c98-46eb-ade2-314c72477c86", + "metadata": {}, + "outputs": [], + "source": [ + "def normalize_index(idx: str) -> str:\n", + " \"\"\"Canonicalize record numbers so table entries and slide filenames compare reliably.\"\"\"\n", + " if idx in INDICES_MAPPING:\n", + " return INDICES_MAPPING[idx]\n", + "\n", + " match = INDEX_PATTERN.match(idx)\n", + "\n", + " if match:\n", + " year, num, rank = match.groups()\n", + " num_padded = num.zfill(5) # pad middle to 5 digits\n", + " if rank:\n", + " return f\"{year}/{num_padded}-{rank}\"\n", + " else:\n", + " return f\"{year}/{num_padded}\"\n", + " else:\n", + " raise ValueError(f\"Flagged (unexpected format or suffix): '{idx}'\")\n", + "\n", + "\n", + "def normalize_indices(result_df):\n", + " \"\"\"Normalize only valid record numbers so invalid rows stay visible for quality review.\"\"\"\n", + " valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + " result_df.loc[valid_record_num, \"record_num\"] = result_df.loc[\n", + " valid_record_num, \"record_num\"\n", + " ].apply(normalize_index)\n", + "\n", + "\n", + "def flag_conflicting_records(result_df):\n", + " \"\"\"Mark duplicate OK records as conflicts while keeping retry/non-final rows separate.\"\"\"\n", + " conflicting_records = []\n", + " grouped = result_df.groupby(\"record_num\")\n", + "\n", + " for record_num, group in grouped:\n", + " if len(group) == 1: # just one entry, correct\n", + " continue\n", + "\n", + " if (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() == 0: # no OK entry, nothing to flag further\n", + " continue\n", + "\n", + " elif (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() == 1: # one OK entry (retry), others non-needed\n", + " result_df.loc[\n", + " (result_df[\"record_num\"] == record_num)\n", + " & (result_df[\"entry_quality\"] != \"OK\"),\n", + " \"entry_quality\",\n", + " ] = \"NON-FINAL\"\n", + "\n", + " elif (\n", + " group[\"entry_quality\"] == \"OK\"\n", + " ).sum() >= 2: # multiple OK entries per one id, conflict\n", + " conflicting_records.append(record_num)\n", + " result_df.loc[result_df[\"record_num\"] == record_num, \"entry_quality\"] = (\n", + " \"CONFLICT\"\n", + " )\n", + "\n", + " print(f\"Conflicting record_nums: {len(conflicting_records)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e92839d6-25dc-4986-821a-8a6b213e463d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conflicting record_nums: 0\n" + ] + } + ], + "source": [ + "normalize_indices(result_df)\n", + "result_df = result_df.drop_duplicates(\n", + " subset=[\"record_num\", \"type\", \"mammaprint_index\", \"entry_quality\"]\n", + ")\n", + "flag_conflicting_records(result_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "same-stem-record-debug", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Record stems with multiple record_nums: 4\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_numsrow_countsource_tablesentry_quality
record_num_stem
2023/00946[2023/00946, 2023/00946-2]4[Mammaprint-surova data:2023, data_2023-09-25:...[NON-FINAL, OK]
2023/03525[2023/03525-1, 2023/03525-2]2[data_test_set_2024-02-16:List1][OK]
2024/02324[2024/02324, 2024/02324-2]2[Mammaprint-surova data:2024][MAMMAPRINT;TYPE]
2025/00968[2025/00968-1, 2025/00968-2]2[Soubor RNA Mammaprint-1_10_25_pro AI:2025][OK]
\n", + "
" + ], + "text/plain": [ + " record_nums row_count \\\n", + "record_num_stem \n", + "2023/00946 [2023/00946, 2023/00946-2] 4 \n", + "2023/03525 [2023/03525-1, 2023/03525-2] 2 \n", + "2024/02324 [2024/02324, 2024/02324-2] 2 \n", + "2025/00968 [2025/00968-1, 2025/00968-2] 2 \n", + "\n", + " source_tables \\\n", + "record_num_stem \n", + "2023/00946 [Mammaprint-surova data:2023, data_2023-09-25:... \n", + "2023/03525 [data_test_set_2024-02-16:List1] \n", + "2024/02324 [Mammaprint-surova data:2024] \n", + "2025/00968 [Soubor RNA Mammaprint-1_10_25_pro AI:2025] \n", + "\n", + " entry_quality \n", + "record_num_stem \n", + "2023/00946 [NON-FINAL, OK] \n", + "2023/03525 [OK] \n", + "2024/02324 [MAMMAPRINT;TYPE] \n", + "2025/00968 [OK] " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def record_num_stem(record_num):\n", + " \"\"\"Return the year/number stem without rank so related ranked records can be inspected together.\"\"\"\n", + " match = INDEX_PATTERN.match(str(record_num))\n", + " if not match:\n", + " return pd.NA\n", + "\n", + " year, num, _rank = match.groups()\n", + " return f\"{year}/{num.zfill(5)}\"\n", + "\n", + "\n", + "valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + "record_stem_debug_df = result_df[valid_record_num].copy()\n", + "record_stem_debug_df[\"record_num_stem\"] = record_stem_debug_df[\"record_num\"].apply(\n", + " record_num_stem\n", + ")\n", + "\n", + "same_stem_counts = record_stem_debug_df.groupby(\"record_num_stem\")[\n", + " \"record_num\"\n", + "].nunique()\n", + "same_stem_stems = same_stem_counts[same_stem_counts > 1].index\n", + "same_stem_records_df = record_stem_debug_df[\n", + " record_stem_debug_df[\"record_num_stem\"].isin(same_stem_stems)\n", + "].sort_values([\"record_num_stem\", \"record_num\", \"source_table\", \"original_index\"])\n", + "\n", + "same_stem_summary_df = same_stem_records_df.groupby(\"record_num_stem\").agg(\n", + " record_nums=(\"record_num\", lambda values: sorted(values.unique())),\n", + " row_count=(\"record_num\", \"size\"),\n", + " source_tables=(\"source_table\", lambda values: sorted(values.unique())),\n", + " entry_quality=(\"entry_quality\", lambda values: sorted(values.unique())),\n", + ")\n", + "\n", + "print(f\"Record stems with multiple record_nums: {len(same_stem_summary_df)}\")\n", + "display(same_stem_summary_df)" + ] + }, + { + "cell_type": "markdown", + "id": "aaad3976-5f93-49e9-9dfe-88f1a358f31e", + "metadata": {}, + "source": [ + "### Table quality results" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c3f3172f-29f0-48a4-96d1-80bf4fb5d68c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Quality Summary ===\n", + "OK: 2084 rows (90.5%)\n", + "NON-FINAL: 114 rows (5.0%)\n", + "MAMMAPRINT;TYPE: 89 rows (3.9%)\n", + "TYPE: 14 rows (0.6%)\n", + "MAMMAPRINT;TYPE;RECORD_NUM: 1 rows (0.0%)\n", + "LUMINAL_INTERVAL: 1 rows (0.0%)\n", + "Total: 2303 rows\n", + "\n", + "=== NON-FINAL ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
112023/00946nelzeNaNdata_2023-09-25:List12023/946NON-FINAL
292023/01108nelzeb luminaldata_2023-09-25:List12023/1108NON-FINAL
472023/01237nelzeNaNdata_2023-09-25:List12023/1237NON-FINAL
562023/01294nelzeNaNdata_2023-09-25:List12023/1294NON-FINAL
772023/01462nelzeNaNdata_2023-09-25:List12023/1462NON-FINAL
852023/01486nelzea luminaldata_2023-09-25:List12023/1486NON-FINAL
892023/01591nevyšetřenoNaNdata_2023-09-25:List12023/1591NON-FINAL
912023/01597nelzea luminaldata_2023-09-25:List12023/1597NON-FINAL
932023/01599nelzeb luminaldata_2023-09-25:List12023/1599NON-FINAL
1632023/02298nelzeNaNdata_2023-09-25:List12023/2298NON-FINAL
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "11 2023/00946 nelze NaN data_2023-09-25:List1 \n", + "29 2023/01108 nelze b luminal data_2023-09-25:List1 \n", + "47 2023/01237 nelze NaN data_2023-09-25:List1 \n", + "56 2023/01294 nelze NaN data_2023-09-25:List1 \n", + "77 2023/01462 nelze NaN data_2023-09-25:List1 \n", + "85 2023/01486 nelze a luminal data_2023-09-25:List1 \n", + "89 2023/01591 nevyšetřeno NaN data_2023-09-25:List1 \n", + "91 2023/01597 nelze a luminal data_2023-09-25:List1 \n", + "93 2023/01599 nelze b luminal data_2023-09-25:List1 \n", + "163 2023/02298 nelze NaN data_2023-09-25:List1 \n", + "\n", + " original_index note entry_quality \n", + "11 2023/946 NON-FINAL \n", + "29 2023/1108 NON-FINAL \n", + "47 2023/1237 NON-FINAL \n", + "56 2023/1294 NON-FINAL \n", + "77 2023/1462 NON-FINAL \n", + "85 2023/1486 NON-FINAL \n", + "89 2023/1591 NON-FINAL \n", + "91 2023/1597 NON-FINAL \n", + "93 2023/1599 NON-FINAL \n", + "163 2023/2298 NON-FINAL " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== MAMMAPRINT;TYPE ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
332023/01141nelzeNaNdata_2023-09-25:List12023/1141MAMMAPRINT;TYPE
532023/01268nevyšetřenoNaNdata_2023-09-25:List12023/1268MAMMAPRINT;TYPE
612023/01332nevyšetřenoNaNdata_2023-09-25:List12023/1332MAMMAPRINT;TYPE
822023/01473nevyšetřenoNaNdata_2023-09-25:List12023/1473MAMMAPRINT;TYPE
1442023/02114nelzeNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1452023/02114specialistaNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1462023/02114NaNNaNdata_2023-09-25:List12023/2114MAMMAPRINT;TYPE
1702023/02353nelzeNaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
1722023/02353specialistaNaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
1732023/023539NaNdata_2023-09-25:List12023/2353MAMMAPRINT;TYPE
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table original_index \\\n", + "33 2023/01141 nelze NaN data_2023-09-25:List1 2023/1141 \n", + "53 2023/01268 nevyšetřeno NaN data_2023-09-25:List1 2023/1268 \n", + "61 2023/01332 nevyšetřeno NaN data_2023-09-25:List1 2023/1332 \n", + "82 2023/01473 nevyšetřeno NaN data_2023-09-25:List1 2023/1473 \n", + "144 2023/02114 nelze NaN data_2023-09-25:List1 2023/2114 \n", + "145 2023/02114 specialista NaN data_2023-09-25:List1 2023/2114 \n", + "146 2023/02114 NaN NaN data_2023-09-25:List1 2023/2114 \n", + "170 2023/02353 nelze NaN data_2023-09-25:List1 2023/2353 \n", + "172 2023/02353 specialista NaN data_2023-09-25:List1 2023/2353 \n", + "173 2023/02353 9 NaN data_2023-09-25:List1 2023/2353 \n", + "\n", + " note entry_quality \n", + "33 MAMMAPRINT;TYPE \n", + "53 MAMMAPRINT;TYPE \n", + "61 MAMMAPRINT;TYPE \n", + "82 MAMMAPRINT;TYPE \n", + "144 MAMMAPRINT;TYPE \n", + "145 MAMMAPRINT;TYPE \n", + "146 MAMMAPRINT;TYPE \n", + "170 MAMMAPRINT;TYPE \n", + "172 MAMMAPRINT;TYPE \n", + "173 MAMMAPRINT;TYPE " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== TYPE ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
3692023/04116-0.841basal typedata_test_set_2024-02-16:List12023/04116unnamed: 3: výsledek z agendieTYPE
6062024/00783-1bazal typeMammaprint-surova data:20242024/783TYPE
6402024/01000-1bazal typeMammaprint-surova data:20242024/1000TYPE
15832025/00341-0.792basal typeSoubor RNA Mammaprint-1_10_25_pro AI:20252025/341podíl nádorových buněk: 0.7TYPE
16582025/00858-0.115her2Soubor RNA Mammaprint-1_10_25_pro AI:20252025/858podíl nádorových buněk: cca 15%TYPE
17942025/01756-1bazalSoubor RNA Mammaprint-1_10_25_pro AI:20252025/1756podíl nádorových buněk: 0.4TYPE
21742025/04071-0.892bazalSoubor RNA Mammaprint-1_10_25_pro AI:20252025/4071podíl nádorových buněk: 0.5TYPE
23692026/00971-1bazalMammaprint_anonymizovany:20262026/971unnamed: 4: 0.8; číslo biopsie: 2026/1934; čís...TYPE
31932025/04908-1bazalMammaprint_anonymizovany:20252025/4908unnamed: 4: 0.4; číslo biopsie: 2025/1033662; ...TYPE
32452025/051770.182NaNMammaprint_anonymizovany:20252025/5177unnamed: 4: 0.5; číslo biopsie: 2025/9864; čís...TYPE
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "369 2023/04116 -0.841 basal type \n", + "606 2024/00783 -1 bazal type \n", + "640 2024/01000 -1 bazal type \n", + "1583 2025/00341 -0.792 basal type \n", + "1658 2025/00858 -0.115 her2 \n", + "1794 2025/01756 -1 bazal \n", + "2174 2025/04071 -0.892 bazal \n", + "2369 2026/00971 -1 bazal \n", + "3193 2025/04908 -1 bazal \n", + "3245 2025/05177 0.182 NaN \n", + "\n", + " source_table original_index \\\n", + "369 data_test_set_2024-02-16:List1 2023/04116 \n", + "606 Mammaprint-surova data:2024 2024/783 \n", + "640 Mammaprint-surova data:2024 2024/1000 \n", + "1583 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/341 \n", + "1658 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/858 \n", + "1794 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/1756 \n", + "2174 Soubor RNA Mammaprint-1_10_25_pro AI:2025 2025/4071 \n", + "2369 Mammaprint_anonymizovany:2026 2026/971 \n", + "3193 Mammaprint_anonymizovany:2025 2025/4908 \n", + "3245 Mammaprint_anonymizovany:2025 2025/5177 \n", + "\n", + " note entry_quality \n", + "369 unnamed: 3: výsledek z agendie TYPE \n", + "606 TYPE \n", + "640 TYPE \n", + "1583 podíl nádorových buněk: 0.7 TYPE \n", + "1658 podíl nádorových buněk: cca 15% TYPE \n", + "1794 podíl nádorových buněk: 0.4 TYPE \n", + "2174 podíl nádorových buněk: 0.5 TYPE \n", + "2369 unnamed: 4: 0.8; číslo biopsie: 2026/1934; čís... TYPE \n", + "3193 unnamed: 4: 0.4; číslo biopsie: 2025/1033662; ... TYPE \n", + "3245 unnamed: 4: 0.5; číslo biopsie: 2025/9864; čís... TYPE " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== MAMMAPRINT;TYPE;RECORD_NUM ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
2220NaNNaNNaNMammaprint_anonymizovany:2026NaNdisek.: anoMAMMAPRINT;TYPE;RECORD_NUM
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "2220 NaN NaN NaN Mammaprint_anonymizovany:2026 \n", + "\n", + " original_index note entry_quality \n", + "2220 NaN disek.: ano MAMMAPRINT;TYPE;RECORD_NUM " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== LUMINAL_INTERVAL ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypesource_tableoriginal_indexnoteentry_quality
32012025/049530b luminalMammaprint_anonymizovany:20252025/4953unnamed: 4: 0.6; číslo biopsie: 2025/53612; ma...LUMINAL_INTERVAL
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type source_table \\\n", + "3201 2025/04953 0 b luminal Mammaprint_anonymizovany:2025 \n", + "\n", + " original_index note \\\n", + "3201 2025/4953 unnamed: 4: 0.6; číslo biopsie: 2025/53612; ma... \n", + "\n", + " entry_quality \n", + "3201 LUMINAL_INTERVAL " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\n=== Quality Summary ===\")\n", + "counts = result_df[\"entry_quality\"].value_counts()\n", + "\n", + "for label, count in counts.items():\n", + " pct = count / len(result_df) * 100\n", + " print(f\"{label}: {count} rows ({pct:.1f}%)\")\n", + "\n", + "print(f\"Total: {len(result_df)} rows\\n\")\n", + "\n", + "\n", + "for label in counts.keys():\n", + " if label == \"OK\":\n", + " continue\n", + "\n", + " # TODO non final print aj hodnotu co nasleduje\n", + " print(f\"=== {label} ===\")\n", + " display(result_df[result_df[\"entry_quality\"] == label].head(10))\n", + " \n", + " # # Save to CSV\n", + " # temp = result_df[result_df[\"entry_quality\"] == label]\n", + " # safe_label = label.replace(\";\", \"_\").replace(\"/\", \"_\")\n", + " # output_file = DATA_PATH / (\"entry_quality_\" + safe_label + \".csv\")\n", + " # temp.to_csv(output_file, index=False, encoding=\"utf-8\")\n", + " \n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "cf84e23f-a37e-4c60-9ca7-565e5e43e66b", + "metadata": {}, + "source": [ + "# Part 3: File System" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e052b41-926d-46b1-871f-1f4747010cb4", + "metadata": {}, + "outputs": [], + "source": "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\nFILESYSTEM_LISTING = DATA_PATH\nprint(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n\n\ndef apply_files_mapping(name):\n \"\"\"Rename raw slide entries via FILES_MAPPING before format checks so remapped\n non-standard names (e.g. two-digit ranks) pass the single-digit FILENAME_PATTERN.\"\"\"\n p = Path(name)\n stem = p.stem if p.suffix == \".mrxs\" else name\n if stem in FILES_MAPPING:\n return FILES_MAPPING[stem] + p.suffix\n return name\n\n\nlisted_entries = [\n apply_files_mapping(Path(line.strip()).name)\n for line in os.listdir(FILESYSTEM_LISTING)\n if line.strip()\n]\n\nall_folders = {\n entry\n for entry in listed_entries\n if Path(entry).suffix == \"\" and entry != \"annotations\" and FILENAME_PATTERN.fullmatch(entry)\n}\nall_files = {\n Path(entry).stem\n for entry in listed_entries\n if Path(entry).suffix == \".mrxs\" and FILENAME_PATTERN.fullmatch(entry[:-5])\n}\n\nprint(f\"Found {len(all_folders)} folders\")\nprint(f\"Found {len(all_files)} .mrxs files\")" + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "00368085-f81c-46db-939d-63b1e25baf45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Perfect 1:1 match between folders and .mrxs files\n" + ] + } + ], + "source": [ + "missing_folders = all_files - all_folders\n", + "missing_files = all_folders - all_files\n", + "\n", + "if not missing_folders and not missing_files:\n", + " print(\"✅ Perfect 1:1 match between folders and .mrxs files\")\n", + "else:\n", + " print(\"⚠️ Mismatch found:\")\n", + " if missing_folders:\n", + " print(\" Files without folders:\", missing_folders)\n", + " if missing_files:\n", + " print(\" Folders without files:\", missing_files)" + ] + }, + { + "cell_type": "markdown", + "id": "7ee9a88a-6fcb-4bbb-944d-b33b563fd4bd", + "metadata": {}, + "source": [ + "### Building mapping between table indices and files" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "bf1297d4-42ae-402c-9c76-51eeadf5b842", + "metadata": {}, + "outputs": [], + "source": [ + "invalid_filename_format_files = set()\n", + "\n", + "\n", + "def normalize_filename(fname):\n", + " \"\"\"Canonicalize slide filename stems so file matching can compare against record numbers.\"\"\"\n", + " if fname in FILES_MAPPING:\n", + " return FILES_MAPPING[fname]\n", + "\n", + " match = FILENAME_PATTERN.match(fname)\n", + " if match:\n", + " p, year, num, rank = match.groups()\n", + " num_padded = num.zfill(5)\n", + " if rank:\n", + " return f\"P{year}_{num_padded}-{rank}\"\n", + " else:\n", + " return f\"P{year}_{num_padded}\"\n", + " else:\n", + " invalid_filename_format_files.add(fname)\n", + " return None\n", + "\n", + "\n", + "def normalize_filename_rankless(fname):\n", + " \"\"\"Canonicalize a filename without rank so sibling slide files can be grouped together.\"\"\"\n", + " fname = normalize_filename(fname)\n", + " if fname is None:\n", + " return None\n", + " if \"-\" in fname:\n", + " return fname.split(\"-\")[0]\n", + " else:\n", + " return fname\n", + "\n", + "\n", + "def build_bi_dict(normalizing_f, data):\n", + " \"\"\"Build forward and reverse lookup dictionaries so normalized IDs can return source files.\"\"\"\n", + " data_to_norm: dict[str, str] = dict()\n", + " norm_to_data: dict[str, list[str]] = dict()\n", + "\n", + " for entry in data:\n", + " norm = normalizing_f(entry)\n", + " if norm is None:\n", + " continue\n", + " data_to_norm[entry] = norm\n", + "\n", + " if norm in norm_to_data:\n", + " norm_to_data[norm].append(entry)\n", + " else:\n", + " norm_to_data[norm] = [entry]\n", + "\n", + " return data_to_norm, norm_to_data\n", + "\n", + "\n", + "def index_to_filename(idx):\n", + " \"\"\"Convert a normalized record number to the matching normalized slide filename stem.\"\"\"\n", + " norm_idx = normalize_index(idx)\n", + " fname = norm_idx.replace(\"/\", \"_\")\n", + " return \"P\" + fname\n", + "\n", + "\n", + "def filename_to_index(fname):\n", + " \"\"\"Convert a normalized slide filename stem back to a record number for missing-entry rows.\"\"\"\n", + " norm_fname = normalize_filename(fname)\n", + " if norm_fname is None:\n", + " return None\n", + " if norm_fname[0] == \"P\":\n", + " norm_fname = norm_fname[1:]\n", + " idx = norm_fname.replace(\"_\", \"/\")\n", + " return idx" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "3dcdd970-4d25-4440-a384-b89322bfdea1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + } + ], + "source": [ + "norm_to_files = build_bi_dict(normalize_filename, all_files)[1]\n", + "norm_to_files_rankless = build_bi_dict(normalize_filename_rankless, all_files)[1]\n", + "if invalid_filename_format_files:\n", + " print(\"Files with unexpected filename format:\")\n", + " for file in sorted(invalid_filename_format_files):\n", + " print(f\" - {file}\")\n", + "\n", + "print([(norm, file) for norm, file in norm_to_files.items() if len(file) > 1])\n", + "\n", + "\n", + "def find_files_by_index(idx, existing_record_nums=None):\n", + " \"\"\"Find slide files while keeping ranked records tied to their exact ranked file.\"\"\"\n", + " existing_record_nums = existing_record_nums or set()\n", + "\n", + " # Step 1: convert table record number, e.g. 2025/00161, to normalized file stem P2025_00161.\n", + " target_file = index_to_filename(idx)\n", + " has_rank = \"-\" in target_file\n", + " rankless_target_file = target_file.split(\"-\")[0]\n", + "\n", + " # Step 2: keep exact normalized filename matches first.\n", + " # Examples: 2025/00161 -> P2025_00161, 2023/00946-2 -> P2023_00946-2.\n", + " exact_matches = norm_to_files.get(target_file, [])\n", + "\n", + " # Step 3: ranked records must not absorb sibling ranks when their exact file exists.\n", + " # Example: 2023/03525-1 should match P2023_03525-35 only, not P2023_03525-23.\n", + " if has_rank and len(exact_matches) > 0:\n", + " return exact_matches\n", + "\n", + " # Step 4: unranked records can collect sibling files that share the same rankless stem.\n", + " # Example: 2025/00161 can match both P2025_00161 and P2025_00161-2.\n", + " sibling_matches = norm_to_files_rankless.get(rankless_target_file, [])\n", + "\n", + " # Step 5: remove files that correspond to a different record already present in the table.\n", + " # Example: 2023/00946 should not also consume P2023_0946-2 when 2023/00946-2 exists.\n", + " res = []\n", + " for file in dict.fromkeys(exact_matches + sibling_matches):\n", + " file_record_num = filename_to_index(file)\n", + " if file_record_num != idx and file_record_num in existing_record_nums:\n", + " continue\n", + " res.append(file)\n", + "\n", + " return res" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "3dc78131-26ae-4261-9413-eefec64b3e98", + "metadata": {}, + "outputs": [], + "source": [ + "matched_files = set()\n", + "file_check_results = []\n", + "\n", + "valid_record_num = ~has_entry_quality_flag(result_df[\"entry_quality\"], \"RECORD_NUM\")\n", + "existing_record_nums = set(result_df.loc[valid_record_num, \"record_num\"])\n", + "\n", + "for idx, row in result_df[valid_record_num].iterrows():\n", + " record_num = row[\"record_num\"]\n", + " files_list = find_files_by_index(record_num, existing_record_nums)\n", + " matched_files.update(files_list)\n", + "\n", + " issue = \"\"\n", + " if len(files_list) == 0:\n", + " issue = \"File missing\"\n", + " elif len(files_list) == 1:\n", + " issue = \"OK\"\n", + " else:\n", + " issue = \"Multiple files\"\n", + "\n", + " file_check_results.append(\n", + " {\n", + " \"record_num\": record_num,\n", + " \"matched_files\": files_list,\n", + " \"file_count\": len(files_list),\n", + " \"file_issue\": issue,\n", + " }\n", + " )\n", + "\n", + "valid_format_files = all_files - invalid_filename_format_files\n", + "unmatched_files = valid_format_files - matched_files\n", + "\n", + "for file in unmatched_files:\n", + " file_check_results.append(\n", + " {\n", + " \"record_num\": filename_to_index(file) + \"M\",\n", + " \"matched_files\": [file],\n", + " \"file_count\": 1,\n", + " \"file_issue\": \"entry missing\",\n", + " }\n", + " )\n", + "\n", + "file_check_df = pd.DataFrame(file_check_results)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "96b23164-4693-4675-af5b-51bbef6c15d1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Filesystem Summary ===\n", + "OK: 2078 rows (95.2%)\n", + "File missing: 79 rows (3.6%)\n", + "Multiple files: 19 rows (0.9%)\n", + "entry missing: 6 rows (0.3%)\n", + "Total: 2182 rows\n", + "\n", + "=== File missing ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
2052023/02656[]0File missing
2062023/02659[]0File missing
2072023/02660[]0File missing
2082023/02661[]0File missing
2092023/02662[]0File missing
2102023/02674[]0File missing
2112023/02689[]0File missing
2122023/02699[]0File missing
2132023/02701[]0File missing
2142023/02702[]0File missing
2152023/02703[]0File missing
2162023/02704[]0File missing
2192023/02720[]0File missing
2902023/03525-1[]0File missing
2912023/03525-2[]0File missing
5082024/00674[]0File missing
5292024/00816[]0File missing
5522024/00988[]0File missing
6682024/01799[]0File missing
6902024/01928[]0File missing
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "205 2023/02656 [] 0 File missing\n", + "206 2023/02659 [] 0 File missing\n", + "207 2023/02660 [] 0 File missing\n", + "208 2023/02661 [] 0 File missing\n", + "209 2023/02662 [] 0 File missing\n", + "210 2023/02674 [] 0 File missing\n", + "211 2023/02689 [] 0 File missing\n", + "212 2023/02699 [] 0 File missing\n", + "213 2023/02701 [] 0 File missing\n", + "214 2023/02702 [] 0 File missing\n", + "215 2023/02703 [] 0 File missing\n", + "216 2023/02704 [] 0 File missing\n", + "219 2023/02720 [] 0 File missing\n", + "290 2023/03525-1 [] 0 File missing\n", + "291 2023/03525-2 [] 0 File missing\n", + "508 2024/00674 [] 0 File missing\n", + "529 2024/00816 [] 0 File missing\n", + "552 2024/00988 [] 0 File missing\n", + "668 2024/01799 [] 0 File missing\n", + "690 2024/01928 [] 0 File missing" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== Multiple files ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
732023/01462[P2023_01462, P2023_01462-1]2Multiple files
792023/01486[P2023_01486-2, P2023_01486-1]2Multiple files
842023/01591[P2023_01591-1, P2023_01591-2]2Multiple files
862023/01597[P2023_01597-1, P2023_01597-2]2Multiple files
882023/01599[P2023_01599-1, P2023_01599-2]2Multiple files
942023/01656[P2023_01656-2, P2023_01656-1]2Multiple files
1752023/02395[P2023_02395-1, P2023_02395-2]2Multiple files
1802023/02444[P2023_02444-1, P2023_02444-2]2Multiple files
2002023/02617[P2023_02617-2, P2023_02617-1]2Multiple files
2172023/02712[P2023_02712-1, P2023_02712-2]2Multiple files
3112023/03714[P2023_03714-3, P2023_03714-4]2Multiple files
6732024/01804[P2024_01804-1, P2024_01804-2]2Multiple files
7972024/02666[P2024_02666-1, P2024_02666-2]2Multiple files
8332024/02851[P2024_02851-2, 2024_02851-1]2Multiple files
10812025/00161[P2025_00161, P2025_00161-2]2Multiple files
15232025/02993[P2025_02993-1, P2025_02993-2]2Multiple files
15272025/03031[P2025_03031-1, P2025_03031-2]2Multiple files
15402025/03126[P2025_03126-2, P2025_03126-1]2Multiple files
16642025/03854[P2025_03854-2, P2025_03854-1]2Multiple files
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "73 2023/01462 [P2023_01462, P2023_01462-1] 2 Multiple files\n", + "79 2023/01486 [P2023_01486-2, P2023_01486-1] 2 Multiple files\n", + "84 2023/01591 [P2023_01591-1, P2023_01591-2] 2 Multiple files\n", + "86 2023/01597 [P2023_01597-1, P2023_01597-2] 2 Multiple files\n", + "88 2023/01599 [P2023_01599-1, P2023_01599-2] 2 Multiple files\n", + "94 2023/01656 [P2023_01656-2, P2023_01656-1] 2 Multiple files\n", + "175 2023/02395 [P2023_02395-1, P2023_02395-2] 2 Multiple files\n", + "180 2023/02444 [P2023_02444-1, P2023_02444-2] 2 Multiple files\n", + "200 2023/02617 [P2023_02617-2, P2023_02617-1] 2 Multiple files\n", + "217 2023/02712 [P2023_02712-1, P2023_02712-2] 2 Multiple files\n", + "311 2023/03714 [P2023_03714-3, P2023_03714-4] 2 Multiple files\n", + "673 2024/01804 [P2024_01804-1, P2024_01804-2] 2 Multiple files\n", + "797 2024/02666 [P2024_02666-1, P2024_02666-2] 2 Multiple files\n", + "833 2024/02851 [P2024_02851-2, 2024_02851-1] 2 Multiple files\n", + "1081 2025/00161 [P2025_00161, P2025_00161-2] 2 Multiple files\n", + "1523 2025/02993 [P2025_02993-1, P2025_02993-2] 2 Multiple files\n", + "1527 2025/03031 [P2025_03031-1, P2025_03031-2] 2 Multiple files\n", + "1540 2025/03126 [P2025_03126-2, P2025_03126-1] 2 Multiple files\n", + "1664 2025/03854 [P2025_03854-2, P2025_03854-1] 2 Multiple files" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "=== entry missing ===\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummatched_filesfile_countfile_issue
23022025/00137M[P2025_00137]1entry missing
23032025/00310M[P2025_00310]1entry missing
23042024/03383M[P2024_03383]1entry missing
23052023/01814M[P2023_01814]1entry missing
23062023/00146-2M[P2023_0146-2]1entry missing
23072025/03567M[P2025_03567]1entry missing
\n", + "
" + ], + "text/plain": [ + " record_num matched_files file_count file_issue\n", + "2302 2025/00137M [P2025_00137] 1 entry missing\n", + "2303 2025/00310M [P2025_00310] 1 entry missing\n", + "2304 2024/03383M [P2024_03383] 1 entry missing\n", + "2305 2023/01814M [P2023_01814] 1 entry missing\n", + "2306 2023/00146-2M [P2023_0146-2] 1 entry missing\n", + "2307 2025/03567M [P2025_03567] 1 entry missing" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\n=== Filesystem Summary ===\")\n", + "file_check_display = file_check_df.drop_duplicates(subset=[\"record_num\", \"file_issue\"])\n", + "counts = file_check_display[\"file_issue\"].value_counts()\n", + "\n", + "for label, count in counts.items():\n", + " pct = count / len(file_check_display) * 100\n", + " print(f\"{label}: {count} rows ({pct:.1f}%)\")\n", + "\n", + "print(f\"Total: {len(file_check_display)} rows\\n\")\n", + "\n", + "\n", + "for label in counts.keys():\n", + " if label == \"OK\":\n", + " continue\n", + "\n", + " print(f\"=== {label} ===\")\n", + " display(file_check_display[file_check_display[\"file_issue\"] == label].head(20))\n", + " print(\"\\n\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "7d34c697-97f0-4e88-a16f-938cbdc50543", + "metadata": {}, + "source": [ + "# Part 4: Merge tables and filesystem" + ] + }, + { + "cell_type": "markdown", + "id": "52f77a9c-8d59-4d6f-a670-cd9070605629", + "metadata": {}, + "source": [ + "### Expand rows for multiple files" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bc477030-59f9-427d-a66b-b7ffb93de021", + "metadata": {}, + "outputs": [], + "source": [ + "final_rows = []\n", + "\n", + "for idx, row in file_check_df.iterrows():\n", + " matched_files = row[\"matched_files\"]\n", + "\n", + " if len(matched_files) == 0:\n", + " # No files found - add row with no path\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = None\n", + " final_rows.append(row_dict)\n", + " elif len(matched_files) == 1:\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = str(DATA_PATH / matched_files[0])\n", + " final_rows.append(row_dict)\n", + " else:\n", + " # Create one row per file\n", + " for file_name in matched_files:\n", + " row_dict = {k: v for k, v in row.items() if k not in [\"matched_files\"]}\n", + " row_dict[\"path\"] = str(DATA_PATH / file_name)\n", + " final_rows.append(row_dict)\n", + "\n", + "expanded_df = pd.DataFrame(final_rows).drop_duplicates()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "ffaae63c-5135-4c8a-8fd6-8bb57f3662f0", + "metadata": {}, + "outputs": [], + "source": [ + "final_df = result_df.merge(\n", + " expanded_df[[\"record_num\", \"path\", \"file_issue\"]], on=\"record_num\", how=\"outer\"\n", + ")\n", + "\n", + "final_df = final_df[\n", + " (final_df[\"file_issue\"].astype(str).isin([\"OK\", \"Multiple files\"]))\n", + " & (final_df[\"entry_quality\"] == \"OK\")\n", + "]\n", + "final_df = final_df[[\"record_num\", \"mammaprint_index\", \"type\", \"path\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "e66fd8a3", + "metadata": {}, + "outputs": [], + "source": [ + "if DROP_ROWS:\n", + " before = len(final_df)\n", + " mask = final_df[\"record_num\"].apply(\n", + " lambda x: any(sub in str(x) for sub in DROP_ROWS)\n", + " )\n", + " final_df = final_df[~mask]\n", + " print(f\"Dropped {before - len(final_df)} rows matching DROP_ROWS ({len(DROP_ROWS)} entries)\")\n", + " print(f\"Remaining: {len(final_df)} rows\")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "b90a7fac-9575-49f9-884c-28c1875dd435", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepath
14102025/00968-10.541a luminal/mnt/data/MOU/breast/mammaprint/P2025_00968
14112025/00968-2-0.076b luminal/mnt/data/MOU/breast/mammaprint/P2025_00968-2
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "1410 2025/00968-1 0.541 a luminal \n", + "1411 2025/00968-2 -0.076 b luminal \n", + "\n", + " path \n", + "1410 /mnt/data/MOU/breast/mammaprint/P2025_00968 \n", + "1411 /mnt/data/MOU/breast/mammaprint/P2025_00968-2 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepath
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [record_num, mammaprint_index, type, path]\n", + "Index: []" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "search_df(final_df, \"2025\", \"968\")\n", + "search_df(final_df, \"2023\", \"3525\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "9b0bc82b-0738-4244-83da-5cbee0c2e1fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "✅ Saved final dataset to: data_mapping_final.csv\n", + " 2040 rows × 4 columns\n" + ] + } + ], + "source": [ + "# Save to CSV\n", + "output_file = \"data_mapping_final.csv\"\n", + "final_df.to_csv(output_file, index=False, encoding=\"utf-8\")\n", + "\n", + "print(f\"\\n✅ Saved final dataset to: {output_file}\")\n", + "print(f\" {len(final_df)} rows × {len(final_df.columns)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "8b80e36f-0baa-406d-b142-cbade0998513", + "metadata": {}, + "source": [ + "# Part 5: Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "a98b1bea-95e5-4672-b8c8-3408a22b0a55", + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_year(record_num):\n", + " \"\"\"Extract the record year so dataset coverage can be summarized by calendar year.\"\"\"\n", + " m = re.match(INDEX_PATTERN, record_num)\n", + " try:\n", + " return int(m.group(1))\n", + " except:\n", + " print(record_num)\n", + "\n", + "\n", + "def count_entries_per_year(df, plot=True, ax=None):\n", + " \"\"\"Count records by year so the final mapping can be checked for temporal coverage.\"\"\"\n", + " years = df[\"record_num\"].apply(_extract_year)\n", + " # years = years.dropna().astype(int)\n", + " counts = years.value_counts().sort_index()\n", + " result = {\"counts\": counts, \"total\": int(counts.sum())}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(8, 4))\n", + " created_ax = True\n", + " ax.bar(counts.index.astype(str), counts.values)\n", + " ax.set_title(\"Count of entries per year\")\n", + " ax.set_xlabel(\"Year\")\n", + " ax.set_ylabel(\"Count\")\n", + " ax.set_xticks(counts.index.astype(int))\n", + " ax.set_xticklabels([str(y) for y in counts.index], rotation=45)\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def mammaprint_distribution(df, plot=True, ax=None, bins=20, ddof=1):\n", + " \"\"\"Summarize Mammaprint scores so score range and distribution shifts are visible.\"\"\"\n", + " s = pd.to_numeric(df[\"mammaprint_index\"], errors=\"coerce\").dropna()\n", + " count = int(s.size)\n", + " mean = float(s.mean()) if count > 0 else float(\"nan\")\n", + " variance = float(s.var(ddof=ddof)) if count > 0 else float(\"nan\")\n", + "\n", + " result = {\"series\": s, \"mean\": mean, \"variance\": variance, \"count\": count}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(7, 4))\n", + " created_ax = True\n", + " ax.hist(s, bins=bins)\n", + " ax.axvline(mean, linestyle=\"--\", linewidth=1)\n", + " ax.set_title(\"MammaPrint index distribution\")\n", + " ax.set_xlabel(\"MammaPrint index\")\n", + " ax.set_ylabel(\"Frequency\")\n", + " ax.grid(True, linestyle=\":\", linewidth=0.5)\n", + " ax.text(\n", + " 0.98,\n", + " 0.95,\n", + " f\"mean={mean:.3f}\\nvar={variance:.4f}\\nn={count}\",\n", + " transform=ax.transAxes,\n", + " ha=\"right\",\n", + " va=\"top\",\n", + " fontsize=9,\n", + " bbox=dict(boxstyle=\"round\", alpha=0.1),\n", + " )\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def mammaprint_distribution_per_type(df, plot=True, bins=20, ddof=1):\n", + " \"\"\"Summarize Mammaprint scores per type so class-specific distributions can be compared.\"\"\"\n", + " results = {}\n", + " grouped = df.groupby(\"type\")\n", + "\n", + " for typ, sub in grouped:\n", + " s = pd.to_numeric(sub[\"mammaprint_index\"], errors=\"coerce\").dropna()\n", + " count = int(s.size)\n", + " mean = float(s.mean()) if count > 0 else float(\"nan\")\n", + " variance = float(s.var(ddof=ddof)) if count > 0 else float(\"nan\")\n", + " results[typ] = {\"series\": s, \"mean\": mean, \"variance\": variance, \"count\": count}\n", + "\n", + " if plot:\n", + " fig, ax = plt.subplots(figsize=(8, 4))\n", + " for typ, info in results.items():\n", + " if info[\"count\"] == 0:\n", + " continue\n", + " ax.hist(\n", + " info[\"series\"],\n", + " bins=bins,\n", + " alpha=0.45,\n", + " label=f\"{typ} (n={info['count']})\",\n", + " )\n", + " ax.set_title(\"MammaPrint index distribution by type\")\n", + " ax.set_xlabel(\"MammaPrint index\")\n", + " ax.set_ylabel(\"Frequency\")\n", + " ax.legend()\n", + " ax.grid(True, linestyle=\":\", linewidth=0.5)\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " summary_rows = []\n", + " for typ, info in results.items():\n", + " summary_rows.append(\n", + " {\n", + " \"type\": typ,\n", + " \"count\": info[\"count\"],\n", + " \"mean\": info[\"mean\"],\n", + " \"variance\": info[\"variance\"],\n", + " }\n", + " )\n", + " summary_df = pd.DataFrame(summary_rows).sort_values(\"count\", ascending=False)\n", + " print(\"\\nSummary (by type):\")\n", + " print(summary_df.to_string(index=False))\n", + "\n", + " return results\n", + "\n", + "\n", + "def count_distinct_types(df, plot=True, ax=None):\n", + " \"\"\"Count labels by type so class balance and missing type values can be reviewed.\"\"\"\n", + " counts = df[\"type\"].fillna(\"MISSING\").value_counts()\n", + " result = {\"counts\": counts, \"total_distinct\": int(counts.size)}\n", + "\n", + " if plot:\n", + " created_ax = False\n", + " if ax is None:\n", + " fig, ax = plt.subplots(figsize=(6, 4))\n", + " created_ax = True\n", + " ax.barh(counts.index.astype(str), counts.values)\n", + " ax.set_title(\"Counts by type\")\n", + " ax.set_xlabel(\"Count\")\n", + " ax.set_ylabel(\"Type\")\n", + " for i, v in enumerate(counts.values):\n", + " ax.text(v, i, f\" {v}\", va=\"center\")\n", + " if created_ax:\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " return result\n", + "\n", + "\n", + "def run_all_stats_and_plot(df, save_fig=None):\n", + " \"\"\"Run all final dataset summaries together so one cell produces the audit plots and metrics.\"\"\"\n", + " results = {}\n", + " results[\"per_year\"] = count_entries_per_year(df, plot=False)\n", + " results[\"mammaprint_overall\"] = mammaprint_distribution(df, plot=False)\n", + " results[\"mammaprint_by_type\"] = mammaprint_distribution_per_type(df, plot=False)\n", + " results[\"counts_by_type\"] = count_distinct_types(df, plot=False)\n", + "\n", + " fig, axs = plt.subplots(2, 2, figsize=(14, 9))\n", + " ax00, ax01, ax10, ax11 = axs.flatten()\n", + "\n", + " counts_year = results[\"per_year\"][\"counts\"]\n", + " if not counts_year.empty:\n", + " ax00.bar(counts_year.index.astype(str), counts_year.values)\n", + " ax00.set_title(\"Entries per year\")\n", + " ax00.set_xlabel(\"Year\")\n", + " ax00.set_ylabel(\"Count\")\n", + " ax00.tick_params(axis=\"x\", rotation=45)\n", + "\n", + " overall_series = results[\"mammaprint_overall\"][\"series\"]\n", + " if overall_series.size > 0:\n", + " ax01.hist(overall_series, bins=20)\n", + " ax01.axvline(results[\"mammaprint_overall\"][\"mean\"], linestyle=\"--\", linewidth=1)\n", + " ax01.set_title(\"MammaPrint index (overall)\")\n", + " ax01.set_xlabel(\"Index\")\n", + " ax01.set_ylabel(\"Frequency\")\n", + "\n", + " counts_type = results[\"counts_by_type\"][\"counts\"]\n", + " if not counts_type.empty:\n", + " ax10.barh(counts_type.index.astype(str), counts_type.values)\n", + " for i, v in enumerate(counts_type.values):\n", + " ax10.text(v, i, f\" {v}\", va=\"center\")\n", + " ax10.set_title(\"Counts by type\")\n", + " ax10.set_xlabel(\"Count\")\n", + "\n", + " for typ, info in results[\"mammaprint_by_type\"].items():\n", + " if info[\"count\"] == 0:\n", + " continue\n", + " ax11.hist(\n", + " info[\"series\"], bins=20, alpha=0.45, label=f\"{typ} (n={info['count']})\"\n", + " )\n", + " ax11.set_title(\"MammaPrint index by type\")\n", + " ax11.set_xlabel(\"Index\")\n", + " ax11.set_ylabel(\"Frequency\")\n", + " if len(results[\"mammaprint_by_type\"]) > 0:\n", + " ax11.legend()\n", + "\n", + " plt.tight_layout()\n", + "\n", + " if save_fig:\n", + " fig.savefig(save_fig, dpi=150)\n", + " print(f\"Saved combined figure to: {save_fig}\")\n", + "\n", + " plt.show()\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "86118b8b-0092-4c58-8758-2e07137b7082", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved combined figure to: stats\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABW0AAAN5CAYAAABzABkEAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAA/JRJREFUeJzs3Xd4FFX7//FPGgmkAaEkhNAk9CogvYbepQpKFwQrRRFQqSr6AA9YeZCmoKIiCFJEmhQRpIP0JhBCb0lISCE5vz/4sV/XJJCEkN0s79d1zXWxZ+45c88wyU7uPXvGyRhjBAAAAAAAAACwC862TgAAAAAAAAAA8H8o2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAYEfefvttOTk52ToNAAAAIE1ef/11eXh4ZPn9TZs2TU5OTrp48WKG9/0gxhhVq1ZNw4YNy/R9PyqlSpVSp06d7tuWmJiocuXK6e23387s9AC7RtEWwGNt8uTJcnJySnH58ssv09Xv66+/LldX14xNFgAAAA9t7969lnu9cePGJRvz5ptvWmJOnDiRyRlmvn+eEycnJzk7Oyt37txq2rSpfvvtt0zJ4eWXX5aXl1em7MtezZ8/X4cPH9abb75p61QylbOzs8aNG6cpU6bo7Nmztk4HsBsUbQFA0tatW2WMSbL07t07U/N49913ZYzJ1H0CAAA8jjw9PfXVV18lufdKSEjQ119/LU9PTxtlZjsTJkyQMUbx8fHauHGjoqKi1KRJE23YsOGB206ePFkxMTGPPkkb7S8zTJw4UV27dlW+fPlsnUqme/rpp5U7d25NnTrV1qkAdoOiLQAAAADgsdOhQwf9/fff2rRpk1X7r7/+qvPnz6tjx442ysz2XFxcVL58ec2dO1cJCQn65JNPbJ2Sw9uwYYOOHDmiHj162DoVm3B2dla3bt301VdfOVwxHkgvirYAkEr3vrIVFRWl559/Xjlz5pSvr6+effZZhYeHW+I6deqkKVOmKCEhweprZve+Wnevn4iICPXt21d+fn4KDg6WlPKctocOHVKXLl2UN29eZcuWTaVKldLUqVOtRoacPn1azz77rAIDA5UjRw6VLVtW48ePV1RUVKqO6+bNm3r22Wfl6+urXLlyqVevXrp69Wq6crnfMf7bZ599JicnJ+3cuTPJupkzZ8rJyUl//PFHmvb/6aefWp17Hx8f1atXTytWrEj22FOTJwAAcCxPPvmkKlSokGQ6rLlz56pOnTrJ3hMcOXLE6h7Dw8ND5cqV0+TJk5WYmGiJW7JkiZycnLR161a9//77CgwMlI+Pj5577jlFRUXJGKP33ntPBQsWVPbs2dWmTRtdvnzZal8Z0Udq801JcHCwXFxcFBoaKkl6/vnnlSdPHt24cUM9e/ZU7ty5Vb58eUnJzzF7Lz4yMlK9e/eWr6+vcubMqV69eunWrVuWuNatW+uzzz5TVFSUVb7nzp1LMbeH2d89CxcuVPny5eXh4aHSpUtr4cKFKe7vr7/+UseOHZUnTx65u7urdOnS+vTTTy3rV61aJRcXF40aNcpqu6VLl8rJyUkTJkxIsW9JWrZsmdzd3VWrVq0k6xITEzVlyhSVKVNG7u7u8vPzU6dOnXTkyBFLTN26dRUcHJzst/ZatmypoKAgq//zH374QbVq1ZKnp6c8PT3VoEEDbd682Wo7f39/9e7dWzt27FC9evWUPXt2jRgxQpLUoEEDy/+Ti4uL/P391b17d50+ffq+x3k/jRo10o0bN5LkATyuKNoCQBq9+uqrat++vc6ePasff/xRy5cvt3pYwI8//qhhw4bJxcXFaqqF4sWLW/UzcOBAtW/fXidOnNDIkSNT3N+uXbv01FNP6fbt29qwYYOuXbum999/X+PHj9cbb7xhiWvZsqVCQ0O1bt063bhxQ4sXL5Yk/fTTT6k6rkGDBqlbt24KDQ3VkiVL9Ntvv6lp06aKi4tLcy5pOcaePXvK29tbn3/+eZJ106dPV6lSpSw3r6nd/8svv2w57wkJCTpw4ICqVaum9u3ba8eOHenKEwAAOJ7evXvrxx9/tHzIff36df38888pTpFVqlQpq/u78+fP6/XXX9fo0aP13//+N0n8tGnT5O3trQMHDmjt2rVas2aNXnvtNU2YMEGenp7av3+/Nm/erJ07d+qll15Kdp8P00da8/23EydOKCEhQUFBQZY2Y4xeeOEFPfPMMzp58qSGDh163z6MMXr55Zf1zDPP6Ny5c1qwYIEWL15sKf5J0vLly/XSSy/J09PTKt+CBQs+MMf07E+6W7Ts0qWLGjdurL///lu//vqrVq5cmewcvlu3blX16tVljNHvv/+uq1evaty4cRo1apTl4VnNmzfXW2+9pQ8++EDLly+XJJ08eVK9evVSixYtHviQrd9//10VKlRQtmzZkqwbOHCgRo0apddee00XL17Upk2bdPHiRdWoUUPHjx+XdLdgfeLEiSQjx8PCwrR69Wr16dNHzs53S0DvvfeeunfvrrZt2+rEiRM6deqUqlevrpCQkCQF03PnzmncuHH6/PPPdfLkSVWvXl3S3ZHB9/6foqOjtXz5cp05c0atW7dO90jZp556SpKSHAPw2DIA8BibNGmSkZTi8vfff1tiX3rpJSPJfPvtt1Z9DBkyxLi5uZmYmBhL27Bhw4yLi0uy+7zXz+zZs5Ose+utt8y/fzXXrFnTFCtWzKp/Y4yZNm2acXFxMaGhoebixYtGkpkxY0ZaT4Eln7lz51q1r1q1ykgyc+bMSVMuDzrG5AwaNMhkz57dXL9+3dK2detWI8lMmjQpzftPSfHixc3AgQOTHHtq8wQAAFnfnj17jCQzdepUc/nyZePm5ma+/PJLY4wxH3/8scmRI4eJiIgwEyZMMJLM8ePHH9jnwIEDTfHixS2vf/rpJyPJPP/881Zxo0ePNtmyZTMDBgywah83bpxxdnY2N27cyNA+UpvvvXMyYcIEY4wxd+7cMQcOHDB16tQxzs7OZt26dcYYY/r162ckmW+++SZJn8OGDTPu7u5WbffiFy9ebNU+aNAg4+HhYRISEixtL730kvH09Hxg7hm1v6JFi5rq1atbxd25c8c88cQTRpK5cOGCpb1y5cqmVKlSJi4uzir+gw8+MNmyZTOXLl0yxhiTkJBgmjRpYnLmzGkOHjxoKlasaAoVKmSuXr36wOPx9/c3bdu2TdJ+8OBBI8kMHz7cqv3SpUsmR44cpmvXrsYYY6KiooyPj4/p0aOHVdy7775rnJyczKlTp4wxxpw+fdq4urqa1157Lcm+6tata2rXrm15nT9/fuPu7m45vgfZu3evkWTWrl1raStZsqTp2LGjVVxybfe4ubklOQbgccVIWwBQyg8iK1KkSJLYVq1aWb0uV66c4uPjLV8bS622bds+MObq1avaunWr2rZtK3d3d6t1jRs3VkJCgrZs2aI8efKoQIECmjhxor766itdunQpTbkkl0+zZs2UPXt2rV+/Pk25pPUYJemll17S7du3rb6e+Pnnn8vNzU09e/ZM8/6jo6P1zjvvqHTp0sqePbvVFBXJPQE6tXkCAADHkjdvXrVo0cJyDzJ37lx16tRJ3t7eKW7z5ZdfqlatWvLx8bHcY/zvf//TqVOnkkw50KJFC6vXpUqVUlxcnEJCQqzaS5curcTExGS/Wv6wfaQl33feeUdOTk5yc3NTnTp15O7urlWrVqlRo0ZWcWm5d3JyckpyDOXKlVNMTIzOnz+f6n4ycn9///23/v777yTH4eLiojZt2li1hYWFac+ePWrfvr3c3Nys1jVu3FhxcXHaunWrpLvzsn7zzTfy8vJSlSpVdPjwYS1cuFB+fn4PzPvmzZvJXnf37sU7dOhg1Z4vXz7VrVtX69atkyTlyJFD3bp106JFixQRESHp7qjjuXPnqlGjRipatKiku9M43LlzR507d06yr5CQEG3bts3qm3Y1atRI9sFoR48eVbdu3VSgQAG5urrKyclJlSpVkqRk77dTy9vbWzdv3kz39oAjoWgLAGmQI0cO+fj4WLXde52Wm4ts2bIpT548D4y7V3z9+OOP5erqKhcXF7m4uMjZ2VnlypWTJF27dk0uLi5avXq1ypcvr4EDB8rf319lypTR+PHjFR0d/cD9uLq6Knfu3Ena8+XLZ5nXNrW5pPUYJals2bKqV6+e/ve//8kYo2vXrmnhwoVq3bq15SYxLfvv2bOnPvnkE73//vs6d+6cEhISZIxRpUqVFB8fb7XvtOQJAAAcT+/evbVx40YtXbpUe/bsSXFqBEn63//+pz59+qh169Y6ePCg4uPjZYzR66+/rsTExCRF0ICAAKvX94pyKbUndz/5MH2kNd8JEybIGKPExETduHFDa9euVZMmTaxifHx85OXllSTPlOTKlSvJ3LPpuX/OyP3du2fMnz9/ku3/3Xbx4kVJ0n/+858k96BVq1a16k+6+0FA27ZtFRMTo0aNGlm+8v8gOXPmtBRb/+le3/7+/knW+fv7W+37+eefV3R0tBYsWCBJ2rhxo06ePKl+/folOZ769etbjsfZ2VnOzs4aO3asEhISrP5fAgMDk82pdu3aOnfunH7++WeFh4fLGGMp1v77fjstIiMjlStXrnRvDzgSirYAkAbJPSQsPf79KX1K7hUTR40apTt37ighIUEJCQlKTEy0jAZ+8cUXJd0tfP7888+6efOmtmzZolatWmncuHGW9fdz584dXb9+PUn75cuXLSMD0pJLWo7xnpdeeknHjh3TunXrNGfOHMXExFjdYKZ2/7du3dJPP/2kF198UU8//bT8/Pws83clN3olrXkCAADH0rp1a/n5+alPnz4qUqSIGjRokGLsvHnzVK1aNY0aNUpBQUFydXWVdHfkZnJSundMyz3lw/SR1nxTI633Thl1/5yR+7t3f5vct9P+3XbvHnT8+PEp3oP27dvXEr9y5UpNnz5d1atX16pVq/TVV1+lKu/ChQtbCqr/dG9gRUq5/nMUb9WqVVWxYkXNmTNHkjRnzhzlypVLTz/9dJLj2bVrl+V47hXw7x3PP0fWJvf/vWLFCl27dk0fffSRqlatKk9PT0kPd11Jd79ZFx8fr8KFCz9UP4CjoGgLAI+Ap6enEhMTdefOnYfqJ3/+/KpWrZp++umnVH9ife+ps5MmTVLDhg1TPZH/smXLrF6vXr1at2/ftnz1Lj25pMXTTz+tgIAAffbZZ5oxY4YKFCig5s2bW9andv9OTk4yxiSZQuFeQRsAAOCf3Nzc9Oyzz+rGjRvq1avXA4t+/77HuHr1qlatWvUoU3woWSVfT09Pq6/lP0pFixZVkSJFktz/JiYmWh4idk/hwoVVvnx5LV68+IH39mfOnFGPHj3UsGFDbdmyRV26dNGLL76ov/7664E51alTR/v3709yDu5NTfHvhwtfuXJFmzZtSjJNRr9+/bR9+3Zt2bJFP/74o5599lmrkcctWrSQi4uLvv/++wfm9CD/vrbmzZv3UP1t375dklS3bt2H6gdwFBRtAeARKFeunIwxWr58uRISEh6qr+nTp+vs2bN6+umntWvXLkVHR1u+itSkSRNdu3ZNe/bsUYcOHbRmzRpdvnxZt2/f1po1a7Rr1y41bNjwgfvw9PS0PDE3MjJSmzZt0oABA1ShQgV17949Tbmkl5ubm/r3768lS5bo5MmT6t27t1xcXNJ8Ljw9PdWoUSPNmDFD27Zt061bt7RixQr95z//UYUKFdKdHwAAcFzTpk2TMUZjx469b1zbtm21ZcsWffXVV7p165b27t2rDh06JCmc2YuslO+950T88ssvD33/nBoTJ07Utm3b9Prrr+vixYsKDQ1V//79VaZMmSSxM2bM0JEjR9S5c2ft2bNH0dHRCg0N1U8//aSGDRsqKipKcXFx6ty5szw8PLRgwQK5uLho1qxZCgoKUqdOnRQZGXnffNq0aaPY2Fj98ccfVu1ly5ZV3759NXXqVM2aNUs3b97UoUOH1LFjR7m6umrcuHFW8c8995w8PDzUrVs33b592+qba5JUrFgxjR8/XpMmTdL48eN1+vRp3b59W0eOHNHHH3+sPn36PPDcNWrUSF5eXhoxYoTCwsJ06dIlTZgwQbGxsQ/c9n7Wr1+vnDlzUrQF/j+KtgAgqWbNmpYHM/xzGTFiRLr669ixowYMGKABAwbIzc3N8hCs9KhSpYp2796tPHnyqF27dsqZM6dq166tr776SiNHjpSfn58qVqyoXr166b///a/Kly+vvHnzavDgwXr99df1+eefp2o/n332mebNm6fAwEC1bdtWdevW1dq1a60+QU9NLg9jwIABlgcZ/PNrZmnd/9dff6169eqpdevWCgwM1MyZM7VgwQJlz579ofIDAACPt9dff11vvfWW3nnnHeXLl08vvPCCxo0bp/Lly9s6tWRlpXy7deumPn36qEePHpb753Pnzj2y/T3zzDP67rvvtGrVKhUuXFiNGzdWkyZNkh3wULNmTe3atUteXl5q3bq1pbD47bffauzYsfL09NTgwYO1Z88e/fDDD5bpBby9vfXjjz/q3Llzev755++bT/369VW6dGnNnz8/ybovvvhCEyZM0JQpU5Q/f37VqVNHefLk0datW1WyZEmr2HvTIYSGhurJJ5+0PBzsn0aNGqXFixdr06ZNqlSpkvz8/NSxY0edPn1aY8aMeeC5K1iwoJYtW6bLly+rRIkSqlKlimJjY/Wf//zngdumJDExUQsWLFDv3r2TzEkMPK6cjDHG1kkAAGzn5Zdf1pdffqlbt27ZOhXdvHlTAQEBqlGjhn777TdbpwMAAABkmq+//loDBw7U33//rbx589o6nUy1aNEiPffcczp69KgKFSpk63QAu8BIWwCA3Vi6dKliYmLUv39/W6cCAAAAZKpnn31WZcqU0cSJE22dSqZKTEzUmDFjNGzYMAq2wD+42joBAAAk6eLFi5o8ebKCg4PVuXNnW6cDAAAAZConJyfLw7geJ87Ozjpw4ICt0wDsDiNtAQA2V6NGDQUFBcnd3V0//PCD3NzcbJ0SAAAAAAA2w5y2AAAAAAAAAGBHGGkLAAAAAAAAAHaEoi0AAAAAAAAA2BEeRPYIJCYm6vz58/L29paTk5Ot0wEAAHAoxhhFRkaqQIECcnZmDEJG414WAADg0UntvSxF20fg/PnzCgoKsnUaAAAADi00NFQFCxa0dRoOh3tZAACAR+9B97IUbR8Bb29vSXdPvo+Pj42zAQAAcCwREREKCgqy3HMhY3EvCwAA8Oik9l6Wou0jcO9rZD4+PtzoAgAAPCJ8df/R4F4WANLu3I1ofbLuhF4JKa6CuXLYOh0AWcCD7mWZBAwAAAAAAOAh3IyO1/c7Q3UzOt7WqQBwEBRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAgIeQx8tdgxo8oTxe7rZOBYCDcLV1AgAAAAAAAFmZv6+H3mxeytZpAHAgjLQFAAAAAAB4CLdi72jryWu6FXvH1qkAcBAUbQEAAAAAAB7C6atR6jZzm05fjbJ1KgAcBEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAA4CG4ujjJ38dDri5Otk4FgINwtXUCGWndunVatGiRLl++rAIFCqh79+6qUaOGVcypU6f06aef6syZMwoODtbgwYPl7++f5hgAAAAAAABJKuXvo22jQmydBgAH4jAjbWfOnKkWLVoof/78euaZZ5QtWzbVrl1bP//8syXm5MmTqlq1qs6fP6/27dtr3759qlq1qq5cuZKmGAAAAAAAAAB4VJyMMcbWSWSEBg0aqFChQpo3b56lrUmTJvLz89N3330nSerVq5cOHjyoHTt2yMnJSXFxcSpevLi6d++uDz74INUxDxIRESFfX1+Fh4fLx8cn4w8WAOxIkRErbJ0CMsjpD1rZOgUgVbjXerQ4vwCQdkcuRqj3nB36sm81lfLPuN+dmX2vzf0g8Oil9l7LYUbalipVSsePH1dCQoIkKTo6WqdPn1aZMmUsMatWrdLTTz8tJ6e7c8xky5ZNbdq00apVq9IUAwAAADwKxhilNKYiLi5Ot27dslqioqJS7Cs+Pv5RpQkA+Jc7CUYXI2J0J8EhxsUBsAMOU7T973//q4oVK6pYsWJq0qSJgoOD1blzZ40aNUrS3SLu5cuXVbBgQavtgoKC9Pfff6c6JjmxsbGKiIiwWgAAAIDU2rhxo1q2bKmcOXPKy8tL9erV07Zt26xi3n//ffn6+srf39+ylC1bNklfn376qQICAuTh4aGiRYvqxx9/zKzDAAAAQAZxmKLtmjVr9MMPP6hnz57q37+/unbtqi+++EJ//vmnpLsjEyQpR44cVtvlyJHDsi41McmZOHGifH19LUtQUFCGHRcAAAAc36effqrXXntNYWFhunTpkipUqKBmzZrp3LlzVnE1a9a0Gml7+vRpq/WLFi3SsGHD9L///U/R0dEaPny4nnnmGW3fvj0TjwYAAAAPy2GKti+++KJeeOEFTZgwQV26dNF///tftW7dWoMHD5YkeXl5ycXFRdevX7fa7tq1a8qZM2eqY5IzcuRIhYeHW5bQ0NCMPDQAAAA4uIULF6pZs2by8vKSl5eXpkyZoqioKK1fvz5J7J07d1KcQmHatGnq0KGD2rVrJ3d3dw0aNEiVK1fWJ5988qgPAQAAABnIIYq2CQkJunz5sp544gmr9uLFiyssLEyS5OrqqrJly2rfvn1WMXv37lXFihVTHZMcd3d3+fj4WC0AAABAel24cEEJCQny8/Ozat+xY4c8PT3l7e2thg0bateuXZZ1CQkJ2rFjh+rVq2e1TcOGDZNMtQAAyFhF8nhqQf8aKpLH09apAHAQDlG0dXFxUZUqVfTdd99ZpjGIiorSokWLVL16dUtcz5499cMPP1i+RrZv3z79+uuv6tmzZ5piAAAAgEclMTFRL7/8skqUKKHGjRtb2gsXLqzFixcrIiJCp06dUpEiRdSwYUOdOXNGkhQeHq7Y2FjlzZvXqr+8efPq8uXLKe6P5zMAwMPzcndVzSf85OXuautUADgIhyjaStKsWbMUFhamYsWKKSQkREWLFpWzs7M++ugjS8yrr76qkJAQVahQQbVr11atWrXUt29fdevWLU0xAAAAwKPy6quvauvWrVq0aJHc3d0t7X369FGrVq3k7u6ufPny6YsvvpCXl5fmzJljtX1iYmKS105OTinuj+czAMDDuxgeow9XHdHF8BhbpwLAQTjMR0DlypXTgQMHdPjwYV26dEmBgYEqWbKk1Q2qm5ubvv/+ex09elRnz55V8eLFVbRoUat+UhMDAAAAPApDhgzRt99+q3Xr1qlcuXL3jXVzc9MTTzyhU6dOSZJy5syp7NmzJxlVe/nyZQUEBKTYz8iRIzV06FDL64iICAq3AJBGV2/FavqGk2pVPkD+vh62TgeAA3CYoq10d5qEcuXKPfAGt2TJkipZsuRDxwAAAAAZZejQofrqq6+0Zs0aVa5c+YHxt2/f1pEjR1S3bl1JkrOzs2rVqqV169bp5ZdftsStWbNGderUSbEfd3d3qxG9AAAAsD2HmR4BAAAAyKqGDx+uWbNmacmSJSpZsqRu3bqlW7duKT4+3hLTokUL/fLLL7pw4YL279+vrl27Kj4+XgMGDLDEvPnmm1q+fLlmzJih0NBQjRkzRseOHdPgwYNtcFQAAABIL4q2AAAAgI198cUXSkxMVMuWLeXv729Zpk6daol57733NHPmTFWtWlWdOnWSr6+vdu3apSJFilhimjRpom+++UbTp09XxYoVtXr1aq1cuVJly5a1wVEBAAAgvRxqegQAAAAgK7p58+YDY5588kktXrz4gXFdunRRly5dMiArAEBq5czhpq5Vg5Qzh5utU3koRUasyLR9nf6gVabtC8iKKNoCAAAAAAA8hIK5cujDThVsnQYAB8L0CAAAAAAAAA8hJj5Bxy5FKiY+wdapAHAQFG0BAAAAAAAewonLt9R06iaduHzL1qkAcBAUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAICHlM2FEguAjONq6wQAAAAAAACysnKBvjr2XgtbpwHAgfAxEAAAAAAAAADYEYq2AAAAAAAAD+HE5Ui1+nizTlyOtHUqABwERVsAAAAAAICHEBOfqIPnIxQTn2jrVAA4CIq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAwEMIypVDn3V/UkG5ctg6FQAOwtXWCQAAAAAAAGRlvjnc1KpCgK3TAOBAGGkLAAAAAADwEK5ExmrW5lO6Ehlr61QAOAiKtgAAAAAAAA/hUkSM3l1xWJciYmydCgAHQdEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAeAjeHq5qXDqfvD1cbZ0KAAfBbxMAAAAAAICHUNjPU7N6VbN1GgAcCCNtAQAAAAAAHkJ8QqKu3YpVfEKirVMB4CAYaQs85oqMWGHrFJCBTn/QytYpAAAAAI+doxcj1fqT37X8lToqF+hr63QAOABG2gIAAAAAAACAHXGYkbZnz55VXFxcknZPT08FBARYtcXExOjKlSvKnz+/smXLlmx/qYkBAAAAAAAAgIzmMCNt+/fvr+bNm1stwcHBGjp0qFXcmDFjlDt3blWqVEl58uTRJ598kqSv1MQAAAAAAAAAwKPgMCNtf/31V6vXO3fuVLVq1fTss89a2ubPn69JkyZp7dq1qlWrln7++Wd16NBBpUqVUpMmTVIdAwAAAAAAAACPisOMtP232bNnKzAwUC1atLC0TZ8+XR07dlStWrUkSW3btlXdunU1ffr0NMUAAAAAAADcUzrAR3+NbarSAT62TgWAg3DIou3t27e1YMEC9e3bVy4uLpKkxMRE7d69WzVr1rSKrVOnjnbu3JnqGAAAAAAAgH9ycXaSt4ebXJydbJ0KAAfhkEXbH3/8UZGRkerXr5+lLTIyUrGxsfLz87OKzZMnj65evZrqmOTExsYqIiLCagEAAAAAAI+Hv69GqcfsP/X31ShbpwLAQThk0Xb27Nlq2rSpChcubGlzdr57qPHx8VaxcXFxltG4qYlJzsSJE+Xr62tZgoKCMuQ4AAAAAACA/YuKvaPNx68qKvaOrVMB4CAcrmh74sQJbdq0Sf3797dq9/b2lq+vry5evGjVfvHiRRUsWDDVMckZOXKkwsPDLUtoaGgGHQ0AAAAAAACAx43DFW3nzJmjfPnyqU2bNknWNWjQQL/++qtV2y+//KIGDRqkKebf3N3d5ePjY7UAAAAAAAAAQHo4VNE2ISFBX331lXr37i03N7ck69966y1t2rRJo0eP1o4dO/TSSy8pNDRUw4YNS1MMAAAAAAAAADwqDlW03bZtm7Jnz67nn38+2fXVqlXT6tWrtWPHDvXu3VthYWHauHGjihcvnqYYAAAAAACAewJ8PTS+XVkF+HrYOhUADsLV1glkpNq1a+vEiRP3jalfv77q16//0DEAAAAAAODxU2TEihTXjV56MBMzAeDIHGqkLQAAAAAAAABkdRRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOuNo6AQAAAAAAgPQqMmKFrVMAgAzHSFsAAAAAAAAAsCOMtAUAAADsQFxcnA4ePKiEhASVKlVKXl5eycadPHlSZ8+eVXBwsAoWLJjuGAAAANgvRtoCAAAANvaf//xHRYsWVb9+/TRgwAAFBgZqxowZVjF37tzRM888o0qVKmnEiBEKDg7WsGHD0hwDAAAA+8dIWwAAAMDGnJ2d9ddffyl37tySpHnz5qlPnz6qWbOmKlSoIEmaOnWq1q5dq7/++ktFihTRzp07VatWLdWqVUsdO3ZMdQwAAADsHyNtAQAAABt7/fXXLQVbSXruuefk5OSknTt3Wtq++uorPfPMMypSpIgkqWrVqmrcuLG+/PLLNMUAAADA/lG0BQAAAOzMn3/+qYSEBJUsWVLS3fluDx8+rMqVK1vFVa5cWfv27Ut1THJiY2MVERFhtQAAAMC2KNoCAAAAdiQ8PFz9+vVTs2bNVLt2bUlSRESEEhMTrUbjSpKfn59u3LiR6pjkTJw4Ub6+vpYlKCgog48IAAAAaUXRFgAAALATUVFRat26tdzd3bVgwQJLe7Zs2SRJt2/ftoqPjo62rEtNTHJGjhyp8PBwyxIaGpohxwIAAID040FkAAAAgB2IiopSq1atFBERofXr1ytXrlyWdT4+PsqdO3eSguq5c+cs89emJiY57u7ucnd3z7DjAAAAwMNjpC0AAABgY9HR0WrVqpVu3LihdevWyc/PL0lMs2bNtHTpUsvruLg4LV++XM2bN09TDAAAAOwfI20BAAAAG2vXrp12796t6dOna+fOnZb2EiVKqFixYpKkMWPG6KmnnlLPnj3VokULzZ8/X5I0ePBgS3xqYgAAAGD/GGkLAAAA2Fi2bNlUq1YtzZ8/X9OmTbMsf/75pyWmZMmS2r59u3x8fLRgwQKVLVtW27dvV968edMUAwAAAPvHSFsAAADAxlasWJGquJIlS+rTTz996BgAAADYN0baAgAAAAAAAIAdoWgLAAAAAAAAAHbE4aZH+PPPP/XLL7/IyclJnTp1UtmyZa3W37hxQ/Pnz9eZM2cUHBysnj17KkeOHGmOAQAAAAAAAIBHwaFG2g4ZMkRNmzbVrVu35OHhoT59+mjDhg2W9ZcuXdKTTz6phQsXKmfOnPriiy9Uo0YN3bp1K00xAAAAAAAAAPCoOMxI24ULF+qTTz7R1q1bVa1aNUnS0KFDdfnyZUvMu+++Kw8PD61du1bu7u569dVXFRwcrI8//lijRo1KdQwAAAAAAAAAPCoOM9L2888/V/PmzS0FW0lyc3NTYGCg5fXSpUvVuXNnubu7S5J8fX3Vpk0bLV26NE0xAAAAAAAAAPCoOEzRdteuXapTp45WrlypN954Q5MmTdLRo0ct62NjYxUaGqqiRYtabVesWDEdP3481THJiY2NVUREhNUCAAAAAAAAAOnhEEVbY4wiIyM1b948TZo0SXny5NGBAwdUsWJF/fTTT5Kk27dvS5K8vb2ttvXx8VF0dHSqY5IzceJE+fr6WpagoKAMOzYAAAAAAAAAjxeHmNPWyclJXl5ecnZ21tq1a+Xi4iJJyp49u4YPH66nn35anp6ecnJy0s2bN622vXHjhnx8fCQpVTHJGTlypIYOHWp5HRERQeEWAAAAAAAAQLo4xEhbSSpXrpwqVKhgKdhKUuXKlXXmzBkZY+Tm5qbg4GAdPnzYarvDhw+rTJkykpSqmOS4u7vLx8fHagEAAAAAAACA9HCYom337t21ZcsWRUVFWdrWrFmjSpUqycnJSZLUtWtXff/997p+/bok6cyZM1q+fLm6du1q2SY1MQAAAAAAAADwqDjE9AiSNHDgQK1du1bly5dXgwYNdPDgQYWGhmr58uWWmDfffFNr165VlSpVVKtWLf3222+qX7++nn/++TTFAAAAAAAAAMCj4jBFWzc3Ny1dulSbN2/W0aNH1bFjRzVo0ECenp6WGE9PT23evFlr167V2bNn1b9/f9WvX98yEje1MQAAAAAAAADwqDhM0faeunXrqm7duimud3FxUbNmze7bR2piAAAAAAAAAOBRcJg5bQEAAAAAAADAEVC0BQAAAAAAAAA74nDTIwAAAAAAANsqMmKFrVMAgCyNkbYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2xNXWCQAAAAAAAODxUmTEikzb1+kPWmXavoCMwkhbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI44TNHWGKM7d+5YLQkJCbZOCwAAAA7s/Pnztk4BAAAADshhirbvvfee3Nzc5OHhYVny58+fJG7u3LkqVqyYXF1dVapUKS1ZsiRdMQAAAEDVqlUVEhKib775Rrdv37Z1OgAAAHAQDlO0laTq1atbjbS9evWq1fqVK1dqwIABevfdd3Xjxg29+OKL6ty5s3bs2JGmGAAAAECSlixZoieeeEIvvviiAgICNGjQIO3cudPWaQEAACCLc6ii7YNMmTJFbdu2Vffu3eXt7a1XX31VTz75pD766KM0xQAAAACS9NRTT+mLL77QhQsX9PHHH+vw4cN66qmnVKFCBU2bNi3JIAIAAAAgNRyqaLt3715lz55duXLlUrNmzbR3717LOmOMtm3bpvr161tt06hRI/3xxx+pjgEAAAD+LUeOHOrZs6c2bNigefPm6dSpUxoyZIgCAwPVt29fXbhwwdYpAgAAIAtxmKJtgQIFNH/+fF2+fFn79+9Xvnz5VK9ePZ05c0aSFBkZqejoaOXNm9dqu3z58unSpUupjklObGysIiIirBYAAAA8Pi5duqRJkyapdOnS6tevn1q1aqU1a9Zo6dKlOn36tBo1amTrFAEAAJCFuNo6gYzSt29fy7+9vb01Z84cFS1aVLNmzdKECRNS3C4xMVFOTk737ftBMRMnTtS4cePSnjQAAACytOXLl2vmzJlauXKlSpQoof79+6tnz57KkyePJaZx48bKnTu3rl27Jj8/PxtmCwAAgKzCYUba/pubm5tKlCihEydOSLpbyPX09NTly5et4q5cuSJ/f/9UxyRn5MiRCg8PtyyhoaEZfDQAAACwR6+99ppy586tDRs26ODBgxo6dKhVwVaSXF1dNX78eHl4eNgoSwAAAGQ1Dlu0jYuL05EjRxQYGChJcnJyUq1atfTbb79Zxa1bt061atVKdUxy3N3d5ePjY7UAAADA8R04cEBz585V7dq17xs3ePBgeXp6ZlJWAAAAyOocpmjboUMHrV+/Xjdu3NDx48fVo0cPRUREaMCAAZaY119/XStWrNCcOXN05coVTZo0Sfv27dOQIUPSFAMAAABIUvbs2bV161YZY6za9+zZo+joaBtlBQAAgKzOYYq2b775pqZMmaLg4GCFhIQoISFB27dvV4kSJSwxTZs21VdffaVJkyapSJEi+uabb7RkyRJVrlw5TTEAAACAJE2fPl3z589P8vyD7du3a9iwYTbKCgAAAFmdk/n3sAA8tIiICPn6+io8PJypEmD3ioxYYesUkIFOf9Aq0/fJNeQ4bHH9AOlhT/daVapU0axZs5J8wB8TE6OAgABdvXpVLi4uqeorISFBy5cv1549e9SpUyeVK1fOav2GDRu0YcMGq7YcOXJo+PDhVm3R0dH66aefdPbsWZUoUULt2rWTq2vqnz9sT+cXQNbFPSLsCfe5sCepvddymJG2AAAAQGYLDw9PMjWCdPdZCbdu3Ur1FAkrV67UE088oZkzZ2rcuHE6cOBAkpgNGzZozpw59+3n+vXrqlq1qiZNmqSLFy/qzTffVEhIiGJjY1N3QAAAALALqf/IHQAAAICVmjVrasqUKZo/f76cnf9vPMTkyZNVokQJeXt7p6qfgIAAbd68WQEBAXJzc0sxrlChQho7dmyK6ydMmKD4+Hjt2LFDnp6eGjlypEqWLKkvvvhCr7zySqqPCwAAALZF0RYAAABIpzFjxqh69eoqV66cGjVqJHd3d23btk07duzQsmXLUt3PvekV7ty5c9+4K1euaPLkycqRI4eeeuopVa1a1Wr9okWL1LNnT3l6ekqS/P391apVK/34448UbQEwZQEAZCFMjwAAAACkU/HixbVnzx41adJEO3fu1MaNGxUcHKxt27apWbNmGb4/V1dXhYaGatu2bapbt64GDBhgWRcdHa3Q0FAFBwdbbRMcHKyjR4+m2GdsbKwiIiKsFgAAANiWzUfa9ujRQ1OnTlWePHnStA4AAACwB4UKFdJHH330yPfTrVs3jR492jINw4svvqg6deqoadOm6tSpk6KioiRJvr6+VtvlzJlTt27dSrHfiRMnaty4cY8ucQAAAKSZzUfa/vbbb4qJiUl23bp16xQfH5/JGQEAAABpExcXp0uXLunixYtWS2JiYobto2TJklbz5taoUUMVK1bUb7/9JkmWKRHCw8Ottrt586a8vLxS7HfkyJEKDw+3LKGhoRmWMwAAANLHZiNtz507pzt37ighIcHy73uMMTp69Khu3LihvHnz2ipFAAAA4L5Onz6t3r17a/PmzckWaC9cuCB/f/9Htn9jjKKjoyVJOXLkUFBQkI4fP24Vc/z4cZUsWTLFPtzd3eXu7v7IcgQAAEDa2axoW6NGDYWFhUm6+9Tdf3NxcdE777wjV1ebz+AAAAAAJGvIkCFydXXVmjVrlC9fviTrM3Karx07dqhatWqW19u2bdP+/fs1ePBgS1vHjh31/fffa9SoUcqRI4cuXryoFStWaMKECRmWBwAAAB49m1VEf//9d925c0d169bVwoULrUYguLi4yN/fX9mzZ7dVegAAAMAD7d+/X8uXL1fp0qUfqp8TJ07o66+/tozW/fHHH3XkyBFVrVpVrVu3liSNGzdOt2/fVqVKlXTlyhUtXLhQPXr00LPPPmvp5+2339avv/6qWrVqqX79+lqxYoUqVqyoF1544aHyAwAAQOayWdG2SJEikqSjR4/K29tbTk5OtkoFAAAASJdChQrd9yFfaeXs7KwxY8Yku2758uX6448/9Oeff6pkyZIaNmyYKlasaBXj5+ennTt3avHixTp79qw+/PBDtWvXjm+vAQAAZDE2v3vz8fFRfHy8jh07pqtXr8oYY7W+Zs2azLEFAAAAuzRkyBC98cYbmj9/voKCgtLdT/HixTV27NgHxtWqVUu1atW6b0yOHDn03HPPpTsXAAAA2J7Ni7Z79uxR+/btdfbs2WTXh4aGqmDBgpmcFQAAAPBgb7zxho4fP65ChQrJy8tLLi4uVuuPHTuW7Fy3AAAAwP3YvGg7YsQINWjQQBMnTkz2hpavcgEAAMBeTZkyRTExMSmu9/X1zcRsAAAA4ChsXhE9c+aMPvroIxUoUMDWqQAAAABpcu8hYQAAAEBGcrZ1AiVLlkxxagQAAAAgKzhz5ozWrVunuLg4W6cCAAAAB2Dzou2gQYP02muvafXq1QoLC9PFixetlsTERFunCAAAACQrKipK7dq1U5EiRdS4cWNdv35dktSnTx8tWLDAxtkBAAAgq7L59AjPP/+8wsLC1KxZs2TX8yAyAAAA2Ktx48bp1q1bOnv2rKpVq2ZpHz58uDp16qRu3brZMDsAAABkVTYv2q5bt07x8fEprvf398/EbAAAAIDU+/XXX/Xll18qKChITk5OlvZSpUrp9OnTioiIkI+Pjw0zBAAAQFZk86JtyZIlbZ0CAAAAkC6RkZHKkSOHJFkVbW/fvq3Y2Fi5utr8dhsAAABZkM3vIo8ePXrfkbalSpXiZhcAAAB2qUaNGlqwYIHGjh1rKdoaYzRmzBhVqlTJUtAFAAAA0sLm1dCQkBCFhYWluJ45bQEAAGCvxo8fr5o1a2rHjh2KiIjQO++8o507d+rQoUP69ddfbZ0eAAAAsiibF2337NmjhIQEq7YzZ85o5MiRCgkJUYECBWyUGQAAAHB/xYsX1549ezR16lTduHFDu3fvVoUKFTR37lxVqlTJ1ukBAAAgi7J50TZv3rxJ2vz9/bVw4UKVK1dOb731lg2yAgAAAFKnYMGCmjJliq3TAAAAgAOxedE2JX5+frp9+7Zu3rypnDlz2jodAAAAIImEhAQZY1Jcz7MZAAAAkB42v4tM7kY3MjJSM2fOlIuLi3x9fW2UGQAAAHB/gYGBunTpUorrL1y4IH9//0zMCAAAAI7A5kXbwoULJ/sgMh8fH82aNcvyFF4AAADA3ixevFhxcXFWbadPn9b777+vLl26KHfu3DbKDAAAAFmZzYu2CxcuVGxsrFWbj4+PSpQoIS8vLxtlBQAAADxYrVq1km2vXbu2WrdurXfffTeTMwIAAIAjsHnRtmbNmrZOAQAAAMhQwcHBunDhgqKiouTp6WnrdAAAAJDF2Lxoe09YWJgOHjyo2NhYlShRQiVLlrR1SgAAAEC6LF68WImJicqRI4etUwEAAEAWZPOibVxcnF599VXNmjVLCQkJcnJykjFGTZs21ZdffqmAgABbpwgAAAAkq2bNmrpy5YpVW2RkpK5evarJkyfzfAYAAACki82Ltu+9955++eUXLV26VA0aNJCbm5v27dunoUOHqk+fPlq1alWa+zx//ry+/fZblSxZUm3atLFaFx8fr19++UVnzpxRcHCwmjZtKmdn5zTHAAAAAIMGDVJ0dLRVm4+Pj6pWraoSJUrYKCsAAABkdTYv2i5ZskQzZ85U06ZNLW3VqlXT0qVL5e/vn+Z5wBITE9W9e3ft2rVLzZo1syraRkZGqlGjRgoPD1fdunU1adIklSpVSsuXL1e2bNlSHQMAAABIUs+ePW2dAgAAAByQzYu20dHRypcvX5J2X19fubm5KSYmJk1F2wkTJsjPz0/169dPsm7ixIm6dOmS9u/fr5w5cyosLExlypTRF198oZdffjnVMQAAAIAkJSQkyBiTqlhXV5vfegMAACCLsPl3/qtXr64JEyZYfa0sMTFR7777rgoUKCA/P79U97V582bNmTNHM2fOTHb9woUL1bVrV+XMmVOSFBgYqNatW+uHH35IUwwAAAAg3b1XdHNzS9Vy8eJFW6cLAACALMLmH/e/++67atiwoYKCglS1alVly5ZN+/fv140bN7Ro0aJU93P9+nU999xzmj17tnLnzp1kfVxcnE6ePKmSJUtatZcqVUqrV69OdUxyYmNjFRsba3kdERGR6rwBAACQdU2aNEmDBg3Siy++qDp16ihbtmzau3evpk2bpr59+6ply5aW2LQMRgAAAMDjzeZF2yJFiujw4cP68ssvtW/fPsXGxurFF19Unz59kp02ISV9+vRRp06d1Lhx42TXR0VFyRhjGUF7T86cORUZGZnqmORMnDhR48aNS3WuAAAAcAyLFy/W9OnT1aNHD0tb8+bN1bJlS7Vr107vv/++DbMDAABAVmXzoq0keXh4aODAgenefs2aNVq1apVGjx6tyZMnS5JOnTolV1dXTZ48WQMGDFCOHDkkJR0FGx4ebpkzNzUxyRk5cqSGDh1qeR0REaGgoKB0Hw8AAACyhr/++ksffvhhkvby5cvr8uXLunXrlry8vGyQGQAAALIym81pGxsbqxEjRighISHZ9R9//LGOHz+eqr4KFCigV155RTdu3NDFixd18eJFy5QFFy9eVEJCgtzd3VW4cGGdPHnSatuTJ08qODhYklIVkxx3d3f5+PhYLQAAAHB8+fLl07x585K0//jjj3Jzc1P27NltkBUAAACyOpuNtJ01a5ZiYmLk4uKS7Pq8efNqwoQJyd4E/1vZsmUtI2zvOXLkiDw8PKza27Vrp4ULF+qdd96Rh4eHrl+/rmXLlumNN95IUwwAAAAg3X0+Q8uWLbVu3TrVrl1bbm5u2rdvn3799Vd99tlnKd7rAgAAAPdjs5G2y5YtU6tWrVJc37p1ay1btixD9/n2228rMTFR9evX11tvvaX69eurcOHCevnll9MUAwAAAEhSo0aNdOjQIVWrVk07d+7Upk2blD9/fv35558PNf0XAAAAHm82G2l76tQpFS1aNMX13t7ecnFx0fXr15U7d+4099+hQwe5ulofXt68ebVnzx59++23Onv2rIYNG6ZnnnlGHh4eaYoBAAAA7ilWrJg+/vhjW6cBAAAAB2Kzoq27u7uuX7+e4vq4uDjdunUr3cXSvn37Jtvu7e2tF1544b7bpiYGAAAAuOfMmTM6ceKE6tatq2zZstk6HQAAAGRxNpse4amnntLXX3+d4vqFCxcqODhYOXLkyMSsAAAAgNSLiopSu3btVKRIETVu3NgyKKFPnz5asGCBjbMDAABAVmWzou2rr76qGTNmaPjw4bpx44alPT4+Xp9//rkGDRqk119/3VbpAQAAAA80btw43bp1S2fPnlX+/Pkt7cOHD9e7775rw8wAAACQldlseoSKFStq7ty56tevnyZPnqwCBQrI3d1dZ8+eVUJCgt5880316tXLVukBAAAAD/Trr7/qyy+/VFBQkJycnCztpUqV0unTpxURESEfHx8bZggAAICsyGZFW0nq3r27GjZsqAULFujQoUO6c+eOihYtqo4dO6pcuXK2TA0AAAB4oMjISMt0Xv8s2t6+fVuxsbFJHowLAP9UZMQKW6cAALBTNr+LDAgI0NChQ22dRpbGG71jOf1BK1unAAAAUqlGjRpasGCBxo4daynaGmM0ZswYVapUieczAAAAIF1sXrQFAAAAsqrx48erZs2a2rFjhyIiIvTOO+9o586dOnTokH799VdbpwcAAIAsiqItAAAAkE7FixfXnj17NHXqVN24cUO7d+9WhQoVNHfuXFWqVMnW6QEAAGX+N5T5Bi0yAkVbAAAAIJ0+/fRT9enTR1OmTLF1KgAAAHAgzrZOAAAAAMiqJk2apBs3btg6DQAAADgYirYAAABAOlWsWFFbt261dRoAAABwMEyPAAAAAKRTp06dNGjQIJ08eVKVKlWSh4eH1fpatWopW7ZsNsoOAAAAWRVFWwAAACCdhg8frmvXrmnkyJHJrr9w4YL8/f0zOSsAAABkdRRtAQAAgHQKCwuTMSbF9a6u3G4DAAAg7ZjTFgAAAEijIkWK6NKlS3JxcZGrq6s+/PBDxcTEyNXV1WoBAAAA0oOiLQAAAJBGMTExViNsP/nkE926dcuGGQEAAMCRULQFAAAAAAAAADtC0RYAAAAAAAAA7AgTbQEAAADpMG3aNHl5eUmSoqKirF7fM2TIEHl6etoiPQAAAGRhFG0BAACANCpQoIC+/vpry2tfX1+r1/e88MILFG0BAACQZhRtAQAAgDTavXu3rVMAAACAA2NOWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAACwAydOnNAbb7yhxo0ba8OGDcnGHD58WIMGDVKrVq00ZMgQnTt3Ll0xAAAAsG8UbQEAAAAbmzFjhlq0aKHcuXNr3bp1unjxYpKYw4cPq3r16oqNjVXv3r114sQJVa9eXZcvX05TDAAAAOwfRVsAAADAxjp27Khjx47pjTfeSDFm3LhxKl++vObMmaPOnTtr0aJFcnZ21tSpU9MUAwAAAPtH0RYAAACwsTx58sjJyem+MatXr1a7du0sr7Nly6ZWrVpp9erVaYoBAACA/aNoCwAAANi5iIgI3bhxQwULFrRqL1iwoE6fPp3qmOTExsYqIiLCagEAAIBtUbQFAAAA7FxcXJwkKXv27FbtOXLksKxLTUxyJk6cKF9fX8sSFBSUkakDAAAgHSjaAgAAAHbOx8dHzs7Oun79ulX7tWvXlCtXrlTHJGfkyJEKDw+3LKGhoRl/AAAAAEgThynaRkdHa9KkSapTp45KlSqltm3bav369Uni1qxZo0aNGumJJ55Q8+bN9eeff6YrBgAAAMgs2bJlU5kyZbR7926r9t27d6tSpUqpjkmOu7u7fHx8rBYAAADYlsMUbd9++21FR0drypQp+umnn1S5cmU1a9ZMW7ZsscRs3bpVrVq1UuPGjbVkyRKVLVtWjRo10tGjR9MUAwAAAGS23r1767vvvtPff/8tSfrzzz+1bt069e7dO00xAAAAsH9Oxhhj6yQyQmJiopydrWvQhQoV0vPPP6/Ro0dLktq2bau4uDitWrXKElOuXDnVqlVLX3zxRapjHiQiIkK+vr4KDw/PlJEKRUaseOT7QOY5/UGrTN0f149jyezrR+IaciS2uH6A9Mjse63MsHPnTo0YMULGGK1fv17lypVT/vz51aZNG7322muSpDt37qh379766aefVKpUKR06dEivvPKK/vOf/1j6SU3Mgzji+QXsFfdRgGPivhr3k9p7LddMzOmR+nfBdt26dbp06ZLq1atnadu4caOlgHtP8+bNtXz58jTFAAAAABmpaNGiGjFihKS7c8zeU7BgQcu/XV1d9fXXX+v06dM6e/asgoODFRAQYNVPamIAAABg/xymaCtJR48eVUhIiKKiohQTE6M5c+aoQYMGkqTIyEhFRETI39/fapv8+fMrLCws1THJiY2NVWxsrOV1REREBh0RAAAAHgd+fn5q3LhxqmKLFCmiIkWKPHQMAAAA7JfDzGkrSU888YS2bdumzZs364033lD//v21efNmSXenT5Dujj74Jzc3NyUkJKQ6JjkTJ06Ur6+vZQkKCsqwYwIAAAAAAADweHGooq2rq6sKFiyocuXKafz48apXr56mTJkiSfL29pa7u7uuXr1qtc3Vq1eVN2/eVMckZ+TIkQoPD7csoaGhGXxkAAAAAAAAAB4XDlW0/bccOXIoOjpa0t05b6tWraotW7ZYxWzevFlPPfVUqmOS4+7uLh8fH6sFAAAAAAAAANLDYYq2r732mk6fPi1JMsZo4cKFWr58uTp37myJeemll/TTTz9p/fr1kqSFCxfq999/14svvpimGAAAAAAAAAB4VBzmQWS1atVSq1atFBYWptjYWAUGBurjjz9W//79LTHdunXT6dOn1b59eyUmJsrd3V3Tp09Xw4YN0xQDAAAAAAAAAI+KwxRtu3btqq5duyoyMlLZsmWTu7t7snEjR47UG2+8oRs3bih37txycXFJVwwAAAAAAAAAPAoOU7S9x9vb+4Exrq6u932wWGpjAAAAAAAAACCjOcyctgAAAAAAAADgCCjaAgAAAAAAAIAdcbjpEQAAAAAAAABbKTJiRabt6/QHrTJtX8hcjLQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwI662TgAAAAAAgJQUGbEiU/d3+oNWmbo/AACSw0hbAAAAAAAAALAjFG0BAAAAAAAAwI4wPQIAAAAAAP9fZk/HAABAchhpCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEccak7bqKgo7dmzR/Hx8Spfvrzy5MmTbNz+/ft15swZBQcHq1SpUumOAQAAAAAAAICM5jAjbcePH68SJUpoxIgRGjt2rAoXLqxJkyZZxcTFxal9+/Zq2LChpk2bpqeeekr9+vWTMSZNMQAAAAAAAADwqDjMSNt8+fLpyJEj8vb2liQtWbJETz/9tBo0aKBq1apJkqZNm6YtW7Zo3759KliwoA4ePKiqVauqQYMG6tGjR6pjAAAAAAAAAOBRcZiRtgMHDrQUbCWpffv2ypYtm/bs2WNpmz9/vrp27aqCBQtKksqWLasWLVpo/vz5aYoBAAAAAAAAgEfFYYq2//b7778rLi5OZcuWlSTduXNHhw8fVoUKFaziKlSooP3796c6JjmxsbGKiIiwWgAAAAAAAAAgPRyyaHvjxg317dtXrVq1Uu3atSVJt27dUkJCgnLlymUV6+fnp5s3b6Y6JjkTJ06Ur6+vZQkKCsrQ4wEAAAAAAADw+HC4om1ERIRatGihnDlz6ttvv7W0u7u7S5Kio6Ot4m/duiUPD49UxyRn5MiRCg8PtyyhoaEZciwAAAAAAAAAHj8O8yAySYqMjFTz5s11584drV27Vj4+PpZ12bNnl7+/v86ePWu1zdmzZ1WsWLFUxyTH3d3dUvAFAAAAAAAAgIfhMCNtb926pebNmysuLk5r1qxRzpw5k8S0aNFCixcvVmJioiQpJiZGy5YtU4sWLdIUAwAAAAAAAACPisOMtG3Tpo3279+vqVOnat26dZb2MmXKqEyZMpKk0aNHq1q1aurYsaNatmyp7777Tq6urho6dKglPjUxAAAAAAAAAPCoOMxI24CAADVr1kyrVq3Sd999Z1kOHDhgiSlSpIh2796tUqVKacOGDapbt6527NghPz+/NMUAAAAAAAAAwKPiMCNt//nQsfsJCgrSxIkTHzoGAAAAAAAAAB4FhxlpCwAAAAAAAACOgKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAABAFvDuu+/Kw8PDailYsGCSuAULFqh8+fLy9fVVtWrVtHbtWhtkCwAAgIdB0RYAAADIAu7cuaMqVaro5s2bluXkyZNWMatWrVLPnj01ePBgHTlyRG3btlXr1q114MABG2UNAACA9KBoCwAAAGQRTk5OViNt3d3drdZPmjRJbdu2Vb9+/RQQEKB33nlHpUqV0tSpU22UMQAAANKDoi0AAACQRezZs0d58uRRoUKF1LlzZx0/ftyyLjExUVu3blXDhg2ttgkJCdGWLVsyO1UAAAA8BIq2AAAAQBaQN29eff755/rrr7+0atUq3blzRzVr1tTFixclSTdv3tTt27eVL18+q+3y5ctniUlObGysIiIirBYAAADYFkVbAAAAIAt46aWX1KtXLwUEBKhMmTL69ttv5ezsrJkzZ1rFOTs7J3ltjEmx34kTJ8rX19eyBAUFPZL8AQAAkHoUbQEAAIAsKHv27CpVqpSOHTsmSfL19ZW7u7uuXLliFXf58mXlz58/xX5Gjhyp8PBwyxIaGvpI8wYAAMCDUbQFAAAAsqD4+HidOHFC/v7+kiQXFxc99dRT2rhxo1Xcb7/9ppo1a6bYj7u7u3x8fKwWAAAA2BZFWwAAACALePbZZ7Vjxw7FxcXp/Pnz6tevn27evKl+/fpZYoYMGaLFixdr0aJFio6O1qeffqp9+/bplVdesWHmAAAASCuKtgAAAEAW0KdPHw0bNkw5c+ZUuXLldOXKFW3ZskWlSpWyxDz99NP66KOP9Nprr8nLy0sfffSRvv/+e1WtWtWGmQMAACCtXG2dAAAAAIAHa9y4sRo3bixjjJycnFKMGzRokAYNGqTExMQkDyUDAABA1sBdHAAAAJCF3K9g+08UbAEAALIuRtoCAACbKTJiha1TQAY6/UErW6cAAAAAOAQ+fgcAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO+Jq6wQAAAAAAAAApF2RESsydX+nP2iVqft7nFG0BQAAAACkSWYXCQAAeNw41PQIhw4d0quvvqo6depo9erVycb89ddf6tu3r0JCQjRw4ECdOnUqXTEAAAAAAAAA8Cg4TNF2xowZ6tSpk4oVK6YtW7bo8uXLSWIOHTqkWrVqydXVVYMHD9b169dVo0YNhYWFpSkGAAAAAAAAAB4Vhynadu3aVYcOHdLgwYNTjJkwYYIqVKigL774Qm3atNGCBQvk5eWlKVOmpCkGAAAAAAAAAB4Vhyna5syZ84Exa9euVZs2bSyvXVxc1Lp1a61ZsyZNMQAAAAAAAADwqDw2DyKLiorS1atXFRgYaNUeGBios2fPpjomObGxsYqNjbW8joiIyMDMAQAAAAAAADxOHGak7YPEx8dLkjw8PKzas2fPrri4uFTHJGfixIny9fW1LEFBQRmZOgAAAAAAAIDHyGNTtPX29pabm5uuXbtm1X7t2jXlzp071THJGTlypMLDwy1LaGhoxh8AAAAAAAAAgMfCYzM9gouLi8qXL69du3ZZtW/fvl2VK1dOdUxy3N3d5e7unvFJAwAAAAAAAHjsPDYjbSWpb9++WrhwoY4cOSJJ2rp1q9atW6e+ffumKQYAAAAAAAAAHhWHGWm7c+dODR482PJ6woQJ+t///qc2bdrozTfflCQNGjRI+/fvV6VKlfTEE0/o5MmTev3119WhQwfLdqmJAQAAAAAAAIBHxWGKtsHBwfrggw+StPv7+1v+7ezsrBkzZmjcuHEKDQ1V0aJFlSdPHqv41MQAAAAAAAAAwKPiMEVbX19f1alTJ1Wx/v7+VsXc9MYAAAAAAAAAQEZ7rOa0BQAAAAAAAAB7R9EWAAAAAAAAAOwIRVsAAAAAAAAAsCMOM6ctAAAAADyuioxYYesUAABABmKkLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB1xtXUCAAAAAOCIioxYYesUAABAFsVIWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADviausEAAAAAAAAANi/IiNWZNq+Tn/QKtP2ZY8YaQsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHmNMWAAAAwGMjM+fiAwAASC9G2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgR5rQFAAAAAAAAYFcyex760x+0ytT9PQgjbQEAAAAAAADAjjDSNgXXr19XWFiYChcuLB8fH1unAwAAAKTavXvZIkWKyNvb29bp3Fdmj6IBAADIChhp+y/GGL322msKCAhQ+/btlS9fPk2YMMHWaQEAAAAPZIzRyy+/bHUvO3HiRFunBQAAgDRipO2/zJw5U3PnztXOnTtVvnx5bdy4UY0bN1bFihXVtm1bW6cHAAAApGj69OmaP3++du/erbJly2r9+vVq2rSpKlasqJYtW9o6PQAAAKQSI23/ZebMmerUqZPKly8vSapfv74aNWqkWbNm2TgzAAAA4P5mzpyprl27qmzZspKkRo0aqX79+tzLAgAAZDEUbf8hISFB+/btU7Vq1azaq1evrt27d9soKwAAAODB4uPj9ddff3EvCwAA4ACYHuEfIiMjFR8fLz8/P6v2PHny6Nq1ayluFxsbq9jYWMvr8PBwSVJERMSjSfRfEmOjM2U/yByZdd3cw/XjWDL7+pG4hhwJ1w8eVmZdQ/f2Y4zJlP1lFeHh4UpISOBeFgAAIB3s7V6Wou0/uLm5SZLVTask3b5927IuORMnTtS4ceOStAcFBWVsgngs+E6zdQbIyrh+8DC4fvCwMvsaioyMlK+vb+bu1I5xLwsAAJB+9nYvS9H2Hzw9PZU7d26dP3/eqv38+fMqVKhQituNHDlSQ4cOtbxOTEzU9evX5efnJycnp0eW7+MkIiJCQUFBCg0NlY+Pj63TQRbD9YOHxTWEh8H1k/GMMYqMjFSBAgVsnYpd8fX1la+vb5a6l+XnI304b2nHOUsfzlv6cN7Sh/OWdpyz9LH1eUvtvSxF238JCQnRihUr9Oabb0q6e9O6cuXK+z5t193dXe7u7lZtOXPmfJRpPrZ8fHz4RYR04/rBw+IawsPg+slYjLBNXqNGjbRixQoNGzZM0t1nNqxcuVIdOnRIcRt7uJfl5yN9OG9pxzlLH85b+nDe0ofzlnacs/Sx5XlLzb0sDyL7l7fffls7d+7Uq6++qjVr1qhXr166fv265cYXAAAAsFejR4/W1q1bNWTIEK1Zs0Y9evRQZGSk1UhaAAAA2D+Ktv9SoUIFbd68WVeuXNHYsWPl4uKiP/74475fKQMAAADsQaVKlbRp0yZduHBBY8aMkbu7u/744w8FBgbaOjUAAACkAdMjJKNKlSpasGCBrdPAP7i7u1v+8ADSiusHD4trCA+D6weZrVq1avruu+9snUaq8PORPpy3tOOcpQ/nLX04b+nDeUs7zln6ZJXz5mSMMbZOAgAAAAAAAABwF9MjAAAAAAAAAIAdoWgLAAAAAAAAAHaEOW0BAAAAPHKnTp3SjRs3VKVKlTRtExcXpxIlSsjZOfnxJqmJyari4uJ0/Phx5cyZ84EPkwsLC9OFCxeSXVelShU5OTkpJiZGBw4cSLK+ZMmS8vb2zpCc7cH169cVFhamwoULy8fH576xFy5cUFhYmFWbq6urKlWq9FD9ZkWnT5/W7du3VaJECbm4uDww/s6dOzp58qRy586tvHnzJll/6NAhRUdHW7XlzZtXhQsXzrCcM0t8fLyOHz8ub29vBQUFZdg26ek3K7l586ZCQ0MVFBSknDlzpmqb8+fP6/bt2ypcuLBcXa1LVpcvX9bZs2et2pycnNL0vpIVhIaGKjIyUsHBwXJzc7tv7NGjRxUZGWnVljt3bhUrVuyh+s1qEhISdOzYMWXPnl1FihS5b+yVK1d05syZZNeVL19e7u7uSkhI0J49e5KsL1asmHLnzp0RKaeOAQAAAIBHZPHixaZ+/frG29vbuLi4pGqbv//+21SuXNn4+vqa/Pnzm8KFC5s///wzzTFZ2eLFi03u3LlNUFCQyZ49u2nWrJkJDw9PMf6TTz4xVapUsVpy5sxpfH19zZ07d4wxxvz1119GkqlYsaJV3M6dOzPrsB6pxMRE8+qrr5ps2bKZYsWKGXd3dzN+/Pj7bjNhwgTj6elpdT4aNGjw0P1mJefPnzfVq1c33t7eJiAgwAQGBppNmzalGH/z5k0zePBg4+vra0qUKGF8fX1NzZo1zdGjR63iKlasaAoWLGh1bidMmPCoDyfD/fLLLyZv3rymYMGCxtPT0zRo0MBcu3btobdJT79ZyYgRI4y7u7vlZ2bEiBH3jf/mm29MyZIlTUBAgClatKjx8/Mzc+bMsYqZOnWq8fDwsLqmqlev/igPI1NdvXrVNGjQwHh6epqCBQuavHnzml9++eW+29SvX98EBARYnZPhw4c/dL9ZyaZNm0xgYKAJCAgw3t7epnr16ub8+fMpxn///fdJ3i/z5ctnXFxczNWrV40xxty4ccNIMqVLl7aKW7lyZWYdljHGGIq2yPLCwsLMrl27TEJCgq1TQRY0Y8YMc/bsWVungSxq9+7dZtCgQbZOA1nYunXrzNy5c01iYqKtUwEemREjRpj169ebuXPnprpoW7NmTdOkSRMTFxdnEhMTTf/+/U1gYKCJjo5OU0xWdebMGePh4WGmTp1qjDHm2rVrpkSJEqZPnz6p7uPOnTsmICDAvPjii5a2e0XbCxcuZHTKdmHGjBnG29vb7N+/3xhjzIYNG4yrq6tZunRpittMmDDhgUWf9PSblTRp0sTUqVPH3L592xhjzJAhQ0yePHlS/JDgr7/+MlOnTjVRUVHGGGOioqJMixYtTPny5a3iKlasaCZNmvRok3/ELl68aLy8vCzF5oiICFOhQgXTuXPnh9omPf1mJd98843Jnj272b59uzHGmD///NO4u7ubb775JsVtRo8ebVX4nzt3rnF2drb0Yczdom3ZsmUfXeI21rlzZ1OpUiUTERFhjDFm3LhxxsvLy1y6dCnFberXr2/eeuutDO83qwgPDzd58uQxr7/+ujHGmNu3b1vuD9KicuXKpm3btpbX94q2O3bsyNB804qiLbKs2NhYM3DgQOPi4mIkmVq1alluNIDUWLx4sZFkihUrRuEWabZ7926TL18+s3DhQlungiwoLi7OtGrVytSsWdPMmzfvvqPnAEeR2qLtgQMHjCSzceNGS1tYWJhxcnIyixYtSnVMVvbuu++aPHnyWEbIGmPM559/bjw8PCyFsgf5+eefjSSzZ88eS9u9ou3OnTvN4cOHTWxsbEanblNVq1ZNUthu2rSpadOmTYrbTJgwwVSrVs0cOXLEnD59OtkP0dLTb1Zx+vRpI8msWLHC0nbt2jXj6upqvvrqq1T389NPPxlJ5vr165a2ihUrmjFjxpj9+/dn2RGk06ZNM97e3iYmJsbSNm/ePOPi4pLiMaVmm/T0m5U0aNAgSQG6Q4cOpmHDhmnqx9fX1/z3v/+1vJ46daopXbq0OXr0qDl16pRDDdy6du2acXFxsSpsR0dHGy8vL/PRRx+luF39+vXNkCFDzP79+82VK1cyrN+s4ssvvzRubm7mxo0blrZ773+nT59OVR+7d+82ksyyZcssbfeKtitWrDAHDhyw2QfCjjXpEx4rffv2VVhYmE6ePKktW7Zo3759Wrt2ra3TQhbToEEDubi4qEGDBgoNDbV1Osgi9uzZo+bNm+uzzz5Tp06dbJ0OsqBPPvlEERER2rx5s3r06OGQcyMC6bV7925JUrVq1SxtBQoUUMGCBS3rUhOTle3evVtPPvmk1byi1atXV0xMjA4dOpSqPmbPnq2qVasmOzdr8+bN1bJlS3l7e2vYsGGKj4/PqNRtJiEhQfv27bO6JqS75+1B18SOHTvUsmVLVa1aVQEBAfrhhx8ypN+sILmfpdy5cys4ODhNx7d3717lzJkzybylEydOVOfOnRUYGKi6devq2LFjGZJ3Ztm9e7dljst7qlevroSEBO3fvz/d26Sn36xk9+7dD/0zc+rUKUVGRiaZn/Tw4cNq3ry5atSoofz582vevHkZkbLN7du3TwkJCVbnLXv27CpfvvwDz9unn36qzp07q3DhwqpWrZrVNfQw/WYFu3fvVnBwsNXvnurVq0tSsnPSJmf27NkKDAxUixYtkqzr3r272rdvL19fX/Xt2zfJ/MGPGg8iQ5a0fft27d69W3v27JG7u7sKFy6scuXKKT4+XsuWLVPNmjWVJ08eW6cJO1eqVCnduHFDGzZsUIMGDdSgQQNt2LBBLi4uioqKUnBwsK1ThB3au3evQkJCNHXqVHXq1ElRUVEaP368fvzxR7m4uKh79+566623HG5yf2SsxYsXq2fPnpaCzMWLFzV9+nSdP39ebdq0Udu2bW2cIZCy/fv3Ky4uLsX17u7uKl++fLr7v3btmrJnz67s2bNbtefJk0fXrl1LdYw9iY6OfmCxNX/+/JYHEV27dk0FChSwWn/v3jY1x3fp0iWtWLFCn332mVW7l5eXfvrpJ7Vv316StG3bNjVt2lTe3t4aO3ZsKo8m85w/f17nz5+/b8y9h6hFRkYqPj5efn5+VusfdE1UrlxZR48eVYkSJWSM0Ycffqju3burWLFiqlq1arr7taWDBw/q9u3bKa53c3NTxYoVJf3f9fTvB+uk5fj27t2rSZMmacyYMXJycrK0v/TSS+revbs8PT11/fp1denSRU8//bT27NmjbNmypfWwbOLatWvJ/t/fW5febdLTb1Zx584dRUREJHt84eHhSkhIeOCD7uLj49W3b1+VK1dOrVu3trSXKVNGBw4cUNmyZSXd/RC8d+/eKlKkiOrVq5fxB5OJ7v2/p/V3Ta9evbR06VL5+voqMjJSPXr0UNu2bXXgwAF5eXmlu9+sIrmfpXuvU3N8MTEx+vbbb/Xyyy9bXZeurq6aPXu2evfuLWdnZx06dEhNmzbVkCFDNGvWrIw9iPugaIss6ddff9Ubb7xh+WRyy5Yt2r17t1566SXdunVLrq6uWrFihWrWrGnjTGHPihcvrpMnTyp//vxWhVtXV1cNHjyYoi2SlSdPHuXOnVuzZ89WixYt1Lp1a+XMmVPDhw/XoUOH9N5772nfvn366aefbJ0q7FhMTIwuX74s6W4BrHHjxqpcubJiY2PVrl07TZgwQW+//baNswSSN3LkSF26dCnF9QUKFNDPP/+c7v7d3NwUHx8vY4xVAej27duWD8RSE2NPzpw5o4EDB943pnv37ho6dKiku8cXGxtrtf5eES41x/fVV1/J3d1d3bp1s2ovUqSI1ai1GjVqaODAgfr666/tsmi7dOlSzZ49+74xM2bMUJUqVSznJbnzdr9z1qpVK8u/nZycNGLECM2dO1ffffedqlatmu5+bWnMmDE6ffp0iutz5sxp+YbivWOIi4uz+hAktcd3/PhxtWzZUh07dtQbb7xhta5///6Wf+fOnVv//e9/VbFiRe3evVs1atRIyyHZjJubm6Kjo63aHvSzmJpt0tNvVuHi4iInJ6dkf2acnZ3l7Hz/L3wnJCSoR48eOnnypDZt2mR1Ppo2bWoV+8orr2ju3Ln69ttvs3zR9n6/a7y9vVPcrk+fPpZ/e3t76+OPP1bhwoX1+++/q3nz5unuN6tI7v0yJibGsu5BFi9erPDwcPXt29eq3cvLy6qtTJkyGj58uEaOHKkvvvjigddxRqFoiyzp1VdfVUJCgiTp0KFD6tChgz755BMNGDBAkZGRat26tQYNGqS9e/faNlHYNTc3NxUoUEDHjx9XqVKl9M033+ipp55Snjx5rD7RBf6pYMGCliJ/yZIl1bx5cy1YsMCyvl69eurUqZNWrlypli1b2jBT2LOaNWvqiy++0JAhQzRgwAB98MEHlhvDd999V+PGjdOAAQOUL18+G2cKJLVixYpH2n+hQoV0584dXb58Wfnz55ckGWN08eJFFSpUKNUx9qR06dLauXNnquMLFSqUZGTuvRGnqTm+OXPm6JlnnknVH+QFChTQuXPnUp1bZho0aJAGDRqUqlhPT0/lzp07ycjc8+fPp/maCAgIsJyTjOw3s/z444+pjr13DOfPn9cTTzxhab9w4YJVQTs5J06cUMOGDdWwYUPNnTvX6gOU5NwbPW6v11tyChUqpPXr11u1PehnMTXbpKffrMLJyUlBQUHJ/swEBQXd9zpJSEhQz5499fvvv2vDhg0qWrToA/dnz7/D0uKfP4sBAQGW9vPnz6tJkyap7icgIEBOTk6Wc5JR/dqrQoUKJZkmMy0/S7Nnz1aTJk2STMORnAIFCig6OlrXr1/PtG92M6ctsiRfX1/LV3iKFCmiBQsW6IUXXpCTk5N8fHw0YsSIVM/3hcdbqVKldPDgQZ0/f17PPfec3nrrLeXKlYs5bnFf9wq3fn5+euutt6zWdezYUcWLF0/TH+d4/AwZMkRXr17Viy++qJMnT1p9kv/KK6/ozp07D/xKMOBI/vrrL8v7bp06dZQtWzar4vDmzZt18+ZNhYSEpDomKwsJCdHOnTstI/IladmyZSpcuLClsHb79m3t3Lkzyfx6v//+u44ePWo1yvGee6OP/mndunUqXbp0Bh+BbYSEhFhdE4mJiVq5cqXVNXH+/HmrgR3/PifXr1/Xvn37rM5JavrNqqpXry5PT0+r49u9e7fCwsKsju/gwYM6c+aM5fXJkyfVsGFD1atXT/PmzUvydfeUrjVJWep6CwkJSXLsy5YtU968eS3TwMTGxmrnzp0KDw9P9TapicnKQkJCtHLlSstrY4yWL19udU1dunRJu3btsrxOTExUr169tHHjRv32228qXrx4kn7/fV1FRkZqx44dWeqaSkmFChWUN29eq5/FU6dO6dChQ1bn7ejRozp16pSku9eeMcaqn/Xr18sYYzknqe03qwoJCdG5c+es5vFdtmyZvLy89NRTT0m6O2XHzp07df36datt//77b/32229per/Mly9fkukYHimbPP4MSIM7d+6YmTNnmi5dupjBgwebCxcuPHCbadOmmdq1a2dCdsgKtmzZYvr27Wuee+45s2nTJqt1w4cPNwMGDDAlSpQwkyZNMsbcffp0cHCweeWVV2yRLuzMxYsXzahRo0zHjh3NRx99ZPWU2tDQUBMfH28Vn5iYaAoUKGC+++67zE4Vduh+72HffPONcXJyMi4uLlZPt127dq3Jmzev1ROlgazs1KlTZseOHWbMmDHGxcXF7Nixw+zYscNERkZaYgoXLmxee+01y+u33nrL5MqVy8ybN88sXrzYFCtWzHTq1Mmq39TEZFVxcXGmUqVKpnbt2mblypVm6tSpxtXV1cyfP98Ss2fPHiPJ/Pbbb1bb9urVy5QvXz7ZfocNG2ZeffVV8/PPP5tff/3V9OnTx7i6upoVK1Y8ysPJNPv27TPZs2c3r7zyilm9erV57rnnTO7cuc2ZM2csMWPGjDG+vr6W19WrVzdTpkwx69evNz/88IN58sknTeHChc3ly5fT1G9WNnHiROPt7W1mz55tli5dakqXLm2aN29uFVO2bFnTr18/Y8zde+VChQqZqlWrmm3btll+pnfs2GFu375tjDFm27Ztpn79+mbevHlm48aNZurUqSZnzpymZ8+emX58DyMhIcHUqlXLVK1a1Sxfvtx89tlnxt3d3fzvf/+zxBw/ftzqyfOp2SY1MVnZsWPHjLe3t+nfv79ZvXq1ef755423t7c5duyYJWbSpEnGxcXF8rpv377Gw8PDLFiwwOqaCg0NtcTUr1/ffPjhh2bdunVm0aJFpkaNGiYgIMCcO3cuU4/vUZk+fbrx8PAwn3/+uVm+fLl58sknTa1ataz+/qhdu7bp2LGjMcaYw4cPmxo1api5c+eajRs3ms8++8zkzZvXtG3bNs39ZmXNmzc3ZcuWNUuXLjWzZ882np6e5oMPPrCsv3LlipFk9R5qzN37iHz58pm4uLgkfU6aNMn07t3bLFq0yKxbt84MGTLEuLi4mFmzZj3y4/knirawazExMaZJkyamXLlypl+/fiZfvnymZMmSlpuB5Bw8eNDkyZMnSXEOj6ePPvrI+Pn5md69e5uqVasaFxcXs3XrVsv6RYsWGUmWgu09ly5dSvaXNx4vf/31l/H39zdt2rQxnTp1Ms7Ozmbo0KH33eaDDz4w5cqVM7GxsZmUJexVat7DvvvuO+Pp6WlKlChhvv/+ezNz5kwTEBBgFi9ebMPMgYz1zjvvmCpVqiRZ9u7da4lp06aNmTJliuV1YmKimT59umncuLGpV6+eeffdd5Pc/6UmJiu7du2aGTp0qKldu7Zp1apVkt8LR48eNVWqVDE7d+60tMXFxZl69eqZL7/8Mtk+4+PjzcyZM027du1Mo0aNzKBBg8zhw4cf6XFktp07d5pnnnnG1KpVy/Tq1cscOXLEav2MGTNMgwYNLK8vX75sRo0aZUJCQkybNm3MhAkTTERERJr7zermzJljmjZtaurWrWtGjx5toqKirNZ37drVTJgwwRhjzObNm5P9ma5SpYo5ceKEZZs///zT9O3b19StW9c888wz5vvvv8/UY8oo4eHh5s033zR16tQxLVq0MAsWLLBaHxoaaqpUqWI2b96c6m1SG5OV7d+/3zz77LOmZs2a5tlnnzX79++3Wj9//nxTvXp1y+uQkJBkr6lp06ZZYq5fv25Gjx5tGjdubFq1amVGjx5tbty4kVmHlCkWLFhgWrRoYerUqWPefPNNEx4ebrW+X79+Zvjw4ZbX+/btMwMGDDD16tUzXbp0MV9++WWyxdgH9ZuVRUVFmdGjR5u6deuapk2bmjlz5litv3nzpqlSpYpZuXKlVXv79u3Nf/7zn2T7TExMNN99953p1KmTadCggenXr5/Zvn37IzuGlDgZ86+x1IAdGTt2rP78808tW7ZMrq6uOnLkiMqXL6/p06fr+eeft4o9ePCgfv75Z33yySeaNm2aunTpYqOsYS/279+vRo0aaceOHSpatKgSExMtc9auWrXKErdt27Ys8zAEZB5jjCpWrKgXX3zR8vCYDz/8UKNGjdLly5etvhYTHx+v9evXa+7cuTp69Kh++umnVM2LBMeW2vewU6dOadKkSfrjjz/k7++vkSNHqkGDBrZLHAAAAIDNUbSF3UpMTFShQoW0fft2y6T1ktSoUSOVL19eH330kVX8ypUrderUKXXq1En+/v6ZnS7s0MCBA1WpUiWrpzV//vnneu+99xQWFmbDzJAVbNiwQWPHjtWGDRssbdevX5efn582bdqkunXrWtqNMZo2bZoKFy6sNm3aZPmn/uLhpfU9DAAAAAD+ydXWCQApuXXrlp5++mmrP3aluw8eu337dpJ4ntKOf3N3d9czzzxj1ZbS9QP82507d5KM6M+dO7e8vb2TXENOTk4aMmRIZqYHO5fW9zAAAAAA+CdnWycApMTHx0effPJJknYPDw8lJiZaXh87dkyHDh3KzNSQRXz00UfKmTOnVdu/r5+4uDirJ5sC9zRu3FjPPfdckvZ/X0O//PKLYmNjMzM1ZAG8hwEAAAB4GBRtYVfWr1+vWbNmad++fSnGODs7W/7gPXbsmEJCQu4bj8fH8ePHNWfOHP3yyy+Kj49PNuaf109cXJw6d+6sL7/8MhOzhL2KjIzUggUL9O233+ry5cspxv3zGpo7d64GDhyoixcvZlaasFPGGK1bt06zZ8/W/v37U4zjPQwAAABAajA9AuxCbGysnn76aR04cECenp46cuSIWrVqpVmzZiWZn9bJyUnGGMsfu++99566detmo8xhLz755BO9/fbbCg4O1l9//aUCBQpozpw5atiwoVXcvevnXsE2W7Zs+vbbb22UNezFvn371LJlS+XNm1dhYWG6deuW3n77bY0cOVLOztafb967hubOnauxY8dq3bp1Kly4sI0yhz2IiYlRu3btdOTIEWXPnl1Hjx5VmzZtNGvWLOXLl88qlvcwAAAAAKnBSFvYhfHjx8vJyUknT57U4cOHtX79eu3bt081atTQyZMnrWKdnJx06tQpyx+7PXv2tFHWsBe7d+/W+PHjtW/fPu3cuVOnTp1SmTJl1LRpU3399ddWsU5OTkpMTLQUbBcsWCBXVz6/epzdux7effdd7d27V+fPn9c777yjMWPGqHv37kpISLCKd3Jy0rx58ywF2+LFi9soc9iLMWPGyMPDQydPntSRI0e0du1a7d69WzVq1NDff/9tFct7GAAAAIDUoGgLu7B8+XJ1797d8sT1hg0b6s8//1T27NnVtGlT3bhxwxLr6uqq33//nT92YbFixQrVqVNHRYoUkSQFBgZq2bJl6t27t3r37q3Vq1dbYl1dXRUdHU3BFhZHjx7V8ePH1aNHD0mSm5ubRo0apUWLFmnx4sV65ZVXrOJdXV31xx9/ULCFxb33sHu/T0JCQrRt2za5ubmpadOmCg8Pt8TyHgYAAAAgNSjawi7kypVLW7ZssWorUKCAVq9erVu3bun111+3tA8cOFDffPMNf+zCIleuXNqxY4fi4uIsbc7OzpoxY4Zat26tXr16KTo6WpJUtWpVTZo0iYItLO49rO6PP/6wam/Xrp1mzZql6dOna82aNZb2Dz74QL/99hsFW1gk9x5WsGBBrV69Wjdv3tTw4cMt7byHAQCQ8Xbs2HHfOeUBICtyMsYYWycBzJo1Sy+++KJ+//13PfXUU1brvvnmG/Xu3VtXr16Vr6+vjTKEPbtw4YKeeOIJDRkyRO+9957VuqtXr6po0aL67LPPKJIgRY0bN9bVq1e1bds2eXh4WK1r06aNnJyc9PPPP9soO9i7//3vf3rttdf0xx9/qEqVKlbrvvrqKw0YMEDXrl2Tl5eXjTIEAMA+7dy5U/Hx8apZs+ZD9dOpUyf5+/vr008/zaDMAMD2GGkLu9C3b1/VqlVLbdu21dGjR63Wde7cWQkJCTp37pyNsoO9CwgI0MSJEzVx4kTNnDnTal2ePHnUqFGjJPNKAv/02Wef6cSJE+ratavViG1J6tKlC9cP7qt///6qVq2a2rRpo+PHj1ut69Kli+Li4hQWFmaj7AAAsF+TJ0/WuHHjbJ0GANglirbIVHFxcXr77bcVFBSkwMBATZo0SdLdr7IvWrRI+fLlU926dfXbb79Zttm/f79y5cqlJ554wlZpw44sWbJETz75pHLlyqUuXbpY5jt+7bXX9Morr+iFF17Q+PHjLQ+Pio2N1ZEjR1StWjVbpg07cerUKbVv3165cuXSk08+qa1bt0qSSpYsqcWLF2v16tVq0aKFLl++bNlm3759XD+QdPf3yVtvvaWCBQsqMDBQ//3vfyVJLi4uWrJkiXLlyqW6detq48aNlm327dunPHnyqGjRorZKGwCALGPz5s06cuSIoqKitHXrVm3dulWxsbFJ4qKjo7Vu3Trt3LlTMTExKfa3f/9+rVq1KsmHqocPH9by5cv1zy8eh4eHa8mSJbp06VLGHRAAPAQmdESmSUxMVPv27RUbG6sPP/xQf/75p4YPH67ChQurS5cu8vPz08aNG9WjRw+FhISoZcuWKlq0qBYuXKgvvvgiyVeW8fiZOXOmxo4dq7ffflsJCQkaM2aMXnjhBf3www+SpI8++kiBgYF6++239f3336tp06basGGDqlevrpYtW9o4e9jaiRMnVKdOHT333HPq2rWrPv74Y7Vr106HDx+Wn5+fmjZtqnXr1umZZ55RyZIl1blzZ928eVO7d+/W77//buv0YWOJiYlq06aNjDH6z3/+oz/++EPDhg1TkSJF1KFDB+XJk0ebN2/Wc889p4YNG6pVq1YqXLiwFi5cqJkzZypbtmy2PgQAAOzeyJEjlS1bNp05c0ZFixbV8ePH5eXlpS1btlieQ3Dw4EE1bdpUOXLkUP78+XX58mXlyJFD/v7+ln4uXbqkp59+WufPn1fJkiW1f/9+1ahRQ999953c3d3l5eWlnj17avTo0Ro8eLAk6YUXXtCxY8e0bds2Gxw5ACTFnLbINN9++62mTZumP/74w/IAqBYtWig+Pl5r1661il29erWWLVsmZ2dn9e7dW5UrV7ZFyrAj169fV4kSJbRx40aVLVtWkjR79mz1799fV69eVe7cuS2xJ0+e1FdffaULFy6oYcOG6tatm5ycnGyVOuxE27ZtVbt2bb355puSpIsXL6pYsWL6+OOP9fzzz1vioqKiNG/ePO3atUvFihXToEGDlCtXLlulDTsxb948TZ8+Xb///rtcXFwkSU2aNJGLi4tWrVplFbtq1SqtWLFCLi4u6tOnjypWrGiLlAEAsHvPPPOMbt68aXkvrVOnjk6cOKHdu3erQIECun37tsqUKaOBAwda7uHq16+vvHnz6ocffpCzs7MWL16sjh076qWXXrLMadusWTMFBQVpxowZcnFxUVRUlOrWrasOHTro7bffliQtXLhQPXr00Pbt27V792699NJL2rVrl0qVKmWbkwEA/8JIW2SauXPnatKkSZaCrXT3AT+TJ09OEtu0aVM1bdo0M9ODnVu4cKG6detmKdhKsox6O336tFXR9oknntD48eNtkSbs1JUrV7R3714tWbLE0ubv76+qVavq1KlTVrGenp4aNGhQJmcIezd37lxNnjzZUrCV7v4OSu6BJ82bN1fz5s0zMz0AABxG586dVaBAAUlS9uzZVbt2bR05ckSSFBYWpk2bNmn79u1ydr4722OHDh2sCq3nzp3T6tWr9fHHH+uXX36RMUbGGJUuXVpr1661FG07d+6slStXqnPnzrpw4YKmTJlCwRaAXaFoi0zTo0cP1atXz6rN399f8fHxNsoIWUmNGjX01FNPWbXly5dPzs7OXEN4IC8vL02ePNlyc38Pv4OQWr1791atWrWs2rh+AADIeH5+flavPTw8dOvWLUnSmTNnJEnFihWzivnn80/ufSC/atUqubm5WcWVK1fO6vXEiRNVsGBBy2heALAnPIgMmaZnz55JvqLu5uZmNfn7iRMnNGrUqMxODVlAxYoVk50mw9XV1XINxcfHa8iQIVYPkQKku6M0unTpkqT937+D5s6dq19++SUzU0MW0atXrwe+hx07dkxvvfVWZqcGAMBj496368LDw63ab968afm3t7e3JOndd9/VkiVLrJZ/f0Nm1KhRKlSokA4fPqwVK1Y82uQBII0o2sKmnJyclJiYKOluwTYkJETBwcE2zgpZyb1rKD4+Xl27dtWZM2eYfxSp9s/fQXPnztXYsWP5HYRU++f1c+zYMYWEhKhkyZI2zgoAAMdVrFgx5c+fXz///LOl7fz589qxY4fldfny5eXv769Zs2Yl2f6fgzsWLVqkb7/9VkuWLNHbb7+tvn37MvgDgF1hegTYlJOTk4wxloLt2LFj1adPH1unhSzEyclJcXFx6tq1qyTp+++/T/I1KCAl934H3SvYrlu3TsWLF7d1Wsgi7l0/9wq27733nnr27GnrtAAAcFjZsmXT+PHj9eqrryoiIkKBgYGaNm2asmfPbolxdXXVzJkz1alTJ928eVPNmjXT9evXtWzZMjVv3lxvvPGGwsLCNGDAAL3//vuqUKGCypYtq9WrV6tfv35atmyZDY8QAP4PRVvYlLOzs27dukXBFunm7OysQYMGqXTp0hRskWbOzs5auXKlFi9eTMEWaebs7KyIiAgKtgAApFO1atUUFRVleV2vXr0kDwOrXLmyYmJiLK8HDBigvHnzatGiRbp06ZImT56sffv2WX3brnXr1tq3b5++/PJLrVixQgULFtTo0aNVv359SdKPP/6obt26aciQIZIkFxcXff311xo6dKh27NihatWqPcrDBoBUcTL/nIwNyGS///676tWrp9mzZ1OwRbr4+/urVq1aFGyRLi+//LKWLl2q3377jYIt0mzDhg0KCQnR3LlzKdgCAAAAyFAUbWFze/fuVaVKlWydBrKoAwcOqGTJkhRskS7h4eG6fv26ihYtautUkEXxHgYAAADgUaBoCwAAAAAAAAB2xNnWCQAAAAAAAAAA/g9FWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAcQHR2tb775RqdOnUqy7sqVK/r666916dIlG2QGAAAAAADSyskYY2ydBADg4T377LM6ePCgtm/frmzZslna27Ztq2vXrmnTpk1ycXGxYYYAAAAAACA1KNoCgIO4efOmypcvr+eee04TJ06UJM2ZM0evvvqq9u3bpyeeeEIJCQnatm2bLl68qCeeeEKVKlWy6uOPP/7QqVOn5OTkpDx58qhy5crKly+fVcyvv/6qwMBA5c2bV9u2bZOvr68aNGiQSUcJAAAAAIDjo2gLAA5k7dq1atGihTZu3KjAwEBVqFBBkyZN0oABA3T27Fm1bt1aCQkJKlmypHbt2qXSpUtryZIl8vDwkCRNnz5dW7ZskSSdP39eO3bs0BdffKFu3bpZ9lGjRg25ubnp3LlzqlSpkurUqaNhw4bZ5HgBAAAAAHBEFG0BwMG88sor+uWXXxQYGCgvLy+tWLFCklS3bl1VqVJF06ZNkyTFxMSoTp06ateund55551k+1q0aJH69++vsLAwZc+eXdLdou3ff/+t/fv3K3/+/JlyTAAAAAAAPE4o2gKAg7l9+7YqVaqkK1eu6ODBgwoICNDJkydVvHhxTZ48WYGBgTLGyBijRYsW6caNG1q/fr1l+wsXLmj//v26evWq4uLi1LdvX+3bt08VKlSQdLdoW6VKlf/H3n1HR1V9fx//pDdSIIVQQiB0KQEiXYogXUVFpKmgiEpHxYIFVPSHqICKCIqAiqiICAiIFBUFQQRpAtJbCL0lIaTnPH/wZL4MSWBSZ4Lv11qzYM7d95x970ySm50z52rKlCn2OkQAAAAAAG5qrvZOAABQsLy8vNS0aVPt379fZcqUkSQdPnxYkvTXX3/Jzc3NKrZatWqW5+PHj9fYsWMVFRWl0NBQS+yZM2esxggNDS3kowAAAAAA4L+Loi0A/Af4+/tLkl588UVFRkZmG3PhwgWNGjVKK1euVNu2bSVdKdbOmTNH134ow8nJqXATBgAAAADgP8zZ3gkAAApfvXr1VKZMGX300UdZth07dkySdPbsWRljVKVKFcu2efPmFVmOAAAAAADgCmbaAsB/gKurq2bNmqV7771Xp0+fVvv27XX27FktWbJEffv21aBBgxQREaG6deuqd+/e6tu3r3bv3q2vvvrK3qkDAAAAAPCfw0xbALgJNWvWTB06dLBq69Chg3bt2qWoqCht2rRJaWlpmjp1qgYNGiRJcnFx0a+//qpOnTppw4YN8vX11YYNG9SnTx/L2riS1LFjR9WpU6dIjwcAAAAAgP8SJ3PtQoUAAAAAAAAAALthpi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AwCItLU3Jycn2TgMAAADFXFJSktLT04v9eMnJyUpLSyvwfq8nPT1dSUlJMsYU6bgAHAtFWwC4joyMDHunkKPCKLCOGDFCgYGBBdpndigOAwAAR5GamqqkpCSlpKTkGJOSkqKkpCSlpqYWYWb2k3lOMh+5vSa+dOmSvLy89M477+Q5h9wUYQtivJyULl1aQ4YMKfB+r2f69Ony8vLSkSNHCnUcisOAY6NoCwDX2LRpk/r06aNy5crJw8NDJUqU0C233KJhw4Zp165d9k7P4oUXXpCPj4+908iT4pw7AAC4uXTv3l1eXl4qWbKk4uPjs2w/f/68fH195eXlpUceecQOGRa9zHMSEBCggIAAubm5KSwsTE899ZRiY2NvuL+Tk5M8PDzk6uqap/HPnj0rLy8vTZo0yab4/I73XzV79mx5eXlpz5499k4FQDYo2gLAVd5//301btxYzs7O+vHHH5WUlKQzZ87ovffe08aNG/XAAw/YO0UAAAAUMHd3d6WmpmrevHlZtn311VdydXWVk5OTHTKzHxcXF8tM2/j4eI0dO1aTJ0/W3XfffcN9fXx8lJSUpJEjRxZBpkU/HgAUBYq2APD//fzzz3rqqac0ePBgzZ49W5GRkXJxcZGXl5fat2+vtWvX6uGHH85x/5w+Npaamprtx+0yP450tWuXDcjpo0qpqamWj4td/dG17OLz83GnnPZNS0vLkvvV+1zv42zXyz03/dp6rrLjyMteAACAoufl5aUuXbros88+y7Jt1qxZuv/+++XsnPXX56uXEbjR8gpXL62Q03XL9a5RCqIPW/O9lre3t/r166devXrp999/17Zt2yRlXcIgLS3NMn5214PXtmWXqzHGco2XeW2YlJR0w3Vl8zpedvJ7XZl53ZrdchqZvwPk9no0u5xutLzB9c5bWlqaJb/M5T8y88pNv9nF2nr+inLNY6A4omgLAP/fG2+8IW9vb73xxhvZbndxcdFzzz1n1bZ8+XI1a9ZM7u7u8vDwUN26dTV79myrmO7du6tu3bpZ+ps6daq8vLx07NgxS1vmmrJxcXF68MEHVaJECXl6eqpbt246f/68Ja5v376aPHmy0tPTLR9bCwgI0KFDhyRJ+/fvV7du3RQYGCgPDw9VrlxZL774ouLi4mw6F+fOnVO3bt3k4+Mjb29v9ejRQ6dOnbJs//TTT+Xl5aUNGzZk2ffjjz+Wl5eX/v7772z7vl7uuek381zdKNdM27Zt09133y1fX1+5ubmpYsWKGjduHAVcAAAgSerXr5/WrFmjAwcOWNq2b9+uzZs357gsQv/+/S3XMt7e3vL19VWnTp0sRc1MjRo10l133aXt27eradOm8vDwUMWKFbVgwQJJ0pYtW9S8eXN5enoqJCRE06ZNyzJWQfRha745qVq1qiQpJiZGx44dk5eXl6ZOnao5c+aoatWq8vDw0Pfff5/tGrNXx8+bN09VqlSRm5ubKleurPnz51vitmzZosqVK0uSXnnlFUu+Q4cOzTGv/IyX6dSpU+rRo4e8vb3l4+Oje++9V2fPns12vNTUVL3xxhuqUqWK3N3d5eXlpTvvvFO7d++WdKVo2aVLF4WGhlqtS5ucnKxGjRqpatWqVtf2OUlJSdHQoUPl7+8vd3d3tW3bVjt37rRsX79+vby8vDRz5sws+65du1ZeXl764osvsu37rbfe0qBBgyRJDRs2tJznpUuX5qrfzPV39+3bd91cMx07dkz9+vVTUFCQ3NzcFBISomHDhikhIeGG5wP4zzEAAJOQkGBcXV1N27Ztbd5n0aJFxtnZ2fTv39+cOHHCnD9/3rz22mtGkpk4caIlrmvXrqZ69epZ9p88ebKRZKKjoy1tgwcPNj4+PqZfv35m5cqVJikpyfz+++8mICDA9O3b12r/Z555xri4uGSbW9WqVU2bNm3MoUOHTEZGhjl8+LAZP368+frrr697TJnjP/DAA5bx//zzT1OpUiVTu3Ztk5iYaIwxJj4+3vj5+ZmHH344Sx9169Y1tWvXvu44OeWem35tzdUYY/7880/j5eVl7rvvPrN//36TnJxsli5dakqVKmWGDh163VwBAMDNrWvXrsbf39+kpqaakJAQ88orr1i2jRgxwlSsWNFkZGQYFxcX06dPnxz7SU9PN3v37jVdu3Y1oaGh5uzZs5ZtkZGRpmHDhua+++4ze/fuNQkJCWbkyJHGzc3N/Prrr6Zz585m9+7d5vLly+aFF14wkszWrVut+i+IPmzNt2vXrtleq/Xo0cNIMv/884+Jjo42kkyXLl3MoEGDzMmTJ82+ffvMH3/8YeLj440kM27cOMu+mfH33HOPGT58uDl16pS5cOGC6d27t3FzczNHjhyxxJ45c8ZIMu+8806O+V8tv+MlJSWZ2rVrm0qVKpn169ebpKQks2rVKtOtWzfj5+dnnnjiCUtsRkaG6dKliwkKCjLz5883CQkJJiYmxtx///2mVKlS5ujRo8YYY06dOmXKli1rGjZsaJKTk40xxgwYMMC4ubmZ9evXX/d4pk6daiSZbt26mS+++MIkJCSYvXv3mmbNmpnAwEBz7NgxS2xkZKRp0KBBlj569eplSpQoYeLj43McZ9asWUaS+ffff7Nss7Xf3OR67NgxU6ZMGdO4cWOzefNmk5KSYv766y/L7y4ZGRnXPS/Afw1FWwAwxuzfv99IMv369bN5n+rVq5saNWqY9PR0q/YuXboYX19fy4VMbou2ksy3335rFZtZ5ExKSsrSdq2TJ08aSebjjz+2+ViuHX/27NlW7atWrTKSzKeffmppGzJkiPH09LS6wF+zZo2RZCZNmnTdca5XcLa139zk2rBhQ1O1alXLBXOmDz/80Dg7O1tdtAMAgP+WzKKtMcY89dRTJjw83GRkZJiUlBQTHBxsXn31VWOMuWHRNi0tzSQmJpqjR49muUaJjIw0bm5uloKeMVcmDXh7exs/Pz9z6NAhS3tiYqIpUaKEGT58uFX/BdGHrflmFm0TExNNYmKiOXnypPnggw+Ms7Ozad++vTHmf0XRmjVrZim2Xa+Iem0h8NSpU8bZ2dmMHTvW0laQRVtbxps5c6aRZFauXGkV++WXXxpJVkXb+fPnG0lZJkMkJCSYkJAQM2TIEEvbmjVrjKurq3nyySfNZ599ZiSZ999//4bHk1kIff75563ajx49atzc3MyIESMsbdOmTTOSzJ9//ml1jO7u7qZ///7XHed6RVtb+81Nro888ojx8fExJ06csIpdvXq1kWR++umn6+YL/NewPAIASJYbSxgb1186fvy49uzZo7vuuivL+mb33Xef4uPjtWnTpjzn07FjR6vntWrVUnp6uo4ePXrDfYODgxUWFqaxY8fq448/tmmfa3Xp0sXqedu2beXt7a1ffvnF0jZo0CAlJSVp1qxZlrapU6fK3d1dDz74YK7HzGu/N8r1zJkz2rhxozp37ixnZ2elpqZa1hlu3ry5MjIytG7dujznCwAAbh6PPPKIjhw5ol9//VVLlizR2bNn1bdv3xzjDx06pD59+igkJERubm7y9/e3LCFw9TILklSzZk2FhYVZnnt7eyssLEzly5dXxYoVLe2enp4KDw+3LHtVkH3kJt+rl7KqWrWqPv74Y7344ov6/vvvreK6dOmSq5u0XXudGxISoqCgoGyPtyDYMt6vv/4qDw8PtW3b1io2u5uuLV26VC4uLurcubNlXdiUlBS5urqqYcOGWrNmjSX2tttu01tvvaVp06ZpwIABeuCBBzRs2DCbc7/2OjcsLEz16tWzuiZ/8MEH5e/vr6lTp1raZsyYoZSUFPXv39/msa6V235tyXXp0qVq0qSJgoKCrK7JGzZsKDc3N6tzB4A1bQFAklSmTBm5ublZrTl1PRcuXJAklS5dOsu2kJAQSbrhOlU5raWaub7Y1TKfx8bG3jA3Z2dnrVq1Sk2bNtWzzz6r8PBwRURE6MUXX1R8fPwN93d1dVXJkiWztAcHB1sdU82aNdWmTRtNmzZNxhidOXNG3333nbp27aqgoKAbjpOT3PRrS66nT5+WJE2ZMkUlSpSQr6+vfH195efnpyZNmsjDw8Om8woAAG5+derUUf369fXZZ59p1qxZat26tVUx9GqXL19Wq1attHv3bi1btkzJyclKTk62rIN67U2oQkNDs/RRokSJHNuzuz7JTx+5zdfFxcVyc6q4uDjt2LFDY8eOlY+Pj1Vc2bJls4x9Pdnl6uvrW2jXY7aMd/78eQUHB2cpPvv6+srT09Oq7fTp00pPT1dISIjl2tLPz09+fn5atWpVlrVZu3fvLnd3d6WmpmrEiBG5yj3z94pr266+Jvfx8VHfvn01d+5cnT9/XhkZGfrkk09Us2ZNNW3aNFfjXS23/dqS65kzZ7R69eos1+SlSpWSs7OzzfffAP4rKNoCgK7cMfj222/Xn3/+aSnIXk9m8fDkyZNZtp04ccIqJiAgINtiaUxMTLZ952amQk6qVaumb7/9VhcuXNDff/+tPn366O2339bAgQNvuG9aWlqWgrMxRqdOnVJwcLBV+6BBg3TgwAGtWLGiQP6in9t+bck189/nnnvO8ovHtY8nnngi3zkDAICbQ79+/TR//nwtW7YsxxuQSdKaNWsUHR2tN954Q1FRUXJzc5OkHGeM5nSNl5trv/z0kdt8bZXZj60K4lq3oMcLCgrSmTNnsnzq7uLFi0pKSrJqCw4Olqenp+Lj47O9rty3b58lNiUlRQ888IBKlSqlqlWr6pFHHtGlS5dszj27m+uePHky22vyzE+q/fjjjzp8+HCBXZPb2q8tuQYFBalLly45XpN/8MEH+c4ZuJlQtAWA/2/06NFKTU3Vc889l+32lJQUjR49WtKVGba1a9fWwoULlZ6ebhU3b948BQQEqGHDhpKkKlWq6OTJk1YXMhkZGVqyZEm+8i1RooTS09OVlpaWY4yLi4saNGigsWPH6vbbb7d5GYCFCxdaPV+2bJmSkpJ0xx13WLV37dpV5cqV04cffqiPP/5YYWFhateuXb5zz02/N8o1JCRETZo00ffff6/k5OQb5gYAAP7b+vTpIxcXF/n7+6tbt245xrm4uEi68smfq3322WeFmV6eFad8M2fzFtW12x133KHk5GQtW7bMqv3apSCkK9epSUlJWrBgwQ37feaZZ7Rp0ybNnTtX8+fP19GjR/XYY4/ZnNe117mHDh3S1q1bs1yTV69eXW3bttW0adM0ZcoUubm56aGHHrph/yVKlJCU83nOTb+25Nq1a1f99ttvlkkuAK6Poi0A/H/NmzfX9OnT9fnnn+uee+7R77//rgsXLujUqVOaN2+eGjRoYHVxNmHCBB0+fFgPPvig9uzZoyNHjmjkyJFauXKl3nrrLXl5eUm6MlvDzc1NQ4YM0dGjR7V37171799f9erVy1e+devWlSR99913SkhIUFJSkowx2rx5szp37qzFixfr6NGjunDhghYvXqy//vrLpoKqj4+PfvnlF33//fc6c+aMVqxYoccff1xRUVHq1auXVayrq6sef/xxLVmyRIcPH1a/fv2yrPGbm9xz26+tuU6bNk0nT57UnXfeqT/++EPnz5/X/v379e2336pFixY6d+7cDXMGAAD/DYGBgYqLi9PZs2fl7e2dY1zjxo1VpkwZvfLKK9q1a5diYmL0zjvv6MyZM0WYre2KU75eXl6qWrWqVq1apZMnTyopKem6ExXyq0ePHoqKitLjjz+u5cuX68yZM1qwYIFWrFghf39/q9iuXbuqV69eeuKJJ/TRRx/p0KFDOnv2rP7880+NHDlSr7/+uiRp7ty5+vDDDzVu3Di1bNlSderU0UcffWRpt8WpU6c0ZcoUnTx5Ups2bdL999+v4OBgjRw5MkvsoEGDtH//fv3000+68847s12u4Fp16tSRk5OTFi5cqLi4OCUlJWVZws3Wfm3J9c0331RQUJA6dOigH3/8UadPn9bRo0e1bNky3XvvvaxpC1yDoi0AXOWRRx7R9u3bVbZsWT3xxBOqVKmSGjRooEmTJqlbt2768ccfLbHt27fXr7/+qosXL6p58+aqV6+eNmzYoAULFlh93L58+fJasGCBDhw4oDp16qhXr17q0aOHWrRoIQ8PD6uPbLm5uWVZN0u6MjPCw8PDqnB57733atiwYXruuecUHBysgIAAHTp0SA0aNNDw4cM1c+ZMtWjRQlWrVtWrr76q0aNH3/ACMXP8jz76SD/88INq1aqlhx9+WJ06ddKKFSuy/fjb448/Ljc3Nzk5OV33I4RXyyn3vPRrS66RkZHaunWrKleurL59+6pixYrq0qWLFi1apPHjxyswMNCmvAEAwM3H3d092+uva3l6esrd3d3y3NfXVz/99JP8/f3VsmVLNWvWTGfOnNG0adPk4eFhdS3i4eFhtW9e2vPbR27yteWcODk5ycPDI8vM3Zy2XS8+u2OYMWOGLl++rOrVqysgIEBDhw7NVS65Gc/NzU0rVqxQp06d1LdvX9WqVUsLFizQxx9/LE9PzyzXwHPmzNG7776rr7/+Wrfeeqtq1aqlkSNHqnz58nr66ae1f/9+DRkyRPfdd5+effZZy379+vXTgAED9NJLL2nz5s05Ho+rq6s8PDw0duxYxcTEqGHDhurQoYMqVKigtWvXZls47dq1q8qXLy9JNi+NUL16dU2YMEFffvmlypQpo4CAAC1dujRP/b7++us3zDUkJESbNm1Sly5d9Nxzz6lq1apq1aqVPvroIz3yyCO67bbbbMob+K9wMrbeKh0AgGycP39eZcuW1W233aZVq1YVWb9DhgzRZ599lqt1wQAAAICbVUREhJKTk3X06FHLchiF3e+0adM0cOBAHTp0KMeb9gHIG2baAgDyZeHChUpOTtaAAQOKRb8AAADAzeavv/7SoUOH9OijjxZowbaw+gVwY1k/JwAAgA1SUlJ0/Phxvfvuu6pRo8Z1b9ThCP0CAAAAN5u0tDRdvHhRY8aMkZ+f33WXkXCEfgHYjpm2AIA8adeunerUqaNSpUpp3rx52a4XVpj95rT+LwAAAPBfMXToUIWHhysmJkbz5s2z6QZkBdlv5vq7V9+nA0DBYE1bAAAAAAAAAHAgzLQFAAAAAAAAAAdC0RYAAAAAAAAAHAg3IisEGRkZOn78uHx9fVnXBQAAoIAZYxQfH6+yZcvK2Zk5CAWNa1kAAIDCY+u1LEXbQnD8+HGFhYXZOw0AAICbWnR0tMqXL2/vNG46XMsCAAAUvhtdy1K0LQS+vr6Srpx8Pz8/O2cDAABwc4mLi1NYWJjlmgsFi2tZAACAwmPrtSxF20KQ+TEyPz8/LnQBAAAKCR/dLxxcywIAABS+G13LsggYAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA2FNWwAAAAAAABSp9PR0paam2jsNoMC5ubnJxcUl3/1QtAUAAAAAAECRMMbo5MmTunjxor1TAQpNQECAQkND83XjXIq2AAAAAAAAKBKZBduQkBB5e3vnq6gFOBpjjC5fvqzTp09LksqUKZPnvijaAgAAAAAAoNClp6dbCraBgYH2TgcoFF5eXpKk06dPKyQkJM9LJXAjMgAAAAAAABS6zDVsvb297ZwJULgy3+P5WbeZoi0AAAAAAACKDEsi4GZXEO9xirYAAAAAAABAAUpMTNSmTZuUlpZWbMeytd+LFy/q33//LdCxHdnFixe1e/fuQh+HNW0BAAAAAACAArRnzx41bNhQZ86cUVBQULEcy9Z+H3/8cUVFRalmzZoFNnZOdu/erbS0NNWuXTvb7cYY7d27V66urqpcubLVtgsXLujAgQPZ7lezZk35+PhYtV26dElHjhxRRESEZZ3azDFatmyptWvXqlq1avk8opxRtAUAAAAAAIBdTf/9YJGNNaBlRJGNVRS8vb0VFRUlNze3Ih97w4YNWrVqlT7//PNCHWfGjBn66KOPtHfvXpUuXVr79+/PErNlyxbdf//9unTpklJSUhQWFqbvv/9eVapUkSRt3LhRL774otU+J06c0PHjx7Vv3z5LXGpqqp555hnNmDFDFStWVGxsrF544QUNGTJEklSyZEkNGDBAo0aN0vz58wvtmFkeAQAAAAAAAMhBWlqaNm3apE2bNmnbtm2Ki4vLdR8XL17Uli1brNqSkpK0adMmy82qLl26pE2bNskYo8TERO3atUvx8fGW+HPnzmnXrl1KTk626icsLEzTpk2zzBS9up+kpCTt3r1bCQkJBX5MkvT++++rV69elpmotoydF5s3b9Ynn3yiZ555JtvtKSkpuu+++9SyZUudPHlSZ86cUfny5dWjRw9LTPv27S3HnPmoVauWmjVrZinYStLgwYO1cOFCbd26VTt37tSBAwdkjLEar1+/flq0aJGio6PzfWw5YaZtIao9ZrmcPbgjIgAA/zWH3+pi7xQAAABQQGJjY/Xkk09KulIc3L9/v+69917NnDlTHh4eNvWxevVqPfjgg7p06ZKl7fDhw2rYsKGio6NVvnx5bd26VS1atNAjjzyiH3/8Ud7e3jp16pRmzpypX375RYsXL5a7u7suX76sH3/8UbfeequkrMsYZPYzYsQIffXVV/L399exY8c0ffp09enTp8COKS0tTUuWLNHs2bMtbbaMLUmbNm26bt++vr6qXr265fmUKVMkSYsXL842fuXKlTp8+LBeffVVOTk5ydXVVa+88oqaNWumzZs3q0GDBln2OXLkiH7++WfNmDHD0nbo0CF9+umn+uqrr1S1alVJkoeHh4YOHWq1b9WqVRUWFqYffvhBgwcPvu6x5BVFWwAAAAAAcPNZNznv+zYbeuMY/GcEBgZaFRnPnDmj22+/XR9++GGOMz/zw9nZWTExMXJxcdETTzyhXr16aeDAgTp27JicnZ3Vp08fjRo1SitXrrxuPxcvXlRMTIxcXV01YcIEDR48WN27d5e7u3uBHNO///6r+Ph41a1bN1djp6enWwrGOYmKitLHH39sUx7SlZm4ISEhCg8Pt7Q1atRITk5OORZtZ82apRIlSuiBBx6wtP38889ycnLSnXfeqRMnTiguLk6VKlWSu7t7lv3r16+v9evXF1rRluURAAAAAAAAgBuIi4vTzp07deTIETVv3ly//PJLoYzz8ssvy8XFRZLUpUsXGWP0yiuvyNnZ2dK2ffv2G/YzZswYubpema/ZvXt3xcbG6siRI1Yx+TmmU6dOSVK2Nym73tguLi5Zlim49pGbgq10ZemIwMBAqzYXFxcFBATo3LlzWeIzMjI0a9Ys9enTR97e//uU/PHjxxUYGKhnnnlGDRo0UJcuXRQUFGSZ6Xu1oKAgnTx5Mld55gYzbQEAAAAAAIAcJCUl6eGHH9YPP/ygChUqyM/PT6dOnVJwcHChjBcaGmr5f2ZB8do2W9aJza6fzP0K4pgyl1FISkqSr6+vzWNLuV8e4Ubc3NyyrPWbmVt2N2hbtWqVjh49qscee8yq3cXFRWfOnJGLi4tiYmLk7Oysr7/+Wn369FFUVJSaNGli1benp6fNOeYWRVsAAADAARhjtGLFCm3ZskVdu3ZVzZo1s8Ts379fa9asUXp6uho2bKjIyMgsMcnJyVqyZImOHj2qatWqqVOnTpaZOQAAIPfee+89bdu2TceOHbPMKn3llVdyXF81O05OTlnaMm9AZg8FcUwRERGSpGPHjuWq2FsYyyNUqFBBp06dUnp6umWW8oULF5SYmKgKFSpkiZ8xY4aioqKyLJuQubzC0KFDLddPvXr10vDhw7V69Wqrou2xY8dUu3Ztm3PMLYq2AAAAgJ2tWLFCgwcPVnh4uH7++WdVrFgxS9H2oYce0saNG9WsWTNJ0lNPPaWHH37Y6uN6Fy9eVOvWrZWSkqIWLVrovffeU7Vq1fTjjz9mO8sEAADc2P79+3XrrbdaLQNwo/VkrxUSEqKEhARduHBBJUuWlHTj2aaFqSCOqVy5cqpcubLWr1+v+vXr27xf5vIIBalNmzZKSEjQb7/9pjZt2ki6ctMyV1dXtWzZ0ir23LlzWrRokd5///1s+3FxcdGJEycs12IJCQmKj4+3Wn4hPT1dmzZt0rBhwwr0OK5G0RYAAACws5IlS+qnn35SeHh4jsXVPn366IsvvrDM1Onfv79uu+029ejRw/LLyJtvvqm4uDht27ZNvr6+OnbsmG655RZNnz5dgwYNKrLjAQDgZtKqVSsNGjRIn3/+ucqXL69Zs2Zp8+bNuuWWW2zuo379+goLC9OTTz6pQYMGae/evXrttdcKMevrK4hjkqR+/frpu+++K/TrjD179ig+Pl7Hjx9XcnKypegbGRkpNzc31apVS3369FH//v317rvvKikpSU8//bRGjBihkJAQq75mz54tV1dX9erVK8s4ZcuW1YgRIzRw4ECNGzdOfn5+mjRpkkJCQnT//fdb4pYvXy4vLy917ty50I6Zoi0AAABgZw0bNpQkpaWl5RjTsWNHq+dNmzaVi4uLDhw4YCnazps3T71797asK1e+fHl17txZ8+bNo2gLAEAePfTQQ4qPj9eXX36p1NRUtWjRQh988IHWrFmT4z7e3t6Kioqy/DHW09NTK1eu1NixYzVmzBjVqlVLc+fO1fDhw+Xu7i7pyjquUVFRVssa+fn5KSoqyqrvkiVLWn2s/9qxsuvHzc1NUVFRlvVlbTmma/vNzsCBAzVp0iTt2rVLt9xyi01j58U777yjrVu3SpJKly5tWV7hxx9/tBRlZ86cqYkTJ2ry5MlydXXVq6++mu31z9atWzV8+HD5+fllO9bbb7+tKlWqaNq0aUpPT1f9+vU1c+ZMywxpSZo6daqeeuopy2tXGJyMMabQev+PiouLk7+/v8JGfCtnj7y/IQEAQPF0+K0u9k7hppZ5rRUbG5vjxXZxlZaWJjc3N3399dfq2bPndWPnz5+v+++/X//8849q166txMREeXt767PPPlPfvn0tcWPGjNHHH3+c492Nk5OTrW7cERcXp7CwsJvy/AL4j1k3Oe/7NhtacHnAIikpSYcOHVKlSpUK9QZOKFozZsxQdHS0Xn31VXunUiQOHjyoJ598UosXL7bcjO1a13uv23oty0xbAAAAoJg5cOCAnnzySQ0cONByA4xLly5Jkvz9/a1iAwICLNuyM27cOLt+PBMAABRv/fv3t3cKRSoiIkIrVqwo9HG4jSwAAABQjBw9elR33HGHWrZsqQ8++MDSnvmRw7i4OKv42NhY+fj45NjfqFGjFBsba3lER0cXTuIAAACwGTNtAQAAgGIiOjparVu3Vr169fTNN9/I1fV/l/M+Pj4qV66c9u/fb7XP/v37Va1atRz79PDwyPGjfQAAALAPZtoCAAAAxcCxY8fUunVrRUZG6ttvv832piD33nuvvv32WyUlJUmSzpw5o6VLl+q+++4r6nQBAACQD8y0BQAAAOzs0KFDmjt3rjIyMiRJP/zwgw4fPqz69eurQ4cOkqQ77rhDZ86cUYMGDTRhwgTLvi1btlSzZs0kSaNHj9ZPP/2kli1b6vbbb9cPP/yg6tWra+DAgUV/UAAAAMgzirYAAACAnaWmpurixYuSpOeff16SdPHiRSUmJlpiunXrpvT0dCUkJFjtm5ycbPl/cHCwNm/erLlz5+ro0aN6+eWX1b17d7m7uxf+QQAAAKDAULQFAAAA7KxatWp66623rhvz5ptv2tSXr6+vHnvssYJICwD+u9ZNzt/+zYYWTB4A/rNY0xYAAAAAAAAAHAhFWwAAAAAAAABwIBRtAQAAAAAAgBxs2LBBQUFBVuvI38iOHTsUFBSk8+fPF2JmhTuWrf2++eabeuaZZwp0bHs5ePCgatasqbi4OHunwpq2AAAAAAAAsLP8riOcG7lcczg1NVXnzp2TMcbmfdLS0nTu3DllZGTkNrtcK6yxbOn32LFjevvtt7Vjx44CHftaGzdu1Ouvv64dO3bI29tbffr00QsvvCBnZ+v5qEuXLtXEiRO1d+9e1a5dW2+99ZYiIyOz9Hf58mW1bt1aBw8e1Nq1a1WjRg1JUkREhBo0aKA333xT48ePL9RjuhFm2gIAAAAAAADFVJ06dXTmzBkFBgYW+dgffvihbr/9doWFhRXaGLt27VKrVq1Uo0YN/fzzz/r44481e/ZsvfDCC1ZxM2fOVLdu3XT//fdr3bp1evHFFzVu3Lhs+xw+fLh8fHx07tw5paWlWW17/PHH9fHHHyshIaHQjskWFG0BAAAAAACAG1i+fLlat26tChUqqFOnTtq7d6/N+y5btkwVK1a0atu/f7+CgoJ04sQJSf9bhuHrr79WmzZtVKFCBbVv31579uzR/Pnz1bhxY1WoUEE9evTQ2bNnLf38+++/qlGjhi5cuGDVz7Jly9S2bVtVrFhR7du3186dOy37nD59WkFBQQoKClKZMmXUrFkzzZ49O9fnZM6cObr33nstz20ZO7e++uorlS5dWu+8844iIiJ02223ady4cXr//fd18eJFSVJcXJxGjBih0aNHa+DAgQoLC1OLFi309ddfZ+nv22+/1V9//aU33ngj2/FatmwpFxcXLV68OM85FwSKtgAAAAAAAMANPPXUU3r11Ve1YsUK+fr6qn379javc5ucnGxVaJX+t/xAenq6pP8twzBx4kS99dZbWrVqlRISEnT77bdr4sSJmjJlipYvX679+/fr2WefzdJP5jIGmf28/PLLeuONN/TLL78oICBA3bp1syzxEBwcrN27d2v37t36+++/9fTTT2vo0KH64YcfbD4fhw4d0rFjx9SwYUNLmy1jp6enWwrGOT26du1q6TMpKUleXl5WY3t7eyslJUXr16+XdKWgHh8fr759+1rFOTk5Zcl52LBh+uqrr+Th4ZHtcTk5Oalhw4ZavXq1zeeiMFC0BQAAAAAAAG5gwoQJat26tWrUqKGZM2fq4sWLmjt3boGPM23aNDVq1EjVqlXTsGHDdOLECX3yySe69dZbVbNmTQ0aNEi//vrrDfv5+OOP1bRpU0VEROi1117Tnj17FBMTI+lKYTKzQFq2bFndf//9evLJJ3M12/bIkSOSpNDQ0FyN7eLiYikY5/T44osvLH116dJFu3fv1qxZs2SM0blz5yzLHhw/flyStHfvXgUGBurPP/9UvXr1VKlSJXXt2lVbtmyx9JOamqpevXrppZdeUq1ata57bGXKlNHRo0dtPheFgaItAAAAAAAAcANNmza1/L9EiRKKjIzUP//8U+Dj1KxZ0/L/zHVqr207d+7cDfu55ZZbLP8PDg6WJKv9Zs6cqdtuu01hYWEKCgrS5MmTLYVYW2TOEHZxccn12Deaaevv72+Jvf322zV9+nSNGTNG3t7eqly5su655x45Oztbzd6NjY3VhAkTNGPGDK1cuVLBwcFq3bq1pfj6yiuvqGTJkhoyZMgNj83FxSXLWrdFjaItAAAAAAAAcAOurq5ZnqekpOS5v8yC47WcnbOW665ty2nfG/WTud8333xjWRLht99+0+7duzVixIhcHU/ZsmUlSadOncrV2LldHkGS+vfvr6NHj+rcuXM6d+6c2rdvr4yMDIWHh0uSQkJClJaWpkmTJikqKkpVqlTRtGnTlJGRoUWLFkmSfvrpJ/3xxx8KDg5WUFCQ7rjjDklSixYtNHToUKvxTp8+rXLlytl8LgqD641DAAAAAAAAgP+2bdu2qW3btpKufNR+586d6tatm037+vv7KzExUampqXJzc5MkHT58uLBSvaFffvlFXbp0UY8ePSxtubmxmiTVqFFDpUqV0t9//61q1arZvF/m8gjXk3mOruXt7S1JmjdvngICAtS8eXNJ/5sFffXat66urnJzc1NqaqokafXq1VazZ7dt26Y77rhDixcvVr169azG+fvvv/Xaa6/ZfEyFgZm2AAAAAAAAwA288MILOnHihFJSUvTyyy8rOTlZvXv3tmnfOnXqyMPDQ9OmTZMkRUdHa/To0YWZ7nWFh4frzz//1IkTJ5Senq4vv/xS33//fa76cHJyUrdu3fTjjz/mevzcLI8gSU8//bTOnTsnY4wWLVqk8ePHa/z48ZYibmRkpNq0aaPXXntN8fHxSk9P11tvvaXLly+rc+fOkqSAgIBsxwgICFCJEiUsY23btk3nzp3T3XffnevjKkjMtAUAAAAAAChGpv9+MFfxA1pGFFIm/y0PPPCAGjRooPPnz6ts2bKaP3++AgICbNo3KChIn376qYYPH67nn39eZcqU0WOPPaZNmzYVbtI5GD58uDZs2KDw8HC5uLiodu3a6tOnj7Zv357rfpo0aaLz58+rVKlShZTtlTVy69Spo4sXLyo0NFRTpkxRv379rGLmzp2rxx57TEFBQXJxcVFERIQWLlyoGjVq5GqsmTNnqnfv3goJCSnAI8g9J2PLIhjIlbi4OPn7+ytsxLdy9vC2dzoAAKCIHX6ri71TuKllXmvFxsbKz8/P3uncdDi/AG4a6ybbb+xmQ28ckw/FtWiblJSkQ4cOqVKlSvL09LR3OjZLS0vTxYsXFRQUJElKSEiQj4/PdfdJT0/XhQsXFBgYKCcnJ6ttiYmJ8vLyssSUKlVKzs7OWcaRrizDEBsba9WWkpKi+Ph4y03Krh0ru36MMTp37pxKlixpdeOwlJQUpaeny8vLS4mJiUpOTrYUoq93DFcbOHCg/Pz8NH78+FyNnRe2nPuUlBQZY+Th4XHduMxcr84rJiZG9evX19atWy1r9ubF9d7rtl5rMdMWAAAAAAAAyIGrq6tVEfJGRUPpyrqtV+9ztcx1V6+NuXYc6crarte2ubu7Wwq2tvbj5OSUbT7u7u5WeV29Juz1juFqH3zwgRISEnI9dl7Ycu6vPqbryS7XoKAg7d+/3yH+cE3RFgAAAAAAAECeuLm52bxMhKPz8PC44QzdosKNyAAAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAUGWOMvVMAClVBvMcp2gIAAAAAAKDQubm5SZIuX75s50yAwpX5Hs98z+eFa0ElAwAAAAAAAOTExcVFAQEBOn36tCTJ29tbTk5Ods4KKDjGGF2+fFmnT59WQECAXFxc8twXRVsAAAAAAAAUidDQUEmyFG6Bm1FAQIDlvZ5XFG0BAAAAAABQJJycnFSmTBmFhIQoNTXV3ukABc7NzS1fM2wzUbQFAAAAAABAkXJxcSmQwhZws+JGZAAAAAAAAADgQJhpCwAAAAAAHM+6yfbO4KYx/feDuYof0DKikDIBYCtm2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAAAAscnvjstzgJmeAbZhpCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOxNXeCQAAAAAAgJvQusn2zgAAii1m2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAAIAdTf/9oL1TAOBgmGkLAAAA2FlaWpq+//57tWvXTkFBQVqwYEG2cYsWLVLTpk1Vrlw53X777frjjz/yFAMAAADHRtEWAAAAsLNJkybpyy+/1FNPPaVz584pOTk5S8yvv/6q+++/Xz179tTq1avVqFEjtWvXTnv27MlVDAAAABwfRVsAAADAzkaOHKnvv/9e7du3zzHmrbfeUufOnTV8+HBVrVpV48ePV0REhCZOnJirGAAAADg+irYAAACAnTk5OV13uzFGa9eu1R133GHV3q5dO61Zs8bmGAAAABQP3IgMAAAAcHAXL17U5cuXVbp0aav20qVL68SJEzbHZCc5OdlqOYa4uLgCzBwAAAB5wUxbAAAAwMEZYyRJLi4uVu2urq7KyMiwOSY748aNk7+/v+URFhZWkKkDAAAgD3JVtE1NTVVaWlquBsjLPnlVWGMV5TEAAICb17p169SxY0eFhYWpRo0aeuONN7Itpi1YsEAtW7ZUmTJl1L59e23dujXb/i5duqR69eopICBA+/btK+TsYU/+/v5yd3fXmTNnrNrPnDmj4OBgm2OyM2rUKMXGxloe0dHRBX8AAAAAyJVcFW07deqkl19+OVcDdO/eXSNGjMjVPnlVWGMV5TEAAICb0/bt29WmTRvdeuutWrdunWbNmqU5c+Zo1KhRVnFTp05V79691bdvX23ZskVjxozRu+++m22fgwYNUlBQkGJjY5Wenl4UhwE7cXFxUVRUlNauXWvV/ttvv6lx48Y2x2THw8NDfn5+Vg8AAADY1021PIK7u7vc3NzsnQYAAEAWc+bMUYUKFfTGG28oLCxMTZs21RtvvKH333/fsobohQsXNHLkSI0dO1b9+/dXaGiomjdvri+//DJLf7Nnz9auXbs0ZsyYoj4U2MmwYcP03Xffafny5UpPT9dnn32mTZs2afDgwbmKAQAAgOMr8huRpaSkyMXFxWqtrdTUVDk5OcnV1TXb58YYqzvqXvs805w5c6zar+0nO2lpaZalDzw9PfN3cAAAADlITk6Wh4eHVZunp6eSk5O1YcMGtWvXTsuWLdPly5f18MMPX7evffv26dlnn9Vvv/2mCxcuFGbaKCK//PKLHnjgAcvzxx9/XEOGDFG/fv0sM6179uypY8eOqVevXrp06ZICAwP12WefqVmzZpb9bIkBAACA48t10fbkyZPq2rWrfvvtN6Wlpal///6aOHFilhse5KRRo0Z68MEHNXLkSEvbQw89pBIlSujTTz+VdGUZhpCQEMXGxmrdunVKTk7Wo48+qmeeeUYDBw7Ur7/+qhIlSmjUqFFW/XTv3l3ly5fXhx9+aOknLCxMly5d0s8//6y0tDT17NlTH330kaWQ+9RTT2n69OlXToarqxo3bqzJkyfrlltuye2pAQAAyNFdd92lDz74QDNnzlS/fv10+vRpjRs3TpIUExMj6UoxtnTp0vr555/1xhtvKD4+XnXr1tXrr7+uBg0aSLryB/AePXro9ddfV/Xq1fXnn3/a7ZhQcFq0aKHdu3dnaffy8rJ6PnLkSD3zzDOKj4/PcRkDW2IAAIVr+u8H7Z0CgGIu18sjfP7552rbtq1OnDihn3/+WXPnztWECRMKPLF58+bpkUce0enTp7V48WJNnTpVjRo10sCBA3Xp0iV9/vnneu6557Rjx47r9jN79mx169ZNJ0+e1Nq1azVv3jzNnDnTsn3y5MlKSkpSUlKSYmJiVKtWLd17771KTU21Odfk5GTFxcVZPQAAAK7Wtm1bffrppxo7dqy8vLx0yy23qHfv3lYx6enpOnfunKZOnapvvvlGv//+u8qUKaM2bdpYbg71wgsvKCwsTI8//rg9DgOFxM3NTUFBQVkePj4+WWKdnJxuWIy1JQYAAACOK9dF2yZNmmjYsGHy8vJS48aN9fzzz2vSpEkFntg999yj+++/X25ubmrbtq1q1aqlTp06qWvXrnJzc9Odd96pihUr3nB2yb333quePXvK3d1ddevWVZcuXbRu3bpsY728vPTGG2/owIED2rVrl825jhs3Tv7+/pZHWFhYro4VAAD8Nzz66KM6dOiQ4uLidO7cOd12222SpIoVK0qSSpcurbS0NL333nuqU6eOKlasqI8++kipqalavHixJGnFihVatWqVAgICFBAQoHbt2km68mmmYcOG2eW4AAAAABSsXBdtMz+alykqKkonT55UbGxsgSUlSREREVbP/fz8VKlSpSxtFy9ezFU/AQEBVvts27ZN7du3l5+fn7y9vRUSEqL09HTLbBZbjBo1SrGxsZZHbvYFAAD/PR4eHnJyctK3336roKAgNW3aVJIs/1699q2rq6tcXV0ta/CvX79eMTExOnz4sA4fPqwFCxZIkn7++WeNHz++iI8EAAAAQGHIddHWGGP1PCMj40pHzrZ1ld0NxDL7uFFcdm15GS+TMUZdunRRzZo1tW/fPqWmpury5ctyc3Oz/GJkCw8PD/n5+Vk9AAAArjV06FCdPn1aGRkZmjdvniZMmKB3333XUqRt0KCBWrdurdGjRys2Nlapqal6/fXXlZaWps6dO0uSfH19LbNsAwICVKJECUv7teufAgAAACiecl203bBhQ5bnYWFh8vX1tWn/kiVL6ty5c1Zte/bsyW0aBSImJkYxMTEaOnSoSpcuLScnJ23atClX69kCAADYql69eqpfv768vLz08ssva8aMGerbt69VzLfffitjjEJCQuTn56dFixbphx9+UJUqVeyUNQAAAICiluui7ebNm/X666/r1KlTWr58ud5++20988wzNu/fqlUrzZkzR1u3btWpU6c0evRobd++PbdpFIjQ0FAFBgZq6tSpOnfunDZu3Kj+/fvbJRcAAHDz69+/v2JiYhQXF6c9e/ZkuRGZJAUHB+v777/X5cuXFRcXp82bN6tt27Y59tmwYUNduHBB1apVK8zUAQAAABShXBVt3d3dNWjQIO3fv18NGzbUI488okGDBmno0KHX3cfNzc3y/Nlnn1WHDh3UqVMnNW/eXImJierevbtVzLX7SFeWIHB1db1u27X7ZdePm5ub3N3dJV1ZI27+/PlavXq1IiIi9NBDD2nQoEEKDAyUi4vLdfsBAADIq6vXrM2Ji4uLTdcfLi4uCggIsHmpKgAAAACOz8lcu0gt8i0uLk7+/v4KG/GtnD287Z0OAAAoYoff6mLvFG5qmddasbGx3EugEHB+ARSYdZPtnYHdTE/jWiAnA1pG3DgIuInZeq3FlAwAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAH4mrvBAAAAAAAgINaN9neGQDAfxIzbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCDciAwAAAAAAABFYvrvB3MVP6BlRCFlAjg2ZtoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADsTV3gkAAAAAsM3ly5e1du1anT59WmXLllWLFi3k5uZmFZORkaE1a9bo6NGjqlatmho3bmynbAEAAJBXFG0BAACAYuCvv/5Sly5dVKFCBdWsWVMbN25USkqKfvnlF1WqVEmSlJCQoM6dO+vgwYNq1KiRfvvtN7Vt21ZfffWVXFxc7HwEAAAAsBXLIwAAAADFwOjRo1WvXj39/fff+vLLL7Vjxw55enpq/Pjxlpjx48frwIED2rJli+bPn6/169dr8eLF+uKLL+yYOQAAAHKLoi0AAABQDBhjVLp0actzNzc3lSpVyirm66+/Vq9evRQUFCRJqlq1qjp27Kivv/66SHMFAABA/rA8AgAAAFAMvPPOO+rVq5eGDh2qGjVqaP369UpPT9fLL78sSUpKStKBAwd0yy23WO1Xq1YtzZgxI8d+k5OTlZycbHkeFxdXOAcAAAAAm1G0BQAAAIoBPz8/lS9fXmvWrNH58+f1999/q169evL29pYkxcfHyxijkiVLWu1XqlSp6xZix40bp9dee61QcwdgR+sm2zsDAEAesDwCAAAAUAx0795dfn5+2rJli+bMmaMdO3boyJEjGjhwoCTJy8tL0pXi7dXi4+Mt27IzatQoxcbGWh7R0dGFdxAAAACwCUVbAAAAwMGlpKRo06ZNuvPOO+Xk5CRJcnV1VceOHfXHH39IkkqUKKHSpUvr8OHDVvseOnRIVapUybFvDw8P+fn5WT0AAABgXxRtAQAAAAfn7u6uMmXKaPv27Vbt27ZtU3h4uOX5XXfdpe+++05paWmSrqxPu2TJEt11111Fmi8AAADyhzVtAQAAgGLgzTff1BNPPKFLly6pdu3aWrdunZYsWaKlS5daYsaMGaNGjRqpc+fOateunebNm6eQkBANHTrUjpkDAAAgt5hpCwAAAOTRtUsRFKZHHnlEGzduVPny5bV3715FRkZq9+7dateunSWmfPny2rp1q9q0aaNDhw6pT58++vPPP+Xr61tkeQIAACD/mGkLAAAA5FHTpk0VERGhRx99VD169FCJEiUKdbzIyEhFRkZeNyYkJEQvvPBCoeYBAACAwsVMWwAAACCPfvrpJ0VFRen5559XaGioHnnkEa1du9beaQEAAKCYo2gLAAAA5FFkZKQ++OADHT9+XDNnztSJEyfUqlUrVa9eXW+99ZZOnDhh7xQBAABQDFG0BQAAAPLJ3d1dDzzwgJYtW6bPPvtMMTExGjVqlMLCwtS7d29FR0fbO0UAAAAUIxRtAQAAgHyKjo7W2LFjVaVKFT3xxBPq1q2b1qxZo1WrVunChQtq06aNvVMEAABAMcKNyAAAAIA8mj9/vj755BOtWrVKkZGReuaZZ9SnTx/5+/tbYlq0aKGSJUvq7NmzCgoKsmO2AAAAKC4o2gIAAAB59OKLL6pNmzb666+/FBUVlW2Mi4uLxo8fL29v7yLODgAAAMUVRVsAAAAgj/755x+5u7vfMG7gwIFFkA0AAABuFqxpCwAAAOSRm5ubVq9eLWOMVfuGDRt06dIlO2UFAACA4o6iLQAAAJBHkydP1oIFC+Tk5GTVvmvXLo0YMcI+SQEAAKDYo2gLAAAA5NGsWbP06KOPZmnv3bu35s+fr7S0NDtkBQAAgOKOoi0AAACQR5cuXVJqamqWdmOM4uPjlZiYaIesAAAAUNxRtAUAAADyqHnz5ho/fnyWGbX/93//p1tuuUW+vr52ygwAAADFmau9EwAAAACKqzFjxqhJkyaqUaOGWrduLQ8PD23YsEE7duzQjz/+aO/0AAAo9qb/fjBX8QNaRhRSJkDRYqYtAAAAkEeVKlXSli1b1K1bN+3fv1/btm3Trbfeqk2bNqlNmzb2Tg8AAADFFDNtAQAAgHwoW7asxo8fb+80AAAAcBOhaAsAAADkU0JCgs6ePStjjFV7WFiYXFxc7JQVAAAAiiuKtgAAAEAeHThwQH369NGGDRuy3X7ixAmFhoYWcVYAAAAo7ijaAgAAAHn01FNPqVSpUvrjjz8UEhKSZXtwcLAdsgIAAEBxR9G2EO14rYP8/PzsnQYAAAAKyc6dO7V06VLVqFHD3qkAAADgJuJs7wQAAACA4qpixYqKjY21dxoAAAC4yVC0BQAAAPJo5MiReuaZZ7Rv3z57pwIAAICbCMsjAAAAAHk0cOBAHTlyRNWqVZObm5ucna3nRBw5ckSlS5e2U3YAAAAorijaAgAAAHn06aefKikpKcftAQEBRZcMAAAAbhoUbQEAAIA8uuOOO+ydAgAAAG5CrGkLAAAA5NOePXu0dOlSJScnS5LS09PtnBEAAACKM4q2AAAAQB7Fx8erQ4cOqlGjhu68805duHBBkvTYY49p9uzZds4OAAAAxRVFWwAAACCPxowZIycnJ50+fdrqhmOjRo3Sm2++acfMAAAAUJyxpi0AAACQR6tWrdLs2bMVHBwsJycnS3vVqlUVHR2t2NhY+fv72zFDAAAAFEfMtAUAAADyKCEhQR4eHpJkVbRNSEhQcnKy3N3d7ZUaAAAAijGKtgAAAEAeNWvWTF988YWk/xVtMzIy9NJLLykqKkpeXl72TA8AAADFFMsjAAAAAHn0+uuvq2nTptqwYYNiY2P13HPPadOmTTp8+LBWrlxp7/QAAABQTDHTFgAAAMijSpUqadu2bWrSpIkaNWqkAwcOqEWLFtq0aZOaN29u7/QAAABQTDHTFgAAAMiH0qVL680337R3GgAAALiJULQFAAAA8ujy5cvKyMjIcbuPj4/VDcoAAAAAW1C0BQAAAPIoIiJCp06dynH7iRMnFBoaWoQZAQAA4GZA0RYAAADIox9//FEpKSlWbYcPH9bYsWN1//33KzAw0E6ZAQAAoDijaAsAAADkUYMGDbK0NWnSRI0bN1a7du302muv2SErAAAAFHcUbQEAAIACVqlSJZ0+fVoJCQny8fGxdzoAirt1k+2dAQCgiDnbOwEAAADgZvPVV19Jkry9ve2cCQAAAIojZtoCAAAAedSgQQOdPn3aqi0+Pl6XL1/W+++/LycnJztlBgAAgOKMoi0AAACQR88//7wSExOt2vz8/NSgQQNVrFjRPkkBAACg2KNoCwAAAORRjx497J0CAAAAbkIUbQEAAIA8unz5sjIyMmyK9fHxYbkEAAAA2IQbkQEAAAB5FBERIV9fX5sep06dyvd4CQkJeuONN9S8eXO1aNFCn3zyiYwxVjG///677rrrLkVGRqp79+76559/8j0uAAAAihZFWwAAACCPJk6cKF9fX73yyitavny5fv31V02aNEnlypXTyy+/rI0bN1oeQUFB+RorKSlJrVu31qJFizR69Gi999572r59uxYsWGCJ+euvv9SuXTvVq1dP06ZNU0BAgFq0aKEjR47k91ABAABQhJzMtX+aR77FxcXJ399fsbGx8vPzs3c6AAAANxVHuta6++671atXL/Xq1cuqfdeuXerUqVOBFkvfeOMNTZw4Ufv371epUqUs7WlpaXJ1dbXkk5ycrOXLl0uSjDGqVq2aOnfurPfff9+mcRzp/AL4/9ZNtncGyKXpaV3sncJ/1oCWEfZOAbguW6+1mGkLAAAA5NHOnTvVoEGDLO01a9bU2bNnFR8fX2Bjff3117rvvvusCraSLAVbY4xWr16tjh07WrY5OTmpU6dO+vXXXwssDwAAABQ+irYAAABAHpUtW1affvpplvavvvpK7u7u8vb2LrCx9uzZo2rVqmn48OGqX7++OnTooBkzZlhuhBYbG6v4+HiVKVPGar/Q0FAdO3Ysx36Tk5MVFxdn9QAAAIB9udo7AQAAAKC4+r//+z916NBBq1atUvPmzeXm5qZt27bpt99+0/Tp0+Xi4lIg46Snpys9PV2vv/66XnjhBX366afasWOHhg0bpmPHjmnMmDFKT0+XJLm7u1vt6+HhobS0tBz7HjdunF577bUCyRMAAAAFg5m2AAAAQB61aNFCe/fuVbt27bR//35t375d1atX15YtW/Too48W2DguLi4qWbKkmjdvrpdffllRUVHq27evhg8frk8++USS5O/vL1dXV509e9Zq37NnzyowMDDHvkeNGqXY2FjLIzo6usDyBgAAQN4w0xYAAADIh/Lly+vtt98u9HEaNmyYZT3bwMBAXbp0SdKVtW3r1aunP//8U48//rglZt26dbr11ltz7NfDw0MeHh6FkzQAAADyhKItAAAAkE979uzR/v37dccdd8jDw0Pp6ekFtjRCpsGDB+vBBx/U9u3bVbduXZ05c0affvqpOnfubIl54okn9NRTT2nQoEG69dZbtWTJEq1du1bLly8v0FwAAHBU038/mKv4AS0jCikTIH8o2gIAAAB5FB8fr/vvv18rVqyQJJ04cUKhoaF67LHH1KZNGz300EMFNtbdd9+tV199VS1btpSfn5/OnDmjLl26aMqUKZaYxx57TPv379dtt90mf39/Xb58WZMmTdIdd9xRYHkAAACg8DkZY4y9k7jZxMXFyd/fX2EjvpWzR8HdMRgAAMARHX6rS5GOl3mtFRsbKz8/vyId+1pPP/20du3apdmzZ6tOnTraunWrQkNDtXfvXt19993avXt3gY+ZnJys48ePq0yZMvL09Mw2JiEhQadOnVLZsmVzjMmJI51f4KaxbrK9M0ARm55WtD8bkXfMtEVRs/Vai5m2AAAAQB6tWrVKs2fPVnBwsJycnCztVatWVXR0tGJjY+Xv71+gY3p4eKhSpUrXjfHx8VFEBL+EAgAAFFfO9k4AAAAAKK4SEhIsN/G6umibkJCg5ORkubu72ys1AAAAFGMUbQEAAIA8atasmb744gtJ/yvaZmRk6KWXXlJUVJS8vLzsmR4AAACKKZZHAAAAAPLo9ddfV9OmTbVhwwbFxsbqueee06ZNm3T48GGtXLnS3ukBAACgmGKmLQAAAJBHlSpV0rZt29SkSRM1atRIBw4cUIsWLbRp0yY1b97c3ukBAACgmGKmLQAAAJBH7777rp588km9+eab9k4FAAAANxFm2gIAAAB59OGHH+rChQv2TgMAAAA3GYq2AAAAQB7Vr19fa9eutXcaAAAAuMmwPAIAAACQR3fffbcGDRqkPXv2qF69evL09LTa3qZNG7m7u9spOwAAABRXFG0BAACAPHrllVcUHx+vN954I9vtMTExKl26dBFnBQAAgOKOoi0AAACQR8eOHbN3CgAAALgJsaYtAAAAkEuhoaE6efKk5fno0aMVHx9vx4wAAABwM6FoCwAAAOTTJ598ooSEBHunAQAAgJsEyyMAAAAAAADgP2n67wdzFT+gZUQhZQJYY6YtAAAAAAAAADgQZtoCAAAAeTB+/Hj5+PhIki5dumT1PNMLL7ygEiVK2CM9AAAAFGMUbQEAAIBcqlSpkhYvXmx5HhoaavU804gRIyjaAgAAINco2gIAAAC5tH79enunAAAAgJsYa9oCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgANxtXcCAAAAAAA4vHWT7Z0BAOA/hJm2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EG5EBgAAAAAAANhg+u8HcxU/oGVEIWWCmx0zbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgbjaOwEAAAAAAGyybnLe9202tODyAACgkDHTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAChmYmNjtXDhQm3atCnb7Tt37tSyZct04MCBIs4MAAAABYGiLQAAAFDM9O/fXw888IDeffddq/aUlBTdc889uu222/TWW28pMjJSgwcPljHGTpkCAAAgL1ztnQAAAAAA202dOlVnz55V69ats2ybMGGC1q1bp3/++Ufly5fX1q1b1bhxY7Vs2VI9evQo+mQBAACQJ8y0BQAAAIqJHTt2aOzYsfriiy/k7Jz1Un727Nnq1auXypcvL0mqV6+e2rVrpy+++KKoUwUAAEA+ULQFAAAAioHLly+rR48emjhxoipUqJBle3Jysvbs2aPIyEir9nr16mn79u059pucnKy4uDirBwAAAOyLoi0AAABQDAwbNkxRUVHq2bNnttvj4+OVkZGhkiVLWrUHBgbq4sWLOfY7btw4+fv7Wx5hYWEFmTYAAADygKItAAAA4OB+//13ffnll7rjjju0cOFCLVy4UKdPn1ZMTIwWLlyopKQkubu7S7oyI/dqly5dkoeHR459jxo1SrGxsZZHdHR0oR4LAAAAbowbkQEAAAAOzs3NTR07dtT3339vaYuJiZGbm5s+++wztWjRQoGBgQoMDMxSdI2OjlalSpVy7NvDw+O6RV0AAAAUPYq2AAAAgINr2rSpFi5caNXWsWNHBQQE6JtvvrG0derUSQsWLNDzzz8vJycnJSUlacmSJXrssceKOGMAAADkB8sjAAAAADeJMWPGaO/everZs6dmzZqlO++8U+7u7hoxYoS9UwMAAEAuULQFAAAAiqFmzZqpYcOGVm1VqlTR5s2bFRYWpmXLlqlp06b666+/VKpUKTtlCQAAgLxgeQQAAACgGBo9enS27ZUqVdK7775bxNkAAACgIDHTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIG42jsBAAAAAAAA4GY0/feDuYof0DKikDJBcUPRFgAAAIUqMTFRzs7O8vDwyDEmLS1Nrq5ZL01TUlKUkpJi1Xbp0qXrjpdTXwAAAEBxUejLI8TExOjUqVOFPUyhjlWUxwAAAHAzSE9P1/fff6927drJx8dHffv2zTbur7/+UoMGDeTp6Sl/f389/fTTSk9Pt2x/7bXXFBoaavUIDw+XJGVkZFjikpOT9dxzzykkJEQlSpRQixYttGPHjsI9SAAAAKCQFHrRdvDgwRo7dmxhD1OoYxXlMQAAANwM9u3bp9mzZ2vkyJHq1KlTtjFnz55Vx44d1bJlS8XGxuqXX37Rl19+qddee80S8+abb+rSpUuWR3x8vMLCwiRJzs7/u5Tt2bOnfvjhB61cuVJJSUmaNGmSFi9eXLgHCQAAABSSm+pGZOXLl1doaKi90wAAAPjPq1GjhhYsWKAOHTrIyckp25jPP/9cGRkZevvtt+Xj46OoqCgNHz5cH374odLS0rLdZ/Xq1Tp06JBV288//6yFCxfqiy++UGRkpCTp1ltv1ahRowr2oAAAAIAikqui7alTp7R7927t3r1bZ86cydOAhw8f1vnz563ajh8/rpMnT1qeR0dHW/q/cOFClqUJjh07pri4uCx9jxo1SgMGDMixn3PnzhXKMQEAACD31q9fr8aNG8vd3d3Sdvvtt+vChQvas2dPtvvMmDFD1apVs2pbvHixKlasqEaNGikjI0PGmELNGwAAAChsubpDw9SpU/XNN99Ikk6fPq2SJUtq1qxZatmypc193HPPPXrwwQc1cuRIS9vTTz+tEiVK6NNPP5UkPfLII/Lw8NCRI0d08eJFnT59Wu3bt9crr7yiRx55RHFxcTp9+rSefPJJffDBB5Z+Bg8erPLly+vDDz+09OPn56djx47p+PHjOnfunFq0aKGFCxfK29u7wI4JAAAAuXf69GlVqFDBqi04ONiyrVatWlbbLl68qPnz5+uVV17RSy+9ZGk/fPiwKleurEGDBumzzz6TJDVp0kTvv/++6tSpU7gHAQAAUICm/37Q5tgBLSMKMRPYW65m2r766quWWannzp3TwIED1atXLyUmJhZ4YqtXr9asWbN07Ngxbd++XStWrFCnTp301Vdf6dixY9qwYYOmTJmiDRs2XLefn376Se+//76OHTumw4cPa+fOnZo2bVqBHlNycrLi4uKsHgAAALixq28mdvXz7JZUmDNnjjIyMtSrVy+rdmOMfv75Z/n6+ur8+fM6fvy4goKC1LlzZyUkJBRe8gAAAEAhydVM20znz5+33DjihRde0K5duxQVFVWgiXXr1k0NGzaUdGVNtFq1aqlhw4aqV6+eJKl+/fqKiIjQli1b1Lhx4+v207RpU0lS6dKl1a5dO23ZsqVAj2ncuHFWN8wAAADAjZUtW1anT5+2ast8XqZMmSzxM2bM0L333qvAwMAs/Xh4eOj//u//5OLiIk9PT40bN05VqlTR33//zSeoAFyxbrK9MwAAwGa5mmm7atUqVatWTeHh4erUqZPuv/9+ZWRkKCYmpsATK1u2rNVzHx+fbNsuXbp03X7KlStn9bxEiRKKj4+3PC+IYxo1apRiY2Mtj+joaJv3BQAA+K+67bbbtH79eqtPOK1cuVIhISGqWrWqVeyWLVu0ZcsWq/sXZGrZsqWMMVY3L0tOTpYkq/VyAQAAgOLC5qJtenq6unfvrv79++vixYs6cOCA/vnnHzk7O2f5WNv1ZPdRt5zuDlzYCuqYPDw85OfnZ/UAAAD4r0tISNClS5eUnp6utLQ0Xbp0SZcvX7Zsf/jhhxUQEKABAwbo0KFD+vHHH/Xee+/pueeek7Oz9WXqjBkzFBERoTZt2mQZp1u3bqpcubIGDhyoo0ePas+ePRoxYoRq166tBg0aFPpxAgAAAAXN5qLtiRMndPHiRXXr1k0uLi6SpN9++03p6em5GjA4OFgnTpywPM/IyND27dtz1UdBKahjAgAAQFbh4eEKDQ3VmjVr9NNPPyk0NNTqxmB+fn76+eefde7cOTVs2FDDhw/Xyy+/rGeeecaqn5SUFP3www964oknsp0A4O7urpUrV+ry5ctq2LChOnbsqHLlyumnn35ipi0AAACKJZvXtC1btqzCw8M1evRoPfPMMzp48KCeeeaZbC+cr6dz584aO3asWrRooTJlymjatGk6dOiQXdYaK6hjAgAAQFZnz569YUyNGjW0bNmy68a4u7vr6NGj140pV66cvvnmm1zlBwAAADgqm4u2zs7O+vHHH/XKK6/o0UcfVZkyZTR16lSNHj1avr6+Oe5Xvnx5hYaGWp4PGTJECQkJGj9+vHx8fHTvvfcqICBAJUqUsMRUqFBBISEhVv2Eh4crKCjIqq1SpUpWN6K4dqzs+rl6u63HdG2/AAAAAAAAAFBYnIwxxt5J3Gzi4uLk7++vsBHfytnD297pAAAAFKrDb3Up0vEyr7ViY2O5l0Ah4PzCoa2bbO8MAJtMTyvan434bxrQMsLeKSAPbL3WsnlNWwAAAAAAAABA4aNoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQV3snAAAAAAD4j1g32d4ZAABQLDDTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAAAAAAAciKu9EwAAAABwYwkJCZoxY4Z+++03paenq1GjRhoyZIj8/Pys4rZt26b33ntPR48eVbVq1fT888+rYsWK9kkaAAAAecJMWwAAAKAYaNasmQ4dOqQ+ffro4Ycf1oIFC9S8eXNdvnzZEvPPP/+oWbNm8vT01IgRI3T69Gk1adJEJ0+etGPmAAAAyC1m2gIAAADFwJo1a6xm1TZr1kxlypTR8uXLde+990qSXn/9dUVFRWnq1KmSpE6dOqly5cqaOHGi3n77bbvkDQAACsf03w/mKn5Ay4hCygSFgZm2AAAAQDFw7TIIJUqUkLOzs5KSkixtq1at0t1332157urqqi5dumjlypVFlicAAADyj5m2AAAAQDH09ttvy9vbW23atJEkxcbG6uLFiypXrpxVXLly5XT06NEc+0lOTlZycrLleVxcXOEkDAAAAJsx0xYAAAAoZr777jv93//9nz7++GOVLl1akpSamipJ8vT0tIr18vJSSkpKjn2NGzdO/v7+lkdYWFjhJQ4AAACbULQFAAAAipEffvhBffr00YcffqjevXtb2v39/eXi4qJz585ZxZ87d06lSpXKsb9Ro0YpNjbW8oiOji603AEAAGAbirYAAABAMbF48WI98MADmjRpkp588kmrbW5ubqpVq5b+/vtvq/a//vpL9evXz7FPDw8P+fn5WT0AAABgXxRtAQAAgGJg6dKl6t69uyZOnKhBgwZlG/Poo4/qm2++0d69eyVJa9eu1erVq/Xoo48WZaoAAADIJ25EBgAAABQDDzzwgNzc3PTVV1/pq6++srQ//vjjevjhhyVJQ4YM0T///KPIyEhFRETowIEDGjVqlO6++257pQ0AAIA8oGgLAAAAFAMrVqyQMSZLe3h4uOX/Li4u+vTTTzV27FhFR0ercuXKCgwMLMo08V+wbrK9MwAA4KZH0RYAAAAoBpo3b25zbJkyZVSmTJlCzAYAAACFiTVtAQAAAAAAAMCBULQFAAAAAAAAAAdC0RYAAAAAAAAAHAhFWwAAAAAAAABwINyIDAAAAAAAALjJTf/9YK7iB7SMKKRMYAtm2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAFDfrJud932ZDCy4PAABQKJhpCwAAAAAAAAAOhKItAAAAAAAAADgQlkcAAAAAgP+S/CytAAAAigQzbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCDciAwAAAAAiho3AwMAANfBTFsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAAByIq70TAAAAAIBiad1ke2cAAABuUsy0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgXAjskK047UO8vPzs3caAAAAAAAAQKGa/vvBXMUPaBlRSJncHJhpCwAAAAAAAAAOhKItAAAAAAAAADgQlkcAAAAA8N+0brK9MwAAwGHldrkDFCxm2gIAAAAAAACAA6FoCwAAAAAAAAAOhOURAAAAAAAAABSpwl5+YUDLiELtv7Ax0xYAAAAAAAAAHAgzbQEAAAAUX9xMDAAA3ISYaQsAAAAAAAAADoSiLQAAAAAAAAA4EJZHAAAAAG4yJ0+eVHR0tCIiIhQYGGjvdAAAAIpcbm905mg3LmOmLQAAAHCTSE9P12OPPaZKlSqpX79+KleunMaMGWPvtAAAAJBLzLQFAAAAbhIffvih5s+fr23btqlatWpau3atbr/9dkVFRenuu++2d3oAAACwEUVbAAAA4CYxc+ZM9ezZU9WqVZMk3XbbbWrdurVmzpzpuEXbdZPtnQEAAIDDoWhbCIwxkqS4uDg7ZwIAAHDzybzGyrzmwhWpqanauXOnhg4datXeqFEjffnllznul5ycrOTkZMvz2NhYSUV4LZuQWDTjAEARSkyLt3cKAHKpqK59bL2WpWhbCM6dOydJCgsLs3MmAAAAN6/4+Hj5+/vbOw2HERsbq/T09Cw3HgsMDNT58+dz3G/cuHF67bXXsrRzLQsA+fG8vRMAkEvDi3i8G13LUrQtBKVKlZIkHT16lF8kHFxcXJzCwsIUHR0tPz8/e6eDHPA6FR+8VsUHr1XxwWuVlTFG8fHxKlu2rL1TcShubm6SpKSkJKv2xMREubu757jfqFGj9PTTT1ueZ2Rk6Pz58woMDJSTk1PhJPv/8f7OG85b7nHO8obzlject7zhvOUe5yxv7H3ebL2WpWhbCJydnSVJ/v7+fNEUE35+frxWxQCvU/HBa1V88FoVH7xW1vjDeFb+/v4KCAhQTEyMVXtMTIwqVKiQ434eHh7y8PCwagsICCiMFHPE+ztvOG+5xznLG85b3nDe8obzlnucs7yx53mz5VrWuQjyAAAAAFAE7rjjDi1evNjyPC0tTUuXLlW7du3smBUAAAByi6ItAAAAcJMYPXq0Nm3apIEDB2rx4sXq2bOnkpKSrJY/AAAAgOOjaFsIPDw8NGbMmCwfM4Pj4bUqHnidig9eq+KD16r44LVCbtSpU0fr1q1TYmKiJk2apODgYP35558KDQ21d2rZ4v2dN5y33OOc5Q3nLW84b3nDecs9zlneFJfz5mSMMfZOAgAAAAAAAABwBTNtAQAAAAAAAMCBULQFAAAAAAAAAAdC0RYAAAAAAAAAHIirvRO42cTFxWnv3r0KCQlRhQoV7J3Of5IxRgcPHlRycrIqV66c48LSp06d0tGjR1WpUiUFBQXlOQb5d/bsWe3evVvh4eEKCwvLsv3QoUM6f/68atSoIR8fn2z7sCUG+RMfH699+/YpPDxcgYGBWbYbY7Rr1y6lpaWpVq1acnXN+iPGlhjkT1xcnA4ePChPT09FRETI3d09S0xKSop27NghHx8fVa9ePdt+bIlB7uzfv18nT55U06ZN5eLikm3Mnj17lJCQoFq1auX486ugYoCi9u+//+r8+fNq3ry5TfHGGP37779KSUlR7dq1c/y5cqOY4uzy5cv6999/FRAQoMqVK1839siRI4qOjs52W/PmzeXk5KTLly9r8+bNWbbXqVNH/v7+BZKzIzh58qSio6MVERGR7TXL1aKjo3XkyBGrNldXVzVp0iRf/RZHu3fvVmJiomrVqpXt9cO1kpKStGfPHpUqVUrly5eXk5OT1fatW7fq0qVLVm2hoaGqUqVKgeZdFJKSkrRr1y75+vqqatWqBbZPXvotTk6fPq0jR44oPDxcISEhN4w3xujQoUNKTExU5cqV5enpabX9+PHjOnjwoFWbk5OTzT9Xiot9+/YpPj5et9xyS5ZzcK1//vlHsbGxVm1BQUGqUaNGvvotblJSUrRz5055eXlle+xXO3HihA4cOJDttltvvVWenp5KT0/X+vXrs2yvXr26goODCyRnmxgUmClTphgvLy9To0YN4+3tbbp27WouX75s77T+Uz7++GMTHh5uKleubKpXr25KlixpZsyYYRWTkZFhBg0aZDw8PMwtt9xiPDw8zPPPP5/rGBSM1NRU06xZM+Ps7GzGjh1rtS02NtbccccdpkSJEqZatWqmRIkS5osvvsh1DPInPT3djBo1ynh7e5vIyEgTHh5uRo4caRWzZ88eU6NGDRMSEmLCwsJMuXLlzLp163Idg/x5/vnnjZeXl4mMjDSVKlUypUuXNgsWLLCKWb58uQkKCjKVKlUypUqVMvXr1zfHjh3LdQxst3jxYtOqVStTsmRJI8lcuHAhS8zx48dNVFSUKVmypImIiDCBgYHmxx9/LJQYoKh98803pnHjxqZkyZLGxcXFpn327dtnbrnlFhMcHGwqVKhgypQpY9auXZvrmOLsm2++MX5+fqZq1arG19fXtGrVypw/fz7H+KlTp5rmzZtbPUqVKmUCAgJMWlqaMcaYf/75x0gyjRo1sorbvHlzUR1WoUpPTzePP/641TX8iy++eN19xo4da3x9fa3OR8eOHfPdb3Fy9OhRExkZaQIDA02lSpVMcHCwWbVqVY7x58+fN08++aQJCAgwdevWNUFBQaZBgwZm586dVnGRkZGmYsWKVuf2rbfeKuzDKXCLFi0yJUuWNFWqVDEBAQGmSZMm5tSpU/neJy/9FicjRoyw+poZMWLEdeNnzpxpKlWqZCpVqmRq1qxpAgICzNSpU61iJk2aZLy9va3eU61atSrEoyhap06dMk2aNDEBAQGmSpUqpmTJkmbRokXX3adVq1YmLCzM6py8/PLL+e63OFm1apUJDg42lSpVMoGBgSYyMtIcPXo0x/j58+dn+XlZpkwZ4+rqas6dO2eMMebChQtGkomMjLSKW758eVEdljHGGIq2BWTjxo3GycnJfP/998YYY06cOGHKly9vnn32WTtn9t/y+uuvm+joaMvzWbNmGWdnZ6sL0WnTphlfX1+zY8cOY4wxGzZsMO7u7mbu3Lm5ikHBGDVqlOnTp48pV65clqLtY489ZqpXr275BWX69OnG1dXV7NmzJ1cxyJ+XX37ZBAYGmu3btxtjrvxRY8qUKVYx9evXN3fddZfll8InnnjClC1b1iQmJuYqBnn3yy+/GEnm119/tbQ9++yzxsfHx6SmphpjjDl37pzx9/c3o0ePNsYYk5SUZJo3b27atWtn2ceWGOTOW2+9ZX755RezePHiHIu2HTt2NM2aNbN8PYwZM8b4+fmZM2fOFHgMUNTGjBlj1q1bZ2bNmmVz0bZhw4amc+fOlp8ZgwcPNqGhoSYhISFXMcXVwYMHjbu7u/noo4+MMVf+SH3LLbeYhx56yOY+UlNTTenSpc3QoUMtbZlF2xMnThR4zo7gww8/NP7+/ubff/81xhizbt064+bmZubPn5/jPmPHjjWNGzcu8H6Lk9atW5vWrVub5ORkY8yVPwKXLFky259Xxhizc+dOM3XqVEt8UlKSufvuu03NmjWt4iIjI80777xTqLkXtuPHjxtvb2/z9ttvG2OMSUhIMA0aNDD33ntvvvbJS7/FyWeffWa8vb3N1q1bjTHGbN682Xh5eZnPP/88x33efPNNc/jwYcvzr7/+2jg5OZn169db2iZNmmRq1apVeInb2T333GNuvfVWy8+xcePGGW9v7+t+z27VqpV56aWXCrzf4uLChQumZMmSlj+kJScnmxYtWpjbb789V/3UrVvX6usvs2i7cePGAs03tyjaFpBBgwaZ2rVrW7W9+uqrJigoyGRkZNgpKxhjjJeXl1WBqVGjRqZfv35WMXfeeafp0KFDrmKQf6tWrTKVKlUyFy9ezFK0TUpKMt7e3ubDDz+0tGVkZJiyZctafijZEoP8uXjxovHy8jITJ07MMWbz5s1Gkvnzzz8tbdHR0cbJyckyy9OWGOTP3LlzjZOTk9UnPBYsWGCcnJzMxYsXjTHGfPLJJ8bT09PEx8dbYhYuXGgkWf4abUsM8ianom1MTIxxcnIyCxcutLRdunTJeHl5WWaYFFQMYE+2Fm23b99uJFnNmj1+/LhxdnY28+bNszmmOHv99ddNSEiISU9Pt7RNmzbNeHh4mEuXLtnUx4IFC4wkyx9djflf0faPP/4wW7ZssfpefzNo0KCBeeyxx6zaOnbsaLp06ZLjPmPHjjVRUVFm69atZvfu3ZY/dOa33+Li4MGDRpL56aefLG0XLlwwbm5uZtasWTb3k3mtkDlLzZgrRduXXnrJ/PXXXyYmJqYg0y4yEydONH5+fpYCtTHGfPnll8bFxcWcPXs2z/vkpd/ipGXLlqZnz55Wbffff3+uZ8UGBASYCRMmWJ5PmjTJ1KhRw2zbts3s2rXLpKSkFES6DuHMmTPG2dnZfPPNN5a2xMRE4+vrayZNmpTjfq1atTJDhw41f/31l9UEtvz2W1zMnDnTuLu7m9jYWEvbkiVLjCRz6NAhm/rYuHGjkWT16bTMou33339vNm3alOMfsQobNyIrIFu2bFFUVJRVW6NGjXT27FkdO3bMTllhx44dSkxMtKybZIzRtm3bsn2ttmzZYnMM8u/MmTPq27evPv/882zXUNuzZ48uX75s9To4OTnp1ltvtbwOtsQgf9atW6fExETdddddOn78uLZs2aK4uDirmMxz3aBBA0tb+fLlVaZMGcs2W2KQP3fddZeaNWumvn37avny5Zo3b55efvllvfTSS5avsS1btqh69eoqUaKEZb9GjRpJurLunK0xKFhbt26VMcbqe5mPj49q1qxp+fooqBigOMh8v179Xi5TpozKly+f5efK9WKKsy1btqh+/fpydv7fr2uNGjVScnKydu3aZVMfM2bMUOPGjVWnTp0s2x544AH17t1bpUqV0uDBg5WSklJgudtLenq6/vnnnzxdw2/ZskW9e/dW27ZtFRoaqtmzZxdIv8VBdl9LAQEBqlq1aq6Ob+PGjSpVqpRKlixp1f7ee+9pwIABqlGjhho1amTz+9dRbNmyRXXq1LFa47dRo0ZKT0/X9u3b87xPXvotTnKqj+TmPbVv3z7FxsZmWQN5z5496tmzpzp06KDg4GBNnz69QHK2t+3btysjI8PqvHl6eqpOnTo3PG/Tp0/XgAEDVLt2bdWtW9dq7fL89FscbNmyRVWrVpWfn5+lLfN3F1uPb8aMGQoLC1OHDh2ybHviiSfUr18/hYSEqHfv3lnWDy5sN9dK/XZ0/vz5LIvRZz4/f/58tjdXQuFKTEzUI488oqZNm+qOO+6QJCUkJCg5OTnb1+r8+fM2xyB/jDF6+OGH1a9fP7Vo0SLbmMxznd3r8O+//9ocg/w5fvy4nJycNGXKFM2dO1fBwcHau3evhg0bpvHjx0u68jr4+fnJzc3Nat+rv2ZsiUH+eHl5adiwYRo2bJj+/fdfxcfHKyQkRD169LDE3Ohnla0xKFjX+1529etSEDFAQdi4caOSk5Nz3O7p6albb701z/2fP39e3t7eWW6Scu37/UYxjuTSpUs3/MNXuXLlVKlSJUlXjq9cuXJW23Pzvfj48eNatmyZPv74Y6t2Pz8/LVu2TB07dpR05RfaNm3aKCAgQG+++aath1Nkjh49qqNHj143JvMmavHx8UpNTc3198CGDRvq4MGDCg8Pl3SlyNivXz9VrVpVTZo0yXO/9rR582Zdvnw5x+1ubm5q3LixpP+9n0qVKmUVk5vj27RpkyZMmKD/+7//s7oZ2VNPPaUePXrI09NTcXFx6tGjh+69915t37692NwkMy/XRf/16620tDTFx8dne3xxcXFKT0/P8YasmVJSUtSvXz/Vr19fnTt3trTXrVtXe/futRRyp0+frieeeEKVK1dWmzZtCv5gilBer+MGDBigpUuXysfHR5cvX1bfvn3VtWtXyw3ubvbrw+y+ljK/n9lyfImJifr666/11FNPWf2h1M3NTXPmzFHv3r0lXbmhcNu2bTVs2DB9/vnnBXgE10fRtoC4ubkpKSnJqi0xMVGSbLrzJgpWSkqK7r//fl28eFG///675Ysvs2CU3WuV+TrZEoP8mTVrlrZt26bnnntOa9eulXTlNTt69Kj+/PNPNWnShNfKQbi5uckYoxMnTujo0aNydXXVunXr1KpVKzVo0EA9evTI9vuflPW1ulEM8mfp0qXq06ePfvnlF8sfQ1599VW1atVK+/btU6lSpWz6WcXPs6J39fcyLy8vS3tiYqLl7rQFFQMUhHHjxun06dM5bi9TpozmzZuX5/7d3NyUnJwsY4xVAejanys3inEkMTExeuGFF64b88ADD2jYsGGS8v+9+PPPP5e3t7fVH+4kqUKFCqpQoYLlef369fXEE0/om2++ccii7fLly2/4y/HkyZNVv379PF8XXjuzasSIEZo2bZrmzZtn8zWpo5kwYYKOHDmS4/aAgAAtWbJE0v9+diQnJ2f52WHL8f3777/q0qWLHnzwQY0YMcJqW9++fS3/9/Pz0zvvvKM6depo8+bNatq0aW4OyW7c3NyyFMBv9LVoyz556be4cHFxkbOzc7ZfM87Ozjcs2Kalpalnz546fvy4fv/9d7m6/q9sdW1hdsCAAfr44481d+7cYl+0vd73mqtnkV6rT58+lv97e3tr0qRJCgsL09q1a9WpU6c891tcZPfzMvO5LV9L3333neLj4/Xoo49atfv4+FgKtpJUpUoVPfvss3ruuec0a9YsqwJvYaJoW0DCw8MVExNj1RYTEyMnJydm2RaxzILtnj17tHr1apUpU8ayzcPDQ6VLl872tcq8gLUlBvkXERGhV155xfI8NjZWy5cvV3R0tJYtW2aZ7RATE2P1sb6YmBjLNltikD8VK1aUJPXv399ywdSsWTPVrVtXa9asUY8ePRQeHq6UlBSdPXtWQUFBkq58lPDUqVOWrxlbYpA/S5YsUd26da1mrw8ePFivvfaa1q5dq7vvvlvh4eFas2aN1X6Z3+uufq1uFIOCdfX3sqs/VhoTE6PbbrutQGOAgvD9998Xav/h4eGWnxGhoaGSpIyMDJ08edLqe9WNYhxJ9erVLX+otkV4eLh27Nhh1Zab78UzZ85Ur169rJa6yUl2172OYsCAARowYIBNsT4+PgoMDCyQa/irz0lB9ltU5syZY3Ps1T87rv4Y+vHjx3XnnXded9/du3erTZs26tKliz755BOrP6Bkp3Tp0paxiovw8HCtWrXKqu1GX4u27JOXfouLzBpIXr5m0tLS1KtXL23evFmrV6+2qZbiyN/DcuPqr8WraxgxMTGqXbu2zf2EhITIycnJck4Kql9HFR4erpUrV1q15eZracaMGerYsaPN77XExESdP3/e8jttYWNN2wLSrl07/fLLL0pISLC0LVq0SE2aNLHpYgkFIzU1Vd27d9euXbu0evVqlS9fPktMu3bttHjxYsvz9PR0LVmyRO3atctVDPLu0Ucf1dq1a60ewcHBGjBggJYtWybpynqnNWrU0A8//GDZ7/Tp01q/fr3ldbAlBvnTuHFjBQQEWF0IpaWl6dSpU5aZey1btpS7u7vV6/DLL78oPj7e8jrYEoP8CQ4O1qlTp5SWlmZpi46OtmyTrnxvO3LkiNVaaYsWLVJAQIAaNmxocwwKVlRUlEqVKmX19fHPP//o0KFDlq+PgooBHNWmTZt06NAhSVKLFi3k4eFh9V7+7bffdPHiRct72ZaY4qxdu3batGmTTpw4YWlbtGiRKlWqpMqVK0u6sqTX2rVrs6yvt3r1au3fvz/bYufVv6tkWrly5U3xi7uU/TX80qVLrd4TmZ/synTtOTlz5oy2bt1qdU5s6be4aty4sXx9fa2+ljZu3Kjjx49bHd+WLVu0f/9+y/M9e/bo9ttvV8eOHfXpp59mKdhevnxZxhirthUrVkiSatWqVRiHUijatWunnTt36sCBA5a2RYsWKTQ01DJpJCkpSWvXrtWFCxds3seWmOKsXbt2WrJkieU9YIzR4sWLrd5TMTEx+uOPPyzP09PT1bt3b23cuFGrV6+2TB652rVfr7Gxsfrrr79uiu9hderUUenSpa2+Fvfu3at///3X6rz9888/2r17t6Qr77309HSrflauXCljjOWc2NpvcdWuXTvFxMRYreO7aNEi+fn5WZaBSUtL09q1a3XmzBmrfffv36/ff//d5p+XK1asUGhoaJblGAqVXW5/dhO6dOmSqVq1qmnbtq1ZuHChGTVqlHF1dTW//PKLvVP7T+nevbvx8fExX3/9tVmzZo3lceTIEUvM7t27ja+vr+nfv7/54YcfTM+ePU1gYKDVXdFtiUHBKleunBk7dqxV24IFC4yLi4sZO3asWbBggWnatKmJjIy0ukuoLTHInylTppjSpUubTz/91Cxbtsx0797dBAYGWt2d9JVXXjEBAQFm+vTpZs6cOaZ8+fLm4YcfturHlhjk3YEDB4yvr6+57777zNKlS83XX39tatasaRo3bmx1J+xOnTqZGjVqmG+//dZMnjzZeHp6mg8++MCqL1tiYLuDBw+aNWvWmPHjxxtJZtmyZWbNmjVWd9ieMmWK8fT0NO+//7759ttvTc2aNU379u2t+imoGKCo7dmzx6xZs8a8+OKLxsXFxXJ9dvWdnsPDw83w4cMtz1977TXj5+dnPvnkE/PVV1+ZChUqmN69e1v1a0tMcZWWlmYaNmxoGjZsaL7//nszbtw44+rqaubOnWuJ2bJli5Fkfv31V6t9H3zwQVOvXr1s+x05cqR5/PHHzbfffmsWLVpkevXqZdzd3c2KFSsK83CKzM6dO42Pj495/PHHzQ8//GC6d+9ugoODzbFjxywxY8aMMf7+/pbnDRs2NG+++ab58ccfzRdffGFq165tqlSpYvU92pZ+i7MJEyYYb29vM2XKFDN37lxTtWpVc/fdd1vF1KpVy/Tv398YY8zRo0dN2bJlza233mp+++03q9+7EhISjDHG/PXXX6Zp06bmk08+McuXLzfjxo0zvr6+ZsCAAUV+fPmRkZFhWrVqZSIjI813331nJkyYYNzc3MyMGTMsMfv27TOSzOLFi23ex5aY4uzAgQMmICDAPPzww+aHH34wDz30kAkICDAHDhywxLzzzjvGxcXF8vyhhx4ynp6eZvbs2VbvqUOHDlliWrZsaV577TWzdOlSM2fOHNOgQQNToUIFc+LEiaI8vEIzY8YM4+7ubiZOnGi+++47U6dOHdOqVSuTkZFhiWnevLnp1q2bMeZK3SIqKspMnTrVLF++3EyYMMGUKlXKdO/ePdf9Fmd33XWXqVatmpk7d66ZMmWK8fLyMhMnTrRsP3PmjJFkZs+ebbXfCy+8YEJDQ61+V8o0YcIE07NnTzNnzhyzdOlSM3DgQOPq6mq++OKLQj+eqzkZc82fv5Bnp0+f1ltvvaVt27YpJCREgwYNyvEmSygc7du3z3bR/b59+1r99WTnzp2WtZ4qV66s5557LstdKW2JQcG57777dM899+jhhx+2al+5cqWmT5+u8+fP69Zbb9Xzzz+f5a60tsQgfxYsWKAvv/xSCQkJqlWrlp5++mmrG6QYY/TZZ59p/vz5SktLU4cOHTRkyBCrG4/ZEoP8OXjwoCZPnqzdu3fL09NTTZo00eDBg60+8ZGYmKiJEyfqt99+k7e3t3r16pVlzUNbYmC7Dz74QN9++22W9nHjxlldJ8ybN09z5szR5cuX1bJlSz399NPy9va22qegYoCiNHbsWC1fvjxL+9SpUy0zyrp3764WLVpY1nSVZPmZkZKSovbt22vo0KFZ1qezJaa4io2N1dtvv60NGzYoICBA/fv3V6dOnSzb9+/fr379+lnWdJWufOqsS5cu6tevn9VafJkyMjL01VdfacmSJYqPj1f16tU1ZMgQRUREFNlxFbZ//vlHEydO1NGjR1W1alU999xzVsc3c+ZMzZs3z/LprgsXLujDDz/Uhg0b5O3trUaNGmnQoEFZvm/eqN/i7uuvv9Y333yjxMRE3X777XrqqaesbvT38MMPq1atWnr++ee1bt06Pffcc9n2M3v2bMsN9bZt26ZPPvlE+/btU7ly5XTvvffq7rvvLpLjKUgJCQl655139Mcff8jX11cPP/yw7rnnHsv2mJgY9ejRQ2+//baaNWtm0z62xhRnu3fv1rvvvquDBw8qIiJCI0eOVI0aNSzbv/nmG02bNk2rV6+WJN155526ePFiln569eqlwYMHS5Li4uI0ZcoUrVu3Th4eHoqKitKQIUPk6+tbFIdUJBYuXKgvvvhC8fHxat68uZ599ln5+PhYtg8aNEiBgYEaO3aspCvneerUqdq9e7dCQ0PVuXPnbK/bb9RvcZaUlKRJkybp119/lZeXl3r27KlevXpZtsfGxqpLly4aPXq02rdvb2l/4IEH1KxZsyzrcWdatGiRvvvuO509e1aVK1fW448/rrp16xb24VihaAsAAAAAAAAADoQ1bQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAFBs/fHHH9q8ebO90wCAAkXRFgCKsZ07d+rpp59Wu3bt1KFDBw0ZMkQbNmywSy6//vqr2rVrZ5exAQAAUPysX79ev/32W777mTRpkmbOnFkAGQGA46BoCwDF1LRp09SgQQMlJSVpxIgRevbZZ1WrVi0988wzmjt3bpHnc+7cOW3cuLHIxwUAAEDx9P7772vcuHH2TgMAHBJFWwAohjZv3qzBgwfr3Xff1UcffaQuXbrojjvu0MCBA7V27Vp16dLFEnvgwAE98cQTatWqlf5fe/cWElXXx3H8N1paVApGlieq8UBpE9JBpzC6sBQqRdDqQgrtQOlFBJWWBdlFYdRFBYpJ3oSE1SRDlJBY3nSOGsXAinJMLY3MzDMV+lyEm3feTk+91TM+7/dztfea/16zlhfD4ufea6elpamystKlr7Nnzyo9Pd2l7fbt24qLizPOr1y5opUrV+rOnTvKzMzUsmXLlJOTo56eHmM8eXl56u3tldVqldVqVVFR0W/8CwAAAODf5urVq3r48KG6urpUU1OjmpoaDQwMfFbX3d2ty5cv6/r16+rr6/tiX0NDQ7p7967sdrvq6+tdPnv48KFsNpuGh4eNtrdv36q8vFzt7e2/dlIA8JPG/NMDAAD8uOLiYgUEBCgrK+uLn0+cOFGS9OrVK8XExCghIUF5eXlyOBxKSUlRWVmZ1qxZI0lqa2tTXV2dy/VdXV26ffu2cf769WtVVVWpra1NeXl5GjdunHJzc9Xc3Kzy8nKFhYUpMzNThw4d0rFjxyRJQUFBv2HmAAAA+Lfav3+/PDw89PLlS82aNUuPHz+WJN25c0d+fn6SpLq6Oi1fvlz+/v6aNm2ampqaNH78eE2bNs3o58WLF0pOTlZvb69mzZolh8OhqKgoVVRUaPz48fLz89PWrVvldDq1a9cuSdKmTZvU2tqqtLS0Pz9xAPgCQlsAGIVqa2s1Z84cjRnz7Z/xw4cPKyAgQGfOnJHJZFJiYqK6urq0e/duI7T9uz5+/CibzSaz2SxJ+vDhg9LT0zU8PCwfHx+Fh4fL09NTVqv1p+cFAACA/2/Pnj2Tw+GQv7+/3r9/r8jISJ08eVJ79uyRJGVnZys+Pt5Y3166dElJSUlaunSp0cf69esVGxurwsJCmUwmDQ4OasmSJTp8+LDy8/MVGBioU6dOae3atYqPj9f9+/dVVVUlh8Px3fU1APwpbI8AAKPQ4OCgfHx8vlt37949JSYmymQyGW0rVqyQ0+lUZ2fnD32nn5+fEdhKUnBwsAYGBtTd3f1D/QAAAABfk5qaKn9/f0mSl5eXFi1aZNxx29raqps3b2rnzp3G+nbVqlWKjIw0rm9ubta1a9cUGhqqCxcu6Pz587p48aLMZrNqamqMupSUFGVkZGj16tXavn27jh8/rrCwsD84UwD4Nv6FBACjUEhIiJxO53frenp6jK0SRoyc9/T0GI+Z/R1jx451OR9ZKP/nXmAAAADA/+K/16fe3t7q7e2V9CmQlaQZM2a41MycOdM4bmpqkiTduHHD5SW5JpNJCxYscLkuPz9fpaWlioqK0oYNG37VFADglyC0BYBRKDk5WdnZ2aqvr5fFYvlqXWhoqBoaGlzaGhoa5OXlpeDgYEnShAkT1N/f71LzMy9g8PDwIMAFAADAbzN58mRJn14aNnI8cj4S5I48jbZv3z7Nmzfvm/3l5OTIbDaroaFBdrtdKSkpv2XcAPAz2B4BAEahjIwMzZ8/X+vWrdOTJ0+M9qGhIdlsNlVXV0uSNm7cKLvdrlu3bkmS3rx5o4KCAmVkZMjT01OSZLFY9Pz5cz148ECS1NnZqRMnTvzwmKZOnare3l62SwAAAMBvYTabFRAQILvdbrS1tLS43FFrsVgUFBSk4uLiz65/+fKlcVxeXi6bzSa73a4DBw5o8+bNamtr+63jB4AfQWgLAKOQt7e3qqurFRMTo+joaEVERMhiscjX11c2m03h4eGSPu1fu2fPHsXHx2v27NmaPn26AgICVFBQYPRltVq1ZcsWLV68WHPnzpXFYlF0dPQPjyk2NlYxMTGKiIiQ1WpVUVHRr5ouAAAAoLFjx+rgwYPau3ev8vLyVFhYqISEBE2aNMmo8fT0VGlpqcrKypSamqqSkhIVFBQoLi5O586dk/Rpm4WsrCwdOXJEkZGRys3NVVRUlDIyMnhyDIDbMA3ziwQAo9rg4KCePn0qDw8Pmc1mjRs37rOad+/e6dmzZ5oyZYpCQkK+2E97e7s6OzsVHh6u/v5+PXr0SLGxsZKkjo4ONTc3uzxi1tfXp/r6ei1cuNC4a3d4eFiNjY3q6OhQYGDgV78LAAAAOHHihPr7+7V7925Jn/aYnTNnjtLS0oyakpISDQ4Oatu2bUZbZWWlKioq5Ovrq6SkJNXW1srX11eZmZlGjdPp1OnTp9XY2Kjg4GAlJycba9vCwkI5nU4dPXrUqG9paVFOTo527Njx2d63APBPILQFAAAAAAAAADfC9ggAAAAAAAAA4EYIbQEAAAAAAADAjRDaAgAAAAAAAIAbIbQFAAAAAAAAADdCaAsAAAAAAAAAboTQFgAAAAAAAADcCKEtAAAAAAAAALgRQlsAAAAAAAAAcCOEtgAAAAAAAADgRghtAQAAAAAAAMCNENoCAAAAAAAAgBshtAUAAAAAAAAAN/IXkFtq+A7H1LAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "{'per_year': {'counts': record_num\n", + " 2023 374\n", + " 2024 718\n", + " 2025 807\n", + " 2026 141\n", + " Name: count, dtype: int64,\n", + " 'total': 2040},\n", + " 'mammaprint_overall': {'series': 1 -0.158\n", + " 2 0.058\n", + " 3 -0.039\n", + " 4 0.065\n", + " 5 -0.429\n", + " ... \n", + " 2304 -0.218\n", + " 2306 -0.141\n", + " 2307 0.192\n", + " 2308 0.396\n", + " 2309 -0.065\n", + " Name: mammaprint_index, Length: 2040, dtype: float64,\n", + " 'mean': -0.007955049019607845,\n", + " 'variance': 0.09011929758116484,\n", + " 'count': 2040},\n", + " 'mammaprint_by_type': {'a luminal': {'series': 2 0.058\n", + " 4 0.065\n", + " 6 0.451\n", + " 7 0.073\n", + " 11 0.090\n", + " ... \n", + " 2300 0.169\n", + " 2302 0.091\n", + " 2303 0.235\n", + " 2307 0.192\n", + " 2308 0.396\n", + " Name: mammaprint_index, Length: 1076, dtype: float64,\n", + " 'mean': 0.22131849442379178,\n", + " 'variance': 0.02272628757391718,\n", + " 'count': 1076},\n", + " 'b luminal': {'series': 1 -0.158\n", + " 3 -0.039\n", + " 5 -0.429\n", + " 8 -0.052\n", + " 9 -0.259\n", + " ... \n", + " 2299 -0.241\n", + " 2301 -0.391\n", + " 2304 -0.218\n", + " 2306 -0.141\n", + " 2309 -0.065\n", + " Name: mammaprint_index, Length: 964, dtype: float64,\n", + " 'mean': -0.26386618257261407,\n", + " 'variance': 0.04115081800368833,\n", + " 'count': 964}},\n", + " 'counts_by_type': {'counts': type\n", + " a luminal 1076\n", + " b luminal 964\n", + " Name: count, dtype: int64,\n", + " 'total_distinct': 2}}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "run_all_stats_and_plot(final_df, \"stats\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 1b97dd7aa18cea8d98be7bb6c91efb3afa68a726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 22 Apr 2026 16:04:14 +0200 Subject: [PATCH 32/79] feat: diff script --- scripts/notebooks/diff_script.ipynb | 128 ++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 scripts/notebooks/diff_script.ipynb diff --git a/scripts/notebooks/diff_script.ipynb b/scripts/notebooks/diff_script.ipynb new file mode 100644 index 00000000..f58472a2 --- /dev/null +++ b/scripts/notebooks/diff_script.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "d06d8e25-a2e4-47d0-8322-040882d4bd58", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmp63gnbhm0\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (3.0.1)\n", + "Requirement already satisfied: numpy in /opt/venv/lib/python3.14/site-packages (2.4.3)\n", + "Requirement already satisfied: openpyxl in /opt/venv/lib/python3.14/site-packages (3.1.5)\n", + "Requirement already satisfied: matplotlib in /opt/venv/lib/python3.14/site-packages (3.10.8)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.14/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: et-xmlfile in /opt/venv/lib/python3.14/site-packages (from openpyxl) (2.0.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (4.62.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.5.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (25.0)\n", + "Requirement already satisfied: pillow>=8 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (12.1.1)\n", + "Requirement already satisfied: pyparsing>=3 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (3.3.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/venv/lib/python3.14/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas numpy openpyxl matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0b50df43-0b03-497c-99ce-227d656fd14c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved 160 full rows (B minus A on 'record_num') to /mnt/projects/mammaprint/data_mapping_diff2.csv\n", + "SUCCESS: A is a subset of B based on key 'record_num'.\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import sys\n", + "\n", + "# ---- Hard-coded file paths ----\n", + "FILE_A = \"/mnt/projects/mammaprint/data_mapping.csv\"\n", + "FILE_B = \"/mnt/projects/mammaprint/data_mapping_new.csv\"\n", + "\n", + "# Output for rows in B but not in A (full rows, not just keys)\n", + "ONLY_IN_B = \"/mnt/projects/mammaprint/data_mapping_diff2.csv\"\n", + "\n", + "KEY = \"record_num\"\n", + "\n", + "\n", + "def main():\n", + " # Load CSVs\n", + " df_a = pd.read_csv(FILE_A)\n", + " df_b = pd.read_csv(FILE_B)\n", + "\n", + " # Ensure key column exists\n", + " if KEY not in df_a.columns or KEY not in df_b.columns:\n", + " print(f\"ERROR: '{KEY}' column is missing in one of the files.\")\n", + " sys.exit(1)\n", + "\n", + " # ---- Compare keys ----\n", + " keys_a = df_a[KEY].drop_duplicates()\n", + " keys_b = df_b[KEY].drop_duplicates()\n", + "\n", + " # Keys in A not found in B\n", + " a_not_in_b = keys_a[~keys_a.isin(keys_b)]\n", + "\n", + " # ---- FULL ROWS from B that do not exist in A (keep all columns) ----\n", + " b_not_in_a_full = df_b[~df_b[KEY].isin(keys_a)]\n", + " b_not_in_a_full.to_csv(ONLY_IN_B, index=False)\n", + "\n", + " print(f\"Saved {len(b_not_in_a_full)} full rows (B minus A on '{KEY}') to {ONLY_IN_B}\")\n", + "\n", + " # Check subset condition\n", + " if not a_not_in_b.empty:\n", + " print(f\"ERROR: A is NOT a subset of B based on key '{KEY}'.\")\n", + " print(f\"Keys in A but missing in B: {len(a_not_in_b)}\")\n", + " print(a_not_in_b)\n", + " sys.exit(1)\n", + "\n", + " print(f\"SUCCESS: A is a subset of B based on key '{KEY}'.\")\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 0f4f1f838f121232e7dfec47f9c87742ec6d17d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 6 Jul 2026 15:14:03 +0200 Subject: [PATCH 33/79] squash data exploration --- scripts/notebooks/data_exploration_script.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/notebooks/data_exploration_script.ipynb b/scripts/notebooks/data_exploration_script.ipynb index 663f99c3..2080b1ac 100644 --- a/scripts/notebooks/data_exploration_script.ipynb +++ b/scripts/notebooks/data_exploration_script.ipynb @@ -1538,7 +1538,7 @@ "id": "2e052b41-926d-46b1-871f-1f4747010cb4", "metadata": {}, "outputs": [], - "source": "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\nFILESYSTEM_LISTING = DATA_PATH\nprint(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n\n\ndef apply_files_mapping(name):\n \"\"\"Rename raw slide entries via FILES_MAPPING before format checks so remapped\n non-standard names (e.g. two-digit ranks) pass the single-digit FILENAME_PATTERN.\"\"\"\n p = Path(name)\n stem = p.stem if p.suffix == \".mrxs\" else name\n if stem in FILES_MAPPING:\n return FILES_MAPPING[stem] + p.suffix\n return name\n\n\nlisted_entries = [\n apply_files_mapping(Path(line.strip()).name)\n for line in os.listdir(FILESYSTEM_LISTING)\n if line.strip()\n]\n\nall_folders = {\n entry\n for entry in listed_entries\n if Path(entry).suffix == \"\" and entry != \"annotations\" and FILENAME_PATTERN.fullmatch(entry)\n}\nall_files = {\n Path(entry).stem\n for entry in listed_entries\n if Path(entry).suffix == \".mrxs\" and FILENAME_PATTERN.fullmatch(entry[:-5])\n}\n\nprint(f\"Found {len(all_folders)} folders\")\nprint(f\"Found {len(all_files)} .mrxs files\")" + "source": "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\nFILESYSTEM_LISTING = DATA_PATH\nprint(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n\nlisted_entries = [\n Path(line.strip()).name\n for line in os.listdir(FILESYSTEM_LISTING)\n if line.strip()\n]\n\n\ndef is_known_filename(stem):\n \"\"\"Accept a slide stem if it matches the standard pattern OR is a FILES_MAPPING key.\n\n The original filename is kept as-is; the FILES_MAPPING alias is only applied later\n at lookup time (in normalize_filename). This just prevents non-standard names\n (e.g. two-digit ranks like P2023_03525-35) from being dropped by the format filter.\n \"\"\"\n return FILENAME_PATTERN.fullmatch(stem) is not None or stem in FILES_MAPPING\n\n\nall_folders = {\n entry\n for entry in listed_entries\n if Path(entry).suffix == \"\" and entry != \"annotations\" and is_known_filename(entry)\n}\nall_files = {\n Path(entry).stem\n for entry in listed_entries\n if Path(entry).suffix == \".mrxs\" and is_known_filename(entry[:-5])\n}\n\nprint(f\"Found {len(all_folders)} folders\")\nprint(f\"Found {len(all_files)} .mrxs files\")" }, { "cell_type": "code", From 648b12f95e9b5609b92491ce64695e0fd09d09a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 6 Jul 2026 15:35:34 +0200 Subject: [PATCH 34/79] data exploration: split --- .../notebooks/data_exploration_script.ipynb | 113 +++++++++++++++--- 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/scripts/notebooks/data_exploration_script.ipynb b/scripts/notebooks/data_exploration_script.ipynb index 2080b1ac..6dd4a20b 100644 --- a/scripts/notebooks/data_exploration_script.ipynb +++ b/scripts/notebooks/data_exploration_script.ipynb @@ -1538,7 +1538,42 @@ "id": "2e052b41-926d-46b1-871f-1f4747010cb4", "metadata": {}, "outputs": [], - "source": "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\nFILESYSTEM_LISTING = DATA_PATH\nprint(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n\nlisted_entries = [\n Path(line.strip()).name\n for line in os.listdir(FILESYSTEM_LISTING)\n if line.strip()\n]\n\n\ndef is_known_filename(stem):\n \"\"\"Accept a slide stem if it matches the standard pattern OR is a FILES_MAPPING key.\n\n The original filename is kept as-is; the FILES_MAPPING alias is only applied later\n at lookup time (in normalize_filename). This just prevents non-standard names\n (e.g. two-digit ranks like P2023_03525-35) from being dropped by the format filter.\n \"\"\"\n return FILENAME_PATTERN.fullmatch(stem) is not None or stem in FILES_MAPPING\n\n\nall_folders = {\n entry\n for entry in listed_entries\n if Path(entry).suffix == \"\" and entry != \"annotations\" and is_known_filename(entry)\n}\nall_files = {\n Path(entry).stem\n for entry in listed_entries\n if Path(entry).suffix == \".mrxs\" and is_known_filename(entry[:-5])\n}\n\nprint(f\"Found {len(all_folders)} folders\")\nprint(f\"Found {len(all_files)} .mrxs files\")" + "source": [ + "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\n", + "FILESYSTEM_LISTING = DATA_PATH\n", + "print(f\"Reading filesystem listing from: {FILESYSTEM_LISTING}\")\n", + "\n", + "listed_entries = [\n", + " Path(line.strip()).name\n", + " for line in os.listdir(FILESYSTEM_LISTING)\n", + " if line.strip()\n", + "]\n", + "\n", + "\n", + "def is_known_filename(stem):\n", + " \"\"\"Accept a slide stem if it matches the standard pattern OR is a FILES_MAPPING key.\n", + "\n", + " The original filename is kept as-is; the FILES_MAPPING alias is only applied later\n", + " at lookup time (in normalize_filename). This just prevents non-standard names\n", + " (e.g. two-digit ranks like P2023_03525-35) from being dropped by the format filter.\n", + " \"\"\"\n", + " return FILENAME_PATTERN.fullmatch(stem) is not None or stem in FILES_MAPPING\n", + "\n", + "\n", + "all_folders = {\n", + " entry\n", + " for entry in listed_entries\n", + " if Path(entry).suffix == \"\" and entry != \"annotations\" and is_known_filename(entry)\n", + "}\n", + "all_files = {\n", + " Path(entry).stem\n", + " for entry in listed_entries\n", + " if Path(entry).suffix == \".mrxs\" and is_known_filename(entry[:-5])\n", + "}\n", + "\n", + "print(f\"Found {len(all_folders)} folders\")\n", + "print(f\"Found {len(all_files)} .mrxs files\")" + ] }, { "cell_type": "code", @@ -2393,6 +2428,66 @@ " print(f\"Remaining: {len(final_df)} rows\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a235c852", + "metadata": {}, + "outputs": [], + "source": [ + "# Assign a train/val/test split.\n", + "#\n", + "# The split is decided here and written into data_mapping.csv as a `split` column,\n", + "# so it propagates through tiling -> embeddings (each stage filters by split) and\n", + "# ends up as three separate embedding artifacts the training datamodule consumes.\n", + "#\n", + "# Split at the record_num (slide) level, not per row: a record can expand to several\n", + "# `path` rows (multi-file slides), and all of them must land in the same split to\n", + "# avoid slide-level leakage. Stratify on `type` and seed for reproducibility.\n", + "import numpy as np\n", + "\n", + "SPLIT_SEED = 42\n", + "SPLIT_RATIOS = {\"train\": 0.7, \"val\": 0.15, \"test\": 0.15}\n", + "\n", + "\n", + "def assign_split(df, ratios=SPLIT_RATIOS, seed=SPLIT_SEED):\n", + " \"\"\"Assign train/val/test per record_num, stratified on type, deterministically.\"\"\"\n", + " assert abs(sum(ratios.values()) - 1.0) < 1e-9, \"ratios must sum to 1\"\n", + " rng = np.random.default_rng(seed)\n", + "\n", + " # One split decision per record_num; use each record's (first) type for strata.\n", + " record_type = df.groupby(\"record_num\")[\"type\"].first()\n", + " split_of_record = {}\n", + "\n", + " for _type, records in record_type.groupby(record_type):\n", + " ids = records.index.to_numpy()\n", + " rng.shuffle(ids) # seeded: reproducible per class\n", + " n = len(ids)\n", + " n_train = int(n * ratios[\"train\"])\n", + " n_val = int(n * ratios[\"val\"])\n", + " for rid in ids[:n_train]:\n", + " split_of_record[rid] = \"train\"\n", + " for rid in ids[n_train : n_train + n_val]:\n", + " split_of_record[rid] = \"val\"\n", + " for rid in ids[n_train + n_val :]:\n", + " split_of_record[rid] = \"test\"\n", + "\n", + " df = df.copy()\n", + " df[\"split\"] = df[\"record_num\"].map(split_of_record)\n", + " return df\n", + "\n", + "\n", + "final_df = assign_split(final_df)\n", + "\n", + "# Sanity: no record_num spans multiple splits, and per-split type balance.\n", + "_per_record_splits = final_df.groupby(\"record_num\")[\"split\"].nunique()\n", + "assert (_per_record_splits == 1).all(), \"a record_num leaked across splits\"\n", + "print(\"Split counts (rows):\")\n", + "print(final_df[\"split\"].value_counts())\n", + "print(\"\\nSplit x type (rows):\")\n", + "print(final_df.groupby([\"split\", \"type\"]).size().unstack(fill_value=0))\n" + ] + }, { "cell_type": "code", "execution_count": 26, @@ -2507,20 +2602,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "9b0bc82b-0738-4244-83da-5cbee0c2e1fe", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "✅ Saved final dataset to: data_mapping_final.csv\n", - " 2040 rows × 4 columns\n" - ] - } - ], + "outputs": [], "source": [ "# Save to CSV\n", "output_file = \"data_mapping_final.csv\"\n", @@ -2860,4 +2945,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} From 2993ee1903bf52585fd2faf275d30b10003f2fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 6 Jul 2026 15:36:23 +0200 Subject: [PATCH 35/79] data exloration script results --- .../notebooks/data_exploration_script.ipynb | 242 +++++++++++------- 1 file changed, 154 insertions(+), 88 deletions(-) diff --git a/scripts/notebooks/data_exploration_script.ipynb b/scripts/notebooks/data_exploration_script.ipynb index 6dd4a20b..78c79e57 100644 --- a/scripts/notebooks/data_exploration_script.ipynb +++ b/scripts/notebooks/data_exploration_script.ipynb @@ -18,13 +18,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Looking in links: /tmp/tmp2ekpckmj\n", + "Looking in links: /tmp/tmpoutm06dy\n", "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", - "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (3.0.3)\n", + "Requirement already satisfied: pandas in /opt/venv/lib/python3.14/site-packages (2.3.3)\n", "Requirement already satisfied: numpy in /opt/venv/lib/python3.14/site-packages (2.4.6)\n", "Requirement already satisfied: openpyxl in /opt/venv/lib/python3.14/site-packages (3.1.5)\n", "Requirement already satisfied: matplotlib in /opt/venv/lib/python3.14/site-packages (3.11.0)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.14/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/venv/lib/python3.14/site-packages (from pandas) (2026.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/venv/lib/python3.14/site-packages (from pandas) (2025.2)\n", "Requirement already satisfied: et-xmlfile in /opt/venv/lib/python3.14/site-packages (from openpyxl) (2.0.0)\n", "Requirement already satisfied: contourpy>=1.0.1 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (1.3.3)\n", "Requirement already satisfied: cycler>=0.10 in /opt/venv/lib/python3.14/site-packages (from matplotlib) (0.12.1)\n", @@ -108,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "9751dfcb-7255-440c-92e5-33a508226e4b", "metadata": {}, "outputs": [], @@ -1534,10 +1536,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "2e052b41-926d-46b1-871f-1f4747010cb4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading filesystem listing from: /mnt/data/MOU/breast/mammaprint\n", + "Found 2124 folders\n", + "Found 2124 .mrxs files\n" + ] + } + ], "source": [ "# FILESYSTEM_LISTING = Path(\"/Users/mickiss/code/rationai/mammaprint-b/subory.txt\")\n", "FILESYSTEM_LISTING = DATA_PATH\n", @@ -1809,8 +1821,8 @@ "text": [ "\n", "=== Filesystem Summary ===\n", - "OK: 2078 rows (95.2%)\n", - "File missing: 79 rows (3.6%)\n", + "OK: 2080 rows (95.3%)\n", + "File missing: 77 rows (3.5%)\n", "Multiple files: 19 rows (0.9%)\n", "entry missing: 6 rows (0.3%)\n", "Total: 2182 rows\n", @@ -1938,20 +1950,6 @@ " File missing\n", " \n", " \n", - " 290\n", - " 2023/03525-1\n", - " []\n", - " 0\n", - " File missing\n", - " \n", - " \n", - " 291\n", - " 2023/03525-2\n", - " []\n", - " 0\n", - " File missing\n", - " \n", - " \n", " 508\n", " 2024/00674\n", " []\n", @@ -1986,6 +1984,20 @@ " 0\n", " File missing\n", " \n", + " \n", + " 699\n", + " 2024/01997\n", + " []\n", + " 0\n", + " File missing\n", + " \n", + " \n", + " 739\n", + " 2024/02324-2\n", + " []\n", + " 0\n", + " File missing\n", + " \n", " \n", "\n", "" @@ -2005,13 +2017,13 @@ "215 2023/02703 [] 0 File missing\n", "216 2023/02704 [] 0 File missing\n", "219 2023/02720 [] 0 File missing\n", - "290 2023/03525-1 [] 0 File missing\n", - "291 2023/03525-2 [] 0 File missing\n", "508 2024/00674 [] 0 File missing\n", "529 2024/00816 [] 0 File missing\n", "552 2024/00988 [] 0 File missing\n", "668 2024/01799 [] 0 File missing\n", - "690 2024/01928 [] 0 File missing" + "690 2024/01928 [] 0 File missing\n", + "699 2024/01997 [] 0 File missing\n", + "739 2024/02324-2 [] 0 File missing" ] }, "metadata": {}, @@ -2065,14 +2077,14 @@ " \n", " 79\n", " 2023/01486\n", - " [P2023_01486-2, P2023_01486-1]\n", + " [P2023_01486-1, P2023_01486-2]\n", " 2\n", " Multiple files\n", " \n", " \n", " 84\n", " 2023/01591\n", - " [P2023_01591-1, P2023_01591-2]\n", + " [P2023_01591-2, P2023_01591-1]\n", " 2\n", " Multiple files\n", " \n", @@ -2086,7 +2098,7 @@ " \n", " 88\n", " 2023/01599\n", - " [P2023_01599-1, P2023_01599-2]\n", + " [P2023_01599-2, P2023_01599-1]\n", " 2\n", " Multiple files\n", " \n", @@ -2100,21 +2112,21 @@ " \n", " 175\n", " 2023/02395\n", - " [P2023_02395-1, P2023_02395-2]\n", + " [P2023_02395-2, P2023_02395-1]\n", " 2\n", " Multiple files\n", " \n", " \n", " 180\n", " 2023/02444\n", - " [P2023_02444-1, P2023_02444-2]\n", + " [P2023_02444-2, P2023_02444-1]\n", " 2\n", " Multiple files\n", " \n", " \n", " 200\n", " 2023/02617\n", - " [P2023_02617-2, P2023_02617-1]\n", + " [P2023_02617-1, P2023_02617-2]\n", " 2\n", " Multiple files\n", " \n", @@ -2149,7 +2161,7 @@ " \n", " 833\n", " 2024/02851\n", - " [P2024_02851-2, 2024_02851-1]\n", + " [2024_02851-1, P2024_02851-2]\n", " 2\n", " Multiple files\n", " \n", @@ -2163,14 +2175,14 @@ " \n", " 1523\n", " 2025/02993\n", - " [P2025_02993-1, P2025_02993-2]\n", + " [P2025_02993-2, P2025_02993-1]\n", " 2\n", " Multiple files\n", " \n", " \n", " 1527\n", " 2025/03031\n", - " [P2025_03031-1, P2025_03031-2]\n", + " [P2025_03031-2, P2025_03031-1]\n", " 2\n", " Multiple files\n", " \n", @@ -2195,22 +2207,22 @@ "text/plain": [ " record_num matched_files file_count file_issue\n", "73 2023/01462 [P2023_01462, P2023_01462-1] 2 Multiple files\n", - "79 2023/01486 [P2023_01486-2, P2023_01486-1] 2 Multiple files\n", - "84 2023/01591 [P2023_01591-1, P2023_01591-2] 2 Multiple files\n", + "79 2023/01486 [P2023_01486-1, P2023_01486-2] 2 Multiple files\n", + "84 2023/01591 [P2023_01591-2, P2023_01591-1] 2 Multiple files\n", "86 2023/01597 [P2023_01597-1, P2023_01597-2] 2 Multiple files\n", - "88 2023/01599 [P2023_01599-1, P2023_01599-2] 2 Multiple files\n", + "88 2023/01599 [P2023_01599-2, P2023_01599-1] 2 Multiple files\n", "94 2023/01656 [P2023_01656-2, P2023_01656-1] 2 Multiple files\n", - "175 2023/02395 [P2023_02395-1, P2023_02395-2] 2 Multiple files\n", - "180 2023/02444 [P2023_02444-1, P2023_02444-2] 2 Multiple files\n", - "200 2023/02617 [P2023_02617-2, P2023_02617-1] 2 Multiple files\n", + "175 2023/02395 [P2023_02395-2, P2023_02395-1] 2 Multiple files\n", + "180 2023/02444 [P2023_02444-2, P2023_02444-1] 2 Multiple files\n", + "200 2023/02617 [P2023_02617-1, P2023_02617-2] 2 Multiple files\n", "217 2023/02712 [P2023_02712-1, P2023_02712-2] 2 Multiple files\n", "311 2023/03714 [P2023_03714-3, P2023_03714-4] 2 Multiple files\n", "673 2024/01804 [P2024_01804-1, P2024_01804-2] 2 Multiple files\n", "797 2024/02666 [P2024_02666-1, P2024_02666-2] 2 Multiple files\n", - "833 2024/02851 [P2024_02851-2, 2024_02851-1] 2 Multiple files\n", + "833 2024/02851 [2024_02851-1, P2024_02851-2] 2 Multiple files\n", "1081 2025/00161 [P2025_00161, P2025_00161-2] 2 Multiple files\n", - "1523 2025/02993 [P2025_02993-1, P2025_02993-2] 2 Multiple files\n", - "1527 2025/03031 [P2025_03031-1, P2025_03031-2] 2 Multiple files\n", + "1523 2025/02993 [P2025_02993-2, P2025_02993-1] 2 Multiple files\n", + "1527 2025/03031 [P2025_03031-2, P2025_03031-1] 2 Multiple files\n", "1540 2025/03126 [P2025_03126-2, P2025_03126-1] 2 Multiple files\n", "1664 2025/03854 [P2025_03854-2, P2025_03854-1] 2 Multiple files" ] @@ -2258,22 +2270,22 @@ " \n", " \n", " 2302\n", - " 2025/00137M\n", - " [P2025_00137]\n", + " 2025/03567M\n", + " [P2025_03567]\n", " 1\n", " entry missing\n", " \n", " \n", " 2303\n", - " 2025/00310M\n", - " [P2025_00310]\n", + " 2025/00137M\n", + " [P2025_00137]\n", " 1\n", " entry missing\n", " \n", " \n", " 2304\n", - " 2024/03383M\n", - " [P2024_03383]\n", + " 2025/00310M\n", + " [P2025_00310]\n", " 1\n", " entry missing\n", " \n", @@ -2286,15 +2298,15 @@ " \n", " \n", " 2306\n", - " 2023/00146-2M\n", - " [P2023_0146-2]\n", + " 2024/03383M\n", + " [P2024_03383]\n", " 1\n", " entry missing\n", " \n", " \n", " 2307\n", - " 2025/03567M\n", - " [P2025_03567]\n", + " 2023/00146-2M\n", + " [P2023_0146-2]\n", " 1\n", " entry missing\n", " \n", @@ -2304,12 +2316,12 @@ ], "text/plain": [ " record_num matched_files file_count file_issue\n", - "2302 2025/00137M [P2025_00137] 1 entry missing\n", - "2303 2025/00310M [P2025_00310] 1 entry missing\n", - "2304 2024/03383M [P2024_03383] 1 entry missing\n", + "2302 2025/03567M [P2025_03567] 1 entry missing\n", + "2303 2025/00137M [P2025_00137] 1 entry missing\n", + "2304 2025/00310M [P2025_00310] 1 entry missing\n", "2305 2023/01814M [P2023_01814] 1 entry missing\n", - "2306 2023/00146-2M [P2023_0146-2] 1 entry missing\n", - "2307 2025/03567M [P2025_03567] 1 entry missing" + "2306 2024/03383M [P2024_03383] 1 entry missing\n", + "2307 2023/00146-2M [P2023_0146-2] 1 entry missing" ] }, "metadata": {}, @@ -2430,10 +2442,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "a235c852", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Split counts (rows):\n", + "split\n", + "train 1429\n", + "test 308\n", + "val 305\n", + "Name: count, dtype: int64\n", + "\n", + "Split x type (rows):\n", + "type a luminal b luminal\n", + "split \n", + "test 162 146\n", + "train 754 675\n", + "val 161 144\n" + ] + } + ], "source": [ "# Assign a train/val/test split.\n", "#\n", @@ -2490,7 +2522,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "id": "b90a7fac-9575-49f9-884c-28c1875dd435", "metadata": {}, "outputs": [ @@ -2519,6 +2551,7 @@ " mammaprint_index\n", " type\n", " path\n", + " split\n", " \n", " \n", " \n", @@ -2528,6 +2561,7 @@ " 0.541\n", " a luminal\n", " /mnt/data/MOU/breast/mammaprint/P2025_00968\n", + " val\n", " \n", " \n", " 1411\n", @@ -2535,6 +2569,7 @@ " -0.076\n", " b luminal\n", " /mnt/data/MOU/breast/mammaprint/P2025_00968-2\n", + " train\n", " \n", " \n", "\n", @@ -2545,9 +2580,9 @@ "1410 2025/00968-1 0.541 a luminal \n", "1411 2025/00968-2 -0.076 b luminal \n", "\n", - " path \n", - "1410 /mnt/data/MOU/breast/mammaprint/P2025_00968 \n", - "1411 /mnt/data/MOU/breast/mammaprint/P2025_00968-2 " + " path split \n", + "1410 /mnt/data/MOU/breast/mammaprint/P2025_00968 val \n", + "1411 /mnt/data/MOU/breast/mammaprint/P2025_00968-2 train " ] }, "metadata": {}, @@ -2578,17 +2613,38 @@ " mammaprint_index\n", " type\n", " path\n", + " split\n", " \n", " \n", " \n", + " \n", + " 375\n", + " 2023/03525-1\n", + " 0.235\n", + " a luminal\n", + " /mnt/data/MOU/breast/mammaprint/P2023_03525-35\n", + " train\n", + " \n", + " \n", + " 376\n", + " 2023/03525-2\n", + " -0.345\n", + " b luminal\n", + " /mnt/data/MOU/breast/mammaprint/P2023_03525-23\n", + " train\n", + " \n", " \n", "\n", "" ], "text/plain": [ - "Empty DataFrame\n", - "Columns: [record_num, mammaprint_index, type, path]\n", - "Index: []" + " record_num mammaprint_index type \\\n", + "375 2023/03525-1 0.235 a luminal \n", + "376 2023/03525-2 -0.345 b luminal \n", + "\n", + " path split \n", + "375 /mnt/data/MOU/breast/mammaprint/P2023_03525-35 train \n", + "376 /mnt/data/MOU/breast/mammaprint/P2023_03525-23 train " ] }, "metadata": {}, @@ -2602,10 +2658,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "9b0bc82b-0738-4244-83da-5cbee0c2e1fe", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "✅ Saved final dataset to: data_mapping_final.csv\n", + " 2042 rows × 5 columns\n" + ] + } + ], "source": [ "# Save to CSV\n", "output_file = \"data_mapping_final.csv\"\n", @@ -2625,7 +2691,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "id": "a98b1bea-95e5-4672-b8c8-3408a22b0a55", "metadata": {}, "outputs": [], @@ -2831,7 +2897,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "id": "86118b8b-0092-4c58-8758-2e07137b7082", "metadata": {}, "outputs": [ @@ -2844,7 +2910,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABW0AAAN5CAYAAABzABkEAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAA/JRJREFUeJzs3Xd4FFX7//FPGgmkAaEkhNAk9CogvYbepQpKFwQrRRFQqSr6AA9YeZCmoKIiCFJEmhQRpIP0JhBCb0lISCE5vz/4sV/XJJCEkN0s79d1zXWxZ+45c88wyU7uPXvGyRhjBAAAAAAAAACwC862TgAAAAAAAAAA8H8o2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAYEfefvttOTk52ToNAAAAIE1ef/11eXh4ZPn9TZs2TU5OTrp48WKG9/0gxhhVq1ZNw4YNy/R9PyqlSpVSp06d7tuWmJiocuXK6e23387s9AC7RtEWwGNt8uTJcnJySnH58ssv09Xv66+/LldX14xNFgAAAA9t7969lnu9cePGJRvz5ptvWmJOnDiRyRlmvn+eEycnJzk7Oyt37txq2rSpfvvtt0zJ4eWXX5aXl1em7MtezZ8/X4cPH9abb75p61QylbOzs8aNG6cpU6bo7Nmztk4HsBsUbQFA0tatW2WMSbL07t07U/N49913ZYzJ1H0CAAA8jjw9PfXVV18lufdKSEjQ119/LU9PTxtlZjsTJkyQMUbx8fHauHGjoqKi1KRJE23YsOGB206ePFkxMTGPPkkb7S8zTJw4UV27dlW+fPlsnUqme/rpp5U7d25NnTrV1qkAdoOiLQAAAADgsdOhQwf9/fff2rRpk1X7r7/+qvPnz6tjx442ysz2XFxcVL58ec2dO1cJCQn65JNPbJ2Sw9uwYYOOHDmiHj162DoVm3B2dla3bt301VdfOVwxHkgvirYAkEr3vrIVFRWl559/Xjlz5pSvr6+effZZhYeHW+I6deqkKVOmKCEhweprZve+Wnevn4iICPXt21d+fn4KDg6WlPKctocOHVKXLl2UN29eZcuWTaVKldLUqVOtRoacPn1azz77rAIDA5UjRw6VLVtW48ePV1RUVKqO6+bNm3r22Wfl6+urXLlyqVevXrp69Wq6crnfMf7bZ599JicnJ+3cuTPJupkzZ8rJyUl//PFHmvb/6aefWp17Hx8f1atXTytWrEj22FOTJwAAcCxPPvmkKlSokGQ6rLlz56pOnTrJ3hMcOXLE6h7Dw8ND5cqV0+TJk5WYmGiJW7JkiZycnLR161a9//77CgwMlI+Pj5577jlFRUXJGKP33ntPBQsWVPbs2dWmTRtdvnzZal8Z0Udq801JcHCwXFxcFBoaKkl6/vnnlSdPHt24cUM9e/ZU7ty5Vb58eUnJzzF7Lz4yMlK9e/eWr6+vcubMqV69eunWrVuWuNatW+uzzz5TVFSUVb7nzp1LMbeH2d89CxcuVPny5eXh4aHSpUtr4cKFKe7vr7/+UseOHZUnTx65u7urdOnS+vTTTy3rV61aJRcXF40aNcpqu6VLl8rJyUkTJkxIsW9JWrZsmdzd3VWrVq0k6xITEzVlyhSVKVNG7u7u8vPzU6dOnXTkyBFLTN26dRUcHJzst/ZatmypoKAgq//zH374QbVq1ZKnp6c8PT3VoEEDbd682Wo7f39/9e7dWzt27FC9evWUPXt2jRgxQpLUoEEDy/+Ti4uL/P391b17d50+ffq+x3k/jRo10o0bN5LkATyuKNoCQBq9+uqrat++vc6ePasff/xRy5cvt3pYwI8//qhhw4bJxcXFaqqF4sWLW/UzcOBAtW/fXidOnNDIkSNT3N+uXbv01FNP6fbt29qwYYOuXbum999/X+PHj9cbb7xhiWvZsqVCQ0O1bt063bhxQ4sXL5Yk/fTTT6k6rkGDBqlbt24KDQ3VkiVL9Ntvv6lp06aKi4tLcy5pOcaePXvK29tbn3/+eZJ106dPV6lSpSw3r6nd/8svv2w57wkJCTpw4ICqVaum9u3ba8eOHenKEwAAOJ7evXvrxx9/tHzIff36df38888pTpFVqlQpq/u78+fP6/XXX9fo0aP13//+N0n8tGnT5O3trQMHDmjt2rVas2aNXnvtNU2YMEGenp7av3+/Nm/erJ07d+qll15Kdp8P00da8/23EydOKCEhQUFBQZY2Y4xeeOEFPfPMMzp58qSGDh163z6MMXr55Zf1zDPP6Ny5c1qwYIEWL15sKf5J0vLly/XSSy/J09PTKt+CBQs+MMf07E+6W7Ts0qWLGjdurL///lu//vqrVq5cmewcvlu3blX16tVljNHvv/+uq1evaty4cRo1apTl4VnNmzfXW2+9pQ8++EDLly+XJJ08eVK9evVSixYtHviQrd9//10VKlRQtmzZkqwbOHCgRo0apddee00XL17Upk2bdPHiRdWoUUPHjx+XdLdgfeLEiSQjx8PCwrR69Wr16dNHzs53S0DvvfeeunfvrrZt2+rEiRM6deqUqlevrpCQkCQF03PnzmncuHH6/PPPdfLkSVWvXl3S3ZHB9/6foqOjtXz5cp05c0atW7dO90jZp556SpKSHAPw2DIA8BibNGmSkZTi8vfff1tiX3rpJSPJfPvtt1Z9DBkyxLi5uZmYmBhL27Bhw4yLi0uy+7zXz+zZs5Ose+utt8y/fzXXrFnTFCtWzKp/Y4yZNm2acXFxMaGhoebixYtGkpkxY0ZaT4Eln7lz51q1r1q1ykgyc+bMSVMuDzrG5AwaNMhkz57dXL9+3dK2detWI8lMmjQpzftPSfHixc3AgQOTHHtq8wQAAFnfnj17jCQzdepUc/nyZePm5ma+/PJLY4wxH3/8scmRI4eJiIgwEyZMMJLM8ePHH9jnwIEDTfHixS2vf/rpJyPJPP/881Zxo0ePNtmyZTMDBgywah83bpxxdnY2N27cyNA+UpvvvXMyYcIEY4wxd+7cMQcOHDB16tQxzs7OZt26dcYYY/r162ckmW+++SZJn8OGDTPu7u5WbffiFy9ebNU+aNAg4+HhYRISEixtL730kvH09Hxg7hm1v6JFi5rq1atbxd25c8c88cQTRpK5cOGCpb1y5cqmVKlSJi4uzir+gw8+MNmyZTOXLl0yxhiTkJBgmjRpYnLmzGkOHjxoKlasaAoVKmSuXr36wOPx9/c3bdu2TdJ+8OBBI8kMHz7cqv3SpUsmR44cpmvXrsYYY6KiooyPj4/p0aOHVdy7775rnJyczKlTp4wxxpw+fdq4urqa1157Lcm+6tata2rXrm15nT9/fuPu7m45vgfZu3evkWTWrl1raStZsqTp2LGjVVxybfe4ubklOQbgccVIWwBQyg8iK1KkSJLYVq1aWb0uV66c4uPjLV8bS622bds+MObq1avaunWr2rZtK3d3d6t1jRs3VkJCgrZs2aI8efKoQIECmjhxor766itdunQpTbkkl0+zZs2UPXt2rV+/Pk25pPUYJemll17S7du3rb6e+Pnnn8vNzU09e/ZM8/6jo6P1zjvvqHTp0sqePbvVFBXJPQE6tXkCAADHkjdvXrVo0cJyDzJ37lx16tRJ3t7eKW7z5ZdfqlatWvLx8bHcY/zvf//TqVOnkkw50KJFC6vXpUqVUlxcnEJCQqzaS5curcTExGS/Wv6wfaQl33feeUdOTk5yc3NTnTp15O7urlWrVqlRo0ZWcWm5d3JyckpyDOXKlVNMTIzOnz+f6n4ycn9///23/v777yTH4eLiojZt2li1hYWFac+ePWrfvr3c3Nys1jVu3FhxcXHaunWrpLvzsn7zzTfy8vJSlSpVdPjwYS1cuFB+fn4PzPvmzZvJXnf37sU7dOhg1Z4vXz7VrVtX69atkyTlyJFD3bp106JFixQRESHp7qjjuXPnqlGjRipatKiku9M43LlzR507d06yr5CQEG3bts3qm3Y1atRI9sFoR48eVbdu3VSgQAG5urrKyclJlSpVkqRk77dTy9vbWzdv3kz39oAjoWgLAGmQI0cO+fj4WLXde52Wm4ts2bIpT548D4y7V3z9+OOP5erqKhcXF7m4uMjZ2VnlypWTJF27dk0uLi5avXq1ypcvr4EDB8rf319lypTR+PHjFR0d/cD9uLq6Knfu3Ena8+XLZ5nXNrW5pPUYJals2bKqV6+e/ve//8kYo2vXrmnhwoVq3bq15SYxLfvv2bOnPvnkE73//vs6d+6cEhISZIxRpUqVFB8fb7XvtOQJAAAcT+/evbVx40YtXbpUe/bsSXFqBEn63//+pz59+qh169Y6ePCg4uPjZYzR66+/rsTExCRF0ICAAKvX94pyKbUndz/5MH2kNd8JEybIGKPExETduHFDa9euVZMmTaxifHx85OXllSTPlOTKlSvJ3LPpuX/OyP3du2fMnz9/ku3/3Xbx4kVJ0n/+858k96BVq1a16k+6+0FA27ZtFRMTo0aNGlm+8v8gOXPmtBRb/+le3/7+/knW+fv7W+37+eefV3R0tBYsWCBJ2rhxo06ePKl+/folOZ769etbjsfZ2VnOzs4aO3asEhISrP5fAgMDk82pdu3aOnfunH7++WeFh4fLGGMp1v77fjstIiMjlStXrnRvDzgSirYAkAbJPSQsPf79KX1K7hUTR40apTt37ighIUEJCQlKTEy0jAZ+8cUXJd0tfP7888+6efOmtmzZolatWmncuHGW9fdz584dXb9+PUn75cuXLSMD0pJLWo7xnpdeeknHjh3TunXrNGfOHMXExFjdYKZ2/7du3dJPP/2kF198UU8//bT8/Pws83clN3olrXkCAADH0rp1a/n5+alPnz4qUqSIGjRokGLsvHnzVK1aNY0aNUpBQUFydXWVdHfkZnJSundMyz3lw/SR1nxTI633Thl1/5yR+7t3f5vct9P+3XbvHnT8+PEp3oP27dvXEr9y5UpNnz5d1atX16pVq/TVV1+lKu/ChQtbCqr/dG9gRUq5/nMUb9WqVVWxYkXNmTNHkjRnzhzlypVLTz/9dJLj2bVrl+V47hXw7x3PP0fWJvf/vWLFCl27dk0fffSRqlatKk9PT0kPd11Jd79ZFx8fr8KFCz9UP4CjoGgLAI+Ap6enEhMTdefOnYfqJ3/+/KpWrZp++umnVH9ife+ps5MmTVLDhg1TPZH/smXLrF6vXr1at2/ftnz1Lj25pMXTTz+tgIAAffbZZ5oxY4YKFCig5s2bW9andv9OTk4yxiSZQuFeQRsAAOCf3Nzc9Oyzz+rGjRvq1avXA4t+/77HuHr1qlatWvUoU3woWSVfT09Pq6/lP0pFixZVkSJFktz/JiYmWh4idk/hwoVVvnx5LV68+IH39mfOnFGPHj3UsGFDbdmyRV26dNGLL76ov/7664E51alTR/v3709yDu5NTfHvhwtfuXJFmzZtSjJNRr9+/bR9+3Zt2bJFP/74o5599lmrkcctWrSQi4uLvv/++wfm9CD/vrbmzZv3UP1t375dklS3bt2H6gdwFBRtAeARKFeunIwxWr58uRISEh6qr+nTp+vs2bN6+umntWvXLkVHR1u+itSkSRNdu3ZNe/bsUYcOHbRmzRpdvnxZt2/f1po1a7Rr1y41bNjwgfvw9PS0PDE3MjJSmzZt0oABA1ShQgV17949Tbmkl5ubm/r3768lS5bo5MmT6t27t1xcXNJ8Ljw9PdWoUSPNmDFD27Zt061bt7RixQr95z//UYUKFdKdHwAAcFzTpk2TMUZjx469b1zbtm21ZcsWffXVV7p165b27t2rDh06JCmc2YuslO+950T88ssvD33/nBoTJ07Utm3b9Prrr+vixYsKDQ1V//79VaZMmSSxM2bM0JEjR9S5c2ft2bNH0dHRCg0N1U8//aSGDRsqKipKcXFx6ty5szw8PLRgwQK5uLho1qxZCgoKUqdOnRQZGXnffNq0aaPY2Fj98ccfVu1ly5ZV3759NXXqVM2aNUs3b97UoUOH1LFjR7m6umrcuHFW8c8995w8PDzUrVs33b592+qba5JUrFgxjR8/XpMmTdL48eN1+vRp3b59W0eOHNHHH3+sPn36PPDcNWrUSF5eXhoxYoTCwsJ06dIlTZgwQbGxsQ/c9n7Wr1+vnDlzUrQF/j+KtgAgqWbNmpYHM/xzGTFiRLr669ixowYMGKABAwbIzc3N8hCs9KhSpYp2796tPHnyqF27dsqZM6dq166tr776SiNHjpSfn58qVqyoXr166b///a/Kly+vvHnzavDgwXr99df1+eefp2o/n332mebNm6fAwEC1bdtWdevW1dq1a60+QU9NLg9jwIABlgcZ/PNrZmnd/9dff6169eqpdevWCgwM1MyZM7VgwQJlz579ofIDAACPt9dff11vvfWW3nnnHeXLl08vvPCCxo0bp/Lly9s6tWRlpXy7deumPn36qEePHpb753Pnzj2y/T3zzDP67rvvtGrVKhUuXFiNGzdWkyZNkh3wULNmTe3atUteXl5q3bq1pbD47bffauzYsfL09NTgwYO1Z88e/fDDD5bpBby9vfXjjz/q3Llzev755++bT/369VW6dGnNnz8/ybovvvhCEyZM0JQpU5Q/f37VqVNHefLk0datW1WyZEmr2HvTIYSGhurJJ5+0PBzsn0aNGqXFixdr06ZNqlSpkvz8/NSxY0edPn1aY8aMeeC5K1iwoJYtW6bLly+rRIkSqlKlimJjY/Wf//zngdumJDExUQsWLFDv3r2TzEkMPK6cjDHG1kkAAGzn5Zdf1pdffqlbt27ZOhXdvHlTAQEBqlGjhn777TdbpwMAAABkmq+//loDBw7U33//rbx589o6nUy1aNEiPffcczp69KgKFSpk63QAu8BIWwCA3Vi6dKliYmLUv39/W6cCAAAAZKpnn31WZcqU0cSJE22dSqZKTEzUmDFjNGzYMAq2wD+42joBAAAk6eLFi5o8ebKCg4PVuXNnW6cDAAAAZConJyfLw7geJ87Ozjpw4ICt0wDsDiNtAQA2V6NGDQUFBcnd3V0//PCD3NzcbJ0SAAAAAAA2w5y2AAAAAAAAAGBHGGkLAAAAAAAAAHaEoi0AAAAAAAAA2BEeRPYIJCYm6vz58/L29paTk5Ot0wEAAHAoxhhFRkaqQIECcnZmDEJG414WAADg0UntvSxF20fg/PnzCgoKsnUaAAAADi00NFQFCxa0dRoOh3tZAACAR+9B97IUbR8Bb29vSXdPvo+Pj42zAQAAcCwREREKCgqy3HMhY3EvCwAA8Oik9l6Wou0jcO9rZD4+PtzoAgAAPCJ8df/R4F4WANLu3I1ofbLuhF4JKa6CuXLYOh0AWcCD7mWZBAwAAAAAAOAh3IyO1/c7Q3UzOt7WqQBwEBRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAgIeQx8tdgxo8oTxe7rZOBYCDcLV1AgAAAAAAAFmZv6+H3mxeytZpAHAgjLQFAAAAAAB4CLdi72jryWu6FXvH1qkAcBAUbQEAAAAAAB7C6atR6jZzm05fjbJ1KgAcBEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAA4CG4ujjJ38dDri5Otk4FgINwtXUCGWndunVatGiRLl++rAIFCqh79+6qUaOGVcypU6f06aef6syZMwoODtbgwYPl7++f5hgAAAAAAABJKuXvo22jQmydBgAH4jAjbWfOnKkWLVoof/78euaZZ5QtWzbVrl1bP//8syXm5MmTqlq1qs6fP6/27dtr3759qlq1qq5cuZKmGAAAAAAAAAB4VJyMMcbWSWSEBg0aqFChQpo3b56lrUmTJvLz89N3330nSerVq5cOHjyoHTt2yMnJSXFxcSpevLi6d++uDz74INUxDxIRESFfX1+Fh4fLx8cn4w8WAOxIkRErbJ0CMsjpD1rZOgUgVbjXerQ4vwCQdkcuRqj3nB36sm81lfLPuN+dmX2vzf0g8Oil9l7LYUbalipVSsePH1dCQoIkKTo6WqdPn1aZMmUsMatWrdLTTz8tJ6e7c8xky5ZNbdq00apVq9IUAwAAADwKxhilNKYiLi5Ot27dslqioqJS7Cs+Pv5RpQkA+Jc7CUYXI2J0J8EhxsUBsAMOU7T973//q4oVK6pYsWJq0qSJgoOD1blzZ40aNUrS3SLu5cuXVbBgQavtgoKC9Pfff6c6JjmxsbGKiIiwWgAAAIDU2rhxo1q2bKmcOXPKy8tL9erV07Zt26xi3n//ffn6+srf39+ylC1bNklfn376qQICAuTh4aGiRYvqxx9/zKzDAAAAQAZxmKLtmjVr9MMPP6hnz57q37+/unbtqi+++EJ//vmnpLsjEyQpR44cVtvlyJHDsi41McmZOHGifH19LUtQUFCGHRcAAAAc36effqrXXntNYWFhunTpkipUqKBmzZrp3LlzVnE1a9a0Gml7+vRpq/WLFi3SsGHD9L///U/R0dEaPny4nnnmGW3fvj0TjwYAAAAPy2GKti+++KJeeOEFTZgwQV26dNF///tftW7dWoMHD5YkeXl5ycXFRdevX7fa7tq1a8qZM2eqY5IzcuRIhYeHW5bQ0NCMPDQAAAA4uIULF6pZs2by8vKSl5eXpkyZoqioKK1fvz5J7J07d1KcQmHatGnq0KGD2rVrJ3d3dw0aNEiVK1fWJ5988qgPAQAAABnIIYq2CQkJunz5sp544gmr9uLFiyssLEyS5OrqqrJly2rfvn1WMXv37lXFihVTHZMcd3d3+fj4WC0AAABAel24cEEJCQny8/Ozat+xY4c8PT3l7e2thg0bateuXZZ1CQkJ2rFjh+rVq2e1TcOGDZNMtQAAyFhF8nhqQf8aKpLH09apAHAQDlG0dXFxUZUqVfTdd99ZpjGIiorSokWLVL16dUtcz5499cMPP1i+RrZv3z79+uuv6tmzZ5piAAAAgEclMTFRL7/8skqUKKHGjRtb2gsXLqzFixcrIiJCp06dUpEiRdSwYUOdOXNGkhQeHq7Y2FjlzZvXqr+8efPq8uXLKe6P5zMAwMPzcndVzSf85OXuautUADgIhyjaStKsWbMUFhamYsWKKSQkREWLFpWzs7M++ugjS8yrr76qkJAQVahQQbVr11atWrXUt29fdevWLU0xAAAAwKPy6quvauvWrVq0aJHc3d0t7X369FGrVq3k7u6ufPny6YsvvpCXl5fmzJljtX1iYmKS105OTinuj+czAMDDuxgeow9XHdHF8BhbpwLAQTjMR0DlypXTgQMHdPjwYV26dEmBgYEqWbKk1Q2qm5ubvv/+ex09elRnz55V8eLFVbRoUat+UhMDAAAAPApDhgzRt99+q3Xr1qlcuXL3jXVzc9MTTzyhU6dOSZJy5syp7NmzJxlVe/nyZQUEBKTYz8iRIzV06FDL64iICAq3AJBGV2/FavqGk2pVPkD+vh62TgeAA3CYoq10d5qEcuXKPfAGt2TJkipZsuRDxwAAAAAZZejQofrqq6+0Zs0aVa5c+YHxt2/f1pEjR1S3bl1JkrOzs2rVqqV169bp5ZdftsStWbNGderUSbEfd3d3qxG9AAAAsD2HmR4BAAAAyKqGDx+uWbNmacmSJSpZsqRu3bqlW7duKT4+3hLTokUL/fLLL7pw4YL279+vrl27Kj4+XgMGDLDEvPnmm1q+fLlmzJih0NBQjRkzRseOHdPgwYNtcFQAAABIL4q2AAAAgI198cUXSkxMVMuWLeXv729Zpk6daol57733NHPmTFWtWlWdOnWSr6+vdu3apSJFilhimjRpom+++UbTp09XxYoVtXr1aq1cuVJly5a1wVEBAAAgvRxqegQAAAAgK7p58+YDY5588kktXrz4gXFdunRRly5dMiArAEBq5czhpq5Vg5Qzh5utU3koRUasyLR9nf6gVabtC8iKKNoCAAAAAAA8hIK5cujDThVsnQYAB8L0CAAAAAAAAA8hJj5Bxy5FKiY+wdapAHAQFG0BAAAAAAAewonLt9R06iaduHzL1qkAcBAUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAICHlM2FEguAjONq6wQAAAAAAACysnKBvjr2XgtbpwHAgfAxEAAAAAAAAADYEYq2AAAAAAAAD+HE5Ui1+nizTlyOtHUqABwERVsAAAAAAICHEBOfqIPnIxQTn2jrVAA4CIq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAwEMIypVDn3V/UkG5ctg6FQAOwtXWCQAAAAAAAGRlvjnc1KpCgK3TAOBAGGkLAAAAAADwEK5ExmrW5lO6Ehlr61QAOAiKtgAAAAAAAA/hUkSM3l1xWJciYmydCgAHQdEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAeAjeHq5qXDqfvD1cbZ0KAAfBbxMAAAAAAICHUNjPU7N6VbN1GgAcCCNtAQAAAAAAHkJ8QqKu3YpVfEKirVMB4CAYaQs85oqMWGHrFJCBTn/QytYpAAAAAI+doxcj1fqT37X8lToqF+hr63QAOABG2gIAAAAAAACAHXGYkbZnz55VXFxcknZPT08FBARYtcXExOjKlSvKnz+/smXLlmx/qYkBAAAAAAAAgIzmMCNt+/fvr+bNm1stwcHBGjp0qFXcmDFjlDt3blWqVEl58uTRJ598kqSv1MQAAAAAAAAAwKPgMCNtf/31V6vXO3fuVLVq1fTss89a2ubPn69JkyZp7dq1qlWrln7++Wd16NBBpUqVUpMmTVIdAwAAAAAAAACPisOMtP232bNnKzAwUC1atLC0TZ8+XR07dlStWrUkSW3btlXdunU1ffr0NMUAAAAAAADcUzrAR3+NbarSAT62TgWAg3DIou3t27e1YMEC9e3bVy4uLpKkxMRE7d69WzVr1rSKrVOnjnbu3JnqGAAAAAAAgH9ycXaSt4ebXJydbJ0KAAfhkEXbH3/8UZGRkerXr5+lLTIyUrGxsfLz87OKzZMnj65evZrqmOTExsYqIiLCagEAAAAAAI+Hv69GqcfsP/X31ShbpwLAQThk0Xb27Nlq2rSpChcubGlzdr57qPHx8VaxcXFxltG4qYlJzsSJE+Xr62tZgoKCMuQ4AAAAAACA/YuKvaPNx68qKvaOrVMB4CAcrmh74sQJbdq0Sf3797dq9/b2lq+vry5evGjVfvHiRRUsWDDVMckZOXKkwsPDLUtoaGgGHQ0AAAAAAACAx43DFW3nzJmjfPnyqU2bNknWNWjQQL/++qtV2y+//KIGDRqkKebf3N3d5ePjY7UAAAAAAAAAQHo4VNE2ISFBX331lXr37i03N7ck69966y1t2rRJo0eP1o4dO/TSSy8pNDRUw4YNS1MMAAAAAAAAADwqDlW03bZtm7Jnz67nn38+2fXVqlXT6tWrtWPHDvXu3VthYWHauHGjihcvnqYYAAAAAACAewJ8PTS+XVkF+HrYOhUADsLV1glkpNq1a+vEiRP3jalfv77q16//0DEAAAAAAODxU2TEihTXjV56MBMzAeDIHGqkLQAAAAAAAABkdRRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOuNo6AQAAAAAAgPQqMmKFrVMAgAzHSFsAAAAAAAAAsCOMtAUAAADsQFxcnA4ePKiEhASVKlVKXl5eycadPHlSZ8+eVXBwsAoWLJjuGAAAANgvRtoCAAAANvaf//xHRYsWVb9+/TRgwAAFBgZqxowZVjF37tzRM888o0qVKmnEiBEKDg7WsGHD0hwDAAAA+8dIWwAAAMDGnJ2d9ddffyl37tySpHnz5qlPnz6qWbOmKlSoIEmaOnWq1q5dq7/++ktFihTRzp07VatWLdWqVUsdO3ZMdQwAAADsHyNtAQAAABt7/fXXLQVbSXruuefk5OSknTt3Wtq++uorPfPMMypSpIgkqWrVqmrcuLG+/PLLNMUAAADA/lG0BQAAAOzMn3/+qYSEBJUsWVLS3fluDx8+rMqVK1vFVa5cWfv27Ut1THJiY2MVERFhtQAAAMC2KNoCAAAAdiQ8PFz9+vVTs2bNVLt2bUlSRESEEhMTrUbjSpKfn59u3LiR6pjkTJw4Ub6+vpYlKCgog48IAAAAaUXRFgAAALATUVFRat26tdzd3bVgwQJLe7Zs2SRJt2/ftoqPjo62rEtNTHJGjhyp8PBwyxIaGpohxwIAAID040FkAAAAgB2IiopSq1atFBERofXr1ytXrlyWdT4+PsqdO3eSguq5c+cs89emJiY57u7ucnd3z7DjAAAAwMNjpC0AAABgY9HR0WrVqpVu3LihdevWyc/PL0lMs2bNtHTpUsvruLg4LV++XM2bN09TDAAAAOwfI20BAAAAG2vXrp12796t6dOna+fOnZb2EiVKqFixYpKkMWPG6KmnnlLPnj3VokULzZ8/X5I0ePBgS3xqYgAAAGD/GGkLAAAA2Fi2bNlUq1YtzZ8/X9OmTbMsf/75pyWmZMmS2r59u3x8fLRgwQKVLVtW27dvV968edMUAwAAAPvHSFsAAADAxlasWJGquJIlS+rTTz996BgAAADYN0baAgAAAAAAAIAdoWgLAAAAAAAAAHbE4aZH+PPPP/XLL7/IyclJnTp1UtmyZa3W37hxQ/Pnz9eZM2cUHBysnj17KkeOHGmOAQAAAAAAAIBHwaFG2g4ZMkRNmzbVrVu35OHhoT59+mjDhg2W9ZcuXdKTTz6phQsXKmfOnPriiy9Uo0YN3bp1K00xAAAAAAAAAPCoOMxI24ULF+qTTz7R1q1bVa1aNUnS0KFDdfnyZUvMu+++Kw8PD61du1bu7u569dVXFRwcrI8//lijRo1KdQwAAAAAAAAAPCoOM9L2888/V/PmzS0FW0lyc3NTYGCg5fXSpUvVuXNnubu7S5J8fX3Vpk0bLV26NE0xAAAAAAAAAPCoOEzRdteuXapTp45WrlypN954Q5MmTdLRo0ct62NjYxUaGqqiRYtabVesWDEdP3481THJiY2NVUREhNUCAAAAAAAAAOnhEEVbY4wiIyM1b948TZo0SXny5NGBAwdUsWJF/fTTT5Kk27dvS5K8vb2ttvXx8VF0dHSqY5IzceJE+fr6WpagoKAMOzYAAAAAAAAAjxeHmNPWyclJXl5ecnZ21tq1a+Xi4iJJyp49u4YPH66nn35anp6ecnJy0s2bN622vXHjhnx8fCQpVTHJGTlypIYOHWp5HRERQeEWAAAAAAAAQLo4xEhbSSpXrpwqVKhgKdhKUuXKlXXmzBkZY+Tm5qbg4GAdPnzYarvDhw+rTJkykpSqmOS4u7vLx8fHagEAAAAAAACA9HCYom337t21ZcsWRUVFWdrWrFmjSpUqycnJSZLUtWtXff/997p+/bok6cyZM1q+fLm6du1q2SY1MQAAAAAAAADwqDjE9AiSNHDgQK1du1bly5dXgwYNdPDgQYWGhmr58uWWmDfffFNr165VlSpVVKtWLf3222+qX7++nn/++TTFAAAAAAAAAMCj4jBFWzc3Ny1dulSbN2/W0aNH1bFjRzVo0ECenp6WGE9PT23evFlr167V2bNn1b9/f9WvX98yEje1MQAAAAAAAADwqDhM0faeunXrqm7duimud3FxUbNmze7bR2piAAAAAAAAAOBRcJg5bQEAAAAAAADAEVC0BQAAAAAAAAA74nDTIwAAAAAAANsqMmKFrVMAgCyNkbYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2xNXWCQAAAAAAAODxUmTEikzb1+kPWmXavoCMwkhbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI44TNHWGKM7d+5YLQkJCbZOCwAAAA7s/Pnztk4BAAAADshhirbvvfee3Nzc5OHhYVny58+fJG7u3LkqVqyYXF1dVapUKS1ZsiRdMQAAAEDVqlUVEhKib775Rrdv37Z1OgAAAHAQDlO0laTq1atbjbS9evWq1fqVK1dqwIABevfdd3Xjxg29+OKL6ty5s3bs2JGmGAAAAECSlixZoieeeEIvvviiAgICNGjQIO3cudPWaQEAACCLc6ii7YNMmTJFbdu2Vffu3eXt7a1XX31VTz75pD766KM0xQAAAACS9NRTT+mLL77QhQsX9PHHH+vw4cN66qmnVKFCBU2bNi3JIAIAAAAgNRyqaLt3715lz55duXLlUrNmzbR3717LOmOMtm3bpvr161tt06hRI/3xxx+pjgEAAAD+LUeOHOrZs6c2bNigefPm6dSpUxoyZIgCAwPVt29fXbhwwdYpAgAAIAtxmKJtgQIFNH/+fF2+fFn79+9Xvnz5VK9ePZ05c0aSFBkZqejoaOXNm9dqu3z58unSpUupjklObGysIiIirBYAAAA8Pi5duqRJkyapdOnS6tevn1q1aqU1a9Zo6dKlOn36tBo1amTrFAEAAJCFuNo6gYzSt29fy7+9vb01Z84cFS1aVLNmzdKECRNS3C4xMVFOTk737ftBMRMnTtS4cePSnjQAAACytOXLl2vmzJlauXKlSpQoof79+6tnz57KkyePJaZx48bKnTu3rl27Jj8/PxtmCwAAgKzCYUba/pubm5tKlCihEydOSLpbyPX09NTly5et4q5cuSJ/f/9UxyRn5MiRCg8PtyyhoaEZfDQAAACwR6+99ppy586tDRs26ODBgxo6dKhVwVaSXF1dNX78eHl4eNgoSwAAAGQ1Dlu0jYuL05EjRxQYGChJcnJyUq1atfTbb79Zxa1bt061atVKdUxy3N3d5ePjY7UAAADA8R04cEBz585V7dq17xs3ePBgeXp6ZlJWAAAAyOocpmjboUMHrV+/Xjdu3NDx48fVo0cPRUREaMCAAZaY119/XStWrNCcOXN05coVTZo0Sfv27dOQIUPSFAMAAABIUvbs2bV161YZY6za9+zZo+joaBtlBQAAgKzOYYq2b775pqZMmaLg4GCFhIQoISFB27dvV4kSJSwxTZs21VdffaVJkyapSJEi+uabb7RkyRJVrlw5TTEAAACAJE2fPl3z589P8vyD7du3a9iwYTbKCgAAAFmdk/n3sAA8tIiICPn6+io8PJypEmD3ioxYYesUkIFOf9Aq0/fJNeQ4bHH9AOlhT/daVapU0axZs5J8wB8TE6OAgABdvXpVLi4uqeorISFBy5cv1549e9SpUyeVK1fOav2GDRu0YcMGq7YcOXJo+PDhVm3R0dH66aefdPbsWZUoUULt2rWTq2vqnz9sT+cXQNbFPSLsCfe5sCepvddymJG2AAAAQGYLDw9PMjWCdPdZCbdu3Ur1FAkrV67UE088oZkzZ2rcuHE6cOBAkpgNGzZozpw59+3n+vXrqlq1qiZNmqSLFy/qzTffVEhIiGJjY1N3QAAAALALqf/IHQAAAICVmjVrasqUKZo/f76cnf9vPMTkyZNVokQJeXt7p6qfgIAAbd68WQEBAXJzc0sxrlChQho7dmyK6ydMmKD4+Hjt2LFDnp6eGjlypEqWLKkvvvhCr7zySqqPCwAAALZF0RYAAABIpzFjxqh69eoqV66cGjVqJHd3d23btk07duzQsmXLUt3PvekV7ty5c9+4K1euaPLkycqRI4eeeuopVa1a1Wr9okWL1LNnT3l6ekqS/P391apVK/34448UbQEwZQEAZCFMjwAAAACkU/HixbVnzx41adJEO3fu1MaNGxUcHKxt27apWbNmGb4/V1dXhYaGatu2bapbt64GDBhgWRcdHa3Q0FAFBwdbbRMcHKyjR4+m2GdsbKwiIiKsFgAAANiWzUfa9ujRQ1OnTlWePHnStA4AAACwB4UKFdJHH330yPfTrVs3jR492jINw4svvqg6deqoadOm6tSpk6KioiRJvr6+VtvlzJlTt27dSrHfiRMnaty4cY8ucQAAAKSZzUfa/vbbb4qJiUl23bp16xQfH5/JGQEAAABpExcXp0uXLunixYtWS2JiYobto2TJklbz5taoUUMVK1bUb7/9JkmWKRHCw8Ottrt586a8vLxS7HfkyJEKDw+3LKGhoRmWMwAAANLHZiNtz507pzt37ighIcHy73uMMTp69Khu3LihvHnz2ipFAAAA4L5Onz6t3r17a/PmzckWaC9cuCB/f/9Htn9jjKKjoyVJOXLkUFBQkI4fP24Vc/z4cZUsWTLFPtzd3eXu7v7IcgQAAEDa2axoW6NGDYWFhUm6+9Tdf3NxcdE777wjV1ebz+AAAAAAJGvIkCFydXXVmjVrlC9fviTrM3Karx07dqhatWqW19u2bdP+/fs1ePBgS1vHjh31/fffa9SoUcqRI4cuXryoFStWaMKECRmWBwAAAB49m1VEf//9d925c0d169bVwoULrUYguLi4yN/fX9mzZ7dVegAAAMAD7d+/X8uXL1fp0qUfqp8TJ07o66+/tozW/fHHH3XkyBFVrVpVrVu3liSNGzdOt2/fVqVKlXTlyhUtXLhQPXr00LPPPmvp5+2339avv/6qWrVqqX79+lqxYoUqVqyoF1544aHyAwAAQOayWdG2SJEikqSjR4/K29tbTk5OtkoFAAAASJdChQrd9yFfaeXs7KwxY8Yku2758uX6448/9Oeff6pkyZIaNmyYKlasaBXj5+ennTt3avHixTp79qw+/PBDtWvXjm+vAQAAZDE2v3vz8fFRfHy8jh07pqtXr8oYY7W+Zs2azLEFAAAAuzRkyBC98cYbmj9/voKCgtLdT/HixTV27NgHxtWqVUu1atW6b0yOHDn03HPPpTsXAAAA2J7Ni7Z79uxR+/btdfbs2WTXh4aGqmDBgpmcFQAAAPBgb7zxho4fP65ChQrJy8tLLi4uVuuPHTuW7Fy3AAAAwP3YvGg7YsQINWjQQBMnTkz2hpavcgEAAMBeTZkyRTExMSmu9/X1zcRsAAAA4ChsXhE9c+aMPvroIxUoUMDWqQAAAABpcu8hYQAAAEBGcrZ1AiVLlkxxagQAAAAgKzhz5ozWrVunuLg4W6cCAAAAB2Dzou2gQYP02muvafXq1QoLC9PFixetlsTERFunCAAAACQrKipK7dq1U5EiRdS4cWNdv35dktSnTx8tWLDAxtkBAAAgq7L59AjPP/+8wsLC1KxZs2TX8yAyAAAA2Ktx48bp1q1bOnv2rKpVq2ZpHz58uDp16qRu3brZMDsAAABkVTYv2q5bt07x8fEprvf398/EbAAAAIDU+/XXX/Xll18qKChITk5OlvZSpUrp9OnTioiIkI+Pjw0zBAAAQFZk86JtyZIlbZ0CAAAAkC6RkZHKkSOHJFkVbW/fvq3Y2Fi5utr8dhsAAABZkM3vIo8ePXrfkbalSpXiZhcAAAB2qUaNGlqwYIHGjh1rKdoaYzRmzBhVqlTJUtAFAAAA0sLm1dCQkBCFhYWluJ45bQEAAGCvxo8fr5o1a2rHjh2KiIjQO++8o507d+rQoUP69ddfbZ0eAAAAsiibF2337NmjhIQEq7YzZ85o5MiRCgkJUYECBWyUGQAAAHB/xYsX1549ezR16lTduHFDu3fvVoUKFTR37lxVqlTJ1ukBAAAgi7J50TZv3rxJ2vz9/bVw4UKVK1dOb731lg2yAgAAAFKnYMGCmjJliq3TAAAAgAOxedE2JX5+frp9+7Zu3rypnDlz2jodAAAAIImEhAQZY1Jcz7MZAAAAkB42v4tM7kY3MjJSM2fOlIuLi3x9fW2UGQAAAHB/gYGBunTpUorrL1y4IH9//0zMCAAAAI7A5kXbwoULJ/sgMh8fH82aNcvyFF4AAADA3ixevFhxcXFWbadPn9b777+vLl26KHfu3DbKDAAAAFmZzYu2CxcuVGxsrFWbj4+PSpQoIS8vLxtlBQAAADxYrVq1km2vXbu2WrdurXfffTeTMwIAAIAjsHnRtmbNmrZOAQAAAMhQwcHBunDhgqKiouTp6WnrdAAAAJDF2Lxoe09YWJgOHjyo2NhYlShRQiVLlrR1SgAAAEC6LF68WImJicqRI4etUwEAAEAWZPOibVxcnF599VXNmjVLCQkJcnJykjFGTZs21ZdffqmAgABbpwgAAAAkq2bNmrpy5YpVW2RkpK5evarJkyfzfAYAAACki82Ltu+9955++eUXLV26VA0aNJCbm5v27dunoUOHqk+fPlq1alWa+zx//ry+/fZblSxZUm3atLFaFx8fr19++UVnzpxRcHCwmjZtKmdn5zTHAAAAAIMGDVJ0dLRVm4+Pj6pWraoSJUrYKCsAAABkdTYv2i5ZskQzZ85U06ZNLW3VqlXT0qVL5e/vn+Z5wBITE9W9e3ft2rVLzZo1syraRkZGqlGjRgoPD1fdunU1adIklSpVSsuXL1e2bNlSHQMAAABIUs+ePW2dAgAAAByQzYu20dHRypcvX5J2X19fubm5KSYmJk1F2wkTJsjPz0/169dPsm7ixIm6dOmS9u/fr5w5cyosLExlypTRF198oZdffjnVMQAAAIAkJSQkyBiTqlhXV5vfegMAACCLsPl3/qtXr64JEyZYfa0sMTFR7777rgoUKCA/P79U97V582bNmTNHM2fOTHb9woUL1bVrV+XMmVOSFBgYqNatW+uHH35IUwwAAAAg3b1XdHNzS9Vy8eJFW6cLAACALMLmH/e/++67atiwoYKCglS1alVly5ZN+/fv140bN7Ro0aJU93P9+nU999xzmj17tnLnzp1kfVxcnE6ePKmSJUtatZcqVUqrV69OdUxyYmNjFRsba3kdERGR6rwBAACQdU2aNEmDBg3Siy++qDp16ihbtmzau3evpk2bpr59+6ply5aW2LQMRgAAAMDjzeZF2yJFiujw4cP68ssvtW/fPsXGxurFF19Unz59kp02ISV9+vRRp06d1Lhx42TXR0VFyRhjGUF7T86cORUZGZnqmORMnDhR48aNS3WuAAAAcAyLFy/W9OnT1aNHD0tb8+bN1bJlS7Vr107vv/++DbMDAABAVmXzoq0keXh4aODAgenefs2aNVq1apVGjx6tyZMnS5JOnTolV1dXTZ48WQMGDFCOHDkkJR0FGx4ebpkzNzUxyRk5cqSGDh1qeR0REaGgoKB0Hw8AAACyhr/++ksffvhhkvby5cvr8uXLunXrlry8vGyQGQAAALIym81pGxsbqxEjRighISHZ9R9//LGOHz+eqr4KFCigV155RTdu3NDFixd18eJFy5QFFy9eVEJCgtzd3VW4cGGdPHnSatuTJ08qODhYklIVkxx3d3f5+PhYLQAAAHB8+fLl07x585K0//jjj3Jzc1P27NltkBUAAACyOpuNtJ01a5ZiYmLk4uKS7Pq8efNqwoQJyd4E/1vZsmUtI2zvOXLkiDw8PKza27Vrp4ULF+qdd96Rh4eHrl+/rmXLlumNN95IUwwAAAAg3X0+Q8uWLbVu3TrVrl1bbm5u2rdvn3799Vd99tlnKd7rAgAAAPdjs5G2y5YtU6tWrVJc37p1ay1btixD9/n2228rMTFR9evX11tvvaX69eurcOHCevnll9MUAwAAAEhSo0aNdOjQIVWrVk07d+7Upk2blD9/fv35558PNf0XAAAAHm82G2l76tQpFS1aNMX13t7ecnFx0fXr15U7d+4099+hQwe5ulofXt68ebVnzx59++23Onv2rIYNG6ZnnnlGHh4eaYoBAAAA7ilWrJg+/vhjW6cBAAAAB2Kzoq27u7uuX7+e4vq4uDjdunUr3cXSvn37Jtvu7e2tF1544b7bpiYGAAAAuOfMmTM6ceKE6tatq2zZstk6HQAAAGRxNpse4amnntLXX3+d4vqFCxcqODhYOXLkyMSsAAAAgNSLiopSu3btVKRIETVu3NgyKKFPnz5asGCBjbMDAABAVmWzou2rr76qGTNmaPjw4bpx44alPT4+Xp9//rkGDRqk119/3VbpAQAAAA80btw43bp1S2fPnlX+/Pkt7cOHD9e7775rw8wAAACQldlseoSKFStq7ty56tevnyZPnqwCBQrI3d1dZ8+eVUJCgt5880316tXLVukBAAAAD/Trr7/qyy+/VFBQkJycnCztpUqV0unTpxURESEfHx8bZggAAICsyGZFW0nq3r27GjZsqAULFujQoUO6c+eOihYtqo4dO6pcuXK2TA0AAAB4oMjISMt0Xv8s2t6+fVuxsbFJHowLAP9UZMQKW6cAALBTNr+LDAgI0NChQ22dRpbGG71jOf1BK1unAAAAUqlGjRpasGCBxo4daynaGmM0ZswYVapUieczAAAAIF1sXrQFAAAAsqrx48erZs2a2rFjhyIiIvTOO+9o586dOnTokH799VdbpwcAAIAsiqItAAAAkE7FixfXnj17NHXqVN24cUO7d+9WhQoVNHfuXFWqVMnW6QEAAGX+N5T5Bi0yAkVbAAAAIJ0+/fRT9enTR1OmTLF1KgAAAHAgzrZOAAAAAMiqJk2apBs3btg6DQAAADgYirYAAABAOlWsWFFbt261dRoAAABwMEyPAAAAAKRTp06dNGjQIJ08eVKVKlWSh4eH1fpatWopW7ZsNsoOAAAAWRVFWwAAACCdhg8frmvXrmnkyJHJrr9w4YL8/f0zOSsAAABkdRRtAQAAgHQKCwuTMSbF9a6u3G4DAAAg7ZjTFgAAAEijIkWK6NKlS3JxcZGrq6s+/PBDxcTEyNXV1WoBAAAA0oOiLQAAAJBGMTExViNsP/nkE926dcuGGQEAAMCRULQFAAAAAAAAADtC0RYAAAAAAAAA7AgTbQEAAADpMG3aNHl5eUmSoqKirF7fM2TIEHl6etoiPQAAAGRhFG0BAACANCpQoIC+/vpry2tfX1+r1/e88MILFG0BAACQZhRtAQAAgDTavXu3rVMAAACAA2NOWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAACwAydOnNAbb7yhxo0ba8OGDcnGHD58WIMGDVKrVq00ZMgQnTt3Ll0xAAAAsG8UbQEAAAAbmzFjhlq0aKHcuXNr3bp1unjxYpKYw4cPq3r16oqNjVXv3r114sQJVa9eXZcvX05TDAAAAOwfRVsAAADAxjp27Khjx47pjTfeSDFm3LhxKl++vObMmaPOnTtr0aJFcnZ21tSpU9MUAwAAAPtH0RYAAACwsTx58sjJyem+MatXr1a7du0sr7Nly6ZWrVpp9erVaYoBAACA/aNoCwAAANi5iIgI3bhxQwULFrRqL1iwoE6fPp3qmOTExsYqIiLCagEAAIBtUbQFAAAA7FxcXJwkKXv27FbtOXLksKxLTUxyJk6cKF9fX8sSFBSUkakDAAAgHSjaAgAAAHbOx8dHzs7Oun79ulX7tWvXlCtXrlTHJGfkyJEKDw+3LKGhoRl/AAAAAEgThynaRkdHa9KkSapTp45KlSqltm3bav369Uni1qxZo0aNGumJJ55Q8+bN9eeff6YrBgAAAMgs2bJlU5kyZbR7926r9t27d6tSpUqpjkmOu7u7fHx8rBYAAADYlsMUbd9++21FR0drypQp+umnn1S5cmU1a9ZMW7ZsscRs3bpVrVq1UuPGjbVkyRKVLVtWjRo10tGjR9MUAwAAAGS23r1767vvvtPff/8tSfrzzz+1bt069e7dO00xAAAAsH9Oxhhj6yQyQmJiopydrWvQhQoV0vPPP6/Ro0dLktq2bau4uDitWrXKElOuXDnVqlVLX3zxRapjHiQiIkK+vr4KDw/PlJEKRUaseOT7QOY5/UGrTN0f149jyezrR+IaciS2uH6A9Mjse63MsHPnTo0YMULGGK1fv17lypVT/vz51aZNG7322muSpDt37qh379766aefVKpUKR06dEivvPKK/vOf/1j6SU3Mgzji+QXsFfdRgGPivhr3k9p7LddMzOmR+nfBdt26dbp06ZLq1atnadu4caOlgHtP8+bNtXz58jTFAAAAABmpaNGiGjFihKS7c8zeU7BgQcu/XV1d9fXXX+v06dM6e/asgoODFRAQYNVPamIAAABg/xymaCtJR48eVUhIiKKiohQTE6M5c+aoQYMGkqTIyEhFRETI39/fapv8+fMrLCws1THJiY2NVWxsrOV1REREBh0RAAAAHgd+fn5q3LhxqmKLFCmiIkWKPHQMAAAA7JfDzGkrSU888YS2bdumzZs364033lD//v21efNmSXenT5Dujj74Jzc3NyUkJKQ6JjkTJ06Ur6+vZQkKCsqwYwIAAAAAAADweHGooq2rq6sKFiyocuXKafz48apXr56mTJkiSfL29pa7u7uuXr1qtc3Vq1eVN2/eVMckZ+TIkQoPD7csoaGhGXxkAAAAAAAAAB4XDlW0/bccOXIoOjpa0t05b6tWraotW7ZYxWzevFlPPfVUqmOS4+7uLh8fH6sFAAAAAAAAANLDYYq2r732mk6fPi1JMsZo4cKFWr58uTp37myJeemll/TTTz9p/fr1kqSFCxfq999/14svvpimGAAAAAAAAAB4VBzmQWS1atVSq1atFBYWptjYWAUGBurjjz9W//79LTHdunXT6dOn1b59eyUmJsrd3V3Tp09Xw4YN0xQDAAAAAAAAAI+KwxRtu3btqq5duyoyMlLZsmWTu7t7snEjR47UG2+8oRs3bih37txycXFJVwwAAAAAAAAAPAoOU7S9x9vb+4Exrq6u932wWGpjAAAAAAAAACCjOcyctgAAAAAAAADgCCjaAgAAAAAAAIAdcbjpEQAAAAAAAABbKTJiRabt6/QHrTJtX8hcjLQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwI662TgAAAAAAgJQUGbEiU/d3+oNWmbo/AACSw0hbAAAAAAAAALAjFG0BAAAAAAAAwI4wPQIAAAAAAP9fZk/HAABAchhpCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAAAAAAAAYEccak7bqKgo7dmzR/Hx8Spfvrzy5MmTbNz+/ft15swZBQcHq1SpUumOAQAAAAAAAICM5jAjbcePH68SJUpoxIgRGjt2rAoXLqxJkyZZxcTFxal9+/Zq2LChpk2bpqeeekr9+vWTMSZNMQAAAAAAAADwqDjMSNt8+fLpyJEj8vb2liQtWbJETz/9tBo0aKBq1apJkqZNm6YtW7Zo3759KliwoA4ePKiqVauqQYMG6tGjR6pjAAAAAAAAAOBRcZiRtgMHDrQUbCWpffv2ypYtm/bs2WNpmz9/vrp27aqCBQtKksqWLasWLVpo/vz5aYoBAAAAAAAAgEfFYYq2//b7778rLi5OZcuWlSTduXNHhw8fVoUKFaziKlSooP3796c6JjmxsbGKiIiwWgAAAAAAAAAgPRyyaHvjxg317dtXrVq1Uu3atSVJt27dUkJCgnLlymUV6+fnp5s3b6Y6JjkTJ06Ur6+vZQkKCsrQ4wEAAAAAAADw+HC4om1ERIRatGihnDlz6ttvv7W0u7u7S5Kio6Ot4m/duiUPD49UxyRn5MiRCg8PtyyhoaEZciwAAAAAAAAAHj8O8yAySYqMjFTz5s11584drV27Vj4+PpZ12bNnl7+/v86ePWu1zdmzZ1WsWLFUxyTH3d3dUvAFAAAAAAAAgIfhMCNtb926pebNmysuLk5r1qxRzpw5k8S0aNFCixcvVmJioiQpJiZGy5YtU4sWLdIUAwAAAAAAAACPisOMtG3Tpo3279+vqVOnat26dZb2MmXKqEyZMpKk0aNHq1q1aurYsaNatmyp7777Tq6urho6dKglPjUxAAAAAAAAAPCoOMxI24CAADVr1kyrVq3Sd999Z1kOHDhgiSlSpIh2796tUqVKacOGDapbt6527NghPz+/NMUAAAAAAAAAwKPiMCNt//nQsfsJCgrSxIkTHzoGAAAAAAAAAB4FhxlpCwAAAAAAAACOgKItAAAAAAAAANgRirYAAAAAAAAAYEco2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgRirYAAABAFvDuu+/Kw8PDailYsGCSuAULFqh8+fLy9fVVtWrVtHbtWhtkCwAAgIdB0RYAAADIAu7cuaMqVaro5s2bluXkyZNWMatWrVLPnj01ePBgHTlyRG3btlXr1q114MABG2UNAACA9KBoCwAAAGQRTk5OViNt3d3drdZPmjRJbdu2Vb9+/RQQEKB33nlHpUqV0tSpU22UMQAAANKDoi0AAACQRezZs0d58uRRoUKF1LlzZx0/ftyyLjExUVu3blXDhg2ttgkJCdGWLVsyO1UAAAA8BIq2AAAAQBaQN29eff755/rrr7+0atUq3blzRzVr1tTFixclSTdv3tTt27eVL18+q+3y5ctniUlObGysIiIirBYAAADYFkVbAAAAIAt46aWX1KtXLwUEBKhMmTL69ttv5ezsrJkzZ1rFOTs7J3ltjEmx34kTJ8rX19eyBAUFPZL8AQAAkHoUbQEAAIAsKHv27CpVqpSOHTsmSfL19ZW7u7uuXLliFXf58mXlz58/xX5Gjhyp8PBwyxIaGvpI8wYAAMCDUbQFAAAAsqD4+HidOHFC/v7+kiQXFxc99dRT2rhxo1Xcb7/9ppo1a6bYj7u7u3x8fKwWAAAA2BZFWwAAACALePbZZ7Vjxw7FxcXp/Pnz6tevn27evKl+/fpZYoYMGaLFixdr0aJFio6O1qeffqp9+/bplVdesWHmAAAASCuKtgAAAEAW0KdPHw0bNkw5c+ZUuXLldOXKFW3ZskWlSpWyxDz99NP66KOP9Nprr8nLy0sfffSRvv/+e1WtWtWGmQMAACCtXG2dAAAAAIAHa9y4sRo3bixjjJycnFKMGzRokAYNGqTExMQkDyUDAABA1sBdHAAAAJCF3K9g+08UbAEAALIuRtoCAACbKTJiha1TQAY6/UErW6cAAAAAOAQ+fgcAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO+Jq6wQAAAAAAAAApF2RESsydX+nP2iVqft7nFG0BQAAAACkSWYXCQAAeNw41PQIhw4d0quvvqo6depo9erVycb89ddf6tu3r0JCQjRw4ECdOnUqXTEAAAAAAAAA8Cg4TNF2xowZ6tSpk4oVK6YtW7bo8uXLSWIOHTqkWrVqydXVVYMHD9b169dVo0YNhYWFpSkGAAAAAAAAAB4Vhynadu3aVYcOHdLgwYNTjJkwYYIqVKigL774Qm3atNGCBQvk5eWlKVOmpCkGAAAAAAAAAB4Vhyna5syZ84Exa9euVZs2bSyvXVxc1Lp1a61ZsyZNMQAAAAAAAADwqDw2DyKLiorS1atXFRgYaNUeGBios2fPpjomObGxsYqNjbW8joiIyMDMAQAAAAAAADxOHGak7YPEx8dLkjw8PKzas2fPrri4uFTHJGfixIny9fW1LEFBQRmZOgAAAAAAAIDHyGNTtPX29pabm5uuXbtm1X7t2jXlzp071THJGTlypMLDwy1LaGhoxh8AAAAAAAAAgMfCYzM9gouLi8qXL69du3ZZtW/fvl2VK1dOdUxy3N3d5e7unvFJAwAAAAAAAHjsPDYjbSWpb9++WrhwoY4cOSJJ2rp1q9atW6e+ffumKQYAAAAAAAAAHhWHGWm7c+dODR482PJ6woQJ+t///qc2bdrozTfflCQNGjRI+/fvV6VKlfTEE0/o5MmTev3119WhQwfLdqmJAQAAAAAAAIBHxWGKtsHBwfrggw+StPv7+1v+7ezsrBkzZmjcuHEKDQ1V0aJFlSdPHqv41MQAAAAAAAAAwKPiMEVbX19f1alTJ1Wx/v7+VsXc9MYAAAAAAAAAQEZ7rOa0BQAAAAAAAAB7R9EWAAAAAAAAAOwIRVsAAAAAAAAAsCMOM6ctAAAAADyuioxYYesUAABABmKkLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB1xtXUCAAAAAOCIioxYYesUAABAFsVIWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADviausEAAAAAAAAANi/IiNWZNq+Tn/QKtP2ZY8YaQsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHmNMWAAAAwGMjM+fiAwAASC9G2gIAAAAAAACAHaFoCwAAAAAAAAB2hKItAAAAAAAAANgR5rQFAAAAAAAAYFcyex760x+0ytT9PQgjbQEAAAAAAADAjjDSNgXXr19XWFiYChcuLB8fH1unAwAAAKTavXvZIkWKyNvb29bp3Fdmj6IBAADIChhp+y/GGL322msKCAhQ+/btlS9fPk2YMMHWaQEAAAAPZIzRyy+/bHUvO3HiRFunBQAAgDRipO2/zJw5U3PnztXOnTtVvnx5bdy4UY0bN1bFihXVtm1bW6cHAAAApGj69OmaP3++du/erbJly2r9+vVq2rSpKlasqJYtW9o6PQAAAKQSI23/ZebMmerUqZPKly8vSapfv74aNWqkWbNm2TgzAAAA4P5mzpyprl27qmzZspKkRo0aqX79+tzLAgAAZDEUbf8hISFB+/btU7Vq1azaq1evrt27d9soKwAAAODB4uPj9ddff3EvCwAA4ACYHuEfIiMjFR8fLz8/P6v2PHny6Nq1ayluFxsbq9jYWMvr8PBwSVJERMSjSfRfEmOjM2U/yByZdd3cw/XjWDL7+pG4hhwJ1w8eVmZdQ/f2Y4zJlP1lFeHh4UpISOBeFgAAIB3s7V6Wou0/uLm5SZLVTask3b5927IuORMnTtS4ceOStAcFBWVsgngs+E6zdQbIyrh+8DC4fvCwMvsaioyMlK+vb+bu1I5xLwsAAJB+9nYvS9H2Hzw9PZU7d26dP3/eqv38+fMqVKhQituNHDlSQ4cOtbxOTEzU9evX5efnJycnp0eW7+MkIiJCQUFBCg0NlY+Pj63TQRbD9YOHxTWEh8H1k/GMMYqMjFSBAgVsnYpd8fX1la+vb5a6l+XnI304b2nHOUsfzlv6cN7Sh/OWdpyz9LH1eUvtvSxF238JCQnRihUr9Oabb0q6e9O6cuXK+z5t193dXe7u7lZtOXPmfJRpPrZ8fHz4RYR04/rBw+IawsPg+slYjLBNXqNGjbRixQoNGzZM0t1nNqxcuVIdOnRIcRt7uJfl5yN9OG9pxzlLH85b+nDe0ofzlnacs/Sx5XlLzb0sDyL7l7fffls7d+7Uq6++qjVr1qhXr166fv265cYXAAAAsFejR4/W1q1bNWTIEK1Zs0Y9evRQZGSk1UhaAAAA2D+Ktv9SoUIFbd68WVeuXNHYsWPl4uKiP/74475fKQMAAADsQaVKlbRp0yZduHBBY8aMkbu7u/744w8FBgbaOjUAAACkAdMjJKNKlSpasGCBrdPAP7i7u1v+8ADSiusHD4trCA+D6weZrVq1avruu+9snUaq8PORPpy3tOOcpQ/nLX04b+nDeUs7zln6ZJXz5mSMMbZOAgAAAAAAAABwF9MjAAAAAAAAAIAdoWgLAAAAAAAAAHaEOW0BAAAAPHKnTp3SjRs3VKVKlTRtExcXpxIlSsjZOfnxJqmJyari4uJ0/Phx5cyZ84EPkwsLC9OFCxeSXVelShU5OTkpJiZGBw4cSLK+ZMmS8vb2zpCc7cH169cVFhamwoULy8fH576xFy5cUFhYmFWbq6urKlWq9FD9ZkWnT5/W7du3VaJECbm4uDww/s6dOzp58qRy586tvHnzJll/6NAhRUdHW7XlzZtXhQsXzrCcM0t8fLyOHz8ub29vBQUFZdg26ek3K7l586ZCQ0MVFBSknDlzpmqb8+fP6/bt2ypcuLBcXa1LVpcvX9bZs2et2pycnNL0vpIVhIaGKjIyUsHBwXJzc7tv7NGjRxUZGWnVljt3bhUrVuyh+s1qEhISdOzYMWXPnl1FihS5b+yVK1d05syZZNeVL19e7u7uSkhI0J49e5KsL1asmHLnzp0RKaeOAQAAAIBHZPHixaZ+/frG29vbuLi4pGqbv//+21SuXNn4+vqa/Pnzm8KFC5s///wzzTFZ2eLFi03u3LlNUFCQyZ49u2nWrJkJDw9PMf6TTz4xVapUsVpy5sxpfH19zZ07d4wxxvz1119GkqlYsaJV3M6dOzPrsB6pxMRE8+qrr5ps2bKZYsWKGXd3dzN+/Pj7bjNhwgTj6elpdT4aNGjw0P1mJefPnzfVq1c33t7eJiAgwAQGBppNmzalGH/z5k0zePBg4+vra0qUKGF8fX1NzZo1zdGjR63iKlasaAoWLGh1bidMmPCoDyfD/fLLLyZv3rymYMGCxtPT0zRo0MBcu3btobdJT79ZyYgRI4y7u7vlZ2bEiBH3jf/mm29MyZIlTUBAgClatKjx8/Mzc+bMsYqZOnWq8fDwsLqmqlev/igPI1NdvXrVNGjQwHh6epqCBQuavHnzml9++eW+29SvX98EBARYnZPhw4c/dL9ZyaZNm0xgYKAJCAgw3t7epnr16ub8+fMpxn///fdJ3i/z5ctnXFxczNWrV40xxty4ccNIMqVLl7aKW7lyZWYdljHGGIq2yPLCwsLMrl27TEJCgq1TQRY0Y8YMc/bsWVungSxq9+7dZtCgQbZOA1nYunXrzNy5c01iYqKtUwEemREjRpj169ebuXPnprpoW7NmTdOkSRMTFxdnEhMTTf/+/U1gYKCJjo5OU0xWdebMGePh4WGmTp1qjDHm2rVrpkSJEqZPnz6p7uPOnTsmICDAvPjii5a2e0XbCxcuZHTKdmHGjBnG29vb7N+/3xhjzIYNG4yrq6tZunRpittMmDDhgUWf9PSblTRp0sTUqVPH3L592xhjzJAhQ0yePHlS/JDgr7/+MlOnTjVRUVHGGGOioqJMixYtTPny5a3iKlasaCZNmvRok3/ELl68aLy8vCzF5oiICFOhQgXTuXPnh9omPf1mJd98843Jnj272b59uzHGmD///NO4u7ubb775JsVtRo8ebVX4nzt3rnF2drb0Yczdom3ZsmUfXeI21rlzZ1OpUiUTERFhjDFm3LhxxsvLy1y6dCnFberXr2/eeuutDO83qwgPDzd58uQxr7/+ujHGmNu3b1vuD9KicuXKpm3btpbX94q2O3bsyNB804qiLbKs2NhYM3DgQOPi4mIkmVq1alluNIDUWLx4sZFkihUrRuEWabZ7926TL18+s3DhQlungiwoLi7OtGrVytSsWdPMmzfvvqPnAEeR2qLtgQMHjCSzceNGS1tYWJhxcnIyixYtSnVMVvbuu++aPHnyWEbIGmPM559/bjw8PCyFsgf5+eefjSSzZ88eS9u9ou3OnTvN4cOHTWxsbEanblNVq1ZNUthu2rSpadOmTYrbTJgwwVSrVs0cOXLEnD59OtkP0dLTb1Zx+vRpI8msWLHC0nbt2jXj6upqvvrqq1T389NPPxlJ5vr165a2ihUrmjFjxpj9+/dn2RGk06ZNM97e3iYmJsbSNm/ePOPi4pLiMaVmm/T0m5U0aNAgSQG6Q4cOpmHDhmnqx9fX1/z3v/+1vJ46daopXbq0OXr0qDl16pRDDdy6du2acXFxsSpsR0dHGy8vL/PRRx+luF39+vXNkCFDzP79+82VK1cyrN+s4ssvvzRubm7mxo0blrZ773+nT59OVR+7d+82ksyyZcssbfeKtitWrDAHDhyw2QfCjjXpEx4rffv2VVhYmE6ePKktW7Zo3759Wrt2ra3TQhbToEEDubi4qEGDBgoNDbV1Osgi9uzZo+bNm+uzzz5Tp06dbJ0OsqBPPvlEERER2rx5s3r06OGQcyMC6bV7925JUrVq1SxtBQoUUMGCBS3rUhOTle3evVtPPvmk1byi1atXV0xMjA4dOpSqPmbPnq2qVasmOzdr8+bN1bJlS3l7e2vYsGGKj4/PqNRtJiEhQfv27bO6JqS75+1B18SOHTvUsmVLVa1aVQEBAfrhhx8ypN+sILmfpdy5cys4ODhNx7d3717lzJkzybylEydOVOfOnRUYGKi6devq2LFjGZJ3Ztm9e7dljst7qlevroSEBO3fvz/d26Sn36xk9+7dD/0zc+rUKUVGRiaZn/Tw4cNq3ry5atSoofz582vevHkZkbLN7du3TwkJCVbnLXv27CpfvvwDz9unn36qzp07q3DhwqpWrZrVNfQw/WYFu3fvVnBwsNXvnurVq0tSsnPSJmf27NkKDAxUixYtkqzr3r272rdvL19fX/Xt2zfJ/MGPGg8iQ5a0fft27d69W3v27JG7u7sKFy6scuXKKT4+XsuWLVPNmjWVJ08eW6cJO1eqVCnduHFDGzZsUIMGDdSgQQNt2LBBLi4uioqKUnBwsK1ThB3au3evQkJCNHXqVHXq1ElRUVEaP368fvzxR7m4uKh79+566623HG5yf2SsxYsXq2fPnpaCzMWLFzV9+nSdP39ebdq0Udu2bW2cIZCy/fv3Ky4uLsX17u7uKl++fLr7v3btmrJnz67s2bNbtefJk0fXrl1LdYw9iY6OfmCxNX/+/JYHEV27dk0FChSwWn/v3jY1x3fp0iWtWLFCn332mVW7l5eXfvrpJ7Vv316StG3bNjVt2lTe3t4aO3ZsKo8m85w/f17nz5+/b8y9h6hFRkYqPj5efn5+VusfdE1UrlxZR48eVYkSJWSM0Ycffqju3burWLFiqlq1arr7taWDBw/q9u3bKa53c3NTxYoVJf3f9fTvB+uk5fj27t2rSZMmacyYMXJycrK0v/TSS+revbs8PT11/fp1denSRU8//bT27NmjbNmypfWwbOLatWvJ/t/fW5febdLTb1Zx584dRUREJHt84eHhSkhIeOCD7uLj49W3b1+VK1dOrVu3trSXKVNGBw4cUNmyZSXd/RC8d+/eKlKkiOrVq5fxB5OJ7v2/p/V3Ta9evbR06VL5+voqMjJSPXr0UNu2bXXgwAF5eXmlu9+sIrmfpXuvU3N8MTEx+vbbb/Xyyy9bXZeurq6aPXu2evfuLWdnZx06dEhNmzbVkCFDNGvWrIw9iPugaIss6ddff9Ubb7xh+WRyy5Yt2r17t1566SXdunVLrq6uWrFihWrWrGnjTGHPihcvrpMnTyp//vxWhVtXV1cNHjyYoi2SlSdPHuXOnVuzZ89WixYt1Lp1a+XMmVPDhw/XoUOH9N5772nfvn366aefbJ0q7FhMTIwuX74s6W4BrHHjxqpcubJiY2PVrl07TZgwQW+//baNswSSN3LkSF26dCnF9QUKFNDPP/+c7v7d3NwUHx8vY4xVAej27duWD8RSE2NPzpw5o4EDB943pnv37ho6dKiku8cXGxtrtf5eES41x/fVV1/J3d1d3bp1s2ovUqSI1ai1GjVqaODAgfr666/tsmi7dOlSzZ49+74xM2bMUJUqVSznJbnzdr9z1qpVK8u/nZycNGLECM2dO1ffffedqlatmu5+bWnMmDE6ffp0iutz5sxp+YbivWOIi4uz+hAktcd3/PhxtWzZUh07dtQbb7xhta5///6Wf+fOnVv//e9/VbFiRe3evVs1atRIyyHZjJubm6Kjo63aHvSzmJpt0tNvVuHi4iInJ6dkf2acnZ3l7Hz/L3wnJCSoR48eOnnypDZt2mR1Ppo2bWoV+8orr2ju3Ln69ttvs3zR9n6/a7y9vVPcrk+fPpZ/e3t76+OPP1bhwoX1+++/q3nz5unuN6tI7v0yJibGsu5BFi9erPDwcPXt29eq3cvLy6qtTJkyGj58uEaOHKkvvvjigddxRqFoiyzp1VdfVUJCgiTp0KFD6tChgz755BMNGDBAkZGRat26tQYNGqS9e/faNlHYNTc3NxUoUEDHjx9XqVKl9M033+ipp55Snjx5rD7RBf6pYMGCliJ/yZIl1bx5cy1YsMCyvl69eurUqZNWrlypli1b2jBT2LOaNWvqiy++0JAhQzRgwAB98MEHlhvDd999V+PGjdOAAQOUL18+G2cKJLVixYpH2n+hQoV0584dXb58Wfnz55ckGWN08eJFFSpUKNUx9qR06dLauXNnquMLFSqUZGTuvRGnqTm+OXPm6JlnnknVH+QFChTQuXPnUp1bZho0aJAGDRqUqlhPT0/lzp07ycjc8+fPp/maCAgIsJyTjOw3s/z444+pjr13DOfPn9cTTzxhab9w4YJVQTs5J06cUMOGDdWwYUPNnTvX6gOU5NwbPW6v11tyChUqpPXr11u1PehnMTXbpKffrMLJyUlBQUHJ/swEBQXd9zpJSEhQz5499fvvv2vDhg0qWrToA/dnz7/D0uKfP4sBAQGW9vPnz6tJkyap7icgIEBOTk6Wc5JR/dqrQoUKJZkmMy0/S7Nnz1aTJk2STMORnAIFCig6OlrXr1/PtG92M6ctsiRfX1/LV3iKFCmiBQsW6IUXXpCTk5N8fHw0YsSIVM/3hcdbqVKldPDgQZ0/f17PPfec3nrrLeXKlYs5bnFf9wq3fn5+euutt6zWdezYUcWLF0/TH+d4/AwZMkRXr17Viy++qJMnT1p9kv/KK6/ozp07D/xKMOBI/vrrL8v7bp06dZQtWzar4vDmzZt18+ZNhYSEpDomKwsJCdHOnTstI/IladmyZSpcuLClsHb79m3t3Lkzyfx6v//+u44ePWo1yvGee6OP/mndunUqXbp0Bh+BbYSEhFhdE4mJiVq5cqXVNXH+/HmrgR3/PifXr1/Xvn37rM5JavrNqqpXry5PT0+r49u9e7fCwsKsju/gwYM6c+aM5fXJkyfVsGFD1atXT/PmzUvydfeUrjVJWep6CwkJSXLsy5YtU968eS3TwMTGxmrnzp0KDw9P9TapicnKQkJCtHLlSstrY4yWL19udU1dunRJu3btsrxOTExUr169tHHjRv32228qXrx4kn7/fV1FRkZqx44dWeqaSkmFChWUN29eq5/FU6dO6dChQ1bn7ejRozp16pSku9eeMcaqn/Xr18sYYzknqe03qwoJCdG5c+es5vFdtmyZvLy89NRTT0m6O2XHzp07df36datt//77b/32229per/Mly9fkukYHimbPP4MSIM7d+6YmTNnmi5dupjBgwebCxcuPHCbadOmmdq1a2dCdsgKtmzZYvr27Wuee+45s2nTJqt1w4cPNwMGDDAlSpQwkyZNMsbcffp0cHCweeWVV2yRLuzMxYsXzahRo0zHjh3NRx99ZPWU2tDQUBMfH28Vn5iYaAoUKGC+++67zE4Vduh+72HffPONcXJyMi4uLlZPt127dq3Jmzev1ROlgazs1KlTZseOHWbMmDHGxcXF7Nixw+zYscNERkZaYgoXLmxee+01y+u33nrL5MqVy8ybN88sXrzYFCtWzHTq1Mmq39TEZFVxcXGmUqVKpnbt2mblypVm6tSpxtXV1cyfP98Ss2fPHiPJ/Pbbb1bb9urVy5QvXz7ZfocNG2ZeffVV8/PPP5tff/3V9OnTx7i6upoVK1Y8ysPJNPv27TPZs2c3r7zyilm9erV57rnnTO7cuc2ZM2csMWPGjDG+vr6W19WrVzdTpkwx69evNz/88IN58sknTeHChc3ly5fT1G9WNnHiROPt7W1mz55tli5dakqXLm2aN29uFVO2bFnTr18/Y8zde+VChQqZqlWrmm3btll+pnfs2GFu375tjDFm27Ztpn79+mbevHlm48aNZurUqSZnzpymZ8+emX58DyMhIcHUqlXLVK1a1Sxfvtx89tlnxt3d3fzvf/+zxBw/ftzqyfOp2SY1MVnZsWPHjLe3t+nfv79ZvXq1ef755423t7c5duyYJWbSpEnGxcXF8rpv377Gw8PDLFiwwOqaCg0NtcTUr1/ffPjhh2bdunVm0aJFpkaNGiYgIMCcO3cuU4/vUZk+fbrx8PAwn3/+uVm+fLl58sknTa1ataz+/qhdu7bp2LGjMcaYw4cPmxo1api5c+eajRs3ms8++8zkzZvXtG3bNs39ZmXNmzc3ZcuWNUuXLjWzZ882np6e5oMPPrCsv3LlipFk9R5qzN37iHz58pm4uLgkfU6aNMn07t3bLFq0yKxbt84MGTLEuLi4mFmzZj3y4/knirawazExMaZJkyamXLlypl+/fiZfvnymZMmSlpuB5Bw8eNDkyZMnSXEOj6ePPvrI+Pn5md69e5uqVasaFxcXs3XrVsv6RYsWGUmWgu09ly5dSvaXNx4vf/31l/H39zdt2rQxnTp1Ms7Ozmbo0KH33eaDDz4w5cqVM7GxsZmUJexVat7DvvvuO+Pp6WlKlChhvv/+ezNz5kwTEBBgFi9ebMPMgYz1zjvvmCpVqiRZ9u7da4lp06aNmTJliuV1YmKimT59umncuLGpV6+eeffdd5Pc/6UmJiu7du2aGTp0qKldu7Zp1apVkt8LR48eNVWqVDE7d+60tMXFxZl69eqZL7/8Mtk+4+PjzcyZM027du1Mo0aNzKBBg8zhw4cf6XFktp07d5pnnnnG1KpVy/Tq1cscOXLEav2MGTNMgwYNLK8vX75sRo0aZUJCQkybNm3MhAkTTERERJr7zermzJljmjZtaurWrWtGjx5toqKirNZ37drVTJgwwRhjzObNm5P9ma5SpYo5ceKEZZs///zT9O3b19StW9c888wz5vvvv8/UY8oo4eHh5s033zR16tQxLVq0MAsWLLBaHxoaaqpUqWI2b96c6m1SG5OV7d+/3zz77LOmZs2a5tlnnzX79++3Wj9//nxTvXp1y+uQkJBkr6lp06ZZYq5fv25Gjx5tGjdubFq1amVGjx5tbty4kVmHlCkWLFhgWrRoYerUqWPefPNNEx4ebrW+X79+Zvjw4ZbX+/btMwMGDDD16tUzXbp0MV9++WWyxdgH9ZuVRUVFmdGjR5u6deuapk2bmjlz5litv3nzpqlSpYpZuXKlVXv79u3Nf/7zn2T7TExMNN99953p1KmTadCggenXr5/Zvn37IzuGlDgZ86+x1IAdGTt2rP78808tW7ZMrq6uOnLkiMqXL6/p06fr+eeft4o9ePCgfv75Z33yySeaNm2aunTpYqOsYS/279+vRo0aaceOHSpatKgSExMtc9auWrXKErdt27Ys8zAEZB5jjCpWrKgXX3zR8vCYDz/8UKNGjdLly5etvhYTHx+v9evXa+7cuTp69Kh++umnVM2LBMeW2vewU6dOadKkSfrjjz/k7++vkSNHqkGDBrZLHAAAAIDNUbSF3UpMTFShQoW0fft2y6T1ktSoUSOVL19eH330kVX8ypUrderUKXXq1En+/v6ZnS7s0MCBA1WpUiWrpzV//vnneu+99xQWFmbDzJAVbNiwQWPHjtWGDRssbdevX5efn582bdqkunXrWtqNMZo2bZoKFy6sNm3aZPmn/uLhpfU9DAAAAAD+ydXWCQApuXXrlp5++mmrP3aluw8eu337dpJ4ntKOf3N3d9czzzxj1ZbS9QP82507d5KM6M+dO7e8vb2TXENOTk4aMmRIZqYHO5fW9zAAAAAA+CdnWycApMTHx0effPJJknYPDw8lJiZaXh87dkyHDh3KzNSQRXz00UfKmTOnVdu/r5+4uDirJ5sC9zRu3FjPPfdckvZ/X0O//PKLYmNjMzM1ZAG8hwEAAAB4GBRtYVfWr1+vWbNmad++fSnGODs7W/7gPXbsmEJCQu4bj8fH8ePHNWfOHP3yyy+Kj49PNuaf109cXJw6d+6sL7/8MhOzhL2KjIzUggUL9O233+ry5cspxv3zGpo7d64GDhyoixcvZlaasFPGGK1bt06zZ8/W/v37U4zjPQwAAABAajA9AuxCbGysnn76aR04cECenp46cuSIWrVqpVmzZiWZn9bJyUnGGMsfu++99566detmo8xhLz755BO9/fbbCg4O1l9//aUCBQpozpw5atiwoVXcvevnXsE2W7Zs+vbbb22UNezFvn371LJlS+XNm1dhYWG6deuW3n77bY0cOVLOztafb967hubOnauxY8dq3bp1Kly4sI0yhz2IiYlRu3btdOTIEWXPnl1Hjx5VmzZtNGvWLOXLl88qlvcwAAAAAKnBSFvYhfHjx8vJyUknT57U4cOHtX79eu3bt081atTQyZMnrWKdnJx06tQpyx+7PXv2tFHWsBe7d+/W+PHjtW/fPu3cuVOnTp1SmTJl1LRpU3399ddWsU5OTkpMTLQUbBcsWCBXVz6/epzdux7effdd7d27V+fPn9c777yjMWPGqHv37kpISLCKd3Jy0rx58ywF2+LFi9soc9iLMWPGyMPDQydPntSRI0e0du1a7d69WzVq1NDff/9tFct7GAAAAIDUoGgLu7B8+XJ1797d8sT1hg0b6s8//1T27NnVtGlT3bhxwxLr6uqq33//nT92YbFixQrVqVNHRYoUkSQFBgZq2bJl6t27t3r37q3Vq1dbYl1dXRUdHU3BFhZHjx7V8ePH1aNHD0mSm5ubRo0apUWLFmnx4sV65ZVXrOJdXV31xx9/ULCFxb33sHu/T0JCQrRt2za5ubmpadOmCg8Pt8TyHgYAAAAgNSjawi7kypVLW7ZssWorUKCAVq9erVu3bun111+3tA8cOFDffPMNf+zCIleuXNqxY4fi4uIsbc7OzpoxY4Zat26tXr16KTo6WpJUtWpVTZo0iYItLO49rO6PP/6wam/Xrp1mzZql6dOna82aNZb2Dz74QL/99hsFW1gk9x5WsGBBrV69Wjdv3tTw4cMt7byHAQCQ8Xbs2HHfOeUBICtyMsYYWycBzJo1Sy+++KJ+//13PfXUU1brvvnmG/Xu3VtXr16Vr6+vjTKEPbtw4YKeeOIJDRkyRO+9957VuqtXr6po0aL67LPPKJIgRY0bN9bVq1e1bds2eXh4WK1r06aNnJyc9PPPP9soO9i7//3vf3rttdf0xx9/qEqVKlbrvvrqKw0YMEDXrl2Tl5eXjTIEAMA+7dy5U/Hx8apZs+ZD9dOpUyf5+/vr008/zaDMAMD2GGkLu9C3b1/VqlVLbdu21dGjR63Wde7cWQkJCTp37pyNsoO9CwgI0MSJEzVx4kTNnDnTal2ePHnUqFGjJPNKAv/02Wef6cSJE+ratavViG1J6tKlC9cP7qt///6qVq2a2rRpo+PHj1ut69Kli+Li4hQWFmaj7AAAsF+TJ0/WuHHjbJ0GANglirbIVHFxcXr77bcVFBSkwMBATZo0SdLdr7IvWrRI+fLlU926dfXbb79Zttm/f79y5cqlJ554wlZpw44sWbJETz75pHLlyqUuXbpY5jt+7bXX9Morr+iFF17Q+PHjLQ+Pio2N1ZEjR1StWjVbpg07cerUKbVv3165cuXSk08+qa1bt0qSSpYsqcWLF2v16tVq0aKFLl++bNlm3759XD+QdPf3yVtvvaWCBQsqMDBQ//3vfyVJLi4uWrJkiXLlyqW6detq48aNlm327dunPHnyqGjRorZKGwCALGPz5s06cuSIoqKitHXrVm3dulWxsbFJ4qKjo7Vu3Trt3LlTMTExKfa3f/9+rVq1KsmHqocPH9by5cv1zy8eh4eHa8mSJbp06VLGHRAAPAQmdESmSUxMVPv27RUbG6sPP/xQf/75p4YPH67ChQurS5cu8vPz08aNG9WjRw+FhISoZcuWKlq0qBYuXKgvvvgiyVeW8fiZOXOmxo4dq7ffflsJCQkaM2aMXnjhBf3www+SpI8++kiBgYF6++239f3336tp06basGGDqlevrpYtW9o4e9jaiRMnVKdOHT333HPq2rWrPv74Y7Vr106HDx+Wn5+fmjZtqnXr1umZZ55RyZIl1blzZ928eVO7d+/W77//buv0YWOJiYlq06aNjDH6z3/+oz/++EPDhg1TkSJF1KFDB+XJk0ebN2/Wc889p4YNG6pVq1YqXLiwFi5cqJkzZypbtmy2PgQAAOzeyJEjlS1bNp05c0ZFixbV8ePH5eXlpS1btlieQ3Dw4EE1bdpUOXLkUP78+XX58mXlyJFD/v7+ln4uXbqkp59+WufPn1fJkiW1f/9+1ahRQ999953c3d3l5eWlnj17avTo0Ro8eLAk6YUXXtCxY8e0bds2Gxw5ACTFnLbINN9++62mTZumP/74w/IAqBYtWig+Pl5r1661il29erWWLVsmZ2dn9e7dW5UrV7ZFyrAj169fV4kSJbRx40aVLVtWkjR79mz1799fV69eVe7cuS2xJ0+e1FdffaULFy6oYcOG6tatm5ycnGyVOuxE27ZtVbt2bb355puSpIsXL6pYsWL6+OOP9fzzz1vioqKiNG/ePO3atUvFihXToEGDlCtXLlulDTsxb948TZ8+Xb///rtcXFwkSU2aNJGLi4tWrVplFbtq1SqtWLFCLi4u6tOnjypWrGiLlAEAsHvPPPOMbt68aXkvrVOnjk6cOKHdu3erQIECun37tsqUKaOBAwda7uHq16+vvHnz6ocffpCzs7MWL16sjh076qWXXrLMadusWTMFBQVpxowZcnFxUVRUlOrWrasOHTro7bffliQtXLhQPXr00Pbt27V792699NJL2rVrl0qVKmWbkwEA/8JIW2SauXPnatKkSZaCrXT3AT+TJ09OEtu0aVM1bdo0M9ODnVu4cKG6detmKdhKsox6O336tFXR9oknntD48eNtkSbs1JUrV7R3714tWbLE0ubv76+qVavq1KlTVrGenp4aNGhQJmcIezd37lxNnjzZUrCV7v4OSu6BJ82bN1fz5s0zMz0AABxG586dVaBAAUlS9uzZVbt2bR05ckSSFBYWpk2bNmn79u1ydr4722OHDh2sCq3nzp3T6tWr9fHHH+uXX36RMUbGGJUuXVpr1661FG07d+6slStXqnPnzrpw4YKmTJlCwRaAXaFoi0zTo0cP1atXz6rN399f8fHxNsoIWUmNGjX01FNPWbXly5dPzs7OXEN4IC8vL02ePNlyc38Pv4OQWr1791atWrWs2rh+AADIeH5+flavPTw8dOvWLUnSmTNnJEnFihWzivnn80/ufSC/atUqubm5WcWVK1fO6vXEiRNVsGBBy2heALAnPIgMmaZnz55JvqLu5uZmNfn7iRMnNGrUqMxODVlAxYoVk50mw9XV1XINxcfHa8iQIVYPkQKku6M0unTpkqT937+D5s6dq19++SUzU0MW0atXrwe+hx07dkxvvfVWZqcGAMBj496368LDw63ab968afm3t7e3JOndd9/VkiVLrJZ/f0Nm1KhRKlSokA4fPqwVK1Y82uQBII0o2sKmnJyclJiYKOluwTYkJETBwcE2zgpZyb1rKD4+Xl27dtWZM2eYfxSp9s/fQXPnztXYsWP5HYRU++f1c+zYMYWEhKhkyZI2zgoAAMdVrFgx5c+fXz///LOl7fz589qxY4fldfny5eXv769Zs2Yl2f6fgzsWLVqkb7/9VkuWLNHbb7+tvn37MvgDgF1hegTYlJOTk4wxloLt2LFj1adPH1unhSzEyclJcXFx6tq1qyTp+++/T/I1KCAl934H3SvYrlu3TsWLF7d1Wsgi7l0/9wq27733nnr27GnrtAAAcFjZsmXT+PHj9eqrryoiIkKBgYGaNm2asmfPbolxdXXVzJkz1alTJ928eVPNmjXT9evXtWzZMjVv3lxvvPGGwsLCNGDAAL3//vuqUKGCypYtq9WrV6tfv35atmyZDY8QAP4PRVvYlLOzs27dukXBFunm7OysQYMGqXTp0hRskWbOzs5auXKlFi9eTMEWaebs7KyIiAgKtgAApFO1atUUFRVleV2vXr0kDwOrXLmyYmJiLK8HDBigvHnzatGiRbp06ZImT56sffv2WX3brnXr1tq3b5++/PJLrVixQgULFtTo0aNVv359SdKPP/6obt26aciQIZIkFxcXff311xo6dKh27NihatWqPcrDBoBUcTL/nIwNyGS///676tWrp9mzZ1OwRbr4+/urVq1aFGyRLi+//LKWLl2q3377jYIt0mzDhg0KCQnR3LlzKdgCAAAAyFAUbWFze/fuVaVKlWydBrKoAwcOqGTJkhRskS7h4eG6fv26ihYtautUkEXxHgYAAADgUaBoCwAAAAAAAAB2xNnWCQAAAAAAAAAA/g9FWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAcQHR2tb775RqdOnUqy7sqVK/r666916dIlG2QGAAAAAADSyskYY2ydBADg4T377LM6ePCgtm/frmzZslna27Ztq2vXrmnTpk1ycXGxYYYAAAAAACA1KNoCgIO4efOmypcvr+eee04TJ06UJM2ZM0evvvqq9u3bpyeeeEIJCQnatm2bLl68qCeeeEKVKlWy6uOPP/7QqVOn5OTkpDx58qhy5crKly+fVcyvv/6qwMBA5c2bV9u2bZOvr68aNGiQSUcJAAAAAIDjo2gLAA5k7dq1atGihTZu3KjAwEBVqFBBkyZN0oABA3T27Fm1bt1aCQkJKlmypHbt2qXSpUtryZIl8vDwkCRNnz5dW7ZskSSdP39eO3bs0BdffKFu3bpZ9lGjRg25ubnp3LlzqlSpkurUqaNhw4bZ5HgBAAAAAHBEFG0BwMG88sor+uWXXxQYGCgvLy+tWLFCklS3bl1VqVJF06ZNkyTFxMSoTp06ateund55551k+1q0aJH69++vsLAwZc+eXdLdou3ff/+t/fv3K3/+/JlyTAAAAAAAPE4o2gKAg7l9+7YqVaqkK1eu6ODBgwoICNDJkydVvHhxTZ48WYGBgTLGyBijRYsW6caNG1q/fr1l+wsXLmj//v26evWq4uLi1LdvX+3bt08VKlSQdLdoW6VKlf/H3n1HR1V9fx//pDdSIIVQQiB0KQEiXYogXUVFpKmgiEpHxYIFVPSHqICKCIqAiqiICAiIFBUFQQRpAtJbCL0lIaTnPH/wZL4MSWBSZ4Lv11qzYM7d95x970ySm50z52rKlCn2OkQAAAAAAG5qrvZOAABQsLy8vNS0aVPt379fZcqUkSQdPnxYkvTXX3/Jzc3NKrZatWqW5+PHj9fYsWMVFRWl0NBQS+yZM2esxggNDS3kowAAAAAA4L+Loi0A/Af4+/tLkl588UVFRkZmG3PhwgWNGjVKK1euVNu2bSVdKdbOmTNH134ow8nJqXATBgAAAADgP8zZ3gkAAApfvXr1VKZMGX300UdZth07dkySdPbsWRljVKVKFcu2efPmFVmOAAAAAADgCmbaAsB/gKurq2bNmqV7771Xp0+fVvv27XX27FktWbJEffv21aBBgxQREaG6deuqd+/e6tu3r3bv3q2vvvrK3qkDAAAAAPCfw0xbALgJNWvWTB06dLBq69Chg3bt2qWoqCht2rRJaWlpmjp1qgYNGiRJcnFx0a+//qpOnTppw4YN8vX11YYNG9SnTx/L2riS1LFjR9WpU6dIjwcAAAAAgP8SJ3PtQoUAAAAAAAAAALthpi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AAAAAAAAAOBCKtgAAAAAAAADgQCjaAgAAAAAAAIADoWgLAAAAAAAAAA6Eoi0AwCItLU3Jycn2TgMAAADFXFJSktLT04v9eMnJyUpLSyvwfq8nPT1dSUlJMsYU6bgAHAtFWwC4joyMDHunkKPCKLCOGDFCgYGBBdpndigOAwAAR5GamqqkpCSlpKTkGJOSkqKkpCSlpqYWYWb2k3lOMh+5vSa+dOmSvLy89M477+Q5h9wUYQtivJyULl1aQ4YMKfB+r2f69Ony8vLSkSNHCnUcisOAY6NoCwDX2LRpk/r06aNy5crJw8NDJUqU0C233KJhw4Zp165d9k7P4oUXXpCPj4+908iT4pw7AAC4uXTv3l1eXl4qWbKk4uPjs2w/f/68fH195eXlpUceecQOGRa9zHMSEBCggIAAubm5KSwsTE899ZRiY2NvuL+Tk5M8PDzk6uqap/HPnj0rLy8vTZo0yab4/I73XzV79mx5eXlpz5499k4FQDYo2gLAVd5//301btxYzs7O+vHHH5WUlKQzZ87ovffe08aNG/XAAw/YO0UAAAAUMHd3d6WmpmrevHlZtn311VdydXWVk5OTHTKzHxcXF8tM2/j4eI0dO1aTJ0/W3XfffcN9fXx8lJSUpJEjRxZBpkU/HgAUBYq2APD//fzzz3rqqac0ePBgzZ49W5GRkXJxcZGXl5fat2+vtWvX6uGHH85x/5w+Npaamprtx+0yP450tWuXDcjpo0qpqamWj4td/dG17OLz83GnnPZNS0vLkvvV+1zv42zXyz03/dp6rrLjyMteAACAoufl5aUuXbros88+y7Jt1qxZuv/+++XsnPXX56uXEbjR8gpXL62Q03XL9a5RCqIPW/O9lre3t/r166devXrp999/17Zt2yRlXcIgLS3NMn5214PXtmWXqzHGco2XeW2YlJR0w3Vl8zpedvJ7XZl53ZrdchqZvwPk9no0u5xutLzB9c5bWlqaJb/M5T8y88pNv9nF2nr+inLNY6A4omgLAP/fG2+8IW9vb73xxhvZbndxcdFzzz1n1bZ8+XI1a9ZM7u7u8vDwUN26dTV79myrmO7du6tu3bpZ+ps6daq8vLx07NgxS1vmmrJxcXF68MEHVaJECXl6eqpbt246f/68Ja5v376aPHmy0tPTLR9bCwgI0KFDhyRJ+/fvV7du3RQYGCgPDw9VrlxZL774ouLi4mw6F+fOnVO3bt3k4+Mjb29v9ejRQ6dOnbJs//TTT+Xl5aUNGzZk2ffjjz+Wl5eX/v7772z7vl7uuek381zdKNdM27Zt09133y1fX1+5ubmpYsWKGjduHAVcAAAgSerXr5/WrFmjAwcOWNq2b9+uzZs357gsQv/+/S3XMt7e3vL19VWnTp0sRc1MjRo10l133aXt27eradOm8vDwUMWKFbVgwQJJ0pYtW9S8eXN5enoqJCRE06ZNyzJWQfRha745qVq1qiQpJiZGx44dk5eXl6ZOnao5c+aoatWq8vDw0Pfff5/tGrNXx8+bN09VqlSRm5ubKleurPnz51vitmzZosqVK0uSXnnlFUu+Q4cOzTGv/IyX6dSpU+rRo4e8vb3l4+Oje++9V2fPns12vNTUVL3xxhuqUqWK3N3d5eXlpTvvvFO7d++WdKVo2aVLF4WGhlqtS5ucnKxGjRqpatWqVtf2OUlJSdHQoUPl7+8vd3d3tW3bVjt37rRsX79+vby8vDRz5sws+65du1ZeXl764osvsu37rbfe0qBBgyRJDRs2tJznpUuX5qrfzPV39+3bd91cMx07dkz9+vVTUFCQ3NzcFBISomHDhikhIeGG5wP4zzEAAJOQkGBcXV1N27Ztbd5n0aJFxtnZ2fTv39+cOHHCnD9/3rz22mtGkpk4caIlrmvXrqZ69epZ9p88ebKRZKKjoy1tgwcPNj4+PqZfv35m5cqVJikpyfz+++8mICDA9O3b12r/Z555xri4uGSbW9WqVU2bNm3MoUOHTEZGhjl8+LAZP368+frrr697TJnjP/DAA5bx//zzT1OpUiVTu3Ztk5iYaIwxJj4+3vj5+ZmHH344Sx9169Y1tWvXvu44OeWem35tzdUYY/7880/j5eVl7rvvPrN//36TnJxsli5dakqVKmWGDh163VwBAMDNrWvXrsbf39+kpqaakJAQ88orr1i2jRgxwlSsWNFkZGQYFxcX06dPnxz7SU9PN3v37jVdu3Y1oaGh5uzZs5ZtkZGRpmHDhua+++4ze/fuNQkJCWbkyJHGzc3N/Prrr6Zz585m9+7d5vLly+aFF14wkszWrVut+i+IPmzNt2vXrtleq/Xo0cNIMv/884+Jjo42kkyXLl3MoEGDzMmTJ82+ffvMH3/8YeLj440kM27cOMu+mfH33HOPGT58uDl16pS5cOGC6d27t3FzczNHjhyxxJ45c8ZIMu+8806O+V8tv+MlJSWZ2rVrm0qVKpn169ebpKQks2rVKtOtWzfj5+dnnnjiCUtsRkaG6dKliwkKCjLz5883CQkJJiYmxtx///2mVKlS5ujRo8YYY06dOmXKli1rGjZsaJKTk40xxgwYMMC4ubmZ9evXX/d4pk6daiSZbt26mS+++MIkJCSYvXv3mmbNmpnAwEBz7NgxS2xkZKRp0KBBlj569eplSpQoYeLj43McZ9asWUaS+ffff7Nss7Xf3OR67NgxU6ZMGdO4cWOzefNmk5KSYv766y/L7y4ZGRnXPS/Afw1FWwAwxuzfv99IMv369bN5n+rVq5saNWqY9PR0q/YuXboYX19fy4VMbou2ksy3335rFZtZ5ExKSsrSdq2TJ08aSebjjz+2+ViuHX/27NlW7atWrTKSzKeffmppGzJkiPH09LS6wF+zZo2RZCZNmnTdca5XcLa139zk2rBhQ1O1alXLBXOmDz/80Dg7O1tdtAMAgP+WzKKtMcY89dRTJjw83GRkZJiUlBQTHBxsXn31VWOMuWHRNi0tzSQmJpqjR49muUaJjIw0bm5uloKeMVcmDXh7exs/Pz9z6NAhS3tiYqIpUaKEGT58uFX/BdGHrflmFm0TExNNYmKiOXnypPnggw+Ms7Ozad++vTHmf0XRmjVrZim2Xa+Iem0h8NSpU8bZ2dmMHTvW0laQRVtbxps5c6aRZFauXGkV++WXXxpJVkXb+fPnG0lZJkMkJCSYkJAQM2TIEEvbmjVrjKurq3nyySfNZ599ZiSZ999//4bHk1kIff75563ajx49atzc3MyIESMsbdOmTTOSzJ9//ml1jO7u7qZ///7XHed6RVtb+81Nro888ojx8fExJ06csIpdvXq1kWR++umn6+YL/NewPAIASJYbSxgb1186fvy49uzZo7vuuivL+mb33Xef4uPjtWnTpjzn07FjR6vntWrVUnp6uo4ePXrDfYODgxUWFqaxY8fq448/tmmfa3Xp0sXqedu2beXt7a1ffvnF0jZo0CAlJSVp1qxZlrapU6fK3d1dDz74YK7HzGu/N8r1zJkz2rhxozp37ixnZ2elpqZa1hlu3ry5MjIytG7dujznCwAAbh6PPPKIjhw5ol9//VVLlizR2bNn1bdv3xzjDx06pD59+igkJERubm7y9/e3LCFw9TILklSzZk2FhYVZnnt7eyssLEzly5dXxYoVLe2enp4KDw+3LHtVkH3kJt+rl7KqWrWqPv74Y7344ov6/vvvreK6dOmSq5u0XXudGxISoqCgoGyPtyDYMt6vv/4qDw8PtW3b1io2u5uuLV26VC4uLurcubNlXdiUlBS5urqqYcOGWrNmjSX2tttu01tvvaVp06ZpwIABeuCBBzRs2DCbc7/2OjcsLEz16tWzuiZ/8MEH5e/vr6lTp1raZsyYoZSUFPXv39/msa6V235tyXXp0qVq0qSJgoKCrK7JGzZsKDc3N6tzB4A1bQFAklSmTBm5ublZrTl1PRcuXJAklS5dOsu2kJAQSbrhOlU5raWaub7Y1TKfx8bG3jA3Z2dnrVq1Sk2bNtWzzz6r8PBwRURE6MUXX1R8fPwN93d1dVXJkiWztAcHB1sdU82aNdWmTRtNmzZNxhidOXNG3333nbp27aqgoKAbjpOT3PRrS66nT5+WJE2ZMkUlSpSQr6+vfH195efnpyZNmsjDw8Om8woAAG5+derUUf369fXZZ59p1qxZat26tVUx9GqXL19Wq1attHv3bi1btkzJyclKTk62rIN67U2oQkNDs/RRokSJHNuzuz7JTx+5zdfFxcVyc6q4uDjt2LFDY8eOlY+Pj1Vc2bJls4x9Pdnl6uvrW2jXY7aMd/78eQUHB2cpPvv6+srT09Oq7fTp00pPT1dISIjl2tLPz09+fn5atWpVlrVZu3fvLnd3d6WmpmrEiBG5yj3z94pr266+Jvfx8VHfvn01d+5cnT9/XhkZGfrkk09Us2ZNNW3aNFfjXS23/dqS65kzZ7R69eos1+SlSpWSs7OzzfffAP4rKNoCgK7cMfj222/Xn3/+aSnIXk9m8fDkyZNZtp04ccIqJiAgINtiaUxMTLZ952amQk6qVaumb7/9VhcuXNDff/+tPn366O2339bAgQNvuG9aWlqWgrMxRqdOnVJwcLBV+6BBg3TgwAGtWLGiQP6in9t+bck189/nnnvO8ovHtY8nnngi3zkDAICbQ79+/TR//nwtW7YsxxuQSdKaNWsUHR2tN954Q1FRUXJzc5OkHGeM5nSNl5trv/z0kdt8bZXZj60K4lq3oMcLCgrSmTNnsnzq7uLFi0pKSrJqCw4Olqenp+Lj47O9rty3b58lNiUlRQ888IBKlSqlqlWr6pFHHtGlS5dszj27m+uePHky22vyzE+q/fjjjzp8+HCBXZPb2q8tuQYFBalLly45XpN/8MEH+c4ZuJlQtAWA/2/06NFKTU3Vc889l+32lJQUjR49WtKVGba1a9fWwoULlZ6ebhU3b948BQQEqGHDhpKkKlWq6OTJk1YXMhkZGVqyZEm+8i1RooTS09OVlpaWY4yLi4saNGigsWPH6vbbb7d5GYCFCxdaPV+2bJmSkpJ0xx13WLV37dpV5cqV04cffqiPP/5YYWFhateuXb5zz02/N8o1JCRETZo00ffff6/k5OQb5gYAAP7b+vTpIxcXF/n7+6tbt245xrm4uEi68smfq3322WeFmV6eFad8M2fzFtW12x133KHk5GQtW7bMqv3apSCkK9epSUlJWrBgwQ37feaZZ7Rp0ybNnTtX8+fP19GjR/XYY4/ZnNe117mHDh3S1q1bs1yTV69eXW3bttW0adM0ZcoUubm56aGHHrph/yVKlJCU83nOTb+25Nq1a1f99ttvlkkuAK6Poi0A/H/NmzfX9OnT9fnnn+uee+7R77//rgsXLujUqVOaN2+eGjRoYHVxNmHCBB0+fFgPPvig9uzZoyNHjmjkyJFauXKl3nrrLXl5eUm6MlvDzc1NQ4YM0dGjR7V37171799f9erVy1e+devWlSR99913SkhIUFJSkowx2rx5szp37qzFixfr6NGjunDhghYvXqy//vrLpoKqj4+PfvnlF33//fc6c+aMVqxYoccff1xRUVHq1auXVayrq6sef/xxLVmyRIcPH1a/fv2yrPGbm9xz26+tuU6bNk0nT57UnXfeqT/++EPnz5/X/v379e2336pFixY6d+7cDXMGAAD/DYGBgYqLi9PZs2fl7e2dY1zjxo1VpkwZvfLKK9q1a5diYmL0zjvv6MyZM0WYre2KU75eXl6qWrWqVq1apZMnTyopKem6ExXyq0ePHoqKitLjjz+u5cuX68yZM1qwYIFWrFghf39/q9iuXbuqV69eeuKJJ/TRRx/p0KFDOnv2rP7880+NHDlSr7/+uiRp7ty5+vDDDzVu3Di1bNlSderU0UcffWRpt8WpU6c0ZcoUnTx5Ups2bdL999+v4OBgjRw5MkvsoEGDtH//fv3000+68847s12u4Fp16tSRk5OTFi5cqLi4OCUlJWVZws3Wfm3J9c0331RQUJA6dOigH3/8UadPn9bRo0e1bNky3XvvvaxpC1yDoi0AXOWRRx7R9u3bVbZsWT3xxBOqVKmSGjRooEmTJqlbt2768ccfLbHt27fXr7/+qosXL6p58+aqV6+eNmzYoAULFlh93L58+fJasGCBDhw4oDp16qhXr17q0aOHWrRoIQ8PD6uPbLm5uWVZN0u6MjPCw8PDqnB57733atiwYXruuecUHBysgIAAHTp0SA0aNNDw4cM1c+ZMtWjRQlWrVtWrr76q0aNH3/ACMXP8jz76SD/88INq1aqlhx9+WJ06ddKKFSuy/fjb448/Ljc3Nzk5OV33I4RXyyn3vPRrS66RkZHaunWrKleurL59+6pixYrq0qWLFi1apPHjxyswMNCmvAEAwM3H3d092+uva3l6esrd3d3y3NfXVz/99JP8/f3VsmVLNWvWTGfOnNG0adPk4eFhdS3i4eFhtW9e2vPbR27yteWcODk5ycPDI8vM3Zy2XS8+u2OYMWOGLl++rOrVqysgIEBDhw7NVS65Gc/NzU0rVqxQp06d1LdvX9WqVUsLFizQxx9/LE9PzyzXwHPmzNG7776rr7/+Wrfeeqtq1aqlkSNHqnz58nr66ae1f/9+DRkyRPfdd5+effZZy379+vXTgAED9NJLL2nz5s05Ho+rq6s8PDw0duxYxcTEqGHDhurQoYMqVKigtWvXZls47dq1q8qXLy9JNi+NUL16dU2YMEFffvmlypQpo4CAAC1dujRP/b7++us3zDUkJESbNm1Sly5d9Nxzz6lq1apq1aqVPvroIz3yyCO67bbbbMob+K9wMrbeKh0AgGycP39eZcuW1W233aZVq1YVWb9DhgzRZ599lqt1wQAAAICbVUREhJKTk3X06FHLchiF3e+0adM0cOBAHTp0KMeb9gHIG2baAgDyZeHChUpOTtaAAQOKRb8AAADAzeavv/7SoUOH9OijjxZowbaw+gVwY1k/JwAAgA1SUlJ0/Phxvfvuu6pRo8Z1b9ThCP0CAAAAN5u0tDRdvHhRY8aMkZ+f33WXkXCEfgHYjpm2AIA8adeunerUqaNSpUpp3rx52a4XVpj95rT+LwAAAPBfMXToUIWHhysmJkbz5s2z6QZkBdlv5vq7V9+nA0DBYE1bAAAAAAAAAHAgzLQFAAAAAAAAAAdC0RYAAAAAAAAAHAg3IisEGRkZOn78uHx9fVnXBQAAoIAZYxQfH6+yZcvK2Zk5CAWNa1kAAIDCY+u1LEXbQnD8+HGFhYXZOw0AAICbWnR0tMqXL2/vNG46XMsCAAAUvhtdy1K0LQS+vr6Srpx8Pz8/O2cDAABwc4mLi1NYWJjlmgsFi2tZAACAwmPrtSxF20KQ+TEyPz8/LnQBAAAKCR/dLxxcywIAABS+G13LsggYAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA2FNWwAAAAAAABSp9PR0paam2jsNoMC5ubnJxcUl3/1QtAUAAAAAAECRMMbo5MmTunjxor1TAQpNQECAQkND83XjXIq2AAAAAAAAKBKZBduQkBB5e3vnq6gFOBpjjC5fvqzTp09LksqUKZPnvijaAgAAAAAAoNClp6dbCraBgYH2TgcoFF5eXpKk06dPKyQkJM9LJXAjMgAAAAAAABS6zDVsvb297ZwJULgy3+P5WbeZoi0AAAAAAACKDEsi4GZXEO9xirYAAAAAAABAAUpMTNSmTZuUlpZWbMeytd+LFy/q33//LdCxHdnFixe1e/fuQh+HNW0BAAAAAACAArRnzx41bNhQZ86cUVBQULEcy9Z+H3/8cUVFRalmzZoFNnZOdu/erbS0NNWuXTvb7cYY7d27V66urqpcubLVtgsXLujAgQPZ7lezZk35+PhYtV26dElHjhxRRESEZZ3azDFatmyptWvXqlq1avk8opxRtAUAAAAAAIBdTf/9YJGNNaBlRJGNVRS8vb0VFRUlNze3Ih97w4YNWrVqlT7//PNCHWfGjBn66KOPtHfvXpUuXVr79+/PErNlyxbdf//9unTpklJSUhQWFqbvv/9eVapUkSRt3LhRL774otU+J06c0PHjx7Vv3z5LXGpqqp555hnNmDFDFStWVGxsrF544QUNGTJEklSyZEkNGDBAo0aN0vz58wvtmFkeAQAAAAAAAMhBWlqaNm3apE2bNmnbtm2Ki4vLdR8XL17Uli1brNqSkpK0adMmy82qLl26pE2bNskYo8TERO3atUvx8fGW+HPnzmnXrl1KTk626icsLEzTpk2zzBS9up+kpCTt3r1bCQkJBX5MkvT++++rV69elpmotoydF5s3b9Ynn3yiZ555JtvtKSkpuu+++9SyZUudPHlSZ86cUfny5dWjRw9LTPv27S3HnPmoVauWmjVrZinYStLgwYO1cOFCbd26VTt37tSBAwdkjLEar1+/flq0aJGio6PzfWw5YaZtIao9ZrmcPbgjIgAA/zWH3+pi7xQAAABQQGJjY/Xkk09KulIc3L9/v+69917NnDlTHh4eNvWxevVqPfjgg7p06ZKl7fDhw2rYsKGio6NVvnx5bd26VS1atNAjjzyiH3/8Ud7e3jp16pRmzpypX375RYsXL5a7u7suX76sH3/8UbfeequkrMsYZPYzYsQIffXVV/L399exY8c0ffp09enTp8COKS0tTUuWLNHs2bMtbbaMLUmbNm26bt++vr6qXr265fmUKVMkSYsXL842fuXKlTp8+LBeffVVOTk5ydXVVa+88oqaNWumzZs3q0GDBln2OXLkiH7++WfNmDHD0nbo0CF9+umn+uqrr1S1alVJkoeHh4YOHWq1b9WqVRUWFqYffvhBgwcPvu6x5BVFWwAAAAAAcPNZNznv+zYbeuMY/GcEBgZaFRnPnDmj22+/XR9++GGOMz/zw9nZWTExMXJxcdETTzyhXr16aeDAgTp27JicnZ3Vp08fjRo1SitXrrxuPxcvXlRMTIxcXV01YcIEDR48WN27d5e7u3uBHNO///6r+Ph41a1bN1djp6enWwrGOYmKitLHH39sUx7SlZm4ISEhCg8Pt7Q1atRITk5OORZtZ82apRIlSuiBBx6wtP38889ycnLSnXfeqRMnTiguLk6VKlWSu7t7lv3r16+v9evXF1rRluURAAAAAAAAgBuIi4vTzp07deTIETVv3ly//PJLoYzz8ssvy8XFRZLUpUsXGWP0yiuvyNnZ2dK2ffv2G/YzZswYubpema/ZvXt3xcbG6siRI1Yx+TmmU6dOSVK2Nym73tguLi5Zlim49pGbgq10ZemIwMBAqzYXFxcFBATo3LlzWeIzMjI0a9Ys9enTR97e//uU/PHjxxUYGKhnnnlGDRo0UJcuXRQUFGSZ6Xu1oKAgnTx5Mld55gYzbQEAAAAAAIAcJCUl6eGHH9YPP/ygChUqyM/PT6dOnVJwcHChjBcaGmr5f2ZB8do2W9aJza6fzP0K4pgyl1FISkqSr6+vzWNLuV8e4Ubc3NyyrPWbmVt2N2hbtWqVjh49qscee8yq3cXFRWfOnJGLi4tiYmLk7Oysr7/+Wn369FFUVJSaNGli1benp6fNOeYWRVsAAADAARhjtGLFCm3ZskVdu3ZVzZo1s8Ts379fa9asUXp6uho2bKjIyMgsMcnJyVqyZImOHj2qatWqqVOnTpaZOQAAIPfee+89bdu2TceOHbPMKn3llVdyXF81O05OTlnaMm9AZg8FcUwRERGSpGPHjuWq2FsYyyNUqFBBp06dUnp6umWW8oULF5SYmKgKFSpkiZ8xY4aioqKyLJuQubzC0KFDLddPvXr10vDhw7V69Wqrou2xY8dUu3Ztm3PMLYq2AAAAgJ2tWLFCgwcPVnh4uH7++WdVrFgxS9H2oYce0saNG9WsWTNJ0lNPPaWHH37Y6uN6Fy9eVOvWrZWSkqIWLVrovffeU7Vq1fTjjz9mO8sEAADc2P79+3XrrbdaLQNwo/VkrxUSEqKEhARduHBBJUuWlHTj2aaFqSCOqVy5cqpcubLWr1+v+vXr27xf5vIIBalNmzZKSEjQb7/9pjZt2ki6ctMyV1dXtWzZ0ir23LlzWrRokd5///1s+3FxcdGJEycs12IJCQmKj4+3Wn4hPT1dmzZt0rBhwwr0OK5G0RYAAACws5IlS+qnn35SeHh4jsXVPn366IsvvrDM1Onfv79uu+029ejRw/LLyJtvvqm4uDht27ZNvr6+OnbsmG655RZNnz5dgwYNKrLjAQDgZtKqVSsNGjRIn3/+ucqXL69Zs2Zp8+bNuuWWW2zuo379+goLC9OTTz6pQYMGae/evXrttdcKMevrK4hjkqR+/frpu+++K/TrjD179ig+Pl7Hjx9XcnKypegbGRkpNzc31apVS3369FH//v317rvvKikpSU8//bRGjBihkJAQq75mz54tV1dX9erVK8s4ZcuW1YgRIzRw4ECNGzdOfn5+mjRpkkJCQnT//fdb4pYvXy4vLy917ty50I6Zoi0AAABgZw0bNpQkpaWl5RjTsWNHq+dNmzaVi4uLDhw4YCnazps3T71797asK1e+fHl17txZ8+bNo2gLAEAePfTQQ4qPj9eXX36p1NRUtWjRQh988IHWrFmT4z7e3t6Kioqy/DHW09NTK1eu1NixYzVmzBjVqlVLc+fO1fDhw+Xu7i7pyjquUVFRVssa+fn5KSoqyqrvkiVLWn2s/9qxsuvHzc1NUVFRlvVlbTmma/vNzsCBAzVp0iTt2rVLt9xyi01j58U777yjrVu3SpJKly5tWV7hxx9/tBRlZ86cqYkTJ2ry5MlydXXVq6++mu31z9atWzV8+HD5+fllO9bbb7+tKlWqaNq0aUpPT1f9+vU1c+ZMywxpSZo6daqeeuopy2tXGJyMMabQev+PiouLk7+/v8JGfCtnj7y/IQEAQPF0+K0u9k7hppZ5rRUbG5vjxXZxlZaWJjc3N3399dfq2bPndWPnz5+v+++/X//8849q166txMREeXt767PPPlPfvn0tcWPGjNHHH3+c492Nk5OTrW7cERcXp7CwsJvy/AL4j1k3Oe/7NhtacHnAIikpSYcOHVKlSpUK9QZOKFozZsxQdHS0Xn31VXunUiQOHjyoJ598UosXL7bcjO1a13uv23oty0xbAAAAoJg5cOCAnnzySQ0cONByA4xLly5Jkvz9/a1iAwICLNuyM27cOLt+PBMAABRv/fv3t3cKRSoiIkIrVqwo9HG4jSwAAABQjBw9elR33HGHWrZsqQ8++MDSnvmRw7i4OKv42NhY+fj45NjfqFGjFBsba3lER0cXTuIAAACwGTNtAQAAgGIiOjparVu3Vr169fTNN9/I1fV/l/M+Pj4qV66c9u/fb7XP/v37Va1atRz79PDwyPGjfQAAALAPZtoCAAAAxcCxY8fUunVrRUZG6ttvv832piD33nuvvv32WyUlJUmSzpw5o6VLl+q+++4r6nQBAACQD8y0BQAAAOzs0KFDmjt3rjIyMiRJP/zwgw4fPqz69eurQ4cOkqQ77rhDZ86cUYMGDTRhwgTLvi1btlSzZs0kSaNHj9ZPP/2kli1b6vbbb9cPP/yg6tWra+DAgUV/UAAAAMgzirYAAACAnaWmpurixYuSpOeff16SdPHiRSUmJlpiunXrpvT0dCUkJFjtm5ycbPl/cHCwNm/erLlz5+ro0aN6+eWX1b17d7m7uxf+QQAAAKDAULQFAAAA7KxatWp66623rhvz5ptv2tSXr6+vHnvssYJICwD+u9ZNzt/+zYYWTB4A/rNY0xYAAAAAAAAAHAhFWwAAAAAAAABwIBRtAQAAAAAAgBxs2LBBQUFBVuvI38iOHTsUFBSk8+fPF2JmhTuWrf2++eabeuaZZwp0bHs5ePCgatasqbi4OHunwpq2AAAAAAAAsLP8riOcG7lcczg1NVXnzp2TMcbmfdLS0nTu3DllZGTkNrtcK6yxbOn32LFjevvtt7Vjx44CHftaGzdu1Ouvv64dO3bI29tbffr00QsvvCBnZ+v5qEuXLtXEiRO1d+9e1a5dW2+99ZYiIyOz9Hf58mW1bt1aBw8e1Nq1a1WjRg1JUkREhBo0aKA333xT48ePL9RjuhFm2gIAAAAAAADFVJ06dXTmzBkFBgYW+dgffvihbr/9doWFhRXaGLt27VKrVq1Uo0YN/fzzz/r44481e/ZsvfDCC1ZxM2fOVLdu3XT//fdr3bp1evHFFzVu3Lhs+xw+fLh8fHx07tw5paWlWW17/PHH9fHHHyshIaHQjskWFG0BAAAAAACAG1i+fLlat26tChUqqFOnTtq7d6/N+y5btkwVK1a0atu/f7+CgoJ04sQJSf9bhuHrr79WmzZtVKFCBbVv31579uzR/Pnz1bhxY1WoUEE9evTQ2bNnLf38+++/qlGjhi5cuGDVz7Jly9S2bVtVrFhR7du3186dOy37nD59WkFBQQoKClKZMmXUrFkzzZ49O9fnZM6cObr33nstz20ZO7e++uorlS5dWu+8844iIiJ02223ady4cXr//fd18eJFSVJcXJxGjBih0aNHa+DAgQoLC1OLFi309ddfZ+nv22+/1V9//aU33ngj2/FatmwpFxcXLV68OM85FwSKtgAAAAAAAMANPPXUU3r11Ve1YsUK+fr6qn379javc5ucnGxVaJX+t/xAenq6pP8twzBx4kS99dZbWrVqlRISEnT77bdr4sSJmjJlipYvX679+/fr2WefzdJP5jIGmf28/PLLeuONN/TLL78oICBA3bp1syzxEBwcrN27d2v37t36+++/9fTTT2vo0KH64YcfbD4fhw4d0rFjx9SwYUNLmy1jp6enWwrGOT26du1q6TMpKUleXl5WY3t7eyslJUXr16+XdKWgHh8fr759+1rFOTk5Zcl52LBh+uqrr+Th4ZHtcTk5Oalhw4ZavXq1zeeiMFC0BQAAAAAAAG5gwoQJat26tWrUqKGZM2fq4sWLmjt3boGPM23aNDVq1EjVqlXTsGHDdOLECX3yySe69dZbVbNmTQ0aNEi//vrrDfv5+OOP1bRpU0VEROi1117Tnj17FBMTI+lKYTKzQFq2bFndf//9evLJJ3M12/bIkSOSpNDQ0FyN7eLiYikY5/T44osvLH116dJFu3fv1qxZs2SM0blz5yzLHhw/flyStHfvXgUGBurPP/9UvXr1VKlSJXXt2lVbtmyx9JOamqpevXrppZdeUq1ata57bGXKlNHRo0dtPheFgaItAAAAAAAAcANNmza1/L9EiRKKjIzUP//8U+Dj1KxZ0/L/zHVqr207d+7cDfu55ZZbLP8PDg6WJKv9Zs6cqdtuu01hYWEKCgrS5MmTLYVYW2TOEHZxccn12Deaaevv72+Jvf322zV9+nSNGTNG3t7eqly5su655x45Oztbzd6NjY3VhAkTNGPGDK1cuVLBwcFq3bq1pfj6yiuvqGTJkhoyZMgNj83FxSXLWrdFjaItAAAAAAAAcAOurq5ZnqekpOS5v8yC47WcnbOW665ty2nfG/WTud8333xjWRLht99+0+7duzVixIhcHU/ZsmUlSadOncrV2LldHkGS+vfvr6NHj+rcuXM6d+6c2rdvr4yMDIWHh0uSQkJClJaWpkmTJikqKkpVqlTRtGnTlJGRoUWLFkmSfvrpJ/3xxx8KDg5WUFCQ7rjjDklSixYtNHToUKvxTp8+rXLlytl8LgqD641DAAAAAAAAgP+2bdu2qW3btpKufNR+586d6tatm037+vv7KzExUampqXJzc5MkHT58uLBSvaFffvlFXbp0UY8ePSxtubmxmiTVqFFDpUqV0t9//61q1arZvF/m8gjXk3mOruXt7S1JmjdvngICAtS8eXNJ/5sFffXat66urnJzc1NqaqokafXq1VazZ7dt26Y77rhDixcvVr169azG+fvvv/Xaa6/ZfEyFgZm2AAAAAAAAwA288MILOnHihFJSUvTyyy8rOTlZvXv3tmnfOnXqyMPDQ9OmTZMkRUdHa/To0YWZ7nWFh4frzz//1IkTJ5Senq4vv/xS33//fa76cHJyUrdu3fTjjz/mevzcLI8gSU8//bTOnTsnY4wWLVqk8ePHa/z48ZYibmRkpNq0aaPXXntN8fHxSk9P11tvvaXLly+rc+fOkqSAgIBsxwgICFCJEiUsY23btk3nzp3T3XffnevjKkjMtAUAAAAAAChGpv9+MFfxA1pGFFIm/y0PPPCAGjRooPPnz6ts2bKaP3++AgICbNo3KChIn376qYYPH67nn39eZcqU0WOPPaZNmzYVbtI5GD58uDZs2KDw8HC5uLiodu3a6tOnj7Zv357rfpo0aaLz58+rVKlShZTtlTVy69Spo4sXLyo0NFRTpkxRv379rGLmzp2rxx57TEFBQXJxcVFERIQWLlyoGjVq5GqsmTNnqnfv3goJCSnAI8g9J2PLIhjIlbi4OPn7+ytsxLdy9vC2dzoAAKCIHX6ri71TuKllXmvFxsbKz8/P3uncdDi/AG4a6ybbb+xmQ28ckw/FtWiblJSkQ4cOqVKlSvL09LR3OjZLS0vTxYsXFRQUJElKSEiQj4/PdfdJT0/XhQsXFBgYKCcnJ6ttiYmJ8vLyssSUKlVKzs7OWcaRrizDEBsba9WWkpKi+Ph4y03Krh0ru36MMTp37pxKlixpdeOwlJQUpaeny8vLS4mJiUpOTrYUoq93DFcbOHCg/Pz8NH78+FyNnRe2nPuUlBQZY+Th4XHduMxcr84rJiZG9evX19atWy1r9ubF9d7rtl5rMdMWAAAAAAAAyIGrq6tVEfJGRUPpyrqtV+9ztcx1V6+NuXYc6crarte2ubu7Wwq2tvbj5OSUbT7u7u5WeV29Juz1juFqH3zwgRISEnI9dl7Ycu6vPqbryS7XoKAg7d+/3yH+cE3RFgAAAAAAAECeuLm52bxMhKPz8PC44QzdosKNyAAAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAUGWOMvVMAClVBvMcp2gIAAAAAAKDQubm5SZIuX75s50yAwpX5Hs98z+eFa0ElAwAAAAAAAOTExcVFAQEBOn36tCTJ29tbTk5Ods4KKDjGGF2+fFmnT59WQECAXFxc8twXRVsAAAAAAAAUidDQUEmyFG6Bm1FAQIDlvZ5XFG0BAAAAAABQJJycnFSmTBmFhIQoNTXV3ukABc7NzS1fM2wzUbQFAAAAAABAkXJxcSmQwhZws+JGZAAAAAAAAADgQJhpCwAAAAAAHM+6yfbO4KYx/feDuYof0DKikDIBYCtm2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAAAAscnvjstzgJmeAbZhpCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOxNXeCQAAAAAAgJvQusn2zgAAii1m2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAAIAdTf/9oL1TAOBgmGkLAAAA2FlaWpq+//57tWvXTkFBQVqwYEG2cYsWLVLTpk1Vrlw53X777frjjz/yFAMAAADHRtEWAAAAsLNJkybpyy+/1FNPPaVz584pOTk5S8yvv/6q+++/Xz179tTq1avVqFEjtWvXTnv27MlVDAAAABwfRVsAAADAzkaOHKnvv/9e7du3zzHmrbfeUufOnTV8+HBVrVpV48ePV0REhCZOnJirGAAAADg+irYAAACAnTk5OV13uzFGa9eu1R133GHV3q5dO61Zs8bmGAAAABQP3IgMAAAAcHAXL17U5cuXVbp0aav20qVL68SJEzbHZCc5OdlqOYa4uLgCzBwAAAB5wUxbAAAAwMEZYyRJLi4uVu2urq7KyMiwOSY748aNk7+/v+URFhZWkKkDAAAgD3JVtE1NTVVaWlquBsjLPnlVWGMV5TEAAICb17p169SxY0eFhYWpRo0aeuONN7Itpi1YsEAtW7ZUmTJl1L59e23dujXb/i5duqR69eopICBA+/btK+TsYU/+/v5yd3fXmTNnrNrPnDmj4OBgm2OyM2rUKMXGxloe0dHRBX8AAAAAyJVcFW07deqkl19+OVcDdO/eXSNGjMjVPnlVWGMV5TEAAICb0/bt29WmTRvdeuutWrdunWbNmqU5c+Zo1KhRVnFTp05V79691bdvX23ZskVjxozRu+++m22fgwYNUlBQkGJjY5Wenl4UhwE7cXFxUVRUlNauXWvV/ttvv6lx48Y2x2THw8NDfn5+Vg8AAADY1021PIK7u7vc3NzsnQYAAEAWc+bMUYUKFfTGG28oLCxMTZs21RtvvKH333/fsobohQsXNHLkSI0dO1b9+/dXaGiomjdvri+//DJLf7Nnz9auXbs0ZsyYoj4U2MmwYcP03Xffafny5UpPT9dnn32mTZs2afDgwbmKAQAAgOMr8huRpaSkyMXFxWqtrdTUVDk5OcnV1TXb58YYqzvqXvs805w5c6zar+0nO2lpaZalDzw9PfN3cAAAADlITk6Wh4eHVZunp6eSk5O1YcMGtWvXTsuWLdPly5f18MMPX7evffv26dlnn9Vvv/2mCxcuFGbaKCK//PKLHnjgAcvzxx9/XEOGDFG/fv0sM6179uypY8eOqVevXrp06ZICAwP12WefqVmzZpb9bIkBAACA48t10fbkyZPq2rWrfvvtN6Wlpal///6aOHFilhse5KRRo0Z68MEHNXLkSEvbQw89pBIlSujTTz+VdGUZhpCQEMXGxmrdunVKTk7Wo48+qmeeeUYDBw7Ur7/+qhIlSmjUqFFW/XTv3l3ly5fXhx9+aOknLCxMly5d0s8//6y0tDT17NlTH330kaWQ+9RTT2n69OlXToarqxo3bqzJkyfrlltuye2pAQAAyNFdd92lDz74QDNnzlS/fv10+vRpjRs3TpIUExMj6UoxtnTp0vr555/1xhtvKD4+XnXr1tXrr7+uBg0aSLryB/AePXro9ddfV/Xq1fXnn3/a7ZhQcFq0aKHdu3dnaffy8rJ6PnLkSD3zzDOKj4/PcRkDW2IAAIVr+u8H7Z0CgGIu18sjfP7552rbtq1OnDihn3/+WXPnztWECRMKPLF58+bpkUce0enTp7V48WJNnTpVjRo10sCBA3Xp0iV9/vnneu6557Rjx47r9jN79mx169ZNJ0+e1Nq1azVv3jzNnDnTsn3y5MlKSkpSUlKSYmJiVKtWLd17771KTU21Odfk5GTFxcVZPQAAAK7Wtm1bffrppxo7dqy8vLx0yy23qHfv3lYx6enpOnfunKZOnapvvvlGv//+u8qUKaM2bdpYbg71wgsvKCwsTI8//rg9DgOFxM3NTUFBQVkePj4+WWKdnJxuWIy1JQYAAACOK9dF2yZNmmjYsGHy8vJS48aN9fzzz2vSpEkFntg999yj+++/X25ubmrbtq1q1aqlTp06qWvXrnJzc9Odd96pihUr3nB2yb333quePXvK3d1ddevWVZcuXbRu3bpsY728vPTGG2/owIED2rVrl825jhs3Tv7+/pZHWFhYro4VAAD8Nzz66KM6dOiQ4uLidO7cOd12222SpIoVK0qSSpcurbS0NL333nuqU6eOKlasqI8++kipqalavHixJGnFihVatWqVAgICFBAQoHbt2km68mmmYcOG2eW4AAAAABSsXBdtMz+alykqKkonT55UbGxsgSUlSREREVbP/fz8VKlSpSxtFy9ezFU/AQEBVvts27ZN7du3l5+fn7y9vRUSEqL09HTLbBZbjBo1SrGxsZZHbvYFAAD/PR4eHnJyctK3336roKAgNW3aVJIs/1699q2rq6tcXV0ta/CvX79eMTExOnz4sA4fPqwFCxZIkn7++WeNHz++iI8EAAAAQGHIddHWGGP1PCMj40pHzrZ1ld0NxDL7uFFcdm15GS+TMUZdunRRzZo1tW/fPqWmpury5ctyc3Oz/GJkCw8PD/n5+Vk9AAAArjV06FCdPn1aGRkZmjdvniZMmKB3333XUqRt0KCBWrdurdGjRys2Nlapqal6/fXXlZaWps6dO0uSfH19LbNsAwICVKJECUv7teufAgAAACiecl203bBhQ5bnYWFh8vX1tWn/kiVL6ty5c1Zte/bsyW0aBSImJkYxMTEaOnSoSpcuLScnJ23atClX69kCAADYql69eqpfv768vLz08ssva8aMGerbt69VzLfffitjjEJCQuTn56dFixbphx9+UJUqVeyUNQAAAICiluui7ebNm/X666/r1KlTWr58ud5++20988wzNu/fqlUrzZkzR1u3btWpU6c0evRobd++PbdpFIjQ0FAFBgZq6tSpOnfunDZu3Kj+/fvbJRcAAHDz69+/v2JiYhQXF6c9e/ZkuRGZJAUHB+v777/X5cuXFRcXp82bN6tt27Y59tmwYUNduHBB1apVK8zUAQAAABShXBVt3d3dNWjQIO3fv18NGzbUI488okGDBmno0KHX3cfNzc3y/Nlnn1WHDh3UqVMnNW/eXImJierevbtVzLX7SFeWIHB1db1u27X7ZdePm5ub3N3dJV1ZI27+/PlavXq1IiIi9NBDD2nQoEEKDAyUi4vLdfsBAADIq6vXrM2Ji4uLTdcfLi4uCggIsHmpKgAAAACOz8lcu0gt8i0uLk7+/v4KG/GtnD287Z0OAAAoYoff6mLvFG5qmddasbGx3EugEHB+ARSYdZPtnYHdTE/jWiAnA1pG3DgIuInZeq3FlAwAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAH4mrvBAAAAAAAgINaN9neGQDAfxIzbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCDciAwAAAAAAABFYvrvB3MVP6BlRCFlAjg2ZtoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADsTV3gkAAAAAsM3ly5e1du1anT59WmXLllWLFi3k5uZmFZORkaE1a9bo6NGjqlatmho3bmynbAEAAJBXFG0BAACAYuCvv/5Sly5dVKFCBdWsWVMbN25USkqKfvnlF1WqVEmSlJCQoM6dO+vgwYNq1KiRfvvtN7Vt21ZfffWVXFxc7HwEAAAAsBXLIwAAAADFwOjRo1WvXj39/fff+vLLL7Vjxw55enpq/Pjxlpjx48frwIED2rJli+bPn6/169dr8eLF+uKLL+yYOQAAAHKLoi0AAABQDBhjVLp0actzNzc3lSpVyirm66+/Vq9evRQUFCRJqlq1qjp27Kivv/66SHMFAABA/rA8AgAAAFAMvPPOO+rVq5eGDh2qGjVqaP369UpPT9fLL78sSUpKStKBAwd0yy23WO1Xq1YtzZgxI8d+k5OTlZycbHkeFxdXOAcAAAAAm1G0BQAAAIoBPz8/lS9fXmvWrNH58+f1999/q169evL29pYkxcfHyxijkiVLWu1XqlSp6xZix40bp9dee61QcwdgR+sm2zsDAEAesDwCAAAAUAx0795dfn5+2rJli+bMmaMdO3boyJEjGjhwoCTJy8tL0pXi7dXi4+Mt27IzatQoxcbGWh7R0dGFdxAAAACwCUVbAAAAwMGlpKRo06ZNuvPOO+Xk5CRJcnV1VceOHfXHH39IkkqUKKHSpUvr8OHDVvseOnRIVapUybFvDw8P+fn5WT0AAABgXxRtAQAAAAfn7u6uMmXKaPv27Vbt27ZtU3h4uOX5XXfdpe+++05paWmSrqxPu2TJEt11111Fmi8AAADyhzVtAQAAgGLgzTff1BNPPKFLly6pdu3aWrdunZYsWaKlS5daYsaMGaNGjRqpc+fOateunebNm6eQkBANHTrUjpkDAAAgt5hpCwAAAOTRtUsRFKZHHnlEGzduVPny5bV3715FRkZq9+7dateunSWmfPny2rp1q9q0aaNDhw6pT58++vPPP+Xr61tkeQIAACD/mGkLAAAA5FHTpk0VERGhRx99VD169FCJEiUKdbzIyEhFRkZeNyYkJEQvvPBCoeYBAACAwsVMWwAAACCPfvrpJ0VFRen5559XaGioHnnkEa1du9beaQEAAKCYo2gLAAAA5FFkZKQ++OADHT9+XDNnztSJEyfUqlUrVa9eXW+99ZZOnDhh7xQBAABQDFG0BQAAAPLJ3d1dDzzwgJYtW6bPPvtMMTExGjVqlMLCwtS7d29FR0fbO0UAAAAUIxRtAQAAgHyKjo7W2LFjVaVKFT3xxBPq1q2b1qxZo1WrVunChQtq06aNvVMEAABAMcKNyAAAAIA8mj9/vj755BOtWrVKkZGReuaZZ9SnTx/5+/tbYlq0aKGSJUvq7NmzCgoKsmO2AAAAKC4o2gIAAAB59OKLL6pNmzb666+/FBUVlW2Mi4uLxo8fL29v7yLODgAAAMUVRVsAAAAgj/755x+5u7vfMG7gwIFFkA0AAABuFqxpCwAAAOSRm5ubVq9eLWOMVfuGDRt06dIlO2UFAACA4o6iLQAAAJBHkydP1oIFC+Tk5GTVvmvXLo0YMcI+SQEAAKDYo2gLAAAA5NGsWbP06KOPZmnv3bu35s+fr7S0NDtkBQAAgOKOoi0AAACQR5cuXVJqamqWdmOM4uPjlZiYaIesAAAAUNxRtAUAAADyqHnz5ho/fnyWGbX/93//p1tuuUW+vr52ygwAAADFmau9EwAAAACKqzFjxqhJkyaqUaOGWrduLQ8PD23YsEE7duzQjz/+aO/0AAAo9qb/fjBX8QNaRhRSJkDRYqYtAAAAkEeVKlXSli1b1K1bN+3fv1/btm3Trbfeqk2bNqlNmzb2Tg8AAADFFDNtAQAAgHwoW7asxo8fb+80AAAAcBOhaAsAAADkU0JCgs6ePStjjFV7WFiYXFxc7JQVAAAAiiuKtgAAAEAeHThwQH369NGGDRuy3X7ixAmFhoYWcVYAAAAo7ijaAgAAAHn01FNPqVSpUvrjjz8UEhKSZXtwcLAdsgIAAEBxR9G2EO14rYP8/PzsnQYAAAAKyc6dO7V06VLVqFHD3qkAAADgJuJs7wQAAACA4qpixYqKjY21dxoAAAC4yVC0BQAAAPJo5MiReuaZZ7Rv3z57pwIAAICbCMsjAAAAAHk0cOBAHTlyRNWqVZObm5ucna3nRBw5ckSlS5e2U3YAAAAorijaAgAAAHn06aefKikpKcftAQEBRZcMAAAAbhoUbQEAAIA8uuOOO+ydAgAAAG5CrGkLAAAA5NOePXu0dOlSJScnS5LS09PtnBEAAACKM4q2AAAAQB7Fx8erQ4cOqlGjhu68805duHBBkvTYY49p9uzZds4OAAAAxRVFWwAAACCPxowZIycnJ50+fdrqhmOjRo3Sm2++acfMAAAAUJyxpi0AAACQR6tWrdLs2bMVHBwsJycnS3vVqlUVHR2t2NhY+fv72zFDAAAAFEfMtAUAAADyKCEhQR4eHpJkVbRNSEhQcnKy3N3d7ZUaAAAAijGKtgAAAEAeNWvWTF988YWk/xVtMzIy9NJLLykqKkpeXl72TA8AAADFFMsjAAAAAHn0+uuvq2nTptqwYYNiY2P13HPPadOmTTp8+LBWrlxp7/QAAABQTDHTFgAAAMijSpUqadu2bWrSpIkaNWqkAwcOqEWLFtq0aZOaN29u7/QAAABQTDHTFgAAAMiH0qVL680337R3GgAAALiJULQFAAAA8ujy5cvKyMjIcbuPj4/VDcoAAAAAW1C0BQAAAPIoIiJCp06dynH7iRMnFBoaWoQZAQAA4GZA0RYAAADIox9//FEpKSlWbYcPH9bYsWN1//33KzAw0E6ZAQAAoDijaAsAAADkUYMGDbK0NWnSRI0bN1a7du302muv2SErAAAAFHcUbQEAAIACVqlSJZ0+fVoJCQny8fGxdzoAirt1k+2dAQCgiDnbOwEAAADgZvPVV19Jkry9ve2cCQAAAIojZtoCAAAAedSgQQOdPn3aqi0+Pl6XL1/W+++/LycnJztlBgAAgOKMoi0AAACQR88//7wSExOt2vz8/NSgQQNVrFjRPkkBAACg2KNoCwAAAORRjx497J0CAAAAbkIUbQEAAIA8unz5sjIyMmyK9fHxYbkEAAAA2IQbkQEAAAB5FBERIV9fX5sep06dyvd4CQkJeuONN9S8eXO1aNFCn3zyiYwxVjG///677rrrLkVGRqp79+76559/8j0uAAAAihZFWwAAACCPJk6cKF9fX73yyitavny5fv31V02aNEnlypXTyy+/rI0bN1oeQUFB+RorKSlJrVu31qJFizR69Gi999572r59uxYsWGCJ+euvv9SuXTvVq1dP06ZNU0BAgFq0aKEjR47k91ABAABQhJzMtX+aR77FxcXJ399fsbGx8vPzs3c6AAAANxVHuta6++671atXL/Xq1cuqfdeuXerUqVOBFkvfeOMNTZw4Ufv371epUqUs7WlpaXJ1dbXkk5ycrOXLl0uSjDGqVq2aOnfurPfff9+mcRzp/AL4/9ZNtncGyKXpaV3sncJ/1oCWEfZOAbguW6+1mGkLAAAA5NHOnTvVoEGDLO01a9bU2bNnFR8fX2Bjff3117rvvvusCraSLAVbY4xWr16tjh07WrY5OTmpU6dO+vXXXwssDwAAABQ+irYAAABAHpUtW1affvpplvavvvpK7u7u8vb2LrCx9uzZo2rVqmn48OGqX7++OnTooBkzZlhuhBYbG6v4+HiVKVPGar/Q0FAdO3Ysx36Tk5MVFxdn9QAAAIB9udo7AQAAAKC4+r//+z916NBBq1atUvPmzeXm5qZt27bpt99+0/Tp0+Xi4lIg46Snpys9PV2vv/66XnjhBX366afasWOHhg0bpmPHjmnMmDFKT0+XJLm7u1vt6+HhobS0tBz7HjdunF577bUCyRMAAAAFg5m2AAAAQB61aNFCe/fuVbt27bR//35t375d1atX15YtW/Too48W2DguLi4qWbKkmjdvrpdffllRUVHq27evhg8frk8++USS5O/vL1dXV509e9Zq37NnzyowMDDHvkeNGqXY2FjLIzo6usDyBgAAQN4w0xYAAADIh/Lly+vtt98u9HEaNmyYZT3bwMBAXbp0SdKVtW3r1aunP//8U48//rglZt26dbr11ltz7NfDw0MeHh6FkzQAAADyhKItAAAAkE979uzR/v37dccdd8jDw0Pp6ekFtjRCpsGDB+vBBx/U9u3bVbduXZ05c0affvqpOnfubIl54okn9NRTT2nQoEG69dZbtWTJEq1du1bLly8v0FwAAHBU038/mKv4AS0jCikTIH8o2gIAAAB5FB8fr/vvv18rVqyQJJ04cUKhoaF67LHH1KZNGz300EMFNtbdd9+tV199VS1btpSfn5/OnDmjLl26aMqUKZaYxx57TPv379dtt90mf39/Xb58WZMmTdIdd9xRYHkAAACg8DkZY4y9k7jZxMXFyd/fX2EjvpWzR8HdMRgAAMARHX6rS5GOl3mtFRsbKz8/vyId+1pPP/20du3apdmzZ6tOnTraunWrQkNDtXfvXt19993avXt3gY+ZnJys48ePq0yZMvL09Mw2JiEhQadOnVLZsmVzjMmJI51f4KaxbrK9M0ARm55WtD8bkXfMtEVRs/Vai5m2AAAAQB6tWrVKs2fPVnBwsJycnCztVatWVXR0tGJjY+Xv71+gY3p4eKhSpUrXjfHx8VFEBL+EAgAAFFfO9k4AAAAAKK4SEhIsN/G6umibkJCg5ORkubu72ys1AAAAFGMUbQEAAIA8atasmb744gtJ/yvaZmRk6KWXXlJUVJS8vLzsmR4AAACKKZZHAAAAAPLo9ddfV9OmTbVhwwbFxsbqueee06ZNm3T48GGtXLnS3ukBAACgmGKmLQAAAJBHlSpV0rZt29SkSRM1atRIBw4cUIsWLbRp0yY1b97c3ukBAACgmGKmLQAAAJBH7777rp588km9+eab9k4FAAAANxFm2gIAAAB59OGHH+rChQv2TgMAAAA3GYq2AAAAQB7Vr19fa9eutXcaAAAAuMmwPAIAAACQR3fffbcGDRqkPXv2qF69evL09LTa3qZNG7m7u9spOwAAABRXFG0BAACAPHrllVcUHx+vN954I9vtMTExKl26dBFnBQAAgOKOoi0AAACQR8eOHbN3CgAAALgJsaYtAAAAkEuhoaE6efKk5fno0aMVHx9vx4wAAABwM6FoCwAAAOTTJ598ooSEBHunAQAAgJsEyyMAAAAAAADgP2n67wdzFT+gZUQhZQJYY6YtAAAAAAAAADgQZtoCAAAAeTB+/Hj5+PhIki5dumT1PNMLL7ygEiVK2CM9AAAAFGMUbQEAAIBcqlSpkhYvXmx5HhoaavU804gRIyjaAgAAINco2gIAAAC5tH79enunAAAAgJsYa9oCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EIq2AAAAAAAAAOBAKNoCAAAAAAAAgANxtXcCAAAAAAA4vHWT7Z0BAOA/hJm2AAAAAAAAAOBAKNoCAAAAAAAAgAOhaAsAAAAAAAAADoSiLQAAAAAAAAA4EG5EBgAAAAAAANhg+u8HcxU/oGVEIWWCmx0zbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgbjaOwEAAAAAAGyybnLe9202tODyAACgkDHTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAChmYmNjtXDhQm3atCnb7Tt37tSyZct04MCBIs4MAAAABYGiLQAAAFDM9O/fXw888IDeffddq/aUlBTdc889uu222/TWW28pMjJSgwcPljHGTpkCAAAgL1ztnQAAAAAA202dOlVnz55V69ats2ybMGGC1q1bp3/++Ufly5fX1q1b1bhxY7Vs2VI9evQo+mQBAACQJ8y0BQAAAIqJHTt2aOzYsfriiy/k7Jz1Un727Nnq1auXypcvL0mqV6+e2rVrpy+++KKoUwUAAEA+ULQFAAAAioHLly+rR48emjhxoipUqJBle3Jysvbs2aPIyEir9nr16mn79u059pucnKy4uDirBwAAAOyLoi0AAABQDAwbNkxRUVHq2bNnttvj4+OVkZGhkiVLWrUHBgbq4sWLOfY7btw4+fv7Wx5hYWEFmTYAAADygKItAAAA4OB+//13ffnll7rjjju0cOFCLVy4UKdPn1ZMTIwWLlyopKQkubu7S7oyI/dqly5dkoeHR459jxo1SrGxsZZHdHR0oR4LAAAAbowbkQEAAAAOzs3NTR07dtT3339vaYuJiZGbm5s+++wztWjRQoGBgQoMDMxSdI2OjlalSpVy7NvDw+O6RV0AAAAUPYq2AAAAgINr2rSpFi5caNXWsWNHBQQE6JtvvrG0derUSQsWLNDzzz8vJycnJSUlacmSJXrssceKOGMAAADkB8sjAAAAADeJMWPGaO/everZs6dmzZqlO++8U+7u7hoxYoS9UwMAAEAuULQFAAAAiqFmzZqpYcOGVm1VqlTR5s2bFRYWpmXLlqlp06b666+/VKpUKTtlCQAAgLxgeQQAAACgGBo9enS27ZUqVdK7775bxNkAAACgIDHTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIG42jsBAAAAAAAA4GY0/feDuYof0DKikDJBcUPRFgAAAIUqMTFRzs7O8vDwyDEmLS1Nrq5ZL01TUlKUkpJi1Xbp0qXrjpdTXwAAAEBxUejLI8TExOjUqVOFPUyhjlWUxwAAAHAzSE9P1/fff6927drJx8dHffv2zTbur7/+UoMGDeTp6Sl/f389/fTTSk9Pt2x/7bXXFBoaavUIDw+XJGVkZFjikpOT9dxzzykkJEQlSpRQixYttGPHjsI9SAAAAKCQFHrRdvDgwRo7dmxhD1OoYxXlMQAAANwM9u3bp9mzZ2vkyJHq1KlTtjFnz55Vx44d1bJlS8XGxuqXX37Rl19+qddee80S8+abb+rSpUuWR3x8vMLCwiRJzs7/u5Tt2bOnfvjhB61cuVJJSUmaNGmSFi9eXLgHCQAAABSSm+pGZOXLl1doaKi90wAAAPjPq1GjhhYsWKAOHTrIyckp25jPP/9cGRkZevvtt+Xj46OoqCgNHz5cH374odLS0rLdZ/Xq1Tp06JBV288//6yFCxfqiy++UGRkpCTp1ltv1ahRowr2oAAAAIAikqui7alTp7R7927t3r1bZ86cydOAhw8f1vnz563ajh8/rpMnT1qeR0dHW/q/cOFClqUJjh07pri4uCx9jxo1SgMGDMixn3PnzhXKMQEAACD31q9fr8aNG8vd3d3Sdvvtt+vChQvas2dPtvvMmDFD1apVs2pbvHixKlasqEaNGikjI0PGmELNGwAAAChsubpDw9SpU/XNN99Ikk6fPq2SJUtq1qxZatmypc193HPPPXrwwQc1cuRIS9vTTz+tEiVK6NNPP5UkPfLII/Lw8NCRI0d08eJFnT59Wu3bt9crr7yiRx55RHFxcTp9+rSefPJJffDBB5Z+Bg8erPLly+vDDz+09OPn56djx47p+PHjOnfunFq0aKGFCxfK29u7wI4JAAAAuXf69GlVqFDBqi04ONiyrVatWlbbLl68qPnz5+uVV17RSy+9ZGk/fPiwKleurEGDBumzzz6TJDVp0kTvv/++6tSpU7gHAQAAUICm/37Q5tgBLSMKMRPYW65m2r766quWWannzp3TwIED1atXLyUmJhZ4YqtXr9asWbN07Ngxbd++XStWrFCnTp301Vdf6dixY9qwYYOmTJmiDRs2XLefn376Se+//76OHTumw4cPa+fOnZo2bVqBHlNycrLi4uKsHgAAALixq28mdvXz7JZUmDNnjjIyMtSrVy+rdmOMfv75Z/n6+ur8+fM6fvy4goKC1LlzZyUkJBRe8gAAAEAhydVM20znz5+33DjihRde0K5duxQVFVWgiXXr1k0NGzaUdGVNtFq1aqlhw4aqV6+eJKl+/fqKiIjQli1b1Lhx4+v207RpU0lS6dKl1a5dO23ZsqVAj2ncuHFWN8wAAADAjZUtW1anT5+2ast8XqZMmSzxM2bM0L333qvAwMAs/Xh4eOj//u//5OLiIk9PT40bN05VqlTR33//zSeoAFyxbrK9MwAAwGa5mmm7atUqVatWTeHh4erUqZPuv/9+ZWRkKCYmpsATK1u2rNVzHx+fbNsuXbp03X7KlStn9bxEiRKKj4+3PC+IYxo1apRiY2Mtj+joaJv3BQAA+K+67bbbtH79eqtPOK1cuVIhISGqWrWqVeyWLVu0ZcsWq/sXZGrZsqWMMVY3L0tOTpYkq/VyAQAAgOLC5qJtenq6unfvrv79++vixYs6cOCA/vnnHzk7O2f5WNv1ZPdRt5zuDlzYCuqYPDw85OfnZ/UAAAD4r0tISNClS5eUnp6utLQ0Xbp0SZcvX7Zsf/jhhxUQEKABAwbo0KFD+vHHH/Xee+/pueeek7Oz9WXqjBkzFBERoTZt2mQZp1u3bqpcubIGDhyoo0ePas+ePRoxYoRq166tBg0aFPpxAgAAAAXN5qLtiRMndPHiRXXr1k0uLi6SpN9++03p6em5GjA4OFgnTpywPM/IyND27dtz1UdBKahjAgAAQFbh4eEKDQ3VmjVr9NNPPyk0NNTqxmB+fn76+eefde7cOTVs2FDDhw/Xyy+/rGeeecaqn5SUFP3www964oknsp0A4O7urpUrV+ry5ctq2LChOnbsqHLlyumnn35ipi0AAACKJZvXtC1btqzCw8M1evRoPfPMMzp48KCeeeaZbC+cr6dz584aO3asWrRooTJlymjatGk6dOiQXdYaK6hjAgAAQFZnz569YUyNGjW0bNmy68a4u7vr6NGj140pV66cvvnmm1zlBwAAADgqm4u2zs7O+vHHH/XKK6/o0UcfVZkyZTR16lSNHj1avr6+Oe5Xvnx5hYaGWp4PGTJECQkJGj9+vHx8fHTvvfcqICBAJUqUsMRUqFBBISEhVv2Eh4crKCjIqq1SpUpWN6K4dqzs+rl6u63HdG2/AAAAAAAAAFBYnIwxxt5J3Gzi4uLk7++vsBHfytnD297pAAAAFKrDb3Up0vEyr7ViY2O5l0Ah4PzCoa2bbO8MAJtMTyvan434bxrQMsLeKSAPbL3WsnlNWwAAAAAAAABA4aNoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EAo2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQV3snAAAAAAD4j1g32d4ZAABQLDDTFgAAAAAAAAAcCEVbAAAAAAAAAHAgFG0BAAAAAAAAwIFQtAUAAAAAAAAAB0LRFgAAAAAAAAAciKu9EwAAAABwYwkJCZoxY4Z+++03paenq1GjRhoyZIj8/Pys4rZt26b33ntPR48eVbVq1fT888+rYsWK9kkaAAAAecJMWwAAAKAYaNasmQ4dOqQ+ffro4Ycf1oIFC9S8eXNdvnzZEvPPP/+oWbNm8vT01IgRI3T69Gk1adJEJ0+etGPmAAAAyC1m2gIAAADFwJo1a6xm1TZr1kxlypTR8uXLde+990qSXn/9dUVFRWnq1KmSpE6dOqly5cqaOHGi3n77bbvkDQAACsf03w/mKn5Ay4hCygSFgZm2AAAAQDFw7TIIJUqUkLOzs5KSkixtq1at0t1332157urqqi5dumjlypVFlicAAADyj5m2AAAAQDH09ttvy9vbW23atJEkxcbG6uLFiypXrpxVXLly5XT06NEc+0lOTlZycrLleVxcXOEkDAAAAJsx0xYAAAAoZr777jv93//9nz7++GOVLl1akpSamipJ8vT0tIr18vJSSkpKjn2NGzdO/v7+lkdYWFjhJQ4AAACbULQFAAAAipEffvhBffr00YcffqjevXtb2v39/eXi4qJz585ZxZ87d06lSpXKsb9Ro0YpNjbW8oiOji603AEAAGAbirYAAABAMbF48WI98MADmjRpkp588kmrbW5ubqpVq5b+/vtvq/a//vpL9evXz7FPDw8P+fn5WT0AAABgXxRtAQAAgGJg6dKl6t69uyZOnKhBgwZlG/Poo4/qm2++0d69eyVJa9eu1erVq/Xoo48WZaoAAADIJ25EBgAAABQDDzzwgNzc3PTVV1/pq6++srQ//vjjevjhhyVJQ4YM0T///KPIyEhFRETowIEDGjVqlO6++257pQ0AAIA8oGgLAAAAFAMrVqyQMSZLe3h4uOX/Li4u+vTTTzV27FhFR0ercuXKCgwMLMo08V+wbrK9MwAA4KZH0RYAAAAoBpo3b25zbJkyZVSmTJlCzAYAAACFiTVtAQAAAAAAAMCBULQFAAAAAAAAAAdC0RYAAAAAAAAAHAhFWwAAAAAAAABwINyIDAAAAAAAALjJTf/9YK7iB7SMKKRMYAtm2gIAAAAAAACAA6FoCwAAAAAAAAAOhKItAAAAAAAAADgQirYAAAAAAAAA4EC4ERkAAAAAFDfrJud932ZDCy4PAABQKJhpCwAAAAAAAAAOhKItAAAAAAAAADgQlkcAAAAAgP+S/CytAAAAigQzbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCDciAwAAAAAiho3AwMAANfBTFsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAAByIq70TAAAAAIBiad1ke2cAAABuUsy0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgXAjskK047UO8vPzs3caAAAAAAAAQKGa/vvBXMUPaBlRSJncHJhpCwAAAAAAAAAOhKItAAAAAAAAADgQlkcAAAAA8N+0brK9MwAAwGHldrkDFCxm2gIAAAAAAACAA6FoCwAAAAAAAAAOhOURAAAAAAAAABSpwl5+YUDLiELtv7Ax0xYAAAAAAAAAHAgzbQEAAAAUX9xMDAAA3ISYaQsAAAAAAAAADoSiLQAAAAAAAAA4EJZHAAAAAG4yJ0+eVHR0tCIiIhQYGGjvdAAAAIpcbm905mg3LmOmLQAAAHCTSE9P12OPPaZKlSqpX79+KleunMaMGWPvtAAAAJBLzLQFAAAAbhIffvih5s+fr23btqlatWpau3atbr/9dkVFRenuu++2d3oAAACwEUVbAAAA4CYxc+ZM9ezZU9WqVZMk3XbbbWrdurVmzpzpuEXbdZPtnQEAAIDDoWhbCIwxkqS4uDg7ZwIAAHDzybzGyrzmwhWpqanauXOnhg4datXeqFEjffnllznul5ycrOTkZMvz2NhYSUV4LZuQWDTjAEARSkyLt3cKAHKpqK59bL2WpWhbCM6dOydJCgsLs3MmAAAAN6/4+Hj5+/vbOw2HERsbq/T09Cw3HgsMDNT58+dz3G/cuHF67bXXsrRzLQsA+fG8vRMAkEvDi3i8G13LUrQtBKVKlZIkHT16lF8kHFxcXJzCwsIUHR0tPz8/e6eDHPA6FR+8VsUHr1XxwWuVlTFG8fHxKlu2rL1TcShubm6SpKSkJKv2xMREubu757jfqFGj9PTTT1ueZ2Rk6Pz58woMDJSTk1PhJPv/8f7OG85b7nHO8obzlject7zhvOUe5yxv7H3ebL2WpWhbCJydnSVJ/v7+fNEUE35+frxWxQCvU/HBa1V88FoVH7xW1vjDeFb+/v4KCAhQTEyMVXtMTIwqVKiQ434eHh7y8PCwagsICCiMFHPE+ztvOG+5xznLG85b3nDe8obzlnucs7yx53mz5VrWuQjyAAAAAFAE7rjjDi1evNjyPC0tTUuXLlW7du3smBUAAAByi6ItAAAAcJMYPXq0Nm3apIEDB2rx4sXq2bOnkpKSrJY/AAAAgOOjaFsIPDw8NGbMmCwfM4Pj4bUqHnidig9eq+KD16r44LVCbtSpU0fr1q1TYmKiJk2apODgYP35558KDQ21d2rZ4v2dN5y33OOc5Q3nLW84b3nDecs9zlneFJfz5mSMMfZOAgAAAAAAAABwBTNtAQAAAAAAAMCBULQFAAAAAAAAAAdC0RYAAAAAAAAAHIirvRO42cTFxWnv3r0KCQlRhQoV7J3Of5IxRgcPHlRycrIqV66c48LSp06d0tGjR1WpUiUFBQXlOQb5d/bsWe3evVvh4eEKCwvLsv3QoUM6f/68atSoIR8fn2z7sCUG+RMfH699+/YpPDxcgYGBWbYbY7Rr1y6lpaWpVq1acnXN+iPGlhjkT1xcnA4ePChPT09FRETI3d09S0xKSop27NghHx8fVa9ePdt+bIlB7uzfv18nT55U06ZN5eLikm3Mnj17lJCQoFq1auX486ugYoCi9u+//+r8+fNq3ry5TfHGGP37779KSUlR7dq1c/y5cqOY4uzy5cv6999/FRAQoMqVK1839siRI4qOjs52W/PmzeXk5KTLly9r8+bNWbbXqVNH/v7+BZKzIzh58qSio6MVERGR7TXL1aKjo3XkyBGrNldXVzVp0iRf/RZHu3fvVmJiomrVqpXt9cO1kpKStGfPHpUqVUrly5eXk5OT1fatW7fq0qVLVm2hoaGqUqVKgeZdFJKSkrRr1y75+vqqatWqBbZPXvotTk6fPq0jR44oPDxcISEhN4w3xujQoUNKTExU5cqV5enpabX9+PHjOnjwoFWbk5OTzT9Xiot9+/YpPj5et9xyS5ZzcK1//vlHsbGxVm1BQUGqUaNGvvotblJSUrRz5055eXlle+xXO3HihA4cOJDttltvvVWenp5KT0/X+vXrs2yvXr26goODCyRnmxgUmClTphgvLy9To0YN4+3tbbp27WouX75s77T+Uz7++GMTHh5uKleubKpXr25KlixpZsyYYRWTkZFhBg0aZDw8PMwtt9xiPDw8zPPPP5/rGBSM1NRU06xZM+Ps7GzGjh1rtS02NtbccccdpkSJEqZatWqmRIkS5osvvsh1DPInPT3djBo1ynh7e5vIyEgTHh5uRo4caRWzZ88eU6NGDRMSEmLCwsJMuXLlzLp163Idg/x5/vnnjZeXl4mMjDSVKlUypUuXNgsWLLCKWb58uQkKCjKVKlUypUqVMvXr1zfHjh3LdQxst3jxYtOqVStTsmRJI8lcuHAhS8zx48dNVFSUKVmypImIiDCBgYHmxx9/LJQYoKh98803pnHjxqZkyZLGxcXFpn327dtnbrnlFhMcHGwqVKhgypQpY9auXZvrmOLsm2++MX5+fqZq1arG19fXtGrVypw/fz7H+KlTp5rmzZtbPUqVKmUCAgJMWlqaMcaYf/75x0gyjRo1sorbvHlzUR1WoUpPTzePP/641TX8iy++eN19xo4da3x9fa3OR8eOHfPdb3Fy9OhRExkZaQIDA02lSpVMcHCwWbVqVY7x58+fN08++aQJCAgwdevWNUFBQaZBgwZm586dVnGRkZGmYsWKVuf2rbfeKuzDKXCLFi0yJUuWNFWqVDEBAQGmSZMm5tSpU/neJy/9FicjRoyw+poZMWLEdeNnzpxpKlWqZCpVqmRq1qxpAgICzNSpU61iJk2aZLy9va3eU61atSrEoyhap06dMk2aNDEBAQGmSpUqpmTJkmbRokXX3adVq1YmLCzM6py8/PLL+e63OFm1apUJDg42lSpVMoGBgSYyMtIcPXo0x/j58+dn+XlZpkwZ4+rqas6dO2eMMebChQtGkomMjLSKW758eVEdljHGGIq2BWTjxo3GycnJfP/998YYY06cOGHKly9vnn32WTtn9t/y+uuvm+joaMvzWbNmGWdnZ6sL0WnTphlfX1+zY8cOY4wxGzZsMO7u7mbu3Lm5ikHBGDVqlOnTp48pV65clqLtY489ZqpXr275BWX69OnG1dXV7NmzJ1cxyJ+XX37ZBAYGmu3btxtjrvxRY8qUKVYx9evXN3fddZfll8InnnjClC1b1iQmJuYqBnn3yy+/GEnm119/tbQ9++yzxsfHx6SmphpjjDl37pzx9/c3o0ePNsYYk5SUZJo3b27atWtn2ceWGOTOW2+9ZX755RezePHiHIu2HTt2NM2aNbN8PYwZM8b4+fmZM2fOFHgMUNTGjBlj1q1bZ2bNmmVz0bZhw4amc+fOlp8ZgwcPNqGhoSYhISFXMcXVwYMHjbu7u/noo4+MMVf+SH3LLbeYhx56yOY+UlNTTenSpc3QoUMtbZlF2xMnThR4zo7gww8/NP7+/ubff/81xhizbt064+bmZubPn5/jPmPHjjWNGzcu8H6Lk9atW5vWrVub5ORkY8yVPwKXLFky259Xxhizc+dOM3XqVEt8UlKSufvuu03NmjWt4iIjI80777xTqLkXtuPHjxtvb2/z9ttvG2OMSUhIMA0aNDD33ntvvvbJS7/FyWeffWa8vb3N1q1bjTHGbN682Xh5eZnPP/88x33efPNNc/jwYcvzr7/+2jg5OZn169db2iZNmmRq1apVeInb2T333GNuvfVWy8+xcePGGW9v7+t+z27VqpV56aWXCrzf4uLChQumZMmSlj+kJScnmxYtWpjbb789V/3UrVvX6usvs2i7cePGAs03tyjaFpBBgwaZ2rVrW7W9+uqrJigoyGRkZNgpKxhjjJeXl1WBqVGjRqZfv35WMXfeeafp0KFDrmKQf6tWrTKVKlUyFy9ezFK0TUpKMt7e3ubDDz+0tGVkZJiyZctafijZEoP8uXjxovHy8jITJ07MMWbz5s1Gkvnzzz8tbdHR0cbJyckyy9OWGOTP3LlzjZOTk9UnPBYsWGCcnJzMxYsXjTHGfPLJJ8bT09PEx8dbYhYuXGgkWf4abUsM8ianom1MTIxxcnIyCxcutLRdunTJeHl5WWaYFFQMYE+2Fm23b99uJFnNmj1+/LhxdnY28+bNszmmOHv99ddNSEiISU9Pt7RNmzbNeHh4mEuXLtnUx4IFC4wkyx9djflf0faPP/4wW7ZssfpefzNo0KCBeeyxx6zaOnbsaLp06ZLjPmPHjjVRUVFm69atZvfu3ZY/dOa33+Li4MGDRpL56aefLG0XLlwwbm5uZtasWTb3k3mtkDlLzZgrRduXXnrJ/PXXXyYmJqYg0y4yEydONH5+fpYCtTHGfPnll8bFxcWcPXs2z/vkpd/ipGXLlqZnz55Wbffff3+uZ8UGBASYCRMmWJ5PmjTJ1KhRw2zbts3s2rXLpKSkFES6DuHMmTPG2dnZfPPNN5a2xMRE4+vrayZNmpTjfq1atTJDhw41f/31l9UEtvz2W1zMnDnTuLu7m9jYWEvbkiVLjCRz6NAhm/rYuHGjkWT16bTMou33339vNm3alOMfsQobNyIrIFu2bFFUVJRVW6NGjXT27FkdO3bMTllhx44dSkxMtKybZIzRtm3bsn2ttmzZYnMM8u/MmTPq27evPv/882zXUNuzZ48uX75s9To4OTnp1ltvtbwOtsQgf9atW6fExETdddddOn78uLZs2aK4uDirmMxz3aBBA0tb+fLlVaZMGcs2W2KQP3fddZeaNWumvn37avny5Zo3b55efvllvfTSS5avsS1btqh69eoqUaKEZb9GjRpJurLunK0xKFhbt26VMcbqe5mPj49q1qxp+fooqBigOMh8v179Xi5TpozKly+f5efK9WKKsy1btqh+/fpydv7fr2uNGjVScnKydu3aZVMfM2bMUOPGjVWnTp0s2x544AH17t1bpUqV0uDBg5WSklJgudtLenq6/vnnnzxdw2/ZskW9e/dW27ZtFRoaqtmzZxdIv8VBdl9LAQEBqlq1aq6Ob+PGjSpVqpRKlixp1f7ee+9pwIABqlGjhho1amTz+9dRbNmyRXXq1LFa47dRo0ZKT0/X9u3b87xPXvotTnKqj+TmPbVv3z7FxsZmWQN5z5496tmzpzp06KDg4GBNnz69QHK2t+3btysjI8PqvHl6eqpOnTo3PG/Tp0/XgAEDVLt2bdWtW9dq7fL89FscbNmyRVWrVpWfn5+lLfN3F1uPb8aMGQoLC1OHDh2ybHviiSfUr18/hYSEqHfv3lnWDy5sN9dK/XZ0/vz5LIvRZz4/f/58tjdXQuFKTEzUI488oqZNm+qOO+6QJCUkJCg5OTnb1+r8+fM2xyB/jDF6+OGH1a9fP7Vo0SLbmMxznd3r8O+//9ocg/w5fvy4nJycNGXKFM2dO1fBwcHau3evhg0bpvHjx0u68jr4+fnJzc3Nat+rv2ZsiUH+eHl5adiwYRo2bJj+/fdfxcfHKyQkRD169LDE3Ohnla0xKFjX+1529etSEDFAQdi4caOSk5Nz3O7p6albb701z/2fP39e3t7eWW6Scu37/UYxjuTSpUs3/MNXuXLlVKlSJUlXjq9cuXJW23Pzvfj48eNatmyZPv74Y6t2Pz8/LVu2TB07dpR05RfaNm3aKCAgQG+++aath1Nkjh49qqNHj143JvMmavHx8UpNTc3198CGDRvq4MGDCg8Pl3SlyNivXz9VrVpVTZo0yXO/9rR582Zdvnw5x+1ubm5q3LixpP+9n0qVKmUVk5vj27RpkyZMmKD/+7//s7oZ2VNPPaUePXrI09NTcXFx6tGjh+69915t37692NwkMy/XRf/16620tDTFx8dne3xxcXFKT0/P8YasmVJSUtSvXz/Vr19fnTt3trTXrVtXe/futRRyp0+frieeeEKVK1dWmzZtCv5gilBer+MGDBigpUuXysfHR5cvX1bfvn3VtWtXyw3ubvbrw+y+ljK/n9lyfImJifr666/11FNPWf2h1M3NTXPmzFHv3r0lXbmhcNu2bTVs2DB9/vnnBXgE10fRtoC4ubkpKSnJqi0xMVGSbLrzJgpWSkqK7r//fl28eFG///675Ysvs2CU3WuV+TrZEoP8mTVrlrZt26bnnntOa9eulXTlNTt69Kj+/PNPNWnShNfKQbi5uckYoxMnTujo0aNydXXVunXr1KpVKzVo0EA9evTI9vuflPW1ulEM8mfp0qXq06ePfvnlF8sfQ1599VW1atVK+/btU6lSpWz6WcXPs6J39fcyLy8vS3tiYqLl7rQFFQMUhHHjxun06dM5bi9TpozmzZuX5/7d3NyUnJwsY4xVAejanys3inEkMTExeuGFF64b88ADD2jYsGGS8v+9+PPPP5e3t7fVH+4kqUKFCqpQoYLlef369fXEE0/om2++ccii7fLly2/4y/HkyZNVv379PF8XXjuzasSIEZo2bZrmzZtn8zWpo5kwYYKOHDmS4/aAgAAtWbJE0v9+diQnJ2f52WHL8f3777/q0qWLHnzwQY0YMcJqW9++fS3/9/Pz0zvvvKM6depo8+bNatq0aW4OyW7c3NyyFMBv9LVoyz556be4cHFxkbOzc7ZfM87Ozjcs2Kalpalnz546fvy4fv/9d7m6/q9sdW1hdsCAAfr44481d+7cYl+0vd73mqtnkV6rT58+lv97e3tr0qRJCgsL09q1a9WpU6c891tcZPfzMvO5LV9L3333neLj4/Xoo49atfv4+FgKtpJUpUoVPfvss3ruuec0a9YsqwJvYaJoW0DCw8MVExNj1RYTEyMnJydm2RaxzILtnj17tHr1apUpU8ayzcPDQ6VLl872tcq8gLUlBvkXERGhV155xfI8NjZWy5cvV3R0tJYtW2aZ7RATE2P1sb6YmBjLNltikD8VK1aUJPXv399ywdSsWTPVrVtXa9asUY8ePRQeHq6UlBSdPXtWQUFBkq58lPDUqVOWrxlbYpA/S5YsUd26da1mrw8ePFivvfaa1q5dq7vvvlvh4eFas2aN1X6Z3+uufq1uFIOCdfX3sqs/VhoTE6PbbrutQGOAgvD9998Xav/h4eGWnxGhoaGSpIyMDJ08edLqe9WNYhxJ9erVLX+otkV4eLh27Nhh1Zab78UzZ85Ur169rJa6yUl2172OYsCAARowYIBNsT4+PgoMDCyQa/irz0lB9ltU5syZY3Ps1T87rv4Y+vHjx3XnnXded9/du3erTZs26tKliz755BOrP6Bkp3Tp0paxiovw8HCtWrXKqu1GX4u27JOXfouLzBpIXr5m0tLS1KtXL23evFmrV6+2qZbiyN/DcuPqr8WraxgxMTGqXbu2zf2EhITIycnJck4Kql9HFR4erpUrV1q15eZracaMGerYsaPN77XExESdP3/e8jttYWNN2wLSrl07/fLLL0pISLC0LVq0SE2aNLHpYgkFIzU1Vd27d9euXbu0evVqlS9fPktMu3bttHjxYsvz9PR0LVmyRO3atctVDPLu0Ucf1dq1a60ewcHBGjBggJYtWybpynqnNWrU0A8//GDZ7/Tp01q/fr3ldbAlBvnTuHFjBQQEWF0IpaWl6dSpU5aZey1btpS7u7vV6/DLL78oPj7e8jrYEoP8CQ4O1qlTp5SWlmZpi46OtmyTrnxvO3LkiNVaaYsWLVJAQIAaNmxocwwKVlRUlEqVKmX19fHPP//o0KFDlq+PgooBHNWmTZt06NAhSVKLFi3k4eFh9V7+7bffdPHiRct72ZaY4qxdu3batGmTTpw4YWlbtGiRKlWqpMqVK0u6sqTX2rVrs6yvt3r1au3fvz/bYufVv6tkWrly5U3xi7uU/TX80qVLrd4TmZ/synTtOTlz5oy2bt1qdU5s6be4aty4sXx9fa2+ljZu3Kjjx49bHd+WLVu0f/9+y/M9e/bo9ttvV8eOHfXpp59mKdhevnxZxhirthUrVkiSatWqVRiHUijatWunnTt36sCBA5a2RYsWKTQ01DJpJCkpSWvXrtWFCxds3seWmOKsXbt2WrJkieU9YIzR4sWLrd5TMTEx+uOPPyzP09PT1bt3b23cuFGrV6+2TB652rVfr7Gxsfrrr79uiu9hderUUenSpa2+Fvfu3at///3X6rz9888/2r17t6Qr77309HSrflauXCljjOWc2NpvcdWuXTvFxMRYreO7aNEi+fn5WZaBSUtL09q1a3XmzBmrfffv36/ff//d5p+XK1asUGhoaJblGAqVXW5/dhO6dOmSqVq1qmnbtq1ZuHChGTVqlHF1dTW//PKLvVP7T+nevbvx8fExX3/9tVmzZo3lceTIEUvM7t27ja+vr+nfv7/54YcfTM+ePU1gYKDVXdFtiUHBKleunBk7dqxV24IFC4yLi4sZO3asWbBggWnatKmJjIy0ukuoLTHInylTppjSpUubTz/91Cxbtsx0797dBAYGWt2d9JVXXjEBAQFm+vTpZs6cOaZ8+fLm4YcfturHlhjk3YEDB4yvr6+57777zNKlS83XX39tatasaRo3bmx1J+xOnTqZGjVqmG+//dZMnjzZeHp6mg8++MCqL1tiYLuDBw+aNWvWmPHjxxtJZtmyZWbNmjVWd9ieMmWK8fT0NO+//7759ttvTc2aNU379u2t+imoGKCo7dmzx6xZs8a8+OKLxsXFxXJ9dvWdnsPDw83w4cMtz1977TXj5+dnPvnkE/PVV1+ZChUqmN69e1v1a0tMcZWWlmYaNmxoGjZsaL7//nszbtw44+rqaubOnWuJ2bJli5Fkfv31V6t9H3zwQVOvXr1s+x05cqR5/PHHzbfffmsWLVpkevXqZdzd3c2KFSsK83CKzM6dO42Pj495/PHHzQ8//GC6d+9ugoODzbFjxywxY8aMMf7+/pbnDRs2NG+++ab58ccfzRdffGFq165tqlSpYvU92pZ+i7MJEyYYb29vM2XKFDN37lxTtWpVc/fdd1vF1KpVy/Tv398YY8zRo0dN2bJlza233mp+++03q9+7EhISjDHG/PXXX6Zp06bmk08+McuXLzfjxo0zvr6+ZsCAAUV+fPmRkZFhWrVqZSIjI813331nJkyYYNzc3MyMGTMsMfv27TOSzOLFi23ex5aY4uzAgQMmICDAPPzww+aHH34wDz30kAkICDAHDhywxLzzzjvGxcXF8vyhhx4ynp6eZvbs2VbvqUOHDlliWrZsaV577TWzdOlSM2fOHNOgQQNToUIFc+LEiaI8vEIzY8YM4+7ubiZOnGi+++47U6dOHdOqVSuTkZFhiWnevLnp1q2bMeZK3SIqKspMnTrVLF++3EyYMMGUKlXKdO/ePdf9Fmd33XWXqVatmpk7d66ZMmWK8fLyMhMnTrRsP3PmjJFkZs+ebbXfCy+8YEJDQ61+V8o0YcIE07NnTzNnzhyzdOlSM3DgQOPq6mq++OKLQj+eqzkZc82fv5Bnp0+f1ltvvaVt27YpJCREgwYNyvEmSygc7du3z3bR/b59+1r99WTnzp2WtZ4qV66s5557LstdKW2JQcG57777dM899+jhhx+2al+5cqWmT5+u8+fP69Zbb9Xzzz+f5a60tsQgfxYsWKAvv/xSCQkJqlWrlp5++mmrG6QYY/TZZ59p/vz5SktLU4cOHTRkyBCrG4/ZEoP8OXjwoCZPnqzdu3fL09NTTZo00eDBg60+8ZGYmKiJEyfqt99+k7e3t3r16pVlzUNbYmC7Dz74QN9++22W9nHjxlldJ8ybN09z5szR5cuX1bJlSz399NPy9va22qegYoCiNHbsWC1fvjxL+9SpUy0zyrp3764WLVpY1nSVZPmZkZKSovbt22vo0KFZ1qezJaa4io2N1dtvv60NGzYoICBA/fv3V6dOnSzb9+/fr379+lnWdJWufOqsS5cu6tevn9VafJkyMjL01VdfacmSJYqPj1f16tU1ZMgQRUREFNlxFbZ//vlHEydO1NGjR1W1alU999xzVsc3c+ZMzZs3z/LprgsXLujDDz/Uhg0b5O3trUaNGmnQoEFZvm/eqN/i7uuvv9Y333yjxMRE3X777XrqqaesbvT38MMPq1atWnr++ee1bt06Pffcc9n2M3v2bMsN9bZt26ZPPvlE+/btU7ly5XTvvffq7rvvLpLjKUgJCQl655139Mcff8jX11cPP/yw7rnnHsv2mJgY9ejRQ2+//baaNWtm0z62xhRnu3fv1rvvvquDBw8qIiJCI0eOVI0aNSzbv/nmG02bNk2rV6+WJN155526ePFiln569eqlwYMHS5Li4uI0ZcoUrVu3Th4eHoqKitKQIUPk6+tbFIdUJBYuXKgvvvhC8fHxat68uZ599ln5+PhYtg8aNEiBgYEaO3aspCvneerUqdq9e7dCQ0PVuXPnbK/bb9RvcZaUlKRJkybp119/lZeXl3r27KlevXpZtsfGxqpLly4aPXq02rdvb2l/4IEH1KxZsyzrcWdatGiRvvvuO509e1aVK1fW448/rrp16xb24VihaAsAAAAAAAAADoQ1bQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAAAAcCAUbQEAAAAAAADAgVC0BQAAAAAAAAAHQtEWAAAAAAAAABwIRVsAAAAAAFBs/fHHH9q8ebO90wCAAkXRFgCKsZ07d+rpp59Wu3bt1KFDBw0ZMkQbNmywSy6//vqr2rVrZ5exAQAAUPysX79ev/32W777mTRpkmbOnFkAGQGA46BoCwDF1LRp09SgQQMlJSVpxIgRevbZZ1WrVi0988wzmjt3bpHnc+7cOW3cuLHIxwUAAEDx9P7772vcuHH2TgMAHBJFWwAohjZv3qzBgwfr3Xff1UcffaQuXbrojjvu0MCBA7V27Vp16dLFEnvgwAE98cQTatWqlf5fe/cWElXXx3H8N1paVApGlieq8UBpE9JBpzC6sBQqRdDqQgrtQOlFBJWWBdlFYdRFBYpJ3oSE1SRDlJBY3nSOGsXAinJMLY3MzDMV+lyEm3feTk+91TM+7/dztfea/16zlhfD4ufea6elpamystKlr7Nnzyo9Pd2l7fbt24qLizPOr1y5opUrV+rOnTvKzMzUsmXLlJOTo56eHmM8eXl56u3tldVqldVqVVFR0W/8CwAAAODf5urVq3r48KG6urpUU1OjmpoaDQwMfFbX3d2ty5cv6/r16+rr6/tiX0NDQ7p7967sdrvq6+tdPnv48KFsNpuGh4eNtrdv36q8vFzt7e2/dlIA8JPG/NMDAAD8uOLiYgUEBCgrK+uLn0+cOFGS9OrVK8XExCghIUF5eXlyOBxKSUlRWVmZ1qxZI0lqa2tTXV2dy/VdXV26ffu2cf769WtVVVWpra1NeXl5GjdunHJzc9Xc3Kzy8nKFhYUpMzNThw4d0rFjxyRJQUFBv2HmAAAA+Lfav3+/PDw89PLlS82aNUuPHz+WJN25c0d+fn6SpLq6Oi1fvlz+/v6aNm2ampqaNH78eE2bNs3o58WLF0pOTlZvb69mzZolh8OhqKgoVVRUaPz48fLz89PWrVvldDq1a9cuSdKmTZvU2tqqtLS0Pz9xAPgCQlsAGIVqa2s1Z84cjRnz7Z/xw4cPKyAgQGfOnJHJZFJiYqK6urq0e/duI7T9uz5+/CibzSaz2SxJ+vDhg9LT0zU8PCwfHx+Fh4fL09NTVqv1p+cFAACA/2/Pnj2Tw+GQv7+/3r9/r8jISJ08eVJ79uyRJGVnZys+Pt5Y3166dElJSUlaunSp0cf69esVGxurwsJCmUwmDQ4OasmSJTp8+LDy8/MVGBioU6dOae3atYqPj9f9+/dVVVUlh8Px3fU1APwpbI8AAKPQ4OCgfHx8vlt37949JSYmymQyGW0rVqyQ0+lUZ2fnD32nn5+fEdhKUnBwsAYGBtTd3f1D/QAAAABfk5qaKn9/f0mSl5eXFi1aZNxx29raqps3b2rnzp3G+nbVqlWKjIw0rm9ubta1a9cUGhqqCxcu6Pz587p48aLMZrNqamqMupSUFGVkZGj16tXavn27jh8/rrCwsD84UwD4Nv6FBACjUEhIiJxO53frenp6jK0SRoyc9/T0GI+Z/R1jx451OR9ZKP/nXmAAAADA/+K/16fe3t7q7e2V9CmQlaQZM2a41MycOdM4bmpqkiTduHHD5SW5JpNJCxYscLkuPz9fpaWlioqK0oYNG37VFADglyC0BYBRKDk5WdnZ2aqvr5fFYvlqXWhoqBoaGlzaGhoa5OXlpeDgYEnShAkT1N/f71LzMy9g8PDwIMAFAADAbzN58mRJn14aNnI8cj4S5I48jbZv3z7Nmzfvm/3l5OTIbDaroaFBdrtdKSkpv2XcAPAz2B4BAEahjIwMzZ8/X+vWrdOTJ0+M9qGhIdlsNlVXV0uSNm7cKLvdrlu3bkmS3rx5o4KCAmVkZMjT01OSZLFY9Pz5cz148ECS1NnZqRMnTvzwmKZOnare3l62SwAAAMBvYTabFRAQILvdbrS1tLS43FFrsVgUFBSk4uLiz65/+fKlcVxeXi6bzSa73a4DBw5o8+bNamtr+63jB4AfQWgLAKOQt7e3qqurFRMTo+joaEVERMhiscjX11c2m03h4eGSPu1fu2fPHsXHx2v27NmaPn26AgICVFBQYPRltVq1ZcsWLV68WHPnzpXFYlF0dPQPjyk2NlYxMTGKiIiQ1WpVUVHRr5ouAAAAoLFjx+rgwYPau3ev8vLyVFhYqISEBE2aNMmo8fT0VGlpqcrKypSamqqSkhIVFBQoLi5O586dk/Rpm4WsrCwdOXJEkZGRys3NVVRUlDIyMnhyDIDbMA3ziwQAo9rg4KCePn0qDw8Pmc1mjRs37rOad+/e6dmzZ5oyZYpCQkK+2E97e7s6OzsVHh6u/v5+PXr0SLGxsZKkjo4ONTc3uzxi1tfXp/r6ei1cuNC4a3d4eFiNjY3q6OhQYGDgV78LAAAAOHHihPr7+7V7925Jn/aYnTNnjtLS0oyakpISDQ4Oatu2bUZbZWWlKioq5Ovrq6SkJNXW1srX11eZmZlGjdPp1OnTp9XY2Kjg4GAlJycba9vCwkI5nU4dPXrUqG9paVFOTo527Njx2d63APBPILQFAAAAAAAAADfC9ggAAAAAAAAA4EYIbQEAAAAAAADAjRDaAgAAAAAAAIAbIbQFAAAAAAAAADdCaAsAAAAAAAAAboTQFgAAAAAAAADcCKEtAAAAAAAAALgRQlsAAAAAAAAAcCOEtgAAAAAAAADgRghtAQAAAAAAAMCNENoCAAAAAAAAgBshtAUAAAAAAAAAN/IXkFtq+A7H1LAAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABW0AAAN5CAYAAABzABkEAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAA+NhJREFUeJzs3Xd4FFX7//FPGgmkAaEkhNAk9CogvfeOVEHpgmClKAIqXdEHeMDKgzQFFRVBkCLSpIj0Kr0JhNBbEhJSSM7vD37s1zUJJCFkN8v7dV1zXeyZe87cM0yyk3vPnnEyxhgBAAAAAAAAAOyCs60TAAAAAAAAAAD8H4q2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAAAAdoSiLQDYkXfffVdOTk62TgMAAABIlTfffFMeHh6Zfn/Tpk2Tk5OTLl26lO59P4wxRlWqVNHQoUMzfN+PS4kSJdSxY8cHtiUkJKhMmTJ69913Mzo9wK5RtAXwRJs8ebKcnJySXb766qs09fvmm2/K1dU1fZMFAADAI9u3b5/lXm/s2LFJxrz99tuWmJMnT2Zwhhnvn+fEyclJzs7Oypkzp5o0aaLff/89Q3J49dVX5eXllSH7slfz58/XkSNH9Pbbb9s6lQzl7OyssWPHasqUKTp37pyt0wHsBkVbAJC0detWGWMSLb169crQPCZMmCBjTIbuEwAA4Enk6empr7/+OtG9V3x8vL755ht5enraKDPbGT9+vIwxiouL08aNGxUZGanGjRtrw4YND9128uTJio6OfvxJ2mh/GWHixInq0qWL8uTJY+tUMtyzzz6rnDlzaurUqbZOBbAbFG0BAAAAAE+c9u3b6++//9amTZus2n/77TdduHBBHTp0sFFmtufi4qKyZctq7ty5io+P16effmrrlBzehg0bdPToUXXv3t3WqdiEs7Ozunbtqq+//trhivFAWlG0BYAUuv+VrcjISL344ovKnj27fH199fzzzyssLMwS17FjR02ZMkXx8fFWXzO7/9W6+/2Eh4erT58+8vPzU3BwsKTk57Q9fPiwOnfurNy5cytLliwqUaKEpk6dajUy5MyZM3r++ecVGBiobNmyqXTp0ho3bpwiIyNTdFy3bt3S888/L19fX+XIkUM9e/bUtWvX0pTLg47x3z7//HM5OTlp165didbNnDlTTk5O+vPPP1O1/88++8zq3Pv4+KhOnTpasWJFkseekjwBAIBjefrpp1WuXLlE02HNnTtXtWrVSvKe4OjRo1b3GB4eHipTpowmT56shIQES9ySJUvk5OSkrVu36oMPPlBgYKB8fHz0wgsvKDIyUsYYvf/++8qfP7+yZs2q1q1b68qVK1b7So8+UppvcoKDg+Xi4qKQkBBJ0osvvqhcuXLp5s2b6tGjh3LmzKmyZctKSnqO2fvxERER6tWrl3x9fZU9e3b17NlTt2/ftsS1atVKn3/+uSIjI63yPX/+fLK5Pcr+7lu4cKHKli0rDw8PlSxZUgsXLkx2f3/99Zc6dOigXLlyyd3dXSVLltRnn31mWb9q1Sq5uLho5MiRVtstXbpUTk5OGj9+fLJ9S9KyZcvk7u6uGjVqJFqXkJCgKVOmqFSpUnJ3d5efn586duyoo0ePWmJq166t4ODgJL+116JFCwUFBVn9n//444+qUaOGPD095enpqXr16mnz5s1W2/n7+6tXr17auXOn6tSpo6xZs2r48OGSpHr16ln+n1xcXOTv769u3brpzJkzDzzOB2nQoIFu3ryZKA/gSUXRFgBS6fXXX1e7du107tw5/fTTT1q+fLnVwwJ++uknDR06VC4uLlZTLRQtWtSqnwEDBqhdu3Y6efKkRowYkez+du/erWeeeUZ37tzRhg0bdP36dX3wwQcaN26c3nrrLUtcixYtFBISonXr1unmzZtavHixJOnnn39O0XENHDhQXbt2VUhIiJYsWaLff/9dTZo0UWxsbKpzSc0x9ujRQ97e3vriiy8SrZs+fbpKlChhuXlN6f5fffVVy3mPj4/XwYMHVaVKFbVr1047d+5MU54AAMDx9OrVSz/99JPlQ+4bN27ol19+SXaKrBIlSljd3124cEFvvvmmRo0apf/+97+J4qdNmyZvb28dPHhQa9eu1Zo1a/TGG29o/Pjx8vT01IEDB7R582bt2rVLr7zySpL7fJQ+Upvvv508eVLx8fEKCgqytBlj9NJLL+m5557TqVOnNGTIkAf2YYzRq6++queee07nz5/XggULtHjxYkvxT5KWL1+uV155RZ6enlb55s+f/6E5pmV/0r2iZefOndWoUSP9/fff+u2337Ry5cok5/DdunWrqlatKmOM/vjjD127dk1jx47VyJEjLQ/Patasmd555x19+OGHWr58uSTp1KlT6tmzp5o3b/7Qh2z98ccfKleunLJkyZJo3YABAzRy5Ei98cYbunTpkjZt2qRLly6pWrVqOnHihKR7BeuTJ08mGjkeGhqq1atXq3fv3nJ2vlcCev/999WtWze1adNGJ0+e1OnTp1W1alU1bNgwUcH0/PnzGjt2rL744gudOnVKVatWlXRvZPD9/6eoqCgtX75cZ8+eVatWrdI8UvaZZ56RpETHADyxDAA8wSZNmmQkJbv8/ffflthXXnnFSDLfffedVR+DBw82bm5uJjo62tI2dOhQ4+LikuQ+7/cze/bsROveeecd8+9fzdWrVzdFihSx6t8YY6ZNm2ZcXFxMSEiIuXTpkpFkZsyYkdpTYMln7ty5Vu2rVq0yksycOXNSlcvDjjEpAwcONFmzZjU3btywtG3dutVIMpMmTUr1/pNTtGhRM2DAgETHntI8AQBA5rd3714jyUydOtVcuXLFuLm5ma+++soYY8wnn3xismXLZsLDw8348eONJHPixImH9jlgwABTtGhRy+uff/7ZSDIvvviiVdyoUaNMlixZTP/+/a3ax44da5ydnc3NmzfTtY+U5nv/nIwfP94YY8zdu3fNwYMHTa1atYyzs7NZt26dMcaYvn37Gknm22+/TdTn0KFDjbu7u1Xb/fjFixdbtQ8cONB4eHiY+Ph4S9srr7xiPD09H5p7eu2vcOHCpmrVqlZxd+/eNU899ZSRZC5evGhpr1ixoilRooSJjY21iv/www9NlixZzOXLl40xxsTHx5vGjRub7Nmzm0OHDpny5cubAgUKmGvXrj30ePz9/U2bNm0StR86dMhIMsOGDbNqv3z5ssmWLZvp0qWLMcaYyMhI4+PjY7p3724VN2HCBOPk5GROnz5tjDHmzJkzxtXV1bzxxhuJ9lW7dm1Ts2ZNy+u8efMad3d3y/E9zL59+4wks3btWktb8eLFTYcOHazikmq7z83NLdExAE8qRtoCgJJ/EFmhQoUSxbZs2dLqdZkyZRQXF2f52lhKtWnT5qEx165d09atW9WmTRu5u7tbrWvUqJHi4+O1ZcsW5cqVS/ny5dPEiRP19ddf6/Lly6nKJal8mjZtqqxZs2r9+vWpyiW1xyhJr7zyiu7cuWP19cQvvvhCbm5u6tGjR6r3HxUVpffee08lS5ZU1qxZraaoSOoJ0CnNEwAAOJbcuXOrefPmlnuQuXPnqmPHjvL29k52m6+++ko1atSQj4+P5R7jf//7n06fPp1oyoHmzZtbvS5RooRiY2PVsGFDq/aSJUsqISEhya+WP2ofqcn3vffek5OTk9zc3FSrVi25u7tr1apVatCggVVcau6dnJycEh1DmTJlFB0drQsXLqS4n/Tc399//62///470XG4uLiodevWVm2hoaHau3ev2rVrJzc3N6t1jRo1UmxsrLZu3Srp3rys3377rby8vFSpUiUdOXJECxculJ+f30PzvnXrVpLX3f178fbt21u158mTR7Vr19a6deskSdmyZVPXrl21aNEihYeHS7o36nju3Llq0KCBChcuLOneNA53795Vp06dEu2rYcOG2rZtm9U37apVq5bkg9GOHTumrl27Kl++fHJ1dZWTk5MqVKggSUneb6eUt7e3bt26lebtAUdC0RYAUiFbtmzy8fGxarv/OjU3F1myZFGuXLkeGne/+PrJJ5/I1dVVLi4ucnFxkbOzs8qUKSNJun79ulxcXLR69WqVLVtWAwYMkL+/v0qVKqVx48YpKirqoftxdXVVzpw5E7XnyZPHMq9tSnNJ7TFKUunSpVWnTh3973//kzFG169f18KFC9WqVSvLTWJq9t+jRw99+umn+uCDD3T+/HnFx8fLGKMKFSooLi7Oat+pyRMAADieXr16aePGjVq6dKn27t2b7NQIkvS///1PvXv3VqtWrXTo0CHFxcXJGKM333xTCQkJiYqgAQEBVq/vF+WSa0/qfvJR+khtvuPHj5cxRgkJCbp586bWrl2rxo0bW8X4+PjIy8srUZ7JyZEjR6K5Z9Ny/5ye+7t/z5g3b95E2/+77dKlS5Kk//znP4nuQStXrmzVn3Tvg4A2bdooOjpaDRo0sHzl/2GyZ89uKbb+0/2+/f39E63z9/e32veLL76oqKgoLViwQJK0ceNGnTp1Sn379k10PHXr1rUcj7Ozs5ydnTVmzBjFx8db/b8EBgYmmVPNmjV1/vx5/fLLLwoLC5MxxlKs/ff9dmpEREQoR44cad4ecCQUbQEgFZJ6SFha/PtT+uTcLyaOHDlSd+/eVXx8vOLj45WQkGAZDfzyyy9Lulf4/OWXX3Tr1i1t2bJFLVu21NixYy3rH+Tu3bu6ceNGovYrV65YRgakJpfUHON9r7zyio4fP65169Zpzpw5io6OtrrBTOn+b9++rZ9//lkvv/yynn32Wfn5+Vnm70pq9Epq8wQAAI6lVatW8vPzU+/evVWoUCHVq1cv2dh58+apSpUqGjlypIKCguTq6irp3sjNpCR375iae8pH6SO1+aZEau+d0uv+OT33d//+Nqlvp/277f496Lhx45K9B+3Tp48lfuXKlZo+fbqqVq2qVatW6euvv05R3gULFrQUVP/p/sCK5HL95yjeypUrq3z58pozZ44kac6cOcqRI4eeffbZRMeze/duy/HcL+DfP55/jqxN6v97xYoVun79uj7++GNVrlxZnp6ekh7tupLufbMuLi5OBQsWfKR+AEdB0RYAHgNPT08lJCTo7t27j9RP3rx5VaVKFf38888p/sT6/lNnJ02apPr166d4Iv9ly5ZZvV69erXu3Llj+epdWnJJjWeffVYBAQH6/PPPNWPGDOXLl0/NmjWzrE/p/p2cnGSMSTSFwv2CNgAAwD+5ubnp+eef182bN9WzZ8+HFv3+fY9x7do1rVq16nGm+EgyS76enp5WX8t/nAoXLqxChQoluv9NSEiwPETsvoIFC6ps2bJavHjxQ+/tz549q+7du6t+/frasmWLOnfurJdffll//fXXQ3OqVauWDhw4kOgc3J+a4t8PF7569ao2bdqUaJqMvn37aseOHdqyZYt++uknPf/881Yjj5s3by4XFxf98MMPD83pYf59bc2bN++R+tuxY4ckqXbt2o/UD+AoKNoCwGNQpkwZGWO0fPlyxcfHP1Jf06dP17lz5/Tss89q9+7dioqKsnwVqXHjxrp+/br27t2r9u3ba82aNbpy5Yru3LmjNWvWaPfu3apfv/5D9+Hp6Wl5Ym5ERIQ2bdqk/v37q1y5curWrVuqckkrNzc39evXT0uWLNGpU6fUq1cvubi4pPpceHp6qkGDBpoxY4a2bdum27dva8WKFfrPf/6jcuXKpTk/AADguKZNmyZjjMaMGfPAuDZt2mjLli36+uuvdfv2be3bt0/t27dPVDizF5kp3/vPifj1118f+f45JSZOnKht27bpzTff1KVLlxQSEqJ+/fqpVKlSiWJnzJiho0ePqlOnTtq7d6+ioqIUEhKin3/+WfXr11dkZKRiY2PVqVMneXh4aMGCBXJxcdGsWbMUFBSkjh07KiIi4oH5tG7dWjExMfrzzz+t2kuXLq0+ffpo6tSpmjVrlm7duqXDhw+rQ4cOcnV11dixY63iX3jhBXl4eKhr1666c+eO1TfXJKlIkSIaN26cJk2apHHjxunMmTO6c+eOjh49qk8++US9e/d+6Llr0KCBvLy8NHz4cIWGhury5csaP368YmJiHrrtg6xfv17Zs2enaAv8fxRtAUBS9erVLQ9m+OcyfPjwNPXXoUMH9e/fX/3795ebm5vlIVhpUalSJe3Zs0e5cuVS27ZtlT17dtWsWVNff/21RowYIT8/P5UvX149e/bUf//7X5UtW1a5c+fWoEGD9Oabb+qLL75I0X4+//xzzZs3T4GBgWrTpo1q166ttWvXWn2CnpJcHkX//v0tDzL459fMUrv/b775RnXq1FGrVq0UGBiomTNnasGCBcqaNesj5QcAAJ5sb775pt555x299957ypMnj1566SWNHTtWZcuWtXVqScpM+Xbt2lW9e/dW9+7dLffP58+ff2z7e+655/T9999r1apVKliwoBo1aqTGjRsnOeChevXq2r17t7y8vNSqVStLYfG7777TmDFj5OnpqUGDBmnv3r368ccfLdMLeHt766efftL58+f14osvPjCfunXrqmTJkpo/f36idV9++aXGjx+vKVOmKG/evKpVq5Zy5cqlrVu3qnjx4lax96dDCAkJ0dNPP215ONg/jRw5UosXL9amTZtUoUIF+fn5qUOHDjpz5oxGjx790HOXP39+LVu2TFeuXFGxYsVUqVIlxcTE6D//+c9Dt01OQkKCFixYoF69eiWakxh4UjkZY4ytkwAA2M6rr76qr776Srdv37Z1Krp165YCAgJUrVo1/f7777ZOBwAAAMgw33zzjQYMGKC///5buXPntnU6GWrRokV64YUXdOzYMRUoUMDW6QB2gZG2AAC7sXTpUkVHR6tfv362TgUAAADIUM8//7xKlSqliRMn2jqVDJWQkKDRo0dr6NChFGyBf3C1dQIAAEjSpUuXNHnyZAUHB6tTp062TgcAAADIUE5OTpaHcT1JnJ2ddfDgQVunAdgdRtoCAGyuWrVqCgoKkru7u3788Ue5ubnZOiUAAAAAAGyGOW0BAAAAAAAAwI4w0hYAAAAAAAAA7AhFWwAAAAAAAACwIzyI7DFISEjQhQsX5O3tLScnJ1unAwAA4FCMMYqIiFC+fPnk7MwYhPTGvSwAAMDjk9J7WYq2j8GFCxcUFBRk6zQAAAAcWkhIiPLnz2/rNBwO97IAAACP38PuZSnaPgbe3t6S7p18Hx8fG2cDAADgWMLDwxUUFGS550L64l4WAADg8UnpvSxF28fg/tfIfHx8uNEFAAB4TPjq/uPBvSwApN75m1H6dN1JvdawqPLnyGbrdABkAg+7l2USMAAAAAAAgEdwKypOP+wK0a2oOFunAsBBULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAHkEuL3cNrPeUcnm52zoVAA7C1dYJAAAAAAAAZGb+vh56u1kJW6cBwIEw0hYAAAAAAOAR3I65q62nrut2zF1bpwLAQVC0BQAAAAAAeARnrkWq68xtOnMt0tapAHAQFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAACAR+Dq4iR/Hw+5ujjZOhUADsLV1gmkp3Xr1mnRokW6cuWK8uXLp27duqlatWpWMadPn9Znn32ms2fPKjg4WIMGDZK/v3+qYwAAAAAAACSphL+Pto1saOs0ADgQhxlpO3PmTDVv3lx58+bVc889pyxZsqhmzZr65ZdfLDGnTp1S5cqVdeHCBbVr10779+9X5cqVdfXq1VTFAAAAAAAAAMDj4mSMMbZOIj3Uq1dPBQoU0Lx58yxtjRs3lp+fn77//ntJUs+ePXXo0CHt3LlTTk5Oio2NVdGiRdWtWzd9+OGHKY55mPDwcPn6+iosLEw+Pj7pf7AAYEcKDV9h6xSQTs582NLWKQApwr3W48X5BYDUO3opXL3m7NRXfaqohH/6/e7M6Htt7geBxy+l91oOM9K2RIkSOnHihOLj4yVJUVFROnPmjEqVKmWJWbVqlZ599lk5Od2bYyZLlixq3bq1Vq1alaoYAAAA4HEwxii5MRWxsbG6ffu21RIZGZlsX3FxcY8rTQDAv9yNN7oUHq278Q4xLg6AHXCYou1///tflS9fXkWKFFHjxo0VHBysTp06aeTIkZLuFXGvXLmi/PnzW20XFBSkv//+O8UxSYmJiVF4eLjVAgAAAKTUxo0b1aJFC2XPnl1eXl6qU6eOtm3bZhXzwQcfyNfXV/7+/paldOnSifr67LPPFBAQIA8PDxUuXFg//fRTRh0GAAAA0onDFG3XrFmjH3/8UT169FC/fv3UpUsXffnll9q+fbukeyMTJClbtmxW22XLls2yLiUxSZk4caJ8fX0tS1BQULodFwAAABzfZ599pjfeeEOhoaG6fPmyypUrp6ZNm+r8+fNWcdWrV7caaXvmzBmr9YsWLdLQoUP1v//9T1FRURo2bJiee+457dixIwOPBgAAAI/KYYq2L7/8sl566SWNHz9enTt31n//+1+1atVKgwYNkiR5eXnJxcVFN27csNru+vXryp49e4pjkjJixAiFhYVZlpCQkPQ8NAAAADi4hQsXqmnTpvLy8pKXl5emTJmiyMhIrV+/PlHs3bt3k51CYdq0aWrfvr3atm0rd3d3DRw4UBUrVtSnn376uA8BAAAA6cghirbx8fG6cuWKnnrqKav2okWLKjQ0VJLk6uqq0qVLa//+/VYx+/btU/ny5VMckxR3d3f5+PhYLQAAAEBaXbx4UfHx8fLz87Nq37lzpzw9PeXt7a369etr9+7dlnXx8fHauXOn6tSpY7VN/fr1E021AABIX4VyeWpBv2oqlMvT1qkAcBAOUbR1cXFRpUqV9P3331umMYiMjNSiRYtUtWpVS1yPHj30448/Wr5Gtn//fv3222/q0aNHqmIAAACAxyUhIUGvvvqqihUrpkaNGlnaCxYsqMWLFys8PFynT59WoUKFVL9+fZ09e1aSFBYWppiYGOXOnduqv9y5c+vKlSvJ7o/nMwDAo/Nyd1X1p/zk5e5q61QAOAiHKNpK0qxZsxQaGqoiRYqoYcOGKly4sJydnfXxxx9bYl5//XU1bNhQ5cqVU82aNVWjRg316dNHXbt2TVUMAAAA8Li8/vrr2rp1qxYtWiR3d3dLe+/evdWyZUu5u7srT548+vLLL+Xl5aU5c+ZYbZ+QkJDotZOTU7L74/kMAPDoLoVF66NVR3UpLNrWqQBwEA7zEVCZMmV08OBBHTlyRJcvX1ZgYKCKFy9udYPq5uamH374QceOHdO5c+dUtGhRFS5c2KqflMQAAAAAj8PgwYP13Xffad26dSpTpswDY93c3PTUU0/p9OnTkqTs2bMra9asiUbVXrlyRQEBAcn2M2LECA0ZMsTyOjw8nMItAKTStdsxmr7hlFqWDZC/r4et0wHgABymaCvdmyahTJkyD73BLV68uIoXL/7IMQAAAEB6GTJkiL7++mutWbNGFStWfGj8nTt3dPToUdWuXVuS5OzsrBo1amjdunV69dVXLXFr1qxRrVq1ku3H3d3dakQvAAAAbM9hpkcAAAAAMqthw4Zp1qxZWrJkiYoXL67bt2/r9u3biouLs8Q0b95cv/76qy5evKgDBw6oS5cuiouLU//+/S0xb7/9tpYvX64ZM2YoJCREo0eP1vHjxzVo0CAbHBUAAADSyqFG2gIAAACZ0ZdffqmEhAS1aNHCqn3UqFEaNmyYJOn999/XhAkTtH37dnl6eqpq1aravXu3ChUqZIlv3Lixvv32W33wwQcaMWKEihcvrpUrV6p06dIZeTgAgEyq0PAVGbavMx+2zLB9AZkRRVsAAADAxm7duvXQmKefflqLFy9+aFznzp3VuXPndMgKAJBS2bO5qUvlIGXP5mbrVAA4CIq2AAAAAAAAjyB/jmz6qGM5W6cBwIEwpy0AAAAAAMAjiI6L1/HLEYqOi7d1KgAcBEVbAAAAAACAR3Dyym01mbpJJ6/ctnUqABwERVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAADgEWVxocQCIP242joBAAAAAACAzKxMoK+Ov9/c1mkAcCB8DAQAAAAAAAAAdoSiLQAAAAAAwCM4eSVCLT/ZrJNXImydCgAHQdEWAAAAAADgEUTHJejQhXBFxyXYOhUADoKiLQAAAAAAAADYEYq2AAAAAAAAAGBHKNoCAAAAAAAAgB2haAsAAAAAAPAIgnJk0+fdnlZQjmy2TgWAg3C1dQIAAAAAAACZmW82N7UsF2DrNAA4EEbaAgAAAAAAPIKrETGatfm0rkbE2DoVAA6Coi0AAAAAAMAjuBwerQkrjuhyeLStUwHgICjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAI/A28NVjUrmkbeHq61TAeAg+G0CAAAAAADwCAr6eWpWzyq2TgOAA2GkLQAAAAAAwCOIi0/Q9dsxiotPsHUqABwEI22BJ1yh4StsnQLS0ZkPW9o6BQAAAOCJc+xShFp9+oeWv1ZLZQJ9bZ0OAAfASFsAAAAAAAAAsCMOM9L23Llzio2NTdTu6empgIAAq7bo6GhdvXpVefPmVZYsWZLsLyUxAAAAAAAAAJDeHGakbb9+/dSsWTOrJTg4WEOGDLGKGz16tHLmzKkKFSooV65c+vTTTxP1lZIYAAAAAAAAAHgcHGak7W+//Wb1eteuXapSpYqef/55S9v8+fM1adIkrV27VjVq1NAvv/yi9u3bq0SJEmrcuHGKYwAAAAAAAADgcXGYkbb/Nnv2bAUGBqp58+aWtunTp6tDhw6qUaOGJKlNmzaqXbu2pk+fnqoYAAAAAACA+0oG+OivMU1UMsDH1qkAcBAOWbS9c+eOFixYoD59+sjFxUWSlJCQoD179qh69epWsbVq1dKuXbtSHAMAAAAAAPBPLs5O8vZwk4uzk61TAeAgHLJo+9NPPykiIkJ9+/a1tEVERCgmJkZ+fn5Wsbly5dK1a9dSHJOUmJgYhYeHWy0AAAAAAODJ8Pe1SHWfvV1/X4u0dSoAHIRDFm1nz56tJk2aqGDBgpY2Z+d7hxoXF2cVGxsbaxmNm5KYpEycOFG+vr6WJSgoKF2OAwAAAAAA2L/ImLvafOKaImPu2joVAA7C4Yq2J0+e1KZNm9SvXz+rdm9vb/n6+urSpUtW7ZcuXVL+/PlTHJOUESNGKCwszLKEhISk09EAAAAAAAAAeNI4XNF2zpw5ypMnj1q3bp1oXb169fTbb79Ztf3666+qV69eqmL+zd3dXT4+PlYLAAAAAAAAAKSFQxVt4+Pj9fXXX6tXr15yc3NLtP6dd97Rpk2bNGrUKO3cuVOvvPKKQkJCNHTo0FTFAAAAAAAAAMDj4lBF223btilr1qx68cUXk1xfpUoVrV69Wjt37lSvXr0UGhqqjRs3qmjRoqmKAQAAAAAAuC/A10Pj2pZWgK+HrVMB4CBcbZ1AeqpZs6ZOnjz5wJi6deuqbt26jxwDAAAAAACePIWGr0h23ailhzIwEwCOzKFG2gIAAAAAAABAZkfRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsiKutEwAAAAAAAEirQsNX2DoFAEh3jLQFAAAAAAAAADvCSFsAAADADsTGxurQoUOKj49XiRIl5OXllWTcqVOndO7cOQUHByt//vxpjgEAAID9YqQtAAAAYGP/+c9/VLhwYfXt21f9+/dXYGCgZsyYYRVz9+5dPffcc6pQoYKGDx+u4OBgDR06NNUxAAAAsH+MtAUAAABszNnZWX/99Zdy5swpSZo3b5569+6t6tWrq1y5cpKkqVOnau3atfrrr79UqFAh7dq1SzVq1FCNGjXUoUOHFMcAAADA/jHSFgAAALCxN99801KwlaQXXnhBTk5O2rVrl6Xt66+/1nPPPadChQpJkipXrqxGjRrpq6++SlUMAAAA7B9FWwAAAMDObN++XfHx8SpevLike/PdHjlyRBUrVrSKq1ixovbv35/imKTExMQoPDzcagEAAIBtUbQFAAAA7EhYWJj69u2rpk2bqmbNmpKk8PBwJSQkWI3GlSQ/Pz/dvHkzxTFJmThxonx9fS1LUFBQOh8RAAAAUouiLQAAAGAnIiMj1apVK7m7u2vBggWW9ixZskiS7ty5YxUfFRVlWZeSmKSMGDFCYWFhliUkJCRdjgUAAABpx4PIAAAAADsQGRmpli1bKjw8XOvXr1eOHDks63x8fJQzZ85EBdXz589b5q9NSUxS3N3d5e7unm7HAQAAgEfHSFsAAADAxqKiotSyZUvdvHlT69atk5+fX6KYpk2baunSpZbXsbGxWr58uZo1a5aqGAAAANg/RtoCAAAANta2bVvt2bNH06dP165duyztxYoVU5EiRSRJo0eP1jPPPKMePXqoefPmmj9/viRp0KBBlviUxAAAAMD+MdIWAAAAsLEsWbKoRo0amj9/vqZNm2ZZtm/fbokpXry4duzYIR8fHy1YsEClS5fWjh07lDt37lTFAAAAwP4x0hYAAACwsRUrVqQornjx4vrss88eOQYAAAD2jZG2AAAAAAAAAGBHKNoCAAAAAAAAgB1xuOkRtm/frl9//VVOTk7q2LGjSpcubbX+5s2bmj9/vs6ePavg4GD16NFD2bJlS3UMAAAAAAAAADwODjXSdvDgwWrSpIlu374tDw8P9e7dWxs2bLCsv3z5sp5++mktXLhQ2bNn15dffqlq1arp9u3bqYoBAAAAAAAAgMfFYUbaLly4UJ9++qm2bt2qKlWqSJKGDBmiK1euWGImTJggDw8PrV27Vu7u7nr99dcVHBysTz75RCNHjkxxDAAAAAAAAAA8Lg4z0vaLL75Qs2bNLAVbSXJzc1NgYKDl9dKlS9WpUye5u7tLknx9fdW6dWstXbo0VTEAAAAAAAAA8Lg4TNF29+7dqlWrllauXKm33npLkyZN0rFjxyzrY2JiFBISosKFC1ttV6RIEZ04cSLFMUmJiYlReHi41QIAAAAAAAAAaeEQRVtjjCIiIjRv3jxNmjRJuXLl0sGDB1W+fHn9/PPPkqQ7d+5Ikry9va229fHxUVRUVIpjkjJx4kT5+vpalqCgoHQ7NgAAAAAAAABPFoeY09bJyUleXl5ydnbW2rVr5eLiIknKmjWrhg0bpmeffVaenp5ycnLSrVu3rLa9efOmfHx8JClFMUkZMWKEhgwZYnkdHh5O4RYAAAAAAABAmjjESFtJKlOmjMqVK2cp2EpSxYoVdfbsWRlj5ObmpuDgYB05csRquyNHjqhUqVKSlKKYpLi7u8vHx8dqAQAAAAAAAIC0cJiibbdu3bRlyxZFRkZa2tasWaMKFSrIyclJktSlSxf98MMPunHjhiTp7NmzWr58ubp06WLZJiUxAAAAAAAAAPC4OMT0CJI0YMAArV27VmXLllW9evV06NAhhYSEaPny5ZaYt99+W2vXrlWlSpVUo0YN/f7776pbt65efPHFVMUAAAAAAAAAwOPiMEVbNzc3LV26VJs3b9axY8fUoUMH1atXT56enpYYT09Pbd68WWvXrtW5c+fUr18/1a1b1zISN6UxAAAAAAAAAPC4OEzR9r7atWurdu3aya53cXFR06ZNH9hHSmIAAAAAAAAA4HFwmDltAQAAAAAAAMARULQFAAAAAAAAADvicNMjAAAAAAAA2yo0fIWtUwCATI2RtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHbE1dYJAAAAAAAA4MlSaPiKDNvXmQ9bZti+gPTCSFsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjjhM0dYYo7t371ot8fHxtk4LAAAADuzChQu2TgEAAAAOyGGKtu+//77c3Nzk4eFhWfLmzZsobu7cuSpSpIhcXV1VokQJLVmyJE0xAAAAQOXKldWwYUN9++23unPnjq3TAQAAgINwmKKtJFWtWtVqpO21a9es1q9cuVL9+/fXhAkTdPPmTb388svq1KmTdu7cmaoYAAAAQJKWLFmip556Si+//LICAgI0cOBA7dq1y9ZpAQAAIJNzqKLtw0yZMkVt2rRRt27d5O3trddff11PP/20Pv7441TFAAAAAJL0zDPP6Msvv9TFixf1ySef6MiRI3rmmWdUrlw5TZs2LdEgAgAAACAlHKpou2/fPmXNmlU5cuRQ06ZNtW/fPss6Y4y2bdumunXrWm3ToEED/fnnnymOAQAAAP4tW7Zs6tGjhzZs2KB58+bp9OnTGjx4sAIDA9WnTx9dvHjR1ikCAAAgE3GYom2+fPk0f/58XblyRQcOHFCePHlUp04dnT17VpIUERGhqKgo5c6d22q7PHny6PLlyymOSUpMTIzCw8OtFgAAADw5Ll++rEmTJqlkyZLq27evWrZsqTVr1mjp0qU6c+aMGjRoYOsUAQAAkIm42jqB9NKnTx/Lv729vTVnzhwVLlxYs2bN0vjx45PdLiEhQU5OTg/s+2ExEydO1NixY1OfNAAAADK15cuXa+bMmVq5cqWKFSumfv36qUePHsqVK5clplGjRsqZM6euX78uPz8/G2YLAACAzMJhRtr+m5ubm4oVK6aTJ09KulfI9fT01JUrV6zirl69Kn9//xTHJGXEiBEKCwuzLCEhIel8NAAAALBHb7zxhnLmzKkNGzbo0KFDGjJkiFXBVpJcXV01btw4eXh42ChLAAAAZDYOW7SNjY3V0aNHFRgYKElycnJSjRo19Pvvv1vFrVu3TjVq1EhxTFLc3d3l4+NjtQAAAMDxHTx4UHPnzlXNmjUfGDdo0CB5enpmUFYAAADI7BymaNu+fXutX79eN2/e1IkTJ9S9e3eFh4erf//+lpg333xTK1as0Jw5c3T16lVNmjRJ+/fv1+DBg1MVAwAAAEhS1qxZtXXrVhljrNr37t2rqKgoG2UFAACAzM5hirZvv/22pkyZouDgYDVs2FDx8fHasWOHihUrZolp0qSJvv76a02aNEmFChXSt99+qyVLlqhixYqpigEAAAAkafr06Zo/f36i5x/s2LFDQ4cOtVFWAAAAyOyczL+HBeCRhYeHy9fXV2FhYUyVALtXaPgKW6eAdHTmw5YZvk+uIcdhi+sHSAt7uteqVKmSZs2alegD/ujoaAUEBOjatWtycXFJUV/x8fFavny59u7dq44dO6pMmTJW6zds2KANGzZYtWXLlk3Dhg2zaouKitLPP/+sc+fOqVixYmrbtq1cXVP+/GF7Or8AMi/uEWFPuM+FPUnpvZbDjLQFAAAAMlpYWFiiqRGke89KuH37doqnSFi5cqWeeuopzZw5U2PHjtXBgwcTxWzYsEFz5sx5YD83btxQ5cqVNWnSJF26dElvv/22GjZsqJiYmJQdEAAAAOxCyj9yBwAAAGClevXqmjJliubPny9n5/8bDzF58mQVK1ZM3t7eKeonICBAmzdvVkBAgNzc3JKNK1CggMaMGZPs+vHjxysuLk47d+6Up6enRowYoeLFi+vLL7/Ua6+9luLjAgAAgG1RtAUAAADSaPTo0apatarKlCmjBg0ayN3dXdu2bdPOnTu1bNmyFPdzf3qFu3fvPjDu6tWrmjx5srJly6ZnnnlGlStXtlq/aNEi9ejRQ56enpIkf39/tWzZUj/99BNFWwBMWQAAmQjTIwAAAABpVLRoUe3du1eNGzfWrl27tHHjRgUHB2vbtm1q2rRpuu/P1dVVISEh2rZtm2rXrq3+/ftb1kVFRSkkJETBwcFW2wQHB+vYsWPJ9hkTE6Pw8HCrBQAAALZl85G23bt319SpU5UrV65UrQMAAADsQYECBfTxxx8/9v107dpVo0aNskzD8PLLL6tWrVpq0qSJOnbsqMjISEmSr6+v1XbZs2fX7du3k+134sSJGjt27ONLHAAAAKlm85G2v//+u6Kjo5Nct27dOsXFxWVwRgAAAEDqxMbG6vLly7p06ZLVkpCQkG77KF68uNW8udWqVVP58uX1+++/S5JlSoSwsDCr7W7duiUvL69k+x0xYoTCwsIsS0hISLrlDAAAgLSx2Ujb8+fP6+7du4qPj7f8+z5jjI4dO6abN28qd+7ctkoRAAAAeKAzZ86oV69e2rx5c5IF2osXL8rf3/+x7d8Yo6ioKElStmzZFBQUpBMnTljFnDhxQsWLF0+2D3d3d7m7uz+2HAEAAJB6NivaVqtWTaGhoZLuPXX331xcXPTee+/J1dXmMzgAAAAASRo8eLBcXV21Zs0a5cmTJ9H69Jzma+fOnapSpYrl9bZt23TgwAENGjTI0tahQwf98MMPGjlypLJly6ZLly5pxYoVGj9+fLrlAQAAgMfPZhXRP/74Q3fv3lXt2rW1cOFCqxEILi4u8vf3V9asWW2VHgAAAPBQBw4c0PLly1WyZMlH6ufkyZP65ptvLKN1f/rpJx09elSVK1dWq1atJEljx47VnTt3VKFCBV29elULFy5U9+7d9fzzz1v6effdd/Xbb7+pRo0aqlu3rlasWKHy5cvrpZdeeqT8AAAAkLFsVrQtVKiQJOnYsWPy9vaWk5OTrVIBAAAA0qRAgQIPfMhXajk7O2v06NFJrlu+fLn+/PNPbd++XcWLF9fQoUNVvnx5qxg/Pz/t2rVLixcv1rlz5/TRRx+pbdu2fHsNAAAgk7H53ZuPj4/i4uJ0/PhxXbt2TcYYq/XVq1dnji0AAADYpcGDB+utt97S/PnzFRQUlOZ+ihYtqjFjxjw0rkaNGqpRo8YDY7Jly6YXXnghzbkAAADA9mxetN27d6/atWunc+fOJbk+JCRE+fPnz+CsAAAAgId76623dOLECRUoUEBeXl5ycXGxWn/8+PEk57oFAAAAHsTmRdvhw4erXr16mjhxYpI3tHyVCwAAAPZqypQpio6OTna9r69vBmYDAAAAR2HziujZs2f18ccfK1++fLZOBQAAAEiV+w8JAwAAANKTs60TKF68eLJTIwAAAACZwdmzZ7Vu3TrFxsbaOhUAAAA4AJsXbQcOHKg33nhDq1evVmhoqC5dumS1JCQk2DpFAAAAIEmRkZFq27atChUqpEaNGunGjRuSpN69e2vBggU2zg4AAACZlc2nR3jxxRcVGhqqpk2bJrmeB5EBAADAXo0dO1a3b9/WuXPnVKVKFUv7sGHD1LFjR3Xt2tWG2QEAACCzsnnRdt26dYqLi0t2vb+/fwZmAwAAAKTcb7/9pq+++kpBQUFycnKytJcoUUJnzpxReHi4fHx8bJghAAAAMiObF22LFy9u6xQAAACANImIiFC2bNkkyapoe+fOHcXExMjV1ea32wAAAMiEbH4XeezYsQeOtC1RogQ3uwAAALBL1apV04IFCzRmzBhL0dYYo9GjR6tChQqWgi4AAACQGjavhjZs2FChoaHJrmdOWwAAANircePGqXr16tq5c6fCw8P13nvvadeuXTp8+LB+++03W6cHAACATMrmRdu9e/cqPj7equ3s2bMaMWKEGjZsqHz58tkoMwAAAODBihYtqr1792rq1Km6efOm9uzZo3Llymnu3LmqUKGCrdMDAABAJmXzom3u3LkTtfn7+2vhwoUqU6aM3nnnHRtkBQAAAKRM/vz5NWXKFFunAQAAAAdi86Jtcvz8/HTnzh3dunVL2bNnt3U6AAAAQCLx8fEyxiS7nmczAAAAIC1sfheZ1I1uRESEZs6cKRcXF/n6+tooMwAAAODBAgMDdfny5WTXX7x4Uf7+/hmYEQAAAByBzYu2BQsWTPJBZD4+Ppo1a5blKbwAAACAvVm8eLFiY2Ot2s6cOaMPPvhAnTt3Vs6cOW2UGQAAADIzmxdtFy5cqJiYGKs2Hx8fFStWTF5eXjbKCgAAAHi4GjVqJNles2ZNtWrVShMmTMjgjAAAAOAIbF60rV69uq1TAAAAANJVcHCwLl68qMjISHl6eto6HQAAAGQyNi/a3hcaGqpDhw4pJiZGxYoVU/HixW2dEgAAAJAmixcvVkJCgrJly2brVAAAAJAJ2bxoGxsbq9dff12zZs1SfHy8nJycZIxRkyZN9NVXXykgIMDWKQIAAABJql69uq5evWrVFhERoWvXrmny5Mk8nwEAAABpYvOi7fvvv69ff/1VS5cuVb169eTm5qb9+/dryJAh6t27t1atWpXqPi9cuKDvvvtOxYsXV+vWra3WxcXF6ddff9XZs2cVHBysJk2ayNnZOdUxAAAAwMCBAxUVFWXV5uPjo8qVK6tYsWI2ygoAAACZnc2LtkuWLNHMmTPVpEkTS1uVKlW0dOlS+fv7p3oesISEBHXr1k27d+9W06ZNrYq2ERERatCggcLCwlS7dm1NmjRJJUqU0PLly5UlS5YUxwAAAACS1KNHD1unAAAAAAdk86JtVFSU8uTJk6jd19dXbm5uio6OTlXRdvz48fLz81PdunUTrZs4caIuX76sAwcOKHv27AoNDVWpUqX05Zdf6tVXX01xDAAAACBJ8fHxMsakKNbV1ea33gAAAMgkbP6d/6pVq2r8+PFWXytLSEjQhAkTlC9fPvn5+aW4r82bN2vOnDmaOXNmkusXLlyoLl26KHv27JKkwMBAtWrVSj/++GOqYgAAAADp3r2im5tbipZLly7ZOl0AAABkEjb/uH/ChAmqX7++goKCVLlyZWXJkkUHDhzQzZs3tWjRohT3c+PGDb3wwguaPXu2cubMmWh9bGysTp06peLFi1u1lyhRQqtXr05xTFJiYmIUExNjeR0eHp7ivAEAAJB5TZo0SQMHDtTLL7+sWrVqKUuWLNq3b5+mTZumPn36qEWLFpbY1AxGAAAAwJPN5kXbQoUK6ciRI/rqq6+0f/9+xcTE6OWXX1bv3r2TnDYhOb1791bHjh3VqFGjJNdHRkbKGGMZQXtf9uzZFRERkeKYpEycOFFjx45Nca4AAABwDIsXL9b06dPVvXt3S1uzZs3UokULtW3bVh988IENswMAAEBmZfOirSR5eHhowIABad5+zZo1WrVqlUaNGqXJkydLkk6fPi1XV1dNnjxZ/fv3V7Zs2SQlHgUbFhZmmTM3JTFJGTFihIYMGWJ5HR4erqCgoDQfDwAAADKHv/76Sx999FGi9rJly+rKlSu6ffu2vLy8bJAZAAAAMjObzWkbExOj4cOHKz4+Psn1n3zyiU6cOJGivvLly6fXXntNN2/e1KVLl3Tp0iXLlAWXLl1SfHy83N3dVbBgQZ06dcpq21OnTik4OFiSUhSTFHd3d/n4+FgtAAAAcHx58uTRvHnzErX/9NNPcnNzU9asWW2QFQAAADI7m420nTVrlqKjo+Xi4pLk+ty5c2v8+PFJ3gT/W+nSpS0jbO87evSoPDw8rNrbtm2rhQsX6r333pOHh4du3LihZcuW6a233kpVDAAAACDdez5DixYttG7dOtWsWVNubm7av3+/fvvtN33++efJ3usCAAAAD2KzkbbLli1Ty5Ytk13fqlUrLVu2LF33+e677yohIUF169bVO++8o7p166pgwYJ69dVXUxUDAAAASFKDBg10+PBhValSRbt27dKmTZuUN29ebd++/ZGm/wIAAMCTzWYjbU+fPq3ChQsnu97b21suLi66ceOGcubMmer+27dvL1dX68PLnTu39u7dq++++07nzp3T0KFD9dxzz8nDwyNVMQAAAMB9RYoU0SeffGLrNAAAAOBAbFa0dXd3140bN5JdHxsbq9u3b6e5WNqnT58k2729vfXSSy89cNuUxAAAAAD3nT17VidPnlTt2rWVJUsWW6cDAACATM5m0yM888wz+uabb5Jdv3DhQgUHBytbtmwZmBUAAACQcpGRkWrbtq0KFSqkRo0aWQYl9O7dWwsWLLBxdgAAAMisbFa0ff311zVjxgwNGzZMN2/etLTHxcXpiy++0MCBA/Xmm2/aKj0AAADgocaOHavbt2/r3Llzyps3r6V92LBhmjBhgg0zAwAAQGZms+kRypcvr7lz56pv376aPHmy8uXLJ3d3d507d07x8fF6++231bNnT1ulBwAAADzUb7/9pq+++kpBQUFycnKytJcoUUJnzpxReHi4fHx8bJghAHtWaPgKW6cAALBTNivaSlK3bt1Uv359LViwQIcPH9bdu3dVuHBhdejQQWXKlLFlapkKb/SO5cyHLW2dAgAASKGIiAjLdF7/LNreuXNHMTExiR6MCwAAAKSEze8iAwICNGTIEFunAQAAAKRatWrVtGDBAo0ZM8ZStDXGaPTo0apQoQLPZwAAAECa2LxoCwAAAGRW48aNU/Xq1bVz506Fh4frvffe065du3T48GH99ttvtk4PAAAAmRRFWwAAACCNihYtqr1792rq1Km6efOm9uzZo3Llymnu3LmqUKGCrdMDAADK+GklmfYQ6YGiLQAAAJBGn332mXr37q0pU6bYOhUAAAA4EGdbJwAAAABkVpMmTdLNmzdtnQYAAAAcDEVbAAAAII3Kly+vrVu32joNAAAAOBimRwAAAADSqGPHjho4cKBOnTqlChUqyMPDw2p9jRo1lCVLFhtlBwAAgMyKoi0AAACQRsOGDdP169c1YsSIJNdfvHhR/v7+GZwVAAAAMjuKtgAAAEAahYaGyhiT7HpXV263AQAAkHrMaQsAAACkUqFChXT58mW5uLjI1dVVH330kaKjo+Xq6mq1AAAAAGlB0RYAAABIpejoaKsRtp9++qlu375tw4wAAADgSCjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEibYAAACANJg2bZq8vLwkSZGRkVav7xs8eLA8PT1tkR4AAAAyMYq2AAAAQCrly5dP33zzjeW1r6+v1ev7XnrpJYq2AAAASDWKtgAAAEAq7dmzx9YpAAAAwIExpy0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAA2IGTJ0/qrbfeUqNGjbRhw4YkY44cOaKBAweqZcuWGjx4sM6fP5+mGAAAANg3irYAAACAjc2YMUPNmzdXzpw5tW7dOl26dClRzJEjR1S1alXFxMSoV69eOnnypKpWraorV66kKgYAAAD2j6ItAAAAYGMdOnTQ8ePH9dZbbyUbM3bsWJUtW1Zz5sxRp06dtGjRIjk7O2vq1KmpigEAAID9o2gLAAAA2FiuXLnk5OT0wJjVq1erbdu2ltdZsmRRy5YttXr16lTFAAAAwP5RtAUAAADsXHh4uG7evKn8+fNbtefPn19nzpxJcUxSYmJiFB4ebrUAAADAtijaAgAAAHYuNjZWkpQ1a1ar9mzZslnWpSQmKRMnTpSvr69lCQoKSs/UAQAAkAYUbQEAAAA75+PjI2dnZ924ccOq/fr168qRI0eKY5IyYsQIhYWFWZaQkJD0PwAAAACkisMUbaOiojRp0iTVqlVLJUqUUJs2bbR+/fpEcWvWrFGDBg301FNPqVmzZtq+fXuaYgAAAICMkiVLFpUqVUp79uyxat+zZ48qVKiQ4pikuLu7y8fHx2oBAACAbTlM0fbdd99VVFSUpkyZop9//lkVK1ZU06ZNtWXLFkvM1q1b1bJlSzVq1EhLlixR6dKl1aBBAx07dixVMQAAAEBG69Wrl77//nv9/fffkqTt27dr3bp16tWrV6piAAAAYP+cjDHG1kmkh4SEBDk7W9egCxQooBdffFGjRo2SJLVp00axsbFatWqVJaZMmTKqUaOGvvzyyxTHPEx4eLh8fX0VFhaWISMVCg1f8dj3gYxz5sOWGbo/rh/HktHXj8Q15Ehscf0AaZHR91oZYdeuXRo+fLiMMVq/fr3KlCmjvHnzqnXr1nrjjTckSXfv3lWvXr30888/q0SJEjp8+LBee+01/ec//7H0k5KYh3HE8wvYK+6jAMfEfTUeJKX3Wq4ZmNNj9e+C7bp163T58mXVqVPH0rZx40ZLAfe+Zs2aafny5amKAQAAANJT4cKFNXz4cEn35pi9L3/+/JZ/u7q66ptvvtGZM2d07tw5BQcHKyAgwKqflMQAAADA/jlM0VaSjh07poYNGyoyMlLR0dGaM2eO6tWrJ0mKiIhQeHi4/P39rbbJmzevQkNDUxyTlJiYGMXExFheh4eHp9MRAQAA4Eng5+enRo0apSi2UKFCKlSo0CPHAAAAwH45zJy2kvTUU09p27Zt2rx5s9566y3169dPmzdvlnRv+gTp3uiDf3Jzc1N8fHyKY5IyceJE+fr6WpagoKB0OyYAAAAAAAAATxaHKtq6uroqf/78KlOmjMaNG6c6depoypQpkiRvb2+5u7vr2rVrVttcu3ZNuXPnTnFMUkaMGKGwsDDLEhISks5HBgAAAAAAAOBJ4VBF23/Lli2boqKiJN2b87Zy5crasmWLVczmzZv1zDPPpDgmKe7u7vLx8bFaAAAAAAAAACAtHKZo+8Ybb+jMmTOSJGOMFi5cqOXLl6tTp06WmFdeeUU///yz1q9fL0lauHCh/vjjD7388supigEAAAAAAACAx8VhHkRWo0YNtWzZUqGhoYqJiVFgYKA++eQT9evXzxLTtWtXnTlzRu3atVNCQoLc3d01ffp01a9fP1UxAAAAAAAAAPC4OEzRtkuXLurSpYsiIiKUJUsWubu7Jxk3YsQIvfXWW7p586Zy5swpFxeXNMUAAAAAAAAAwOPgMEXb+7y9vR8a4+rq+sAHi6U0BgAAAAAAAADSm8PMaQsAAAAAAAAAjoCiLQAAAAAAAADYEYebHgEAAAAAAACwlULDV2TYvs582DLD9oWMxUhbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO+Jq6wQAAAAAAEhOoeErMnR/Zz5smaH7AwAgKYy0BQAAAAAAAAA7QtEWAAAAAAAAAOwI0yMAAAAAAPD/ZfR0DAAAJIWRtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHbEoea0jYyM1N69exUXF6eyZcsqV65cScYdOHBAZ8+eVXBwsEqUKJHmGAAAAAAAAABIbw4z0nbcuHEqVqyYhg8frjFjxqhgwYKaNGmSVUxsbKzatWun+vXra9q0aXrmmWfUt29fGWNSFQMAAAAAAAAAj4vDjLTNkyePjh49Km9vb0nSkiVL9Oyzz6pevXqqUqWKJGnatGnasmWL9u/fr/z58+vQoUOqXLmy6tWrp+7du6c4BgAAAAAAAAAeF4cZaTtgwABLwVaS2rVrpyxZsmjv3r2Wtvnz56tLly7Knz+/JKl06dJq3ry55s+fn6oYAAAAAAAAAHhcHKZo+29//PGHYmNjVbp0aUnS3bt3deTIEZUrV84qrly5cjpw4ECKY5ISExOj8PBwqwUAAAAAAAAA0sIhi7Y3b95Unz591LJlS9WsWVOSdPv2bcXHxytHjhxWsX5+frp161aKY5IyceJE+fr6WpagoKB0PR4AAAAAAAAATw6HK9qGh4erefPmyp49u7777jtLu7u7uyQpKirKKv727dvy8PBIcUxSRowYobCwMMsSEhKSLscCAAAAAAAA4MnjMA8ik6SIiAg1a9ZMd+/e1dq1a+Xj42NZlzVrVvn7++vcuXNW25w7d05FihRJcUxS3N3dLQVfAAAAAAAAAHgUDjPS9vbt22rWrJliY2O1Zs0aZc+ePVFM8+bNtXjxYiUkJEiSoqOjtWzZMjVv3jxVMQAAAAAAAADwuDjMSNvWrVvrwIEDmjp1qtatW2dpL1WqlEqVKiVJGjVqlKpUqaIOHTqoRYsW+v777+Xq6qohQ4ZY4lMSAwAAAAAAAACPi8OMtA0ICFDTpk21atUqff/995bl4MGDlphChQppz549KlGihDZs2KDatWtr586d8vPzS1UMAAAAAAAAADwuDjPS9p8PHXuQoKAgTZw48ZFjAAAAAAAAAOBxcJiRtgAAAAAAAADgCCjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAZAITJkyQh4eH1ZI/f/5EcQsWLFDZsmXl6+urKlWqaO3atTbIFgAAAI+Coi0AAACQCdy9e1eVKlXSrVu3LMupU6esYlatWqUePXpo0KBBOnr0qNq0aaNWrVrp4MGDNsoaAAAAaUHRFgAAAMgknJycrEbauru7W62fNGmS2rRpo759+yogIEDvvfeeSpQooalTp9ooYwAAAKQFRVsAAAAgk9i7d69y5cqlAgUKqFOnTjpx4oRlXUJCgrZu3ar69etbbdOwYUNt2bIlo1MFAADAI6BoCwAAAGQCuXPn1hdffKG//vpLq1at0t27d1W9enVdunRJknTr1i3duXNHefLksdouT548lpikxMTEKDw83GoBAACAbVG0BQAAADKBV155RT179lRAQIBKlSql7777Ts7Ozpo5c6ZVnLOzc6LXxphk+504caJ8fX0tS1BQ0GPJHwAAAClH0RYAAADIhLJmzaoSJUro+PHjkiRfX1+5u7vr6tWrVnFXrlxR3rx5k+1nxIgRCgsLsywhISGPNW8AAAA8HEVbAAAAIBOKi4vTyZMn5e/vL0lycXHRM888o40bN1rF/f7776pevXqy/bi7u8vHx8dqAQAAgG1RtAUAAAAygeeff147d+5UbGysLly4oL59++rWrVvq27evJWbw4MFavHixFi1apKioKH322Wfav3+/XnvtNRtmDgAAgNSiaAsAAABkAr1799bQoUOVPXt2lSlTRlevXtWWLVtUokQJS8yzzz6rjz/+WG+88Ya8vLz08ccf64cfflDlypVtmDkAAABSy9XWCQAAAAB4uEaNGqlRo0YyxsjJySnZuIEDB2rgwIFKSEhI9FAyAAAAZA7cxQEAAACZyIMKtv9EwRYAACDzYqQtAACwmULDV9g6BaSjMx+2tHUKAAAAgEPg43cAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCMUbQEAAAAAAADAjlC0BQAAAAAAAAA7QtEWAAAAAAAAAOwIRVsAAAAAAAAAsCOutk4AAAAAAAAAQOoVGr4iQ/d35sOWGbq/JxlFWwAAAABAqmR0kQAAgCeNQ02PcPjwYb3++uuqVauWVq9enWTMX3/9pT59+qhhw4YaMGCATp8+naYYAAAAAAAAAHgcHKZoO2PGDHXs2FFFihTRli1bdOXKlUQxhw8fVo0aNeTq6qpBgwbpxo0bqlatmkJDQ1MVAwAAAAAAAACPi8MUbbt06aLDhw9r0KBBycaMHz9e5cqV05dffqnWrVtrwYIF8vLy0pQpU1IVAwAAAAAAAACPi8MUbbNnz/7QmLVr16p169aW1y4uLmrVqpXWrFmTqhgAAAAAAAAAeFyemAeRRUZG6tq1awoMDLRqDwwM1Llz51Ick5SYmBjFxMRYXoeHh6dj5gAAAAAAAACeJA4z0vZh4uLiJEkeHh5W7VmzZlVsbGyKY5IyceJE+fr6WpagoKD0TB0AAAAAAADAE+SJKdp6e3vLzc1N169ft2q/fv26cubMmeKYpIwYMUJhYWGWJSQkJP0PAAAAAAAAAMAT4YmZHsHFxUVly5bV7t27rdp37NihihUrpjgmKe7u7nJ3d0//pAEAAAAAAAA8cZ6YkbaS1KdPHy1cuFBHjx6VJG3dulXr1q1Tnz59UhUDAAAAAAAAAI+Lw4y03bVrlwYNGmR5PX78eP3vf/9T69at9fbbb0uSBg4cqAMHDqhChQp66qmndOrUKb355ptq3769ZbuUxAAAAAAAAADA4+IwRdvg4GB9+OGHidr9/f0t/3Z2dtaMGTM0duxYhYSEqHDhwsqVK5dVfEpiAAAAAAAAAOBxcZiira+vr2rVqpWiWH9/f6tiblpjAAAAAAAAACC9PVFz2gIAAAAAAACAvaNoCwAAAAAAAAB2hKItAAAAAAAAANgRh5nTFgAAAACeVIWGr7B1CgAAIB0x0hYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOULQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwIxRtAQAAAAAAAMCOuNo6AQAAAABwRIWGr7B1CgAAIJNipC0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdoWgLAAAAAAAAAHaEoi0AAAAAAAAA2BGKtgAAAAAAAABgRyjaAgAAAAAAAIAdcbV1AgAAAAAAAADsX6HhKzJsX2c+bJlh+7JHjLQFAAAAAAAAADtC0RYAAAAAAAAA7AhFWwAAAAAAAACwI8xpCwAAAOCJkZFz8QEAAKQVI20BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCHPaAgAAAAAAALArGT0P/ZkPW2bo/h6GkbYAAAAAAAAAYEcYaZuMGzduKDQ0VAULFpSPj4+t0wEAAABS7P69bKFCheTt7W3rdB4oo0fRAAAAZAaMtP0XY4zeeOMNBQQEqF27dsqTJ4/Gjx9v67QAAACAhzLG6NVXX7W6l504caKt0wIAAEAqMdL2X2bOnKm5c+dq165dKlu2rDZu3KhGjRqpfPnyatOmja3TAwAAAJI1ffp0zZ8/X3v27FHp0qW1fv16NWnSROXLl1eLFi1snR4AAABSiJG2/zJz5kx17NhRZcuWlSTVrVtXDRo00KxZs2ycGQAAAPBgM2fOVJcuXVS6dGlJUoMGDVS3bl3uZQEAADIZirb/EB8fr/3796tKlSpW7VWrVtWePXtslBUAAADwcHFxcfrrr7+4lwUAAHAATI/wDxEREYqLi5Ofn59Ve65cuXT9+vVkt4uJiVFMTIzldVhYmCQpPDz88ST6LwkxURmyH2SMjLpu7uP6cSwZff1IXEOOhOsHjyqjrqH7+zHGZMj+MouwsDDFx8dzLwsAAJAG9nYvS9H2H9zc3CTJ6qZVku7cuWNZl5SJEydq7NixidqDgoLSN0E8EXyn2ToDZGZcP3gUXD94VBl9DUVERMjX1zdjd2rHuJcFAABIO3u7l6Vo+w+enp7KmTOnLly4YNV+4cIFFShQINntRowYoSFDhlheJyQk6MaNG/Lz85OTk9Njy/dJEh4erqCgIIWEhMjHx8fW6SCT4frBo+IawqPg+kl/xhhFREQoX758tk7Frvj6+srX1zdT3cvy85E2nLfU45ylDectbThvacN5Sz3OWdrY+ryl9F6Wou2/NGzYUCtWrNDbb78t6d5N68qVKx/4tF13d3e5u7tbtWXPnv1xpvnE8vHx4RcR0ozrB4+KawiPgusnfTHCNmkNGjTQihUrNHToUEn3ntmwcuVKtW/fPtlt7OFelp+PtOG8pR7nLG04b2nDeUsbzlvqcc7SxpbnLSX3sjyI7F/effdd7dq1S6+//rrWrFmjnj176saNG5YbXwAAAMBejRo1Slu3btXgwYO1Zs0ade/eXREREVYjaQEAAGD/KNr+S7ly5bR582ZdvXpVY8aMkYuLi/78888HfqUMAAAAsAcVKlTQpk2bdPHiRY0ePVru7u76888/FRgYaOvUAAAAkApMj5CESpUqacGCBbZOA//g7u5u+cMDSC2uHzwqriE8Cq4fZLQqVaro+++/t3UaKcLPR9pw3lKPc5Y2nLe04bylDect9ThnaZNZzpuTMcbYOgkAAAAAAAAAwD1MjwAAAAAAAAAAdoSiLQAAAAAAAADYEea0BQAAAPDYnT59Wjdv3lSlSpVStU1sbKyKFSsmZ+ekx5ukJCazio2N1YkTJ5Q9e/aHPkwuNDRUFy9eTHJdpUqV5OTkpOjoaB08eDDR+uLFi8vb2ztdcrYHN27cUGhoqAoWLCgfH58Hxl68eFGhoaFWba6urqpQocIj9ZsZnTlzRnfu3FGxYsXk4uLy0Pi7d+/q1KlTypkzp3Lnzp1o/eHDhxUVFWXVljt3bhUsWDDdcs4ocXFxOnHihLy9vRUUFJRu26Sl38zk1q1bCgkJUVBQkLJnz56ibS5cuKA7d+6oYMGCcnW1LllduXJF586ds2pzcnJK1ftKZhASEqKIiAgFBwfLzc3tgbHHjh1TRESEVVvOnDlVpEiRR+o3s4mPj9fx48eVNWtWFSpU6IGxV69e1dmzZ5NcV7ZsWbm7uys+Pl579+5NtL5IkSLKmTNneqScMgYAAAAAHpPFixebunXrGm9vb+Pi4pKibf7++29TsWJF4+vra/LmzWsKFixotm/fnuqYzGzx4sUmZ86cJigoyGTNmtU0bdrUhIWFJRv/6aefmkqVKlkt2bNnN76+vubu3bvGGGP++usvI8mUL1/eKm7Xrl0ZdViPVUJCgnn99ddNlixZTJEiRYy7u7sZN27cA7cZP3688fT0tDof9erVe+R+M5MLFy6YqlWrGm9vbxMQEGACAwPNpk2bko2/deuWGTRokPH19TXFihUzvr6+pnr16ubYsWNWceXLlzf58+e3Orfjx49/3IeT7n799VeTO3dukz9/fuPp6Wnq1atnrl+//sjbpKXfzGT48OHG3d3d8jMzfPjwB8Z/++23pnjx4iYgIMAULlzY+Pn5mTlz5ljFTJ061Xh4eFhdU1WrVn2ch5Ghrl27ZurVq2c8PT1N/vz5Te7cuc2vv/76wG3q1q1rAgICrM7JsGHDHrnfzGTTpk0mMDDQBAQEGG9vb1O1alVz4cKFZON/+OGHRO+XefLkMS4uLubatWvGGGNu3rxpJJmSJUtaxa1cuTKjDssYYwxFW2R6oaGhZvfu3SY+Pt7WqSATmjFjhjl37pyt00AmtWfPHjNw4EBbp4FMbN26dWbu3LkmISHB1qkAj83w4cPN+vXrzdy5c1NctK1evbpp3LixiY2NNQkJCaZfv34mMDDQREVFpSomszp79qzx8PAwU6dONcYYc/36dVOsWDHTu3fvFPdx9+5dExAQYF5++WVL2/2i7cWLF9M7ZbswY8YM4+3tbQ4cOGCMMWbDhg3G1dXVLF26NNltxo8f/9CiT1r6zUwaN25satWqZe7cuWOMMWbw4MEmV65cyX5I8Ndff5mpU6eayMhIY4wxkZGRpnnz5qZs2bJWceXLlzeTJk16vMk/ZpcuXTJeXl6WYnN4eLgpV66c6dSp0yNtk5Z+M5Nvv/3WZM2a1ezYscMYY8z27duNu7u7+fbbb5PdZtSoUVaF/7lz5xpnZ2dLH8bcK9qWLl368SVuY506dTIVKlQw4eHhxhhjxo4da7y8vMzly5eT3aZu3brmnXfeSfd+M4uwsDCTK1cu8+abbxpjjLlz547l/iA1KlasaNq0aWN5fb9ou3PnznTNN7Uo2iLTiomJMQMGDDAuLi5GkqlRo4blRgNIicWLFxtJpkiRIhRukWp79uwxefLkMQsXLrR1KsiEYmNjTcuWLU316tXNvHnzHjh6DnAUKS3aHjx40EgyGzdutLSFhoYaJycns2jRohTHZGYTJkwwuXLlsoyQNcaYL774wnh4eFgKZQ/zyy+/GElm7969lrb7Rdtdu3aZI0eOmJiYmPRO3aYqV66cqLDdpEkT07p162S3GT9+vKlSpYo5evSoOXPmTJIfoqWl38zizJkzRpJZsWKFpe369evG1dXVfP311ynu5+effzaSzI0bNyxt5cuXN6NHjzYHDhzItCNIp02bZry9vU10dLSlbd68ecbFxSXZY0rJNmnpNzOpV69eogJ0+/btTf369VPVj6+vr/nvf/9reT116lRTsmRJc+zYMXP69GmHGrh1/fp14+LiYlXYjoqKMl5eXubjjz9Odru6deuawYMHmwMHDpirV6+mW7+ZxVdffWXc3NzMzZs3LW333//OnDmToj727NljJJlly5ZZ2u4XbVesWGEOHjxosw+EHWvSJzxR+vTpo9DQUJ06dUpbtmzR/v37tXbtWlunhUymXr16cnFxUb169RQSEmLrdJBJ7N27V82aNdPnn3+ujh072jodZEKffvqpwsPDtXnzZnXv3t0h50YE0mrPnj2SpCpVqlja8uXLp/z581vWpSQmM9uzZ4+efvppq3lFq1atqujoaB0+fDhFfcyePVuVK1dOcm7WZs2aqUWLFvL29tbQoUMVFxeXXqnbTHx8vPbv3291TUj3ztvDromdO3eqRYsWqly5sgICAvTjjz+mS7+ZQVI/Szlz5lRwcHCqjm/fvn3Knj17onlLJ06cqE6dOikwMFC1a9fW8ePH0yXvjLJnzx7LHJf3Va1aVfHx8Tpw4ECat0lLv5nJnj17Hvln5vTp04qIiEg0P+mRI0fUrFkzVatWTXnz5tW8efPSI2Wb279/v+Lj463OW9asWVW2bNmHnrfPPvtMnTp1UsGCBVWlShWra+hR+s0M9uzZo+DgYKvfPVWrVpWkJOekTcrs2bMVGBio5s2bJ1rXrVs3tWvXTr6+vurTp0+i+YMfNx5Ehkxpx44d2rNnj/bu3St3d3cVLFhQZcqUUVxcnJYtW6bq1asrV65ctk4Tdq5EiRK6efOmNmzYoHr16qlevXrasGGDXFxcFBkZqeDgYFunCDu0b98+NWzYUFOnTlXHjh0VGRmpcePG6aeffpKLi4u6deumd955x+Em90f6Wrx4sXr06GEpyFy6dEnTp0/XhQsX1Lp1a7Vp08bGGQLJO3DggGJjY5Nd7+7urrJly6a5/+vXrytr1qzKmjWrVXuuXLl0/fr1FMfYk6ioqIcWW/PmzWt5ENH169eVL18+q/X3721TcnyXL1/WihUr9Pnnn1u1e3l56eeff1a7du0kSdu2bVOTJk3k7e2tMWPGpPBoMs6FCxd04cKFB8bcf4haRESE4uLi5OfnZ7X+YddExYoVdezYMRUrVkzGGH300Ufq1q2bihQposqVK6e5X1s6dOiQ7ty5k+x6Nzc3lS9fXtL/XU//frBOao5v3759mjRpkkaPHi0nJydL+yuvvKJu3brJ09NTN27cUOfOnfXss89q7969ypIlS2oPyyauX7+e5P/9/XVp3SYt/WYWd+/eVXh4eJLHFxYWpvj4+Ic+6C4uLk59+vRRmTJl1KpVK0t7qVKldPDgQZUuXVrSvQ/Be/XqpUKFCqlOnTrpfzAZ6P7/e2p/1/Ts2VNLly6Vr6+vIiIi1L17d7Vp00YHDx6Ul5dXmvvNLJL6Wbr/OiXHFx0dre+++06vvvqq1XXp6uqq2bNnq1evXnJ2dtbhw4fVpEkTDR48WLNmzUrfg3gAirbIlH777Te99dZblk8mt2zZoj179uiVV17R7du35erqqhUrVqh69eo2zhT2rGjRojp16pTy5s1rVbh1dXXVoEGDKNoiSbly5VLOnDk1e/ZsNW/eXK1atVL27Nk1bNgwHT58WO+//77279+vn3/+2dapwo5FR0frypUrku4VwBo1aqSKFSsqJiZGbdu21fjx4/Xuu+/aOEsgaSNGjNDly5eTXZ8vXz798ssvae7fzc1NcXFxMsZYFYDu3Llj+UAsJTH25OzZsxowYMADY7p166YhQ4ZIund8MTExVuvvF+FScnxff/213N3d1bVrV6v2QoUKWY1aq1atmgYMGKBvvvnGLou2S5cu1ezZsx8YM2PGDFWqVMlyXpI6bw86Zy1btrT828nJScOHD9fcuXP1/fffq3Llymnu15ZGjx6tM2fOJLs+e/bslm8o3j+G2NhYqw9BUnp8J06cUIsWLdShQwe99dZbVuv69etn+XfOnDn13//+V+XLl9eePXtUrVq11BySzbi5uSkqKsqq7WE/iynZJi39ZhYuLi5ycnJK8mfG2dlZzs4P/sJ3fHy8unfvrlOnTmnTpk1W56NJkyZWsa+99prmzp2r7777LtMXbR/0u8bb2zvZ7Xr37m35t7e3tz755BMVLFhQf/zxh5o1a5bmfjOLpN4vo6OjLeseZvHixQoLC1OfPn2s2r28vKzaSpUqpWHDhmnEiBH68ssvH3odpxeKtsiUXn/9dcXHx0uSDh8+rPbt2+vTTz9V//79FRERoVatWmngwIHat2+fbROFXXNzc1O+fPl04sQJlShRQt9++62eeeYZ5cqVy+oTXeCf8ufPbynyFy9eXM2aNdOCBQss6+vUqaOOHTtq5cqVatGihQ0zhT2rXr26vvzySw0ePFj9+/fXhx9+aLkxnDBhgsaOHav+/fsrT548Ns4USGzFihWPtf8CBQro7t27unLlivLmzStJMsbo0qVLKlCgQIpj7EnJkiW1a9euFMcXKFAg0cjc+yNOU3J8c+bM0XPPPZeiP8jz5cun8+fPpzi3jDRw4EANHDgwRbGenp7KmTNnopG5Fy5cSPU1ERAQYDkn6dlvRvnpp59SHHv/GC5cuKCnnnrK0n7x4kWrgnZSTp48qfr166t+/fqaO3eu1QcoSbk/etxer7ekFChQQOvXr7dqe9jPYkq2SUu/mYWTk5OCgoKS/JkJCgp64HUSHx+vHj166I8//tCGDRtUuHDhh+7Pnn+HpcY/fxYDAgIs7RcuXFDjxo1T3E9AQICcnJws5yS9+rVXBQoUSDRNZmp+lmbPnq3GjRsnmoYjKfny5VNUVJRu3LiRYd/sZk5bZEq+vr6Wr/AUKlRICxYs0EsvvSQnJyf5+Pho+PDhKZ7vC0+2EiVK6NChQ7pw4YJeeOEFvfPOO8qRIwdz3OKB7hdu/fz89M4771it69Chg4oWLZqqP87x5Bk8eLCuXbuml19+WadOnbL6JP+1117T3bt3H/qVYMCR/PXXX5b33Vq1ailLlixWxeHNmzfr1q1batiwYYpjMrOGDRtq165dlhH5krRs2TIVLFjQUli7c+eOdu3alWh+vT/++EPHjh2zGuV43/3RR/+0bt06lSxZMp2PwDYaNmxodU0kJCRo5cqVVtfEhQsXrAZ2/Puc3LhxQ/v377c6JynpN7OqWrWqPD09rY5vz549Cg0NtTq+Q4cO6ezZs5bXp06dUv369VWnTh3Nmzcv0dfdk7vWJGWq661hw4aJjn3ZsmXKnTu3ZRqYmJgY7dq1S2FhYSneJiUxmVnDhg21cuVKy2tjjJYvX251TV2+fFm7d++2vE5ISFDPnj21ceNG/f777ypatGiifv99XUVERGjnzp2Z6ppKTrly5ZQ7d26rn8XTp0/r8OHDVuft2LFjOn36tKR7154xxqqf9evXyxhjOScp7Tezatiwoc6fP281j++yZcvk5eWlZ555RtK9KTt27dqlGzduWG37999/6/fff0/V+2WePHkSTcfwWNnk8WdAKty9e9fMnDnTdO7c2QwaNMhcvHjxodtMmzbN1KxZMwOyQ2awZcsW06dPH/PCCy+YTZs2Wa0bNmyY6d+/vylWrJiZNGmSMebe06eDg4PNa6+9Zot0YWcuXbpkRo4caTp06GA+/vhjq6fUhoSEmLi4OKv4hIQEky9fPvP9999ndKqwQw96D/v222+Nk5OTcXFxsXq67dq1a03u3LmtnigNZGanT582O3fuNKNHjzYuLi5m586dZufOnSYiIsISU7BgQfPGG29YXr/zzjsmR44cZt68eWbx4sWmSJEipmPHjlb9piQms4qNjTUVKlQwNWvWNCtXrjRTp041rq6uZv78+ZaYvXv3Gknm999/t9q2Z8+epmzZskn2O3ToUPP666+bX375xfz222+md+/extXV1axYseJxHk6G2b9/v8maNat57bXXzOrVq80LL7xgcubMac6ePWuJGT16tPH19bW8rlq1qpkyZYpZv369+fHHH83TTz9tChYsaK5cuZKqfjOziRMnGm9vbzN79myzdOlSU7JkSdOsWTOrmNKlS5u+ffsaY+7dKxcoUMBUrlzZbNu2zfIzvXPnTnPnzh1jjDHbtm0zdevWNfPmzTMbN240U6dONdmzZzc9evTI8ON7FPHx8aZGjRqmcuXKZvny5ebzzz837u7u5n//+58l5sSJE1ZPnk/JNimJycyOHz9uvL29Tb9+/czq1avNiy++aLy9vc3x48ctMZMmTTIuLi6W13369DEeHh5mwYIFVtdUSEiIJaZu3brmo48+MuvWrTOLFi0y1apVMwEBAeb8+fMZenyPy/Tp042Hh4f54osvzPLly83TTz9tatSoYfX3R82aNU2HDh2MMcYcOXLEVKtWzcydO9ds3LjRfP755yZ37tymTZs2qe43M2vWrJkpXbq0Wbp0qZk9e7bx9PQ0H374oWX91atXjSSr91Bj7t1H5MmTx8TGxibqc9KkSaZXr15m0aJFZt26dWbw4MHGxcXFzJo167Efzz9RtIVdi46ONo0bNzZlypQxffv2NXny5DHFixe33Awk5dChQyZXrlyJinN4Mn388cfGz8/P9OrVy1SuXNm4uLiYrVu3WtYvWrTISLIUbO+7fPlykr+88WT566+/jL+/v2ndurXp2LGjcXZ2NkOGDHngNh9++KEpU6aMiYmJyaAsYa9S8h72/fffG09PT1OsWDHzww8/mJkzZ5qAgACzePFiG2YOpK/33nvPVKpUKdGyb98+S0zr1q3NlClTLK8TEhLM9OnTTaNGjUydOnXMhAkTEt3/pSQmM7t+/boZMmSIqVmzpmnZsmWi3wvHjh0zlSpVMrt27bK0xcbGmjp16pivvvoqyT7j4uLMzJkzTdu2bU2DBg3MwIEDzZEjRx7rcWS0Xbt2meeee87UqFHD9OzZ0xw9etRq/YwZM0y9evUsr69cuWJGjhxpGjZsaFq3bm3Gjx9vwsPDU91vZjdnzhzTpEkTU7t2bTNq1CgTGRlptb5Lly5m/PjxxhhjNm/enOTPdKVKlczJkyct22zfvt306dPH1K5d2zz33HPmhx9+yNBjSi9hYWHm7bffNrVq1TLNmzc3CxYssFofEhJiKlWqZDZv3pzibVIak5kdOHDAPP/886Z69erm+eefNwcOHLBaP3/+fFO1alXL64YNGyZ5TU2bNs0Sc+PGDTNq1CjTqFEj07JlSzNq1Chz8+bNjDqkDLFgwQLTvHlzU6tWLfP222+bsLAwq/V9+/Y1w4YNs7zev3+/6d+/v6lTp47p3Lmz+eqrr5Isxj6s38wsMjLSjBo1ytSuXds0adLEzJkzx2r9rVu3TKVKlczKlSut2tu1a2f+85//JNlnQkKC+f77703Hjh1NvXr1TN++fc2OHTse2zEkx8mYf42lBuzImDFjtH37di1btkyurq46evSoypYtq+nTp+vFF1+0ij106JB++eUXffrpp5o2bZo6d+5so6xhLw4cOKAGDRpo586dKly4sBISEixz1q5atcoSt23btkzzMARkHGOMypcvr5dfftny8JiPPvpII0eO1JUrV6y+FhMXF6f169dr7ty5OnbsmH7++ecUzYsEx5bS97DTp09r0qRJ+vPPP+Xv768RI0aoXr16tkscAAAAgM1RtIXdSkhIUIECBbRjxw7LpPWS1KBBA5UtW1Yff/yxVfzKlSt1+vRpdezYUf7+/hmdLuzQgAEDVKFCBaunNX/xxRd6//33FRoaasPMkBls2LBBY8aM0YYNGyxtN27ckJ+fnzZt2qTatWtb2o0xmjZtmgoWLKjWrVtn+qf+4tGl9j0MAAAAAP7J1dYJAMm5ffu2nn32Was/dqV7Dx67c+dOonie0o5/c3d313PPPWfVltz1A/zb3bt3E43oz5kzp7y9vRNdQ05OTho8eHBGpgc7l9r3MAAAAAD4J2dbJwAkx8fHR59++mmidg8PDyUkJFheHz9+XIcPH87I1JBJfPzxx8qePbtV27+vn9jYWKsnmwL3NWrUSC+88EKi9n9fQ7/++qtiYmIyMjVkAryHAQAAAHgUFG1hV9avX69Zs2Zp//79ycY4Oztb/uA9fvy4GjZs+MB4PDlOnDihOXPm6Ndff1VcXFySMf+8fmJjY9WpUyd99dVXGZgl7FVERIQWLFig7777TleuXEk27p/X0Ny5czVgwABdunQpo9KEnTLGaN26dZo9e7YOHDiQbBzvYQAAAABSgukRYBdiYmL07LPP6uDBg/L09NTRo0fVsmVLzZo1K9H8tE5OTjLGWP7Yff/999W1a1cbZQ578emnn+rdd99VcHCw/vrrL+XLl09z5sxR/fr1reLuXz/3C7ZZsmTRd999Z6OsYS/279+vFi1aKHfu3AoNDdXt27f17rvvasSIEXJ2tv588/41NHfuXI0ZM0br1q1TwYIFbZQ57EF0dLTatm2ro0ePKmvWrDp27Jhat26tWbNmKU+ePFaxvIcBAAAASAlG2sIujBs3Tk5OTjp16pSOHDmi9evXa//+/apWrZpOnTplFevk5KTTp09b/tjt0aOHjbKGvdizZ4/GjRun/fv3a9euXTp9+rRKlSqlJk2a6JtvvrGKdXJyUkJCgqVgu2DBArm68vnVk+z+9TBhwgTt27dPFy5c0HvvvafRo0erW7duio+Pt4p3cnLSvHnzLAXbokWL2ihz2IvRo0fLw8NDp06d0tGjR7V27Vrt2bNH1apV099//20Vy3sYAAAAgJSgaAu7sHz5cnXr1s3yxPX69etr+/btypo1q5o0aaKbN29aYl1dXfXHH3/wxy4sVqxYoVq1aqlQoUKSpMDAQC1btky9evVSr169tHr1akusq6uroqKiKNjC4tixYzpx4oS6d+8uSXJzc9PIkSO1aNEiLV68WK+99ppVvKurq/78808KtrC4/x52//dJw4YNtW3bNrm5ualJkyYKCwuzxPIeBgAAACAlKNrCLuTIkUNbtmyxasuXL59Wr16t27dv680337S0DxgwQN9++y1/7MIiR44c2rlzp2JjYy1tzs7OmjFjhlq1aqWePXsqKipKklS5cmVNmjSJgi0s7j+s7s8//7Rqb9u2rWbNmqXp06drzZo1lvYPP/xQv//+OwVbWCT1HpY/f36tXr1at27d0rBhwyztvIcBAJD+du7c+cA55QEgM3IyxhhbJwHMmjVLL7/8sv744w8988wzVuu+/fZb9erVS9euXZOvr6+NMoQ9u3jxop566ikNHjxY77//vtW6a9euqXDhwvr8888pkiBZjRo10rVr17Rt2zZ5eHhYrWvdurWcnJz0yy+/2Cg72Lv//e9/euONN/Tnn3+qUqVKVuu+/vpr9e/fX9evX5eXl5eNMgQAwD7t2rVLcXFxql69+iP107FjR/n7++uzzz5Lp8wAwPYYaQu70KdPH9WoUUNt2rTRsWPHrNZ16tRJ8fHxOn/+vI2yg70LCAjQxIkTNXHiRM2cOdNqXa5cudSgQYNE80oC//T555/r5MmT6tKli9WIbUnq3Lkz1w8eqF+/fqpSpYpat26tEydOWK3r3LmzYmNjFRoaaqPsAACwX5MnT9bYsWNtnQYA2CWKtshQsbGxevfddxUUFKTAwEBNmjRJ0r2vsi9atEh58uRR7dq19fvvv1u2OXDggHLkyKGnnnrKVmnDjixZskRPP/20cuTIoc6dO1vmO37jjTf02muv6aWXXtK4ceMsD4+KiYnR0aNHVaVKFVumDTtx+vRptWvXTjly5NDTTz+trVu3SpKKFy+uxYsXa/Xq1WrevLmuXLli2Wb//v1cP5B07/fJO++8o/z58yswMFD//e9/JUkuLi5asmSJcuTIodq1a2vjxo2Wbfbv369cuXKpcOHCtkobAIBMY/PmzTp69KgiIyO1detWbd26VTExMYnioqKitG7dOu3atUvR0dHJ9nfgwAGtWrUq0YeqR44c0fLly/XPLx6HhYVpyZIlunz5cvodEAA8AiZ0RIZJSEhQu3btFBMTo48++kjbt2/XsGHDVLBgQXXu3Fl+fn7auHGjunfvroYNG6pFixYqXLiwFi5cqC+//DLRV5bx5Jk5c6bGjBmjd999V/Hx8Ro9erReeukl/fjjj5Kkjz/+WIGBgXr33Xf1ww8/qEmTJtqwYYOqVq2qFi1a2Dh72NrJkydVq1YtvfDCC+rSpYs++eQTtW3bVkeOHJGfn5+aNGmidevW6bnnnlPx4sXVqVMn3bp1S3v27NEff/xh6/RhYwkJCWrdurWMMfrPf/6jP//8U0OHDlWhQoXUvn175cqVS5s3b9YLL7yg+vXrq2XLlipYsKAWLlyomTNnKkuWLLY+BAAA7N6IESOUJUsWnT17VoULF9aJEyfk5eWlLVu2WJ5DcOjQITVp0kTZsmVT3rx5deXKFWXLlk3+/v6Wfi5fvqxnn31WFy5cUPHixXXgwAFVq1ZN33//vdzd3eXl5aUePXpo1KhRGjRokCTppZde0vHjx7Vt2zYbHDkAJMactsgw3333naZNm6Y///zT8gCo5s2bKy4uTmvXrrWKXb16tZYtWyZnZ2f16tVLFStWtEXKsCM3btxQsWLFtHHjRpUuXVqSNHv2bPXr10/Xrl1Tzpw5LbGnTp3S119/rYsXL6p+/frq2rWrnJycbJU67ESbNm1Us2ZNvf3225KkS5cuqUiRIvrkk0/04osvWuIiIyM1b9487d69W0WKFNHAgQOVI0cOW6UNOzFv3jxNnz5df/zxh1xcXCRJjRs3louLi1atWmUVu2rVKq1YsUIuLi7q3bu3ypcvb4uUAQCwe88995xu3bpleS+tVauWTp48qT179ihfvny6c+eOSpUqpQEDBlju4erWravcuXPrxx9/lLOzsxYvXqwOHTrolVdescxp27RpUwUFBWnGjBlycXFRZGSkateurfbt2+vdd9+VJC1cuFDdu3fXjh07tGfPHr3yyivavXu3SpQoYZuTAQD/wkhbZJi5c+dq0qRJloKtdO8BP5MnT04U26RJEzVp0iQj04OdW7hwobp27Wop2EqyjHo7c+aMVdH2qaee0rhx42yRJuzU1atXtW/fPi1ZssTS5u/vr8qVK+v06dNWsZ6enho4cGAGZwh7N3fuXE2ePNlSsJXu/Q5K6oEnzZo1U7NmzTIyPQAAHEanTp2UL18+SVLWrFlVs2ZNHT16VJIUGhqqTZs2aceOHXJ2vjfbY/v27a0KrefPn9fq1av1ySef6Ndff5UxRsYYlSxZUmvXrrUUbTt16qSVK1eqU6dOunjxoqZMmULBFoBdoWiLDNO9e3fVqVPHqs3f319xcXE2ygiZSbVq1fTMM89YteXJk0fOzs5cQ3goLy8vTZ482XJzfx+/g5BSvXr1Uo0aNazauH4AAEh/fn5+Vq89PDx0+/ZtSdLZs2clSUWKFLGK+efzT+5/IL9q1Sq5ublZxZUpU8bq9cSJE5U/f37LaF4AsCc8iAwZpkePHom+ou7m5mY1+fvJkyc1cuTIjE4NmUD58uWTnCbD1dXVcg3FxcVp8ODBVg+RAqR7ozQ6d+6cqP3fv4Pmzp2rX3/9NSNTQybRs2fPh76HHT9+XO+8805GpwYAwBPj/rfrwsLCrNpv3bpl+be3t7ckacKECVqyZInV8u9vyIwcOVIFChTQkSNHtGLFisebPACkEkVb2JSTk5MSEhIk3SvYNmzYUMHBwTbOCpnJ/WsoLi5OXbp00dmzZ5l/FCn2z99Bc+fO1ZgxY/gdhBT75/Vz/PhxNWzYUMWLF7dxVgAAOK4iRYoob968+uWXXyxtFy5c0M6dOy2vy5YtK39/f82aNSvR9v8c3LFo0SJ99913WrJkid5991316dOHwR8A7ArTI8CmnJycZIyxFGzHjBmj3r172zotZCJOTk6KjY1Vly5dJEk//PBDoq9BAcm5/zvofsF23bp1Klq0qK3TQiZx//q5X7B9//331aNHD1unBQCAw8qSJYvGjRun119/XeHh4QoMDNS0adOUNWtWS4yrq6tmzpypjh076tatW2ratKlu3LihZcuWqVmzZnrrrbcUGhqq/v3764MPPlC5cuVUunRprV69Wn379tWyZctseIQA8H8o2sKmnJ2ddfv2bQq2SDNnZ2cNHDhQJUuWpGCLVHN2dtbKlSu1ePFiCrZINWdnZ4WHh1OwBQAgjapUqaLIyEjL6zp16iR6GFjFihUVHR1ted2/f3/lzp1bixYt0uXLlzV58mTt37/f6tt2rVq10v79+/XVV19pxYoVyp8/v0aNGqW6detKkn766Sd17dpVgwcPliS5uLjom2++0ZAhQ7Rz505VqVLlcR42AKSIk/nnZGxABvvjjz9Up04dzZ49m4It0sTf3181atSgYIs0efXVV7V06VL9/vvvFGyRahs2bFDDhg01d+5cCrYAAAAA0hVFW9jcvn37VKFCBVungUzq4MGDKl68OAVbpElYWJhu3LihwoUL2zoVZFK8hwEAAAB4HCjaAgAAAAAAAIAdcbZ1AgAAAAAAAACA/0PRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BAAAAAAAAwI5QtAUAAAAAAAAAO0LRFgAAAAAAAADsCEVbAAAAAAAAALAjFG0BwAFERUXp22+/1enTpxOtu3r1qr755htdvnzZBpkBAAAAAIDUcjLGGFsnAQB4dM8//7wOHTqkHTt2KEuWLJb2Nm3a6Pr169q0aZNcXFxsmCEAAAAAAEgJirYA4CBu3bqlsmXL6oUXXtDEiRMlSXPmzNHrr7+u/fv366mnnlJ8fLy2bdumS5cu6amnnlKFChWs+vjzzz91+vRpOTk5KVeuXKpYsaLy5MljFfPbb78pMDBQuXPn1rZt2+Tr66t69epl0FECAAAAAOD4KNoCgANZu3atmjdvro0bNyowMFDlypXTpEmT1L9/f507d06tWrVSfHy8ihcvrt27d6tkyZJasmSJPDw8JEnTp0/Xli1bJEkXLlzQzp079eWXX6pr166WfVSrVk1ubm46f/68KlSooFq1amno0KE2OV4AAAAAABwRRVsAcDCvvfaafv31VwUGBsrLy0srVqyQJNWuXVuVKlXStGnTJEnR0dGqVauW2rZtq/feey/JvhYtWqR+/fopNDRUWbNmlXSvaPv333/rwIEDyps3b4YcEwAAAAAATxKKtgDgYO7cuaMKFSro6tWrOnTokAICAnTq1CkVLVpUkydPVmBgoIwxMsZo0aJFunnzptavX2/Z/uLFizpw4ICuXbum2NhY9enTR/v371e5cuUk3Sva/j/27js8iur9+/gnZRMSSIGEEEoIhCJSRAkd6VKjINLBAiIqTVFsWEBFvoAKqIiACKjYEEGQohQFBVEEaQLSS0LoLQkhhSTn+YMn+2NJApu6G3y/rmsvnTP3nLlnNuXk5uyZ8PBwTZ061VGXCAAAAADALc3d0QkAAPKWl5eXGjVqpAMHDqh06dKSpCNHjkiS/vrrL1ksFpvYqlWrWrcnTJigMWPGKDw8XMHBwdbYM2fO2JwjODg4n68CAAAAAID/Loq2APAf4OfnJ0l6+eWXVbt27UxjLly4oJEjR2rVqlVq3bq1pKvF2i+//FLXfyjDxcUlfxMGAAAAAOA/zNXRCQAA8t+dd96p0qVL66OPPsqw79ixY5Kks2fPyhijypUrW/fNnz+/wHIEAAAAAABXMdMWAP4D3N3dNWfOHHXp0kWnT59W27ZtdfbsWS1dulSPPPKIBg8erLCwMN1xxx3q06ePHnnkEe3Zs0dfffWVo1MHAAAAAOA/h5m2AHALaty4sdq1a2fT1q5dO+3evVvh4eHavHmzUlJSNG3aNA0ePFiS5ObmpjVr1qhDhw7auHGjfHx8tHHjRvXt29e6Nq4ktW/fXrVq1SrQ6wEAAAAA4L/ExVy/UCEAAAAAAAAAwGGYaQsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAsEpJSVFSUpKj0wAAAEAhl5iYqNTU1EJ/vqSkJKWkpOR5vzeSmpqqxMREGWMK9LwAnAtFWwC4gbS0NEenkKX8KLAOHz5cAQEBedpnZigOAwAAZ3HlyhUlJiYqOTk5y5jk5GQlJibqypUrBZiZ46Tfk/RXdsfEly5dkpeXl955550c55CdImxenC8rpUqV0tChQ/O83xuZOXOmvLy8dPTo0Xw9D8VhwLlRtAWA62zevFl9+/ZV2bJl5enpqWLFiql69ep66qmntHv3bkenZ/XSSy+paNGijk4jRwpz7gAA4NbSvXt3eXl5qXjx4oqLi8uw//z58/Lx8ZGXl5f69+/vgAwLXvo98ff3l7+/vywWi0JCQvTMM88oJibmpse7uLjI09NT7u7uOTr/2bNn5eXlpcmTJ9sVn9vz/VfNnTtXXl5e2rt3r6NTAZAJirYAcI33339fDRo0kKurq5YvX67ExESdOXNG7733njZt2qQePXo4OkUAAADkMQ8PD125ckXz58/PsO+rr76Su7u7XFxcHJCZ47i5uVln2sbFxWnMmDGaMmWKOnXqdNNjixYtqsTERD333HMFkGnBnw8ACgJFWwD4/37++Wc988wzGjJkiObOnavatWvLzc1NXl5eatu2rdavX6+HH344y+Oz+tjYlStXMv24XfrHka51/bIBWX1U6cqVK9aPi1370bXM4nPzcaesjk1JScmQ+7XH3OjjbDfKPTv92nuvMuPMy14AAICC5+XlpYiICH366acZ9s2ZM0fdunWTq2vGP5+vXUbgZssrXLu0QlbjlhuNUfKiD3vzvZ63t7f69eun3r1767ffftP27dslZVzCICUlxXr+zMaD17dllqsxxjrGSx8bJiYm3nRd2ZyeLzO5HVemj1szW04j/W+A7I5HM8vpZssb3Oi+paSkWPNLX/4jPa/s9JtZrL33ryDXPAYKI4q2APD/vfXWW/L29tZbb72V6X43Nze98MILNm0rVqxQ48aN5eHhIU9PT91xxx2aO3euTUz37t11xx13ZOhv2rRp8vLy0rFjx6xt6WvKxsbG6sEHH1SxYsVUpEgRde3aVefPn7fGPfLII5oyZYpSU1OtH1vz9/fX4cOHJUkHDhxQ165dFRAQIE9PT1WqVEkvv/yyYmNj7boX586dU9euXVW0aFF5e3urZ8+eOnXqlHX/J598Ii8vL23cuDHDsTNmzJCXl5f+/vvvTPu+Ue7Z6Tf9Xt0s13Tbt29Xp06d5OPjI4vFogoVKmjcuHEUcAEAgCSpX79+WrdunQ4ePGht27Fjh7Zs2ZLlsggDBgywjmW8vb3l4+OjDh06WIua6erXr6/77rtPO3bsUKNGjeTp6akKFSro+++/lyRt3bpVTZo0UZEiRRQUFKTp06dnOFde9GFvvlmpUqWKJCk6OlrHjh2Tl5eXpk2bpi+//FJVqlSRp6enFi5cmOkas9fGz58/X5UrV5bFYlGlSpW0YMECa9zWrVtVqVIlSdJrr71mzXfYsGFZ5pWb86U7deqUevbsKW9vbxUtWlRdunTR2bNnMz3flStX9NZbb6ly5cry8PCQl5eX7r33Xu3Zs0fS1aJlRESEgoODbdalTUpKUv369VWlShWbsX1WkpOTNWzYMPn5+cnDw0OtW7fWrl27rPv/+OMPeXl5afbs2RmOXb9+vby8vPT5559n2vf48eM1ePBgSVK9evWs93nZsmXZ6jd9/d39+/ffMNd0x44dU79+/RQYGCiLxaKgoCA99dRTio+Pv+n9AP5zDADAxMfHG3d3d9O6dWu7j1m8eLFxdXU1AwYMMCdOnDDnz583b7zxhpFkJk2aZI3r3Lmzue222zIcP2XKFCPJREVFWduGDBliihYtavr162dWrVplEhMTzW+//Wb8/f3NI488YnP8iBEjjJubW6a5ValSxbRq1cocPnzYpKWlmSNHjpgJEyaYr7/++obXlH7+Hj16WM//559/mooVK5qaNWuahIQEY4wxcXFxxtfX1zz88MMZ+rjjjjtMzZo1b3ierHLPTr/25mqMMX/++afx8vIyDzzwgDlw4IBJSkoyy5YtMyVKlDDDhg27Ya4AAODW1rlzZ+Pn52euXLligoKCzGuvvWbdN3z4cFOhQgWTlpZm3NzcTN++fbPsJzU11ezbt8907tzZBAcHm7Nnz1r31a5d29SrV8888MADZt++fSY+Pt4899xzxmKxmDVr1piOHTuaPXv2mMuXL5uXXnrJSDLbtm2z6T8v+rA3386dO2c6VuvZs6eRZP755x8TFRVlJJmIiAgzePBgc/LkSbN//37z+++/m7i4OCPJjBs3znpsevz9999vnn76aXPq1Clz4cIF06dPH2OxWMzRo0etsWfOnDGSzDvvvJNl/tfK7fkSExNNzZo1TcWKFc0ff/xhEhMTzerVq03Xrl2Nr6+veeKJJ6yxaWlpJiIiwgQGBpoFCxaY+Ph4Ex0dbbp162ZKlChhIiMjjTHGnDp1ypQpU8bUq1fPJCUlGWOMGThwoLFYLOaPP/644fVMmzbNSDJdu3Y1n3/+uYmPjzf79u0zjRs3NgEBAebYsWPW2Nq1a5s6depk6KN3796mWLFiJi4uLsvzzJkzx0gy//77b4Z99vabnVyPHTtmSpcubRo0aGC2bNlikpOTzV9//WX92yUtLe2G9wX4r6FoCwDGmAMHDhhJpl+/fnYfc9ttt5lq1aqZ1NRUm/aIiAjj4+NjHchkt2gryXz77bc2selFzsTExAxt1zt58qSRZGbMmGH3tVx//rlz59q0r1692kgyn3zyibVt6NChpkiRIjYD/HXr1hlJZvLkyTc8z40Kzvb2m51c69WrZ6pUqWIdMKf78MMPjaurq82gHQAA/LekF22NMeaZZ54xoaGhJi0tzSQnJ5uSJUua119/3Rhjblq0TUlJMQkJCSYyMjLDGKV27drGYrFYC3rGXJ004O3tbXx9fc3hw4et7QkJCaZYsWLm6aeftuk/L/qwN9/0om1CQoJJSEgwJ0+eNB988IFxdXU1bdu2Ncb8X1H09ttvz1Bsu1ER9fpC4KlTp4yrq6sZM2aMtS0vi7b2nG/27NlGklm1apVN7BdffGEk2RRtFyxYYCRlmAwRHx9vgoKCzNChQ61t69atM+7u7ubJJ580n376qZFk3n///ZteT3oh9MUXX7Rpj4yMNBaLxQwfPtzaNn36dCPJ/PnnnzbX6OHhYQYMGHDD89yoaGtvv9nJtX///qZo0aLmxIkTNrFr1641ksxPP/10w3yB/xqWRwAAyfpgCWPn+kvHjx/X3r17dd9992VY3+yBBx5QXFycNm/enON82rdvb7Ndo0YNpaamKjIy8qbHlixZUiEhIRozZoxmzJhh1zHXi4iIsNlu3bq1vL299csvv1jbBg8erMTERM2ZM8faNm3aNHl4eOjBBx/M9jlz2u/Ncj1z5ow2bdqkjh07ytXVVVeuXLGuM9ykSROlpaVpw4YNOc4XAADcOvr376+jR49qzZo1Wrp0qc6ePatHHnkky/jDhw+rb9++CgoKksVikZ+fn3UJgWuXWZCk22+/XSEhIdZtb29vhYSEqFy5cqpQoYK1vUiRIgoNDbUue5WXfWQn32uXsqpSpYpmzJihl19+WQsXLrSJi4iIyNZD2q4f5wYFBSkwMDDT680L9pxvzZo18vT0VOvWrW1iM3vo2rJly+Tm5qaOHTta14VNTk6Wu7u76tWrp3Xr1llj7777bo0fP17Tp0/XwIED1aNHDz311FN25379ODckJER33nmnzZj8wQcflJ+fn6ZNm2ZtmzVrlpKTkzVgwAC7z3W97PZrT67Lli1Tw4YNFRgYaDMmr1evniwWi829A8CatgAgSSpdurQsFovNmlM3cuHCBUlSqVKlMuwLCgqSpJuuU5XVWqrp64tdK307Jibmprm5urpq9erVatSokZ5//nmFhoYqLCxML7/8suLi4m56vLu7u4oXL56hvWTJkjbXdPvtt6tVq1aaPn26jDE6c+aMvvvuO3Xu3FmBgYE3PU9WstOvPbmePn1akjR16lQVK1ZMPj4+8vHxka+vrxo2bChPT0+77isAALj11apVS3fddZc+/fRTzZkzRy1atLAphl7r8uXLat68ufbs2aMff/xRSUlJSkpKsq6Dev1DqIKDgzP0UaxYsSzbMxuf5KaP7Obr5uZmfThVbGysdu7cqTFjxqho0aI2cWXKlMlw7hvJLFcfH598G4/Zc77z58+rZMmSGYrPPj4+KlKkiE3b6dOnlZqaqqCgIOvY0tfXV76+vlq9enWGtVm7d+8uDw8PXblyRcOHD89W7ul/V1zfdu2YvGjRonrkkUc0b948nT9/Xmlpafr44491++23q1GjRtk637Wy2689uZ45c0Zr167NMCYvUaKEXF1d7X7+BvBfQdEWAHT1icEtW7bUn3/+aS3I3kh68fDkyZMZ9p04ccImxt/fP9NiaXR0dKZ9Z2emQlaqVq2qb7/9VhcuXNDff/+tvn376u2339agQYNuemxKSkqGgrMxRqdOnVLJkiVt2gcPHqyDBw9q5cqVefIv+tnt155c0//7wgsvWP/wuP71xBNP5DpnAABwa+jXr58WLFigH3/8McsHkEnSunXrFBUVpbfeekvh4eGyWCySlOWM0azGeNkZ++Wmj+zma6/0fuyVF2PdvD5fYGCgzpw5k+FTdxcvXlRiYqJNW8mSJVWkSBHFxcVlOq7cv3+/NTY5OVk9evRQiRIlVKVKFfXv31+XLl2yO/fMHq578uTJTMfk6Z9UW758uY4cOZJnY3J7+7Un18DAQEVERGQ5Jv/ggw9ynTNwK6FoCwD/36hRo3TlyhW98MILme5PTk7WqFGjJF2dYVuzZk0tWrRIqampNnHz58+Xv7+/6tWrJ0mqXLmyTp48aTOQSUtL09KlS3OVb7FixZSamqqUlJQsY9zc3FSnTh2NGTNGLVu2tHsZgEWLFtls//jjj0pMTNQ999xj0965c2eVLVtWH374oWbMmKGQkBC1adMm17lnp9+b5RoUFKSGDRtq4cKFSkpKumluAADgv61v375yc3OTn5+funbtmmWcm5ubpKuf/LnWp59+mp/p5Vhhyjd9Nm9Bjd3uueceJSUl6ccff7Rpv34pCOnqODUxMVHff//9TfsdMWKENm/erHnz5mnBggWKjIzUY489Znde149zDx8+rG3btmUYk992221q3bq1pk+frqlTp8piseihhx66af/FihWTlPV9zk6/9uTauXNn/frrr9ZJLgBujKItAPx/TZo00cyZM/XZZ5/p/vvv12+//aYLFy7o1KlTmj9/vurUqWMzOJs4caKOHDmiBx98UHv37tXRo0f13HPPadWqVRo/fry8vLwkXZ2tYbFYNHToUEVGRmrfvn0aMGCA7rzzzlzle8cdd0iSvvvuO8XHxysxMVHGGG3ZskUdO3bUkiVLFBkZqQsXLmjJkiX666+/7CqoFi1aVL/88osWLlyoM2fOaOXKlXr88ccVHh6u3r1728S6u7vr8ccf19KlS3XkyBH169cvwxq/2ck9u/3am+v06dN18uRJ3Xvvvfr99991/vx5HThwQN9++62aNm2qc+fO3TRnAADw3xAQEKDY2FidPXtW3t7eWcY1aNBApUuX1muvvabdu3crOjpa77zzjs6cOVOA2dqvMOXr5eWlKlWqaPXq1Tp58qQSExNvOFEht3r27Knw8HA9/vjjWrFihc6cOaPvv/9eK1eulJ+fn01s586d1bt3bz3xxBP66KOPdPjwYZ09e1Z//vmnnnvuOb355puSpHnz5unDDz/UuHHj1KxZM9WqVUsfffSRtd0ep06d0tSpU3Xy5Elt3rxZ3bp1U8mSJfXcc89liB08eLAOHDign376Sffee2+myxVcr1atWnJxcdGiRYsUGxurxMTEDEu42duvPbmOHTtWgYGBateunZYvX67Tp08rMjJSP/74o7p06cKatsB1KNoCwDX69++vHTt2qEyZMnriiSdUsWJF1alTR5MnT1bXrl21fPlya2zbtm21Zs0aXbx4UU2aNNGdd96pjRs36vvvv7f5uH25cuX0/fff6+DBg6pVq5Z69+6tnj17qmnTpvL09LT5yJbFYsmwbpZ0dWaEp6enTeGyS5cueuqpp/TCCy+oZMmS8vf31+HDh1WnTh09/fTTmj17tpo2baoqVaro9ddf16hRo246QEw//0cffaQffvhBNWrU0MMPP6wOHTpo5cqVmX787fHHH5fFYpGLi8sNP0J4raxyz0m/9uRau3Ztbdu2TZUqVdIjjzyiChUqKCIiQosXL9aECRMUEBBgV94AAODW4+Hhken463pFihSRh4eHddvHx0c//fST/Pz81KxZMzVu3FhnzpzR9OnT5enpaTMW8fT0tDk2J+257SM7+dpzT1xcXOTp6Zlh5m5W+24Un9k1zJo1S5cvX9Ztt90mf39/DRs2LFu5ZOd8FotFK1euVIcOHfTII4+oRo0a+v777zVjxgwVKVIkwxj4yy+/1Lvvvquvv/5adevWVY0aNfTcc8+pXLlyevbZZ3XgwAENHTpUDzzwgJ5//nnrcf369dPAgQP1yiuvaMuWLVlej7u7uzw9PTVmzBhFR0erXr16ateuncqXL6/169dnWjjt3LmzypUrJ0l2L41w2223aeLEifriiy9UunRp+fv7a9myZTnq980337xprkFBQdq8ebMiIiL0wgsvqEqVKmrevLk++ugj9e/fX3fffbddeQP/FS7G3kelAwCQifPnz6tMmTK6++67tXr16gLrd+jQofr000+ztS4YAAAAcKsKCwtTUlKSIiMjrcth5He/06dP16BBg3T48OEsH9oHIGeYaQsAyJVFixYpKSlJAwcOLBT9AgAAALeav/76S4cPH9ajjz6apwXb/OoXwM1l/JwAAAB2SE5O1vHjx/Xuu++qWrVqN3xQhzP0CwAAANxqUlJSdPHiRY0ePVq+vr43XEbCGfoFYD9m2gIAcqRNmzaqVauWSpQoofnz52e6Xlh+9pvV+r8AAADAf8WwYcMUGhqq6OhozZ8/364HkOVlv+nr7177nA4AeYM1bQEAAAAAAADAiTDTFgAAAAAAAACcCEVbAAAAAAAAAHAiPIgsH6Slpen48ePy8fFhXRcAAIA8ZoxRXFycypQpI1dX5iDkNcayAAAA+cfesSxF23xw/PhxhYSEODoNAACAW1pUVJTKlSvn6DRuOYxlAQAA8t/NxrIUbfOBj4+PpKs339fX18HZAAAA3FpiY2MVEhJiHXMhbzGWBQAAyD/2jmUp2uaD9I+R+fr6MtAFAADIJ3x0P38wlgUAAMh/NxvLsggYAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE2FNWwAAAAAAABSo1NRUXblyxdFpAHnOYrHIzc0t1/1QtAUAAAAAAECBMMbo5MmTunjxoqNTAfKNv7+/goODc/XgXIq2AAAAAAAAKBDpBdugoCB5e3vnqqgFOBtjjC5fvqzTp09LkkqXLp3jvijaAgAAAAAAIN+lpqZaC7YBAQGOTgfIF15eXpKk06dPKygoKMdLJfAgMgAAAAAAAOS79DVsvb29HZwJkL/Sv8Zzs24zRVsAAAAAAAAUGJZEwK0uL77GKdoCAAAAAAAAeSghIUGbN29WSkpKoT2Xvf1evHhR//77b56e25ldvHhRe/bsyffzsKYtAAAAAAAAkIf27t2revXq6cyZMwoMDCyU57K338cff1zh4eG6/fbb8+zcWdmzZ49SUlJUs2bNTPcbY7Rv3z65u7urUqVKNvsuXLiggwcPZnrc7bffrqJFi9oVY4xRs2bNtH79elWtWjV3F3QDFG0BAAAAAADgUDN/O1Rg5xrYLKzAzlUQvL29FR4eLovFUuDn3rhxo1avXq3PPvssX88za9YsffTRR9q3b59KlSqlAwcOZIjZunWrunXrpkuXLik5OVkhISFauHChKleuLEnatGmTXn75ZZtjTpw4oePHj2v//v2qXLmyXTHFixfXwIEDNXLkSC1YsCDfrpnlEQAAAAAAAIAspKSkaPPmzdq8ebO2b9+u2NjYbPdx8eJFbd261aYtMTFRmzdvtj6s6tKlS9q8ebOMMUpISNDu3bsVFxdnjT937px2796tpKQkm35CQkI0ffp0FS1aNEM/iYmJ2rNnj+Lj4/P8miTp/fffV+/eveXl5WX3uXNiy5Yt+vjjjzVixIhM9ycnJ+uBBx5Qs2bNdPLkSZ05c0blypVTz549rTFt27a1XnP6q0aNGmrcuLG1sGtPjCT169dPixcvVlRUVK6vLSvMtM1HNUevkKsnT0QEAOC/6Mj4CEenAAAAgDwQExOjJ598UtLV4uCBAwfUpUsXzZ49W56ennb1sXbtWj344IO6dOmSte3IkSOqV6+eoqKiVK5cOW3btk1NmzZV//79tXz5cnl7e+vUqVOaPXu2fvnlFy1ZskQeHh66fPmyli9frrp160rKuIxBej/Dhw/XV199JT8/Px07dkwzZ85U37598+yaUlJStHTpUs2dO9faZs+5JWnz5s037NvHx0e33XabdXvq1KmSpCVLlmQav2rVKh05ckSvv/66XFxc5O7urtdee02NGzfWli1bVKdOnQzHHD16VD///LNmzZqVZR5ZxVSpUkUhISH64YcfNGTIkBteS05RtAUAAAAAALeeDVNyfmzjYXmXBwq9gIAAmyLjmTNn1LJlS3344YdZzvzMDVdXV0VHR8vNzU1PPPGEevfurUGDBunYsWNydXVV3759NXLkSK1ateqG/Vy8eFHR0dFyd3fXxIkTNWTIEHXv3l0eHh55ck3//vuv4uLidMcdd2Tr3KmpqdaCcVbCw8M1Y8YMu/KQrs7EDQoKUmhoqLWtfv36cnFxybJoO2fOHBUrVkw9evTIst8bxdx11136448/8q1oy/IIAAAAAAAAwE3ExsZq165dOnr0qJo0aaJffvklX87z6quvys3NTZIUEREhY4xee+01ubq6Wtt27Nhx035Gjx4td/er8zW7d++umJgYHT161CYmN9d06tQpScr0IWU3Orebm1uGJQiuf2WnYCtdXToiICDAps3NzU3+/v46d+5chvi0tDTNmTNHffv2lbd35p+Sv1lMYGCgTp48ma08s4OZtgAAAAAAAEAWEhMT9fDDD+uHH35Q+fLl5evrq1OnTqlkyZL5cr7g4GDr/6cXC69vs2ed2Mz6ST8uL64pfRmFxMRE+fj42H1uKfvLI9yMxWLJsNZvem6ZPaBt9erVioyM1GOPPZZlnzeLSUxMVJEiRezOMbso2gIAAABOwBijlStXauvWrercubNuv/32DDEHDhzQunXrlJqaqnr16ql27doZYpKSkrR06VJFRkaqatWq6tChg3VmDgAAyL733ntP27dv17Fjx6yzSl977bUs11fNjIuLS4a29AeQOUJeXFNYWJgk6dixY9kq9ubH8gjly5fXqVOnlJqaap2lfOHCBSUkJKh8+fIZ4mfNmqXw8PBMl02wN+bYsWOqWbOm3TlmF0VbAAAAwMFWrlypIUOGKDQ0VD///LMqVKiQoWj70EMPadOmTWrcuLEk6ZlnntHDDz9sfTCHdHX9uBYtWig5OVlNmzbVe++9p6pVq2r58uWZzjIBAAA3d+DAAdWtW9dmGYCbrSd7vaCgIMXHx+vChQsqXry4pJvPNs1PeXFNZcuWVaVKlfTHH3/orrvusvu49OUR8lKrVq0UHx+vX3/9Va1atZJ09aFl7u7uatasmU3suXPntHjxYr3//vtZ9nezmNTUVG3evFlPPfVU3l3EdSjaAgAAAA5WvHhx/fTTTwoNDc2yuNq3b199/vnn1pk6AwYM0N13362ePXta/xgZO3asYmNjtX37dvn4+OjYsWOqXr26Zs6cqcGDBxfY9QAAcCtp3ry5Bg8erM8++0zlypXTnDlztGXLFlWvXt3uPu666y6FhIToySef1ODBg7Vv3z698cYb+Zj1jeXFNUlSv3799N133+X7OGPv3r2Ki4vT8ePHlZSUZC361q5dWxaLRTVq1FDfvn01YMAAvfvuu0pMTNSzzz6r4cOHKygoyKavuXPnyt3dXb17987yfDeLWbFihby8vNSxY8e8u8jrULQFAAAAHKxevXqSpJSUlCxj2rdvb7PdqFEjubm56eDBg9ai7fz589WnTx/runLlypVTx44dNX/+fIq2AADk0EMPPaS4uDh98cUXunLlipo2baoPPvhA69aty/IYb29vhYeHW/8xtkiRIlq1apXGjBmj0aNHq0aNGpo3b56efvppeXh4SLq6jmt4eLjNska+vr4KDw+36bt48eI2H9m//lyZ9WOxWBQeHm5dX9aea7q+38wMGjRIkydP1u7du1W9enW7zp0T77zzjrZt2yZJKlWqlHV5heXLl1uLsrNnz9akSZM0ZcoUubu76/XXX890/LNt2zY9/fTT8vX1zfJ8N4uZNm2annnmGet7lx9cjDEm33r/j4qNjZWfn59Chn8rV8+cf0ECAIDC68j4CEencMtKH2vFxMTccLBdGKWkpMhisejrr79Wr169bhi7YMECdevWTf/8849q1qyphIQEeXt769NPP9UjjzxijRs9erRmzJiR5dONk5KSbB7cERsbq5CQkFvy/gL4j9kwJefHNh6Wd3nAKjExUYcPH1bFihXz9QFOKFizZs1SVFSUXn/9dUenUiAOHTqkJ598UkuWLLE+jO16N/pat3csy0xbAAAAoJA5ePCgnnzySQ0aNMj6AIxLly5Jkvz8/Gxi/f39rfsyM27cOId+PBMAABRuAwYMcHQKBSosLEwrV67M9/PwGFkAAACgEImMjNQ999yjZs2a6YMPPrC2p3/kMDY21iY+JiZGRYsWzbK/kSNHKiYmxvqKiorKn8QBAABgN2baAgAAAIVEVFSUWrRooTvvvFPffPON3N3/bzhftGhRlS1bVgcOHLA55sCBA6patWqWfXp6emb50T4AAAA4BjNtAQAAgELg2LFjatGihWrXrq1vv/0204eCdOnSRd9++60SExMlSWfOnNGyZcv0wAMPFHS6AAAAyAVm2gIAAAAOdvjwYc2bN09paWmSpB9++EFHjhzRXXfdpXbt2kmS7rnnHp05c0Z16tTRxIkTrcc2a9ZMjRs3liSNGjVKP/30k5o1a6aWLVvqhx9+0G233aZBgwYV/EUBAAAgxyjaAgAAAA525coVXbx4UZL04osvSpIuXryohIQEa0zXrl2Vmpqq+Ph4m2OTkpKs/1+yZElt2bJF8+bNU2RkpF599VV1795dHh4e+X8RAAAAyDMUbQEAAAAHq1q1qsaPH3/DmLFjx9rVl4+Pjx577LG8SAsA/rs2TMnd8Y2H5U0eAP6zWNMWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAIAsbNy4UYGBgTbryN/Mzp07FRgYqPPnz+djZvl7Lnv7HTt2rEaMGJGn53aUQ4cO6fbbb1dsbKyjU2FNWwAAAAAAADhYbtcRzo5srjl85coVnTt3TsYYu49JSUnRuXPnlJaWlt3ssi2/zmVPv8eOHdPbb7+tnTt35um5r7dp0ya9+eab2rlzp7y9vdW3b1+99NJLcnW1nY+6bNkyTZo0Sfv27VPNmjU1fvx41a5d27o/ODhYKSkpNseMHTtWTzzxhCQpLCxMderU0dixYzVhwoR8vaabYaYtAAAAAAAAUEjVqlVLZ86cUUBAQIGf+8MPP1TLli0VEhKSb+fYvXu3mjdvrmrVqunnn3/WjBkzNHfuXL300ks2cbNnz1bXrl3VrVs3bdiwQS+//LLGjRtnE3P27Fl99NFH2rNnj/X1yCOP2MQ8/vjjmjFjhuLj4/PtmuxB0RYAAAAAAAC4iRUrVqhFixYqX768OnTooH379tl97I8//qgKFSrYtB04cECBgYE6ceKEpP9bhuHrr79Wq1atVL58ebVt21Z79+7VggUL1KBBA5UvX149e/bU2bNnrf38+++/qlatmi5cuGDTz48//qjWrVurQoUKatu2rXbt2mU95vTp0woMDFRgYKBKly6txo0ba+7cudm+J19++aW6dOli3bbn3Nn11VdfqVSpUnrnnXcUFhamu+++W+PGjdP777+vixcvSpJiY2M1fPhwjRo1SoMGDVJISIiaNm2qr7/+OkN/vr6+1msPDAxUkSJFbPY3a9ZMbm5uWrJkSY5zzgsUbQEAAAAAAICbeOaZZ/T6669r5cqV8vHxUdu2be1e5zYpKcmm0Cr93/IDqampkv5vGYZJkyZp/PjxWr16teLj49WyZUtNmjRJU6dO1YoVK3TgwAE9//zzGfpJX8YgvZ9XX31Vb731ln755Rf5+/ura9eu1iUeSpYsaZ1p+vfff+vZZ5/VsGHD9MMPP9h9Pw4fPqxjx46pXr161jZ7zp2ammpTNM3s1blzZ2ufiYmJ8vLysjm3t7e3kpOT9ccff0i6WlCPi4vLMGvWxcUlQ96DBg1SuXLl1LRpU82ePTvDshcuLi6qV6+e1q5da/e9yA+saQsAAAAAAADcxMSJE9WiRQtJVz+KX65cOc2bN08PP/xwnp5n+vTpCg8PlyQ99dRT6tWrl1atWqUaNWpIkgYPHqwxY8bctJ8ZM2aobt26kqQ33nhD1atXV3R0tMqVKycXFxcFBgZaY7t166bNmzdr7ty56tSpk115Hj16VNLVdWKzc243Nzft2bPnhn1bLBbr/0dERGjSpEmaM2eO+vXrp/Pnz1uXPTh+/Lgkad++fQoICNCff/6pMWPGKCYmRnfccYdef/113XXXXda+OnbsqBEjRqh8+fL6+eefNXz4cB09elRvvPGGzflLly6tyMhIu+5DfmGmLQAAAAAAAHATjRo1sv5/sWLFVLt2bf3zzz95fp7bb7/d+v/p69Re33bu3Lmb9lO9enXr/5csWVKSbI6bPXu27r77boWEhCgwMFBTpkyxFmLtkT5D2M3NLdvnvtlMWz8/P2tsy5YtNXPmTI0ePVre3t6qVKmS7r//frm6utrM3o2JidHEiRM1a9YsrVq1SiVLllSLFi1siq8//PCDmjdvrooVK+qxxx7TqFGjNH78eF25csUmfzc3twwPLCtoFG0BAAAAAACAm3B3d8+wnZycnOP+rv9YfjpX14zluuvbsjr2Zv2kH/fNN99Yl0T49ddftWfPHg0fPjxb11OmTBlJ0qlTp7J17uwujyBJAwYMUGRkpM6dO6dz586pbdu2SktLU2hoqCQpKChIKSkpmjx5ssLDw1W5cmVNnz5daWlpWrx4cZbXUK9ePSUnJ+vw4cM27adPn1bZsmXtvhf5geURAAAAAAAAgJvYvn27WrduLenq2q27du1S165d7TrWz89PCQkJunLlivWj/0eOHMmvVG/ql19+UUREhHr27Glty86D1SSpWrVqKlGihP7++29VrVrV7uOyuzzCtby9vSVJ8+fPl7+/v5o0aSLp/2ZBX7v2rbu7uywWS4ZZtNfav3+/XFxcrDOa0/39998ZlkwoaMy0BQAAAAAAAG7ipZde0okTJ5ScnKxXX31VSUlJ6tOnj13H1qpVS56enpo+fbokKSoqSqNGjcrPdG8oNDRUf/75p06cOKHU1FR98cUXWrhwYbb6cHFxUdeuXbV8+fJsnz87yyNI0rPPPqtz587JGKPFixdrwoQJmjBhgrWIW7t2bbVq1UpvvPGG4uLilJqaqvHjx+vy5cvq2LGjpKuF3k8//VSxsbEyxmj9+vUaNWqUunXrZlO03b59u86dO2f32r75hZm2AAAAAAAAhcjM3w5lK35gs7B8yuS/pUePHqpTp47Onz+vMmXKaMGCBfL397fr2MDAQH3yySd6+umn9eKLL6p06dJ67LHHtHnz5vxNOgtPP/20Nm7cqNDQULm5ualmzZrq27evduzYke1+GjZsqPPnz6tEiRL5lO3VNXJr1aqlixcvKjg4WFOnTlW/fv1sYubNm6fHHntMgYGBcnNzU1hYmBYtWqRq1apJuro27uuvv64RI0bo8uXL8vf3V79+/fTaa6/Z9DN79mz16dNHQUFB+XY99nAx9iyCgWyJjY2Vn5+fQoZ/K1dPb0enAwAAHODI+AhHp3DLSh9rxcTEyNfX19Hp3HK4vwBuGRumOO7cjYfla/eFtWibmJiow4cPq2LFiipSpIij07FbSkqKLl68qMDAQElSfHy8ihYtesNjUlNTdeHCBQUEBMjFxcVmX0JCgry8vKwxJUqUkKura4bzSFeXYYiJibFpS05OVlxcnHV26PXnyqwfY4zOnTun4sWL2zw4LDk5WampqfLy8lJCQoKSkpKshegbXcO1Bg0aJF9fX02YMCFb584Je+59cnKyjDHy9PTMMib9PbhedHS07rrrLm3bts26Zm9O3Ohr3d6xFjNtAQAAAAAAgCy4u7vbFCFvVjSUrq7beu0x10ovFl4fc/15pKtru17f5uHhYfNxfnv6cXFxyTQfDw8Pm7yuLWTe6Bqu9cEHHyg+Pj7b584Je+79tdeUlcwKttLVGdEHDhxwin+4pmgLAAAAAAAAIEcsFovdy0Q4O09PzxvO0C1IPIgMAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAQIExxjg6BSBf5cXXOEVbAAAAAAAA5DuLxSJJunz5soMzAfJX+td4+td8TrjnVTIAAAAAAABAVtzc3OTv76/Tp09Lkry9veXi4uLgrIC8Y4zR5cuXdfr0afn7+8vNzS3HfVG0BQAAAAAAQIEIDg6WJGvhFrgV+fv7W7/Wc4qiLQAAAAAAAAqEi4uLSpcuraCgIF25csXR6QB5zmKx5GqGbTqKtgAAAAAAAChQbm5ueVLYAm5VPIgMAAAAAAAAAJwIM20BAAAAAIDz2TDF0RncMmb+dihb8QObheVTJgDsxUxbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCA8iAwAAAAAAgFV2H1yWHTzkDLAPM20BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIm4OzoBAAAAAABwC9owxdEZAEChxUxbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCA8iAwAAAAAAcKCZvx1ydAoAnAwzbQEAAAAHS0lJ0cKFC9WmTRsFBgbq+++/zzRu8eLFatSokcqWLauWLVvq999/z1EMAAAAnBtFWwAAAMDBJk+erC+++ELPPPOMzp07p6SkpAwxa9asUbdu3dSrVy+tXbtW9evXV5s2bbR3795sxQAAAMD5UbQFAAAAHOy5557TwoUL1bZt2yxjxo8fr44dO+rpp59WlSpVNGHCBIWFhWnSpEnZigEAAIDzo2gLAAAAOJiLi8sN9xtjtH79et1zzz027W3atNG6devsjgEAAEDhwIPIAAAAACd38eJFXb58WaVKlbJpL1WqlE6cOGF3TGaSkpJslmOIjY3Nw8wBAACQE8y0BQAAAJycMUaS5ObmZtPu7u6utLQ0u2MyM27cOPn5+VlfISEheZk6AAAAciBbRdsrV64oJSUlWyfIyTE5lV/nKshrAAAAt64tW7bovvvuU/ny5VWlShW9+uqrunLlSoa4n376Sa1atVKZMmXUsmVL/fnnnzb7Q0ND5e/vb/N69913C+oy4AB+fn7y8PDQmTNnbNrPnDmjkiVL2h2TmZEjRyomJsb6ioqKyvsLAAAAQLZkq2jboUMHvfrqq9k6Qffu3TV8+PBsHZNT+XWugrwGAABwazpw4ICaN2+usLAwrVu3Tt98842WL1+uoUOH2sR99dVX6ty5s7p06aLNmzfr7bff1sSJE21iYmJi9MEHH+jIkSPW17BhwwryclDA3NzcFB4ervXr19u0//rrr2rQoIHdMZnx9PSUr6+vzQsAAACOdUstj+Dh4SGLxeLoNAAAADKYP3++vL29NXnyZIWGhio8PFzvvPOOPvnkE+t6owkJCRo2bJief/55DRs2TGXKlFG9evU0f/78DP15e3vbzLT19PQs6EtCAXvqqaf03XffacWKFUpNTdWnn36qzZs3a8iQIdmKAQAAgPMr8AeRJScny83NzWatrStXrsjFxUXu7u6ZbhtjbJ6oe/12ui+//NKm/fp+MpOSkmJd+qBIkSK5uzgAAIAsJCUlycPDQ66u//dv5kWKFFFaWprWr1+v7t27a82aNTp//rwefvjhm/b3zDPPaOjQoQoNDdWDDz6owYMHZ1jLFIXHL7/8oh49eli3H3/8cQ0dOlT9+vWzLn3Rq1cvHTt2TL1799alS5cUEBCgTz/9VI0bN7YeZ08MAAAAnF+2i7YnT55U586d9euvvyolJUUDBgzQpEmT7P4joX79+nrwwQf13HPPWdseeughFStWTJ988omkq8swBAUFKSYmRhs2bFBSUpIeffRRjRgxQoMGDdKaNWtUrFgxjRw50qaf7t27q1y5cvrwww+t/YSEhOjSpUv6+eeflZKSol69eumjjz6yFnKfeeYZzZw58+rNcHdXgwYNNGXKFFWvXj27twYAACBLERERGjNmjN577z0NGzZMFy9e1JtvvilJio6OliTt379fRYoU0b///qvu3bvr3LlzqlGjhkaPHm1TdGvbtq2GDBmisLAw/fbbb3rqqad06NAhTZ482SHXhtxr2rSp9uzZk6Hdy8vLZvu5557TiBEjFBcXl+UyBvbEAADy18zfDjk6BQCFXLaXR/jss8/UunVrnThxQj///LPmzZuXYZ21vDB//nz1799fp0+f1pIlSzRt2jTVr19fgwYN0qVLl/TZZ5/phRde0M6dO2/Yz9y5c9W1a1edPHlS69ev1/z58zV79mzr/ilTpigxMVGJiYmKjo5WjRo11KVLl0wfCpKVpKQkxcbG2rwAAACuVa9ePX311VeaOnWqvL29FRYWpvbt28vb29sak5qaquTkZI0dO1Zz5szR77//rurVq+uee+7R3r17rXHffvutmjdvrpCQEPXt21fjxo3TBx98oLi4OEdcGvKAxWJRYGBghlfRokUzxLq4uNy0GGtPDAAAAJxXtou2DRs21FNPPSUvLy81aNBAL774Yr7M6rj//vvVrVs3WSwWtW7dWjVq1FCHDh3UuXNnWSwW3XvvvapQoUKGpylfr0uXLurVq5c8PDx0xx13KCIiQhs2bMg01svLS2+99ZYOHjyo3bt3253ruHHj5OfnZ32FhIRk61oBAMB/Q8+ePbV//37FxcXp4sWL6tq1qy5fvqwKFSpIkkqVKqW0tDSNHz9ederUUWhoqCZNmiQfHx8tWLAgy37r1q2rtLQ07d+/v4CuBAAAAEB+ynbRtk6dOjbb4eHhOnnypGJiYvIsKUkKCwuz2fb19VXFihUztF28eDFb/fj7+9scs337drVt21a+vr7y9vZWUFCQUlNTFRUVZXeuI0eOVExMjPWVnWMBAMB/j4eHh1xcXPTtt9+qaNGiatmypSSpUaNGkmTzUDEXFxd5eHhY1+DPTPrH6kuWLJmPWQMAAAAoKNku2hpjbLbT0tKuduRqX1eZPUAsvY+bxWXWlpPzpTPGKCIiQrfffrv279+vK1eu6PLly7JYLDf8w+h6np6e8vX1tXkBAABc79lnn9Xx48dljNGyZcv05ptvauzYsfLz85N09R+b77//fr355ps6d+6cUlNTNXnyZJ06dUqdOnWSJC1evFhTp07VuXPnlJaWpg0bNmjkyJG69957+bQPAAAAcIvIdtF248aNGbZDQkLk4+Nj1/HFixfXuXPnbNquXaOtIEVHRys6OlrDhg1TqVKl5OLios2bN2drPVsAAAB71a9fX40bN5aXl5eGDRumiRMn6umnn7aJ+fTTTxUYGKiyZcuqWLFimjVrlhYuXKg777xTktSiRQsdOXJENWrUkJeXl7p3766ePXvq66+/dsAVAQAAAMgP2S7abtmyRW+++aZOnTqlFStW6O2339aIESPsPr558+b68ssvtW3bNp06dUqjRo3Sjh07sptGnggODlZAQICmTZumc+fOadOmTRowYIBDcgEAALe+Xr166ciRI4qJidGhQ4c0cODADDF+fn768ssvdfnyZcXGxmrnzp269957bfa/8847OnnypOLi4hQdHa23335bxYoVK8hLAQAAAJCPslW09fDw0ODBg3XgwAHVq1dP/fv31+DBgzVs2LAbHmOxWKzbzz//vNq1a6cOHTqoSZMmSkhIUPfu3W1irj9GuroEgbu7+w3brj8us34sFos8PDwkSe7u7lqwYIHWrl2rsLAwPfTQQxo8eLACAgLk5uZ2w34AAABy6to1a7Pi6up60/FH+pgGAAAAwK3FxVy/SC1yLTY2Vn5+fgoZ/q1cPb0dnQ4AAHCAI+MjHJ3CLSt9rBUTE8OzBPIB9xdAntkwxdEZOMzMFMYBWRnYLOzmQcAtzN6xVraXRwAAAAAAAAAA5B+KtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIATcXd0AgAAAAAAwEltmOLoDADgP4mZtgAAAAAAAADgRJhpCwAAAAAAgAIx87dD2Yof2CwsnzIBnBszbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAibg7OgEAAAAA9rl8+bLWr1+v06dPq0yZMmratKksFotNTFpamtatW6fIyEhVrVpVDRo0cFC2AAAAyCmKtgAAAEAh8NdffykiIkLly5fX7bffrk2bNik5OVm//PKLKlasKEmKj49Xx44ddejQIdWvX1+//vqrWrdura+++kpubm4OvgIAAADYi+URAAAAgEJg1KhRuvPOO/X333/riy++0M6dO1WkSBFNmDDBGjNhwgQdPHhQW7du1YIFC/THH39oyZIl+vzzzx2YOQAAALKLoi0AAABQCBhjVKpUKeu2xWJRiRIlbGK+/vpr9e7dW4GBgZKkKlWqqH379vr6668LNFcAAADkDssjAAAAAIXAO++8o969e2vYsGGqVq2a/vjjD6WmpurVV1+VJCUmJurgwYOqXr26zXE1atTQrFmzsuw3KSlJSUlJ1u3Y2Nj8uQAAAADYjaItAAAAUAj4+vqqXLlyWrdunc6fP6+///5bd955p7y9vSVJcXFxMsaoePHiNseVKFHihoXYcePG6Y033sjX3AE40IYpjs4AAJADLI8AAAAAFALdu3eXr6+vtm7dqi+//FI7d+7U0aNHNWjQIEmSl5eXpKvF22vFxcVZ92Vm5MiRiomJsb6ioqLy7yIAAABgF4q2AAAAgJNLTk7W5s2bde+998rFxUWS5O7urvbt2+v333+XJBUrVkylSpXSkSNHbI49fPiwKleunGXfnp6e8vX1tXkBAADAsSjaAgAAAE7Ow8NDpUuX1o4dO2zat2/frtDQUOv2fffdp++++04pKSmSrq5Pu3TpUt13330Fmi8AAAByhzVtAQAAgEJg7NixeuKJJ3Tp0iXVrFlTGzZs0NKlS7Vs2TJrzOjRo1W/fn117NhRbdq00fz58xUUFKRhw4Y5MHMAAABkFzNtAQAAgBy6fimC/NS/f39t2rRJ5cqV0759+1S7dm3t2bNHbdq0scaUK1dO27ZtU6tWrXT48GH17dtXf/75p3x8fAosTwAAAOQeM20BAACAHGrUqJHCwsL06KOPqmfPnipWrFi+nq927dqqXbv2DWOCgoL00ksv5WseAAAAyF/MtAUAAABy6KefflJ4eLhefPFFBQcHq3///lq/fr2j0wIAAEAhR9EWAAAAyKHatWvrgw8+0PHjxzV79mydOHFCzZs312233abx48frxIkTjk4RAAAAhRBFWwAAACCXPDw81KNHD/3444/69NNPFR0drZEjRyokJER9+vRRVFSUo1MEAABAIULRFgAAAMilqKgojRkzRpUrV9YTTzyhrl27at26dVq9erUuXLigVq1aOTpFAAAAFCI8iAwAAADIoQULFujjjz/W6tWrVbt2bY0YMUJ9+/aVn5+fNaZp06YqXry4zp49q8DAQAdmCwAAgMKCoi0AAACQQy+//LJatWqlv/76S+Hh4ZnGuLm5acKECfL29i7g7AAAAFBYUbQFAAAAcuiff/6Rh4fHTeMGDRpUANkAAADgVsGatgAAAEAOWSwWrV27VsYYm/aNGzfq0qVLDsoKAAAAhR1FWwAAACCHpkyZou+//14uLi427bt379bw4cMdkxQAAAAKPYq2AAAAQA7NmTNHjz76aIb2Pn36aMGCBUpJSXFAVgAAACjsKNoCAAAAOXTp0iVduXIlQ7sxRnFxcUpISHBAVgAAACjsKNoCAAAAOdSkSRNNmDAhw4za//3vf6pevbp8fHwclBkAAAAKM3dHJwAAAAAUVqNHj1bDhg1VrVo1tWjRQp6entq4caN27typ5cuXOzo9AAAKvZm/HcpW/MBmYfmUCVCwmGkLAAAA5FDFihW1detWde3aVQcOHND27dtVt25dbd68Wa1atXJ0egAAACikmGkLAAAA5EKZMmU0YcIER6cBAACAWwhFWwAAACCX4uPjdfbsWRljbNpDQkLk5ubmoKwAAABQWFG0BQAAAHLo4MGD6tu3rzZu3Jjp/hMnTig4OLiAswIAAEBhR9EWAAAAyKFnnnlGJUqU0O+//66goKAM+0uWLOmArAAAAFDYUbTNRzvfaCdfX19HpwEAAIB8smvXLi1btkzVqlVzdCoAAAC4hbg6OgEAAACgsKpQoYJiYmIcnQYAAABuMRRtAQAAgBx67rnnNGLECO3fv9/RqQAAAOAWwvIIAAAAQA4NGjRIR48eVdWqVWWxWOTqajsn4ujRoypVqpSDsgMAAEBhRdEWAAAAyKFPPvlEiYmJWe739/cvuGQAAABwy6BoCwAAAOTQPffc4+gUAAAAcAtiTVsAAAAgl/bu3atly5YpKSlJkpSamurgjAAAAFCYUbQFAAAAciguLk7t2rVTtWrVdO+99+rChQuSpMcee0xz5851cHYAAAAorCjaAgAAADk0evRoubi46PTp0zYPHBs5cqTGjh3rwMwAAABQmLGmLQAAAJBDq1ev1ty5c1WyZEm5uLhY26tUqaKoqCjFxMTIz8/PgRkCAACgMGKmLQAAAJBD8fHx8vT0lCSbom18fLySkpLk4eHhqNQAAABQiFG0BQAAAHKocePG+vzzzyX9X9E2LS1Nr7zyisLDw+Xl5eXI9AAAAFBIsTwCAAAAkENvvvmmGjVqpI0bNyomJkYvvPCCNm/erCNHjmjVqlWOTg8AAACFFDNtAQAAgByqWLGitm/froYNG6p+/fo6ePCgmjZtqs2bN6tJkyaOTg8AAACFFDNtAQAAgFwoVaqUxo4d6+g0AAAAcAuhaAsAAADk0OXLl5WWlpbl/qJFi9o8oAwAAACwB0VbAAAAIIfCwsJ06tSpLPefOHFCwcHBBZgRgFvShimOzgAAUMAo2gIAAAA5tHz5ciUnJ9u0HTlyRGPGjFG3bt0UEBDgoMwAAABQmFG0BQAAAHKoTp06GdoaNmyoBg0aqE2bNnrjjTcckBUAAAAKO1dHJwAAAADcaipWrKjTp08rPj7e0akAAACgEKJoCwAAAOSxr776SpLk7e3t4EwAAABQGLE8AgAAAJBDderU0enTp23a4uLidPnyZb3//vtycXFxUGYAAAAozCjaAgAAADn04osvKiEhwabN19dXderUUYUKFRyTFAAAAAo9irYAAABADvXs2dPRKQAAAOAWRNEWAAAAyKHLly8rLS3NrtiiRYuyXAIAAADswoPIAAAAgBwKCwuTj4+PXa9Tp07l+nzx8fF666231KRJEzVt2lQff/yxjDE2Mb/99pvuu+8+1a5dW927d9c///yT6/MCAACgYFG0BQAAAHJo0qRJ8vHx0WuvvaYVK1ZozZo1mjx5ssqWLatXX31VmzZtsr4CAwNzda7ExES1aNFCixcv1qhRo/Tee+9px44d+v77760xf/31l9q0aaM777xT06dPl7+/v5o2baqjR4/m9lIBAABQgFzM9f80j1yLjY2Vn5+fYmJi5Ovr6+h0AAAAbinONNbq1KmTevfurd69e9u07969Wx06dMjTYulbb72lSZMm6cCBAypRooS1PSUlRe7u7tZ8kpKStGLFCkmSMUZVq1ZVx44d9f7779t1Hme6vwD+vw1THJ0BsmlmSoSjU/jPGtgszNEpADdk71iLmbYAAABADu3atUt16tTJ0H777bfr7NmziouLy7Nzff3113rggQdsCraSrAVbY4zWrl2r9u3bW/e5uLioQ4cOWrNmTZ7lAQAAgPxH0RYAAADIoTJlyuiTTz7J0P7VV1/Jw8ND3t7eeXauvXv3qmrVqnr66ad11113qV27dpo1a5b1QWgxMTGKi4tT6dKlbY4LDg7WsWPHsuw3KSlJsbGxNi8AAAA4lrujEwAAAAAKq//9739q166dVq9erSZNmshisWj79u369ddfNXPmTLm5ueXJeVJTU5Wamqo333xTL730kj755BPt3LlTTz31lI4dO6bRo0crNTVVkuTh4WFzrKenp1JSUrLse9y4cXrjjTfyJE8AAADkDWbaAgAAADnUtGlT7du3T23atNGBAwe0Y8cO3Xbbbdq6daseffTRPDuPm5ubihcvriZNmujVV19VeHi4HnnkET399NP6+OOPJUl+fn5yd3fX2bNnbY49e/asAgICsux75MiRiomJsb6ioqLyLG8AAADkDDNtAQAAgFwoV66c3n777Xw/T7169TKsZxsQEKBLly5Jurq27Z133qk///xTjz/+uDVmw4YNqlu3bpb9enp6ytPTM3+SBgAAQI5QtAUAAAByae/evTpw4IDuueceeXp6KjU1Nc+WRkg3ZMgQPfjgg9qxY4fuuOMOnTlzRp988ok6duxojXniiSf0zDPPaPDgwapbt66WLl2q9evXa8WKFXmaCwAAzmrmb4eyFT+wWVg+ZQLkDkVbAAAAIIfi4uLUrVs3rVy5UpJ04sQJBQcH67HHHlOrVq300EMP5dm5OnXqpNdff13NmjWTr6+vzpw5o4iICE2dOtUa89hjj+nAgQO6++675efnp8uXL2vy5Mm655578iwPAAAA5D8XY4xxdBK3mtjYWPn5+Slk+Ldy9cy7JwYDAAA4oyPjIwr0fOljrZiYGPn6+hboua/37LPPavfu3Zo7d65q1aqlbdu2KTg4WPv27VOnTp20Z8+ePD9nUlKSjh8/rtKlS6tIkSKZxsTHx+vUqVMqU6ZMljFZcab7C9wyNkxxdAYoYDNTCvZ3I3KOmbYoaPaOtZhpCwAAAOTQ6tWrNXfuXJUsWVIuLi7W9ipVqigqKkoxMTHy8/PL03N6enqqYsWKN4wpWrSowsL4IxQAAKCwcnV0AgAAAEBhFR8fb32I17VF2/j4eCUlJcnDw8NRqQEAAKAQo2gLAAAA5FDjxo31+eefS/q/om1aWppeeeUVhYeHy8vLy5HpAQAAoJBieQQAAAAgh9588001atRIGzduVExMjF544QVt3rxZR44c0apVqxydHgAAAAopZtoCAAAAOVSxYkVt375dDRs2VP369XXw4EE1bdpUmzdvVpMmTRydHgAAAAopZtoCAAAAOfTuu+/qySef1NixYx2dCgAAAG4hzLQFAAAAcujDDz/UhQsXHJ0GAAAAbjEUbQEAAIAcuuuuu7R+/XpHpwEAAIBbDMsjAAAAADnUqVMnDR48WHv37tWdd96pIkWK2Oxv1aqVPDw8HJQdAAAACiuKtgAAAEAOvfbaa4qLi9Nbb72V6f7o6GiVKlWqgLMCAABAYUfRFgAAAMihY8eOOToFAAAA3IJY0xYAAADIpuDgYJ08edK6PWrUKMXFxTkwIwAAANxKKNoCAAAAufTxxx8rPj7e0WkAAADgFsHyCAAAAAAAAPhPmvnboWzFD2wWlk+ZALaYaQsAAAAAAAAAToSZtgAAAEAOTJgwQUWLFpUkXbp0yWY73UsvvaRixYo5Ij0AAAAUYhRtAQAAgGyqWLGilixZYt0ODg622U43fPhwirYAAADINoq2AAAAQDb98ccfjk4BAAAAtzDWtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ+Lu6AQAAAAAAHB6G6Y4OgMAwH8IM20BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAiPIgMAAAAAAAAsMPM3w5lK35gs7B8ygS3OmbaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIATcXd0AgAAAAAA2GXDlJwf23hY3uUBAEA+Y6YtAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAUMjExMRo0aJF2rx5c6b7d+3apR9//FEHDx4s4MwAAACQFyjaAgAAAIXMgAED1KNHD7377rs27cnJybr//vt19913a/z48apdu7aGDBkiY4yDMgUAAEBOuDs6AQAAAAD2mzZtms6ePasWLVpk2Ddx4kRt2LBB//zzj8qVK6dt27apQYMGatasmXr27FnwyQIAACBHmGkLAAAAFBI7d+7UmDFj9Pnnn8vVNeNQfu7cuerdu7fKlSsnSbrzzjvVpk0bff755wWdKgAAAHKBoi0AAABQCFy+fFk9e/bUpEmTVL58+Qz7k5KStHfvXtWuXdum/c4779SOHTuy7DcpKUmxsbE2LwAAADgWRVsAAACgEHjqqacUHh6uXr16Zbo/Li5OaWlpKl68uE17QECALl68mGW/48aNk5+fn/UVEhKSl2kDAAAgByjaAgAAAE7ut99+0xdffKF77rlHixYt0qJFi3T69GlFR0dr0aJFSkxMlIeHh6SrM3KvdenSJXl6embZ98iRIxUTE2N9RUVF5eu1AAAA4OZ4EBkAAADg5CwWi9q3b6+FCxda26Kjo2WxWPTpp5+qadOmCggIUEBAQIaia1RUlCpWrJhl356enjcs6gIAAKDgUbQFAAAAnFyjRo20aNEim7b27dvL399f33zzjbWtQ4cO+v777/Xiiy/KxcVFiYmJWrp0qR577LECzhgAAAC5wfIIAAAAwC1i9OjR2rdvn3r16qU5c+bo3nvvlYeHh4YPH+7o1AAAAJANFG0BAACAQqhx48aqV6+eTVvlypW1ZcsWhYSE6Mcff1SjRo30119/qUSJEg7KEgAAADnB8ggAAABAITRq1KhM2ytWrKh33323gLMBAABAXmKmLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4EYq2AAAAAAAAAOBEKNoCAAAAAAAAgBNxd3QCAAAAAAAAwK1o5m+HshU/sFlYPmWCwoaiLQAAAPJVQkKCXF1d5enpmWVMSkqK3N0zDk2Tk5OVnJxs03bp0qWbxqQrVqxYDjIGAAAAHCvfl0eIjo7WqVOn8vs0+XqugrwGAACAW0FqaqoWLlyoNm3aqGjRonrkkUcyjfvrr79Up04dFSlSRH5+fnr22WeVmppq3f/GG28oODjY5hUaGipJSktLyzKmePHiCgoKssYAAAAAhUm+F22HDBmiMWPG5Pdp8vVcBXkNAAAAt4L9+/dr7ty5eu6559ShQ4dMY86ePav27durWbNmiomJ0S+//KIvvvhCb7zxhjVm7NixunTpkvUVFxenkJAQSZKrq2uWMaGhoerevbs1BgAAAChMbqlRbLly5RQcHOzoNAAAAP7zqlWrpu+//17t2rWTi4tLpjGfffaZ0tLS9Pbbb6to0aIKDw/X008/rQ8//FApKSmZHrN27VodPnz4hudeu3atDh48qIEDB+b6OgAAAABHyFbR9tSpU9qzZ4/27NmjM2fO5OiER44c0fnz523ajh8/rpMnT1q3o6KirP1fuHAhw9IEx44dU2xsbIa+R44caTM4v76fc+fO5cs1AQAAIPv++OMPNWjQQB4eHta2li1b6sKFC9q7d2+mx8yaNUtVq1a9Yb+zZs1StWrVdPfdd+dpvgAAAEBBydaDyKZNm6ZvvvlGknT69GkVL15cc+bMUbNmzezu4/7779eDDz6o5557ztr27LPPqlixYvrkk08kSf3795enp6eOHj2qixcv6vTp02rbtq1ee+019e/fX7GxsTp9+rSefPJJffDBB9Z+hgwZonLlyunDDz+09uPr66tjx47p+PHjOnfunJo2bapFixbJ29s7z64JAAAA2Xf69GmVL1/epq1kyZLWfTVq1LDZd/HiRS1YsECvvfaaXnnllUz7TI8ZO3Zs/iQNAACQj2b+dsju2IHNwvIxEzhatmbavv7669ZZqefOndOgQYPUu3dvJSQk5Hlia9eu1Zw5c3Ts2DHt2LFDK1euVIcOHfTVV1/p2LFj2rhxo6ZOnaqNGzfesJ+ffvpJ77//vo4dO6YjR45o165dmj59ep5eU1JSkmJjY21eAAAAuLnrHxSWvp3Zkgpffvml0tLS1Lt37yz7S495+OGH8zZRAAAAoABla6ZtuvPnz1sfHPHSSy9p9+7dCg8Pz9PEunbtqnr16km6uiZajRo1VK9ePd15552SpLvuukthYWHaunWrGjRocMN+GjVqJEkqVaqU2rRpo61bt+bpNY0bN87mgRkAAAC4uTJlyuj06dM2benbpUuXzhA/a9YsdenSRQEBAVn2mR4TGBiYt8kCKPw2THF0BgAA2C1bM21Xr16tqlWrKjQ0VB06dFC3bt2Ulpam6OjoPE+sTJkyNttFixbNtO3SpUs37Kds2bI228WKFVNcXJx1Oy+uaeTIkYqJibG+oqKi7D4WAADgv+ruu+/WH3/8YfMJp1WrVikoKEhVqlSxid26dau2bt16w4eL2RMDAAAAFAZ2F21TU1PVvXt3DRgwQBcvXtTBgwf1zz//yNXVNcPH2m4ks4+6ZfV04PyWV9fk6ekpX19fmxcAAMB/XXx8vC5duqTU1FSlpKTo0qVLunz5snX/ww8/LH9/fw0cOFCHDx/W8uXL9d577+mFF16Qq6vtMHXWrFkKCwtTq1atsjyfPTEAAABAYWD38ggnTpzQxYsX1bVrV7m5uUmSfv31V6WmpmbrhCVLltSJEyes22lpadqxY4dDHvyVV9cEAACAjEJDQ5WYmGjdDg4OVqlSpXTw4EFJkq+vr37++Wc988wzqlevnooXL65XX31VI0aMsOknOTlZP/zwg4YOHZrpBAB7YwAAAIDCwu6ibZkyZRQaGqpRo0ZpxIgROnTokEaMGJHtQXHHjh01ZswYNW3aVKVLl9b06dN1+PBhhxRt8+qaAAAAkNHZs2dvGlOtWjX9+OOPN4zx8PBQZGRkrmMAAACAwsLuoq2rq6uWL1+u1157TY8++qhKly6tadOmadSoUfLx8cnyuHLlyik4ONi6PXToUMXHx2vChAkqWrSounTpIn9/fxUrVswaU758eQUFBdn0ExoamuGBEhUrVrR5EMX158qsn2v323tN1/cLAAAAAAAAAPnFxRhjHJ3ErSY2NlZ+fn4KGf6tXD29HZ0OAABAvjoyPqJAz5c+1oqJieFZAvmA+wuntmGKozMA7DIzpWB/N+K/aWCzMEengBywd6xl94PIAAAAAAAAAAD5j6ItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOhKItAAAAAAAAADgRirYAAAAAAAAA4ETcHZ0AAAAAAOA/YsMUR2cAAEChwExbAAAAAAAAAHAiFG0BAAAAAAAAwIlQtAUAAAAAAAAAJ0LRFgAAAAAAAACcCEVbAAAAAAAAAHAi7o5OAAAAAMDNxcfHa9asWfr111+Vmpqq+vXra+jQofL19bWJ2759u9577z1FRkaqatWqevHFF1WhQgXHJA0AAIAcYaYtAAAAUAg0btxYhw8fVt++ffXwww/r+++/V5MmTXT58mVrzD///KPGjRurSJEiGj58uE6fPq2GDRvq5MmTDswcAAAA2cVMWwAAAKAQWLdunc2s2saNG6t06dJasWKFunTpIkl68803FR4ermnTpkmSOnTooEqVKmnSpEl6++23HZI3AADIHzN/O5St+IHNwvIpE+QHZtoCAAAAhcD1yyAUK1ZMrq6uSkxMtLatXr1anTp1sm67u7srIiJCq1atKrA8AQAAkHvMtAUAAAAKobffflve3t5q1aqVJCkmJkYXL15U2bJlbeLKli2ryMjILPtJSkpSUlKSdTs2NjZ/EgYAAIDdmGkLAAAAFDLfffed/ve//2nGjBkqVaqUJOnKlSuSpCJFitjEenl5KTk5Ocu+xo0bJz8/P+srJCQk/xIHAACAXSjaAgAAAIXIDz/8oL59++rDDz9Unz59rO1+fn5yc3PTuXPnbOLPnTunEiVKZNnfyJEjFRMTY31FRUXlW+4AAACwD0VbAAAAoJBYsmSJevToocmTJ+vJJ5+02WexWFSjRg39/fffNu1//fWX7rrrriz79PT0lK+vr80LAAAAjkXRFgAAACgEli1bpu7du2vSpEkaPHhwpjGPPvqovvnmG+3bt0+StH79eq1du1aPPvpoQaYKAACAXOJBZAAAAEAh0KNHD1ksFn311Vf66quvrO2PP/64Hn74YUnS0KFD9c8//6h27doKCwvTwYMHNXLkSHXq1MlRaQMAACAHKNoCAAAAhcDKlStljMnQHhoaav1/Nzc3ffLJJxozZoyioqJUqVIlBQQEFGSa+C/YMMXRGQAAcMujaAsAAAAUAk2aNLE7tnTp0ipdunQ+ZgMAAID8xJq2AAAAAAAAAOBEKNoCAAAAAAAAgBOhaAsAAAAAAAAAToSiLQAAAAAAAAA4ER5EBgAAAAAAANziZv52KFvxA5uF5VMmsAczbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCI8iAwAAAAACpsNU3J+bONheZcHAADIF8y0BQAAAAAAAAAnQtEWAAAAAAAAAJwIyyMAAAAAwH9JbpZWAAAABYKZtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBEeRAYAAAAABY2HgQEAgBtgpi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRCjaAgAAAAAAAIAToWgLAAAAAAAAAE7E3dEJAAAAAEChtGGKozMAAAC3KGbaAgAAAAAAAIAToWgLAAAAAAAAAE6Eoi0AAAAAAAAAOBGKtgAAAAAAAADgRHgQWT7a+UY7+fr6OjoNAAAAAAAAIF/N/O1QtuIHNgvLp0xuDcy0BQAAAAAAAAAnQtEWAAAAAAAAAJwIyyMAAAAA+G/aMMXRGQAA4LSyu9wB8hYzbQEAAAAAAADAiVC0BQAAAAAAAAAnwvIIAAAAAAAAAApUfi+/MLBZWL72n9+YaQsAAAAAAAAAToSZtgAAAAAKLx4mBgAAbkHMtAUAAAAAAAAAJ0LRFgAAAAAAAACcCMsjAAAAALeYkydPKioqSmFhYQoICHB0OgAAAAUuuw86c7YHlzHTFgAAALhFpKam6rHHHlPFihXVr18/lS1bVqNHj3Z0WgAAAMgmZtoCAAAAt4gPP/xQCxYs0Pbt21W1alWtX79eLVu2VHh4uDp16uTo9AAAAGAnirYAAADALWL27Nnq1auXqlatKkm6++671aJFC82ePdt5i7Ybpjg6AwAAAKdD0TYfGGMkSbGxsQ7OBAAA4NaTPsZKH3PhqitXrmjXrl0aNmyYTXv9+vX1xRdfZHlcUlKSkpKSrNsxMTGSCnAsG59QMOcBgAKUkBLn6BQAZFNBjX3sHctStM0H586dkySFhIQ4OBMAAIBbV1xcnPz8/BydhtOIiYlRampqhgePBQQE6Pz581keN27cOL3xxhsZ2hnLAkBuvOjoBABk09MFfL6bjWUp2uaDEiVKSJIiIyP5Q8KJxcbGKiQkRFFRUfL19XV0OrgB3qvCg/eq8OC9Khx4nzJnjFFcXJzKlCnj6FScisVikSQlJibatCckJMjDwyPL40aOHKlnn33Wup2Wlqbz588rICBALi4u+ZPs/8fXeM5w37KPe5Yz3Lec4b7lDPct+7hnOePo+2bvWJaibT5wdXWVJPn5+fFNUwj4+vryPhUSvFeFB+9V4cF7VTjwPmXEP4xn5OfnJ39/f0VHR9u0R0dHq3z58lke5+npKU9PT5s2f3///EgxS3yN5wz3Lfu4ZznDfcsZ7lvOcN+yj3uWM468b/aMZV0LIA8AAAAABeCee+7RkiVLrNspKSlatmyZ2rRp48CsAAAAkF0UbQEAAIBbxKhRo7R582YNGjRIS5YsUa9evZSYmGiz/AEAAACcH0XbfODp6anRo0dn+JgZnAvvU+HBe1V48F4VHrxXhQPvE7KrVq1a2rBhgxISEjR58mSVLFlSf/75p4KDgx2dWqb4Gs8Z7lv2cc9yhvuWM9y3nOG+ZR/3LGcKy31zMcYYRycBAAAAAAAAALiKmbYAAAAAAAAA4EQo2gIAAAAAAACAE6FoCwAAAAAAAABOxN3RCdxqYmNjtW/fPgUFBal8+fKOTuc/KS0tTfv375eLi4sqVqwoi8WSadyxY8d08uRJVa5cWf7+/jmOQe4dPXpUUVFRqlGjhooXL26zzxijf//9V8nJyapZs6bc3TP+2LInBrl3/PhxnTp1StWrV890wfbLly/r33//lb+/vypVqpRpH/bEIHeioqJ06tQplS5dWmXLls005tSpU4qMjFTFihUVGBiY4xjYLykpSX///beCg4MVFhaWaYw9Y4i8igEc4d9//9X58+fVpEkTu+IZA2Tv92b6eCozTZo0kYuLiy5fvqwtW7Zk2F+rVi35+fnlSc7O4OTJk4qKilJYWJgCAgJuGBsVFaWjR4/atLm7u6thw4a56rcw2rNnjxISElSjRg15eHjcND4xMVF79+5ViRIlVK5cObm4uNjs37Ztmy5dumTTFhwcrMqVK+dp3gUhMTFRu3fvlo+Pj6pUqZJnx+Sk38Lk9OnTOnr0qEJDQxUUFHTTeGOMDh8+rISEBFWqVElFihSx2X/8+HEdOnTIps3FxcXu3yuFxf79+xUXF6fq1atnuAfX++effxQTE2PTFhgYqGrVquWq38ImOTlZu3btkpeXV6bXfq0TJ07o4MGDme6rW7euihQpotTUVP3xxx8Z9t92220qWbJknuRsF4M8M3XqVOPl5WWqVatmvL29TefOnc3ly5cdndZ/yvjx403p0qVNtWrVTMWKFU3p0qXNwoULbWKSkpJMz549jZeXl7n99ttNkSJFzDvvvJPtGOSNM2fOmJCQECPJLFmyxGbf/v37TfXq1U3JkiVN+fLlTenSpc369euzHYPcOXXqlGnXrp3x8fExdevWNRUqVDCLFi2yifnmm2+Mr6+vqVKlivHx8THNmzc358+fz3YMci4qKsrUr1/fFC9e3ISHhxtfX1/TvHlzc+bMGWtMWlqaGTx4sPH09DTVq1c3np6e5sUXX7Tpx54Y2O/cuXPm+eefN2XKlDHe3t5myJAhmcbZM4bIqxigoH3zzTemQYMGpnjx4sbNzc2uYxgDZP/35rRp00yTJk1sXiVKlDD+/v4mJSXFGGPMP//8YySZ+vXr28Rt2bKloC4rX6WmpprHH3/c5nfYyy+/fMNjxowZY3x8fGzuR/v27XPdb2ESGRlpateubQICAkzFihVNyZIlzerVq7OMP3/+vHnyySeNv7+/ueOOO0xgYKCpU6eO2bVrl01c7dq1TYUKFWzu7fjx4/P7cvLc4sWLTfHixU3lypWNv7+/adiwoTl16lSuj8lJv4XJ8OHDbb5nhg8ffsP42bNnm4oVK5qKFSua22+/3fj7+5tp06bZxEyePNl4e3vbfE01b948H6+iYJ06dco0bNjQ+Pv7m8qVK5vixYubxYsX3/CY5s2bm5CQEJt78uqrr+a638Jk9erVpmTJkqZixYomICDA1K5d20RGRmYZv2DBggy/L0uXLm3c3d3NuXPnjDHGXLhwwUgytWvXtolbsWJFQV2WMcYYirZ5ZNOmTcbFxcVaIDxx4oQpV66cef755x2c2X/LK6+8Yk6fPm3dHjdunPH09DRHjx61tr3++usmODjY+k28fPly4+LiYn799ddsxSD30tLSTEREhHnxxRczLdrWq1fPdOzY0fqHxpAhQ0xwcLCJj4/PVgxyLiUlxdStW9e0aNHCxMTEGGOu/gL74osvrDGHDh0yHh4e5qOPPjLGGBMTE2OqV69uHnrooWzFIHf69OljatWqZf3av3DhgqlYsaJNkXD69OnGx8fH7Ny50xhjzMaNG42Hh4eZN29etmJgvx07dpjx48eb06dPmwYNGmRatLVnDJFXMYAjjB492mzYsMHMmTPH7qLtf30MkBe/N69cuWJKlSplhg0bZm1LL9qeOHEiz3N2Bh9++KHx8/Mz//77rzHGmA0bNhiLxWIWLFiQ5TFjxowxDRo0yPN+C5MWLVqYFi1amKSkJGOMMS+++KIpXry4uXDhQqbxu3btMtOmTbPGJyYmmk6dOpnbb7/dJq527dqFfuLL8ePHjbe3t3n77beNMcbEx8ebOnXqmC5duuTqmJz0W5h8+umnxtvb22zbts0YY8yWLVuMl5eX+eyzz7I8ZuzYsebIkSPW7a+//tq4uLiYP/74w9o2efJkU6NGjfxL3MHuv/9+U7duXevvsXHjxhlvb+8b/sxu3ry5eeWVV/K838LiwoULpnjx4tZ/SEtKSjJNmzY1LVu2zFY/d9xxh833X3rRdtOmTXmab3ZRtM0jgwcPNjVr1rRpe/31101gYKBJS0tzUFa4ePGikWQzoCpfvrx56aWXbOLq1q1rHnnkkWzFIPcmTpxoWrZsaU6fPp2haLtjxw4jyWbGzPHjx42rq6uZP3++3THInQULFhhJZu/evVnGvPnmmyYoKMikpqZa26ZPn248PT3NpUuX7I5B7rRp08Y8+OCDNm0RERGmR48e1u369eubfv362cTce++9pl27dtmKQc5kVbS1ZwyRVzGAI9lbtGUMkDe/N7///nsjyezYscPall60/f33383WrVtNXFxcnufuSHXq1DGPPfaYTVv79u1NRERElseMGTPGhIeHm23btpk9e/aYK1eu5Em/hcWhQ4eMJPPTTz9Z2y5cuGAsFouZM2eO3f0sWrTISLLOUjPmatH2lVdeMX/99ZeJjo7Oy7QLzKRJk4yvr6+1QG2MMV988YVxc3MzZ8+ezfExOem3MGnWrJnp1auXTVu3bt2yPSvW39/fTJw40bo9efJkU61aNbN9+3aze/duk5ycnBfpOoUzZ84YV1dX880331jbEhISjI+Pj5k8eXKWxzVv3twMGzbM/PXXXyYqKirP+i0sZs+ebTw8PKwTjIwxZunSpUaSOXz4sF19bNq0yUgyy5cvt7alF20XLlxoNm/enOU/YuU3HkSWR7Zu3arw8HCbtvr16+vs2bM6duyYg7LCpk2bJMm6btL58+cVGRmZ6Xu1detWu2OQe3///bfeeecdff755xnWv5JkvdfXvg+lS5dWuXLlrPvsiUHu/Pzzz7rttttUtWpV7d6927pu4LW2bt2qu+66S66u//crpX79+kpKStLu3bvtjkHuvPzyy1q1apXef/99/fzzzxo/frz+/vtvvfjii5KurhG2ffv2G/5ssycGec+eMURexQCFAWOAvPm9OWvWLDVo0EC1atXKsK9Hjx7q06ePSpQooSFDhmT43V4Ypaam6p9//snR77CtW7eqT58+at26tYKDgzV37tw86bcwyOx7yd/fX1WqVMnW9W3atEklSpTI8HyK9957TwMHDlS1atVUv379Qjfu27p1q2rVqmWzxm/9+vWVmpqqHTt25PiYnPRbmGQ1JsnO19T+/fsVExOTYQ3kvXv3qlevXmrXrp1KliypmTNn5knOjrZjxw6lpaXZ3LciRYqoVq1aN71vM2fO1MCBA1WzZk3dcccdNmuX56bfwmDr1q2qUqWKfH19rW3169e37rPHrFmzFBISonbt2mXY98QTT6hfv34KCgpSnz59MqwfnN9urZX6Hej8+fMZFqNP3z5//rxCQkIckdZ/2oULFzRo0CB16tRJd9xxh6Sr74WkTN+r9H32xCB34uLi1KtXL33wwQcqV66czp49myHm/Pnz8vb2zrBA+vXv1c1ikDvHjx+Xr6+vmjVrprNnzyoxMVHx8fGaMWOG7r//fklX34frH3h17c8/e2OQO/Xq1VPnzp31+uuvq2LFijp06JAeffRR1ahRQ5IUHx+vpKSkG/5ssycGec+eMURexQB5YdOmTUpKSspyf5EiRVS3bt0c938rjgEuXbqkbdu23TCmbNmyqlixoqTc/948fvy4fvzxR82YMcOm3dfXVz/++KPat28v6eoftK1atZK/v7/Gjh1r7+UUmMjISEVGRt4wJv0hanFxcbpy5Uq2f4fVq1dPhw4dUmhoqKSrRcZ+/fqpSpUqatiwYY77daQtW7bo8uXLWe63WCxq0KCBpP/7eipRooRNTHaub/PmzZo4caL+97//2UzGeOaZZ9SzZ08VKVJEsbGx6tmzp7p06aIdO3Zk+lBbZ3Sz3605PSYn/RYWKSkpiouLy/T6YmNjlZqaKjc3txv2kZycrH79+umuu+5Sx44dre133HGH9u3bZy3kzpw5U0888YQqVaqkVq1a5f3FFKCc1iEGDhyoZcuWqWjRorp8+bIeeeQRde7c2fqAu1u9vpHZ91L6zzN7ri8hIUFff/21nnnmGZt/KLVYLPryyy/Vp08fSdKBAwfUunVrPfXUU/rss8/y8ApujKJtHrFYLEpMTLRpS0hIkCS7nryJvHXp0iVFRETIz89Pn3/+ubXdYrFIUqbvVfr7ZE8McueVV15RyZIlVbp0aa1fv976r1W7d+9WaGioatWqJYvFoqSkJBljbAZ/179XN4tB7lgsFm3atEmzZ89W//79JUmjR4/Wgw8+qEOHDikoKMiun3/8jMx/Dz74oE6ePKnIyEj5+PjowoULatasmWJjY/XJJ5/w88+J5dX3EN9nKCjjxo3T6dOns9xfunRpzZ8/P8f934pjgOjoaL300ks3jOnRo4eeeuopSbn/fv7ss8/k7e2tnj172rSXL19e5cuXt27fddddeuKJJ/TNN984ZdF2xYoVN/3jeMqUKbrrrrty/Dvs+plVw4cP1/Tp0zV//nw1bNiwUP5unDhxoo4ePZrlfn9/fy1dulTS//3uT0pKkpeXlzXG3uv7999/FRERoQcffFDDhw+32ffII49Y/9/X11fvvPOOatWqpS1btqhRo0bZuSSHsVgsGQrgN/tetOeYnPRbWLi5ucnV1TXT7xlXV9ebFmxTUlLUq1cvHT9+XL/99pvc3f+vbHV9YXbgwIGaMWOG5s2bV+iLtjf6WXPtLNLr9e3b1/r/3t7emjx5skJCQrR+/Xp16NAhx/0WFpn9vkzftud76bvvvlNcXJweffRRm/aiRYtaC7bS1U9vP//883rhhRc0Z84cmwJvfqJom0dCQ0MVHR1t0xYdHS0XFxdmthSwS5cuqUOHDkpMTNTq1avl5+dn3VemTBlZLJZM36v0Aaw9McidYsWKydXV1frHS0pKiqSrf2BERUVpypQpCg0NVWpqqk6dOqXg4GBJUlpamk6ePGl9H+yJQe5UqFBBRYoUUb9+/axtTz75pN58801t2bJF7du3V2hoqHbu3GlzXPr3z7Xv1c1ikHPGGC1btkzvv/++fHx8JEnFixdXnz59NHnyZEmSp6enSpUqdcOfbfbEIO/ZM4bIqxggLyxcuDBf+78VxwC33Xab1q9fb3d8bn9vzp49W71791axYsVuGpvZz31nMXDgQA0cONCu2KJFiyogICBPfodde0/yst+C8uWXX9odmz7DODo62uZj6MePH9e99957w2P37NmjVq1aKSIiQh9//HGmS55dq1SpUtZzFRahoaFavXq1TdvNvhftOSYn/RYW6eOOnHzPpKSkqHfv3tqyZYvWrl1r1/jFmX+GZce134ulS5e2tkdHR6tmzZp29xMUFCQXFxfrPcmrfp1VaGioVq1aZdOWne+lWbNmqX379nZ/rSUkJOj8+fMKDAzMWcLZxJq2eaRNmzb65ZdfFB8fb21bvHixGjZsaNdgCXkjPj5eHTt2VHx8vFavXp3hYz4Wi0XNmzfXDz/8YG1LTEzUihUr1KZNG7tjkDv/+9//tH79eusr/V/6J0yYoClTpkiSmjZtKk9PT5v34ddff9XFixet74M9Mciddu3aKSkpSWfOnLG2pa+NWbJkSUlXf/5t3rxZJ06csMYsXrxYFStWVKVKleyOQc65uLgoMDAww7qlUVFR1vdJuvo+LFmyxLqdmpqqpUuX2ny/2BODvGXPGCKvYgBntXnzZh0+fFgSYwDJvt+b8fHxNp9YSrd27VodOHAg02LntT8f0q1ateqW+MNdyvx32LJly2y+JiIjI/Xnn39at6+/J2fOnNG2bdts7ok9/RZWDRo0kI+Pj8330qZNm3T8+HGb69u6dasOHDhg3d67d69atmyp9u3b65NPPslQsL18+bKMMTZtK1eulCTr0k2FQZs2bbRr1y4dPHjQ2rZ48WIFBwdb14tOTEzU+vXrdeHCBbuPsSemMGvTpo2WLl1q/RowxmjJkiU2X1PR0dH6/fffrdupqanq06ePNm3apLVr16pChQoZ+r3++zUmJkZ//fXXLfEzrFatWipVqpTN9+K+ffv077//2ty3f/75R3v27JF09WsvNTXVpp9Vq1bJGGO9J/b2W1i1adNG0dHRNuv4Ll68WL6+vtZlYFJSUrR+/Xqbv2mlq0se/Pbbb3b/vly5cqWCg4MzLMeQrxzy+LNb0KVLl0yVKlVM69atzaJFi8zIkSONu7u7+eWXXxyd2n9GSkqKadGihQkICDCLFy8269ats76OHz9ujfvjjz+Mh4eHGTFihFm8eLHp0KGDKV++vM3TAO2JQd45c+aMkWSWLFli0/7GG28YX19f8/HHH5uvvvrKlC9f3vTp0yfbMcidjh07miZNmpjFixebefPmmerVq5s2bdpYn0afkpJi6tWrZ+rVq2cWLlxoxo0bZ9zd3c28efOsfdgTg9wZN26c8fLyMm+//bZZuXKleeutt4zFYjHTpk2zxuzZs8f4+PiYAQMGmB9++MH06tXLBAQEmMjIyGzFwH6pqanW30XVq1c3DzzwgFm3bp3ZsmWLNcaeMURexQCOsHfvXrNu3Trz8ssvGzc3N+v3xLVPeg4NDTVPP/20dfu/Pgaw5/fm1q1bjSSzZs0am2MffPBBc+edd2ba73PPPWcef/xx8+2335rFixeb3r17Gw8PD7Ny5cr8vJwCs2vXLlO0aFHz+OOPmx9++MF0797dlCxZ0hw7dswaM3r0aOPn52fdrlevnhk7dqxZvny5+fzzz03NmjVN5cqVzblz57LVb2E2ceJE4+3tbaZOnWrmzZtnqlSpYjp16mQTU6NGDTNgwABjjDGRkZGmTJkypm7duubXX3+1+bsrPj7eGGPMX3/9ZRo1amQ+/vhjs2LFCjNu3Djj4+NjBg4cWODXlxtpaWmmefPmpnbt2ua7774zEydONBaLxcyaNcsas3//fpu/Zew5xp6YwuzgwYPG39/fPPzww+aHH34wDz30kPH39zcHDx60xrzzzjvGzc3Nuv3QQw+ZIkWKmLlz59p8TR0+fNga06xZM/PGG2+YZcuWmS+//NLUqVPHlC9f3pw4caIgLy/fzJo1y3h4eJhJkyaZ7777ztSqVcs0b97c+neXMcY0adLEdO3a1RhzddweHh5upk2bZlasWGEmTpxoSpQoYbp3757tfguz++67z1StWtXMmzfPTJ061Xh5eZlJkyZZ96fXG+bOnWtz3EsvvWSCg4PNlStXMvQ5ceJE06tXL/Pll1+aZcuWmUGDBhl3d3fz+eef5/v1XMvFmOv++Qs5dvr0aY0fP17bt29XUFCQBg8erKZNmzo6rf+MhISELP+l6Nlnn9UDDzxg3d64caOmTJmiEydOqEaNGnrxxRczPOzBnhjkjZiYGEVEROjtt99W48aNbfZ9+umnWrBggZKTk9W2bVsNGzYsw9o09sQg5xITE/X+++/rl19+kbe3t5o1a6bBgwfbPEAiJiZGb7/9tjZu3Ch/f38NGDBAHTp0sOnHnhjkzvfff68FCxbo1KlTKlOmjHr16pXhHu/atcu61l2lSpX0wgsvZHgqrz0xsE9Wv5vCwsJs1ly3ZwyRVzFAQRszZoxWrFiRoX3atGnWGWXdu3dX06ZNrWu6SowBbvZ788CBA+rXr591TVdJunLliiIiItSvXz+btfjSpaWl6auvvtLSpUsVFxen2267TUOHDlVYWFiBXVd+++effzRp0iRFRkaqSpUqeuGFF2yub/bs2Zo/f75+/PFHSVcfXvzhhx9q48aN8vb2Vv369TV48GB5e3tnq9/C7uuvv9Y333yjhIQEtWzZUs8884zNg/4efvhh699EGzZs0AsvvJBpP3PnzrU+UG/79u36+OOPtX//fpUtW1ZdunRRp06dCuR68lJ8fLzeeecd/f777/Lx8dHDDz9sfSCvdHXGaM+ePW3+lrnZMfbGFGZ79uzRu+++q0OHDiksLEzPPfecqlWrZt3/zTffaPr06Vq7dq0k6d5779XFixcz9NO7d28NGTJEkhQbG6upU6dqw4YN8vT0VHh4uIYOHWpdHuxWsGjRIn3++eeKi4tTkyZN9Pzzz6to0aLW/YMHD1ZAQIDGjBkj6ep9njZtmvbs2aPg4GB17Ngxw3rm9vRbmCUmJmry5Mlas2aNvLy81KtXL/Xu3du6P73eMGrUKLVt29ba3qNHDzVu3DjDetzpFi9erO+++05nz55VpUqV9Pjjj1sfcl9QKNoCAAAAAAAAgBNhTVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAAAJwIRVsAAAAAAAAAcCIUbQEAAAAAAADAiVC0BQAAAAAAAAAnQtEWAAAAAAAUWr///ru2bNni6DQAIE9RtAWAQmzXrl169tln1aZNG7Vr105Dhw7Vxo0bHZLLmjVr1KZNG4ecGwAAAIXPH3/8oV9//TXX/UyePFmzZ8/Og4wAwHlQtAWAQmr69OmqU6eOEhMTNXz4cD3//POqUaOGRowYoXnz5hV4PufOndOmTZsK/LwAAAAonN5//32NGzfO0WkAgFOiaAsAhdCWLVs0ZMgQvfvuu/roo48UERGhe+65R4MGDdL69esVERFhjT148KCeeOIJNW/eXN26ddPy5ctt+po3b5769u1r0/bnn3/q7rvvtm6vWLFCERER2rhxo/r376977rlHL7zwguLi4qz5vPzyy7p06ZIaNmyohg0b6qOPPsrHOwAAAIBbzc8//6ydO3fq4sWLWrNmjdasWaOEhIQMcbGxsVq2bJnWr1+v+Pj4TPtKS0vTX3/9pUWLFumff/6x2bdz50599913MsZY2y5cuKBvvvlGJ0+ezNuLAoAccnd0AgCA7Js+fbpKly6tQYMGZbq/WLFikqRTp06pfv36atu2rV5++WVt3bpV999/v7744gv16NFDknTixAlt377d5viLFy/qzz//tG6fOXNGK1eu1IkTJ/Tyyy+rSJEievHFFxUZGalvvvlGlStXVv/+/fW///1P7733niSpbNmy+XDlAAAAuFWNHj1arq6uOn78uKpVq6a9e/dKkjZu3KgSJUpIkrZv3642bdooKChIwcHBOnLkiLy8vBQcHGztJzo6Wp06ddKlS5dUrVo1bd26VTVq1NDChQvl5eWlEiVK6Mknn9Thw4f1/PPPS5Iee+wxHTt2TN26dSv4CweATFC0BYBCaNu2bapZs6bc3W/8Y3zChAkqXbq0vvrqK7m4uKhdu3a6ePGiXnrpJWvR1l4pKSn67rvvFBYWJkm6cuWK+vbtK2OMfH19VaVKFbm5ualhw4Y5vi4AAAD8tx08eFBbt25VUFCQkpOTVb16dc2YMUMjR46UJA0ePFitW7e2jm+XLl2q++67T82bN7f28fDDD6tBgwaaOnWqXFxclJiYqKZNm2rChAl6/fXXVaZMGX3yySfq2bOnWrdurb///lsrV67U1q1bbzq+BoCCwvIIAFAIJSYmytfX96ZxmzZtUrt27eTi4mJt69ixow4fPqzz589n65wlSpSwFmwlqVy5ckpISFBsbGy2+gEAAACy0rVrVwUFBUmSPDw81KhRI+uM22PHjmnDhg167rnnrOPbe++9V9WrV7ceHxkZqV9++UWVKlXSggULNH/+fP3www8KCwvTmjVrrHH333+/+vXrp+7du2v48OF6//33Vbly5QK8UgC4Mf4JCQAKoZCQEB0+fPimcXFxcdalEtKlb8fFxVk/ZmYPi8Vis50+UL52LTAAAAAgN64fn3p6eurSpUuSrhZkJalChQo2MRUrVrT+/5EjRyRJv//+u81Dcl1cXFS3bt3/1979gyTXxmEcv56kPxDhEBGVERgKZSekSCXagrYkqDWwIqKGlsD+PA01BEJNghGCSzRESDg1ubdF4GBEKSVUgwlBiVO9QyRE7/NCD8V7gu9nOufmd27vs8jh4ty/8+66tbU1xWIxuVwuTU5OftUtAMCXILQFgB/I7/drbm5OqVRKhmH8sa69vV3pdPrdWDqdVlVVlWw2mySptrZWxWLxXc3ffIChoqKCABcAAADfpr6+XtLrR8Pejt/O34Lct91oq6ur6unp+c/5gsGg7Ha70um0EomERkZGvmXdAPA3aI8AAD9QIBBQb2+vxsfHdX5+Xh5/fn5WPB5XMpmUJE1NTSmRSOj4+FiSdH9/r1AopEAgIIvFIkkyDENXV1c6OTmRJBUKBYXD4U+vqbGxUY+Pj7RLAAAAwLew2+1qampSIpEoj+VyuXdv1BqGoZaWFu3s7Hy4/ubmpny8v7+veDyuRCKh9fV1TU9P6/b29lvXDwCfQWgLAD9QdXW1ksmkPB6P3G63nE6nDMOQ1WpVPB6Xw+GQ9Nq/dnl5WYODg+ro6FBbW5uampoUCoXKc/l8Ps3MzKi/v1/d3d0yDENut/vTa/J6vfJ4PHI6nfL5fNre3v6q2wUAAABUWVmpjY0N/f79WysrK4pEIhoaGlJdXV25xmKxKBaLaW9vT6Ojo4pGowqFQhoYGNDBwYGk1zYLs7Oz2tzcVGdnpxYXF+VyuRQIBNg5BsA0fr3wjwQAP1qpVNLFxYUqKipkt9tVU1Pzoebh4UGXl5dqaGhQa2vrv85zd3enQqEgh8OhYrGos7Mzeb1eSVI+n9f19fW7LWZPT09KpVLq6+srv7X78vKiTCajfD6v5ubmP/4WAAAAEA6HVSwWtbS0JOm1x2xXV5fGxsbKNdFoVKVSSfPz8+Wxo6MjHR4eymq1anh4WKenp7JarZqYmCjXZLNZ7e7uKpPJyGazye/3l59tI5GIstmstra2yvW5XE7BYFALCwsfet8CwP+B0BYAAAAAAAAATIT2CAAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCKEtgAAAAAAAABgIoS2AAAAAAAAAGAihLYAAAAAAAAAYCL/AKOrK3g1aPnlAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -2856,12 +2922,12 @@ "data": { "text/plain": [ "{'per_year': {'counts': record_num\n", - " 2023 374\n", + " 2023 376\n", " 2024 718\n", " 2025 807\n", " 2026 141\n", " Name: count, dtype: int64,\n", - " 'total': 2040},\n", + " 'total': 2042},\n", " 'mammaprint_overall': {'series': 1 -0.158\n", " 2 0.058\n", " 3 -0.039\n", @@ -2873,10 +2939,10 @@ " 2307 0.192\n", " 2308 0.396\n", " 2309 -0.065\n", - " Name: mammaprint_index, Length: 2040, dtype: float64,\n", - " 'mean': -0.007955049019607845,\n", - " 'variance': 0.09011929758116484,\n", - " 'count': 2040},\n", + " Name: mammaprint_index, Length: 2042, dtype: float64,\n", + " 'mean': -0.008001126346718901,\n", + " 'variance': 0.09011556584390465,\n", + " 'count': 2042},\n", " 'mammaprint_by_type': {'a luminal': {'series': 2 0.058\n", " 4 0.065\n", " 6 0.451\n", @@ -2888,10 +2954,10 @@ " 2303 0.235\n", " 2307 0.192\n", " 2308 0.396\n", - " Name: mammaprint_index, Length: 1076, dtype: float64,\n", - " 'mean': 0.22131849442379178,\n", - " 'variance': 0.02272628757391718,\n", - " 'count': 1076},\n", + " Name: mammaprint_index, Length: 1077, dtype: float64,\n", + " 'mean': 0.22133119777158775,\n", + " 'variance': 0.022705340289735015,\n", + " 'count': 1077},\n", " 'b luminal': {'series': 1 -0.158\n", " 3 -0.039\n", " 5 -0.429\n", @@ -2903,18 +2969,18 @@ " 2304 -0.218\n", " 2306 -0.141\n", " 2309 -0.065\n", - " Name: mammaprint_index, Length: 964, dtype: float64,\n", - " 'mean': -0.26386618257261407,\n", - " 'variance': 0.04115081800368833,\n", - " 'count': 964}},\n", + " Name: mammaprint_index, Length: 965, dtype: float64,\n", + " 'mean': -0.26395025906735753,\n", + " 'variance': 0.041114951880119543,\n", + " 'count': 965}},\n", " 'counts_by_type': {'counts': type\n", - " a luminal 1076\n", - " b luminal 964\n", + " a luminal 1077\n", + " b luminal 965\n", " Name: count, dtype: int64,\n", " 'total_distinct': 2}}" ] }, - "execution_count": 29, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } From 711ef58ad55312acaaff2091f8757d602cd1f269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 13 Jul 2026 12:16:31 +0200 Subject: [PATCH 36/79] feat: old mask coverage --- .../notebooks/check_mask_coverage_old.ipynb | 717 ++++++++++++++++++ 1 file changed, 717 insertions(+) create mode 100644 scripts/notebooks/check_mask_coverage_old.ipynb diff --git a/scripts/notebooks/check_mask_coverage_old.ipynb b/scripts/notebooks/check_mask_coverage_old.ipynb new file mode 100644 index 00000000..4c7579a7 --- /dev/null +++ b/scripts/notebooks/check_mask_coverage_old.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Check mask coverage and tiled datasets against data_mapping\n", + "\n", + "Verifies 1-1 mapping between slides in `data_mapping.csv` and:\n", + "- **MLflow masks** (downloaded from MLflow): tissue, epithelium\n", + "- **QC masks**: blur (`Piqe_piqe_median_activity_mask_`), folding (`FoldingFunction_folding_test_`), residual (`ResidualArtifactsAndCoverage_coverage_mask_`)\n", + "- **Slide files** listed in the `path` column: checks that every slide can be opened without error\n", + "- **Tiled datasets** (MLflow): checks `slides.parquet` ↔ `data_mapping` and `tiles.parquet` ↔ `slides.parquet` consistency" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "155f9f1e-5529-448d-8d5e-7c20208def37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmprwve6ksi\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas mlflow>3 openslide-python tifffile matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import mlflow\n", + "from mlflow.artifacts import download_artifacts\n", + "from openslide import OpenSlide\n", + "import os\n", + "from tifffile import TiffFile" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\n", + "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", + "\n", + "PROJECT_PATH = Path(\"/mnt/projects/mammaprint\")\n", + "DATA_MAPPING_CSV = PROJECT_PATH / \"data_mapping.csv\"\n", + "QC_MASKS_PATH = PROJECT_PATH / \"qc_masks3\"\n", + "\n", + "MLFLOW_MASKS = {\n", + " # \"tissue\": \"mlflow-artifacts:/3/50c597ac29784808bd89d55d9912f699/artifacts/tissue_masks\",\n", + " # \"epithelium\": \"mlflow-artifacts:/3/c5e6c447e4784e5b991bff258195b5df/artifacts/epithelium_masks\",\n", + " \"tissue\": \"mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\",\n", + " \"epithelium\": \"mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\",\n", + "}\n", + "\n", + "QC_MASK_PREFIXES = {\n", + " \"blur\": \"Piqe_piqe_median_activity_mask_\",\n", + " \"folding\": \"FoldingFunction_folding_test_\",\n", + " \"residual\": \"ResidualArtifactsAndCoverage_coverage_mask_\",\n", + "}\n", + "\n", + "TILED_DATASETS = {\n", + " # \"mou_3_224\": \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\",\n", + " # \"mou_2_224\": \"mlflow-artifacts:/3/1ebe489fd3ca4f5682c9a4b3d6f74620/artifacts/mou_2_224\",\n", + "}\n", + "\n", + "EMBEDDINGS = {\n", + " # \"embedding3\": \"mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2042 slides from /mnt/projects/mammaprint/data_mapping.csv\n", + "Columns: ['record_num', 'mammaprint_index', 'type', 'path', 'split']\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplit
02023/00835-0.158b luminal/mnt/data/MOU/breast/mammaprint/P2023_0835test
12023/008360.058a luminal/mnt/data/MOU/breast/mammaprint/P2023_0836train
22023/00837-0.039b luminal/mnt/data/MOU/breast/mammaprint/P2023_0837train
32023/008380.065a luminal/mnt/data/MOU/breast/mammaprint/P2023_0838test
42023/00839-0.429b luminal/mnt/data/MOU/breast/mammaprint/P2023_0839test
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2023/00835 -0.158 b luminal \n", + "1 2023/00836 0.058 a luminal \n", + "2 2023/00837 -0.039 b luminal \n", + "3 2023/00838 0.065 a luminal \n", + "4 2023/00839 -0.429 b luminal \n", + "\n", + " path split \n", + "0 /mnt/data/MOU/breast/mammaprint/P2023_0835 test \n", + "1 /mnt/data/MOU/breast/mammaprint/P2023_0836 train \n", + "2 /mnt/data/MOU/breast/mammaprint/P2023_0837 train \n", + "3 /mnt/data/MOU/breast/mammaprint/P2023_0838 test \n", + "4 /mnt/data/MOU/breast/mammaprint/P2023_0839 test " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Load data mapping\n", + "df = pd.read_csv(DATA_MAPPING_CSV)\n", + "print(f\"Loaded {len(df)} slides from {DATA_MAPPING_CSV}\")\n", + "print(f\"Columns: {df.columns.tolist()}\")\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 unique slide tiff names expected\n" + ] + } + ], + "source": [ + "# Derive the .tiff name that the tiling pipeline expects for each slide\n", + "df[\"tiff_name\"] = df[\"path\"].apply(lambda p: Path(p).with_suffix(\".tiff\").name)\n", + "expected_names = set(df[\"tiff_name\"])\n", + "print(f\"{len(expected_names)} unique slide tiff names expected\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-slide-open-check", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Verify data_mapping slide files can be opened ----\n", + "SLIDE_SUFFIX_CANDIDATES = (\".mrxs\", \".tiff\", \".tif\")\n", + "\n", + "\n", + "def _slide_path_candidates(raw_path: str) -> list[Path]:\n", + " path = Path(raw_path)\n", + " candidates = [path]\n", + " if not path.suffix:\n", + " candidates.extend(Path(f\"{raw_path}{suffix}\") for suffix in SLIDE_SUFFIX_CANDIDATES)\n", + " return candidates\n", + "\n", + "\n", + "def _resolve_slide_path(raw_path: str) -> Path:\n", + " for candidate in _slide_path_candidates(raw_path):\n", + " if candidate.is_file():\n", + " return candidate\n", + " return Path(raw_path)\n", + "\n", + "\n", + "def _try_open_slide(raw_path: str) -> tuple[Path, str | None]:\n", + " slide_path = _resolve_slide_path(raw_path)\n", + " try:\n", + " str_path = str(slide_path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as slide:\n", + " _ = len(slide.pages)\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " _ = slide.dimensions\n", + " return slide_path, None\n", + " except Exception as e:\n", + " return slide_path, f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "# slide_open_rows = []\n", + "# for row in df.itertuples(index=False):\n", + "# opened_path, error = _try_open_slide(row.path)\n", + "# if error:\n", + "# slide_open_rows.append(\n", + "# {\n", + "# \"record_num\": row.record_num,\n", + "# \"path\": row.path,\n", + "# \"resolved_path\": str(opened_path),\n", + "# \"error\": error,\n", + "# }\n", + "# )\n", + "\n", + "# slide_open_errors = pd.DataFrame(slide_open_rows)\n", + "# if slide_open_errors.empty:\n", + "# print(f\"All {len(df)} slide files opened successfully.\")\n", + "# else:\n", + "# print(f\"{len(slide_open_errors)} slide files failed to open:\")\n", + "# display(slide_open_errors)\n", + "# raise AssertionError(f\"{len(slide_open_errors)} slide files failed to open\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cell-5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 tissue mask files found in /tmp/tmpm4swn0wz/tissue_masks\n" + ] + }, + { + "ename": "MlflowException", + "evalue": "The following failures occurred while downloading one or more artifacts from http://mlflow-s3.rationai-mlflow/api/2.0/mlflow-artifacts/artifacts/3/0176fb9485de49699b1e16f362cff5fa/artifacts:\n##### File epithelium_masks/P2023_01448.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2023_03021.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2024_02850.tiff #####\nResponse ended prematurely", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mMlflowException\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 3\u001b[39m mlflow_files: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mset\u001b[39m[\u001b[38;5;28mstr\u001b[39m]] = {}\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m mask_name, uri \u001b[38;5;129;01min\u001b[39;00m MLFLOW_MASKS.items():\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m mask_dir = Path(\u001b[43mdownload_artifacts\u001b[49m\u001b[43m(\u001b[49m\u001b[43muri\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[32m 7\u001b[39m mlflow_dirs[mask_name] = mask_dir\n\u001b[32m 8\u001b[39m found = {p.name \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m mask_dir.iterdir() \u001b[38;5;28;01mif\u001b[39;00m p.is_file()}\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/artifacts/__init__.py:84\u001b[39m, in \u001b[36mdownload_artifacts\u001b[39m\u001b[34m(artifact_uri, run_id, artifact_path, dst_path, tracking_uri, registry_uri)\u001b[39m\n\u001b[32m 81\u001b[39m pathlib.Path(dst_path).mkdir(exist_ok=\u001b[38;5;28;01mTrue\u001b[39;00m, parents=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 83\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m artifact_uri \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m84\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_download_artifact_from_uri\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 85\u001b[39m \u001b[43m \u001b[49m\u001b[43martifact_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdst_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtracking_uri\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtracking_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mregistry_uri\u001b[49m\u001b[43m=\u001b[49m\u001b[43mregistry_uri\u001b[49m\n\u001b[32m 86\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 88\u001b[39m \u001b[38;5;66;03m# Use `runs://` to download both run and model (if exists) artifacts\u001b[39;00m\n\u001b[32m 89\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m run_id \u001b[38;5;129;01mand\u001b[39;00m artifact_path:\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/tracking/artifact_utils.py:128\u001b[39m, in \u001b[36m_download_artifact_from_uri\u001b[39m\u001b[34m(artifact_uri, output_path, lineage_header_info, tracking_uri, registry_uri)\u001b[39m\n\u001b[32m 122\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(repo, ModelsArtifactRepository):\n\u001b[32m 123\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m repo.download_artifacts(\n\u001b[32m 124\u001b[39m artifact_path=artifact_path,\n\u001b[32m 125\u001b[39m dst_path=output_path,\n\u001b[32m 126\u001b[39m lineage_header_info=lineage_header_info,\n\u001b[32m 127\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m128\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrepo\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdownload_artifacts\u001b[49m\u001b[43m(\u001b[49m\u001b[43martifact_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43martifact_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdst_path\u001b[49m\u001b[43m=\u001b[49m\u001b[43moutput_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 129\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 130\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m artifact_uri.startswith(\u001b[33m\"\u001b[39m\u001b[33mm-\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 131\u001b[39m \u001b[38;5;66;03m# When a Model ID like string is passed, suggest using 'models:/{artifact_uri}' instead.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/opt/venv/lib/python3.14/site-packages/mlflow/store/artifact/artifact_repo.py:366\u001b[39m, in \u001b[36mArtifactRepository.download_artifacts\u001b[39m\u001b[34m(self, artifact_path, dst_path)\u001b[39m\n\u001b[32m 358\u001b[39m error_codes = {\n\u001b[32m 359\u001b[39m e.error_code\n\u001b[32m 360\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m e \u001b[38;5;129;01min\u001b[39;00m failed_downloads.values()\n\u001b[32m 361\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, MlflowException) \u001b[38;5;129;01mand\u001b[39;00m e.error_code != \u001b[33m\"\u001b[39m\u001b[33mINTERNAL_ERROR\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 362\u001b[39m }\n\u001b[32m 363\u001b[39m error_code = (\n\u001b[32m 364\u001b[39m ErrorCode.Value(error_codes.pop()) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_codes) == \u001b[32m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m INTERNAL_ERROR\n\u001b[32m 365\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m366\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MlflowException(\n\u001b[32m 367\u001b[39m message=(\n\u001b[32m 368\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mThe following failures occurred while downloading one or more\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 369\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m artifacts from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.artifact_uri\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m_truncate_error(failures)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 370\u001b[39m ),\n\u001b[32m 371\u001b[39m error_code=error_code,\n\u001b[32m 372\u001b[39m )\n\u001b[32m 374\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m os.path.join(dst_path, artifact_path)\n", + "\u001b[31mMlflowException\u001b[39m: The following failures occurred while downloading one or more artifacts from http://mlflow-s3.rationai-mlflow/api/2.0/mlflow-artifacts/artifacts/3/0176fb9485de49699b1e16f362cff5fa/artifacts:\n##### File epithelium_masks/P2023_01448.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2023_03021.tiff #####\nResponse ended prematurely\n##### File epithelium_masks/P2024_02850.tiff #####\nResponse ended prematurely" + ] + } + ], + "source": [ + "# ---- MLflow masks (download from MLflow) ----\n", + "mlflow_dirs: dict[str, Path] = {}\n", + "mlflow_files: dict[str, set[str]] = {}\n", + "\n", + "for mask_name, uri in MLFLOW_MASKS.items():\n", + " mask_dir = Path(download_artifacts(uri))\n", + " mlflow_dirs[mask_name] = mask_dir\n", + " found = {p.name for p in mask_dir.iterdir() if p.is_file()}\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files found in {mask_dir}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- QC masks (local directory) ----\n", + "qc_files: dict[str, set[str]] = {}\n", + "for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + " found = {\n", + " p.name.removeprefix(prefix)\n", + " for p in QC_MASKS_PATH.iterdir()\n", + " if p.is_file() and p.name.startswith(prefix)\n", + " }\n", + " qc_files[mask_type] = found\n", + " print(f\"{len(found)} {mask_type} mask files found (prefix: {prefix})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Build per-slide report ----\n", + "rows = []\n", + "for name in sorted(expected_names):\n", + " row = {\"tiff_name\": name}\n", + " for mask_name in MLFLOW_MASKS:\n", + " row[mask_name] = name in mlflow_files[mask_name]\n", + " for mask_type in QC_MASK_PREFIXES:\n", + " row[mask_type] = name in qc_files[mask_type]\n", + " rows.append(row)\n", + "\n", + "mask_cols = [*MLFLOW_MASKS.keys(), *QC_MASK_PREFIXES.keys()]\n", + "report = pd.DataFrame(rows)\n", + "report[\"all_present\"] = report[mask_cols].all(axis=1)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "im0bfkx60ei", + "metadata": {}, + "outputs": [], + "source": [ + "# # ---- Try opening each mask file to verify readability ----\n", + "# def _try_open_mask(path: Path) -> str | None:\n", + "# \"\"\"Try to open a mask file. Returns None on success or an error message.\"\"\"\n", + "# try:\n", + "# str_path = str(path)\n", + "# if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + "# with TiffFile(str_path) as slide:\n", + "# pass\n", + "# else:\n", + "# with OpenSlide(str_path) as slide:\n", + "# pass\n", + "# return None\n", + "# except Exception as e:\n", + "# return f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "# open_errors: list[dict[str, str]] = []\n", + "\n", + "# # MLflow masks\n", + "# for mask_name in MLFLOW_MASKS:\n", + "# for name in sorted(expected_names & mlflow_files[mask_name]):\n", + "# path = mlflow_dirs[mask_name] / name\n", + "# err = _try_open_mask(path)\n", + "# if err:\n", + "# open_errors.append({\"mask_type\": mask_name, \"tiff_name\": name, \"error\": err})\n", + "\n", + "# # QC masks\n", + "# for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + "# for name in sorted(expected_names & qc_files[mask_type]):\n", + "# path = QC_MASKS_PATH / f\"{prefix}{name}\"\n", + "# err = _try_open_mask(path)\n", + "# if err:\n", + "# open_errors.append({\"mask_type\": mask_type, \"tiff_name\": name, \"error\": err})\n", + "\n", + "# if open_errors:\n", + "# errors_df = pd.DataFrame(open_errors)\n", + "# print(f\"{len(open_errors)} mask files failed to open:\")\n", + "# display(errors_df)\n", + "# else:\n", + "# print(\"All existing mask files opened successfully.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-8", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Summary ----\n", + "summary = report[mask_cols].sum().to_frame(\"found\")\n", + "summary[\"missing\"] = len(expected_names) - summary[\"found\"]\n", + "summary[\"total\"] = len(expected_names)\n", + "print(\"Coverage summary:\")\n", + "summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-9", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Slides missing at least one mask ----\n", + "missing = report[~report[\"all_present\"]].drop(columns=[\"all_present\"])\n", + "print(f\"{len(missing)} slides missing at least one mask:\")\n", + "missing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Extra masks not in data_mapping ----\n", + "for mask_name in MLFLOW_MASKS:\n", + " extra = mlflow_files[mask_name] - expected_names\n", + " print(f\"{len(extra)} extra {mask_name} masks with no matching slide in data_mapping:\")\n", + " if extra:\n", + " print(sorted(extra))\n", + " print()\n", + "\n", + "for mask_type in QC_MASK_PREFIXES:\n", + " extra = qc_files[mask_type] - expected_names\n", + " print(f\"{len(extra)} extra {mask_type} masks with no matching slide:\")\n", + " if extra:\n", + " print(sorted(extra))" + ] + }, + { + "cell_type": "markdown", + "id": "53d0f240", + "metadata": {}, + "source": [ + "## Visual inspection\n", + "\n", + "Display 5 sample masks for each mask type (MLflow + QC) in a grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0b337bd", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Visual inspection: 5 sample masks per type ----\n", + "N_SAMPLES = 5\n", + "THUMB_SIZE = (512, 512)\n", + "\n", + "\n", + "def _read_mask_thumbnail(path: Path) -> np.ndarray | None:\n", + " \"\"\"Read a mask file and return a thumbnail as a numpy array.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as tif:\n", + " page = tif.pages[0]\n", + " # Read full page then downsample\n", + " img = page.asarray()\n", + " # Resize to thumbnail via slicing (nearest-neighbor)\n", + " step_y = max(1, img.shape[0] // THUMB_SIZE[1])\n", + " step_x = max(1, img.shape[1] // THUMB_SIZE[0])\n", + " return img[::step_y, ::step_x]\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " thumb = slide.get_thumbnail(THUMB_SIZE)\n", + " return np.array(thumb)\n", + " except Exception as e:\n", + " print(f\" Could not read {path.name}: {e}\")\n", + " return None\n", + "\n", + "\n", + "# Collect mask paths: list of (mask_type, [paths])\n", + "all_mask_types: list[tuple[str, list[Path]]] = []\n", + "\n", + "# Pick 5 sample tiff names that exist across all available mask types\n", + "sample_names = sorted(expected_names)[:N_SAMPLES]\n", + "\n", + "for mask_name in MLFLOW_MASKS:\n", + " paths = [mlflow_dirs[mask_name] / name for name in sample_names if name in mlflow_files[mask_name]]\n", + " all_mask_types.append((mask_name, paths[:N_SAMPLES]))\n", + "\n", + "for mask_type, prefix in QC_MASK_PREFIXES.items():\n", + " paths = [QC_MASKS_PATH / f\"{prefix}{name}\" for name in sample_names if name in qc_files[mask_type]]\n", + " all_mask_types.append((mask_type, paths[:N_SAMPLES]))\n", + "\n", + "# Plot grid: rows = mask types, cols = samples\n", + "n_rows = len(all_mask_types)\n", + "n_cols = N_SAMPLES\n", + "fig, axes = plt.subplots(n_rows, n_cols, figsize=(3 * n_cols, 3 * n_rows))\n", + "if n_rows == 1:\n", + " axes = [axes]\n", + "\n", + "for row_idx, (mask_type, paths) in enumerate(all_mask_types):\n", + " for col_idx in range(n_cols):\n", + " ax = axes[row_idx][col_idx]\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " if col_idx == 0:\n", + " ax.set_ylabel(mask_type, fontsize=12, fontweight=\"bold\")\n", + " if col_idx < len(paths):\n", + " thumb = _read_mask_thumbnail(paths[col_idx])\n", + " if thumb is not None:\n", + " ax.imshow(thumb, cmap=\"gray\")\n", + " ax.set_title(paths[col_idx].name, fontsize=7)\n", + " else:\n", + " ax.text(0.5, 0.5, \"read error\", ha=\"center\", va=\"center\", transform=ax.transAxes)\n", + " else:\n", + " ax.text(0.5, 0.5, \"N/A\", ha=\"center\", va=\"center\", transform=ax.transAxes, color=\"gray\")\n", + "\n", + "fig.suptitle(\"Sample masks (5 per type)\", fontsize=14, fontweight=\"bold\")\n", + "fig.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e5a23332", + "metadata": {}, + "source": [ + "## Tiled dataset validation\n", + "\n", + "For each tiled dataset, download `slides.parquet` and `tiles.parquet` from MLflow, then check:\n", + "1. Every slide in `data_mapping.csv` has a row in `slides.parquet` (and vice versa)\n", + "2. Every slide in `slides.parquet` has at least one tile in `tiles.parquet`\n", + "3. Every `slide_id` in `tiles.parquet` maps to a slide in `slides.parquet`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe9ffadc", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Download tiled datasets from MLflow ----\n", + "tiled_data: dict[str, tuple[pd.DataFrame, pd.DataFrame]] = {}\n", + "\n", + "for ds_name, uri in TILED_DATASETS.items():\n", + " local_dir = Path(download_artifacts(uri))\n", + " slides = pd.read_parquet(local_dir / \"slides.parquet\")\n", + " tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n", + " tiled_data[ds_name] = (slides, tiles)\n", + " print(f\"{ds_name}: {len(slides)} slides, {len(tiles)} tiles\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2828c601", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Validate tiled datasets ----\n", + "expected_stems = {Path(p).stem for p in df[\"path\"]}\n", + "\n", + "for ds_name, (slides, tiles) in tiled_data.items():\n", + " print(f\"=== {ds_name} ===\\n\")\n", + "\n", + " # 1. slides.parquet path stem ↔ data_mapping path stem\n", + " slide_stems = {Path(p).stem for p in slides[\"path\"]}\n", + " missing_from_dataset = expected_stems - slide_stems\n", + " extra_in_dataset = slide_stems - expected_stems\n", + "\n", + " print(f\"Slides in data_mapping: {len(expected_stems)}\")\n", + " print(f\"Slides in dataset: {len(slide_stems)}\")\n", + " print(f\"Missing from dataset: {len(missing_from_dataset)}\")\n", + " print(f\"Extra in dataset: {len(extra_in_dataset)}\")\n", + "\n", + " if missing_from_dataset:\n", + " print(f\"\\nSlides in data_mapping but NOT in {ds_name}:\")\n", + " for p in sorted(missing_from_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " if extra_in_dataset:\n", + " print(f\"\\nSlides in {ds_name} but NOT in data_mapping:\")\n", + " for p in sorted(extra_in_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " # 2. Every slide has at least one tile\n", + " slide_ids = set(slides[\"id\"])\n", + " slide_ids_in_tiles = set(tiles[\"slide_id\"])\n", + "\n", + " slides_without_tiles = slide_ids - slide_ids_in_tiles\n", + " print(f\"\\nSlides without any tiles: {len(slides_without_tiles)}\")\n", + " if slides_without_tiles:\n", + " for s in sorted(slides_without_tiles):\n", + " print(f\" {s}\")\n", + "\n", + " # 3. No orphan tiles (tile references a slide not in slides.parquet)\n", + " orphan_tile_ids = slide_ids_in_tiles - slide_ids\n", + " print(f\"Orphan tile slide_ids: {len(orphan_tile_ids)}\")\n", + " if orphan_tile_ids:\n", + " for sid in sorted(orphan_tile_ids):\n", + " count = int((tiles[\"slide_id\"] == sid).sum())\n", + " print(f\" slide_id={sid}: {count} tiles\")\n", + "\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "ea2eae31", + "metadata": {}, + "source": [ + "## Embeddings validation\n", + "\n", + "For each embeddings artifact, check that every slide in `data_mapping.csv` has a corresponding `.parquet` file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc5ce48a", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- Download and validate embeddings ----\n", + "# Embedding parquet files are named as \"{record_num}.parquet\" (e.g. \"2023/00835.parquet\")\n", + "expected_record_nums = set(df[\"path\"])\n", + "\n", + "for emb_name, uri in EMBEDDINGS.items():\n", + " print(f\"=== {emb_name} ===\\n\")\n", + " emb_dir = Path(download_artifacts(uri))\n", + " emb_files = {p.stem for p in emb_dir.iterdir() if p.is_file() and p.suffix == \".parquet\"}\n", + " print(f\"Embedding files found: {len(emb_files)}\")\n", + " print(f\"Slides in data_mapping: {len(expected_record_nums)}\")\n", + "\n", + " missing = expected_record_nums - emb_files\n", + " extra = emb_files - expected_record_nums\n", + "\n", + " print(f\"Missing embeddings: {len(missing)}\")\n", + " if missing:\n", + " for s in sorted(missing):\n", + " print(f\" {s}\")\n", + "\n", + " print(f\"Extra embeddings: {len(extra)}\")\n", + " if extra:\n", + " for s in sorted(extra):\n", + " print(f\" {s}\")\n", + "\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From c6f2d0fd86b87e1ed610c62f719ab6f05446dd22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 13 Jul 2026 12:16:38 +0200 Subject: [PATCH 37/79] feat: new mask coverage --- scripts/notebooks/check_mask_coverage.ipynb | 1329 +++++++++++++++++++ 1 file changed, 1329 insertions(+) create mode 100644 scripts/notebooks/check_mask_coverage.ipynb diff --git a/scripts/notebooks/check_mask_coverage.ipynb b/scripts/notebooks/check_mask_coverage.ipynb new file mode 100644 index 00000000..3a98f321 --- /dev/null +++ b/scripts/notebooks/check_mask_coverage.ipynb @@ -0,0 +1,1329 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Check mask coverage and tiled datasets against data_mapping\n", + "\n", + "Verifies 1-1 mapping between slides in `data_mapping.csv` and:\n", + "- **MLflow masks** (downloaded from MLflow): tissue, epithelium\n", + "- **QC masks**: blur (`Piqe_piqe_median_activity_mask_`), folding (`FoldingFunction_folding_test_`), residual (`ResidualArtifactsAndCoverage_coverage_mask_`)\n", + "- **Slide files** listed in the `path` column: checks that every slide can be opened without error\n", + "- **Tiled datasets** (MLflow): checks `slides.parquet` ↔ `data_mapping` and `tiles.parquet` ↔ `slides.parquet` consistency" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "155f9f1e-5529-448d-8d5e-7c20208def37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: /tmp/tmpxhfqrklg\n", + "Requirement already satisfied: pip in /opt/venv/lib/python3.14/site-packages (25.3)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.3\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3 install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install pandas mlflow>3 openslide-python tifffile matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mlflow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from mlflow.artifacts import download_artifacts, list_artifacts\n", + "from openslide import OpenSlide\n", + "from tifffile import TiffFile" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\n", + "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", + "\n", + "PROJECT_PATH = Path(\"/mnt/projects/mammaprint\")\n", + "DATA_MAPPING_CSV = PROJECT_PATH / \"data_mapping.csv\"\n", + "\n", + "# QC masks may live in several directories. They are treated as if everything\n", + "# lived in a single location: all listed dirs are merged together.\n", + "QC_MASKS_PATHS = [\n", + " PROJECT_PATH / \"qc_masks\",\n", + " PROJECT_PATH / \"qc_masks3\",\n", + " PROJECT_PATH / \"qc_masks_big_diff\",\n", + "]\n", + "\n", + "MLFLOW_MASKS = {\n", + " \"tissue\": \"mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\",\n", + " \"epithelium\": \"mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\",\n", + "}\n", + "\n", + "QC_MASK_PREFIXES = {\n", + " \"blur\": \"Piqe_piqe_median_activity_mask_\",\n", + " \"folding\": \"FoldingFunction_folding_test_\",\n", + " \"residual\": \"ResidualArtifactsAndCoverage_coverage_mask_\",\n", + "}\n", + "\n", + "TILED_DATASETS = {\n", + " \"mou_3_224\": \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\",\n", + " # \"mou_2_224\": \"mlflow-artifacts:/3/1ebe489fd3ca4f5682c9a4b3d6f74620/artifacts/mou_2_224\",\n", + "}\n", + "\n", + "EMBEDDINGS = {\n", + " \"embedding3\": \"mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "a55ee407", + "metadata": {}, + "source": [ + "# Functions\n", + "\n", + "All logic is defined below as individually-callable functions. Run every cell in this section once\n", + "to define them, then use the **Calls** section at the bottom to run whichever checks you need.\n", + "\n", + "For every mask source there are two modes:\n", + "- **list-only** (`ls`-equivalent): enumerate filenames without downloading (`list_mlflow_masks`,\n", + " `list_qc_masks`, `list_embeddings`).\n", + "- **download + open**: download artifacts and open each file to verify readability\n", + " (`download_mlflow_masks` + `open_masks`, `plot_sample_masks`)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-3", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B1. Data-mapping loader ----\n", + "def load_data_mapping(csv_path=DATA_MAPPING_CSV):\n", + " \"\"\"Load data_mapping.csv, derive expected .tiff names, return (df, expected_names).\"\"\"\n", + " df = pd.read_csv(csv_path)\n", + " df[\"tiff_name\"] = df[\"path\"].apply(lambda p: Path(p).with_suffix(\".tiff\").name)\n", + " expected_names = set(df[\"tiff_name\"])\n", + " print(f\"Loaded {len(df)} slides from {csv_path}\")\n", + " print(f\"Columns: {df.columns.tolist()}\")\n", + " print(f\"{len(expected_names)} unique slide tiff names expected\")\n", + " return df, expected_names" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-slide-open-check", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B2. Slide-open verification ----\n", + "SLIDE_SUFFIX_CANDIDATES = (\".mrxs\", \".tiff\", \".tif\")\n", + "\n", + "\n", + "def _slide_path_candidates(raw_path: str) -> list[Path]:\n", + " path = Path(raw_path)\n", + " candidates = [path]\n", + " if not path.suffix:\n", + " candidates.extend(Path(f\"{raw_path}{suffix}\") for suffix in SLIDE_SUFFIX_CANDIDATES)\n", + " return candidates\n", + "\n", + "\n", + "def _resolve_slide_path(raw_path: str) -> Path:\n", + " for candidate in _slide_path_candidates(raw_path):\n", + " if candidate.is_file():\n", + " return candidate\n", + " return Path(raw_path)\n", + "\n", + "\n", + "def _try_open_slide(raw_path: str) -> tuple[Path, str | None]:\n", + " slide_path = _resolve_slide_path(raw_path)\n", + " try:\n", + " str_path = str(slide_path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as slide:\n", + " _ = len(slide.pages)\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " _ = slide.dimensions\n", + " return slide_path, None\n", + " except Exception as e:\n", + " return slide_path, f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "def check_slides_open(df: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Try to open every slide file in df. Returns a DataFrame of failures (empty => all OK).\"\"\"\n", + " slide_open_rows = []\n", + " for row in df.itertuples(index=False):\n", + " opened_path, error = _try_open_slide(row.path)\n", + " if error:\n", + " slide_open_rows.append(\n", + " {\n", + " \"record_num\": row.record_num,\n", + " \"path\": row.path,\n", + " \"resolved_path\": str(opened_path),\n", + " \"error\": error,\n", + " }\n", + " )\n", + "\n", + " slide_open_errors = pd.DataFrame(slide_open_rows)\n", + " if slide_open_errors.empty:\n", + " print(f\"All {len(df)} slide files opened successfully.\")\n", + " else:\n", + " print(f\"{len(slide_open_errors)} slide files failed to open.\")\n", + " return slide_open_errors" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-5", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B3. MLflow masks: two modes ----\n", + "def list_mlflow_masks(mlflow_masks=MLFLOW_MASKS) -> dict[str, set[str]]:\n", + " \"\"\"LIST ONLY: enumerate MLflow mask filenames via list_artifacts, without downloading.\"\"\"\n", + " mlflow_files: dict[str, set[str]] = {}\n", + " for mask_name, uri in mlflow_masks.items():\n", + " found = {\n", + " Path(info.path).name\n", + " for info in list_artifacts(artifact_uri=uri)\n", + " if not info.is_dir\n", + " }\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files listed (no download) at {uri}\")\n", + " return mlflow_files\n", + "\n", + "\n", + "def download_mlflow_masks(mlflow_masks=MLFLOW_MASKS) -> tuple[dict[str, Path], dict[str, set[str]]]:\n", + " \"\"\"DOWNLOAD: fetch MLflow mask dirs and enumerate them. Returns (dirs, files).\"\"\"\n", + " mlflow_dirs: dict[str, Path] = {}\n", + " mlflow_files: dict[str, set[str]] = {}\n", + " for mask_name, uri in mlflow_masks.items():\n", + " mask_dir = Path(download_artifacts(uri))\n", + " mlflow_dirs[mask_name] = mask_dir\n", + " found = {p.name for p in mask_dir.iterdir() if p.is_file()}\n", + " mlflow_files[mask_name] = found\n", + " print(f\"{len(found)} {mask_name} mask files found in {mask_dir}\")\n", + " return mlflow_dirs, mlflow_files" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B4. QC masks (local): list ----\n", + "def list_qc_masks(qc_paths=QC_MASKS_PATHS, prefixes=QC_MASK_PREFIXES) -> dict[str, set[str]]:\n", + " \"\"\"LIST ONLY: enumerate local QC mask filenames (prefix-stripped), without opening them.\n", + "\n", + " Accepts one or more directories; all are listed and merged, as if every mask lived in a\n", + " single location. A single Path is also accepted for convenience.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " qc_files: dict[str, set[str]] = {mask_type: set() for mask_type in prefixes}\n", + " for qc_path in qc_paths:\n", + " for mask_type, prefix in prefixes.items():\n", + " found = {\n", + " p.name.removeprefix(prefix)\n", + " for p in qc_path.iterdir()\n", + " if p.is_file() and p.name.startswith(prefix)\n", + " }\n", + " qc_files[mask_type] |= found\n", + "\n", + " for mask_type, found in qc_files.items():\n", + " prefix = prefixes[mask_type]\n", + " print(f\"{len(found)} {mask_type} mask files found (prefix: {prefix})\")\n", + " return qc_files\n", + "\n", + "\n", + "def _find_qc_mask(filename: str, qc_paths) -> Path | None:\n", + " \"\"\"Return the full path of a QC mask file across the given dirs, or None if not found.\"\"\"\n", + " for qc_path in qc_paths:\n", + " candidate = Path(qc_path) / filename\n", + " if candidate.is_file():\n", + " return candidate\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B6. Coverage report + summary + extras ----\n", + "DATA_MAPPING_COLUMNS = [\"record_num\", \"mammaprint_index\", \"type\", \"path\"]\n", + "\n", + "\n", + "def build_coverage_report(\n", + " expected_names: set[str],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + ") -> pd.DataFrame:\n", + " \"\"\"Per-slide presence table across all mask types, plus an `all_present` column.\"\"\"\n", + " rows = []\n", + " for name in sorted(expected_names):\n", + " row = {\"tiff_name\": name}\n", + " for mask_name in mlflow_files:\n", + " row[mask_name] = name in mlflow_files[mask_name]\n", + " for mask_type in qc_files:\n", + " row[mask_type] = name in qc_files[mask_type]\n", + " rows.append(row)\n", + "\n", + " mask_cols = [*mlflow_files.keys(), *qc_files.keys()]\n", + " report = pd.DataFrame(rows)\n", + " report[\"all_present\"] = report[mask_cols].all(axis=1)\n", + " return report\n", + "\n", + "\n", + "def coverage_summary(report: pd.DataFrame, expected_names: set[str]) -> pd.DataFrame:\n", + " \"\"\"Found/missing/total counts per mask type.\"\"\"\n", + " mask_cols = [c for c in report.columns if c not in (\"tiff_name\", \"all_present\")]\n", + " summary = report[mask_cols].sum().to_frame(\"found\")\n", + " summary[\"missing\"] = len(expected_names) - summary[\"found\"]\n", + " summary[\"total\"] = len(expected_names)\n", + " return summary\n", + "\n", + "\n", + "def slides_missing_masks(report: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Rows for slides missing at least one mask.\"\"\"\n", + " missing = report[~report[\"all_present\"]].drop(columns=[\"all_present\"])\n", + " print(f\"{len(missing)} slides missing at least one mask.\")\n", + " return missing\n", + "\n", + "\n", + "def missing_mask_data_mapping(\n", + " df: pd.DataFrame,\n", + " report: pd.DataFrame,\n", + " mask_types: str | list[str],\n", + " out_csv: str | Path | None = None,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Return a data_mapping.csv-style frame of the slides MISSING the given mask type(s).\n", + "\n", + " ``mask_types`` is a mask column name from the report (e.g. \"epithelium\") or a list of them;\n", + " a row is included if it is missing ANY of the requested masks. The result keeps only the\n", + " original data_mapping columns. If ``out_csv`` is given, it is also written there.\n", + " \"\"\"\n", + " if isinstance(mask_types, str):\n", + " mask_types = [mask_types]\n", + "\n", + " unknown = [m for m in mask_types if m not in report.columns]\n", + " if unknown:\n", + " raise KeyError(f\"Unknown mask type(s) {unknown}. Available: \"\n", + " f\"{[c for c in report.columns if c not in ('tiff_name', 'all_present')]}\")\n", + "\n", + " # tiff_names missing at least one of the requested masks\n", + " missing_mask = ~report[mask_types].all(axis=1)\n", + " missing_names = set(report.loc[missing_mask, \"tiff_name\"])\n", + "\n", + " out = df[df[\"tiff_name\"].isin(missing_names)]\n", + " keep = [c for c in DATA_MAPPING_COLUMNS if c in out.columns]\n", + " out = out[keep].reset_index(drop=True)\n", + "\n", + " print(f\"{len(out)} slides missing {mask_types}.\")\n", + " if out_csv is not None:\n", + " out.to_csv(out_csv, index=False)\n", + " print(f\"Wrote {out_csv}\")\n", + " return out\n", + "\n", + "\n", + "def extra_masks(\n", + " expected_names: set[str],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + ") -> None:\n", + " \"\"\"Print mask files that have no matching slide in data_mapping.\"\"\"\n", + " for mask_name in mlflow_files:\n", + " extra = mlflow_files[mask_name] - expected_names\n", + " print(f\"{len(extra)} extra {mask_name} masks with no matching slide in data_mapping:\")\n", + " if extra:\n", + " print(sorted(extra))\n", + " print()\n", + "\n", + " for mask_type in qc_files:\n", + " extra = qc_files[mask_type] - expected_names\n", + " print(f\"{len(extra)} extra {mask_type} masks with no matching slide:\")\n", + " if extra:\n", + " print(sorted(extra))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "im0bfkx60ei", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B5. Mask readability check (open each existing mask file) ----\n", + "def _try_open_mask(path: Path) -> str | None:\n", + " \"\"\"Try to open a mask file. Returns None on success or an error message.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path):\n", + " pass\n", + " else:\n", + " with OpenSlide(str_path):\n", + " pass\n", + " return None\n", + " except Exception as e:\n", + " return f\"{type(e).__name__}: {e}\"\n", + "\n", + "\n", + "def open_masks(\n", + " expected_names: set[str],\n", + " mlflow_dirs: dict[str, Path],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + " qc_paths=QC_MASKS_PATHS,\n", + " prefixes=QC_MASK_PREFIXES,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Open each existing mask file to verify readability. Returns a DataFrame of failures.\n", + "\n", + " Pass MLflow dirs/files from ``download_mlflow_masks`` (opening implies a prior download).\n", + " Pass QC files from ``list_qc_masks`` (already local; may span multiple ``qc_paths``, which are\n", + " searched in order). Any group may be an empty dict to skip it.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " open_errors: list[dict[str, str]] = []\n", + "\n", + " # MLflow masks (require downloaded dirs)\n", + " for mask_name in mlflow_files:\n", + " mask_dir = mlflow_dirs[mask_name]\n", + " for name in sorted(expected_names & mlflow_files[mask_name]):\n", + " err = _try_open_mask(mask_dir / name)\n", + " if err:\n", + " open_errors.append({\"mask_type\": mask_name, \"tiff_name\": name, \"error\": err})\n", + "\n", + " # QC masks (local; located across qc_paths)\n", + " for mask_type in qc_files:\n", + " prefix = prefixes[mask_type]\n", + " for name in sorted(expected_names & qc_files[mask_type]):\n", + " path = _find_qc_mask(f\"{prefix}{name}\", qc_paths)\n", + " if path is None:\n", + " open_errors.append(\n", + " {\"mask_type\": mask_type, \"tiff_name\": name, \"error\": \"FileNotFound in qc_paths\"}\n", + " )\n", + " continue\n", + " err = _try_open_mask(path)\n", + " if err:\n", + " open_errors.append({\"mask_type\": mask_type, \"tiff_name\": name, \"error\": err})\n", + "\n", + " errors_df = pd.DataFrame(open_errors)\n", + " if errors_df.empty:\n", + " print(\"All checked mask files opened successfully.\")\n", + " else:\n", + " print(f\"{len(errors_df)} mask files failed to open.\")\n", + " return errors_df" + ] + }, + { + "cell_type": "markdown", + "id": "53d0f240", + "metadata": {}, + "source": [ + "## Visual inspection (function)\n", + "\n", + "`plot_sample_masks(...)` displays N sample masks for each mask type (MLflow + QC) in a grid.\n", + "Requires downloaded MLflow dirs (from `download_mlflow_masks`)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f0b337bd", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B7. Visual inspection ----\n", + "THUMB_SIZE = (512, 512)\n", + "\n", + "\n", + "def _read_mask_thumbnail(path: Path, thumb_size=THUMB_SIZE) -> np.ndarray | None:\n", + " \"\"\"Read a mask file and return a thumbnail as a numpy array.\"\"\"\n", + " try:\n", + " str_path = str(path)\n", + " if str_path.lower().endswith((\".ome.tiff\", \".ome.tif\")):\n", + " with TiffFile(str_path) as tif:\n", + " page = tif.pages[0]\n", + " # Read full page then downsample via slicing (nearest-neighbor)\n", + " img = page.asarray()\n", + " step_y = max(1, img.shape[0] // thumb_size[1])\n", + " step_x = max(1, img.shape[1] // thumb_size[0])\n", + " return img[::step_y, ::step_x]\n", + " else:\n", + " with OpenSlide(str_path) as slide:\n", + " thumb = slide.get_thumbnail(thumb_size)\n", + " return np.array(thumb)\n", + " except Exception as e:\n", + " print(f\" Could not read {path.name}: {e}\")\n", + " return None\n", + "\n", + "\n", + "def plot_sample_masks(\n", + " expected_names: set[str],\n", + " mlflow_dirs: dict[str, Path],\n", + " mlflow_files: dict[str, set[str]],\n", + " qc_files: dict[str, set[str]],\n", + " n_samples: int = 5,\n", + " qc_paths=QC_MASKS_PATHS,\n", + " prefixes=QC_MASK_PREFIXES,\n", + ") -> None:\n", + " \"\"\"Display n_samples sample masks per type (MLflow + QC) in a grid.\n", + "\n", + " QC masks are located across ``qc_paths`` (searched in order), so masks spread over several\n", + " directories are handled as if they lived in a single location.\n", + " \"\"\"\n", + " if isinstance(qc_paths, (str, Path)):\n", + " qc_paths = [qc_paths]\n", + " qc_paths = [Path(p) for p in qc_paths]\n", + "\n", + " # Collect mask paths: list of (mask_type, [paths])\n", + " all_mask_types: list[tuple[str, list[Path]]] = []\n", + " sample_names = sorted(expected_names)[:n_samples]\n", + "\n", + " for mask_name in mlflow_files:\n", + " paths = [mlflow_dirs[mask_name] / name for name in sample_names if name in mlflow_files[mask_name]]\n", + " all_mask_types.append((mask_name, paths[:n_samples]))\n", + "\n", + " for mask_type in qc_files:\n", + " prefix = prefixes[mask_type]\n", + " paths = [\n", + " found\n", + " for name in sample_names\n", + " if name in qc_files[mask_type]\n", + " and (found := _find_qc_mask(f\"{prefix}{name}\", qc_paths)) is not None\n", + " ]\n", + " all_mask_types.append((mask_type, paths[:n_samples]))\n", + "\n", + " # Plot grid: rows = mask types, cols = samples\n", + " n_rows = len(all_mask_types)\n", + " n_cols = n_samples\n", + " fig, axes = plt.subplots(n_rows, n_cols, figsize=(3 * n_cols, 3 * n_rows))\n", + " if n_rows == 1:\n", + " axes = [axes]\n", + "\n", + " for row_idx, (mask_type, paths) in enumerate(all_mask_types):\n", + " for col_idx in range(n_cols):\n", + " ax = axes[row_idx][col_idx]\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " if col_idx == 0:\n", + " ax.set_ylabel(mask_type, fontsize=12, fontweight=\"bold\")\n", + " if col_idx < len(paths):\n", + " thumb = _read_mask_thumbnail(paths[col_idx])\n", + " if thumb is not None:\n", + " ax.imshow(thumb, cmap=\"gray\")\n", + " ax.set_title(paths[col_idx].name, fontsize=7)\n", + " else:\n", + " ax.text(0.5, 0.5, \"read error\", ha=\"center\", va=\"center\", transform=ax.transAxes)\n", + " else:\n", + " ax.text(0.5, 0.5, \"N/A\", ha=\"center\", va=\"center\", transform=ax.transAxes, color=\"gray\")\n", + "\n", + " fig.suptitle(f\"Sample masks ({n_samples} per type)\", fontsize=14, fontweight=\"bold\")\n", + " fig.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e5a23332", + "metadata": {}, + "source": [ + "## Tiled dataset validation (functions)\n", + "\n", + "`download_tiled_datasets(...)` downloads `slides.parquet` / `tiles.parquet` per dataset.\n", + "`validate_tiled_datasets(df, tiled_data)` checks:\n", + "1. Every slide in `data_mapping.csv` has a row in `slides.parquet` (and vice versa)\n", + "2. Every slide in `slides.parquet` has at least one tile in `tiles.parquet`\n", + "3. Every `slide_id` in `tiles.parquet` maps to a slide in `slides.parquet`" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "fe9ffadc", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B8a. Tiled datasets ----\n", + "def download_tiled_datasets(tiled=TILED_DATASETS) -> dict[str, tuple[pd.DataFrame, pd.DataFrame]]:\n", + " \"\"\"Download slides.parquet / tiles.parquet per dataset. Returns {name: (slides, tiles)}.\"\"\"\n", + " tiled_data: dict[str, tuple[pd.DataFrame, pd.DataFrame]] = {}\n", + " for ds_name, uri in tiled.items():\n", + " local_dir = Path(download_artifacts(uri))\n", + " slides = pd.read_parquet(local_dir / \"slides.parquet\")\n", + " tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n", + " tiled_data[ds_name] = (slides, tiles)\n", + " print(f\"{ds_name}: {len(slides)} slides, {len(tiles)} tiles\")\n", + " return tiled_data\n", + "\n", + "\n", + "def validate_tiled_datasets(df: pd.DataFrame, tiled_data: dict) -> None:\n", + " \"\"\"Check slides.parquet <-> data_mapping and tiles.parquet <-> slides.parquet consistency.\"\"\"\n", + " expected_stems = {Path(p).stem for p in df[\"path\"]}\n", + "\n", + " for ds_name, (slides, tiles) in tiled_data.items():\n", + " print(f\"=== {ds_name} ===\\n\")\n", + "\n", + " # 1. slides.parquet path stem <-> data_mapping path stem\n", + " slide_stems = {Path(p).stem for p in slides[\"path\"]}\n", + " missing_from_dataset = expected_stems - slide_stems\n", + " extra_in_dataset = slide_stems - expected_stems\n", + "\n", + " print(f\"Slides in data_mapping: {len(expected_stems)}\")\n", + " print(f\"Slides in dataset: {len(slide_stems)}\")\n", + " print(f\"Missing from dataset: {len(missing_from_dataset)}\")\n", + " print(f\"Extra in dataset: {len(extra_in_dataset)}\")\n", + "\n", + " if missing_from_dataset:\n", + " print(f\"\\nSlides in data_mapping but NOT in {ds_name}:\")\n", + " for p in sorted(missing_from_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " if extra_in_dataset:\n", + " print(f\"\\nSlides in {ds_name} but NOT in data_mapping:\")\n", + " for p in sorted(extra_in_dataset):\n", + " print(f\" {p}\")\n", + "\n", + " # 2. Every slide has at least one tile\n", + " slide_ids = set(slides[\"id\"])\n", + " slide_ids_in_tiles = set(tiles[\"slide_id\"])\n", + "\n", + " slides_without_tiles = slide_ids - slide_ids_in_tiles\n", + " print(f\"\\nSlides without any tiles: {len(slides_without_tiles)}\")\n", + " if slides_without_tiles:\n", + " for s in sorted(slides_without_tiles):\n", + " print(f\" {s}\")\n", + "\n", + " # 3. No orphan tiles (tile references a slide not in slides.parquet)\n", + " orphan_tile_ids = slide_ids_in_tiles - slide_ids\n", + " print(f\"Orphan tile slide_ids: {len(orphan_tile_ids)}\")\n", + " if orphan_tile_ids:\n", + " for sid in sorted(orphan_tile_ids):\n", + " count = int((tiles[\"slide_id\"] == sid).sum())\n", + " print(f\" slide_id={sid}: {count} tiles\")\n", + "\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "ea2eae31", + "metadata": {}, + "source": [ + "## Embeddings validation (functions)\n", + "\n", + "`list_embeddings(df)` checks coverage using `list_artifacts` (no download).\n", + "`validate_embeddings(df)` downloads the artifact dir and checks the same, for parity with the\n", + "download+open mask mode. Both verify every slide in `data_mapping.csv` has a matching `.parquet` file." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cc5ce48a", + "metadata": {}, + "outputs": [], + "source": [ + "# ---- B8b. Embeddings: two modes ----\n", + "# Embedding parquet files are named as \"{record_num}.parquet\" (e.g. \"2023/00835.parquet\"),\n", + "# i.e. the stem matches the data_mapping `path` column.\n", + "def _report_embeddings(emb_name: str, emb_files: set[str], expected_record_nums: set[str]) -> None:\n", + " print(f\"=== {emb_name} ===\\n\")\n", + " print(f\"Embedding files found: {len(emb_files)}\")\n", + " print(f\"Slides in data_mapping: {len(expected_record_nums)}\")\n", + "\n", + " missing = expected_record_nums - emb_files\n", + " extra = emb_files - expected_record_nums\n", + "\n", + " print(f\"Missing embeddings: {len(missing)}\")\n", + " if missing:\n", + " for s in sorted(missing):\n", + " print(f\" {s}\")\n", + "\n", + " print(f\"Extra embeddings: {len(extra)}\")\n", + " if extra:\n", + " for s in sorted(extra):\n", + " print(f\" {s}\")\n", + " print()\n", + "\n", + "\n", + "def list_embeddings(df: pd.DataFrame, embeddings=EMBEDDINGS) -> None:\n", + " \"\"\"LIST ONLY: check embedding coverage via list_artifacts, without downloading.\"\"\"\n", + " expected_record_nums = set(df[\"path\"])\n", + " for emb_name, uri in embeddings.items():\n", + " emb_files = {\n", + " Path(info.path).stem\n", + " for info in list_artifacts(artifact_uri=uri)\n", + " if not info.is_dir and info.path.endswith(\".parquet\")\n", + " }\n", + " _report_embeddings(emb_name, emb_files, expected_record_nums)\n", + "\n", + "\n", + "def validate_embeddings(df: pd.DataFrame, embeddings=EMBEDDINGS) -> None:\n", + " \"\"\"DOWNLOAD: fetch the embeddings dir and check coverage against data_mapping.\"\"\"\n", + " expected_record_nums = set(df[\"path\"])\n", + " for emb_name, uri in embeddings.items():\n", + " emb_dir = Path(download_artifacts(uri))\n", + " emb_files = {p.stem for p in emb_dir.iterdir() if p.is_file() and p.suffix == \".parquet\"}\n", + " _report_embeddings(emb_name, emb_files, expected_record_nums)" + ] + }, + { + "cell_type": "markdown", + "id": "fd9a315e", + "metadata": {}, + "source": [ + "# Calls\n", + "\n", + "Run the cells below to execute individual checks. **Comment out whatever you don't need.**\n", + "\n", + "The cheap, network-light path is left uncommented by default (list-only mask enumeration + coverage\n", + "report). The download / open / visual / tiled / embeddings cells are commented out — uncomment them\n", + "when you want to actually fetch and open the files to verify everything works end-to-end." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a9b7a86c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2042 slides from /mnt/projects/mammaprint/data_mapping.csv\n", + "Columns: ['record_num', 'mammaprint_index', 'type', 'path', 'split', 'tiff_name']\n", + "2042 unique slide tiff names expected\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepathsplittiff_name
02023/00835-0.158b luminal/mnt/data/MOU/breast/mammaprint/P2023_0835testP2023_0835.tiff
12023/008360.058a luminal/mnt/data/MOU/breast/mammaprint/P2023_0836trainP2023_0836.tiff
22023/00837-0.039b luminal/mnt/data/MOU/breast/mammaprint/P2023_0837trainP2023_0837.tiff
32023/008380.065a luminal/mnt/data/MOU/breast/mammaprint/P2023_0838testP2023_0838.tiff
42023/00839-0.429b luminal/mnt/data/MOU/breast/mammaprint/P2023_0839testP2023_0839.tiff
\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2023/00835 -0.158 b luminal \n", + "1 2023/00836 0.058 a luminal \n", + "2 2023/00837 -0.039 b luminal \n", + "3 2023/00838 0.065 a luminal \n", + "4 2023/00839 -0.429 b luminal \n", + "\n", + " path split tiff_name \n", + "0 /mnt/data/MOU/breast/mammaprint/P2023_0835 test P2023_0835.tiff \n", + "1 /mnt/data/MOU/breast/mammaprint/P2023_0836 train P2023_0836.tiff \n", + "2 /mnt/data/MOU/breast/mammaprint/P2023_0837 train P2023_0837.tiff \n", + "3 /mnt/data/MOU/breast/mammaprint/P2023_0838 test P2023_0838.tiff \n", + "4 /mnt/data/MOU/breast/mammaprint/P2023_0839 test P2023_0839.tiff " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# --- Load data mapping (required by everything below) ---\n", + "df, expected_names = load_data_mapping()\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "533d7fbf", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Slides open check (verifies every slide file in data_mapping opens) ---\n", + "# slide_errors = check_slides_open(df)\n", + "# display(slide_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "23f1e1ad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2042 tissue mask files listed (no download) at mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks\n", + "310 epithelium mask files listed (no download) at mlflow-artifacts:/3/0176fb9485de49699b1e16f362cff5fa/artifacts/epithelium_masks\n" + ] + } + ], + "source": [ + "# --- MLflow masks: LIST ONLY (no download) ---\n", + "mlflow_files = list_mlflow_masks()\n", + "\n", + "# --- MLflow masks: DOWNLOAD + OPEN (uncomment to actually fetch & verify readability) ---\n", + "# mlflow_dirs, mlflow_files = download_mlflow_masks()\n", + "# mlflow_open_errors = open_masks(expected_names, mlflow_dirs, mlflow_files, {})\n", + "# display(mlflow_open_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "2df7d0f5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1800 blur mask files found (prefix: Piqe_piqe_median_activity_mask_)\n", + "1800 folding mask files found (prefix: FoldingFunction_folding_test_)\n", + "1800 residual mask files found (prefix: ResidualArtifactsAndCoverage_coverage_mask_)\n" + ] + } + ], + "source": [ + "# --- QC masks: LIST ONLY (local ls, no open) ---\n", + "qc_files = list_qc_masks()\n", + "\n", + "# --- QC masks: OPEN (uncomment to verify each local QC mask is readable) ---\n", + "# qc_open_errors = open_masks(expected_names, {}, {}, qc_files)\n", + "# display(qc_open_errors)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "7073481f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tiff_nametissueepitheliumblurfoldingresidualall_present
02024_02851-1.tiffTrueTrueTrueTrueTrueTrue
12024_02865.tiffTrueFalseTrueTrueTrueFalse
22024_02890.tiffTrueFalseTrueTrueTrueFalse
3P2023_01000.tiffTrueFalseTrueTrueTrueFalse
4P2023_01001.tiffTrueFalseTrueTrueTrueFalse
........................
2037P2026_00970.tiffTrueFalseFalseFalseFalseFalse
2038P2026_00973.tiffTrueFalseFalseFalseFalseFalse
2039P2026_00974.tiffTrueFalseFalseFalseFalseFalse
2040P2026_00976.tiffTrueFalseFalseFalseFalseFalse
2041P2026_00977.tiffTrueFalseFalseFalseFalseFalse
\n", + "

2042 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " tiff_name tissue epithelium blur folding residual \\\n", + "0 2024_02851-1.tiff True True True True True \n", + "1 2024_02865.tiff True False True True True \n", + "2 2024_02890.tiff True False True True True \n", + "3 P2023_01000.tiff True False True True True \n", + "4 P2023_01001.tiff True False True True True \n", + "... ... ... ... ... ... ... \n", + "2037 P2026_00970.tiff True False False False False \n", + "2038 P2026_00973.tiff True False False False False \n", + "2039 P2026_00974.tiff True False False False False \n", + "2040 P2026_00976.tiff True False False False False \n", + "2041 P2026_00977.tiff True False False False False \n", + "\n", + " all_present \n", + "0 True \n", + "1 False \n", + "2 False \n", + "3 False \n", + "4 False \n", + "... ... \n", + "2037 False \n", + "2038 False \n", + "2039 False \n", + "2040 False \n", + "2041 False \n", + "\n", + "[2042 rows x 7 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- Coverage report (per-slide presence across all mask types) ---\n", + "report = build_coverage_report(expected_names, mlflow_files, qc_files)\n", + "display(report)\n", + "\n", + "# --- Summary / missing / extras (uncomment as needed) ---\n", + "# display(coverage_summary(report, expected_names))\n", + "# display(slides_missing_masks(report))\n", + "# extra_masks(expected_names, mlflow_files, qc_files)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "47dac634", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "243 slides missing ['blur', 'folding', 'residual'].\n", + "Wrote data_mapping_missing_qc.csv\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
record_nummammaprint_indextypepath
02024/04733-0.122b luminal/mnt/data/MOU/breast/mammaprint/P2024_04733
12025/001610.263a luminal/mnt/data/MOU/breast/mammaprint/P2025_00161-2
22025/04084-0.215b luminal/mnt/data/MOU/breast/mammaprint/P2025_04084
32025/04262-0.023b luminal/mnt/data/MOU/breast/mammaprint/P2025_04262
42025/04267-0.309b luminal/mnt/data/MOU/breast/mammaprint/P2025_04267
...............
2382026/00970-0.218b luminal/mnt/data/MOU/breast/mammaprint/P2026_00970
2392026/00973-0.141b luminal/mnt/data/MOU/breast/mammaprint/P2026_00973
2402026/009740.192a luminal/mnt/data/MOU/breast/mammaprint/P2026_00974
2412026/009760.396a luminal/mnt/data/MOU/breast/mammaprint/P2026_00976
2422026/00977-0.065b luminal/mnt/data/MOU/breast/mammaprint/P2026_00977
\n", + "

243 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " record_num mammaprint_index type \\\n", + "0 2024/04733 -0.122 b luminal \n", + "1 2025/00161 0.263 a luminal \n", + "2 2025/04084 -0.215 b luminal \n", + "3 2025/04262 -0.023 b luminal \n", + "4 2025/04267 -0.309 b luminal \n", + ".. ... ... ... \n", + "238 2026/00970 -0.218 b luminal \n", + "239 2026/00973 -0.141 b luminal \n", + "240 2026/00974 0.192 a luminal \n", + "241 2026/00976 0.396 a luminal \n", + "242 2026/00977 -0.065 b luminal \n", + "\n", + " path \n", + "0 /mnt/data/MOU/breast/mammaprint/P2024_04733 \n", + "1 /mnt/data/MOU/breast/mammaprint/P2025_00161-2 \n", + "2 /mnt/data/MOU/breast/mammaprint/P2025_04084 \n", + "3 /mnt/data/MOU/breast/mammaprint/P2025_04262 \n", + "4 /mnt/data/MOU/breast/mammaprint/P2025_04267 \n", + ".. ... \n", + "238 /mnt/data/MOU/breast/mammaprint/P2026_00970 \n", + "239 /mnt/data/MOU/breast/mammaprint/P2026_00973 \n", + "240 /mnt/data/MOU/breast/mammaprint/P2026_00974 \n", + "241 /mnt/data/MOU/breast/mammaprint/P2026_00976 \n", + "242 /mnt/data/MOU/breast/mammaprint/P2026_00977 \n", + "\n", + "[243 rows x 4 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# --- Export a data_mapping.csv-style file of slides MISSING a chosen mask ---\n", + "# Set mask_types to the mask column you care about, e.g. \"epithelium\", \"tissue\",\n", + "# \"blur\", \"folding\", \"residual\" (or a list of several -> missing ANY of them).\n", + "missing_epithelium = missing_mask_data_mapping(\n", + " df, report, mask_types=[\"blur\", \"folding\", \"residual\"], out_csv=\"data_mapping_missing_qc.csv\"\n", + ")\n", + "display(missing_epithelium)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "b360e3bd", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Visual grid: sample masks per type (needs downloaded MLflow dirs; uncomment to run) ---\n", + "# mlflow_dirs, mlflow_files = download_mlflow_masks()\n", + "# plot_sample_masks(expected_names, mlflow_dirs, mlflow_files, qc_files)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "3e10cb93", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Tiled datasets (downloads slides/tiles parquet; uncomment to run) ---\n", + "# tiled_data = download_tiled_datasets()\n", + "# validate_tiled_datasets(df, tiled_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "d91e48a5", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Embeddings ---\n", + "# list_embeddings(df) # LIST ONLY (no download)\n", + "# validate_embeddings(df) # DOWNLOAD + check coverage" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 12628bcf423a39e2cd4a7a7b9fd838f70236e375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 13 Jul 2026 19:01:20 +0200 Subject: [PATCH 38/79] feat: mask thresholds --- .../notebooks/mask_coverage_threshold.ipynb | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 scripts/notebooks/mask_coverage_threshold.ipynb diff --git a/scripts/notebooks/mask_coverage_threshold.ipynb b/scripts/notebooks/mask_coverage_threshold.ipynb new file mode 100644 index 00000000..5b7af36f --- /dev/null +++ b/scripts/notebooks/mask_coverage_threshold.ipynb @@ -0,0 +1,173 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6eb94fd9", + "metadata": {}, + "source": [ + "# Mask coverage → pick a filtering threshold\n", + "\n", + "The tiling pipeline (`preprocessing/tiling.py`) already computes a **per-tile coverage score** for\n", + "each mask type and stores it in the tiled dataset's `tiles.parquet` as these columns:\n", + "`tissue_overlap`, `epithelium_overlap`, `blur_overlap`, `folding_overlap`, `residual_overlap`.\n", + "Each is in `[0, 1]` = fraction of desired (non-artifact) pixels; tiles are kept/dropped by\n", + "thresholding these.\n", + "\n", + "This notebook loads those columns straight from a tiled-dataset MLflow artifact and shows their\n", + "**distribution + CDF** so you can pick sensible thresholds.\n", + "\n", + "> **Note on \"AUC\".** A true ROC/AUC needs a per-tile keep/discard ground truth, which we don't have.\n", + "> Instead we plot the **coverage distribution** and its **CDF** — the CDF axis directly answers\n", + "> *\"if I set the threshold at T, what fraction of tiles do I keep vs drop?\"*, which is exactly the\n", + "> decision you want to make. An optional luminal A/B overlay lets you check the two classes behave\n", + "> similarly under a given cutoff.\n", + "\n", + "No masks or OpenSlide access needed — the overlaps are read as-is from the parquet, so the numbers\n", + "are exactly what the pipeline produced.\n", + "\n", + "Edit the **Config** cell (tiled-dataset URI + which overlaps to look at), then run the **Calls** cells." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73e2c71a", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m ensurepip --upgrade\n", + "!python -m pip install \"pandas\" \"mlflow>3\" pyarrow matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6472a2b0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mlflow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from mlflow.artifacts import download_artifacts" + ] + }, + { + "cell_type": "markdown", + "id": "4a898180", + "metadata": {}, + "source": [ + "# Config\n", + "\n", + "This is the one cell you edit. Point `TILED_DATASET_URI` at the tiling run's MLflow artifact dir\n", + "(the one containing `slides.parquet` / `tiles.parquet`), and pick which overlaps to plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37f50707", + "metadata": {}, + "outputs": [], + "source": "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\nos.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\nmlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n\n# Tiled-dataset artifact dir (contains slides.parquet + tiles.parquet). Same style as the\n# TILED_DATASETS entries in check_mask_coverage.ipynb. Point this at your dataset.\nTILED_DATASET_URI = \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\"\n\n# The overlap columns to analyze (as stored in tiles.parquet). Drop any you don't care about.\nOVERLAP_COLUMNS = [\n \"tissue_overlap\",\n \"epithelium_overlap\",\n \"blur_overlap\",\n \"folding_overlap\",\n \"residual_overlap\",\n]\n\n# Columns whose stored values are inverted relative to the convention you want\n# (\"fraction of DESIRED pixels\", where higher = cleaner/keep). For these, the notebook plots\n# `1 - overlap` everywhere (loaded once, so summary + plots stay consistent).\n# NOTE: this only relabels the interpretation for viewing; it does NOT change the parquet.\n# If an upstream struct_field is genuinely wrong, fix it in preprocessing/tiling.py instead.\nINVERT_COLUMNS = [\"folding_overlap\", \"residual_overlap\"]\n\n# Column holding the luminal type on each tile row (for the optional A/B overlay). Set to None\n# to disable the per-class split. tiling.py copies `type` onto every tile.\nTYPE_COLUMN = \"type\"" + }, + { + "cell_type": "markdown", + "id": "992db56f", + "metadata": {}, + "source": [ + "# Functions\n", + "\n", + "Run every cell in this section once, then use the **Calls** section at the bottom." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e128468", + "metadata": {}, + "outputs": [], + "source": "# ---- Load tiles.parquet from a tiled-dataset MLflow artifact ----\n# type_label mirrors ml/data/datasets/labels.py:_map_luminal_type (a luminal -> 1, b luminal -> 0)\n_LUMINAL_MAP = {\"a luminal\": 1, \"b luminal\": 0}\n\n\ndef load_tiles(uri=TILED_DATASET_URI, overlap_columns=OVERLAP_COLUMNS,\n invert_columns=INVERT_COLUMNS, type_column=TYPE_COLUMN):\n \"\"\"Download the tiled dataset's tiles.parquet and return it as a DataFrame.\n\n Any column in `invert_columns` is replaced with `1 - value` (fixes overlaps stored with the\n opposite convention, so higher = cleaner/keep everywhere). Adds `type_label` (luminal A=1/B=0)\n if `type_column` is present. Warns about requested overlap columns missing from the parquet.\n \"\"\"\n local_dir = Path(download_artifacts(uri))\n tiles = pd.read_parquet(local_dir / \"tiles.parquet\")\n print(f\"Loaded {len(tiles):,} tiles from {local_dir / 'tiles.parquet'}\")\n print(f\"Columns: {tiles.columns.tolist()}\")\n\n missing = [c for c in overlap_columns if c not in tiles.columns]\n if missing:\n print(f\"WARNING: requested overlap columns not in parquet: {missing}\")\n\n for col_name in invert_columns:\n if col_name in tiles.columns:\n tiles[col_name] = 1.0 - tiles[col_name]\n print(f\"Inverted '{col_name}' -> plotting 1 - {col_name}\")\n\n if type_column and type_column in tiles.columns:\n tiles[\"type_label\"] = tiles[type_column].str.strip().str.lower().map(_LUMINAL_MAP)\n print(tiles[type_column].value_counts(dropna=False).to_string())\n return tiles" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8b0baa0", + "metadata": {}, + "outputs": [], + "source": "# ---- Per-column quantile table (read cutoffs straight off it) ----\ndef summary_table(tiles, overlap_columns=OVERLAP_COLUMNS):\n \"\"\"Count + quantiles of each overlap column.\n\n Includes fine LOW-end quantiles (1/2/5/10%) because for right-skewed overlaps the useful\n threshold sits in the bottom tail: e.g. the 5% row tells you the cutoff that drops ~5% of tiles.\n \"\"\"\n qs = [0.0, 0.01, 0.02, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0]\n cols = [c for c in overlap_columns if c in tiles.columns]\n out = (\n tiles[cols]\n .describe(percentiles=qs)\n .drop(index=[\"mean\", \"std\"], errors=\"ignore\")\n .T\n )\n return out" + }, + { + "cell_type": "code", + "execution_count": null, + "id": "428367e0", + "metadata": {}, + "outputs": [], + "source": "# ---- Plot: coverage histogram + CDF, with candidate thresholds ----\n_CLASS_NAMES = {1: \"a luminal\", 0: \"b luminal\"}\n\n\ndef auto_xlim(values, tail=0.02, pad=0.01):\n \"\"\"Zoom window that frames the informative tail of a skewed distribution.\n\n Returns (lo, hi) spanning the `tail` quantile (e.g. 2%) up to the max, padded a little. When\n almost every tile has coverage ~1.0, this zooms in near 1.0 where the threshold decision lives.\n Falls back to the full (0, 1) range for empty/all-NaN or degenerate (all-identical) input.\n \"\"\"\n values = np.asarray(pd.Series(values).dropna())\n if values.size == 0:\n return (0.0, 1.0)\n lo = float(np.quantile(values, tail))\n hi = float(values.max())\n lo = max(0.0, lo - pad)\n hi = min(1.0, hi + pad)\n if hi - lo < 1e-6: # degenerate (all identical) -> fall back to full range\n return (0.0, 1.0)\n return (lo, hi)\n\n\ndef plot_overlap(tiles, column, thresholds=(0.1, 0.25, 0.5), by_class=True,\n bins=50, xlim=(0.0, 1.0)):\n \"\"\"Histogram (left) + CDF (right) of one overlap column.\n\n The CDF reads directly as 'fraction of tiles dropped if threshold = x'; keep-fraction is\n printed for each candidate threshold. Optional luminal A/B overlay.\n\n `xlim` sets the x-axis window (bins + histogram range track it), so for heavily skewed\n columns you can zoom toward 1.0. Pass `xlim=auto_xlim(tiles[column].dropna())` to auto-frame\n the tail. The keep/drop stats below always use the FULL data, regardless of the zoom window.\n \"\"\"\n if column not in tiles.columns:\n print(f\"Column '{column}' not in tiles.\")\n return\n values_all = tiles[column].dropna()\n if values_all.empty:\n print(f\"No values for '{column}' (empty or all-NaN) -- skipping.\")\n return\n\n lo, hi = xlim\n fig, (ax_hist, ax_cdf) = plt.subplots(1, 2, figsize=(13, 4.5))\n\n def _plot_series(values, label):\n values = np.asarray(values)\n ax_hist.hist(values, bins=bins, range=(lo, hi), alpha=0.5, density=True, label=label)\n xs = np.sort(values)\n ys = np.arange(1, len(xs) + 1) / len(xs) # P(overlap <= x) = fraction dropped\n ax_cdf.plot(xs, ys, label=label)\n\n if by_class and \"type_label\" in tiles.columns and tiles[\"type_label\"].notna().any():\n for lbl, group in tiles.dropna(subset=[column]).groupby(\"type_label\", dropna=True):\n _plot_series(group[column], _CLASS_NAMES.get(lbl, str(lbl)))\n else:\n _plot_series(values_all, \"all\")\n\n for t in thresholds:\n if lo <= t <= hi:\n ax_hist.axvline(t, color=\"k\", ls=\"--\", lw=0.8)\n ax_cdf.axvline(t, color=\"k\", ls=\"--\", lw=0.8)\n\n zoom_note = \"\" if (lo, hi) == (0.0, 1.0) else f\" [zoom {lo:.3f}–{hi:.3f}]\"\n ax_hist.set(title=f\"{column}: distribution{zoom_note}\", xlabel=\"coverage\", ylabel=\"density\",\n xlim=(lo, hi))\n ax_hist.legend()\n ax_cdf.set(title=f\"{column}: CDF (= fraction dropped at threshold){zoom_note}\",\n xlabel=\"threshold\", ylabel=\"fraction of tiles with coverage ≤ threshold\",\n xlim=(lo, hi))\n ax_cdf.legend()\n fig.tight_layout()\n plt.show()\n\n total = len(values_all)\n print(f\"{column}: {total:,} tiles\")\n for t in thresholds:\n kept = int((values_all > t).sum())\n print(f\" threshold > {t:>5}: keep {kept:>9,} ({100 * kept / total:5.1f}%), \"\n f\"drop {total - kept:>9,} ({100 * (total - kept) / total:5.1f}%)\")" + }, + { + "cell_type": "markdown", + "id": "5c7488ad", + "metadata": {}, + "source": [ + "# Calls\n", + "\n", + "Run these to load the dataset and inspect each overlap. Adjust `thresholds=` to your candidate\n", + "cutoffs and re-run cell 3 as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6020e97", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Load tiles.parquet from the tiled-dataset artifact.\n", + "tiles = load_tiles()\n", + "tiles.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b681e8af", + "metadata": {}, + "outputs": [], + "source": [ + "# 2. Quantile table — read candidate thresholds straight off it.\n", + "summary_table(tiles)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a37d329b", + "metadata": {}, + "outputs": [], + "source": "# 3. Distribution + CDF per overlap column.\n# - `thresholds=` : candidate cutoffs to annotate + print keep/drop for.\n# - `xlim=` : x-axis zoom window. Only folding/residual are zoomed here (their mass sits\n# near 1.0); the rest are shown full-range. auto_xlim frames the tail --\n# lower `tail=` to zoom tighter, or pass a manual window like (0.9, 1.0).\nZOOM_COLUMNS = [\"folding_overlap\", \"residual_overlap\"]\n\nfor column in OVERLAP_COLUMNS:\n if column not in tiles.columns:\n continue\n if column in ZOOM_COLUMNS:\n xlim = auto_xlim(tiles[column].dropna()) # or e.g. (0.9, 1.0)\n else:\n xlim = (0.0, 1.0)\n plot_overlap(tiles, column, thresholds=(0.1, 0.25, 0.5), xlim=xlim)" + }, + { + "cell_type": "markdown", + "id": "29607684", + "source": "## How to pick a threshold\n\nThere's no single \"correct\" cutoff — you're trading off **tile quality** against **how much data you keep**. Use the plots + quantile table together:\n\n1. **Look for a gap / elbow in the distribution.** If the histogram is bimodal — a big clean lump near 1.0 and a smaller junk lump near 0 — put the threshold *in the valley between them*. That's the most defensible cutoff: it separates two populations rather than cutting into one.\n2. **If there's no clear gap (just a smooth skew toward 1.0),** there's no natural boundary, so pick by **data-loss budget** instead. Decide how many tiles you're willing to drop (e.g. \"≤5%\") and read the cutoff off the matching low-end quantile in `summary_table` (the 5% row) or off the CDF (find where y = 0.05, take the x). A skewed column usually means the mask rarely fires, so a low cutoff (drop only the worst 1–5%) is typically enough.\n3. **Match the cutoff to what the mask is *for*.**\n - `tissue_overlap` — the gate that decides a tile is on tissue at all; a moderate cutoff (e.g. 0.25–0.5) is normal. (The pipeline already drops `tissue_overlap == 0`.)\n - `epithelium_overlap` — how much tumor epithelium a tile must contain; set by how epithelium-pure you want tiles, not by data loss.\n - `blur` / `folding` / `residual` (QC) — these only remove artifacts, so keep them **lenient**: drop just the clearly-bad tail (often 0.01–0.1) so you don't throw away good tissue.\n4. **Sanity-check the A/B overlay.** A QC/tissue threshold should affect luminal A and B tiles *about equally*. If one class loses far more tiles than the other at your cutoff, you're introducing class bias — loosen it or reconsider.\n5. **Confirm with the printed keep/drop line.** Once a candidate looks right, add it to `thresholds=` and check the exact keep/drop counts are acceptable before committing.\n\nIn short: **cut at a gap if one exists; otherwise cut by an acceptable data-loss budget — lenient for pure-artifact masks, and balanced across A/B.**", + "metadata": {} + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 2ac1ebdbf5b7570997fe49faa323e7255cb0e39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 6 Jul 2026 12:11:54 +0200 Subject: [PATCH 39/79] fix: change repo --- scripts/preprocessing/run_qc.py | 5 +++-- scripts/preprocessing/run_tissue_masks.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/preprocessing/run_qc.py b/scripts/preprocessing/run_qc.py index 9842c9ba..2797ca95 100644 --- a/scripts/preprocessing/run_qc.py +++ b/scripts/preprocessing/run_qc.py @@ -10,9 +10,10 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", "uv run -m preprocessing.quality_control", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], diff --git a/scripts/preprocessing/run_tissue_masks.py b/scripts/preprocessing/run_tissue_masks.py index ad149d49..7c803de1 100644 --- a/scripts/preprocessing/run_tissue_masks.py +++ b/scripts/preprocessing/run_tissue_masks.py @@ -10,10 +10,11 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", - "uv run -m preprocessing.tissue_masks", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + "uv run python -m preprocessing.tissue_masks", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], ) From 78aa569c150e4bd2147137687270cab8b9c11578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 6 Jul 2026 16:02:29 +0200 Subject: [PATCH 40/79] squash epithel --- scripts/preprocessing/run_epithelium_masks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/preprocessing/run_epithelium_masks.py b/scripts/preprocessing/run_epithelium_masks.py index 6d77dab9..7f7aca47 100644 --- a/scripts/preprocessing/run_epithelium_masks.py +++ b/scripts/preprocessing/run_epithelium_masks.py @@ -9,9 +9,10 @@ gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "uv sync", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", "uv run -m preprocessing.epithelium_masks", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], From 538ab5a75e474e5635d455bce39cfa4b340b8fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 13 Jul 2026 12:06:39 +0200 Subject: [PATCH 41/79] fix: epith --- configs/preprocessing/epithelium_masks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/preprocessing/epithelium_masks.yaml b/configs/preprocessing/epithelium_masks.yaml index c699cf54..46cfc133 100644 --- a/configs/preprocessing/epithelium_masks.yaml +++ b/configs/preprocessing/epithelium_masks.yaml @@ -6,7 +6,7 @@ dataset: mlflow_uris: tissue_masks: ??? -max_concurrent: 4 +max_concurrent: 100 timeout: 3000 model_name: episeg-1 From 61df822c088d7dfde7796a81ed5c4c51c15631b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 16:15:08 +0200 Subject: [PATCH 42/79] feat: demo tiled data --- configs/data/sources/demo_single_l3_tiled.yaml | 5 +++++ configs/data/tiled/demo_l3_tiled.yaml | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 configs/data/sources/demo_single_l3_tiled.yaml create mode 100644 configs/data/tiled/demo_l3_tiled.yaml diff --git a/configs/data/sources/demo_single_l3_tiled.yaml b/configs/data/sources/demo_single_l3_tiled.yaml new file mode 100644 index 00000000..91a516ef --- /dev/null +++ b/configs/data/sources/demo_single_l3_tiled.yaml @@ -0,0 +1,5 @@ +# @package _global_ +# DEMO single-scale raw-tile source: level-3 tiled card + raw-tile template. +defaults: + - /data/_template_raw + - /data/tiled/demo_l3_tiled diff --git a/configs/data/tiled/demo_l3_tiled.yaml b/configs/data/tiled/demo_l3_tiled.yaml new file mode 100644 index 00000000..6a22fff6 --- /dev/null +++ b/configs/data/tiled/demo_l3_tiled.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# DEMO level-3 RAW-TILE card (OpenSlide tiled artifact: slides.parquet + tiles.parquet). +# Per-split raw_uris produced by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224 \ +# --name l3_tiled +name: demo_l3_tiled +level: 3 +mpp: 1.94 +tile_extent: 224 +raw_uris: + train: "mlflow-artifacts:/3/59bc4eba00734fd1becac92949065792/artifacts/l3_tiled_train" + val: "mlflow-artifacts:/3/1c3b8348dab84158a95abac79267e71b/artifacts/l3_tiled_val" + test: "mlflow-artifacts:/3/594c785fe3064c9784c2ec50375cc59b/artifacts/l3_tiled_test" From ed2d1e263c2e19709f77a53d0ded0522dd873903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 16:25:17 +0200 Subject: [PATCH 43/79] feat: demo embed --- configs/data/sources/demo_single_l3_embed.yaml | 5 +++++ configs/data/tiled/demo_l3_embed.yaml | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 configs/data/sources/demo_single_l3_embed.yaml create mode 100644 configs/data/tiled/demo_l3_embed.yaml diff --git a/configs/data/sources/demo_single_l3_embed.yaml b/configs/data/sources/demo_single_l3_embed.yaml new file mode 100644 index 00000000..393474bb --- /dev/null +++ b/configs/data/sources/demo_single_l3_embed.yaml @@ -0,0 +1,5 @@ +# @package _global_ +# DEMO single-scale embeddings source: level-3 embedding card + embeddings template. +defaults: + - /data/_template + - /data/tiled/demo_l3_embed diff --git a/configs/data/tiled/demo_l3_embed.yaml b/configs/data/tiled/demo_l3_embed.yaml new file mode 100644 index 00000000..fd31952b --- /dev/null +++ b/configs/data/tiled/demo_l3_embed.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# DEMO level-3 EMBEDDINGS card (precomputed Virchow2 features, mpp 1.94). +# Per-split uris produced by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri mlflow-artifacts:/3/16e8f2a772ec48d2a21dc8cf704f0e10/artifacts/embeddings \ +# --name l3_embed +name: demo_l3_embed +level: 3 +mpp: 1.94 +tile_extent: 224 +uris: + train: "mlflow-artifacts:/3/6f1e6c8f3bd048eebc5addd1c3a9e36c/artifacts/l3_embed_train" + val: "mlflow-artifacts:/3/5b589be048504c169cdb6e81d585f520/artifacts/l3_embed_val" + test: "mlflow-artifacts:/3/4735b43e2fd342bea6961cf4321460bf/artifacts/l3_embed_test" From 9c5cf2a3b0809e1d5fc023b002d0dc6175b80e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 25 Jul 2026 15:24:25 +0200 Subject: [PATCH 44/79] feat: eval test set after fit --- ml/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ml/train.py b/ml/train.py index 630964f8..28f0d860 100644 --- a/ml/train.py +++ b/ml/train.py @@ -12,6 +12,7 @@ uv run -m ml.train +experiment=ml/train_mil_embeddings mode=test checkpoint=/path/best.ckpt """ +import logging import logging import random From e2a02fd7a32706ecf3bfc267da89e69e3674fd9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 10:48:23 +0200 Subject: [PATCH 45/79] feat: continuous tiling feat: tiling with constants --- configs/preprocessing/tiling.yaml | 14 ++++- preprocessing/tiling.py | 88 ++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/configs/preprocessing/tiling.yaml b/configs/preprocessing/tiling.yaml index 7fc4812d..6a999c42 100644 --- a/configs/preprocessing/tiling.yaml +++ b/configs/preprocessing/tiling.yaml @@ -5,6 +5,13 @@ tile_extent: ??? stride: ??? data_name: ??? +# Tile-filtering thresholds on the *_overlap scores (higher = cleaner). A tile is kept only when +# its overlap exceeds the threshold. See scripts/notebooks/mask_coverage_threshold.ipynb. +tissue_threshold: ??? +blur_threshold: ??? +folding_threshold: ??? +residual_threshold: ??? + metadata: run_name: Mammaprint Tiling ${dataset.name} description: | @@ -14,9 +21,14 @@ metadata: * assigns tissue/epithelium/QC mask paths and computes overlap scores * keeps tiles with tissue in the centered 50% ROI * computes full-tile overlaps for epithelium, blur, folding, and residual masks + * filters tiles by the per-mask coverage thresholds * saves slide and tile datasets via MLflow hyperparams: mpp: ${mpp} tile_extent: ${tile_extent} stride: ${stride} - data_name: ${data_name} \ No newline at end of file + data_name: ${data_name} + tissue_threshold: ${tissue_threshold} + blur_threshold: ${blur_threshold} + folding_threshold: ${folding_threshold} + residual_threshold: ${residual_threshold} \ No newline at end of file diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index cf66793a..e2c590cd 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -2,7 +2,7 @@ from functools import partial from pathlib import Path -from typing import Any +from typing import Any, Literal import hydra import numpy as np @@ -32,17 +32,29 @@ def add_tile_overlap( roi: Polygon, path_col: str, overlap_col: str, - struct_field: str, + keep: Literal["zero", "nonzero"], + background: str = "0", ) -> Dataset: - """Compute per-tile overlap score against a mask overlay. + """Compute a per-tile coverage score against a mask overlay. - This helper uses `tile_overlay_overlap` to compute, for each tile, - how much of the given ROI (tile sub-region) overlaps a binary mask. + This helper uses `tile_overlay_overlap` to obtain, for each tile, the fraction of the ROI + covered by every distinct pixel value in the mask, then reduces it to a single score in + ``[0, 1]`` where **higher = keep**. - The mask file path is read from the column ``path_col`` in the Ray Dataset. The - function expects the dataset to contain tile coordinates and microns-per-pixel columns - (``tile_x``, ``tile_y``, ``mpp_x``, ``mpp_y``), as produced by the upstream tiling - pipeline. + The masks are downsampled to a continuous ``[0, 255]`` range (they are NOT strictly binary): + the background/clean pixel value is ``0`` and *any* non-zero value marks foreground (tissue) or + an artifact, depending on the mask. Two reductions are therefore supported via ``keep``: + + * ``keep="nonzero"`` (tissue/epithelium): score = fraction of non-zero pixels = foreground + coverage. Keep tiles with a lot of tissue/epithelium. + * ``keep="zero"`` (QC artifact masks: blur/folding/residual): score = fraction of ``0`` pixels = + clean fraction. Keep tiles that are mostly artifact-free. + + In both cases a downstream ``overlap > threshold`` filter keeps the desirable tiles. + + The mask file path is read from the column ``path_col`` in the Ray Dataset. The function expects + the dataset to contain tile coordinates and microns-per-pixel columns (``tile_x``, ``tile_y``, + ``mpp_x``, ``mpp_y``), as produced by the upstream tiling pipeline. Parameters ---------- @@ -54,14 +66,12 @@ def add_tile_overlap( path_col: Column name containing the mask image path for each tile. overlap_col: - Name of the output column to store the computed overlap score in [0, 1]. - struct_field: - The pixel value (as string: "0" or "255") in the binary mask representing the - undesired artifact or background. The mask contains only values 0 and 255. - The overlap is computed as ``1 - (fraction of pixels with this value)``, yielding - the fraction of desired (non-artifact) pixels. - For example, when folding masks have folds as white (255), pass "255" to compute - the amount of non-folded tissue. + Name of the output column to store the computed score in [0, 1]. + keep: + Which pixels count as desirable: ``"zero"`` scores the clean (background) fraction, + ``"nonzero"`` scores the foreground fraction. + background: + Pixel value (as string) of the clean/background class. Defaults to ``"0"``. Returns: ------- @@ -82,12 +92,16 @@ def add_tile_overlap( ) def extract_value(batch: DataBatch) -> DataBatch: - batch[overlap_col] = np.array( - [ - 1.0 - (s.get(struct_field, 1.0) or 1.0) if isinstance(s, dict) else 0.0 - for s in batch[overlap_struct_col] - ] - ) + def score(s: Any) -> float: + # No mask / no readable pixels in the ROI -> score 0 so the tile is dropped. + if not isinstance(s, dict) or not s: + return 0.0 + # Fraction of clean/background (== background value) pixels in the ROI. + zero_fraction = s.get(background, 0.0) or 0.0 + # keep="zero": clean fraction; keep="nonzero": foreground (1 - clean) fraction. + return zero_fraction if keep == "zero" else 1.0 - zero_fraction + + batch[overlap_col] = np.array([score(s) for s in batch[overlap_struct_col]]) return batch tiles = tiles.map_batches( @@ -226,21 +240,35 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: # Full tile full_roi = box(0, 0, config.tile_extent, config.tile_extent) - # Add tissue overlaps + # Add coverage scores (all in [0, 1], higher = keep). The masks are downsampled continuous + # [0, 255] where 0 = clean/background and any non-zero = foreground/artifact (per the QC docs). + # tissue/epithelium keep the non-zero (foreground) fraction; the QC artifact masks keep the + # zero (clean) fraction. See scripts/notebooks/mask_coverage_threshold.ipynb for the evidence. tiles = add_tile_overlap( - tiles, tissue_roi, "tissue_mask_path", "tissue_overlap", "0" + tiles, tissue_roi, "tissue_mask_path", "tissue_overlap", keep="nonzero" ) - tiles = tiles.filter(expr=col("tissue_overlap") > 0.0) - tiles = add_tile_overlap(tiles, full_roi, "blur_mask_path", "blur_overlap", "255") tiles = add_tile_overlap( - tiles, full_roi, "folding_mask_path", "folding_overlap", "255" + tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" ) tiles = add_tile_overlap( - tiles, full_roi, "residual_mask_path", "residual_overlap", "0" + tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" ) tiles = add_tile_overlap( - tiles, full_roi, "epithelium_mask_path", "epithelium_overlap", "0" + tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" ) + tiles = add_tile_overlap( + tiles, full_roi, "epithelium_mask_path", "epithelium_overlap", keep="nonzero" + ) + + # Filter tiles by mask coverage. Thresholds come from the experiment config (see + # configs/experiment/preprocessing/tiling/*.yaml) and were chosen from the coverage + # distributions in scripts/notebooks/mask_coverage_threshold.ipynb: a moderate tissue gate, and + # lenient artifact-only QC cutoffs (they only remove damaged tissue). Epithelium is not filtered + # here because that mask is not always populated; add a cutoff once it is, by desired tumor purity. + tiles = tiles.filter(expr=col("tissue_overlap") > config.tissue_threshold) + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) # Drop unnecessary columns tiles = tiles.drop_columns( From 022ac6c222ab970b3274fd0a503162da0feb4395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 11:47:50 +0200 Subject: [PATCH 46/79] feat: new data --- configs/data/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/data/default.yaml b/configs/data/default.yaml index 84879366..b80ceaf6 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -6,5 +6,5 @@ dataset: qc_masks: ${project_path}/qc_masks data_mapping: ${project_path}/data_mapping.csv mlflow_uris: - tissue_masks: mlflow-artifacts:/98/aa6301ad8ae341ab935ab5325cde9369/artifacts/tissue_masks + tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks epithelium_masks: null From 69f6da32a779770786bfdcec15430dcf2f4be9aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 16 Jul 2026 12:30:24 +0200 Subject: [PATCH 47/79] fix: ignore null epit mask --- preprocessing/tiling.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index e2c590cd..1028c267 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -27,6 +27,12 @@ BATCH_SIZE = 256 +def add_missing_epithelium_overlap(batch: DataBatch) -> DataBatch: + """Fill a NaN ``epithelium_overlap`` when no epithelium mask is configured.""" + batch["epithelium_overlap"] = np.nan + return batch + + def add_tile_overlap( tiles: Dataset, roi: Polygon, @@ -256,9 +262,24 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: tiles = add_tile_overlap( tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" ) - tiles = add_tile_overlap( - tiles, full_roi, "epithelium_mask_path", "epithelium_overlap", keep="nonzero" - ) + if epithelium_masks_path: + print("[INFO] Computing overlap: epithelium_overlap from epithelium_mask_path") + tiles = add_tile_overlap( + tiles, + full_roi, + "epithelium_mask_path", + "epithelium_overlap", + keep="nonzero", + ) + else: + print( + "[INFO] Skipping epithelium overlap: " + "dataset.mlflow_uris.epithelium_masks is not set" + ) + tiles = tiles.map_batches( + add_missing_epithelium_overlap, + batch_format="pandas", + ) # Filter tiles by mask coverage. Thresholds come from the experiment config (see # configs/experiment/preprocessing/tiling/*.yaml) and were chosen from the coverage From 238257e535f2ac6fcdf6ebec412c54434319a58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 20 Jun 2026 18:01:45 +0200 Subject: [PATCH 48/79] feat: remove memory estimates tiling --- preprocessing/tiling.py | 96 +++++++++++++++++++++++------ scripts/preprocessing/run_tiling.py | 22 ++++++- 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 1028c267..ff1154ea 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -1,5 +1,6 @@ """Script for creating slides and tiles datasets for mammaprint prediction using Ray Data.""" +import os from functools import partial from pathlib import Path from typing import Any, Literal @@ -24,7 +25,48 @@ from shapely.geometry.polygon import Polygon -BATCH_SIZE = 256 +# Rows per block after the tiling flat_map. Larger blocks amortize Ray's per-block +# scheduling overhead and the per-batch mask file-open in the overlap UDFs (one open per +# (mask, block) instead of one per 256 tiles), at the cost of more object-store memory per +# block. Kept well within the object store sized below. +BATCH_SIZE = 4096 + +# Fraction of the pod's memory to give Ray's object store when RAY_MEMORY_BYTES is set. +# The rest is left for Ray's heap and the Python worker processes. +RAY_OBJECT_STORE_FRACTION = 0.4 + + +def _ray_init_kwargs() -> dict[str, Any]: + """Build ``ray.init`` sizing kwargs from the pod's resources. + + On the K8s cluster ``ray.init()`` auto-detection was unreliable: it over-detected CPUs + (saw the 192-core node instead of the pod's cgroup limit) and under-sized the object + store (~9.5 GiB), which backpressured the repartition queue and collapsed the + overlap+filter stage to a single task. + + To avoid duplicating the pod's resource numbers, the launcher + (``scripts/preprocessing/run_tiling.py``) exports them as environment variables derived + from its own ``submit_job(cpu=..., memory=...)`` args: + + * ``RAY_NUM_CPUS`` -- number of CPUs to give Ray. + * ``RAY_MEMORY_BYTES`` -- the pod's total memory in bytes; the object store is sized as + a fraction of it. + + When unset (e.g. a plain local run), we fall back to Ray's own auto-detection for both. + """ + kwargs: dict[str, Any] = {} + + num_cpus = os.environ.get("RAY_NUM_CPUS") + if num_cpus: + kwargs["num_cpus"] = int(float(num_cpus)) + + memory_bytes = os.environ.get("RAY_MEMORY_BYTES") + if memory_bytes: + # Fraction of pod memory for the object store. The rest is left for Ray's heap and + # the Python worker processes. Capped by /dev/shm at the pod level (see run_tiling.py). + kwargs["object_store_memory"] = int(float(memory_bytes) * RAY_OBJECT_STORE_FRACTION) + + return kwargs def add_missing_epithelium_overlap(batch: DataBatch) -> DataBatch: @@ -93,9 +135,9 @@ def add_tile_overlap( col("mpp_x"), col("mpp_y"), ) - tiles = tiles.with_column( - overlap_struct_col, overlap_struct, num_cpus=1, memory=BATCH_SIZE * 8 * 512**2 - ) + # num_cpus=1 so Ray schedules one task per core across the pod. Without it the fused + # overlap+filter operator ran with a single task and left 47 of 48 cores idle. + tiles = tiles.with_column(overlap_struct_col, overlap_struct, num_cpus=1) def extract_value(batch: DataBatch) -> DataBatch: def score(s: Any) -> float: @@ -114,7 +156,6 @@ def score(s: Any) -> float: extract_value, batch_format="pandas", num_cpus=1, - memory=BATCH_SIZE * 8 * 512**2, ) return tiles.drop_columns([overlap_struct_col]) @@ -185,7 +226,7 @@ def tiling( @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: if not ray.is_initialized(): - ray.init() + ray.init(**_ray_init_kwargs()) tissue_masks_path = ( None @@ -223,8 +264,13 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: join_type="left_outer", ) - # Create unique slide IDs and save slide metadata - slides = slides.map(row_hash, num_cpus=0.1, memory=128 * 1024**2) + # Create unique slide IDs and save slide metadata. + # Fractional num_cpus so these cheap coordinate-generation stages don't win Ray's + # per-operator CPU reservation. Ray's ReservationOpResourceAllocator prioritizes upstream + # operators, so without a cap row_hash/tiling grabbed ~22 of 48 CPUs and starved the + # expensive downstream overlap+filter stage to a single task while its input queue backed + # up. Keeping these sub-1.0 leaves the CPU budget for the mask-reading UDFs (num_cpus=1). + slides = slides.map(row_hash, num_cpus=0.1) # Expand slides into tile coordinates tiles = slides.flat_map( @@ -234,8 +280,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: qc_masks_path=qc_masks_path, epithelium_masks_path=epithelium_masks_path, ), - num_cpus=0.2, - memory=BATCH_SIZE * 1024**2, + num_cpus=0.25, ).repartition(target_num_rows_per_block=BATCH_SIZE) # Compute masks and filter tissue tiles @@ -250,18 +295,39 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: # [0, 255] where 0 = clean/background and any non-zero = foreground/artifact (per the QC docs). # tissue/epithelium keep the non-zero (foreground) fraction; the QC artifact masks keep the # zero (clean) fraction. See scripts/notebooks/mask_coverage_threshold.ipynb for the evidence. + # + # We compute each overlap and immediately filter on it, rather than computing all overlaps up + # front and filtering at the end. The final result is identical (the keep set is the AND of all + # thresholds), but interleaving means a tile discarded by an early gate never pays for the + # remaining mask reads. Only ~2.5% of tiles survive, so this roughly halves total overlap work. + # Thresholds come from the experiment config (see configs/experiment/preprocessing/tiling/*.yaml) + # and the coverage distributions in scripts/notebooks/mask_coverage_threshold.ipynb: a moderate + # tissue gate, and lenient artifact-only QC cutoffs (they only remove damaged tissue). + # + # Order matters for speed (not for the result): the tissue gate runs first because it is both the + # most selective and the cheapest (center-50% ROI = smallest region read). tiles = add_tile_overlap( tiles, tissue_roi, "tissue_mask_path", "tissue_overlap", keep="nonzero" ) + tiles = tiles.filter(expr=col("tissue_overlap") > config.tissue_threshold) + tiles = add_tile_overlap( tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" ) + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + tiles = add_tile_overlap( tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" ) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + tiles = add_tile_overlap( tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" ) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) + + # Epithelium is computed but NOT filtered here (that mask is not always populated; add a cutoff + # once it is, by desired tumor purity). Computed last so it only runs on the surviving tiles. if epithelium_masks_path: print("[INFO] Computing overlap: epithelium_overlap from epithelium_mask_path") tiles = add_tile_overlap( @@ -281,16 +347,6 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: batch_format="pandas", ) - # Filter tiles by mask coverage. Thresholds come from the experiment config (see - # configs/experiment/preprocessing/tiling/*.yaml) and were chosen from the coverage - # distributions in scripts/notebooks/mask_coverage_threshold.ipynb: a moderate tissue gate, and - # lenient artifact-only QC cutoffs (they only remove damaged tissue). Epithelium is not filtered - # here because that mask is not always populated; add a cutoff once it is, by desired tumor purity. - tiles = tiles.filter(expr=col("tissue_overlap") > config.tissue_threshold) - tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) - tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) - tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) - # Drop unnecessary columns tiles = tiles.drop_columns( [ diff --git a/scripts/preprocessing/run_tiling.py b/scripts/preprocessing/run_tiling.py index 134a55d6..692c88fb 100644 --- a/scripts/preprocessing/run_tiling.py +++ b/scripts/preprocessing/run_tiling.py @@ -1,17 +1,33 @@ +from humanfriendly import parse_size from kube_jobs import storage, submit_job +# Pod resources. These are the single source of truth: they size the K8s request/limit AND +# are forwarded to Ray via environment variables so tiling.py does not hard-code them. +CPU = 48 +MEMORY = "64Gi" + +# Ray's auto-detection is unreliable on this cluster (it over-detects CPUs from the node and +# under-sizes the object store), so we hand it the pod's real numbers. tiling.py reads these +# and derives ray.init(num_cpus=..., object_store_memory=fraction * RAY_MEMORY_BYTES). +ray_env = [ + f"export RAY_NUM_CPUS={CPU}", + f"export RAY_MEMORY_BYTES={parse_size(MEMORY, binary=True)}", +] + submit_job( job_name="mammaprint-tiling-...", username=..., image="cerit.io/rationai/base:2.0.6", - cpu=8, - memory="32Gi", + cpu=CPU, + memory=MEMORY, gpu=None, public=False, script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/mammaprint workdir", + "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + *ray_env, "uv sync --frozen", "uv run -m preprocessing.tiling +experiment=preprocessing/tiling/...", ], From bde510018d10e7a96d1eb057cdcbd5717b410a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 20 Jul 2026 13:56:12 +0200 Subject: [PATCH 49/79] feat: tiling diagnostics --- configs/preprocessing/tiling.yaml | 2 + preprocessing/tiling.py | 141 +++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 12 deletions(-) diff --git a/configs/preprocessing/tiling.yaml b/configs/preprocessing/tiling.yaml index 6a999c42..4c44f275 100644 --- a/configs/preprocessing/tiling.yaml +++ b/configs/preprocessing/tiling.yaml @@ -12,6 +12,8 @@ blur_threshold: ??? folding_threshold: ??? residual_threshold: ??? +diagnostics: false + metadata: run_name: Mammaprint Tiling ${dataset.name} description: | diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index ff1154ea..0d5fa0e9 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -161,6 +161,93 @@ def score(s: Any) -> float: return tiles.drop_columns([overlap_struct_col]) +def count_candidate_tiles(slides_df: pd.DataFrame) -> int: + """Number of candidate tiles across all slides, computed analytically (no mask IO). + + Mirrors ``grid_tiles(..., last="keep")``: per axis it yields + ``ceil((extent - tile_extent) / stride) + 1`` tiles, so the per-slide count is the product + over the x and y axes. Summed over slides this is the pre-filter tile count -- computed from + the slide metadata alone, so we never have to materialize the ~250M-row candidate set to count + it. + """ + ex = np.asarray(slides_df["extent_x"], dtype=float) + ey = np.asarray(slides_df["extent_y"], dtype=float) + tx = np.asarray(slides_df["tile_extent_x"], dtype=float) + ty = np.asarray(slides_df["tile_extent_y"], dtype=float) + sx = np.asarray(slides_df["stride_x"], dtype=float) + sy = np.asarray(slides_df["stride_y"], dtype=float) + nx = np.ceil((ex - tx) / sx) + 1 + ny = np.ceil((ey - ty) / sy) + 1 + return int(np.sum(nx * ny)) + + +def log_drop_funnel( + tissue_survivors: Dataset, + n_candidates: int, + thresholds: dict[str, float], + logger: MLFlowLogger, +) -> None: + """Quantify how many tiles each QC threshold drops and log the funnel to MLflow. + + ``tissue_survivors`` must be the dataset AFTER the tissue gate but with ``blur_overlap``, + ``folding_overlap`` and ``residual_overlap`` already computed and NOT yet filtered (so the + dropped tiles' scores are still present). Reports two views: + + * **independent** -- how many tiles each threshold removes on its own. Tissue is measured + against the full candidate grid (``n_candidates``); the QC gates are measured among the + tissue survivors (the only affordable and meaningful denominator). + * **sequential** -- the funnel in pipeline order (tissue -> blur -> folding -> residual), where + each gate is credited only for tiles it is the first to drop. These sum to the total dropped. + + A tile is dropped by a gate when ``overlap <= threshold`` (the strict complement of the + production ``overlap > threshold`` filter; NaN/empty overlaps already score 0.0 and so count as + dropped, matching the run). + """ + df = tissue_survivors.to_pandas() + n_tissue = len(df) + + b = df["blur_overlap"] <= thresholds["blur"] + f = df["folding_overlap"] <= thresholds["folding"] + r = df["residual_overlap"] <= thresholds["residual"] + + independent = { + "tissue": n_candidates - n_tissue, + "blur": int(b.sum()), + "folding": int(f.sum()), + "residual": int(r.sum()), + } + # Sequential attribution in pipeline order: each gate credited only for tiles that survived + # every earlier gate (tissue already applied by construction of this dataset). + sequential = { + "tissue": n_candidates - n_tissue, + "blur": int(b.sum()), + "folding": int((~b & f).sum()), + "residual": int((~b & ~f & r).sum()), + } + kept_final = int((~b & ~f & ~r).sum()) + + def pct(n: int) -> float: + return round(100.0 * n / n_candidates, 3) if n_candidates else 0.0 + + print("\n[DIAGNOSTICS] Tile drop funnel") + print(f" candidate tiles (pre-filter): {n_candidates:,}") + print(f" survived tissue gate: {n_tissue:,}") + print(" independent drops (each gate alone; QC gates among tissue survivors):") + for k in ("tissue", "blur", "folding", "residual"): + print(f" {k:9} {independent[k]:>12,} ({pct(independent[k]):5.2f}% of candidates)") + print(" sequential drops (pipeline order; mutually exclusive):") + for k in ("tissue", "blur", "folding", "residual"): + print(f" {k:9} {sequential[k]:>12,} ({pct(sequential[k]):5.2f}% of candidates)") + print(f" kept (survived all gates): {kept_final:,} ({pct(kept_final):5.2f}%)") + + metrics = {"diag_candidate_tiles": n_candidates, "diag_tissue_survivors": n_tissue, + "diag_kept_final": kept_final} + for k in ("tissue", "blur", "folding", "residual"): + metrics[f"diag_independent_drop_{k}"] = independent[k] + metrics[f"diag_sequential_drop_{k}"] = sequential[k] + logger.log_metrics(metrics) + + def tiling( row: dict[str, Any], tissue_masks_path: str | None, @@ -311,20 +398,50 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: ) tiles = tiles.filter(expr=col("tissue_overlap") > config.tissue_threshold) - tiles = add_tile_overlap( - tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" - ) - tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + if config.get("diagnostics", False): + # Diagnostic path: quantify per-gate drops. Compute ALL three QC overlaps on the tissue + # survivors WITHOUT filtering (so dropped tiles' scores are retained), materialize once so + # the funnel scan and the subsequent production filters both reuse the same pinned blocks + # (no re-reading of masks), log the funnel, then apply the identical QC filters below so the + # written dataset is byte-identical to a normal run. + tiles = add_tile_overlap( + tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" + ) + tiles = add_tile_overlap( + tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" + ) + tiles = add_tile_overlap( + tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" + ) + tiles = tiles.materialize() + + n_candidates = count_candidate_tiles(slides.to_pandas()) + thresholds = { + "tissue": config.tissue_threshold, + "blur": config.blur_threshold, + "folding": config.folding_threshold, + "residual": config.residual_threshold, + } + log_drop_funnel(tiles, n_candidates, thresholds, logger) - tiles = add_tile_overlap( - tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" - ) - tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) + else: + tiles = add_tile_overlap( + tiles, full_roi, "blur_mask_path", "blur_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("blur_overlap") > config.blur_threshold) - tiles = add_tile_overlap( - tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" - ) - tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) + tiles = add_tile_overlap( + tiles, full_roi, "folding_mask_path", "folding_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("folding_overlap") > config.folding_threshold) + + tiles = add_tile_overlap( + tiles, full_roi, "residual_mask_path", "residual_overlap", keep="zero" + ) + tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) # Epithelium is computed but NOT filtered here (that mask is not always populated; add a cutoff # once it is, by desired tumor purity). Computed last so it only runs on the surviving tiles. From da9b40455bd0de7eaa2835d2858b7ee9e29ff1f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 23 Jul 2026 15:23:05 +0200 Subject: [PATCH 50/79] feat: change file suffixes --- preprocessing/epithelium_masks.py | 2 +- preprocessing/quality_control.py | 2 +- preprocessing/tiling.py | 2 +- preprocessing/tissue_masks.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/preprocessing/epithelium_masks.py b/preprocessing/epithelium_masks.py index 60258397..f78f20ff 100644 --- a/preprocessing/epithelium_masks.py +++ b/preprocessing/epithelium_masks.py @@ -54,7 +54,7 @@ async def _bounded_process(client: AsyncClient, path: str) -> None: @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: slides_df = pd.read_csv(config.dataset.paths.data_mapping) - slides: list[str] = slides_df.apply(lambda x: x["path"] + ".mrxs", axis=1).tolist() + slides: list[str] = slides_df["path"].tolist() with tempfile.TemporaryDirectory( prefix="episeg_", dir=config.project_path diff --git a/preprocessing/quality_control.py b/preprocessing/quality_control.py index 3081dee6..3b6e2536 100644 --- a/preprocessing/quality_control.py +++ b/preprocessing/quality_control.py @@ -184,7 +184,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: output_path.mkdir(exist_ok=True, parents=True) df = pd.read_csv(config.dataset.paths.data_mapping) - slides = [Path(path + ".mrxs") for path in df["path"]] + slides = [Path(path) for path in df["path"]] semaphore = asyncio.Semaphore(config.request_limit) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 0d5fa0e9..c8de9cc6 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -329,7 +329,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: # Read slide paths from data mapping slide_labels_df = pd.read_csv(config.dataset.paths.data_mapping) - slide_labels_df["slide_path"] = slide_labels_df["path"] + ".mrxs" + slide_labels_df["slide_path"] = slide_labels_df["path"] slide_paths = slide_labels_df["slide_path"].tolist() metadata_ds = rd.from_pandas( slide_labels_df[["slide_path", "type", "mammaprint_index"]] diff --git a/preprocessing/tissue_masks.py b/preprocessing/tissue_masks.py index 59a1aa72..fe45213c 100644 --- a/preprocessing/tissue_masks.py +++ b/preprocessing/tissue_masks.py @@ -35,7 +35,7 @@ def process_slide(slide_path: Path, level: int, output_path: Path) -> None: @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: df = pd.read_csv(config.dataset.paths.data_mapping) - slides = [Path(path + ".mrxs") for path in df["path"]] + slides = [Path(path) for path in df["path"]] with TemporaryDirectory() as output_dir: process_items( From d8ba4470e0090b1f9ea13aa1305a458cad844df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 27 Jul 2026 15:39:42 +0200 Subject: [PATCH 51/79] fix: mlkit update for embed --- uv.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 2c6c4a7a..baa11a1b 100644 --- a/uv.lock +++ b/uv.lock @@ -711,7 +711,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -2019,8 +2018,8 @@ dependencies = [ [[package]] name = "rationai-mlkit" -version = "0.4.0" -source = { git = "https://github.com/RationAI/mlkit.git#55314049191dbfd43b21074ba2a3b6e7f6612647" } +version = "0.4.1" +source = { git = "https://github.com/RationAI/mlkit.git#db38501bb363cd3d76c6a3d31bec702f16002dd1" } dependencies = [ { name = "datasets" }, { name = "gitpython" }, From 7ed15174691e927356a5642bd5b7f9874f2f2915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 27 Jul 2026 15:49:08 +0200 Subject: [PATCH 52/79] fix: tiling renaming --- preprocessing/tiling.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index c8de9cc6..80053ac8 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -477,6 +477,12 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: ] ) + # Rename tile coordinates to the schema mlkit's OpenSlideTilesDataset expects. + # The ratiopath overlap helpers (tile_overlay_overlap) require "tile_x"/"tile_y", so those + # names are kept through the whole pipeline above; mlkit's OpenSlideTilesDataset.__getitem__ + # reads tile["x"]/tile["y"], so we rename only now, right before writing. + tiles = tiles.rename_columns({"tile_x": "x", "tile_y": "y"}) + # Convert Ray Datasets to pandas DataFrames slides_df = slides.to_pandas() tiles_df = tiles.to_pandas() From 3ab00816699d7657115fb9ac181fcc054522545e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 27 Jul 2026 21:48:09 +0200 Subject: [PATCH 53/79] fix: 1 tb ram :( --- preprocessing/tiling.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 80053ac8..bbe0d06e 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -9,7 +9,6 @@ import numpy as np import pandas as pd import ray -import ray.data as rd from mlflow.artifacts import download_artifacts from omegaconf import DictConfig from rationai.mlkit import autolog, with_cli_args @@ -329,27 +328,35 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: # Read slide paths from data mapping slide_labels_df = pd.read_csv(config.dataset.paths.data_mapping) - slide_labels_df["slide_path"] = slide_labels_df["path"] - slide_paths = slide_labels_df["slide_path"].tolist() - metadata_ds = rd.from_pandas( - slide_labels_df[["slide_path", "type", "mammaprint_index"]] - ) + slide_paths = slide_labels_df["path"].tolist() print(f"\n[INFO] Processing {len(slide_paths)} slides using Ray Data pipeline\n") + # Attach labels (type, mammaprint_index) by path. This is a tiny per-slide lookup + # (~2k rows) that we already hold in the driver, so we broadcast it as a dict and merge + # in a map_batches rather than a Ray hash-shuffle join. The distributed join was pure + # overhead here -- worse, its shuffle produced a wildly inflated per-task memory estimate + # (~1 TiB/task) that no node could satisfy, so the autoscaler rejected it forever and the + # whole pipeline deadlocked at Join: 0/1 before row_hash/tiling ever ran. + labels_by_path = slide_labels_df.set_index("path")[ + ["type", "mammaprint_index"] + ].to_dict("index") + + def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: + labels = batch["path"].map(labels_by_path) + batch["type"] = [lp["type"] if isinstance(lp, dict) else None for lp in labels] + batch["mammaprint_index"] = [ + lp["mammaprint_index"] if isinstance(lp, dict) else None for lp in labels + ] + return batch + # Read slides slides = read_slides( slide_paths, mpp=config.mpp, tile_extent=config.tile_extent, stride=config.stride, - ).join( - metadata_ds, - on=("path",), - right_on=("slide_path",), - num_partitions=16, - join_type="left_outer", - ) + ).map_batches(attach_labels, batch_format="pandas", num_cpus=0.1) # Create unique slide IDs and save slide metadata. # Fractional num_cpus so these cheap coordinate-generation stages don't win Ray's From 2dca108196abb1a5429ab8323049a036c6c853b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 28 Jul 2026 10:39:29 +0200 Subject: [PATCH 54/79] fix: change train configs --- ml/train.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ml/train.py b/ml/train.py index 28f0d860..1010bc32 100644 --- a/ml/train.py +++ b/ml/train.py @@ -8,8 +8,10 @@ Run with, e.g.:: - uv run -m ml.train +experiment=ml/train_mil_embeddings # fit (default) - uv run -m ml.train +experiment=ml/train_mil_embeddings mode=test checkpoint=/path/best.ckpt + uv run -m ml.train +experiment=ml/train_mil_embeddings \ + data/embedded=demo_l3_embed + uv run -m ml.train +experiment=ml/train_mil_embeddings \ + data/embedded=demo_l3_embed mode=test checkpoint=/path/best.ckpt """ import logging From bf2abb0d65b4a56ef053cb3c0ad627cf73ae527c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 27 Jul 2026 22:41:59 +0200 Subject: [PATCH 55/79] feat: more configs --- configs/data/embedded/tissue_only/l4.yaml | 17 +++++++++++++++++ configs/data/embedded/tissue_only/l5.yaml | 17 +++++++++++++++++ configs/data/tiled/tissue_only/l1.yaml | 17 +++++++++++++++++ configs/data/tiled/tissue_only/l2.yaml | 2 +- configs/data/tiled/tissue_only/l3.yaml | 2 +- configs/data/tiled/tissue_only/l4.yaml | 17 +++++++++++++++++ configs/data/tiled/tissue_only/l5.yaml | 17 +++++++++++++++++ .../embeddings/tissue_only/l0.yaml | 19 +++++++++++++++++++ .../embeddings/tissue_only/l1.yaml | 19 +++++++++++++++++++ .../embeddings/tissue_only/l4.yaml | 19 +++++++++++++++++++ .../embeddings/tissue_only/l5.yaml | 19 +++++++++++++++++++ 11 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 configs/data/embedded/tissue_only/l4.yaml create mode 100644 configs/data/embedded/tissue_only/l5.yaml create mode 100644 configs/data/tiled/tissue_only/l1.yaml create mode 100644 configs/data/tiled/tissue_only/l4.yaml create mode 100644 configs/data/tiled/tissue_only/l5.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml create mode 100644 configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml diff --git a/configs/data/embedded/tissue_only/l4.yaml b/configs/data/embedded/tissue_only/l4.yaml new file mode 100644 index 00000000..30be4951 --- /dev/null +++ b/configs/data/embedded/tissue_only/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l4; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l4_tissue_embed +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_tissue_embed +uris: + all: "mlflow-artifacts:/3/ecfc0ecdba5b44d9becc5c9df1103a16/artifacts/embeddings" + train: "mlflow-artifacts:/3/1f67a719b2414301a22144c8a6ca926b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/9be7148759d946e68c64f2a0a498c3ab/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/981c02d861cf4d00b4cd42d5458bd7b3/artifacts/embeddings_test" diff --git a/configs/data/embedded/tissue_only/l5.yaml b/configs/data/embedded/tissue_only/l5.yaml new file mode 100644 index 00000000..d82eaa60 --- /dev/null +++ b/configs/data/embedded/tissue_only/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l5; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l5_tissue_embed +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_tissue_embed +uris: + all: "mlflow-artifacts:/3/e4beda97fdab43f08e8b71fc5bea733c/artifacts/embeddings" + train: "mlflow-artifacts:/3/cac061824af449af90376addd37bbc4b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d0c2242662ed4f81b3d10fa6864a36fd/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/74a96b20a8504705a4295212f2163c07/artifacts/embeddings_test" diff --git a/configs/data/tiled/tissue_only/l1.yaml b/configs/data/tiled/tissue_only/l1.yaml new file mode 100644 index 00000000..5a26fc40 --- /dev/null +++ b/configs/data/tiled/tissue_only/l1.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.1 +# Level-1 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l1; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l1_tissue +defaults: + - /data/levels/l1@_here_ + - _self_ + +name: l1_tissue +raw_uris: + all: "mlflow-artifacts:/3/43602e9f922d46f18dc06910b66a3be1/artifacts/mou_1_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l1_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l1_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l1_tissue_test" diff --git a/configs/data/tiled/tissue_only/l2.yaml b/configs/data/tiled/tissue_only/l2.yaml index 92849f62..f29e2ff4 100644 --- a/configs/data/tiled/tissue_only/l2.yaml +++ b/configs/data/tiled/tissue_only/l2.yaml @@ -5,7 +5,7 @@ defaults: name: l2_tissue raw_uris: - all: "mlflow-artifacts:/3/0d8e51f080df435bbaa9b189b7973aac/artifacts/mou_2_224_tissue" + all: "mlflow-artifacts:/3/c65f9ba257da4595837750ede54d4f25/artifacts/mou_2_224_tissue" train: "mlflow-artifacts:/3//artifacts/l2_tissue_train" val: "mlflow-artifacts:/3//artifacts/l2_tissue_val" test: "mlflow-artifacts:/3//artifacts/l2_tissue_test" diff --git a/configs/data/tiled/tissue_only/l3.yaml b/configs/data/tiled/tissue_only/l3.yaml index 631badef..6f85f6e3 100644 --- a/configs/data/tiled/tissue_only/l3.yaml +++ b/configs/data/tiled/tissue_only/l3.yaml @@ -5,7 +5,7 @@ defaults: name: l3_tissue raw_uris: - all: "mlflow-artifacts:/3/6b1ba82c3d29423f87004e5223e4cf99/artifacts/mou_3_224_tissue" + all: "mlflow-artifacts:/3/6948538ec16544ffb0e1ffda2b0a0eec/artifacts/mou_3_224_tissue" train: "mlflow-artifacts:/3//artifacts/l3_tissue_train" val: "mlflow-artifacts:/3//artifacts/l3_tissue_val" test: "mlflow-artifacts:/3//artifacts/l3_tissue_test" diff --git a/configs/data/tiled/tissue_only/l4.yaml b/configs/data/tiled/tissue_only/l4.yaml new file mode 100644 index 00000000..be3dbcd0 --- /dev/null +++ b/configs/data/tiled/tissue_only/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_tissue +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_tissue +raw_uris: + all: "mlflow-artifacts:/3/46b689cdd7a2450ea5c2f997a86e4561/artifacts/mou_4_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l4_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l4_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l4_tissue_test" diff --git a/configs/data/tiled/tissue_only/l5.yaml b/configs/data/tiled/tissue_only/l5.yaml new file mode 100644 index 00000000..f33ed802 --- /dev/null +++ b/configs/data/tiled/tissue_only/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (tissue masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_tissue +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_tissue +raw_uris: + all: "mlflow-artifacts:/3/f939531a6ff849cb8b97b34d030f8725/artifacts/mou_5_224_tissue" + train: "mlflow-artifacts:/3//artifacts/l5_tissue_train" + val: "mlflow-artifacts:/3//artifacts/l5_tissue_val" + test: "mlflow-artifacts:/3//artifacts/l5_tissue_test" diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml new file mode 100644 index 00000000..a35d4cd5 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l0.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-0 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l0 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l0 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l0_tissue_embed + path: ${dataset.levels.0.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml new file mode 100644 index 00000000..b3d00615 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l1.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-1 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l1 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l1 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l1_tissue_embed + path: ${dataset.levels.1.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml new file mode 100644 index 00000000..94c90b83 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l4.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-4 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l4 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l4 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l4_tissue_embed + path: ${dataset.levels.4.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml b/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml new file mode 100644 index 00000000..9619a9fe --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/tissue_only/l5.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-5 embeddings (tissue masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/tissue_only/l5 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/tissue_only/l5 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l5_tissue_embed + path: ${dataset.levels.5.raw_uris.all} From bde73f5f24d74ca98a4bbe6729852592fd12b0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 1 Aug 2026 19:11:44 +0200 Subject: [PATCH 56/79] feat: transformer configs --- .../ml/train_mil_embeddings_transformer.yaml | 47 +++++++++++++++++ ...rain_mil_embeddings_transformer_joint.yaml | 52 +++++++++++++++++++ ...mil_embeddings_transformer_regression.yaml | 49 +++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 configs/experiment/ml/train_mil_embeddings_transformer.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml diff --git a/configs/experiment/ml/train_mil_embeddings_transformer.yaml b/configs/experiment/ml/train_mil_embeddings_transformer.yaml new file mode 100644 index 00000000..38147102 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer.yaml @@ -0,0 +1,47 @@ +# @package _global_ +# TransMIL-style transformer MIL on precomputed level-3 embeddings, CLASSIFICATION +# of luminal type: identity encoder (frozen embeddings) -> transformer aggregation +# (CLS-token self-attention over the tile bag) -> linear head (out_dim=1). +# Transformer twin of train_mil_embeddings_att (label_mode=type, BCE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml new file mode 100644 index 00000000..be73c430 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml @@ -0,0 +1,52 @@ +# @package _global_ +# Multi-task transformer MIL on precomputed level-3 embeddings: identity encoder -> +# transformer aggregation (CLS-token self-attention over the tile bag) -> MLP head +# with out_dim=2. The head emits a class logit (col 0) and a regression scalar +# (col 1); JointLoss = w_cls * BCE(type) + w_reg * MSE(index) trains both at once. +# Transformer twin of train_mil_embeddings_joint (label_mode="both"). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_auroc + - joint_f1 + - joint_mse + - joint_pearson + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: TransMIL-style CLS-token self-attention aggregation + 2-output + MLP head trained jointly on luminal type (BCE) and the continuous MammaPrint + index (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml new file mode 100644 index 00000000..2cc2e6cd --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml @@ -0,0 +1,49 @@ +# @package _global_ +# TransMIL-style transformer MIL on precomputed level-3 embeddings, REGRESSION of +# the MammaPrint index: identity encoder (frozen embeddings) -> transformer +# aggregation (CLS-token self-attention over the tile bag) -> linear head (out_dim=1). +# Regression twin of train_mil_embeddings_transformer (label_mode=index, MSE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data: _template + - optional /data/embedded: null + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8) + +metadata: + experiment_name: MammaPrint + run_name: Transformer MIL — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention aggregation + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} From 4570f75cfd9aa1c9cdd7a2f4416b51ecaeb134ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 1 Aug 2026 21:50:46 +0200 Subject: [PATCH 57/79] feat: spatial --- configs/data/_spatial_template.yaml | 28 +++ ...in_mil_embeddings_spatial_transformer.yaml | 58 ++++++ ..._embeddings_spatial_transformer_joint.yaml | 62 +++++++ ...ddings_spatial_transformer_regression.yaml | 59 +++++++ .../ml/aggregator/spatial_transformer.yaml | 19 ++ ml/data/__init__.py | 2 + ml/data/datasets/spatial_scale.py | 54 ++++++ ml/models/aggregators/__init__.py | 2 + ml/models/aggregators/spatial_transformer.py | 166 ++++++++++++++++++ ml/typing.py | 8 +- 10 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 configs/data/_spatial_template.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml create mode 100644 configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml create mode 100644 configs/ml/aggregator/spatial_transformer.yaml create mode 100644 ml/data/datasets/spatial_scale.py create mode 100644 ml/models/aggregators/spatial_transformer.py diff --git a/configs/data/_spatial_template.yaml b/configs/data/_spatial_template.yaml new file mode 100644 index 00000000..533a6377 --- /dev/null +++ b/configs/data/_spatial_template.yaml @@ -0,0 +1,28 @@ +# @package _global_ +# Coordinate-aware single-scale data shape: identical to _template but the DataModule +# runs over SpatialScaleDataset, which appends each tile's level-0 (x, y) as the last +# two bag columns ((N, D + 2)). Pair with the spatial_transformer aggregator, which +# slices those columns off. Select one artifact card at launch, e.g. +# data/embedded=demo_l3_embed. +dataset: + levels: ??? # filled by the selected data/embedded/* card + data_mapping: ${project_path}/data_mapping.csv + +datamodule: + _target_: ml.data.DataModule + batch_size: 1 + num_workers: 0 + pin_memory: false + persistent_workers: false + train: &dataset + _target_: ml.data.SpatialScaleDataset + split: train + levels: ${dataset.levels} + data_mapping: ${dataset.data_mapping} + label_mode: ${label_mode} + val: + <<: *dataset + split: val + test: + <<: *dataset + split: test diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml new file mode 100644 index 00000000..97987f5b --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer.yaml @@ -0,0 +1,58 @@ +# @package _global_ +# Spatially-aware transformer MIL on precomputed level-3 embeddings, CLASSIFICATION +# of luminal type: identity encoder (frozen embeddings) -> SpatialTransformerMIL +# (TransMIL self-attention + 2D sinusoidal PE from each tile's (x, y)) -> linear head +# (out_dim=1). Coord-aware twin of train_mil_embeddings_transformer: swaps the data +# template to SpatialScaleDataset (bag carries the x/y columns) and the aggregator to +# spatial_transformer. label_mode=type, BCE. +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: bce + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - binary_accuracy + - binary_auroc + - binary_f1 + - binary_precision + - binary_recall + - binary_specificity + - binary_cohen_kappa + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: type +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL — luminal classification (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning classification of luminal type (A/B) over frozen + per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention with a 2D sinusoidal positional encoding built from each tile's + physical (x, y) coordinate, + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml new file mode 100644 index 00000000..caeb4118 --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_joint.yaml @@ -0,0 +1,62 @@ +# @package _global_ +# Multi-task spatially-aware transformer MIL on precomputed level-3 embeddings: +# identity encoder -> SpatialTransformerMIL (TransMIL self-attention + 2D sinusoidal +# PE from each tile's (x, y)) -> MLP head with out_dim=2. The head emits a class logit +# (col 0) and a regression scalar (col 1); JointLoss = w_cls * BCE(type) + w_reg * +# MSE(index) trains both at once. Coord-aware twin of +# train_mil_embeddings_transformer_joint (label_mode="both"). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: joint + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - joint_auroc + - joint_f1 + - joint_mse + - joint_pearson + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: both +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +# Two outputs: [class logit, regression scalar]. +ml: + head: + out_dim: 2 + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL joint — luminal classification + MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multi-task multiple-instance-learning over frozen per-slide level-3 Virchow2 + tile embeddings: TransMIL-style CLS-token self-attention with a 2D sinusoidal + positional encoding built from each tile's physical (x, y) coordinate, + 2-output + MLP head trained jointly on luminal type (BCE) and the continuous MammaPrint index + (MSE) via JointLoss. + hyperparameters: + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + w_cls: ${ml.loss.w_cls} + w_reg: ${ml.loss.w_reg} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml new file mode 100644 index 00000000..cefaf4cc --- /dev/null +++ b/configs/experiment/ml/train_mil_embeddings_spatial_transformer_regression.yaml @@ -0,0 +1,59 @@ +# @package _global_ +# Spatially-aware transformer MIL on precomputed level-3 embeddings, REGRESSION of +# the MammaPrint index: identity encoder (frozen embeddings) -> SpatialTransformerMIL +# (TransMIL self-attention + 2D sinusoidal PE from each tile's (x, y)) -> linear head +# (out_dim=1). Regression twin of train_mil_embeddings_spatial_transformer +# (label_mode=index, MSE). +defaults: + - /ml: module + - /ml/encoder@ml.encoder: identity + - /ml/aggregator@ml.aggregator: spatial_transformer + - /ml/head@ml.head: linear + - /ml/output_activation@ml.output_activation: identity + - /ml/loss@ml.loss: mse + - /ml/optimizer@ml.optimizer: adam + - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau + - /ml/metrics: + - mae + - mse + - pearson + - spearman + - r2 + - thresholded_f1 + - thresholded_precision + - thresholded_recall + - thresholded_specificity + - /trainer: default + - /data: _spatial_template + - optional /data/embedded: null + - _self_ + +label_mode: index +feature_dim: 2560 # Virchow2 embedding size (divisible by num_heads=8 and by 4) + +# Tile footprint facts for the PE grid-normalisation, pulled from the level-3 card. +# extent = tile_extent * mpp (level-0 px). Change the level index if you swap cards. +spatial: + mpp: ${dataset.levels.3.mpp} + tile_extent: ${dataset.levels.3.tile_extent} + +metadata: + experiment_name: MammaPrint + run_name: Spatial transformer MIL — MammaPrint index regression (level-3 Virchow2 embeddings) + description: | + Multiple-instance-learning regression of the continuous MammaPrint index over + frozen per-slide level-3 Virchow2 tile embeddings. TransMIL-style CLS-token + self-attention with a 2D sinusoidal positional encoding built from each tile's + physical (x, y) coordinate, + linear head. + hyperparameters: + encoder: ${ml.encoder._target_} + aggregator: ${ml.aggregator._target_} + num_heads: ${ml.aggregator.num_heads} + num_layers: ${ml.aggregator.num_layers} + pe_scale: ${ml.aggregator.pe_scale} + pe_span: ${ml.aggregator.pe_span} + head: ${ml.head._target_} + loss: ${ml.loss._target_} + lr: ${ml.optimizer.lr} + batch_size: ${datamodule.batch_size} + max_epochs: ${trainer.max_epochs} diff --git a/configs/ml/aggregator/spatial_transformer.yaml b/configs/ml/aggregator/spatial_transformer.yaml new file mode 100644 index 00000000..26640bd7 --- /dev/null +++ b/configs/ml/aggregator/spatial_transformer.yaml @@ -0,0 +1,19 @@ +# TransMIL-style self-attention MIL with an added 2D sinusoidal positional encoding +# built from each tile's level-0 pixel (x, y). Coordinates ride as the last two +# columns of the bag ((N, feature_dim + 2)); the aggregator slices them off, so +# feature_dim (NOT feature_dim + 2) must be divisible by num_heads and by 4. +# +# REQUIRES a coord-aware dataset (ml.data.SpatialScaleDataset) — select the spatial +# data template. tile_extent / mpp come from the chosen level card and set the +# level-0 tile footprint (extent = tile_extent * mpp) used to grid-normalise coords. +_target_: ml.models.aggregators.SpatialTransformerMIL +feature_dim: ${feature_dim} +tile_extent: ${spatial.tile_extent} +mpp: ${spatial.mpp} +num_heads: 8 +num_layers: 2 +mlp_ratio: 4.0 +dropout: 0.1 +pe_scale: 0.1 +pe_base: 10000.0 +pe_span: 64.0 diff --git a/ml/data/__init__.py b/ml/data/__init__.py index 6b1864f3..3aa9b514 100644 --- a/ml/data/__init__.py +++ b/ml/data/__init__.py @@ -6,6 +6,7 @@ # from ml.data.datasets.pyramid import PyramidSlideDataset from ml.data.datasets.raw_tiles import RawTileSlideDataset from ml.data.datasets.single_scale import SingleScaleDataset +from ml.data.datasets.spatial_scale import SpatialScaleDataset # Grouped (single-level first, multilevel last) to make the toggle obvious, so this @@ -14,6 +15,7 @@ "DataModule", "RawTileSlideDataset", "SingleScaleDataset", + "SpatialScaleDataset", "mil_collate", # ── Multilevel ── # "PyramidSlideDataset", diff --git a/ml/data/datasets/spatial_scale.py b/ml/data/datasets/spatial_scale.py new file mode 100644 index 00000000..08f0890a --- /dev/null +++ b/ml/data/datasets/spatial_scale.py @@ -0,0 +1,54 @@ +"""Coordinate-aware single-scale slide dataset. + +Identical to :class:`~ml.data.datasets.single_scale.SingleScaleDataset` except each +bag is widened from ``(N, D)`` to ``(N, D + 2)`` by appending the tile's level-0 +pixel ``(x, y)`` coordinates (the top-left corner) as the last two columns. Those +columns are already present in every embedding parquet (written by +``preprocessing/embeddings.py`` alongside the ``embedding`` column); this dataset +simply stops discarding them. + +The wide bag is consumed only by +:class:`~ml.models.aggregators.spatial_transformer.SpatialTransformerMIL`, which +slices the coordinate columns back off inside its ``forward``. Pair the two together +(coord-aware dataset + spatial aggregator); no other aggregator understands the extra +columns, and no other dataset produces them, so the flat single-scale path is +unaffected. +""" + +import numpy as np +import pandas as pd +import torch + +from ml.data.datasets.labels import get_label +from ml.data.datasets.single_scale import SingleScaleDataset +from ml.typing import MILSample, SlideMetadata + + +class SpatialScaleDataset(SingleScaleDataset): + """Per-slide bags of embeddings with appended ``(x, y)`` tile coordinates. + + Returns a standard :data:`~ml.typing.MILSample`; the only difference from the + parent is the bag width — ``(N, D + 2)`` instead of ``(N, D)``, with the last two + columns the level-0 pixel coordinates. Construction (split resolution, artifact + download, slide filtering, label mode) is inherited unchanged. + """ + + def __getitem__(self, idx: int) -> MILSample: + row = self.slides.iloc[idx] + frame = pd.read_parquet( + (self.embeddings_dir / row["name"]).with_suffix(".parquet") + ) + features = torch.from_numpy( + np.stack(frame["embedding"].to_numpy()) + ).float() # (N, D) + coords = torch.from_numpy( + frame[["x", "y"]].to_numpy() + ).float() # (N, 2), level-0 pixel top-left corners + bag = torch.cat([features, coords], dim=1) # (N, D + 2) + + label = get_label(row, self.label_mode) + metadata: SlideMetadata = {"slide_id": row["name"]} + return bag, label, metadata + + +__all__ = ["SpatialScaleDataset"] diff --git a/ml/models/aggregators/__init__.py b/ml/models/aggregators/__init__.py index 6fff616b..dc0ac140 100644 --- a/ml/models/aggregators/__init__.py +++ b/ml/models/aggregators/__init__.py @@ -5,6 +5,7 @@ from ml.models.aggregators.base import Aggregator from ml.models.aggregators.max import MaxPool from ml.models.aggregators.mean import MeanPool +from ml.models.aggregators.spatial_transformer import SpatialTransformerMIL from ml.models.aggregators.transformer import TransformerMIL @@ -17,6 +18,7 @@ "AttentionMIL", "MaxPool", "MeanPool", + "SpatialTransformerMIL", "TransformerMIL", # ── Multilevel ── # "MultiScaleMIL", diff --git a/ml/models/aggregators/spatial_transformer.py b/ml/models/aggregators/spatial_transformer.py new file mode 100644 index 00000000..09f7a0c9 --- /dev/null +++ b/ml/models/aggregators/spatial_transformer.py @@ -0,0 +1,166 @@ +"""Spatially-aware Transformer MIL aggregator (TransMIL + 2D positional encoding). + +The plain :class:`~ml.models.aggregators.transformer.TransformerMIL` is +permutation-invariant: it treats a slide's tiles as an unordered *set* and has no +notion of where each tile sits on the slide. This variant injects that spatial +signal by adding a **2D sinusoidal positional encoding**, computed from each tile's +physical ``(x, y)`` coordinate, to the tile features before the transformer runs. +Everything downstream (CLS-token self-attention, slide-vector readout, per-tile +attention map) is identical — this module *composes* ``TransformerMIL`` rather than +reimplementing it, so the readout has a single source of truth. + +Why sinusoidal (and not TransMIL's own PPEG / a learned coordinate table): + +* WSI tiles are a **sparse, non-rectangular** subset of the slide (tissue/mask + regions only), variable in count. PPEG-style schemes reshape the token sequence + into a dense ``sqrt(N) x sqrt(N)`` grid and convolve — inventing adjacency across real + tissue gaps and treats *sequence order* as position. That is wrong here. A + sinusoidal PE is a pure per-tile function of that tile's own coordinate, so gaps + and irregular shapes cost nothing and no ordering is assumed. +* It adds **zero learned parameters**, the safest choice against overfitting on the + small (per-slide-labelled) cohorts typical of pathology. + +Coordinates arrive as the **last two columns of the bag** (``(N, feature_dim + 2)``); +the coord-aware dataset (:class:`~ml.data.datasets.spatial_scale.SpatialScaleDataset`) +packs them there. Only this aggregator, paired with that dataset, ever sees the wide +bag — the flat aggregators keep receiving a plain ``(N, feature_dim)`` tensor. + +Positional context for MIL over strong frozen foundation-model features (Virchow2) +is a genuinely marginal signal for bag-of-tiles tasks (subtype / continuous-index), +so treat this as an ablation to compare against the permutation-invariant baselines, +not a default. See the module docstring rationale above for the correctness traps it +deliberately avoids. +""" + +import torch +from torch import Tensor + +from ml.models.aggregators.base import Aggregator +from ml.models.aggregators.transformer import TransformerMIL +from ml.typing import Bag + + +class SpatialTransformerMIL(Aggregator[Bag]): + """TransMIL self-attention with an added 2D sinusoidal positional encoding. + + The incoming bag is ``(N, feature_dim + 2)``: the first ``feature_dim`` columns + are tile features, the last two are the tile's level-0 pixel ``(x, y)`` (top-left + corner). Coordinates are converted to per-slide-normalised grid coordinates, a + sinusoidal PE is built from them and added to the features, and the result is fed + to an internal :class:`~ml.models.aggregators.transformer.TransformerMIL`. + + Args: + feature_dim: Dimensionality of the tile **features** ``D`` (not counting the + 2 coordinate columns). Must be divisible by ``num_heads`` and by 4 (the + PE splits ``D`` into two axis-halves, each a sin/cos pair stack). + tile_extent: Tile edge length in tiling pixels (from the level card). With + ``mpp`` this gives the level-0 footprint ``extent = tile_extent * mpp`` + used to turn pixel coordinates into grid indices. + mpp: Microns-per-pixel of the tiling level (from the level card). + num_heads: Attention heads (must divide ``feature_dim``). + num_layers: Transformer encoder layers. + mlp_ratio: Feed-forward width as a multiple of ``feature_dim``. + dropout: Dropout inside the transformer layers. + pe_scale: Multiplier on the PE before it is added to the features. Kept small + (default 0.1) so the encoding does not dominate the feature norm. + pe_base: Sinusoidal frequency base ``θ`` (as in Vaswani et al.). + pe_span: Span constant mapping normalised ``[0, 1]`` coordinates into a + usable phase range (larger = finer positional resolution). + """ + + def __init__( + self, + feature_dim: int, + tile_extent: int, + mpp: float, + num_heads: int = 8, + num_layers: int = 2, + mlp_ratio: float = 4.0, + dropout: float = 0.1, + pe_scale: float = 0.1, + pe_base: float = 10000.0, + pe_span: float = 64.0, + ) -> None: + super().__init__() + if feature_dim % 4 != 0: + raise ValueError( + f"feature_dim ({feature_dim}) must be divisible by 4 for the 2D " + "sin/cos positional encoding (two axis-halves of sin/cos pairs)." + ) + # feature_dim % num_heads is validated inside the composed TransformerMIL. + self.feature_dim = feature_dim + self.extent = tile_extent * mpp + if self.extent <= 0: + raise ValueError( + f"tile_extent * mpp must be positive; got {tile_extent} * {mpp}." + ) + self.pe_scale = pe_scale + self.pe_base = pe_base + self.pe_span = pe_span + + # Compose (don't copy) the plain transformer: it owns the CLS token, the + # encoder stack and the attention readout, so the interpretability map and + # slide-vector logic stay identical to the non-spatial aggregator. + self.transformer = TransformerMIL( + feature_dim=feature_dim, + num_heads=num_heads, + num_layers=num_layers, + mlp_ratio=mlp_ratio, + dropout=dropout, + ) + + @property + def out_dim(self) -> int: + return self.feature_dim + + def _positional_encoding(self, coords: Tensor) -> Tensor: + """Build a 2D sinusoidal PE ``(N, D)`` from level-0 pixel coords ``(N, 2)``. + + Pixels -> grid indices (``/ extent``) -> per-slide bbox min-max to ``[0, 1]`` + (translation-invariant, so absolute tissue position on the glass — a scanner + / sectioning artefact — cannot leak into the encoding). Half the channels + encode the x axis, half the y axis; within each half, sin/cos frequency pairs. + """ + grid = coords / self.extent # (N, 2), float grid indices (kept continuous) + + gmin = grid.min(dim=0).values # (2,) + gmax = grid.max(dim=0).values # (2,) + # clamp_min in *grid units* (1.0 = one tile): a single tile or a bag whose + # tiles share an x- or y-line has zero span on that axis -> normalise to 0 + # (a constant coordinate), which yields a constant PE and cannot divide by 0. + span = (gmax - gmin).clamp_min(1.0) + norm = (grid - gmin) / span # (N, 2) in [0, 1] + + half = self.feature_dim // 2 # channels per axis + num_freqs = half // 2 # sin/cos pairs per axis + freq_idx = torch.arange(num_freqs, device=coords.device, dtype=torch.float32) + # ω_k = 1 / θ^(2k / half); phase = normalised_coord * span * ω_k + inv_freq = 1.0 / (self.pe_base ** (2.0 * freq_idx / half)) # (num_freqs,) + + pe = coords.new_zeros(coords.shape[0], self.feature_dim) # (N, D) + for axis in (0, 1): + phase = norm[:, axis : axis + 1] * self.pe_span * inv_freq # (N, num_freqs) + base = axis * half + pe[:, base : base + half : 2] = torch.sin(phase) + pe[:, base + 1 : base + half : 2] = torch.cos(phase) + return pe + + def forward(self, bag: Tensor) -> tuple[Tensor, Tensor]: + # bag: (N, feature_dim + 2) — features then the (x, y) coordinate columns. + expected = self.feature_dim + 2 + if bag.shape[-1] != expected: + raise ValueError( + f"SpatialTransformerMIL expects a bag of width feature_dim + 2 " + f"({expected}); got {bag.shape[-1]}. Is it paired with " + "SpatialScaleDataset (which appends the x/y columns)?" + ) + features = bag[:, : self.feature_dim] # (N, D) + coords = bag[:, self.feature_dim :] # (N, 2) + + pe = self._positional_encoding(coords) # (N, D) + # The CLS token is added *inside* the composed transformer, after this — so it + # receives no positional term (it has no coordinate), as intended. + return self.transformer(features + self.pe_scale * pe) + + +__all__ = ["SpatialTransformerMIL"] diff --git a/ml/typing.py b/ml/typing.py index 9d4298ba..3265d868 100644 --- a/ml/typing.py +++ b/ml/typing.py @@ -38,7 +38,13 @@ class LevelSpec(TypedDict): # ── Single-level types (always shipped) ────────────────────────────────────── type Bag = Tensor -"""A bag of per-slide items: tile features ``(N, D)`` or raw tiles ``(N, C, H, W)``.""" +"""A bag of per-slide items: tile features ``(N, D)`` or raw tiles ``(N, C, H, W)``. + +The coordinate-aware single-scale path (``SpatialScaleDataset`` + +``SpatialTransformerMIL``) packs a wider ``(N, D + 2)`` tensor whose last two columns +are each tile's level-0 pixel ``(x, y)``; the spatial aggregator slices them off. No +other dataset or aggregator produces or consumes those extra columns. +""" type MILSample = tuple[Bag, Tensor, SlideMetadata] """A single-level slide-level training sample: ``(bag, label, metadata)``.""" From 5303c27aaf1d6adf863d465d108c7a580ee28e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 3 Aug 2026 22:26:20 +0200 Subject: [PATCH 58/79] feat: resume test --- ml/resume_test.py | 251 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 ml/resume_test.py diff --git a/ml/resume_test.py b/ml/resume_test.py new file mode 100644 index 00000000..f91509bc --- /dev/null +++ b/ml/resume_test.py @@ -0,0 +1,251 @@ +"""Re-run the test-set inference for a run whose post-fit test step died, and append +the results to that **same MLflow run**. + +Background: ``ml/train.py`` does ``fit`` then, if ``test_after_fit``, ``test`` on the +best checkpoint. When a slide download fails during that test step, the run ends up +with a fully trained model (its best checkpoint is logged under the run's +``checkpoints/best`` artifacts) and train/val metrics — but **no test metrics**. There +is no need to retrain; we only need to re-run inference and write it back. + +This entrypoint, given a run id (or run name), does exactly that: + + 1. reads the run's own logged Hydra overrides (``configs/hydra.yaml`` artifact), + so the data shape / model pieces match the original run exactly, + 2. downloads the run's best checkpoint (``checkpoints/best/checkpoint.ckpt``), + 3. re-invokes ``ml.train`` in ``mode=test`` with ``checkpoint=`` and + ``+logger.run_id=`` — Lightning's MLFlowLogger resumes the existing run + when ``run_id`` is set, so the new ``test/*`` metrics append to it. + +Console log: mlkit's StreamCapture re-logs the run's whole console to a single +``console.log`` artifact from an empty buffer, so the resume's ``test`` output would +overwrite the original training log. Before launching, we copy the existing +``console.log`` to ``console.train.log`` to preserve it (disable with +``--no-keep-console-log``). + +Must run in an environment with the data mounts and MLflow reachable — i.e. the same +pod shape the training job used (see ``scripts/ml/submit_resume_test.py``). + +Usage (inside such a pod): + uv run -m ml.resume_test --run-id + uv run -m ml.resume_test --run-name --experiment MammaPrint + uv run -m ml.resume_test --run-id --dry-run # print the command, run nothing +""" + +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +from mlflow import MlflowClient +from omegaconf import OmegaConf + + +DEFAULT_EXPERIMENT = "MammaPrint" + +# Artifact paths written by rationai.mlkit (autolog._log_config / MLFlowLogger). +HYDRA_ARTIFACT = "configs/hydra.yaml" # holds hydra.overrides.task (the CLI args) +BEST_CKPT_ARTIFACT = "checkpoints/best/checkpoint.ckpt" + +# mlkit's StreamCapture re-logs the run's whole console to this single path, from an +# empty buffer — so the resume's `test` output would OVERWRITE the training console +# log. We back the original up under this path before launching the subprocess. +CONSOLE_LOG = "console.log" +CONSOLE_LOG_BACKUP = "console.train.log" + +# Overrides we must drop from the replay: they steer stage/checkpoint/run identity, +# which resume_test sets itself. Anything matching `key=...` (with these keys) is cut. +DROP_OVERRIDE_KEYS = {"mode", "checkpoint", "test_after_fit"} + + +def _client(tracking_uri: str | None) -> MlflowClient: + # tracking_uri=None -> MlflowClient reads MLFLOW_TRACKING_URI from the env. + return MlflowClient(tracking_uri=tracking_uri) + + +def resolve_run_id( + mlflow_client: MlflowClient, run_id: str | None, run_name: str | None, experiment: str +) -> str: + """Return the run id, looking it up by run name in `experiment` if needed.""" + if run_id: + return run_id + exp = mlflow_client.get_experiment_by_name(experiment) + if exp is None: + raise SystemExit(f"MLflow experiment {experiment!r} not found.") + safe = run_name.replace("'", "\\'") + runs = mlflow_client.search_runs( + experiment_ids=[exp.experiment_id], + filter_string=f"attributes.run_name = '{safe}'", + ) + if not runs: + raise SystemExit(f"No MLflow run named {run_name!r} in {experiment!r}.") + if len(runs) > 1: + ids = ", ".join(r.info.run_id for r in runs) + raise SystemExit( + f"{len(runs)} runs named {run_name!r} in {experiment!r} ({ids}); " + f"pass --run-id to disambiguate." + ) + return runs[0].info.run_id + + +def original_overrides(mlflow_client: MlflowClient, run_id: str, dst: Path) -> list[str]: + """Read the run's original Hydra CLI overrides, minus the stage/checkpoint ones. + + ``autolog._log_config`` saves ``hydra.yaml`` via ``OmegaConf.save(HydraConfig.get())``. + ``HydraConfig.get()`` returns the *contents* of the ``hydra`` node, so the CLI args + live at ``overrides.task`` (no ``hydra.`` prefix). We try both keys to be safe. + """ + path = mlflow_client.download_artifacts(run_id, HYDRA_ARTIFACT, str(dst)) + hydra_cfg = OmegaConf.load(path) + task = OmegaConf.select(hydra_cfg, "overrides.task") + if task is None: + task = OmegaConf.select(hydra_cfg, "hydra.overrides.task") + task = list(task or []) + if not task: + raise SystemExit( + f"Run {run_id}: no Hydra overrides found in {HYDRA_ARTIFACT} " + f"(looked at overrides.task and hydra.overrides.task). Without them the " + f"experiment/data config can't be reconstructed. Pass the overrides " + f"manually, or inspect the artifact." + ) + + kept = [] + for ov in task: + key = ov.split("=", 1)[0].lstrip("+~") + if key in DROP_OVERRIDE_KEYS: + continue + kept.append(ov) + return kept + + +def download_checkpoint(mlflow_client: MlflowClient, run_id: str, dst: Path) -> str: + """Download the run's best checkpoint; return its local path.""" + # Fail early with a clear message if the fit never logged a checkpoint. + arts = {a.path for a in mlflow_client.list_artifacts(run_id, "checkpoints/best")} + if BEST_CKPT_ARTIFACT not in arts: + raise SystemExit( + f"Run {run_id} has no {BEST_CKPT_ARTIFACT!r} artifact " + f"(found: {sorted(arts) or 'nothing'}). Cannot resume test without a " + f"trained checkpoint — the fit likely did not complete." + ) + return mlflow_client.download_artifacts(run_id, BEST_CKPT_ARTIFACT, str(dst)) + + +def backup_console_log( + mlflow_client: MlflowClient, run_id: str, dst: Path, dry_run: bool +) -> bool: + """Preserve the original training console log before the resume overwrites it. + + The resume subprocess writes a fresh ``console.log`` (mlkit's StreamCapture starts + from an empty buffer and log_text overwrites), which would clobber the training + console output. We copy the existing ``console.log`` to ``console.train.log`` first. + Idempotent-ish: if a backup already exists we don't re-copy (so re-running the + resume doesn't overwrite the true original with a test-only log). Returns True if a + backup was made (or already present), False if there was no original to save. + """ + top = {a.path for a in mlflow_client.list_artifacts(run_id)} + if CONSOLE_LOG_BACKUP in top: + print(f" {CONSOLE_LOG_BACKUP!r} already exists; leaving it as the original.") + return True + if CONSOLE_LOG not in top: + print(f" No {CONSOLE_LOG!r} to preserve (nothing logged yet).") + return False + + if dry_run: + print(f" [dry-run] would copy {CONSOLE_LOG!r} -> {CONSOLE_LOG_BACKUP!r}.") + return True + + local = mlflow_client.download_artifacts(run_id, CONSOLE_LOG, str(dst)) + text = Path(local).read_text(encoding="utf-8") + mlflow_client.log_text(run_id, text, CONSOLE_LOG_BACKUP) + print(f" Preserved training log as {CONSOLE_LOG_BACKUP!r}.") + return True + + +def build_command(overrides: list[str], checkpoint: str, run_id: str) -> list[str]: + """The `ml.train` test-mode command that appends results to the existing run.""" + return [ + "uv", "run", "-m", "ml.train", + *overrides, + "mode=test", + f"checkpoint={checkpoint}", + "test_after_fit=false", + f"+logger.run_id={run_id}", + ] + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + src = parser.add_mutually_exclusive_group(required=True) + src.add_argument("--run-id", help="MLflow run id to resume test for.") + src.add_argument("--run-name", help="MLflow run name (== job name) to look up.") + parser.add_argument( + "--experiment", + default=DEFAULT_EXPERIMENT, + help=f"Experiment to search when using --run-name (default {DEFAULT_EXPERIMENT}).", + ) + parser.add_argument( + "--tracking-uri", + default=None, + help="MLflow tracking URI (defaults to MLFLOW_TRACKING_URI in the env).", + ) + parser.add_argument( + "--override", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra Hydra override to pass to ml.train, repeatable. When given, these " + "REPLACE the overrides read from the run's logged config (use if the run's " + "hydra.yaml has none, or to fix them).", + ) + parser.add_argument( + "--keep-console-log", + action=argparse.BooleanOptionalAction, + default=True, + help="Before running, copy the run's existing console.log to console.train.log " + "so the resume's test output doesn't clobber the training log (default: on).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Resolve everything and print the command, but don't run test.", + ) + args = parser.parse_args() + + mlflow_client = _client(args.tracking_uri) + run_id = resolve_run_id(mlflow_client, args.run_id, args.run_name, args.experiment) + print(f"Resuming test for run {run_id}.") + + # Keep the downloaded config + checkpoint for the whole subprocess lifetime. + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + if args.override: + overrides = args.override + print("Using overrides passed on the command line (skipping logged config).") + else: + overrides = original_overrides(mlflow_client, run_id, tmp / "cfg") + checkpoint = download_checkpoint(mlflow_client, run_id, tmp / "ckpt") + print(f"Replaying overrides: {overrides}") + print(f"Checkpoint: {checkpoint}") + + if args.keep_console_log: + print("Preserving the original console log:") + backup_console_log(mlflow_client, run_id, tmp / "console", args.dry_run) + + cmd = build_command(overrides, checkpoint, run_id) + printable = " ".join(cmd) + if args.dry_run: + print(f"[dry-run] would run:\n {printable}") + return 0 + + print(f"Running:\n {printable}") + # Inherit stdout/stderr so the test output streams; MLflow also captures it + # into the same run via the logger's stream capture. + result = subprocess.run(cmd) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) From 15a03bf156fcb3b6549b711a7f9b69393ae88479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 18 Aug 2026 12:48:49 +0200 Subject: [PATCH 59/79] feat: cancer masks --- configs/data/default.yaml | 1 + .../embeddings/cancer_mask/l0.yaml | 19 ++++++++ .../embeddings/cancer_mask/l1.yaml | 19 ++++++++ .../embeddings/cancer_mask/l2.yaml | 19 ++++++++ .../embeddings/cancer_mask/l3.yaml | 19 ++++++++ .../embeddings/cancer_mask/l4.yaml | 19 ++++++++ .../embeddings/cancer_mask/l5.yaml | 19 ++++++++ .../tiling/cancer_mask/mou_0_224.yaml | 11 +++++ .../tiling/cancer_mask/mou_1_224.yaml | 11 +++++ .../tiling/cancer_mask/mou_2_224.yaml | 11 +++++ .../tiling/cancer_mask/mou_3_224.yaml | 11 +++++ .../tiling/cancer_mask/mou_4_224.yaml | 11 +++++ .../tiling/cancer_mask/mou_5_224.yaml | 11 +++++ configs/preprocessing/tiling.yaml | 6 ++- preprocessing/tiling.py | 47 +++++++++++++++++++ 15 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml create mode 100644 configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml diff --git a/configs/data/default.yaml b/configs/data/default.yaml index b80ceaf6..5041b6ec 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -8,3 +8,4 @@ dataset: mlflow_uris: tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks epithelium_masks: null + cancer_masks: mlflow-artifacts:/2/70b1c915966d43e3b5be46668075bd93/artifacts/heatmaps diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml new file mode 100644 index 00000000..b9047c86 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l0.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-0 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l0 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l0 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l0_cancer_mask_embed + path: ${dataset.levels.0.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml new file mode 100644 index 00000000..0ceb1f8b --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-1 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l1 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l1 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l1_cancer_mask_embed + path: ${dataset.levels.1.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml new file mode 100644 index 00000000..5cd2e14b --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-2 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l2 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l2 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l2_cancer_mask_embed + path: ${dataset.levels.2.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml new file mode 100644 index 00000000..ab74847d --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-3 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l3 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l3 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l3_cancer_mask_embed + path: ${dataset.levels.3.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml new file mode 100644 index 00000000..f61093fe --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-4 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l4 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l4 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l4_cancer_mask_embed + path: ${dataset.levels.4.raw_uris.all} diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml new file mode 100644 index 00000000..f6d203e4 --- /dev/null +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml @@ -0,0 +1,19 @@ +# @package _global_ +# Level-5 embeddings (cancer_mask masks, Virchow2 features) over the WHOLE tiled +# dataset. Embed once, then split with +# scripts/preprocessing/split_dataset.py --kind embedded. +# +# Run: +# uv run -m preprocessing.embeddings \ +# +experiment=preprocessing/embeddings/cancer_mask/l5 +# +# The whole (un-split) tiled artifact URI lives in the matching raw-tile card's +# `raw_uris.all`; embeddings.py reads it via `dataset.path` below. +defaults: + - /data/tiled/cancer_mask/l5 + - /preprocessing/tile_encoder: virchow2 + - _self_ + +dataset: + name: l5_cancer_mask_embed + path: ${dataset.levels.5.raw_uris.all} diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml new file mode 100644 index 00000000..b0ad8795 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_0_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-0 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l0; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l0@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_0_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml new file mode 100644 index 00000000..0104c776 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_1_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-1 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l1; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l1@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_1_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml new file mode 100644 index 00000000..2ebc3707 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_2_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-2 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l2; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml new file mode 100644 index 00000000..9137334e --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_3_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-3 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l3; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml new file mode 100644 index 00000000..c8e36285 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_4_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-4 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l4; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_cancer_mask diff --git a/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml new file mode 100644 index 00000000..68a2e6df --- /dev/null +++ b/configs/experiment/preprocessing/tiling/cancer_mask/mou_5_224.yaml @@ -0,0 +1,11 @@ +# @package _global_ +# Level-5 tiling (cancer_mask masks). mpp/tile_extent/stride come from +# /data/levels/l5; coverage thresholds from /experiment/preprocessing/tiling/_thresholds. +# Only the output data_name is variant-specific. +# To re-derive thresholds for this level/variant, add overrides below _self_. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_cancer_mask diff --git a/configs/preprocessing/tiling.yaml b/configs/preprocessing/tiling.yaml index 4c44f275..301821d2 100644 --- a/configs/preprocessing/tiling.yaml +++ b/configs/preprocessing/tiling.yaml @@ -11,6 +11,7 @@ tissue_threshold: ??? blur_threshold: ??? folding_threshold: ??? residual_threshold: ??? +cancer_threshold: ??? diagnostics: false @@ -22,7 +23,7 @@ metadata: * reads slide metadata and builds a grid of tiles with the given stride * assigns tissue/epithelium/QC mask paths and computes overlap scores * keeps tiles with tissue in the centered 50% ROI - * computes full-tile overlaps for epithelium, blur, folding, and residual masks + * computes full-tile overlaps for epithelium, cancer, blur, folding, and residual masks * filters tiles by the per-mask coverage thresholds * saves slide and tile datasets via MLflow hyperparams: @@ -33,4 +34,5 @@ metadata: tissue_threshold: ${tissue_threshold} blur_threshold: ${blur_threshold} folding_threshold: ${folding_threshold} - residual_threshold: ${residual_threshold} \ No newline at end of file + residual_threshold: ${residual_threshold} + cancer_threshold: ${cancer_threshold} diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index bbe0d06e..0fc90fb7 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -74,6 +74,12 @@ def add_missing_epithelium_overlap(batch: DataBatch) -> DataBatch: return batch +def add_missing_cancer_overlap(batch: DataBatch) -> DataBatch: + """Fill a NaN ``cancer_overlap`` when no cancer mask is configured.""" + batch["cancer_overlap"] = np.nan + return batch + + def add_tile_overlap( tiles: Dataset, roi: Polygon, @@ -252,6 +258,7 @@ def tiling( tissue_masks_path: str | None, qc_masks_path: str | None, epithelium_masks_path: str | None, + cancer_masks_path: str | None, ) -> list[dict[str, Any]]: """Generate tile coordinates for a single slide.""" slide_path = Path(row["path"]) @@ -267,6 +274,10 @@ def tiling( else "" ) + cancer_mask = ( + str(Path(cancer_masks_path) / tiff_slide_name) if cancer_masks_path else "" + ) + if qc_masks_path: qc_path = Path(qc_masks_path) blur_mask = str(qc_path / f"Piqe_piqe_median_activity_mask_{tiff_slide_name}") @@ -288,6 +299,7 @@ def tiling( "mpp_y": row["mpp_y"], "tissue_mask_path": tissue_mask, "epithelium_mask_path": epithelium_mask, + "cancer_mask_path": cancer_mask, "blur_mask_path": blur_mask, "folding_mask_path": folding_mask, "residual_mask_path": residual_mask, @@ -324,6 +336,10 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: if config.dataset.mlflow_uris.epithelium_masks is None else download_artifacts(config.dataset.mlflow_uris.epithelium_masks) ) + cancer_masks_uri = config.dataset.mlflow_uris.get("cancer_masks") + cancer_masks_path = ( + None if cancer_masks_uri is None else download_artifacts(cancer_masks_uri) + ) qc_masks_path = config.dataset.paths.qc_masks or None # Read slide paths from data mapping @@ -373,6 +389,7 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: tissue_masks_path=tissue_masks_path, qc_masks_path=qc_masks_path, epithelium_masks_path=epithelium_masks_path, + cancer_masks_path=cancer_masks_path, ), num_cpus=0.25, ).repartition(target_num_rows_per_block=BATCH_SIZE) @@ -471,11 +488,41 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: batch_format="pandas", ) + # Cancer-mask discovery mode records the foreground fraction without filtering. + # Once a cutoff has been selected, setting cancer_threshold applies the production gate. + if cancer_masks_path: + print("[INFO] Computing overlap: cancer_overlap from cancer_mask_path") + tiles = add_tile_overlap( + tiles, + full_roi, + "cancer_mask_path", + "cancer_overlap", + keep="nonzero", + ) + cancer_threshold = config.get("cancer_threshold") + if cancer_threshold is None: + print( + "[INFO] Cancer threshold is unset; recording overlap without filtering" + ) + else: + print(f"[INFO] Filtering cancer_overlap > {cancer_threshold}") + tiles = tiles.filter(expr=col("cancer_overlap") > cancer_threshold) + else: + print( + "[INFO] Skipping cancer overlap: " + "dataset.mlflow_uris.cancer_masks is not set" + ) + tiles = tiles.map_batches( + add_missing_cancer_overlap, + batch_format="pandas", + ) + # Drop unnecessary columns tiles = tiles.drop_columns( [ "tissue_mask_path", "epithelium_mask_path", + "cancer_mask_path", "blur_mask_path", "folding_mask_path", "residual_mask_path", From 0907ab7a630d5aaa72457baf84a3d3bf3c137df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 18 Aug 2026 21:56:58 +0200 Subject: [PATCH 60/79] feat: missing masks --- preprocessing/tiling.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 0fc90fb7..32f9177f 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -19,7 +19,7 @@ from ratiopath.tiling.utils import row_hash from ray.data import Dataset from ray.data.block import DataBatch -from ray.data.expressions import col +from ray.data.expressions import col, lit from shapely.geometry import box from shapely.geometry.polygon import Polygon @@ -68,18 +68,6 @@ def _ray_init_kwargs() -> dict[str, Any]: return kwargs -def add_missing_epithelium_overlap(batch: DataBatch) -> DataBatch: - """Fill a NaN ``epithelium_overlap`` when no epithelium mask is configured.""" - batch["epithelium_overlap"] = np.nan - return batch - - -def add_missing_cancer_overlap(batch: DataBatch) -> DataBatch: - """Fill a NaN ``cancer_overlap`` when no cancer mask is configured.""" - batch["cancer_overlap"] = np.nan - return batch - - def add_tile_overlap( tiles: Dataset, roi: Polygon, @@ -483,10 +471,7 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: "[INFO] Skipping epithelium overlap: " "dataset.mlflow_uris.epithelium_masks is not set" ) - tiles = tiles.map_batches( - add_missing_epithelium_overlap, - batch_format="pandas", - ) + tiles = tiles.with_column("epithelium_overlap", lit(float("nan"))) # Cancer-mask discovery mode records the foreground fraction without filtering. # Once a cutoff has been selected, setting cancer_threshold applies the production gate. @@ -512,10 +497,7 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: "[INFO] Skipping cancer overlap: " "dataset.mlflow_uris.cancer_masks is not set" ) - tiles = tiles.map_batches( - add_missing_cancer_overlap, - batch_format="pandas", - ) + tiles = tiles.with_column("cancer_overlap", lit(float("nan"))) # Drop unnecessary columns tiles = tiles.drop_columns( From eda2b3ba8a648428c80b971d57f8a25aaa632571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Wed, 19 Aug 2026 18:09:00 +0200 Subject: [PATCH 61/79] fix cancer heatmap probability scoring --- preprocessing/tiling.py | 27 +++++++++++++++++++++------ scripts/preprocessing/run_tiling.py | 18 +++++++++++++++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 32f9177f..c8d10d52 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -73,7 +73,7 @@ def add_tile_overlap( roi: Polygon, path_col: str, overlap_col: str, - keep: Literal["zero", "nonzero"], + keep: Literal["zero", "nonzero", "mean"], background: str = "0", ) -> Dataset: """Compute a per-tile coverage score against a mask overlay. @@ -84,14 +84,17 @@ def add_tile_overlap( The masks are downsampled to a continuous ``[0, 255]`` range (they are NOT strictly binary): the background/clean pixel value is ``0`` and *any* non-zero value marks foreground (tissue) or - an artifact, depending on the mask. Two reductions are therefore supported via ``keep``: + an artifact, depending on the mask. Three reductions are supported via ``keep``: * ``keep="nonzero"`` (tissue/epithelium): score = fraction of non-zero pixels = foreground coverage. Keep tiles with a lot of tissue/epithelium. * ``keep="zero"`` (QC artifact masks: blur/folding/residual): score = fraction of ``0`` pixels = clean fraction. Keep tiles that are mostly artifact-free. + * ``keep="mean"`` (cancer probability heatmaps): score = mean pixel intensity divided by 255. + OpenSlide exposes grayscale TIFFs as repeated RGB channels, so the RGB mean remains the + original mean probability. - In both cases a downstream ``overlap > threshold`` filter keeps the desirable tiles. + In all cases a downstream ``overlap > threshold`` filter keeps the desirable tiles. The mask file path is read from the column ``path_col`` in the Ray Dataset. The function expects the dataset to contain tile coordinates and microns-per-pixel columns (``tile_x``, ``tile_y``, @@ -110,7 +113,8 @@ def add_tile_overlap( Name of the output column to store the computed score in [0, 1]. keep: Which pixels count as desirable: ``"zero"`` scores the clean (background) fraction, - ``"nonzero"`` scores the foreground fraction. + ``"nonzero"`` scores the foreground fraction, and ``"mean"`` scores normalized mean + intensity. background: Pixel value (as string) of the clean/background class. Defaults to ``"0"``. @@ -137,6 +141,16 @@ def score(s: Any) -> float: # No mask / no readable pixels in the ROI -> score 0 so the tile is dropped. if not isinstance(s, dict) or not s: return 0.0 + if keep == "mean": + weighted_sum = 0.0 + total_fraction = 0.0 + for value, fraction in s.items(): + fraction = float(fraction or 0.0) + weighted_sum += float(value) * fraction + total_fraction += fraction + if total_fraction == 0.0: + return 0.0 + return weighted_sum / total_fraction / 255.0 # Fraction of clean/background (== background value) pixels in the ROI. zero_fraction = s.get(background, 0.0) or 0.0 # keep="zero": clean fraction; keep="nonzero": foreground (1 - clean) fraction. @@ -473,7 +487,8 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: ) tiles = tiles.with_column("epithelium_overlap", lit(float("nan"))) - # Cancer-mask discovery mode records the foreground fraction without filtering. + # Cancer heatmaps store sigmoid probabilities as uint8 intensity (probability * 255). + # Discovery mode records the mean normalized probability without filtering. # Once a cutoff has been selected, setting cancer_threshold applies the production gate. if cancer_masks_path: print("[INFO] Computing overlap: cancer_overlap from cancer_mask_path") @@ -482,7 +497,7 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: full_roi, "cancer_mask_path", "cancer_overlap", - keep="nonzero", + keep="mean", ) cancer_threshold = config.get("cancer_threshold") if cancer_threshold is None: diff --git a/scripts/preprocessing/run_tiling.py b/scripts/preprocessing/run_tiling.py index 692c88fb..ba8aa946 100644 --- a/scripts/preprocessing/run_tiling.py +++ b/scripts/preprocessing/run_tiling.py @@ -6,6 +6,12 @@ # are forwarded to Ray via environment variables so tiling.py does not hard-code them. CPU = 48 MEMORY = "64Gi" +SHM = "32Gi" + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +EXPERIMENT = "preprocessing/tiling/cancer_mask/mou_3_224" +DATA_NAME = "mou_3_224_cancer_probability_threshold_scan" # Ray's auto-detection is unreliable on this cluster (it over-detects CPUs from the node and # under-sizes the object store), so we hand it the pod's real numbers. tiling.py reads these @@ -16,20 +22,26 @@ ] submit_job( - job_name="mammaprint-tiling-...", - username=..., + job_name="mammaprint-tiling-cancer-probability-l3-threshold-diagnostics", + username=USERNAME, image="cerit.io/rationai/base:2.0.6", cpu=CPU, memory=MEMORY, + shm=SHM, gpu=None, public=False, script=[ "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", + f"git checkout {GIT_BRANCH}", "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", *ray_env, "uv sync --frozen", - "uv run -m preprocessing.tiling +experiment=preprocessing/tiling/...", + ( + f"uv run -m preprocessing.tiling +experiment={EXPERIMENT} " + "cancer_threshold=null diagnostics=true " + f"data_name={DATA_NAME}" + ), ], storage=[storage.secure.DATA, storage.secure.PROJECTS], ) From a53ff45f4d2c550e518b04939e271c2735698867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 21 Aug 2026 10:57:28 +0200 Subject: [PATCH 62/79] fix: remove duplicate logging import --- ml/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ml/train.py b/ml/train.py index 1010bc32..4085614d 100644 --- a/ml/train.py +++ b/ml/train.py @@ -14,7 +14,6 @@ data/embedded=demo_l3_embed mode=test checkpoint=/path/best.ckpt """ -import logging import logging import random From f334c694499687826d01d15ebe38d3e2f1b996d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 21 Aug 2026 11:24:22 +0200 Subject: [PATCH 63/79] fix: restore level 2 embedded data config --- configs/data/embedded/tissue_only/l2.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 configs/data/embedded/tissue_only/l2.yaml diff --git a/configs/data/embedded/tissue_only/l2.yaml b/configs/data/embedded/tissue_only/l2.yaml new file mode 100644 index 00000000..18cbbbd2 --- /dev/null +++ b/configs/data/embedded/tissue_only/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 EMBEDDINGS card (tissue masks) — precomputed Virchow2 features. +# Physical facts come from /data/levels/l2; only name + URIs live here. +# `uris.all` is the whole (un-split) embeddings artifact written by the +# embeddings step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind embedded \ +# --uri --name l2_tissue_embed +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_tissue_embed +uris: + all: "mlflow-artifacts:/3//artifacts/l2_tissue_embed" + train: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_train" + val: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_val" + test: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_test" From 47994812eead50d9678e1fb27f5bfd9e7b42c854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 21 Aug 2026 11:31:57 +0200 Subject: [PATCH 64/79] feat: mask coverage threshold --- scripts/notebooks/mask_coverage_threshold.py | 316 ++++++++++++++++++ .../run_mask_coverage_threshold.py | 22 ++ 2 files changed, 338 insertions(+) create mode 100644 scripts/notebooks/mask_coverage_threshold.py create mode 100644 scripts/preprocessing/run_mask_coverage_threshold.py diff --git a/scripts/notebooks/mask_coverage_threshold.py b/scripts/notebooks/mask_coverage_threshold.py new file mode 100644 index 00000000..d4f49fce --- /dev/null +++ b/scripts/notebooks/mask_coverage_threshold.py @@ -0,0 +1,316 @@ +"""Analyze cancer-mask tile scores and choose a production filtering threshold. + +This is the non-interactive cluster version of mask_coverage_threshold.ipynb. +It is preconfigured for the corrected level-3 cancer-probability tiling run. +The script prints quantiles and exact keep/drop counts, then logs CSV tables and +a histogram/CDF figure to a small MLflow analysis run. +""" + +from __future__ import annotations + +import argparse +import os +import tempfile +from pathlib import Path + +import matplotlib + + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt +import mlflow +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +from mlflow.artifacts import download_artifacts + + +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DEFAULT_DATASET_URI = ( + "mlflow-artifacts:/3/41a0cfd625574ed9a5c547a8c254bc8a/artifacts/" + "mou_3_224_cancer_probability_threshold_scan" +) +DEFAULT_COLUMN = "cancer_overlap" +DEFAULT_THRESHOLDS = ( + 0.05, + 0.10, + 0.15, + 0.20, + 0.25, + 0.30, + 0.40, + 0.50, + 0.60, + 0.70, + 0.75, + 0.80, + 0.85, + 0.90, + 0.95, +) +QUANTILES = ( + 0.0, + 0.001, + 0.01, + 0.02, + 0.05, + 0.10, + 0.25, + 0.50, + 0.75, + 0.90, + 0.95, + 0.99, + 0.999, + 1.0, +) +LUMINAL_MAP = {"a luminal": "A", "b luminal": "B"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Analyze cancer_overlap from a tiled MLflow dataset.", + ) + parser.add_argument("--dataset-uri", default=DEFAULT_DATASET_URI) + parser.add_argument("--column", default=DEFAULT_COLUMN) + parser.add_argument("--tracking-uri", default=DEFAULT_TRACKING_URI) + parser.add_argument( + "--thresholds", + type=float, + nargs="+", + default=list(DEFAULT_THRESHOLDS), + ) + parser.add_argument("--experiment", default="MammaPrint") + parser.add_argument( + "--run-name", + default="Cancer probability threshold analysis l3", + ) + return parser.parse_args() + + +def resolve_tiles_path(dataset_uri: str) -> Path: + local_path = Path(download_artifacts(dataset_uri)) + if local_path.is_file(): + return local_path + tiles_path = local_path / "tiles.parquet" + if not tiles_path.is_file(): + raise FileNotFoundError( + f"Downloaded '{dataset_uri}', but '{tiles_path}' does not exist." + ) + return tiles_path + + +def load_tiles(tiles_path: Path, column: str) -> pd.DataFrame: + available_columns = set(pq.read_schema(tiles_path).names) + if column not in available_columns: + raise ValueError( + f"Column '{column}' is missing from {tiles_path}. " + f"Available columns: {sorted(available_columns)}" + ) + + columns = [column] + if "type" in available_columns: + columns.append("type") + tiles = pd.read_parquet(tiles_path, columns=columns, engine="pyarrow") + tiles[column] = pd.to_numeric(tiles[column], errors="coerce") + if "type" in tiles: + tiles["luminal_class"] = ( + tiles["type"].astype("string").str.strip().str.lower().map(LUMINAL_MAP) + ) + return tiles + + +def quantile_table(tiles: pd.DataFrame, column: str) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + groups.extend( + (f"luminal_{label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ) + + for group_name, values in groups: + clean = values.dropna() + quantiles = clean.quantile(QUANTILES) + for quantile, value in quantiles.items(): + rows.append( + { + "group": group_name, + "quantile": float(quantile), + "cancer_probability": float(value), + "tile_count": len(clean), + } + ) + return pd.DataFrame(rows) + + +def threshold_table( + tiles: pd.DataFrame, + column: str, + thresholds: list[float], +) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + groups.extend( + (f"luminal_{label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ) + + for group_name, values in groups: + clean = values.dropna() + total = len(clean) + for threshold in thresholds: + kept = int((clean > threshold).sum()) + dropped = total - kept + rows.append( + { + "group": group_name, + "threshold": threshold, + "total_tiles": total, + "kept_tiles": kept, + "dropped_tiles": dropped, + "kept_percent": 100.0 * kept / total if total else 0.0, + "dropped_percent": 100.0 * dropped / total if total else 0.0, + } + ) + return pd.DataFrame(rows) + + +def class_bias_table(thresholds: pd.DataFrame) -> pd.DataFrame: + class_rows = thresholds[thresholds["group"].isin(["luminal_A", "luminal_B"])] + if class_rows["group"].nunique() != 2: + return pd.DataFrame() + pivot = class_rows.pivot( + index="threshold", + columns="group", + values="kept_percent", + ).reset_index() + pivot["absolute_keep_rate_gap_pp"] = ( + pivot["luminal_A"] - pivot["luminal_B"] + ).abs() + return pivot + + +def plot_distribution( + tiles: pd.DataFrame, + column: str, + thresholds: list[float], + output_path: Path, +) -> None: + bins = np.linspace(0.0, 1.0, 201) + figure, (histogram_axis, cdf_axis) = plt.subplots(1, 2, figsize=(14, 5)) + + groups: list[tuple[str, pd.Series]] = [("all", tiles[column])] + if "luminal_class" in tiles: + class_groups = [ + (f"luminal {label}", group[column]) + for label, group in tiles.groupby("luminal_class", dropna=True) + ] + if class_groups: + groups = class_groups + + for label, values in groups: + clean = values.dropna().to_numpy() + counts, edges = np.histogram(clean, bins=bins) + centers = (edges[:-1] + edges[1:]) / 2 + density = counts / counts.sum() / np.diff(edges) if counts.sum() else counts + cdf = np.cumsum(counts) / counts.sum() if counts.sum() else counts + histogram_axis.plot(centers, density, label=label) + cdf_axis.plot(edges[1:], cdf, label=label) + + for threshold in thresholds: + histogram_axis.axvline(threshold, color="black", alpha=0.18, linewidth=0.7) + cdf_axis.axvline(threshold, color="black", alpha=0.18, linewidth=0.7) + + histogram_axis.set( + title=f"{column}: probability distribution", + xlabel="mean cancer probability", + ylabel="density", + xlim=(0.0, 1.0), + ) + cdf_axis.set( + title="CDF: fraction dropped at threshold", + xlabel="threshold", + ylabel="fraction with score ≤ threshold", + xlim=(0.0, 1.0), + ylim=(0.0, 1.0), + ) + histogram_axis.legend() + cdf_axis.legend() + figure.tight_layout() + figure.savefig(output_path, dpi=180) + plt.close(figure) + + +def main() -> None: + args = parse_args() + os.environ["MLFLOW_TRACKING_URI"] = args.tracking_uri + mlflow.set_tracking_uri(args.tracking_uri) + + print(f"Downloading tiled dataset: {args.dataset_uri}") + tiles_path = resolve_tiles_path(args.dataset_uri) + tiles = load_tiles(tiles_path, args.column) + values = tiles[args.column].dropna() + + print(f"Loaded {len(tiles):,} tile rows from {tiles_path}") + print(f"Valid {args.column} values: {len(values):,}") + print(f"Unique {args.column} values: {values.nunique():,}") + print("\nCancer probability summary:") + print( + values.describe( + percentiles=[0.001, 0.01, 0.02, 0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99, 0.999] + ).to_string() + ) + + quantiles = quantile_table(tiles, args.column) + thresholds = threshold_table(tiles, args.column, args.thresholds) + bias = class_bias_table(thresholds) + + print("\nThreshold keep/drop table (all tiles):") + print( + thresholds[thresholds["group"] == "all"].to_string( + index=False, + formatters={ + "kept_percent": "{:.2f}".format, + "dropped_percent": "{:.2f}".format, + }, + ) + ) + if not bias.empty: + print("\nLuminal A/B keep-rate comparison:") + print(bias.to_string(index=False, float_format=lambda value: f"{value:.2f}")) + + mlflow.set_experiment(args.experiment) + with tempfile.TemporaryDirectory() as directory: + output_dir = Path(directory) + quantiles.to_csv(output_dir / "cancer_probability_quantiles.csv", index=False) + thresholds.to_csv(output_dir / "cancer_threshold_keep_drop.csv", index=False) + if not bias.empty: + bias.to_csv(output_dir / "cancer_threshold_class_bias.csv", index=False) + plot_distribution( + tiles, + args.column, + args.thresholds, + output_dir / "cancer_probability_distribution_cdf.png", + ) + + with mlflow.start_run(run_name=args.run_name) as run: + mlflow.log_param("source_dataset_uri", args.dataset_uri) + mlflow.log_param("score_column", args.column) + mlflow.log_param("candidate_thresholds", ",".join(map(str, args.thresholds))) + mlflow.log_metric("tile_count", len(values)) + mlflow.log_metric("cancer_probability_mean", float(values.mean())) + mlflow.log_metric("cancer_probability_median", float(values.median())) + mlflow.log_artifacts(str(output_dir), artifact_path="threshold_analysis") + print(f"\nLogged analysis to MLflow run: {run.info.run_id}") + print( + "Analysis artifacts: " + f"mlflow-artifacts:/{run.info.experiment_id}/{run.info.run_id}/" + "artifacts/threshold_analysis" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_mask_coverage_threshold.py b/scripts/preprocessing/run_mask_coverage_threshold.py new file mode 100644 index 00000000..09f03479 --- /dev/null +++ b/scripts/preprocessing/run_mask_coverage_threshold.py @@ -0,0 +1,22 @@ +"""Submit the prefilled cancer-threshold analysis to the cluster.""" + +from kube_jobs import submit_job + + +submit_job( + job_name="mammaprint-cancer-probability-threshold-analysis-l3", + username="kissmi", + image="cerit.io/rationai/base:2.0.6", + cpu=4, + memory="16Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + "git checkout feat/tiling-values", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + "uv run python scripts/notebooks/mask_coverage_threshold.py", + ], +) From 87d7a49777aed85b29349f90a53fbc3e15b199ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 21 Aug 2026 11:41:16 +0200 Subject: [PATCH 65/79] feat: cancer threshold --- configs/experiment/preprocessing/tiling/_thresholds.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/experiment/preprocessing/tiling/_thresholds.yaml b/configs/experiment/preprocessing/tiling/_thresholds.yaml index 8cc2d41b..a6b2c548 100644 --- a/configs/experiment/preprocessing/tiling/_thresholds.yaml +++ b/configs/experiment/preprocessing/tiling/_thresholds.yaml @@ -8,6 +8,7 @@ # folding 0.1 - kept lenient: folding masks are slide-inconsistent, so a low gate avoids uneven # filtering while still removing majority-folded tiles. # residual 0.5 - drops heavily-contaminated tiles (some slides are ~66% residual artifact). +# cancer 0.5 - keeps tiles with mean cancer probability above 0.5. # Inherited across all levels; re-verify against a level's own distributions if # you re-tile it (mask resolution differs by mpp) by overriding these in that # experiment after `_thresholds` in its defaults list. @@ -15,3 +16,4 @@ tissue_threshold: 0.25 blur_threshold: 0.5 folding_threshold: 0.1 residual_threshold: 0.5 +cancer_threshold: 0.5 From ddbe971092f4ceecf582fca2a393e6e922e17931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 24 Aug 2026 12:20:19 +0200 Subject: [PATCH 66/79] fix: empty slides --- ml/data/datasets/single_scale.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ml/data/datasets/single_scale.py b/ml/data/datasets/single_scale.py index bf38b41f..6183312b 100644 --- a/ml/data/datasets/single_scale.py +++ b/ml/data/datasets/single_scale.py @@ -14,6 +14,7 @@ import numpy as np import pandas as pd +import pyarrow.parquet as pq import torch from torch.utils.data import Dataset @@ -30,6 +31,17 @@ logger = logging.getLogger(__name__) +def _empty_embedding_slides(directory: Path, slide_ids: set[str]) -> tuple[str, ...]: + """Return slide IDs whose Parquet files contain no tile rows.""" + return tuple( + sorted( + slide_id + for slide_id in slide_ids + if pq.read_metadata(directory / f"{slide_id}.parquet").num_rows == 0 + ) + ) + + class SingleScaleDataset(Dataset[MILSample]): """Per-slide flat bags of precomputed tile embeddings (one level). @@ -62,6 +74,17 @@ def __init__( slides = load_labeled_slides(data_mapping, self.label_mode) present = available_slides({self.level: self.embeddings_dir}) + self.empty_slides = _empty_embedding_slides(self.embeddings_dir, present) + if self.empty_slides: + logger.warning( + "Excluding %d slide(s) with empty embedding bags from the %s " + "split at level %d: %s", + len(self.empty_slides), + split, + self.level, + ", ".join(self.empty_slides), + ) + present.difference_update(self.empty_slides) self.slides = slides[slides["name"].isin(present)].reset_index(drop=True) def __len__(self) -> int: From c4a9139f6e36fe6373f3c01231e27a9035a627eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 28 Aug 2026 09:20:46 +0200 Subject: [PATCH 67/79] feat: split dataset --- preprocessing/split_dataset.py | 152 +++++++++++++++++++++ scripts/preprocessing/run_split_dataset.py | 118 ++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 preprocessing/split_dataset.py create mode 100644 scripts/preprocessing/run_split_dataset.py diff --git a/preprocessing/split_dataset.py b/preprocessing/split_dataset.py new file mode 100644 index 00000000..20acbd2f --- /dev/null +++ b/preprocessing/split_dataset.py @@ -0,0 +1,152 @@ +"""Materialise per-split MLflow artifacts from a whole dataset artifact.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import tempfile +from pathlib import Path + +import mlflow +import pandas as pd +import yaml +from mlflow.artifacts import download_artifacts +from rationai.tiling.writers import save_mlflow_dataset + + +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" +MLFLOW_EXPERIMENT = "MammaPrint" +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +SPLITS = ("train", "val", "test") + + +def _load_split_map(data_mapping_path: str) -> dict[str, str]: + """Return ``slide_stem -> split`` from data_mapping.csv, keyed by path stem.""" + df = pd.read_csv(data_mapping_path) + if "split" not in df.columns: + raise ValueError(f"'{data_mapping_path}' has no 'split' column.") + + df = df[df["path"].notna() & df["split"].notna()] + stems = df["path"].apply(lambda path: Path(str(path)).stem) + split_map = dict(zip(stems, df["split"].astype(str), strict=True)) + bad = {split for split in split_map.values() if split not in SPLITS} + if bad: + raise ValueError(f"Unexpected split values in '{data_mapping_path}': {bad}") + return split_map + + +def _assign_or_die(stems: list[str], split_map: dict[str, str]) -> dict[str, str]: + """Map each artifact stem to its split; abort if any stem is unmapped.""" + unmapped = sorted(stem for stem in stems if stem not in split_map) + if unmapped: + raise ValueError( + f"{len(unmapped)} slide(s) in the artifact have no split row in " + f"data_mapping.csv: {unmapped}. Add them (or fix the artifact) first." + ) + return {stem: split_map[stem] for stem in stems} + + +def _configure_mlflow() -> None: + tracking_uri = os.environ.get("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + mlflow.set_tracking_uri(tracking_uri) + mlflow.set_experiment(MLFLOW_EXPERIMENT) + print(f"Using MLflow tracking URI: {tracking_uri} (experiment: {MLFLOW_EXPERIMENT})") + + +def _uri_from_card(card_path: str) -> str: + """Read the whole artifact URI from a tiled or embedded level card.""" + card = yaml.safe_load(Path(card_path).read_text()) + uris = card.get("raw_uris") or card.get("uris") or {} + all_uri = uris.get("all") + if not all_uri: + raise ValueError( + f"Card '{card_path}' has no `raw_uris.all` / `uris.all` URI to split." + ) + return all_uri + + +def _name_from_uri(uri: str) -> str: + name = Path(uri.rstrip("/")).name + if name in {"", ".", "artifacts"}: + raise ValueError(f"Could not infer a dataset name from --uri '{uri}'. Pass --name.") + return name + + +def _split_embedded(local_dir: Path, name: str, split_map: dict[str, str]) -> None: + files = sorted(local_dir.glob("*.parquet")) + if not files: + raise ValueError(f"No .parquet files found under '{local_dir}'.") + assignment = _assign_or_die([file.stem for file in files], split_map) + + with tempfile.TemporaryDirectory(dir=os.getcwd()) as tmp: + split_dirs = {split: Path(tmp) / split for split in SPLITS} + for directory in split_dirs.values(): + directory.mkdir() + for file in files: + shutil.copy2(file, split_dirs[assignment[file.stem]] / file.name) + + for split in SPLITS: + count = sum(1 for value in assignment.values() if value == split) + dataset_name = f"{name}_{split}" + with mlflow.start_run(run_name=f"split-{dataset_name}"): + mlflow.log_artifacts(str(split_dirs[split]), dataset_name) + print(f" {split}: {count} slides -> {dataset_name}") + + +def _split_tiled(local_dir: Path, name: str, split_map: dict[str, str]) -> None: + slides = pd.read_parquet(local_dir / "slides.parquet") + tiles = pd.read_parquet(local_dir / "tiles.parquet") + slides = slides.assign(_stem=slides["path"].apply(lambda path: Path(str(path)).stem)) + assignment = _assign_or_die(slides["_stem"].tolist(), split_map) + slides = slides.assign(split=slides["_stem"].map(assignment)) + + for split in SPLITS: + split_slide_ids = slides.loc[slides["split"] == split, "id"] + split_slides = slides[slides["split"] == split].drop(columns=["_stem", "split"]) + split_tiles = tiles[tiles["slide_id"].isin(split_slide_ids)] + dataset_name = f"{name}_{split}" + with mlflow.start_run(run_name=f"split-{dataset_name}"): + save_mlflow_dataset( + slides=split_slides.reset_index(drop=True), + tiles=split_tiles.reset_index(drop=True), + dataset_name=dataset_name, + ) + print( + f" {split}: {len(split_slides)} slides, {len(split_tiles)} tiles " + f"-> {dataset_name}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Materialise per-split MLflow artifacts from data_mapping.csv." + ) + parser.add_argument("--kind", choices=["embedded", "tiled"], required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--card", help="Level card whose `all` URI is the artifact to split.") + source.add_argument("--uri", help="Whole artifact URI (overrides --card).") + parser.add_argument("--name", help="Output base name (default: inferred from URI).") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + args = parser.parse_args() + + split_map = _load_split_map(args.data_mapping) + _configure_mlflow() + uri = args.uri or _uri_from_card(args.card) + name = args.name or _name_from_uri(uri) + local_dir = Path(download_artifacts(artifact_uri=uri)) + print(f"Splitting {args.kind} artifact '{uri}' as '{name}_{{train,val,test}}'") + + if args.kind == "embedded": + _split_embedded(local_dir, name, split_map) + else: + _split_tiled(local_dir, name, split_map) + + print( + "Done. Point the level card at the three MLflow artifacts named:\n" + f" {name}_train\n {name}_val\n {name}_test" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_split_dataset.py b/scripts/preprocessing/run_split_dataset.py new file mode 100644 index 00000000..d6b56a21 --- /dev/null +++ b/scripts/preprocessing/run_split_dataset.py @@ -0,0 +1,118 @@ +"""Submit a Kubernetes job that materialises train/val/test MLflow artifacts. + +Examples:: + + uv run python scripts/preprocessing/run_split_dataset.py --kind tiled \ + --card configs/data/tiled/tissue_only/l5.yaml --name l5_tissue + + uv run python scripts/preprocessing/run_split_dataset.py --kind embedded \ + --uri mlflow-artifacts:/3//artifacts/embeddings \ + --name l5_tissue_embed + +The submitted pod runs ``preprocessing.split_dataset``. The projects volume is +mounted so the worker can read ``/mnt/projects/mammaprint/data_mapping.csv``. +""" + +from __future__ import annotations + +import argparse +import re +import shlex +from pathlib import Path + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +IMAGE = "cerit.io/rationai/base:2.0.6" +CPU = 8 +MEMORY = "64Gi" +SHM = "16Gi" +TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" + + +def _slug(value: str) -> str: + """Convert a dataset identifier to a Kubernetes-compatible name fragment.""" + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + + +def _default_job_name( + kind: str, card: str | None, uri: str | None, name: str | None +) -> str: + if name: + identifier = name + elif card: + path = Path(card) + identifier = f"{path.parent.name}-{path.stem}" + else: + identifier = Path((uri or "dataset").rstrip("/")).name + + job_name = f"mammaprint-split-{kind}-{_slug(identifier)}" + return job_name[:63].rstrip("-") + + +def _worker_command(args: argparse.Namespace) -> str: + command = [ + "uv", + "run", + "-m", + "preprocessing.split_dataset", + "--kind", + args.kind, + "--data-mapping", + args.data_mapping, + ] + if args.card: + command.extend(["--card", args.card]) + else: + command.extend(["--uri", args.uri]) + if args.name: + command.extend(["--name", args.name]) + return shlex.join(command) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Submit a Kubernetes job that splits one MLflow dataset artifact." + ) + parser.add_argument("--kind", choices=["embedded", "tiled"], required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--card", help="Repository-relative level card containing an `all` URI." + ) + source.add_argument("--uri", help="Whole MLflow artifact URI to split.") + parser.add_argument("--name", help="Output base name (default: inferred by the worker).") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + parser.add_argument("--job-name", help="Override the generated Kubernetes job name.") + args = parser.parse_args() + + job_name = args.job_name or _default_job_name( + args.kind, args.card, args.uri, args.name + ) + print(f"Submitting {job_name} with {CPU} CPUs and {MEMORY} RAM.") + submit_job( + job_name=job_name, + username=USERNAME, + image=IMAGE, + cpu=CPU, + memory=MEMORY, + shm=SHM, + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {shlex.quote(GIT_BRANCH)}", + f"export MLFLOW_TRACKING_URI={shlex.quote(TRACKING_URI)}", + "uv sync --frozen", + _worker_command(args), + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + print(f"Submitted {job_name}.") + + +if __name__ == "__main__": + main() From 71967c3fde0c9eec657677e8aa0831b2eb3c0d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Fri, 21 Aug 2026 21:25:23 +0200 Subject: [PATCH 68/79] feat: cancer masks --- configs/data/embedded/cancer_mask_5/l2.yaml | 14 ++ configs/data/embedded/cancer_mask_5/l3.yaml | 14 ++ configs/data/embedded/cancer_mask_5/l4.yaml | 14 ++ configs/data/embedded/cancer_mask_5/l5.yaml | 14 ++ configs/data/embedded/cancer_mask_9/l2.yaml | 14 ++ configs/data/embedded/cancer_mask_9/l3.yaml | 14 ++ configs/data/embedded/cancer_mask_9/l4.yaml | 14 ++ configs/data/embedded/cancer_mask_9/l5.yaml | 14 ++ configs/data/embedded/tissue_only/l2.yaml | 8 +- configs/data/tiled/cancer_mask_5/l1.yaml | 17 ++ configs/data/tiled/cancer_mask_5/l2.yaml | 17 ++ configs/data/tiled/cancer_mask_5/l3.yaml | 17 ++ configs/data/tiled/cancer_mask_5/l4.yaml | 17 ++ configs/data/tiled/cancer_mask_5/l5.yaml | 17 ++ configs/data/tiled/cancer_mask_9/l2.yaml | 17 ++ configs/data/tiled/cancer_mask_9/l3.yaml | 17 ++ configs/data/tiled/cancer_mask_9/l4.yaml | 17 ++ configs/data/tiled/cancer_mask_9/l5.yaml | 17 ++ .../ml/train_mil_embeddings_transformer.yaml | 1 - ...rain_mil_embeddings_transformer_joint.yaml | 5 +- ...mil_embeddings_transformer_regression.yaml | 3 - .../embeddings/cancer_mask/l1.yaml | 2 +- .../embeddings/cancer_mask/l2.yaml | 2 +- .../embeddings/cancer_mask/l3.yaml | 2 +- .../embeddings/cancer_mask/l4.yaml | 2 +- .../embeddings/cancer_mask/l5.yaml | 2 +- scripts/ml/run_train_sweep.py | 191 ++++++++++++++++++ 27 files changed, 469 insertions(+), 14 deletions(-) create mode 100644 configs/data/embedded/cancer_mask_5/l2.yaml create mode 100644 configs/data/embedded/cancer_mask_5/l3.yaml create mode 100644 configs/data/embedded/cancer_mask_5/l4.yaml create mode 100644 configs/data/embedded/cancer_mask_5/l5.yaml create mode 100644 configs/data/embedded/cancer_mask_9/l2.yaml create mode 100644 configs/data/embedded/cancer_mask_9/l3.yaml create mode 100644 configs/data/embedded/cancer_mask_9/l4.yaml create mode 100644 configs/data/embedded/cancer_mask_9/l5.yaml create mode 100644 configs/data/tiled/cancer_mask_5/l1.yaml create mode 100644 configs/data/tiled/cancer_mask_5/l2.yaml create mode 100644 configs/data/tiled/cancer_mask_5/l3.yaml create mode 100644 configs/data/tiled/cancer_mask_5/l4.yaml create mode 100644 configs/data/tiled/cancer_mask_5/l5.yaml create mode 100644 configs/data/tiled/cancer_mask_9/l2.yaml create mode 100644 configs/data/tiled/cancer_mask_9/l3.yaml create mode 100644 configs/data/tiled/cancer_mask_9/l4.yaml create mode 100644 configs/data/tiled/cancer_mask_9/l5.yaml create mode 100644 scripts/ml/run_train_sweep.py diff --git a/configs/data/embedded/cancer_mask_5/l2.yaml b/configs/data/embedded/cancer_mask_5/l2.yaml new file mode 100644 index 00000000..6a775896 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l2.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.2 +# Level-2 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/ac2fdbe10e6047a99186e16ac149825c/artifacts/embeddings" + train: "mlflow-artifacts:/3/b88229631c214c8e8d1161ce5f2ef23b/artifacts/l2_cancer_mask_5_embed_train" + val: "mlflow-artifacts:/3/d47d498ad0524ad2b14a140779a66e4f/artifacts/l2_cancer_mask_5_embed_val" + test: "mlflow-artifacts:/3/b1e75bf1510a4dc581d1c60ae4a061c5/artifacts/l2_cancer_mask_5_embed_test" diff --git a/configs/data/embedded/cancer_mask_5/l3.yaml b/configs/data/embedded/cancer_mask_5/l3.yaml new file mode 100644 index 00000000..45d849cf --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l3.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# Level-3 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/1c71b92b718642efb3698e6c5c5e7436/artifacts/embeddings" + train: "mlflow-artifacts:/3/1cbd1a158b6f44c6924608c8b99b8c7b/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c6680ed10e66455fa06e61b367f44ae0/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/48ffa80398c2468cba34738a54adc84e/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_5/l4.yaml b/configs/data/embedded/cancer_mask_5/l4.yaml new file mode 100644 index 00000000..e64066a1 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l4.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.4 +# Level-4 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/3492b1dae7be474f80cea02afe37d252/artifacts/embeddings" + train: "mlflow-artifacts:/3/2e018211e7bb4c299a3068678e798872/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/bb56dcfc27f64c62900e0413e881402a/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c6ed24afdc894c8bab90a160b9d96211/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_5/l5.yaml b/configs/data/embedded/cancer_mask_5/l5.yaml new file mode 100644 index 00000000..085d8479 --- /dev/null +++ b/configs/data/embedded/cancer_mask_5/l5.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.5 +# Level-5 embeddings selected with a cancer-probability threshold of 0.5. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask_5_embed +uris: + all: "mlflow-artifacts:/3/d2a3870bbe784ded95a878998513f8fd/artifacts/embeddings" + train: "mlflow-artifacts:/3/4d2342aed93d4686b1a5a464d47fc742/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/9b3c03fdc4844405b9df7877020fad70/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/7c99af8a7d2d4728bfa10c129bcdf862/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l2.yaml b/configs/data/embedded/cancer_mask_9/l2.yaml new file mode 100644 index 00000000..795f76c1 --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l2.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.2 +# Level-2 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/3f9706ef68f6434f89ed97d9031a8a6a/artifacts/embeddings" + train: "mlflow-artifacts:/3/e6e63894393440a49e809b826014606c/artifacts/l2_cancer_mask_9_embed_train" + val: "mlflow-artifacts:/3/5a487183ee21496b95a7d784190d4ee3/artifacts/l2_cancer_mask_9_embed_val" + test: "mlflow-artifacts:/3/e36817eb35de4d46a061ee8a8d8a5b7b/artifacts/l2_cancer_mask_9_embed_test" diff --git a/configs/data/embedded/cancer_mask_9/l3.yaml b/configs/data/embedded/cancer_mask_9/l3.yaml new file mode 100644 index 00000000..010d8669 --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l3.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.3 +# Level-3 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the complete whole-dataset Virchow2 artifact. Replace the +# split placeholders after running scripts/preprocessing/split_dataset.py. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/709170ca84d64c21bc37c487942ec48f/artifacts/embeddings" + train: "mlflow-artifacts:/3/a3af7a4b837248b18ca8b387b4262f9a/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/70d7ac7c2d154291857d47ed7def18df/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e5e5ef2e3b1b4a35a9ebb4b04dfdde14/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l4.yaml b/configs/data/embedded/cancer_mask_9/l4.yaml new file mode 100644 index 00000000..6a76261d --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l4.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.4 +# Level-4 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/84dda274e69c4eb8adc3a875eec8e482/artifacts/embeddings" + train: "mlflow-artifacts:/3/a0367e263d6a4bf7a70a5c8b3316c7c1/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/d1eda30624384c8eae75edf4ec07a508/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/8798109c8ec640079b2bb778590b0bde/artifacts/embeddings_test" diff --git a/configs/data/embedded/cancer_mask_9/l5.yaml b/configs/data/embedded/cancer_mask_9/l5.yaml new file mode 100644 index 00000000..9f9859fb --- /dev/null +++ b/configs/data/embedded/cancer_mask_9/l5.yaml @@ -0,0 +1,14 @@ +# @package dataset.levels.5 +# Level-5 embeddings selected with a cancer-probability threshold of 0.9. +# `uris.all` is the finished whole-dataset Virchow2 artifact. Materialise the +# split artifacts with scripts/preprocessing/split_dataset.py before training. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask_9_embed +uris: + all: "mlflow-artifacts:/3/3bdfcabbdf884380b3ece1e0f37bf3b7/artifacts/embeddings" + train: "mlflow-artifacts:/3/146e36cdab604071a47e25a2d2720b36/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/b4305a97ff34482b8210b31108c99f31/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/d89478ba8003457080794fe147b4f6f0/artifacts/embeddings_test" diff --git a/configs/data/embedded/tissue_only/l2.yaml b/configs/data/embedded/tissue_only/l2.yaml index 18cbbbd2..7bde776e 100644 --- a/configs/data/embedded/tissue_only/l2.yaml +++ b/configs/data/embedded/tissue_only/l2.yaml @@ -11,7 +11,7 @@ defaults: name: l2_tissue_embed uris: - all: "mlflow-artifacts:/3//artifacts/l2_tissue_embed" - train: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_train" - val: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_val" - test: "mlflow-artifacts:/3//artifacts/l2_tissue_embed_test" + all: "mlflow-artifacts:/3/d70b72d842604ebe93d3fbd9e1dfd2f5/artifacts/embeddings" + train: "mlflow-artifacts:/3/b11db2e5ec074e1da3c172ea41880c5c/artifacts/l2_tissue_only_mask_embed_train" + val: "mlflow-artifacts:/3/8134ea19af804e8aaf51680157575cf2/artifacts/l2_tissue_only_mask_embed_val" + test: "mlflow-artifacts:/3/d8479367bf224405b3a77bbc7b7ac289/artifacts/l2_tissue_only_mask_embed_test" diff --git a/configs/data/tiled/cancer_mask_5/l1.yaml b/configs/data/tiled/cancer_mask_5/l1.yaml new file mode 100644 index 00000000..4cbc190a --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l1.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.1 +# Level-1 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l1; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l1_cancer_mask +defaults: + - /data/levels/l1@_here_ + - _self_ + +name: l1_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/7651a68400f6484db36d3626b1d64e83/artifacts/mou_1_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l1_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l2.yaml b/configs/data/tiled/cancer_mask_5/l2.yaml new file mode 100644 index 00000000..81f77b8a --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l2; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l2_cancer_mask +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/698ec2dcda3a4b7aa57b250dbbbfaec0/artifacts/mou_2_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l3.yaml b/configs/data/tiled/cancer_mask_5/l3.yaml new file mode 100644 index 00000000..e82214f2 --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l3.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.3 +# Level-3 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l3; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l3_cancer_mask +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/06de9794199446c79c7edf732cfc3f82/artifacts/mou_3_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l4.yaml b/configs/data/tiled/cancer_mask_5/l4.yaml new file mode 100644 index 00000000..4518c0fd --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_cancer_mask +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/b02182a7b72241b184970623aadc1fbd/artifacts/mou_4_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_5/l5.yaml b/configs/data/tiled/cancer_mask_5/l5.yaml new file mode 100644 index 00000000..8884ed95 --- /dev/null +++ b/configs/data/tiled/cancer_mask_5/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_cancer_mask +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/e033d2f90707481a983326931159d89f/artifacts/mou_5_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l2.yaml b/configs/data/tiled/cancer_mask_9/l2.yaml new file mode 100644 index 00000000..a33078a0 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l2.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.2 +# Level-2 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l2; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l2_cancer_mask +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/f555b4f833f44c13b60789326ab83a5c/artifacts/mou_2_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l2_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l3.yaml b/configs/data/tiled/cancer_mask_9/l3.yaml new file mode 100644 index 00000000..92357ab9 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l3.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.3 +# Level-3 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l3; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l3_cancer_mask +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/182661b3b71c4f2d9a684d700f55dc7b/artifacts/mou_3_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l3_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l4.yaml b/configs/data/tiled/cancer_mask_9/l4.yaml new file mode 100644 index 00000000..44eb24c5 --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l4.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.4 +# Level-4 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l4; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l4_cancer_mask +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/931879ecd8704c99904a70cd15c0c3fc/artifacts/mou_4_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l4_cancer_mask_test" diff --git a/configs/data/tiled/cancer_mask_9/l5.yaml b/configs/data/tiled/cancer_mask_9/l5.yaml new file mode 100644 index 00000000..23ad67cb --- /dev/null +++ b/configs/data/tiled/cancer_mask_9/l5.yaml @@ -0,0 +1,17 @@ +# @package dataset.levels.5 +# Level-5 RAW-TILE card (cancer_mask masks). Physical facts come from +# /data/levels/l5; only the variant name + artifact URIs live here. +# `raw_uris.all` is the whole (un-split) tiled artifact read by the embeddings +# step. Per-split URIs are produced from it by: +# python scripts/preprocessing/split_dataset.py --kind tiled \ +# --uri --name l5_cancer_mask +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_cancer_mask +raw_uris: + all: "mlflow-artifacts:/3/a145a861300d46f19c2d3f7fe02ef5a6/artifacts/mou_5_224_cancer_mask" + train: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_train" + val: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_val" + test: "mlflow-artifacts:/3//artifacts/l5_cancer_mask_test" diff --git a/configs/experiment/ml/train_mil_embeddings_transformer.yaml b/configs/experiment/ml/train_mil_embeddings_transformer.yaml index 38147102..8a2bbf72 100644 --- a/configs/experiment/ml/train_mil_embeddings_transformer.yaml +++ b/configs/experiment/ml/train_mil_embeddings_transformer.yaml @@ -19,7 +19,6 @@ defaults: - binary_precision - binary_recall - binary_specificity - - binary_cohen_kappa - /trainer: default - /data: _template - optional /data/embedded: null diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml index be73c430..573938cb 100644 --- a/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml +++ b/configs/experiment/ml/train_mil_embeddings_transformer_joint.yaml @@ -14,10 +14,13 @@ defaults: - /ml/optimizer@ml.optimizer: adam - /ml/lr_scheduler@ml.lr_scheduler: reduce_lr_on_plateau - /ml/metrics: + - joint_accuracy - joint_auroc - joint_f1 + - joint_precision + - joint_recall + - joint_specificity - joint_mse - - joint_pearson - /trainer: default - /data: _template - optional /data/embedded: null diff --git a/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml index 2cc2e6cd..e26c8f22 100644 --- a/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml +++ b/configs/experiment/ml/train_mil_embeddings_transformer_regression.yaml @@ -15,9 +15,6 @@ defaults: - /ml/metrics: - mae - mse - - pearson - - spearman - - r2 - thresholded_f1 - thresholded_precision - thresholded_recall diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml index 0ceb1f8b..4b9e16e7 100644 --- a/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l1.yaml @@ -10,7 +10,7 @@ # The whole (un-split) tiled artifact URI lives in the matching raw-tile card's # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - - /data/tiled/cancer_mask/l1 + - /data/tiled/cancer_mask_5/l1 - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml index 5cd2e14b..82803462 100644 --- a/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l2.yaml @@ -10,7 +10,7 @@ # The whole (un-split) tiled artifact URI lives in the matching raw-tile card's # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - - /data/tiled/cancer_mask/l2 + - /data/tiled/cancer_mask_5/l2 - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml index ab74847d..7e0e29f7 100644 --- a/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l3.yaml @@ -10,7 +10,7 @@ # The whole (un-split) tiled artifact URI lives in the matching raw-tile card's # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - - /data/tiled/cancer_mask/l3 + - /data/tiled/cancer_mask_5/l3 - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml index f61093fe..bb64ffdb 100644 --- a/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l4.yaml @@ -10,7 +10,7 @@ # The whole (un-split) tiled artifact URI lives in the matching raw-tile card's # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - - /data/tiled/cancer_mask/l4 + - /data/tiled/cancer_mask_5/l4 - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml index f6d203e4..62d4e567 100644 --- a/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml +++ b/configs/experiment/preprocessing/embeddings/cancer_mask/l5.yaml @@ -10,7 +10,7 @@ # The whole (un-split) tiled artifact URI lives in the matching raw-tile card's # `raw_uris.all`; embeddings.py reads it via `dataset.path` below. defaults: - - /data/tiled/cancer_mask/l5 + - /data/tiled/cancer_mask_5/l5 - /preprocessing/tile_encoder: virchow2 - _self_ diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py new file mode 100644 index 00000000..78fd8fe4 --- /dev/null +++ b/scripts/ml/run_train_sweep.py @@ -0,0 +1,191 @@ +import itertools +import os +import time + +from kube_jobs import storage, submit_job + + +# Each objective has a self-consistent experiment (head out_dim, label_mode, +# output activation, loss and metrics are set together). The generic mean-pool +# experiments are used as the base because their metadata remains valid when the +# aggregator is overridden; the transformer-specific experiments interpolate +# transformer-only fields and cannot be composed with mean/max/attention. +EXPERIMENTS = { + "bce": "ml/train_mil_embeddings", + "regression": "ml/train_mil_embeddings_regression", + "joint": "ml/train_mil_embeddings_joint", +} +# The cancer_mask_5/l4 split is currently being materialised. Do not launch the +# sweep until its train/validation/test placeholders have been replaced. +DATASETS = { + "cm5-l4": "cancer_mask_5/l4", + "cm5-l5": "cancer_mask_5/l5", + "cm9-l4": "cancer_mask_9/l4", + "cm9-l5": "cancer_mask_9/l5", +} +AGGREGATORS = ("mean", "max", "attention", "transformer") +HEADS = ("mlp", "linear") +GPUS = ["A40"] +CODE_VERSION = 2 +SUBMISSION_INTERVAL_SECONDS = 45 * 60 +USERNAME = "kissmi" +POD_MLFLOW_TRACKING_URI = os.getenv( + "POD_MLFLOW_TRACKING_URI", "http://mlflow-s3.rationai-mlflow" +) +MLFLOW_LOOKUP_URI = os.getenv("MLFLOW_LOOKUP_URI") +MLFLOW_EXPERIMENT = "MammaPrint" +JOB_NAME_PREFIX = "mammaprint-train-cmv2-" + + +def _job_name( + dataset_key: str, + head: str, + aggregator: str, + objective: str, +) -> str: + return f"{JOB_NAME_PREFIX}{dataset_key}-{head}-{aggregator}-{objective}" + + +def _sweep_jobs() -> list[tuple[int, str, str, str, str, str, str]]: + combinations = itertools.product( + DATASETS.items(), + EXPERIMENTS.items(), + AGGREGATORS, + HEADS, + ) + return [ + ( + job_index, + dataset_key, + dataset_config, + objective, + experiment, + aggregator, + head, + ) + for job_index, ( + (dataset_key, dataset_config), + (objective, experiment), + aggregator, + head, + ) in enumerate(combinations) + ] + + +def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: + """Fetch sweep runs that already exist, including failed/running runs.""" + if MLFLOW_LOOKUP_URI is None: + print( + "Skipping MLflow lookup: set MLFLOW_LOOKUP_URI to a tracking endpoint " + "reachable from this machine to enable it." + ) + return set() + + from mlflow import MlflowClient + + mlflow_client = MlflowClient(tracking_uri=MLFLOW_LOOKUP_URI) + experiment = mlflow_client.get_experiment_by_name(MLFLOW_EXPERIMENT) + if experiment is None: + print(f"MLflow experiment {MLFLOW_EXPERIMENT!r} does not exist yet.") + return set() + + runs = mlflow_client.search_runs( + experiment_ids=[experiment.experiment_id], + filter_string=f"attributes.run_name LIKE '{JOB_NAME_PREFIX}%'", + max_results=50_000, + ) + matching_runs = [run for run in runs if run.info.run_name in expected_names] + existing = {run.info.run_name for run in matching_runs} + print(f"Fetched {len(existing)} existing sweep run(s) from MLflow.") + return existing + + +def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: + """Fetch jobs that were submitted but may not have reached MLflow yet.""" + from kube_jobs.constants import CLUSTER, JOB_NAMESPACE + from kubernetes import client + from kubernetes.config import load_kube_config + + load_kube_config(context=CLUSTER) + batch = client.BatchV1Api() + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={USERNAME}", + ).items + matching_jobs = [ + job + for job in jobs + if job.metadata is not None and job.metadata.name in expected_names + ] + existing = {job.metadata.name for job in matching_jobs} + print(f"Fetched {len(existing)} existing sweep job(s) from Kubernetes.") + return existing + + +def main() -> None: + jobs = _sweep_jobs() + expected_names = { + _job_name(dataset_key, head, aggregator, objective) + for _, dataset_key, _, objective, _, aggregator, head in jobs + } + kubernetes_names = _existing_kubernetes_job_names(expected_names) + mlflow_names = _existing_mlflow_run_names(expected_names) + existing_names = mlflow_names | kubernetes_names + pending_jobs = [ + job + for job in jobs + if _job_name(job[1], job[6], job[5], job[3]) not in existing_names + ] + + print( + f"Sweep contains {len(jobs)} jobs: {len(existing_names)} already submitted, " + f"{len(pending_jobs)} pending." + ) + + for pending_index, ( + job_index, + dataset_key, + dataset_config, + objective, + experiment, + aggregator, + head, + ) in enumerate(pending_jobs): + if pending_index > 0: + print("Waiting 45 minutes before submitting the next job.") + time.sleep(SUBMISSION_INTERVAL_SECONDS) + + name = _job_name(dataset_key, head, aggregator, objective) + gpu = GPUS[job_index % len(GPUS)] + print(f"Submitting {name} to {gpu} ({pending_index + 1}/{len(pending_jobs)}).") + submit_job( + job_name=name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=16, + memory="48Gi", + gpu=gpu, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + "git checkout feat/tiling-values", + f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", + # "export HF_TOKEN=", + "uv sync --frozen", + f""" + uv run -m ml.train +experiment={experiment} \ + data/embedded={dataset_config} \ + ml/aggregator={aggregator} \ + ml/head={head} \ + metadata.run_name={name} \ + +logger.tags.code_version={CODE_VERSION} \ + +logger.tags.data_variant={dataset_key} \ + """, + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + + +if __name__ == "__main__": + main() From dc04b723669c09b1777f9d19de76f115176ac5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 1 Sep 2026 11:27:43 +0200 Subject: [PATCH 69/79] feat: add all-mask tiling threshold scan --- configs/data/default.yaml | 2 +- .../preprocessing/tiling/_thresholds.yaml | 3 ++ .../tiling/threshold_scan/mou_3_224.yaml | 16 ++++++++ configs/preprocessing/tiling.yaml | 4 +- preprocessing/tiling.py | 37 +++++++++++++++---- .../notebooks/mask_coverage_threshold.ipynb | 11 +++--- scripts/preprocessing/run_tiling.py | 11 ++---- 7 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml diff --git a/configs/data/default.yaml b/configs/data/default.yaml index 5041b6ec..58c25533 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -7,5 +7,5 @@ dataset: data_mapping: ${project_path}/data_mapping.csv mlflow_uris: tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks - epithelium_masks: null + epithelium_masks: mlflow-artifacts:/3/7692760921564a069e2552f92bb5092a/artifacts/epithelium_masks cancer_masks: mlflow-artifacts:/2/70b1c915966d43e3b5be46668075bd93/artifacts/heatmaps diff --git a/configs/experiment/preprocessing/tiling/_thresholds.yaml b/configs/experiment/preprocessing/tiling/_thresholds.yaml index a6b2c548..77b2b3e7 100644 --- a/configs/experiment/preprocessing/tiling/_thresholds.yaml +++ b/configs/experiment/preprocessing/tiling/_thresholds.yaml @@ -8,6 +8,8 @@ # folding 0.1 - kept lenient: folding masks are slide-inconsistent, so a low gate avoids uneven # filtering while still removing majority-folded tiles. # residual 0.5 - drops heavily-contaminated tiles (some slides are ~66% residual artifact). +# epithelium null - discovery mode: record epithelium coverage without filtering until a cutoff +# is selected from the threshold-scan dataset. # cancer 0.5 - keeps tiles with mean cancer probability above 0.5. # Inherited across all levels; re-verify against a level's own distributions if # you re-tile it (mask resolution differs by mpp) by overriding these in that @@ -16,4 +18,5 @@ tissue_threshold: 0.25 blur_threshold: 0.5 folding_threshold: 0.1 residual_threshold: 0.5 +epithelium_threshold: null cancer_threshold: 0.5 diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml new file mode 100644 index 00000000..f988a0d0 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_3_224.yaml @@ -0,0 +1,16 @@ +# @package _global_ +# Full-cohort level-3 threshold-discovery run with every available mask attached. +# +# The established tissue/QC cutoffs remain active so the persisted dataset contains the same +# production-eligible tile population. diagnostics=true additionally logs the tissue/QC drop +# funnel before those filters are applied. Epithelium and cancer filtering stay disabled so their +# complete score distributions on the surviving tiles can be inspected in the threshold notebook. +defaults: + - /data/levels/l3@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_3_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/preprocessing/tiling.yaml b/configs/preprocessing/tiling.yaml index 301821d2..f70ab851 100644 --- a/configs/preprocessing/tiling.yaml +++ b/configs/preprocessing/tiling.yaml @@ -11,6 +11,7 @@ tissue_threshold: ??? blur_threshold: ??? folding_threshold: ??? residual_threshold: ??? +epithelium_threshold: ??? cancer_threshold: ??? diagnostics: false @@ -24,7 +25,7 @@ metadata: * assigns tissue/epithelium/QC mask paths and computes overlap scores * keeps tiles with tissue in the centered 50% ROI * computes full-tile overlaps for epithelium, cancer, blur, folding, and residual masks - * filters tiles by the per-mask coverage thresholds + * optionally filters tiles by the per-mask coverage thresholds * saves slide and tile datasets via MLflow hyperparams: mpp: ${mpp} @@ -35,4 +36,5 @@ metadata: blur_threshold: ${blur_threshold} folding_threshold: ${folding_threshold} residual_threshold: ${residual_threshold} + epithelium_threshold: ${epithelium_threshold} cancer_threshold: ${cancer_threshold} diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index c8d10d52..0bcc2727 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -63,7 +63,9 @@ def _ray_init_kwargs() -> dict[str, Any]: if memory_bytes: # Fraction of pod memory for the object store. The rest is left for Ray's heap and # the Python worker processes. Capped by /dev/shm at the pod level (see run_tiling.py). - kwargs["object_store_memory"] = int(float(memory_bytes) * RAY_OBJECT_STORE_FRACTION) + kwargs["object_store_memory"] = int( + float(memory_bytes) * RAY_OBJECT_STORE_FRACTION + ) return kwargs @@ -241,14 +243,23 @@ def pct(n: int) -> float: print(f" survived tissue gate: {n_tissue:,}") print(" independent drops (each gate alone; QC gates among tissue survivors):") for k in ("tissue", "blur", "folding", "residual"): - print(f" {k:9} {independent[k]:>12,} ({pct(independent[k]):5.2f}% of candidates)") + print( + f" {k:9} {independent[k]:>12,} ({pct(independent[k]):5.2f}% of candidates)" + ) print(" sequential drops (pipeline order; mutually exclusive):") for k in ("tissue", "blur", "folding", "residual"): - print(f" {k:9} {sequential[k]:>12,} ({pct(sequential[k]):5.2f}% of candidates)") - print(f" kept (survived all gates): {kept_final:,} ({pct(kept_final):5.2f}%)") + print( + f" {k:9} {sequential[k]:>12,} ({pct(sequential[k]):5.2f}% of candidates)" + ) + print( + f" kept (survived all gates): {kept_final:,} ({pct(kept_final):5.2f}%)" + ) - metrics = {"diag_candidate_tiles": n_candidates, "diag_tissue_survivors": n_tissue, - "diag_kept_final": kept_final} + metrics = { + "diag_candidate_tiles": n_candidates, + "diag_tissue_survivors": n_tissue, + "diag_kept_final": kept_final, + } for k in ("tissue", "blur", "folding", "residual"): metrics[f"diag_independent_drop_{k}"] = independent[k] metrics[f"diag_sequential_drop_{k}"] = sequential[k] @@ -469,8 +480,9 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: ) tiles = tiles.filter(expr=col("residual_overlap") > config.residual_threshold) - # Epithelium is computed but NOT filtered here (that mask is not always populated; add a cutoff - # once it is, by desired tumor purity). Computed last so it only runs on the surviving tiles. + # Epithelium is computed last so it only runs on tissue/QC survivors. Discovery runs leave the + # threshold unset to preserve the complete score distribution; production runs can set a + # tile-level coverage cutoff once the desired epithelial purity has been selected. if epithelium_masks_path: print("[INFO] Computing overlap: epithelium_overlap from epithelium_mask_path") tiles = add_tile_overlap( @@ -480,6 +492,15 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: "epithelium_overlap", keep="nonzero", ) + epithelium_threshold = config.get("epithelium_threshold") + if epithelium_threshold is None: + print( + "[INFO] Epithelium threshold is unset; recording overlap without " + "filtering" + ) + else: + print(f"[INFO] Filtering epithelium_overlap > {epithelium_threshold}") + tiles = tiles.filter(expr=col("epithelium_overlap") > epithelium_threshold) else: print( "[INFO] Skipping epithelium overlap: " diff --git a/scripts/notebooks/mask_coverage_threshold.ipynb b/scripts/notebooks/mask_coverage_threshold.ipynb index 5b7af36f..34d434a6 100644 --- a/scripts/notebooks/mask_coverage_threshold.ipynb +++ b/scripts/notebooks/mask_coverage_threshold.ipynb @@ -9,9 +9,10 @@ "\n", "The tiling pipeline (`preprocessing/tiling.py`) already computes a **per-tile coverage score** for\n", "each mask type and stores it in the tiled dataset's `tiles.parquet` as these columns:\n", - "`tissue_overlap`, `epithelium_overlap`, `blur_overlap`, `folding_overlap`, `residual_overlap`.\n", - "Each is in `[0, 1]` = fraction of desired (non-artifact) pixels; tiles are kept/dropped by\n", - "thresholding these.\n", + "`tissue_overlap`, `epithelium_overlap`, `cancer_overlap`, `blur_overlap`, `folding_overlap`,\n", + "`residual_overlap`. Each is in `[0, 1]`: coverage columns are fractions of desired pixels and\n", + "`cancer_overlap` is the mean normalized cancer probability. Tiles can be kept/dropped by\n", + "thresholding these scores.\n", "\n", "This notebook loads those columns straight from a tiled-dataset MLflow artifact and shows their\n", "**distribution + CDF** so you can pick sensible thresholds.\n", @@ -73,7 +74,7 @@ "id": "37f50707", "metadata": {}, "outputs": [], - "source": "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\nos.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\nmlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n\n# Tiled-dataset artifact dir (contains slides.parquet + tiles.parquet). Same style as the\n# TILED_DATASETS entries in check_mask_coverage.ipynb. Point this at your dataset.\nTILED_DATASET_URI = \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\"\n\n# The overlap columns to analyze (as stored in tiles.parquet). Drop any you don't care about.\nOVERLAP_COLUMNS = [\n \"tissue_overlap\",\n \"epithelium_overlap\",\n \"blur_overlap\",\n \"folding_overlap\",\n \"residual_overlap\",\n]\n\n# Columns whose stored values are inverted relative to the convention you want\n# (\"fraction of DESIRED pixels\", where higher = cleaner/keep). For these, the notebook plots\n# `1 - overlap` everywhere (loaded once, so summary + plots stay consistent).\n# NOTE: this only relabels the interpretation for viewing; it does NOT change the parquet.\n# If an upstream struct_field is genuinely wrong, fix it in preprocessing/tiling.py instead.\nINVERT_COLUMNS = [\"folding_overlap\", \"residual_overlap\"]\n\n# Column holding the luminal type on each tile row (for the optional A/B overlay). Set to None\n# to disable the per-class split. tiling.py copies `type` onto every tile.\nTYPE_COLUMN = \"type\"" + "source": "MLFLOW_TRACKING_URI = \"http://mlflow-s3.rationai-mlflow\"\nos.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\nmlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n\n# Tiled-dataset artifact dir (contains slides.parquet + tiles.parquet). Same style as the\n# TILED_DATASETS entries in check_mask_coverage.ipynb. Point this at your dataset.\nTILED_DATASET_URI = \"mlflow-artifacts:/3/0b32eceb1c434abf94104bb2c54b792e/artifacts/mou_3_224\"\n\n# The overlap columns to analyze (as stored in tiles.parquet). Drop any you don't care about.\nOVERLAP_COLUMNS = [\n \"tissue_overlap\",\n \"epithelium_overlap\",\n \"cancer_overlap\",\n \"blur_overlap\",\n \"folding_overlap\",\n \"residual_overlap\",\n]\n\n# All current overlap columns already use the notebook convention: higher means a larger\n# fraction of the desired/keep signal. Only list legacy columns here if an older artifact stored\n# them in the opposite direction.\nINVERT_COLUMNS = []\n\n# Column holding the luminal type on each tile row (for the optional A/B overlay). Set to None\n# to disable the per-class split. tiling.py copies `type` onto every tile.\nTYPE_COLUMN = \"type\"" }, { "cell_type": "markdown", @@ -170,4 +171,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/scripts/preprocessing/run_tiling.py b/scripts/preprocessing/run_tiling.py index ba8aa946..0042e7fb 100644 --- a/scripts/preprocessing/run_tiling.py +++ b/scripts/preprocessing/run_tiling.py @@ -10,8 +10,7 @@ USERNAME = "kissmi" GIT_BRANCH = "feat/tiling-values" -EXPERIMENT = "preprocessing/tiling/cancer_mask/mou_3_224" -DATA_NAME = "mou_3_224_cancer_probability_threshold_scan" +EXPERIMENT = "preprocessing/tiling/threshold_scan/mou_3_224" # Ray's auto-detection is unreliable on this cluster (it over-detects CPUs from the node and # under-sizes the object store), so we hand it the pod's real numbers. tiling.py reads these @@ -22,7 +21,7 @@ ] submit_job( - job_name="mammaprint-tiling-cancer-probability-l3-threshold-diagnostics", + job_name="mammaprint-tiling-all-masks-l3-threshold-scan", username=USERNAME, image="cerit.io/rationai/base:2.0.6", cpu=CPU, @@ -37,11 +36,7 @@ "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", *ray_env, "uv sync --frozen", - ( - f"uv run -m preprocessing.tiling +experiment={EXPERIMENT} " - "cancer_threshold=null diagnostics=true " - f"data_name={DATA_NAME}" - ), + f"uv run -m preprocessing.tiling +experiment={EXPERIMENT}", ], storage=[storage.secure.DATA, storage.secure.PROJECTS], ) From efeba15545519b945e483527e8a8610c365dd21d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 3 Sep 2026 13:02:33 +0200 Subject: [PATCH 70/79] feat: epithelium masks --- configs/data/default.yaml | 2 +- preprocessing/tiling.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/configs/data/default.yaml b/configs/data/default.yaml index 58c25533..f6b75a2c 100644 --- a/configs/data/default.yaml +++ b/configs/data/default.yaml @@ -4,8 +4,8 @@ dataset: name: full_dataset paths: qc_masks: ${project_path}/qc_masks + epithelium_masks: ${project_path}/epithelium_onnx_masks data_mapping: ${project_path}/data_mapping.csv mlflow_uris: tissue_masks: mlflow-artifacts:/3/1d67612f123e46f6a3c6183257106512/artifacts/tissue_masks - epithelium_masks: mlflow-artifacts:/3/7692760921564a069e2552f92bb5092a/artifacts/epithelium_masks cancer_masks: mlflow-artifacts:/2/70b1c915966d43e3b5be46668075bd93/artifacts/heatmaps diff --git a/preprocessing/tiling.py b/preprocessing/tiling.py index 0bcc2727..c69ebb37 100644 --- a/preprocessing/tiling.py +++ b/preprocessing/tiling.py @@ -344,11 +344,9 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: if config.dataset.mlflow_uris.tissue_masks is None else download_artifacts(config.dataset.mlflow_uris.tissue_masks) ) - epithelium_masks_path = ( - None - if config.dataset.mlflow_uris.epithelium_masks is None - else download_artifacts(config.dataset.mlflow_uris.epithelium_masks) - ) + # Epithelium masks are generated into persistent project storage, so use them in + # place just like the QC masks rather than trying to download an MLflow artifact. + epithelium_masks_path = config.dataset.paths.get("epithelium_masks") or None cancer_masks_uri = config.dataset.mlflow_uris.get("cancer_masks") cancer_masks_path = ( None if cancer_masks_uri is None else download_artifacts(cancer_masks_uri) @@ -504,7 +502,7 @@ def attach_labels(batch: pd.DataFrame) -> pd.DataFrame: else: print( "[INFO] Skipping epithelium overlap: " - "dataset.mlflow_uris.epithelium_masks is not set" + "dataset.paths.epithelium_masks is not set" ) tiles = tiles.with_column("epithelium_overlap", lit(float("nan"))) From 8ba088eb2823bef3e210151a9ad9c5ee2fef74db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 7 Sep 2026 10:48:07 +0200 Subject: [PATCH 71/79] feat: add staggered all-mask tiling scans --- .../tiling/threshold_scan/mou_2_224.yaml | 13 +++ .../tiling/threshold_scan/mou_4_224.yaml | 13 +++ .../tiling/threshold_scan/mou_5_224.yaml | 13 +++ .../run_tiling_threshold_scans.py | 83 +++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml create mode 100644 configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml create mode 100644 scripts/preprocessing/run_tiling_threshold_scans.py diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml new file mode 100644 index 00000000..f9c9cc83 --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_2_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-2 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l2@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_2_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml new file mode 100644 index 00000000..4ca1ae3c --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_4_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-4 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l4@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_4_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml b/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml new file mode 100644 index 00000000..861d1b1f --- /dev/null +++ b/configs/experiment/preprocessing/tiling/threshold_scan/mou_5_224.yaml @@ -0,0 +1,13 @@ +# @package _global_ +# Full-cohort level-5 scan with every available mask attached. Tissue/QC use the established +# production gates; epithelium and cancer are scored without filtering so downstream variants can +# be derived from exactly the same tile population. +defaults: + - /data/levels/l5@_global_ + - /experiment/preprocessing/tiling/_thresholds + - _self_ + +data_name: mou_5_224_all_masks_threshold_scan +epithelium_threshold: null +cancer_threshold: null +diagnostics: true diff --git a/scripts/preprocessing/run_tiling_threshold_scans.py b/scripts/preprocessing/run_tiling_threshold_scans.py new file mode 100644 index 00000000..9878d3ad --- /dev/null +++ b/scripts/preprocessing/run_tiling_threshold_scans.py @@ -0,0 +1,83 @@ +import argparse +import time + +from humanfriendly import parse_size +from kube_jobs import storage, submit_job + + +CPU = 48 +MEMORY = "64Gi" +SHM = "32Gi" + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +DEFAULT_LEVELS = (2, 4, 5) +DEFAULT_INTERVAL_MINUTES = 30 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Submit full-cohort all-mask tiling scans at a fixed interval." + ) + parser.add_argument( + "--levels", + nargs="+", + type=int, + choices=range(6), + default=DEFAULT_LEVELS, + ) + parser.add_argument( + "--interval-minutes", + type=float, + default=DEFAULT_INTERVAL_MINUTES, + ) + return parser.parse_args() + + +def submit_level(level: int) -> None: + experiment = f"preprocessing/tiling/threshold_scan/mou_{level}_224" + job_name = f"mammaprint-tiling-all-masks-l{level}-threshold-scan" + ray_env = [ + f"export RAY_NUM_CPUS={CPU}", + f"export RAY_MEMORY_BYTES={parse_size(MEMORY, binary=True)}", + ] + + print(f"Submitting {job_name}.", flush=True) + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=CPU, + memory=MEMORY, + shm=SHM, + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + *ray_env, + "uv sync --frozen", + f"uv run -m preprocessing.tiling +experiment={experiment}", + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + + +def main() -> None: + args = parse_args() + interval_seconds = args.interval_minutes * 60 + + for index, level in enumerate(args.levels): + if index: + print( + f"Waiting {args.interval_minutes:g} minutes before the next submission.", + flush=True, + ) + time.sleep(interval_seconds) + submit_level(level) + + +if __name__ == "__main__": + main() From f929849bcc1b82d72d063e55f1cce1e495b77cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Mon, 7 Sep 2026 15:21:24 +0200 Subject: [PATCH 72/79] feat: filter embedded tiles by mask scores --- preprocessing/filter_embeddings.py | 436 ++++++++++++++++++ .../preprocessing/run_filter_embeddings.py | 90 ++++ 2 files changed, 526 insertions(+) create mode 100644 preprocessing/filter_embeddings.py create mode 100644 scripts/preprocessing/run_filter_embeddings.py diff --git a/preprocessing/filter_embeddings.py b/preprocessing/filter_embeddings.py new file mode 100644 index 00000000..f344345f --- /dev/null +++ b/preprocessing/filter_embeddings.py @@ -0,0 +1,436 @@ +"""Filter existing tissue-only tile embeddings using scored mask overlaps.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import mlflow +import numpy as np +import pandas as pd +from mlflow.artifacts import download_artifacts + + +DEFAULT_DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" +DEFAULT_EXPERIMENT = "MammaPrint" +DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +SPLITS = ("train", "val", "test") +REQUIRED_SCORE_COLUMNS = ( + "slide_id", + "x", + "y", + "epithelium_overlap", + "cancer_overlap", +) +REQUIRED_EMBEDDING_COLUMNS = ("x", "y", "embedding") + + +@dataclass(frozen=True) +class Strategy: + slug: str + logic: str + + +def _threshold_token(value: float) -> str: + return f"{value:g}".replace(".", "") + + +def _strategies( + epithelium_threshold: float, cancer_threshold: float +) -> tuple[Strategy, ...]: + epithelium = _threshold_token(epithelium_threshold) + cancer = _threshold_token(cancer_threshold) + return ( + Strategy(f"epithel_{epithelium}", "epithelium_only"), + Strategy(f"epithel_or_cancer_{epithelium}_{cancer}", "or"), + Strategy(f"epithel_and_cancer_{epithelium}_{cancer}", "and"), + ) + + +def _selection_masks( + scores: pd.DataFrame, + epithelium_threshold: float, + cancer_threshold: float, +) -> dict[str, np.ndarray]: + epithelium = scores["epithelium_overlap"].to_numpy() > epithelium_threshold + cancer = scores["cancer_overlap"].to_numpy() > cancer_threshold + return { + "epithelium_only": epithelium, + "or": epithelium | cancer, + "and": epithelium & cancer, + } + + +def _require_columns( + frame: pd.DataFrame, required: tuple[str, ...], source: Path +) -> None: + missing = sorted(set(required) - set(frame.columns)) + if missing: + raise ValueError(f"{source} is missing required columns: {missing}") + + +def _load_split_map(data_mapping_path: Path) -> dict[str, str]: + mapping = pd.read_csv(data_mapping_path) + _require_columns(mapping, ("path", "split"), data_mapping_path) + mapping = mapping[mapping["path"].notna() & mapping["split"].notna()] + stems = mapping["path"].map(lambda path: Path(str(path)).stem) + if stems.duplicated().any(): + duplicates = sorted(stems[stems.duplicated(keep=False)].unique()) + raise ValueError( + f"Duplicate slide stems in {data_mapping_path}: {duplicates[:10]}" + ) + + split_map = dict(zip(stems, mapping["split"].astype(str), strict=True)) + unexpected = sorted(set(split_map.values()) - set(SPLITS)) + if unexpected: + raise ValueError( + f"Unexpected split values in {data_mapping_path}: {unexpected}" + ) + return split_map + + +def _load_scores( + tiling_dir: Path, expected_level: int +) -> tuple[dict[str, pd.DataFrame], int]: + slides_path = tiling_dir / "slides.parquet" + tiles_path = tiling_dir / "tiles.parquet" + slides = pd.read_parquet(slides_path, columns=["id", "path", "level"]) + tiles = pd.read_parquet(tiles_path, columns=list(REQUIRED_SCORE_COLUMNS)) + _require_columns(slides, ("id", "path", "level"), slides_path) + _require_columns(tiles, REQUIRED_SCORE_COLUMNS, tiles_path) + + actual_levels = sorted(slides["level"].dropna().unique().tolist()) + if actual_levels != [expected_level]: + raise ValueError( + f"Expected a level-{expected_level} tiling artifact, found levels {actual_levels}" + ) + + if slides["id"].duplicated().any(): + raise ValueError(f"Duplicate slide IDs in {slides_path}") + slides = slides.assign(stem=slides["path"].map(lambda path: Path(str(path)).stem)) + if slides["stem"].duplicated().any(): + duplicates = sorted( + slides.loc[slides["stem"].duplicated(keep=False), "stem"].unique() + ) + raise ValueError(f"Duplicate slide stems in {slides_path}: {duplicates[:10]}") + + id_to_stem = dict(zip(slides["id"], slides["stem"], strict=True)) + unknown_ids = sorted(set(tiles["slide_id"].unique()) - set(id_to_stem)) + if unknown_ids: + raise ValueError(f"Tiles reference unknown slide IDs: {unknown_ids[:10]}") + if tiles[["epithelium_overlap", "cancer_overlap"]].isna().any().any(): + null_counts = tiles[["epithelium_overlap", "cancer_overlap"]].isna().sum() + raise ValueError(f"Mask-score columns contain null values:\n{null_counts}") + + empty_scores = tiles.drop(columns="slide_id").iloc[:0].copy() + by_stem = {stem: empty_scores.copy() for stem in slides["stem"]} + for slide_id, frame in tiles.groupby("slide_id", sort=False): + scores = frame.drop(columns="slide_id").reset_index(drop=True) + if scores.duplicated(["x", "y"]).any(): + raise ValueError( + f"Duplicate scored coordinates for slide {id_to_stem[slide_id]}" + ) + by_stem[id_to_stem[slide_id]] = scores + return by_stem, len(tiles) + + +def _prepare_output_dirs( + root: Path, strategies: tuple[Strategy, ...] +) -> dict[str, dict[str, Path]]: + output_dirs: dict[str, dict[str, Path]] = {} + for strategy in strategies: + directories = {"all": root / strategy.slug / "embeddings"} + directories.update( + {split: root / strategy.slug / f"embeddings_{split}" for split in SPLITS} + ) + for directory in directories.values(): + directory.mkdir(parents=True) + output_dirs[strategy.logic] = directories + return output_dirs + + +def _link_to_split(source: Path, destination: Path) -> None: + try: + os.link(source, destination) + except OSError: + # A hard link can fail if a caller places temporary output across filesystems. + shutil.copy2(source, destination) + + +def filter_embeddings( + scores_by_stem: dict[str, pd.DataFrame], + embeddings_dir: Path, + output_dirs: dict[str, dict[str, Path]], + split_map: dict[str, str], + epithelium_threshold: float, + cancer_threshold: float, + logger: logging.Logger, +) -> tuple[dict[str, dict[str, int]], int]: + embedding_files = sorted(embeddings_dir.glob("*.parquet")) + if not embedding_files: + raise ValueError(f"No per-slide Parquet files found in {embeddings_dir}") + + embedding_stems = {path.stem for path in embedding_files} + missing_embeddings = sorted(set(scores_by_stem) - embedding_stems) + if missing_embeddings: + raise ValueError( + f"{len(missing_embeddings)} scored slides have no embedding file: " + f"{missing_embeddings[:10]}" + ) + unmapped_splits = sorted(embedding_stems - set(split_map)) + if unmapped_splits: + raise ValueError( + f"{len(unmapped_splits)} embedded slides have no split mapping: {unmapped_splits[:10]}" + ) + + counts = { + logic: {"tiles": 0, "empty_slides": 0, "slides_below_100_tiles": 0} + for logic in output_dirs + } + source_embedding_tiles = 0 + + for index, embeddings_path in enumerate(embedding_files, start=1): + stem = embeddings_path.stem + source_embeddings = pd.read_parquet(embeddings_path) + _require_columns(source_embeddings, REQUIRED_EMBEDDING_COLUMNS, embeddings_path) + if source_embeddings.duplicated(["x", "y"]).any(): + raise ValueError(f"Duplicate embedded coordinates in {embeddings_path}") + + scores = scores_by_stem.get(stem) + if scores is None: + # Extra embedding slides are allowed only when the scored tiling set contains no tiles + # for them. They are preserved as empty bags in every derived dataset. + scores = pd.DataFrame( + columns=["x", "y", "epithelium_overlap", "cancer_overlap"] + ) + + merged = scores.merge( + source_embeddings, + on=["x", "y"], + how="left", + validate="one_to_one", + indicator=True, + ) + unmatched = merged["_merge"].ne("both") + if unmatched.any(): + examples = merged.loc[unmatched, ["x", "y"]].head().to_dict("records") + raise ValueError( + f"{int(unmatched.sum())} scored coordinates for {stem} have no embedding: {examples}" + ) + merged = merged.drop(columns="_merge") + source_embedding_tiles += len(source_embeddings) + masks = _selection_masks(merged, epithelium_threshold, cancer_threshold) + split = split_map[stem] + + for logic, selected in masks.items(): + output = merged.loc[selected, list(REQUIRED_EMBEDDING_COLUMNS)] + output_path = output_dirs[logic]["all"] / embeddings_path.name + output.to_parquet(output_path, index=False, engine="pyarrow") + _link_to_split( + output_path, output_dirs[logic][split] / embeddings_path.name + ) + counts[logic]["tiles"] += len(output) + counts[logic]["empty_slides"] += int(output.empty) + counts[logic]["slides_below_100_tiles"] += int(len(output) < 100) + + if index % 100 == 0 or index == len(embedding_files): + logger.info("Filtered %d/%d slides", index, len(embedding_files)) + + return counts, source_embedding_tiles + + +def _log_outputs( + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + output_dirs: dict[str, dict[str, Path]], + strategies: tuple[Strategy, ...], + counts: dict[str, dict[str, int]], + scored_tiles: int, + source_embedding_tiles: int, + epithelium_threshold: float, + cancer_threshold: float, + console_log: Path, + logger: logging.Logger, +) -> None: + for strategy in strategies: + dataset_name = f"l{level}_{strategy.slug}_embed" + manifest = { + "schema_version": 1, + "dataset_name": dataset_name, + "level": level, + "strategy": strategy.logic, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + "comparison": ">", + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + "scored_tiles": scored_tiles, + "source_embedding_tiles": source_embedding_tiles, + **counts[strategy.logic], + } + + with mlflow.start_run(run_name=f"filter-{dataset_name}") as run: + mlflow.log_params( + { + "dataset_name": dataset_name, + "level": level, + "strategy": strategy.logic, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + } + ) + mlflow.set_tags( + { + "data_variant": strategy.slug, + "source_tiling_uri": tiling_uri, + "source_embeddings_uri": embeddings_uri, + } + ) + mlflow.log_metrics( + {key: float(value) for key, value in counts[strategy.logic].items()} + ) + for split_name, directory in output_dirs[strategy.logic].items(): + artifact_path = ( + "embeddings" if split_name == "all" else f"embeddings_{split_name}" + ) + logger.info("Logging %s to %s", dataset_name, artifact_path) + mlflow.log_artifacts(str(directory), artifact_path=artifact_path) + mlflow.log_dict(manifest, "filter_manifest.json") + for handler in logger.handlers: + handler.flush() + experiment_id = run.info.experiment_id + run_id = run.info.run_id + logger.info("Created %s (run %s)", dataset_name, run_id) + for split_name in ("all", *SPLITS): + artifact_path = ( + "embeddings" if split_name == "all" else f"embeddings_{split_name}" + ) + logger.info( + " %s: mlflow-artifacts:/%s/%s/artifacts/%s", + split_name, + experiment_id, + run_id, + artifact_path, + ) + for handler in logger.handlers: + handler.flush() + mlflow.log_artifact(str(console_log)) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Filter existing tissue-only embeddings into epithelium-only, epithelium OR cancer, " + "and epithelium AND cancer datasets." + ) + ) + parser.add_argument("--level", type=int, required=True) + parser.add_argument("--tiling-uri", required=True) + parser.add_argument("--embeddings-uri", required=True) + parser.add_argument( + "--embeddings-path", + help="Existing mounted embedding directory; falls back to --embeddings-uri if absent.", + ) + parser.add_argument("--epithelium-threshold", type=float, default=0.25) + parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument("--data-mapping", default=DEFAULT_DATA_MAPPING) + parser.add_argument("--work-dir", default="/mnt/projects/mammaprint") + parser.add_argument( + "--tracking-uri", default=os.getenv("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + ) + parser.add_argument("--experiment-name", default=DEFAULT_EXPERIMENT) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + mlflow.set_tracking_uri(args.tracking_uri) + mlflow.set_experiment(args.experiment_name) + strategies = _strategies(args.epithelium_threshold, args.cancer_threshold) + + with tempfile.TemporaryDirectory( + prefix=f"filter-embeddings-l{args.level}-", dir=args.work_dir + ) as tmp: + root = Path(tmp) + console_log = root / "console.log" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s", + handlers=[logging.StreamHandler(), logging.FileHandler(console_log)], + force=True, + ) + logger = logging.getLogger(__name__) + logger.info("Downloading scored tiling artifact: %s", args.tiling_uri) + tiling_download_dir = root / "source-tiling" + tiling_download_dir.mkdir() + tiling_dir = Path( + download_artifacts(args.tiling_uri, dst_path=str(tiling_download_dir)) + ) + + mounted_embeddings = ( + Path(args.embeddings_path) if args.embeddings_path is not None else None + ) + if mounted_embeddings is not None and mounted_embeddings.is_dir(): + embeddings_dir = mounted_embeddings + logger.info("Using mounted tissue embeddings: %s", embeddings_dir) + else: + if mounted_embeddings is not None: + logger.warning( + "Mounted embedding directory %s is unavailable; downloading %s", + mounted_embeddings, + args.embeddings_uri, + ) + else: + logger.info( + "Downloading tissue embedding artifact: %s", args.embeddings_uri + ) + embeddings_download_dir = root / "source-embeddings" + embeddings_download_dir.mkdir() + embeddings_dir = Path( + download_artifacts( + args.embeddings_uri, + dst_path=str(embeddings_download_dir), + ) + ) + + scores_by_stem, scored_tiles = _load_scores(tiling_dir, args.level) + split_map = _load_split_map(Path(args.data_mapping)) + output_dirs = _prepare_output_dirs(root / "outputs", strategies) + counts, source_embedding_tiles = filter_embeddings( + scores_by_stem=scores_by_stem, + embeddings_dir=embeddings_dir, + output_dirs=output_dirs, + split_map=split_map, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + logger=logger, + ) + logger.info("Selection summary:\n%s", json.dumps(counts, indent=2)) + _log_outputs( + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + output_dirs=output_dirs, + strategies=strategies, + counts=counts, + scored_tiles=scored_tiles, + source_embedding_tiles=source_embedding_tiles, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + console_log=console_log, + logger=logger, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_filter_embeddings.py b/scripts/preprocessing/run_filter_embeddings.py new file mode 100644 index 00000000..8c518843 --- /dev/null +++ b/scripts/preprocessing/run_filter_embeddings.py @@ -0,0 +1,90 @@ +"""Submit one CPU-only embedding-filtering job.""" + +from __future__ import annotations + +import argparse +import shlex + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +TISSUE_EMBEDDING_URIS = { + 2: "mlflow-artifacts:/3/d70b72d842604ebe93d3fbd9e1dfd2f5/artifacts/embeddings", + 3: "mlflow-artifacts:/3/c513c87eac3d4792ba9757a2d7300eb6/artifacts/embeddings", + 4: "mlflow-artifacts:/3/ecfc0ecdba5b44d9becc5c9df1103a16/artifacts/embeddings", + 5: "mlflow-artifacts:/3/e4beda97fdab43f08e8b71fc5bea733c/artifacts/embeddings", +} +TISSUE_EMBEDDING_PATHS = { + level: f"/mnt/projects/mammaprint/embeddings/l{level}_tissue_embed" + for level in TISSUE_EMBEDDING_URIS +} + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--level", type=int, choices=sorted(TISSUE_EMBEDDING_URIS), required=True + ) + parser.add_argument("--tiling-uri", required=True) + parser.add_argument("--embeddings-uri") + parser.add_argument("--embeddings-path") + parser.add_argument("--epithelium-threshold", type=float, default=0.25) + parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def _token(value: float) -> str: + return f"{value:g}".replace(".", "") + + +def main() -> None: + args = _parse_args() + embeddings_uri = args.embeddings_uri or TISSUE_EMBEDDING_URIS[args.level] + embeddings_path = args.embeddings_path or TISSUE_EMBEDDING_PATHS[args.level] + job_name = ( + f"mammaprint-filter-embeddings-l{args.level}-" + f"epi{_token(args.epithelium_threshold)}-c{_token(args.cancer_threshold)}" + ) + filter_command = " ".join( + [ + "uv run -m preprocessing.filter_embeddings", + f"--level {args.level}", + f"--tiling-uri {shlex.quote(args.tiling_uri)}", + f"--embeddings-uri {shlex.quote(embeddings_uri)}", + f"--embeddings-path {shlex.quote(embeddings_path)}", + f"--epithelium-threshold {args.epithelium_threshold:g}", + f"--cancer-threshold {args.cancer_threshold:g}", + ] + ) + + if args.dry_run: + print(f"Job: {job_name}") + print(f"Command: {filter_command}") + return + + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=16, + memory="64Gi", + shm="16Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + "export MLFLOW_TRACKING_URI=http://mlflow-s3.rationai-mlflow", + "uv sync --frozen", + filter_command, + ], + storage=[storage.secure.PROJECTS], + ) + + +if __name__ == "__main__": + main() From 758c88f4eb7d14a2fa242d1c85ead1cf2f8dcb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 8 Sep 2026 15:34:04 +0200 Subject: [PATCH 73/79] feat: epithel masks fix: download with retries Add level 2 filtered embedding datasets --- configs/data/embedded/epithel/l2.yaml | 13 +++++ configs/data/embedded/epithel/l3.yaml | 13 +++++ configs/data/embedded/epithel/l4.yaml | 13 +++++ configs/data/embedded/epithel/l5.yaml | 13 +++++ .../embedded/epithel_and_cancer_5/l2.yaml | 13 +++++ .../embedded/epithel_and_cancer_5/l3.yaml | 13 +++++ .../embedded/epithel_and_cancer_5/l4.yaml | 13 +++++ .../embedded/epithel_and_cancer_5/l5.yaml | 13 +++++ .../data/embedded/epithel_or_cancer_5/l2.yaml | 13 +++++ .../data/embedded/epithel_or_cancer_5/l3.yaml | 13 +++++ .../data/embedded/epithel_or_cancer_5/l4.yaml | 13 +++++ .../data/embedded/epithel_or_cancer_5/l5.yaml | 13 +++++ ml/data/datasets/_sources.py | 49 +++++++++++++++++-- ml/data/datasets/raw_tiles.py | 10 ++-- 14 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 configs/data/embedded/epithel/l2.yaml create mode 100644 configs/data/embedded/epithel/l3.yaml create mode 100644 configs/data/embedded/epithel/l4.yaml create mode 100644 configs/data/embedded/epithel/l5.yaml create mode 100644 configs/data/embedded/epithel_and_cancer_5/l2.yaml create mode 100644 configs/data/embedded/epithel_and_cancer_5/l3.yaml create mode 100644 configs/data/embedded/epithel_and_cancer_5/l4.yaml create mode 100644 configs/data/embedded/epithel_and_cancer_5/l5.yaml create mode 100644 configs/data/embedded/epithel_or_cancer_5/l2.yaml create mode 100644 configs/data/embedded/epithel_or_cancer_5/l3.yaml create mode 100644 configs/data/embedded/epithel_or_cancer_5/l4.yaml create mode 100644 configs/data/embedded/epithel_or_cancer_5/l5.yaml diff --git a/configs/data/embedded/epithel/l2.yaml b/configs/data/embedded/epithel/l2.yaml new file mode 100644 index 00000000..12a8cf3f --- /dev/null +++ b/configs/data/embedded/epithel/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings" + train: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/f8d20bbc5a5f47e0ad0c20e87375b051/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l3.yaml b/configs/data/embedded/epithel/l3.yaml new file mode 100644 index 00000000..e7a9553d --- /dev/null +++ b/configs/data/embedded/epithel/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings" + train: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/7e30eeb156de459b8d666f0468e23ccc/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l4.yaml b/configs/data/embedded/epithel/l4.yaml new file mode 100644 index 00000000..c9eec19e --- /dev/null +++ b/configs/data/embedded/epithel/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings" + train: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/57b73f4841da4af5bc9f5507febbfffa/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel/l5.yaml b/configs/data/embedded/epithel/l5.yaml new file mode 100644 index 00000000..96c79682 --- /dev/null +++ b/configs/data/embedded/epithel/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25. +# The whole dataset and all three predefined splits are stored in one MLflow run. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_025_embed +uris: + all: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings" + train: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/4bec58458eef40a8b7b369d50f3d4f07/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l2.yaml b/configs/data/embedded/epithel_and_cancer_5/l2.yaml new file mode 100644 index 00000000..3ffacf21 --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings" + train: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c7532f294516428cb6a572bb5d583bf4/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l3.yaml b/configs/data/embedded/epithel_and_cancer_5/l3.yaml new file mode 100644 index 00000000..0f97a28f --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings" + train: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e91fc1099b114d389e63d381fde4d365/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l4.yaml b/configs/data/embedded/epithel_and_cancer_5/l4.yaml new file mode 100644 index 00000000..5855b9bd --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings" + train: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/eeeca3a1bc7b4725ab06ff27f2dc9101/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_and_cancer_5/l5.yaml b/configs/data/embedded/epithel_and_cancer_5/l5.yaml new file mode 100644 index 00000000..e2ab0691 --- /dev/null +++ b/configs/data/embedded/epithel_and_cancer_5/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# AND cancer probability is greater than 0.5. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_and_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings" + train: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/330c4f5f1a334def95884a9c5cd3aabc/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l2.yaml b/configs/data/embedded/epithel_or_cancer_5/l2.yaml new file mode 100644 index 00000000..76081123 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l2.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.2 +# Level-2 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l2@_here_ + - _self_ + +name: l2_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings" + train: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/85f1195a0b754aff844cecc896af1598/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l3.yaml b/configs/data/embedded/epithel_or_cancer_5/l3.yaml new file mode 100644 index 00000000..9dc1dcc4 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l3.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.3 +# Level-3 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l3@_here_ + - _self_ + +name: l3_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings" + train: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/482b36130caf4415859155be11857ecb/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l4.yaml b/configs/data/embedded/epithel_or_cancer_5/l4.yaml new file mode 100644 index 00000000..f381cab4 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l4.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.4 +# Level-4 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l4@_here_ + - _self_ + +name: l4_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings" + train: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/c65a1976537946168ed537b97f472a48/artifacts/embeddings_test" diff --git a/configs/data/embedded/epithel_or_cancer_5/l5.yaml b/configs/data/embedded/epithel_or_cancer_5/l5.yaml new file mode 100644 index 00000000..302b96e3 --- /dev/null +++ b/configs/data/embedded/epithel_or_cancer_5/l5.yaml @@ -0,0 +1,13 @@ +# @package dataset.levels.5 +# Level-5 Virchow2 embeddings retained when epithelium overlap is greater than 0.25 +# OR cancer probability is greater than 0.5. +defaults: + - /data/levels/l5@_here_ + - _self_ + +name: l5_epithel_or_cancer_025_05_embed +uris: + all: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings" + train: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_train" + val: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_val" + test: "mlflow-artifacts:/3/e4b3945541e24c7ba6b38b24e8faf600/artifacts/embeddings_test" diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py index 80272c0d..fa2fbdce 100644 --- a/ml/data/datasets/_sources.py +++ b/ml/data/datasets/_sources.py @@ -12,17 +12,60 @@ """ import logging +import time from collections.abc import Mapping from pathlib import Path import pandas as pd -from mlflow.artifacts import download_artifacts +from mlflow.artifacts import download_artifacts as _download_artifacts +from mlflow.exceptions import MlflowException from ml.data.datasets.labels import LabelMode, get_target_columns, process_slides logger = logging.getLogger(__name__) +MLFLOW_DOWNLOAD_MAX_RETRIES = 3 +MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS = 1.0 + + +def download_artifacts_with_retries( + artifact_uri: str, + *, + max_retries: int = MLFLOW_DOWNLOAD_MAX_RETRIES, + initial_backoff_seconds: float = MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS, +) -> Path: + """Download an MLflow artifact, retrying transient download failures. + + MLflow raises one aggregate :class:`MlflowException` when any file in a + directory fails to download. Retrying the directory lets a short-lived + storage or network failure recover instead of aborting dataset setup. + """ + if max_retries < 0: + raise ValueError("max_retries must be non-negative.") + if initial_backoff_seconds < 0: + raise ValueError("initial_backoff_seconds must be non-negative.") + + for retry in range(max_retries + 1): + try: + return Path(_download_artifacts(artifact_uri=artifact_uri)) + except MlflowException: + if retry == max_retries: + raise + + delay = initial_backoff_seconds * 2**retry + logger.warning( + "MLflow artifact download failed for %s; retrying in %.1f seconds " + "(%d/%d).", + artifact_uri, + delay, + retry + 1, + max_retries, + ) + time.sleep(delay) + + raise AssertionError("unreachable") + def load_labeled_slides( data_mapping: str | Path, @@ -56,8 +99,7 @@ def load_labeled_slides( def download_level_sources(sources: Mapping[int, str]) -> dict[int, Path]: """Download each pyramid level's artifact and return ``level -> local dir``.""" return { - level: Path(download_artifacts(artifact_uri=uri)) - for level, uri in sources.items() + level: download_artifacts_with_retries(uri) for level, uri in sources.items() } @@ -85,6 +127,7 @@ def split_uri(card: Mapping[str, object], key: str, split: str, level: int) -> s __all__ = [ "available_slides", + "download_artifacts_with_retries", "download_level_sources", "load_labeled_slides", "split_uri", diff --git a/ml/data/datasets/raw_tiles.py b/ml/data/datasets/raw_tiles.py index 8994532c..de12ea57 100644 --- a/ml/data/datasets/raw_tiles.py +++ b/ml/data/datasets/raw_tiles.py @@ -21,7 +21,11 @@ import torch from torch.utils.data import Dataset -from ml.data.datasets._sources import load_labeled_slides, split_uri +from ml.data.datasets._sources import ( + download_artifacts_with_retries, + load_labeled_slides, + split_uri, +) from ml.data.datasets.labels import LabelMode, get_label from ml.typing import LevelSpec, MILSample, SlideMetadata @@ -66,8 +70,6 @@ def __init__( "RawTileSlideDataset is single-scale: pass exactly one level." ) - from mlflow.artifacts import download_artifacts - from ml.data.datasets import SlideDataset # lazy: pulls OpenSlide self.label_mode = LabelMode(label_mode) @@ -76,7 +78,7 @@ def __init__( (card,) = levels.values() uri = split_uri(dict(card), "raw_uris", split, 0) - local_dir = Path(download_artifacts(artifact_uri=uri)) + local_dir = download_artifacts_with_retries(uri) transform = _normalize_transform() slide_dataset = SlideDataset(paths=[local_dir], transforms=transform) From 726b72cc9709f5023e166105a3ce819ff1922018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 8 Sep 2026 17:00:27 +0200 Subject: [PATCH 74/79] Add resumable training sweeps and artifact downloads --- .../d_optimal_36_combinations.csv | 37 ++ .../d_optimal_48_combinations.csv | 49 +++ main.py | 118 +++++++ ml/data/datasets/_sources.py | 86 ++++- scripts/ml/retry_failed_jobs.py | 237 +++++++++++++ scripts/ml/run_train_d_optimal_sweep.py | 324 ++++++++++++++++++ scripts/ml/run_train_sweep.py | 105 ++++-- 7 files changed, 913 insertions(+), 43 deletions(-) create mode 100644 configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv create mode 100644 configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv create mode 100644 main.py create mode 100644 scripts/ml/retry_failed_jobs.py create mode 100644 scripts/ml/run_train_d_optimal_sweep.py diff --git a/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv b/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv new file mode 100644 index 00000000..05fae806 --- /dev/null +++ b/configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv @@ -0,0 +1,37 @@ +level,head,aggregator,loss,mask_type +2,mlp,mean,bce,epithel +5,linear,attention,bce,epithel_and_cancer_5 +2,linear,max,regression,tissue_only +3,mlp,mean,regression,cancer_mask_9 +5,mlp,max,bce,cancer_mask_5 +2,linear,mean,joint,epithel_or_cancer_5 +3,linear,attention,bce,cancer_mask_9 +3,mlp,max,regression,epithel +4,mlp,attention,regression,epithel +3,linear,transformer,joint,epithel +3,linear,attention,joint,tissue_only +5,linear,max,bce,epithel_or_cancer_5 +2,linear,transformer,bce,cancer_mask_9 +3,linear,mean,joint,epithel_and_cancer_5 +2,linear,max,joint,cancer_mask_5 +4,mlp,attention,joint,tissue_only +4,mlp,mean,bce,epithel_or_cancer_5 +5,mlp,attention,joint,epithel_or_cancer_5 +4,mlp,max,joint,epithel_and_cancer_5 +2,linear,attention,bce,cancer_mask_5 +4,linear,max,joint,cancer_mask_9 +2,mlp,transformer,regression,epithel_or_cancer_5 +5,mlp,transformer,bce,tissue_only +4,linear,transformer,regression,epithel_and_cancer_5 +3,linear,max,regression,epithel_or_cancer_5 +4,linear,mean,regression,cancer_mask_5 +2,mlp,attention,regression,epithel_and_cancer_5 +5,linear,mean,regression,epithel +5,linear,mean,regression,tissue_only +5,mlp,max,regression,cancer_mask_9 +2,linear,max,bce,epithel +3,mlp,max,bce,tissue_only +3,mlp,transformer,regression,cancer_mask_5 +5,mlp,transformer,joint,epithel +4,linear,transformer,bce,epithel_or_cancer_5 +2,mlp,transformer,joint,cancer_mask_9 diff --git a/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv b/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv new file mode 100644 index 00000000..ac67b6bb --- /dev/null +++ b/configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv @@ -0,0 +1,49 @@ +level,head,aggregator,loss,mask_type +5,mlp,mean,regression,epithel +3,linear,attention,bce,epithel_and_cancer_5 +3,linear,attention,joint,cancer_mask_9 +4,mlp,mean,joint,cancer_mask_5 +2,mlp,transformer,bce,cancer_mask_9 +2,mlp,attention,joint,epithel_and_cancer_5 +3,mlp,max,joint,cancer_mask_9 +5,linear,mean,joint,epithel_and_cancer_5 +4,linear,attention,bce,epithel_or_cancer_5 +3,mlp,mean,regression,epithel_or_cancer_5 +5,linear,mean,bce,tissue_only +2,mlp,transformer,joint,cancer_mask_5 +5,linear,max,bce,cancer_mask_5 +2,linear,mean,regression,tissue_only +4,mlp,mean,joint,epithel +2,linear,attention,joint,epithel +2,mlp,attention,bce,epithel_or_cancer_5 +4,mlp,max,joint,epithel_and_cancer_5 +2,linear,max,regression,cancer_mask_5 +4,mlp,attention,regression,tissue_only +5,mlp,attention,regression,tissue_only +5,linear,transformer,bce,epithel +5,linear,attention,regression,cancer_mask_9 +3,mlp,mean,bce,tissue_only +3,linear,transformer,regression,epithel_and_cancer_5 +2,linear,transformer,bce,tissue_only +2,linear,max,regression,epithel_or_cancer_5 +2,mlp,mean,bce,epithel_and_cancer_5 +3,mlp,max,joint,tissue_only +2,mlp,max,regression,epithel +4,mlp,transformer,regression,epithel +2,linear,mean,joint,cancer_mask_5 +2,mlp,transformer,joint,cancer_mask_9 +5,linear,max,regression,cancer_mask_9 +4,mlp,transformer,regression,epithel_and_cancer_5 +4,linear,max,joint,tissue_only +4,mlp,attention,bce,cancer_mask_5 +4,linear,transformer,joint,epithel_or_cancer_5 +5,mlp,max,joint,epithel_or_cancer_5 +5,mlp,max,bce,epithel_and_cancer_5 +3,mlp,max,bce,epithel +3,linear,attention,joint,epithel +5,linear,transformer,joint,tissue_only +4,linear,mean,bce,cancer_mask_9 +4,linear,mean,regression,cancer_mask_9 +3,linear,transformer,regression,cancer_mask_5 +5,mlp,transformer,joint,epithel_or_cancer_5 +5,mlp,attention,joint,cancer_mask_5 diff --git a/main.py b/main.py new file mode 100644 index 00000000..1de32428 --- /dev/null +++ b/main.py @@ -0,0 +1,118 @@ +import numpy as np +import pandas as pd +from sklearn.preprocessing import OneHotEncoder + +# --------------------------------------------------------- +# 1. DEFINICE VAŠICH FAKTORŮ A JEJICH HLADIN +# --------------------------------------------------------- +factors = { + 'level': [2, 3, 4, 5], # 4 + 'head': ['linear', 'mlp'], # 2 + 'aggregator': ['mean', 'max', 'attention', 'transformer'], # 4 + 'loss': ['bce', 'regression', 'joint'], # 3 + 'mask_type': ['tissue_only', 'cancer_mask_5', 'cancer_mask_9', 'epithel', 'epithel_and_cancer_5', 'epithel_or_cancer_5'] # 5 +} + +# Vytvoření plné mřížky (Full Grid Search = 4 * 2 * 4 * 3 * 3 = 288 kombinací) +grid_tuples = pd.MultiIndex.from_product(factors.values(), names=factors.keys()) +full_grid = grid_tuples.to_frame().reset_index(drop=True) + +print(f"Celkový počet kombinací v plné mřížce (Full Grid): {len(full_grid)}") + +# --------------------------------------------------------- +# 2. PŘEVOD NA KATEGORICKOU MATICI (Dummy / One-Hot Encoding) +# --------------------------------------------------------- +# Jelikož jsou parametry kategorické, převedeme je na 0/1 matici pro výpočet determinantu +encoder = OneHotEncoder(drop='first', sparse_output=False) +X_full = encoder.fit_transform(full_grid) + +# Přidání sloupce jedniček pro absolutní člen (intercept b0) +X_full_intercept = np.hstack([np.ones((X_full.shape[0], 1)), X_full]) +p = X_full_intercept.shape[1] # Počet odhadovaných parametrů v modelu + +print(f"Počet volných parametrů k odhadnutí v modelu (p): {p}") + +# --------------------------------------------------------- +# 3. ALGORITMUS PRO D-OPTIMAL SELECTION (Fedorov's Exchange Idea) +# --------------------------------------------------------- +def compute_d_efficiency(X_sub, X_full_matrix): + """Spočítá D-efektivitu vybraného vzorku vůči celkovému vzorku.""" + N_sub = X_sub.shape[0] + N_full = X_full_matrix.shape[0] + + # Matice informace M = X^T * X + M_sub = np.dot(X_sub.T, X_sub) + M_full = np.dot(X_full_matrix.T, X_full_matrix) + + sign_sub, logdet_sub = np.linalg.slogdet(M_sub) + sign_full, logdet_full = np.linalg.slogdet(M_full) + + if sign_sub <= 0: + return 0.0 # Singulární matice (nedostatek dat pro odhad) + + # Relativní D-efficiency vzorec v logaritmickém tvaru pro numerickou stabilitu + det_ratio_per_param = np.exp((logdet_sub - logdet_full) / p) + d_eff = (det_ratio_per_param) * (N_full / N_sub) + return d_eff * 100 + +def generate_d_optimal_design(X_matrix, full_df, n_samples, n_iterations=100): + """ + Vybere n_samples řádků tak, aby maximizoval determinant X^T * X. + """ + best_det = -np.inf + best_indices = None + + np.random.seed(42) # Pro reprodukovatelnost + + for _ in range(n_iterations): + # Náhodný počáteční výběr + current_indices = np.random.choice(X_matrix.shape[0], size=n_samples, replace=False) + + # Lokální vylepšování determinantu (Exchange algorithm) + for i in range(n_samples): + current_X = X_matrix[current_indices] + sign, current_det = np.linalg.slogdet(np.dot(current_X.T, current_X)) + + if sign <= 0: + current_det = -np.inf + + # Zkusíme zaměnit i-tý prvek za jiný kandidát + candidates = np.setdiff1d(np.arange(X_matrix.shape[0]), current_indices) + for cand in np.random.choice(candidates, size=min(30, len(candidates)), replace=False): + temp_indices = current_indices.copy() + temp_indices[i] = cand + + temp_X = X_matrix[temp_indices] + temp_sign, temp_det = np.linalg.slogdet(np.dot(temp_X.T, temp_X)) + + if temp_sign > 0 and temp_det > current_det: + current_det = temp_det + current_indices = temp_indices.copy() + + if current_det > best_det: + best_det = current_det + best_indices = current_indices + + selected_df = full_df.iloc[best_indices].copy().reset_index(drop=True) + d_eff = compute_d_efficiency(X_matrix[best_indices], X_matrix) + + return selected_df, d_eff + +# --------------------------------------------------------- +# 4. GENEROVÁNÍ DESIGNU PRO 36 A 48 KOMBINACÍ +# --------------------------------------------------------- +print("\nGeneruji D-Optimal Design pro 36 kombinací...") +design_36, d_eff_36 = generate_d_optimal_design(X_full_intercept, full_grid, n_samples=36) +print(f"-> D-Efektivita pro 36 kombinací: {d_eff_36:.2f} %") + +print("\nGeneruji D-Optimal Design pro 48 kombinací...") +design_48, d_eff_48 = generate_d_optimal_design(X_full_intercept, full_grid, n_samples=48) +print(f"-> D-Efektivita pro 48 kombinací: {d_eff_48:.2f} %") + +# --------------------------------------------------------- +# 5. ULOŽENÍ VÝSLEDKŮ DO CSV +# --------------------------------------------------------- +design_36.to_csv("configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv", index=False) +design_48.to_csv("configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv", index=False) + +print("\nSoubory 'configs/sweeps/d_optimal_mask_comparison/d_optimal_36_combinations.csv' a 'configs/sweeps/d_optimal_mask_comparison/d_optimal_48_combinations.csv' byly úspěšně vytvořeny.") diff --git a/ml/data/datasets/_sources.py b/ml/data/datasets/_sources.py index fa2fbdce..1690849c 100644 --- a/ml/data/datasets/_sources.py +++ b/ml/data/datasets/_sources.py @@ -12,13 +12,22 @@ """ import logging +import tempfile import time from collections.abc import Mapping from pathlib import Path import pandas as pd -from mlflow.artifacts import download_artifacts as _download_artifacts +from mlflow.artifacts import ( + download_artifacts as _download_artifacts, +) +from mlflow.artifacts import ( + list_artifacts as _list_artifacts, +) +from mlflow.entities import FileInfo from mlflow.exceptions import MlflowException +from mlflow.tracking.artifact_utils import _get_root_uri_and_artifact_path +from mlflow.utils.uri import append_to_uri_path from ml.data.datasets.labels import LabelMode, get_target_columns, process_slides @@ -29,26 +38,93 @@ MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS = 1.0 +def _artifact_files(root_uri: str, artifact_path: str) -> list[FileInfo]: + """Return every file below an artifact directory using metadata only.""" + files: list[FileInfo] = [] + pending = [artifact_path] + while pending: + current_path = pending.pop() + current_uri = append_to_uri_path(root_uri, current_path) + for item in _list_artifacts(artifact_uri=current_uri): + if item.is_dir: + pending.append(item.path) + else: + files.append(item) + return files + + +def _is_complete_download(file_info: FileInfo, destination: Path) -> bool: + local_path = destination / file_info.path + if not local_path.is_file(): + return False + return ( + file_info.file_size is None or local_path.stat().st_size == file_info.file_size + ) + + +def _download_incomplete_artifacts( + root_uri: str, + artifact_path: str, + destination: Path, +) -> Path: + remote_files = _artifact_files(root_uri, artifact_path) + if not remote_files: + # The URI may point to a single file rather than a directory. + uri = append_to_uri_path(root_uri, artifact_path) + return Path(_download_artifacts(artifact_uri=uri, dst_path=str(destination))) + + incomplete = [ + file_info + for file_info in remote_files + if not _is_complete_download(file_info, destination) + ] + logger.info( + "Retrying %d incomplete MLflow artifact(s); keeping %d already downloaded.", + len(incomplete), + len(remote_files) - len(incomplete), + ) + for file_info in incomplete: + file_uri = append_to_uri_path(root_uri, file_info.path) + _download_artifacts(artifact_uri=file_uri, dst_path=str(destination)) + + return destination / artifact_path + + def download_artifacts_with_retries( artifact_uri: str, *, max_retries: int = MLFLOW_DOWNLOAD_MAX_RETRIES, initial_backoff_seconds: float = MLFLOW_DOWNLOAD_INITIAL_BACKOFF_SECONDS, ) -> Path: - """Download an MLflow artifact, retrying transient download failures. + """Download an MLflow artifact, resuming transient download failures. MLflow raises one aggregate :class:`MlflowException` when any file in a - directory fails to download. Retrying the directory lets a short-lived - storage or network failure recover instead of aborting dataset setup. + directory fails. The initial parallel download uses a stable temporary + destination; retries preserve complete files and fetch only missing or + size-mismatched files. """ if max_retries < 0: raise ValueError("max_retries must be non-negative.") if initial_backoff_seconds < 0: raise ValueError("initial_backoff_seconds must be non-negative.") + root_uri, artifact_path = _get_root_uri_and_artifact_path(artifact_uri) + destination = Path(tempfile.mkdtemp(prefix="mammaprint-mlflow-artifacts-")) + for retry in range(max_retries + 1): try: - return Path(_download_artifacts(artifact_uri=artifact_uri)) + if retry == 0: + return Path( + _download_artifacts( + artifact_uri=artifact_uri, + dst_path=str(destination), + ) + ) + return _download_incomplete_artifacts( + root_uri, + artifact_path, + destination, + ) except MlflowException: if retry == max_retries: raise diff --git a/scripts/ml/retry_failed_jobs.py b/scripts/ml/retry_failed_jobs.py new file mode 100644 index 00000000..92982ee7 --- /dev/null +++ b/scripts/ml/retry_failed_jobs.py @@ -0,0 +1,237 @@ +"""Log this user's training jobs, retry the most-recently-failed one, wait, repeat. + +The jobs submitted by ``run_train_sweep*.py`` set ``backoff_limit=0`` — one pod +failure marks the whole Job ``Failed`` with no automatic retry. This helper: + + 1. lists your jobs (label ``created_by=``) with their status, + 2. finds the most recent FAILED job, + 3. re-creates it under a bumped name (its own manifest, so image / resources / + command are preserved exactly — works for any job, transformer or spatial), + 4. sleeps ``WAIT_MINUTES``, then loops. + +A failed job whose name ends in ``-`` is retried as ``-``; otherwise ``-2`` +is appended. The old failed Job is deleted first (its name is a unique key, so a +re-create with the same name would 409). Kubernetes' own TTL may already have +removed it — that's fine. + +Usage: + uv run python scripts/ml/retry_failed_jobs.py # loop forever, 30 min + uv run python scripts/ml/retry_failed_jobs.py --once # one pass, no wait + uv run python scripts/ml/retry_failed_jobs.py --wait 30 # 30-minute cadence + uv run python scripts/ml/retry_failed_jobs.py --dry-run # log + show plan, no changes + +Reads/writes only jobs labelled with your username. Requires a working kubectl +context (the same one kube_jobs submits to). +""" + +import argparse +import copy +import re +import time + +from kube_jobs.constants import CLUSTER, JOB_NAMESPACE +from kubernetes import client +from kubernetes.config import load_kube_config + + +USER = "kissmi" # created_by label — the jobs this script is allowed to touch +WAIT_MINUTES = 30 +# Only consider these when picking "the last failed job", so we never resurrect an +# unrelated failure (e.g. a preprocessing job). Narrow this if you want. +NAME_PREFIXES = ( + "mammaprint-train-", + "mammaprint-sweep-", + "mp-doe36-", + "mp-doe48-", +) + + +def _batch_api() -> client.BatchV1Api: + load_kube_config(context=CLUSTER) + return client.BatchV1Api() + + +def _is_failed(job: client.V1Job) -> bool: + return bool(job.status and job.status.failed) + + +def _is_active(job: client.V1Job) -> bool: + return bool(job.status and job.status.active) + + +def _is_succeeded(job: client.V1Job) -> bool: + return bool(job.status and job.status.succeeded) + + +def _status_str(job: client.V1Job) -> str: + if _is_active(job): + return "ACTIVE" + if _is_succeeded(job): + return "SUCCEEDED" + if _is_failed(job): + return "FAILED" + return "UNKNOWN" + + +def list_jobs(batch: client.BatchV1Api) -> list[client.V1Job]: + """This user's jobs, newest first (by start time).""" + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, label_selector=f"created_by={USER}" + ).items + + def start_key(j: client.V1Job) -> str: + st = j.status.start_time if j.status else None + return st.isoformat() if st else "" + + return sorted(jobs, key=start_key, reverse=True) + + +def log_jobs(jobs: list[client.V1Job]) -> None: + print(f"\n{'STATUS':<10} {'START':<22} NAME") + print("-" * 90) + for j in jobs: + st = j.status.start_time if j.status else None + start = st.strftime("%Y-%m-%d %H:%M:%S") if st else "-" + print(f"{_status_str(j):<10} {start:<22} {j.metadata.name}") + + +def _bump_name(name: str) -> str: + """`...-bce-1` -> `...-bce-2`; a name without a trailing `-` gets `-2`.""" + m = re.match(r"^(.*)-(\d+)$", name) + if m: + return f"{m.group(1)}-{int(m.group(2)) + 1}" + return f"{name}-2" + + +def _retry_manifest(job: client.V1Job, new_name: str) -> client.V1Job: + """Copy a Job's spec into a fresh Job named `new_name`, stripping runtime fields. + + Kubernetes injects immutable fields into a running Job's pod template (the + `controller-uid`/`job-name` selector labels and `batch.kubernetes.io/*`). Those + must be removed or the create is rejected — we rebuild a clean metadata/spec and + carry over only what we set at submit time. + """ + src = job.spec + tmpl = copy.deepcopy(src.template) + + # Drop the auto-injected selector labels; keep our own app/created_by labels. + labels = dict(tmpl.metadata.labels or {}) if tmpl.metadata else {} + for k in list(labels): + if k.startswith("batch.kubernetes.io/") or k in {"controller-uid", "job-name"}: + del labels[k] + labels["app"] = new_name + tmpl.metadata = client.V1ObjectMeta(labels=labels) + + # Rename the container so its name matches the new job (cosmetic, matches submit). + for c in tmpl.spec.containers: + c.name = new_name + + new_spec = client.V1JobSpec( + backoff_limit=src.backoff_limit, + ttl_seconds_after_finished=src.ttl_seconds_after_finished, + template=tmpl, + ) + return client.V1Job( + api_version="batch/v1", + kind="Job", + metadata=client.V1ObjectMeta(name=new_name, labels={"created_by": USER}), + spec=new_spec, + ) + + +def last_failed(jobs: list[client.V1Job]) -> client.V1Job | None: + for j in jobs: + if j.metadata.name.startswith(NAME_PREFIXES) and _is_failed(j): + return j + return None + + +def retry_last_failed( + batch: client.BatchV1Api, jobs: list[client.V1Job], dry_run: bool +) -> str | None: + """Delete the newest failed job and re-create it under a bumped name. + + Returns the new job name, or None if there was nothing to retry. + """ + target = last_failed(jobs) + if target is None: + print("\nNo failed job to retry. ✅") + return None + + old = target.metadata.name + new = _bump_name(old) + print(f"\nLast failed job: {old} -> retry as: {new}") + + if dry_run: + print( + "[dry-run] would delete the old job and create the retry; no changes made." + ) + return new + + manifest = _retry_manifest(target, new) + + # Delete the old failed Job so its name is free (may already be TTL-collected). + try: + batch.delete_namespaced_job( + name=old, namespace=JOB_NAMESPACE, propagation_policy="Background" + ) + print(f"Deleted old failed job {old}.") + except client.ApiException as e: + if e.status == 404: + print(f"Old job {old} already gone (TTL); nothing to delete.") + else: + raise + + try: + resp = batch.create_namespaced_job(body=manifest, namespace=JOB_NAMESPACE) + print(f"Retry submitted: {resp.metadata.name} 🚀") + return resp.metadata.name + except client.ApiException as e: + print(f"Retry creation failed: {e.reason} ({e.status})") + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--wait", + type=float, + default=WAIT_MINUTES, + help=f"Minutes to wait between passes (default {WAIT_MINUTES}).", + ) + parser.add_argument( + "--once", + action="store_true", + help="Run a single pass (log + retry) and exit, no waiting.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Log jobs and show the retry plan without changing anything.", + ) + args = parser.parse_args() + + batch = _batch_api() + pass_num = 0 + while True: + pass_num += 1 + print(f"\n{'=' * 90}\nPASS {pass_num}\n{'=' * 90}") + jobs = list_jobs(batch) + log_jobs(jobs) + retry_last_failed(batch, jobs, dry_run=args.dry_run) + + if args.once: + break + + wait_s = args.wait * 60 + print(f"\nSleeping {args.wait:g} min before the next pass (Ctrl-C to stop)...") + try: + time.sleep(wait_s) + except KeyboardInterrupt: + print("\nStopped.") + break + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ml/run_train_d_optimal_sweep.py b/scripts/ml/run_train_d_optimal_sweep.py new file mode 100644 index 00000000..c8939f6a --- /dev/null +++ b/scripts/ml/run_train_d_optimal_sweep.py @@ -0,0 +1,324 @@ +"""Submit the configured D-optimal mask-comparison training sweep. + +Edit the settings block below, then run this file without command-line flags. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shlex +import time +from dataclasses import dataclass +from pathlib import Path + +from kube_jobs import storage, submit_job + + +EXPERIMENTS = { + "bce": "ml/train_mil_embeddings", + "regression": "ml/train_mil_embeddings_regression", + "joint": "ml/train_mil_embeddings_joint", +} +MASK_CONFIGS = { + "tissue_only": ("tissue_only", "tis"), + "cancer_mask_5": ("cancer_mask_5", "c05"), + "cancer_mask_9": ("cancer_mask_9", "c09"), + "epithel_and_cancer_5": ("epithel_and_cancer_5", "eac05"), + "epithel_or_cancer_5": ("epithel_or_cancer_5", "eoc05"), +} +DESIGN_DIR = Path("configs/sweeps/d_optimal_mask_comparison") +GIT_REF = "feat/tiling-values" +USERNAME = "kissmi" +IMAGE = "cerit.io/rationai/base:2.0.6" +POD_MLFLOW_TRACKING_URI = os.getenv( + "POD_MLFLOW_TRACKING_URI", "http://mlflow-s3.rationai-mlflow" +) +MLFLOW_LOOKUP_URI = os.getenv( + "MLFLOW_LOOKUP_URI", + "https://mlflow-s3.rationai.cloud.trusted.e-infra.cz", +) +MLFLOW_EXPERIMENT = "MammaPrint" + +# Sweep settings. Edit these values before running this file. +DESIGN_SIZE = 48 +DATA_MAPPING: str | None = None # e.g. "/mnt/projects/mammaprint/my_mapping.csv" +REPEATS = 20 +SEED_START = 0 +GPUS = ("H100", "A40") +SUBMISSION_INTERVAL_MINUTES = 30.0 +MAX_JOBS: int | None = None +DRY_RUN = False + + +@dataclass(frozen=True) +class SweepJob: + design_row: int + level: int + head: str + aggregator: str + loss: str + mask_type: str + data_config: str + job_token: str + + +@dataclass(frozen=True) +class WorkItem: + job: SweepJob + repeat: int + seed: int + + +def _validate_settings() -> str: + if DESIGN_SIZE not in (36, 48): + raise ValueError(f"DESIGN_SIZE must be 36 or 48, got {DESIGN_SIZE}.") + if DATA_MAPPING is None: + raise ValueError( + "Set DATA_MAPPING in the settings block to a path visible inside the pod." + ) + if not GPUS: + raise ValueError("GPUS must contain at least one GPU type.") + unexpected_gpus = sorted(set(GPUS) - {"H100", "A40"}) + if unexpected_gpus: + raise ValueError(f"Unsupported GPUs: {unexpected_gpus}") + if SUBMISSION_INTERVAL_MINUTES < 0: + raise ValueError("SUBMISSION_INTERVAL_MINUTES cannot be negative.") + if MAX_JOBS is not None and MAX_JOBS <= 0: + raise ValueError("MAX_JOBS must be positive or None.") + return DATA_MAPPING + + +def _load_jobs(design_size: int) -> list[SweepJob]: + path = DESIGN_DIR / f"d_optimal_{design_size}_combinations.csv" + if not path.is_file(): + raise FileNotFoundError( + f"Missing design table {path}; run `uv run python main.py`." + ) + + with path.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + if len(rows) != design_size: + raise RuntimeError(f"Expected {design_size} rows in {path}, found {len(rows)}.") + + jobs = [] + for design_row, row in enumerate(rows, start=1): + loss = row["loss"] + if loss not in EXPERIMENTS: + raise ValueError(f"Unknown loss {loss!r} in {path}.") + mask_type = row["mask_type"] + try: + data_config, job_token = MASK_CONFIGS[mask_type] + except KeyError as error: + raise ValueError(f"Unknown mask type {mask_type!r} in {path}.") from error + jobs.append( + SweepJob( + design_row=design_row, + level=int(row["level"]), + head=row["head"], + aggregator=row["aggregator"], + loss=loss, + mask_type=mask_type, + data_config=data_config, + job_token=job_token, + ) + ) + return jobs + + +def _mapping_id(data_mapping: str) -> str: + return hashlib.sha256(data_mapping.encode()).hexdigest()[:6] + + +def _work_items(jobs: list[SweepJob], repeats: int, seed_start: int) -> list[WorkItem]: + if repeats <= 0: + raise ValueError(f"REPEATS must be positive, got {repeats}.") + return [ + WorkItem(job=job, repeat=repeat, seed=seed_start + repeat) + for repeat in range(repeats) + for job in jobs + ] + + +def _job_name(item: WorkItem, design_size: int, mapping_id: str) -> str: + job = item.job + name = ( + f"mp-doe{design_size}-{job.job_token}-l{job.level}-" + f"{job.head}-{job.aggregator}-{job.loss}-s{item.seed}-m{mapping_id}" + ) + # Leave room for the retry helper's '-2', '-3', ... suffixes. + if len(name) > 60: + raise ValueError(f"Kubernetes job name leaves no retry suffix room: {name}") + return name + + +def _gpu_for(item: WorkItem, gpus: tuple[str, ...]) -> str: + # Rotate each configuration across devices between repetitions instead of + # permanently coupling a model configuration to one GPU type. + gpu_index = (item.job.design_row - 1 + item.repeat) % len(gpus) + return gpus[gpu_index] + + +def _validate_dataset_cards(jobs: list[SweepJob]) -> None: + repo_root = Path(__file__).resolve().parents[2] + checked = set() + for job in jobs: + data_override = f"{job.data_config}/l{job.level}" + if data_override in checked: + continue + checked.add(data_override) + card = repo_root / "configs" / "data" / "embedded" / f"{data_override}.yaml" + if not card.is_file(): + raise FileNotFoundError(f"Missing dataset card: {card}") + text = card.read_text() + missing = [ + split + for split in ("all", "train", "val", "test") + if f" {split}:" not in text + ] + if missing or "" in text: + problem = ", ".join(missing) if missing else "placeholder URI" + raise RuntimeError(f"Dataset card {card} is incomplete: {problem}.") + + +def _existing_mlflow_run_names(expected_names: set[str], prefix: str) -> set[str]: + from mlflow import MlflowClient + + client = MlflowClient(tracking_uri=MLFLOW_LOOKUP_URI) + experiment = client.get_experiment_by_name(MLFLOW_EXPERIMENT) + if experiment is None: + return set() + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + filter_string=f"attributes.run_name LIKE '{prefix}%'", + max_results=50_000, + ) + return {run.info.run_name for run in runs if run.info.run_name in expected_names} + + +def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: + from kube_jobs.constants import CLUSTER, JOB_NAMESPACE + from kubernetes import client + from kubernetes.config import load_kube_config + + load_kube_config(context=CLUSTER) + batch = client.BatchV1Api() + jobs = batch.list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={USERNAME}", + ).items + existing = set() + for job in jobs: + if job.metadata is None: + continue + name = job.metadata.name + match = re.match(r"^(.*)-(\d+)$", name) + base_name = match.group(1) if match else name + if base_name in expected_names: + existing.add(base_name) + return existing + + +def _submit( + item: WorkItem, + design_size: int, + gpu: str, + mapping_id: str, + data_mapping: str, +) -> None: + job = item.job + name = _job_name(item, design_size, mapping_id) + experiment = EXPERIMENTS[job.loss] + data_override = f"{job.data_config}/l{job.level}" + data_mapping_override = ( + f" {shlex.quote(f'dataset.data_mapping={json.dumps(data_mapping)}')}" + ) + submit_job( + job_name=name, + username=USERNAME, + image=IMAGE, + cpu=16, + memory="48Gi", + gpu=gpu, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_REF}", + f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", + "uv sync --frozen", + f""" + uv run -m ml.train +experiment={experiment} \ + data/embedded={data_override} \ + ml/aggregator={job.aggregator} \ + ml/head={job.head} \ + seed={item.seed}{data_mapping_override} \ + test_after_fit=false \ + metadata.run_name={name} \ + +logger.tags.data_variant={job.mask_type} \ + +logger.tags.d_optimal_design=d_optimal_{design_size} \ + +logger.tags.d_optimal_design_row=row_{job.design_row} \ + +logger.tags.repeat=repeat_{item.repeat + 1} \ + +logger.tags.data_mapping_id=map_{mapping_id} + """, + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + + +def main() -> None: + data_mapping = _validate_settings() + jobs = _load_jobs(DESIGN_SIZE) + _validate_dataset_cards(jobs) + items = _work_items(jobs, REPEATS, SEED_START) + mapping_id = _mapping_id(data_mapping) + expected_names = {_job_name(item, DESIGN_SIZE, mapping_id) for item in items} + prefix = f"mp-doe{DESIGN_SIZE}-" + + if DRY_RUN: + print( + f"D-optimal design: {len(jobs)} configurations x {REPEATS} seeds " + f"= {len(items)} jobs" + ) + print(f"GPUs: {', '.join(GPUS)}") + print(f"Interval: {SUBMISSION_INTERVAL_MINUTES:g} minutes") + print(f"Data mapping: {data_mapping}") + print(f"Data mapping ID: {mapping_id}") + for item in items: + gpu = _gpu_for(item, GPUS) + print(f"{_job_name(item, DESIGN_SIZE, mapping_id)} -> {gpu}") + return + + kubernetes_names = _existing_kubernetes_job_names(expected_names) + mlflow_names = _existing_mlflow_run_names(expected_names, prefix) + existing_names = kubernetes_names | mlflow_names + pending = [ + item + for item in items + if _job_name(item, DESIGN_SIZE, mapping_id) not in existing_names + ] + if MAX_JOBS is not None: + pending = pending[:MAX_JOBS] + + print( + f"Design has {len(items)} jobs; {len(existing_names)} already exist and " + f"{len(pending)} will be submitted now." + ) + interval_seconds = SUBMISSION_INTERVAL_MINUTES * 60 + for pending_index, item in enumerate(pending): + if pending_index > 0: + print( + f"Waiting {SUBMISSION_INTERVAL_MINUTES:g} minutes before the next job." + ) + time.sleep(interval_seconds) + gpu = _gpu_for(item, GPUS) + name = _job_name(item, DESIGN_SIZE, mapping_id) + print(f"Submitting {name} to {gpu} ({pending_index + 1}/{len(pending)}).") + _submit(item, DESIGN_SIZE, gpu, mapping_id, data_mapping) + + +if __name__ == "__main__": + main() diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py index 78fd8fe4..533a81f0 100644 --- a/scripts/ml/run_train_sweep.py +++ b/scripts/ml/run_train_sweep.py @@ -5,50 +5,63 @@ from kube_jobs import storage, submit_job -# Each objective has a self-consistent experiment (head out_dim, label_mode, -# output activation, loss and metrics are set together). The generic mean-pool -# experiments are used as the base because their metadata remains valid when the -# aggregator is overridden; the transformer-specific experiments interpolate -# transformer-only fields and cannot be composed with mean/max/attention. +# Each objective points to the matching training experiment. Edit these values, +# together with the dataset/model lists below, to define the Cartesian sweep. EXPERIMENTS = { "bce": "ml/train_mil_embeddings", "regression": "ml/train_mil_embeddings_regression", "joint": "ml/train_mil_embeddings_joint", } -# The cancer_mask_5/l4 split is currently being materialised. Do not launch the -# sweep until its train/validation/test placeholders have been replaced. DATASETS = { - "cm5-l4": "cancer_mask_5/l4", - "cm5-l5": "cancer_mask_5/l5", - "cm9-l4": "cancer_mask_9/l4", - "cm9-l5": "cancer_mask_9/l5", + "tissue": "tissue_only", + "cm5": "cancer_mask_5", + "cm9": "cancer_mask_9", + "e025": "epithel", + "eor5": "epithel_or_cancer_5", + "ean5": "epithel_and_cancer_5", } +LEVELS = ( + # 2, + 3, 4, 5) AGGREGATORS = ("mean", "max", "attention", "transformer") HEADS = ("mlp", "linear") -GPUS = ["A40"] -CODE_VERSION = 2 -SUBMISSION_INTERVAL_SECONDS = 45 * 60 +SEEDS = tuple(range(20)) +GPUS = [ + "mig-2g.20gb", + # "H100", + # "A40", +] +GIT_REF = "feat/tiling-values" +SUBMISSION_INTERVAL_SECONDS = 10 USERNAME = "kissmi" +IMAGE = "cerit.io/rationai/base:2.0.6" POD_MLFLOW_TRACKING_URI = os.getenv( "POD_MLFLOW_TRACKING_URI", "http://mlflow-s3.rationai-mlflow" ) -MLFLOW_LOOKUP_URI = os.getenv("MLFLOW_LOOKUP_URI") +MLFLOW_LOOKUP_URI = os.getenv( + "MLFLOW_LOOKUP_URI", + "https://mlflow-s3.rationai.cloud.trusted.e-infra.cz", +) MLFLOW_EXPERIMENT = "MammaPrint" -JOB_NAME_PREFIX = "mammaprint-train-cmv2-" +JOB_NAME_PREFIX = "mammaprint-sweep-" def _job_name( dataset_key: str, + level: int, head: str, aggregator: str, objective: str, + seed: int, ) -> str: - return f"{JOB_NAME_PREFIX}{dataset_key}-{head}-{aggregator}-{objective}" + return f"{JOB_NAME_PREFIX}{dataset_key}-l{level}-{head}-{aggregator}-{objective}-s{seed}" -def _sweep_jobs() -> list[tuple[int, str, str, str, str, str, str]]: +def _sweep_jobs() -> list[tuple[int, str, str, int, str, str, str, str, int]]: combinations = itertools.product( + SEEDS, DATASETS.items(), + LEVELS, EXPERIMENTS.items(), AGGREGATORS, HEADS, @@ -58,13 +71,17 @@ def _sweep_jobs() -> list[tuple[int, str, str, str, str, str, str]]: job_index, dataset_key, dataset_config, + level, objective, experiment, aggregator, head, + seed, ) for job_index, ( + seed, (dataset_key, dataset_config), + level, (objective, experiment), aggregator, head, @@ -72,15 +89,20 @@ def _sweep_jobs() -> list[tuple[int, str, str, str, str, str, str]]: ] +def _gpu_for_job(job_index: int) -> str: + configurations_per_seed = ( + len(DATASETS) + * len(LEVELS) + * len(EXPERIMENTS) + * len(AGGREGATORS) + * len(HEADS) + ) + seed_index, configuration_index = divmod(job_index, configurations_per_seed) + return GPUS[(configuration_index + seed_index) % len(GPUS)] + + def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: """Fetch sweep runs that already exist, including failed/running runs.""" - if MLFLOW_LOOKUP_URI is None: - print( - "Skipping MLflow lookup: set MLFLOW_LOOKUP_URI to a tracking endpoint " - "reachable from this machine to enable it." - ) - return set() - from mlflow import MlflowClient mlflow_client = MlflowClient(tracking_uri=MLFLOW_LOOKUP_URI) @@ -125,8 +147,8 @@ def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: def main() -> None: jobs = _sweep_jobs() expected_names = { - _job_name(dataset_key, head, aggregator, objective) - for _, dataset_key, _, objective, _, aggregator, head in jobs + _job_name(dataset_key, level, head, aggregator, objective, seed) + for _, dataset_key, _, level, objective, _, aggregator, head, seed in jobs } kubernetes_names = _existing_kubernetes_job_names(expected_names) mlflow_names = _existing_mlflow_run_names(expected_names) @@ -134,7 +156,8 @@ def main() -> None: pending_jobs = [ job for job in jobs - if _job_name(job[1], job[6], job[5], job[3]) not in existing_names + if _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + not in existing_names ] print( @@ -146,41 +169,47 @@ def main() -> None: job_index, dataset_key, dataset_config, + level, objective, experiment, aggregator, head, + seed, ) in enumerate(pending_jobs): if pending_index > 0: - print("Waiting 45 minutes before submitting the next job.") + print( + f"Waiting {SUBMISSION_INTERVAL_SECONDS / 60} minutes before submitting the next job." + ) time.sleep(SUBMISSION_INTERVAL_SECONDS) - name = _job_name(dataset_key, head, aggregator, objective) - gpu = GPUS[job_index % len(GPUS)] + name = _job_name(dataset_key, level, head, aggregator, objective, seed) + gpu = _gpu_for_job(job_index) print(f"Submitting {name} to {gpu} ({pending_index + 1}/{len(pending_jobs)}).") submit_job( job_name=name, username=USERNAME, - image="cerit.io/rationai/base:2.0.6", - cpu=16, - memory="48Gi", + image=IMAGE, + cpu=10, + memory="36Gi", gpu=gpu, public=False, script=[ "git clone https://github.com/rationAI/mammaprint workdir", "cd workdir", - "git checkout feat/tiling-values", + f"git checkout {GIT_REF}", f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", # "export HF_TOKEN=", "uv sync --frozen", f""" uv run -m ml.train +experiment={experiment} \ - data/embedded={dataset_config} \ + data/embedded={dataset_config}/l{level} \ ml/aggregator={aggregator} \ ml/head={head} \ + seed={seed} \ metadata.run_name={name} \ - +logger.tags.code_version={CODE_VERSION} \ - +logger.tags.data_variant={dataset_key} \ + +logger.tags.data_variant={dataset_config} \ + +logger.tags.level=level_{level} \ + +logger.tags.sweep_seed=seed_{seed} \ """, ], storage=[storage.secure.DATA, storage.secure.PROJECTS], From 954a8c0e11c26db604a7b6db882862c1e4eaeb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Sat, 19 Sep 2026 12:06:49 +0200 Subject: [PATCH 75/79] Recover filtered embedding artifacts --- preprocessing/filter_embeddings.py | 352 +++++++++++++++--- preprocessing/recover_embedding_splits.py | 326 ++++++++++++++++ .../preprocessing/run_filter_embeddings.py | 58 ++- .../run_recover_embedding_splits.py | 84 +++++ tests/preprocessing/__init__.py | 0 tests/preprocessing/test_filter_embeddings.py | 83 +++++ .../test_recover_embedding_splits.py | 50 +++ 7 files changed, 890 insertions(+), 63 deletions(-) create mode 100644 preprocessing/recover_embedding_splits.py create mode 100644 scripts/preprocessing/run_recover_embedding_splits.py create mode 100644 tests/preprocessing/__init__.py create mode 100644 tests/preprocessing/test_filter_embeddings.py create mode 100644 tests/preprocessing/test_recover_embedding_splits.py diff --git a/preprocessing/filter_embeddings.py b/preprocessing/filter_embeddings.py index f344345f..a561498c 100644 --- a/preprocessing/filter_embeddings.py +++ b/preprocessing/filter_embeddings.py @@ -8,19 +8,29 @@ import os import shutil import tempfile +import time +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING, TypedDict, cast import mlflow import numpy as np import pandas as pd from mlflow.artifacts import download_artifacts +from mlflow.exceptions import MlflowException + + +if TYPE_CHECKING: + from collections.abc import Iterator DEFAULT_DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" DEFAULT_EXPERIMENT = "MammaPrint" DEFAULT_TRACKING_URI = "http://mlflow-s3.rationai-mlflow" SPLITS = ("train", "val", "test") +STRATEGY_LOGICS = ("epithelium_only", "or", "and") +FILTER_STATE_FILENAME = "filter_state.json" REQUIRED_SCORE_COLUMNS = ( "slide_id", "x", @@ -37,20 +47,37 @@ class Strategy: logic: str +class FilterState(TypedDict): + schema_version: int + level: int + tiling_uri: str + embeddings_uri: str + strategies: list[str] + counts: dict[str, dict[str, int]] + scored_tiles: int + source_embedding_tiles: int + epithelium_threshold: float + cancer_threshold: float + + def _threshold_token(value: float) -> str: return f"{value:g}".replace(".", "") def _strategies( - epithelium_threshold: float, cancer_threshold: float + epithelium_threshold: float, + cancer_threshold: float, + selected_logics: tuple[str, ...] = STRATEGY_LOGICS, ) -> tuple[Strategy, ...]: epithelium = _threshold_token(epithelium_threshold) cancer = _threshold_token(cancer_threshold) - return ( + definitions = ( Strategy(f"epithel_{epithelium}", "epithelium_only"), Strategy(f"epithel_or_cancer_{epithelium}_{cancer}", "or"), Strategy(f"epithel_and_cancer_{epithelium}_{cancer}", "and"), ) + selected = set(selected_logics) + return tuple(strategy for strategy in definitions if strategy.logic in selected) def _selection_masks( @@ -150,7 +177,7 @@ def _prepare_output_dirs( {split: root / strategy.slug / f"embeddings_{split}" for split in SPLITS} ) for directory in directories.values(): - directory.mkdir(parents=True) + directory.mkdir(parents=True, exist_ok=True) output_dirs[strategy.logic] = directories return output_dirs @@ -228,7 +255,8 @@ def filter_embeddings( masks = _selection_masks(merged, epithelium_threshold, cancer_threshold) split = split_map[stem] - for logic, selected in masks.items(): + for logic in output_dirs: + selected = masks[logic] output = merged.loc[selected, list(REQUIRED_EMBEDDING_COLUMNS)] output_path = output_dirs[logic]["all"] / embeddings_path.name output.to_parquet(output_path, index=False, engine="pyarrow") @@ -245,6 +273,30 @@ def filter_embeddings( return counts, source_embedding_tiles +def _log_artifacts_with_retries( + local_dir: Path, + artifact_path: str, + logger: logging.Logger, + max_retries: int, +) -> None: + for attempt in range(max_retries + 1): + try: + mlflow.log_artifacts(str(local_dir), artifact_path=artifact_path) + return + except MlflowException: + if attempt == max_retries: + raise + delay = 2**attempt + logger.warning( + "Uploading %s failed; retrying in %d second(s) (%d/%d)", + artifact_path, + delay, + attempt + 1, + max_retries, + ) + time.sleep(delay) + + def _log_outputs( *, level: int, @@ -259,6 +311,7 @@ def _log_outputs( cancer_threshold: float, console_log: Path, logger: logging.Logger, + upload_max_retries: int, ) -> None: for strategy in strategies: dataset_name = f"l{level}_{strategy.slug}_embed" @@ -299,12 +352,20 @@ def _log_outputs( mlflow.log_metrics( {key: float(value) for key, value in counts[strategy.logic].items()} ) - for split_name, directory in output_dirs[strategy.logic].items(): + # Upload the training inputs first. If the optional combined artifact later + # fails, the split datasets are already usable and can be resumed separately. + for split_name in (*SPLITS, "all"): + directory = output_dirs[strategy.logic][split_name] artifact_path = ( "embeddings" if split_name == "all" else f"embeddings_{split_name}" ) logger.info("Logging %s to %s", dataset_name, artifact_path) - mlflow.log_artifacts(str(directory), artifact_path=artifact_path) + _log_artifacts_with_retries( + directory, + artifact_path, + logger, + max_retries=upload_max_retries, + ) mlflow.log_dict(manifest, "filter_manifest.json") for handler in logger.handlers: handler.flush() @@ -327,6 +388,120 @@ def _log_outputs( mlflow.log_artifact(str(console_log)) +@contextmanager +def _working_root( + *, level: int, work_dir: Path, output_dir: Path | None +) -> Iterator[Path]: + if output_dir is not None: + output_dir.mkdir(parents=True, exist_ok=True) + yield output_dir + return + + with tempfile.TemporaryDirectory( + prefix=f"filter-embeddings-l{level}-", dir=work_dir + ) as tmp: + yield Path(tmp) + + +def _state_payload( + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + strategies: tuple[Strategy, ...], + counts: dict[str, dict[str, int]], + scored_tiles: int, + source_embedding_tiles: int, + epithelium_threshold: float, + cancer_threshold: float, +) -> FilterState: + return { + "schema_version": 1, + "level": level, + "tiling_uri": tiling_uri, + "embeddings_uri": embeddings_uri, + "strategies": [strategy.logic for strategy in strategies], + "counts": counts, + "scored_tiles": scored_tiles, + "source_embedding_tiles": source_embedding_tiles, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + } + + +def _load_filter_state( + state_path: Path, + *, + level: int, + tiling_uri: str, + embeddings_uri: str, + strategies: tuple[Strategy, ...], + epithelium_threshold: float, + cancer_threshold: float, +) -> FilterState: + if not state_path.is_file(): + raise FileNotFoundError( + f"Cannot use --upload-only because {state_path} does not exist" + ) + state = cast("FilterState", json.loads(state_path.read_text())) + expected = { + "level": level, + "tiling_uri": tiling_uri, + "embeddings_uri": embeddings_uri, + "epithelium_threshold": epithelium_threshold, + "cancer_threshold": cancer_threshold, + } + mismatches = { + key: (state.get(key), expected_value) + for key, expected_value in expected.items() + if state.get(key) != expected_value + } + if mismatches: + raise ValueError(f"Saved filter state does not match this command: {mismatches}") + requested_strategies = {strategy.logic for strategy in strategies} + saved_strategies = set(state.get("strategies", [])) + missing_strategies = sorted(requested_strategies - saved_strategies) + if missing_strategies: + raise ValueError( + f"Saved filter state does not contain requested strategies: {missing_strategies}" + ) + return state + + +def _validate_output_dirs(output_dirs: dict[str, dict[str, Path]]) -> None: + for logic, directories in output_dirs.items(): + all_files = { + path.name: path for path in directories["all"].glob("*.parquet") + } + if not all_files: + raise ValueError(f"No filtered Parquet files found for strategy {logic}") + + split_names: set[str] = set() + for split in SPLITS: + for split_path in directories[split].glob("*.parquet"): + if split_path.name in split_names: + raise ValueError( + f"{split_path.name} appears in more than one split for {logic}" + ) + source_path = all_files.get(split_path.name) + if source_path is None: + raise ValueError( + f"{split_path.name} appears in {split} but not in all for {logic}" + ) + if split_path.stat().st_size != source_path.stat().st_size: + raise ValueError( + f"Size mismatch for {logic}/{split}/{split_path.name}" + ) + split_names.add(split_path.name) + + missing_split_files = sorted(set(all_files) - split_names) + if missing_split_files: + raise ValueError( + f"Filtered files have no split assignment for {logic}: " + f"{missing_split_files[:10]}" + ) + + def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -343,8 +518,28 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument("--epithelium-threshold", type=float, default=0.25) parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument( + "--strategies", + nargs="+", + choices=STRATEGY_LOGICS, + default=list(STRATEGY_LOGICS), + help="Strategies to materialise; defaults to all three.", + ) parser.add_argument("--data-mapping", default=DEFAULT_DATA_MAPPING) parser.add_argument("--work-dir", default="/mnt/projects/mammaprint") + parser.add_argument( + "--output-dir", + help=( + "Persistent working directory. Filtered files and filter_state.json are kept " + "here so a failed upload can be retried with --upload-only." + ), + ) + parser.add_argument( + "--upload-only", + action="store_true", + help="Upload an existing --output-dir without filtering again.", + ) + parser.add_argument("--upload-max-retries", type=int, default=3) parser.add_argument( "--tracking-uri", default=os.getenv("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) ) @@ -354,14 +549,36 @@ def _parse_args() -> argparse.Namespace: def main() -> None: args = _parse_args() + if args.upload_max_retries < 0: + raise ValueError("--upload-max-retries must be non-negative") + if args.upload_only and args.output_dir is None: + raise ValueError("--upload-only requires --output-dir") + persistent_output = Path(args.output_dir) if args.output_dir else None + if ( + persistent_output is not None + and persistent_output.is_dir() + and any(persistent_output.iterdir()) + and not args.upload_only + ): + raise FileExistsError( + f"Persistent output directory is not empty: {persistent_output}. " + "Use --upload-only to reuse completed filtering or choose a new directory." + ) + mlflow.set_tracking_uri(args.tracking_uri) mlflow.set_experiment(args.experiment_name) - strategies = _strategies(args.epithelium_threshold, args.cancer_threshold) + selected_logics = tuple(dict.fromkeys(args.strategies)) + strategies = _strategies( + args.epithelium_threshold, + args.cancer_threshold, + selected_logics, + ) - with tempfile.TemporaryDirectory( - prefix=f"filter-embeddings-l{args.level}-", dir=args.work_dir - ) as tmp: - root = Path(tmp) + with _working_root( + level=args.level, + work_dir=Path(args.work_dir), + output_dir=persistent_output, + ) as root: console_log = root / "console.log" logging.basicConfig( level=logging.INFO, @@ -370,51 +587,83 @@ def main() -> None: force=True, ) logger = logging.getLogger(__name__) - logger.info("Downloading scored tiling artifact: %s", args.tiling_uri) - tiling_download_dir = root / "source-tiling" - tiling_download_dir.mkdir() - tiling_dir = Path( - download_artifacts(args.tiling_uri, dst_path=str(tiling_download_dir)) - ) - - mounted_embeddings = ( - Path(args.embeddings_path) if args.embeddings_path is not None else None - ) - if mounted_embeddings is not None and mounted_embeddings.is_dir(): - embeddings_dir = mounted_embeddings - logger.info("Using mounted tissue embeddings: %s", embeddings_dir) + output_dirs = _prepare_output_dirs(root / "outputs", strategies) + state_path = root / FILTER_STATE_FILENAME + + if args.upload_only: + state = _load_filter_state( + state_path, + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + strategies=strategies, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + ) + counts = state["counts"] + scored_tiles = int(state["scored_tiles"]) + source_embedding_tiles = int(state["source_embedding_tiles"]) + logger.info("Reusing filtered output recorded in %s", state_path) else: - if mounted_embeddings is not None: - logger.warning( - "Mounted embedding directory %s is unavailable; downloading %s", - mounted_embeddings, - args.embeddings_uri, - ) + logger.info("Downloading scored tiling artifact: %s", args.tiling_uri) + tiling_download_dir = root / "source-tiling" + tiling_download_dir.mkdir(exist_ok=True) + tiling_dir = Path( + download_artifacts(args.tiling_uri, dst_path=str(tiling_download_dir)) + ) + + mounted_embeddings = ( + Path(args.embeddings_path) if args.embeddings_path is not None else None + ) + if mounted_embeddings is not None and mounted_embeddings.is_dir(): + embeddings_dir = mounted_embeddings + logger.info("Using mounted tissue embeddings: %s", embeddings_dir) else: - logger.info( - "Downloading tissue embedding artifact: %s", args.embeddings_uri - ) - embeddings_download_dir = root / "source-embeddings" - embeddings_download_dir.mkdir() - embeddings_dir = Path( - download_artifacts( - args.embeddings_uri, - dst_path=str(embeddings_download_dir), + if mounted_embeddings is not None: + logger.warning( + "Mounted embedding directory %s is unavailable; downloading %s", + mounted_embeddings, + args.embeddings_uri, + ) + else: + logger.info( + "Downloading tissue embedding artifact: %s", args.embeddings_uri + ) + embeddings_download_dir = root / "source-embeddings" + embeddings_download_dir.mkdir(exist_ok=True) + embeddings_dir = Path( + download_artifacts( + args.embeddings_uri, + dst_path=str(embeddings_download_dir), + ) ) + + scores_by_stem, scored_tiles = _load_scores(tiling_dir, args.level) + split_map = _load_split_map(Path(args.data_mapping)) + counts, source_embedding_tiles = filter_embeddings( + scores_by_stem=scores_by_stem, + embeddings_dir=embeddings_dir, + output_dirs=output_dirs, + split_map=split_map, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + logger=logger, ) + state = _state_payload( + level=args.level, + tiling_uri=args.tiling_uri, + embeddings_uri=args.embeddings_uri, + strategies=strategies, + counts=counts, + scored_tiles=scored_tiles, + source_embedding_tiles=source_embedding_tiles, + epithelium_threshold=args.epithelium_threshold, + cancer_threshold=args.cancer_threshold, + ) + state_path.write_text(json.dumps(state, indent=2)) + logger.info("Saved resumable filter state to %s", state_path) - scores_by_stem, scored_tiles = _load_scores(tiling_dir, args.level) - split_map = _load_split_map(Path(args.data_mapping)) - output_dirs = _prepare_output_dirs(root / "outputs", strategies) - counts, source_embedding_tiles = filter_embeddings( - scores_by_stem=scores_by_stem, - embeddings_dir=embeddings_dir, - output_dirs=output_dirs, - split_map=split_map, - epithelium_threshold=args.epithelium_threshold, - cancer_threshold=args.cancer_threshold, - logger=logger, - ) + _validate_output_dirs(output_dirs) logger.info("Selection summary:\n%s", json.dumps(counts, indent=2)) _log_outputs( level=args.level, @@ -429,6 +678,7 @@ def main() -> None: cancer_threshold=args.cancer_threshold, console_log=console_log, logger=logger, + upload_max_retries=args.upload_max_retries, ) diff --git a/preprocessing/recover_embedding_splits.py b/preprocessing/recover_embedding_splits.py new file mode 100644 index 00000000..23067a64 --- /dev/null +++ b/preprocessing/recover_embedding_splits.py @@ -0,0 +1,326 @@ +"""Resume split artifact uploads from a complete per-slide embedding artifact.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import tempfile +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING + +import mlflow +from mlflow import MlflowClient +from mlflow.entities import FileInfo +from mlflow.exceptions import MlflowException + +from preprocessing.filter_embeddings import ( + DEFAULT_DATA_MAPPING, + DEFAULT_TRACKING_URI, + SPLITS, + _load_split_map, +) + + +if TYPE_CHECKING: + from collections.abc import Callable + + +@dataclass(frozen=True) +class ArtifactFile: + path: str + size: int + + @property + def name(self) -> str: + return Path(self.path).name + + +@dataclass(frozen=True) +class RecoveryPlan: + expected: dict[str, dict[str, ArtifactFile]] + existing: dict[str, dict[str, ArtifactFile]] + missing: dict[str, dict[str, ArtifactFile]] + + @property + def missing_files(self) -> int: + return sum(len(files) for files in self.missing.values()) + + @property + def missing_bytes(self) -> int: + return sum( + file.size for files in self.missing.values() for file in files.values() + ) + + +def _artifact_files(items: list[FileInfo], artifact_path: str) -> dict[str, ArtifactFile]: + directories = [item.path for item in items if item.is_dir] + if directories: + raise ValueError(f"Unexpected directories under {artifact_path}: {directories[:10]}") + + files: dict[str, ArtifactFile] = {} + for item in items: + name = Path(item.path).name + if Path(name).suffix != ".parquet": + raise ValueError(f"Unexpected non-Parquet artifact: {item.path}") + if name in files: + raise ValueError(f"Duplicate artifact name under {artifact_path}: {name}") + files[name] = ArtifactFile(item.path, int(item.file_size or 0)) + return files + + +def _build_recovery_plan( + source: dict[str, ArtifactFile], + destinations: dict[str, dict[str, ArtifactFile]], + split_map: dict[str, str], +) -> RecoveryPlan: + if not source: + raise ValueError("The source embedding artifact is empty") + + unmapped = sorted(Path(name).stem for name in source if Path(name).stem not in split_map) + if unmapped: + raise ValueError(f"Source slides have no split mapping: {unmapped[:10]}") + + expected: dict[str, dict[str, ArtifactFile]] = {split: {} for split in SPLITS} + for name, file in source.items(): + expected[split_map[Path(name).stem]][name] = file + + for split in SPLITS: + unexpected = sorted(set(destinations[split]) - set(expected[split])) + if unexpected: + raise ValueError( + f"embeddings_{split} contains unexpected files: {unexpected[:10]}" + ) + mismatched = sorted( + name + for name, file in destinations[split].items() + if file.size != expected[split][name].size + ) + if mismatched: + raise ValueError( + f"embeddings_{split} contains size-mismatched files: {mismatched[:10]}" + ) + + missing = { + split: { + name: file + for name, file in expected[split].items() + if name not in destinations[split] + } + for split in SPLITS + } + return RecoveryPlan(expected=expected, existing=destinations, missing=missing) + + +def _retry[T]( + operation: Callable[[], T], *, label: str, max_retries: int, logger: logging.Logger +) -> T: + for attempt in range(max_retries + 1): + try: + return operation() + except MlflowException: + if attempt == max_retries: + raise + delay = 2**attempt + logger.warning( + "%s failed; retrying in %d second(s) (%d/%d)", + label, + delay, + attempt + 1, + max_retries, + ) + time.sleep(delay) + raise AssertionError("unreachable") + + +def _load_plan( + client: MlflowClient, + *, + run_id: str, + source_artifact_path: str, + split_map: dict[str, str], +) -> RecoveryPlan: + source = _artifact_files( + client.list_artifacts(run_id, source_artifact_path), source_artifact_path + ) + destinations = { + split: _artifact_files( + client.list_artifacts(run_id, f"embeddings_{split}"), + f"embeddings_{split}", + ) + for split in SPLITS + } + return _build_recovery_plan(source, destinations, split_map) + + +def _format_bytes(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024 or unit == "TiB": + return f"{value:.1f} {unit}" + value /= 1024 + raise AssertionError("unreachable") + + +def _print_plan(plan: RecoveryPlan) -> None: + for split in SPLITS: + print( + f"{split}: {len(plan.existing[split])}/{len(plan.expected[split])} present; " + f"{len(plan.missing[split])} missing" + ) + print( + f"Total transfer: {plan.missing_files} files, " + f"{_format_bytes(plan.missing_bytes)}" + ) + + +def _recover( + client: MlflowClient, + *, + run_id: str, + plan: RecoveryPlan, + max_retries: int, + logger: logging.Logger, +) -> None: + total = plan.missing_files + completed = 0 + for split in SPLITS: + artifact_path = f"embeddings_{split}" + for name, source in sorted(plan.missing[split].items()): + with tempfile.TemporaryDirectory(prefix="recover-embedding-split-") as tmp: + local_path = Path( + _retry( + partial( + client.download_artifacts, + run_id, + source.path, + dst_path=tmp, + ), + label=f"Download {source.path}", + max_retries=max_retries, + logger=logger, + ) + ) + if local_path.stat().st_size != source.size: + raise ValueError( + f"Downloaded size mismatch for {source.path}: " + f"{local_path.stat().st_size} != {source.size}" + ) + _retry( + partial( + client.log_artifact, + run_id, + str(local_path), + artifact_path=artifact_path, + ), + label=f"Upload {artifact_path}/{name}", + max_retries=max_retries, + logger=logger, + ) + completed += 1 + if completed % 25 == 0 or completed == total: + logger.info("Recovered %d/%d split files", completed, total) + + +def _log_recovery_manifest( + client: MlflowClient, + *, + run_id: str, + source_artifact_path: str, + plan: RecoveryPlan, +) -> None: + manifest = { + "schema_version": 1, + "run_id": run_id, + "source_artifact_path": source_artifact_path, + "recovered_at": datetime.now(UTC).isoformat(), + "file_counts": {split: len(plan.expected[split]) for split in SPLITS}, + } + with tempfile.TemporaryDirectory(prefix="embedding-recovery-manifest-") as tmp: + path = Path(tmp) / "split_recovery_manifest.json" + path.write_text(json.dumps(manifest, indent=2)) + client.log_artifact(run_id, str(path)) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True) + parser.add_argument("--source-artifact-path", default="embeddings") + parser.add_argument("--data-mapping", default=DEFAULT_DATA_MAPPING) + parser.add_argument( + "--tracking-uri", default=os.getenv("MLFLOW_TRACKING_URI", DEFAULT_TRACKING_URI) + ) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument( + "--apply", + action="store_true", + help="Perform uploads. Without this flag, only print the recovery plan.", + ) + parser.add_argument( + "--mark-finished", + action="store_true", + help="Mark the recovered MLflow run FINISHED after complete validation.", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.max_retries < 0: + raise ValueError("--max-retries must be non-negative") + if args.mark_finished and not args.apply: + raise ValueError("--mark-finished requires --apply") + + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logger = logging.getLogger(__name__) + mlflow.set_tracking_uri(args.tracking_uri) + client = MlflowClient() + split_map = _load_split_map(Path(args.data_mapping)) + plan = _load_plan( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + split_map=split_map, + ) + _print_plan(plan) + if not args.apply: + print("Dry run only; pass --apply to upload missing files.") + return + + _recover( + client, + run_id=args.run_id, + plan=plan, + max_retries=args.max_retries, + logger=logger, + ) + final_plan = _load_plan( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + split_map=split_map, + ) + if final_plan.missing_files: + raise RuntimeError( + f"Recovery finished with {final_plan.missing_files} files still missing" + ) + + _log_recovery_manifest( + client, + run_id=args.run_id, + source_artifact_path=args.source_artifact_path, + plan=final_plan, + ) + client.set_tag(run_id=args.run_id, key="split_upload_recovered", value="true") + if args.mark_finished: + client.set_terminated(args.run_id, status="FINISHED") + print("Recovery complete; all split artifact files are present and size-validated.") + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/run_filter_embeddings.py b/scripts/preprocessing/run_filter_embeddings.py index 8c518843..d02447b9 100644 --- a/scripts/preprocessing/run_filter_embeddings.py +++ b/scripts/preprocessing/run_filter_embeddings.py @@ -20,6 +20,7 @@ level: f"/mnt/projects/mammaprint/embeddings/l{level}_tissue_embed" for level in TISSUE_EMBEDDING_URIS } +STRATEGY_CHOICES = ("epithelium_only", "or", "and") def _parse_args() -> argparse.Namespace: @@ -32,6 +33,15 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--embeddings-path") parser.add_argument("--epithelium-threshold", type=float, default=0.25) parser.add_argument("--cancer-threshold", type=float, default=0.5) + parser.add_argument( + "--strategies", + nargs="+", + choices=STRATEGY_CHOICES, + default=list(STRATEGY_CHOICES), + ) + parser.add_argument("--output-dir") + parser.add_argument("--upload-only", action="store_true") + parser.add_argument("--upload-max-retries", type=int, default=3) parser.add_argument("--dry-run", action="store_true") return parser.parse_args() @@ -42,23 +52,47 @@ def _token(value: float) -> str: def main() -> None: args = _parse_args() + if args.upload_only and not args.output_dir: + raise ValueError("--upload-only requires --output-dir") embeddings_uri = args.embeddings_uri or TISSUE_EMBEDDING_URIS[args.level] embeddings_path = args.embeddings_path or TISSUE_EMBEDDING_PATHS[args.level] + strategies = tuple(dict.fromkeys(args.strategies)) + strategy_token = "-".join( + {"epithelium_only": "epi", "or": "or", "and": "and"}[strategy] + for strategy in strategies + ) job_name = ( f"mammaprint-filter-embeddings-l{args.level}-" - f"epi{_token(args.epithelium_threshold)}-c{_token(args.cancer_threshold)}" - ) - filter_command = " ".join( - [ - "uv run -m preprocessing.filter_embeddings", - f"--level {args.level}", - f"--tiling-uri {shlex.quote(args.tiling_uri)}", - f"--embeddings-uri {shlex.quote(embeddings_uri)}", - f"--embeddings-path {shlex.quote(embeddings_path)}", - f"--epithelium-threshold {args.epithelium_threshold:g}", - f"--cancer-threshold {args.cancer_threshold:g}", - ] + f"{strategy_token}-epi{_token(args.epithelium_threshold)}-" + f"c{_token(args.cancer_threshold)}" ) + command = [ + "uv", + "run", + "-m", + "preprocessing.filter_embeddings", + "--level", + str(args.level), + "--tiling-uri", + args.tiling_uri, + "--embeddings-uri", + embeddings_uri, + "--embeddings-path", + embeddings_path, + "--epithelium-threshold", + f"{args.epithelium_threshold:g}", + "--cancer-threshold", + f"{args.cancer_threshold:g}", + "--strategies", + *strategies, + "--upload-max-retries", + str(args.upload_max_retries), + ] + if args.output_dir: + command.extend(["--output-dir", args.output_dir]) + if args.upload_only: + command.append("--upload-only") + filter_command = shlex.join(command) if args.dry_run: print(f"Job: {job_name}") diff --git a/scripts/preprocessing/run_recover_embedding_splits.py b/scripts/preprocessing/run_recover_embedding_splits.py new file mode 100644 index 00000000..14ea2c22 --- /dev/null +++ b/scripts/preprocessing/run_recover_embedding_splits.py @@ -0,0 +1,84 @@ +"""Submit a resumable job that completes split uploads from an MLflow run's all artifact.""" + +from __future__ import annotations + +import argparse +import re +import shlex + +from kube_jobs import storage, submit_job + + +USERNAME = "kissmi" +GIT_BRANCH = "feat/tiling-values" +TRACKING_URI = "http://mlflow-s3.rationai-mlflow" +DATA_MAPPING = "/mnt/projects/mammaprint/data_mapping.csv" + + +def _job_name(run_id: str) -> str: + token = re.sub(r"[^a-z0-9]+", "-", run_id.lower()).strip("-") + return f"mammaprint-recover-embedding-splits-{token[:12]}" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True) + parser.add_argument("--source-artifact-path", default="embeddings") + parser.add_argument("--data-mapping", default=DATA_MAPPING) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument("--mark-finished", action="store_true") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + command = [ + "uv", + "run", + "-m", + "preprocessing.recover_embedding_splits", + "--run-id", + args.run_id, + "--source-artifact-path", + args.source_artifact_path, + "--data-mapping", + args.data_mapping, + "--max-retries", + str(args.max_retries), + "--apply", + ] + if args.mark_finished: + command.append("--mark-finished") + worker_command = shlex.join(command) + job_name = _job_name(args.run_id) + + if args.dry_run: + print(f"Job: {job_name}") + print(f"Command: {worker_command}") + return + + submit_job( + job_name=job_name, + username=USERNAME, + image="cerit.io/rationai/base:2.0.6", + cpu=4, + memory="16Gi", + shm="4Gi", + gpu=None, + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_BRANCH}", + f"export MLFLOW_TRACKING_URI={TRACKING_URI}", + "uv sync --frozen", + worker_command, + ], + storage=[storage.secure.PROJECTS], + ) + print(f"Submitted {job_name}.") + + +if __name__ == "__main__": + main() diff --git a/tests/preprocessing/__init__.py b/tests/preprocessing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/preprocessing/test_filter_embeddings.py b/tests/preprocessing/test_filter_embeddings.py new file mode 100644 index 00000000..d9bb4318 --- /dev/null +++ b/tests/preprocessing/test_filter_embeddings.py @@ -0,0 +1,83 @@ +import json +import logging + +import pandas as pd + +from preprocessing.filter_embeddings import ( + _load_filter_state, + _prepare_output_dirs, + _strategies, + _validate_output_dirs, + filter_embeddings, +) + + +def test_filter_embeddings_materialises_only_selected_strategies(tmp_path): + embeddings_dir = tmp_path / "source" + embeddings_dir.mkdir() + pd.DataFrame( + { + "x": [0, 1, 2], + "y": [10, 11, 12], + "embedding": [[1.0], [2.0], [3.0]], + } + ).to_parquet(embeddings_dir / "slide.parquet", index=False) + scores = pd.DataFrame( + { + "x": [0, 1, 2], + "y": [10, 11, 12], + "epithelium_overlap": [0.3, 0.1, 0.4], + "cancer_overlap": [0.1, 0.8, 0.9], + } + ) + strategies = _strategies(0.25, 0.5, ("or", "and")) + output_dirs = _prepare_output_dirs(tmp_path / "outputs", strategies) + + counts, source_tiles = filter_embeddings( + scores_by_stem={"slide": scores}, + embeddings_dir=embeddings_dir, + output_dirs=output_dirs, + split_map={"slide": "train"}, + epithelium_threshold=0.25, + cancer_threshold=0.5, + logger=logging.getLogger(__name__), + ) + + assert source_tiles == 3 + assert counts["or"]["tiles"] == 3 + assert counts["and"]["tiles"] == 1 + assert set(output_dirs) == {"or", "and"} + assert len(pd.read_parquet(output_dirs["or"]["all"] / "slide.parquet")) == 3 + assert len(pd.read_parquet(output_dirs["and"]["all"] / "slide.parquet")) == 1 + _validate_output_dirs(output_dirs) + + +def test_upload_only_accepts_subset_of_saved_strategies(tmp_path): + state_path = tmp_path / "filter_state.json" + state_path.write_text( + json.dumps( + { + "level": 2, + "tiling_uri": "tiling", + "embeddings_uri": "embeddings", + "strategies": ["or", "and"], + "epithelium_threshold": 0.25, + "cancer_threshold": 0.5, + "counts": {}, + "scored_tiles": 3, + "source_embedding_tiles": 3, + } + ) + ) + + state = _load_filter_state( + state_path, + level=2, + tiling_uri="tiling", + embeddings_uri="embeddings", + strategies=_strategies(0.25, 0.5, ("and",)), + epithelium_threshold=0.25, + cancer_threshold=0.5, + ) + + assert state["strategies"] == ["or", "and"] diff --git a/tests/preprocessing/test_recover_embedding_splits.py b/tests/preprocessing/test_recover_embedding_splits.py new file mode 100644 index 00000000..c72a5522 --- /dev/null +++ b/tests/preprocessing/test_recover_embedding_splits.py @@ -0,0 +1,50 @@ +import pytest + +from preprocessing.recover_embedding_splits import ( + ArtifactFile, + _build_recovery_plan, +) + + +def _file(path: str, size: int = 10) -> ArtifactFile: + return ArtifactFile(path=path, size=size) + + +def test_recovery_plan_reuses_valid_files_and_finds_missing_files(): + source = { + "train.parquet": _file("embeddings/train.parquet", 10), + "val.parquet": _file("embeddings/val.parquet", 20), + "test.parquet": _file("embeddings/test.parquet", 30), + } + destinations = { + "train": { + "train.parquet": _file("embeddings_train/train.parquet", 10), + }, + "val": {}, + "test": {}, + } + + plan = _build_recovery_plan( + source, + destinations, + {"train": "train", "val": "val", "test": "test"}, + ) + + assert plan.missing_files == 2 + assert plan.missing_bytes == 50 + assert set(plan.missing["val"]) == {"val.parquet"} + assert set(plan.missing["test"]) == {"test.parquet"} + + +def test_recovery_plan_rejects_size_mismatch(): + source = {"slide.parquet": _file("embeddings/slide.parquet", 10)} + destinations = { + "train": { + "slide.parquet": _file("embeddings_train/slide.parquet", 9), + }, + "val": {}, + "test": {}, + } + + with pytest.raises(ValueError, match="size-mismatched"): + _build_recovery_plan(source, destinations, {"slide": "train"}) From 205e8f77acc30807bb89d2b0398020c026d3ecb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 24 Sep 2026 19:14:54 +0200 Subject: [PATCH 76/79] feat: sweep runner feat: list pending gpus --- scripts/ml/list_pending_gpu_jobs.py | 138 ++++++++++++++++ scripts/ml/run_train_sweep.py | 236 ++++++++++++++++++++++++---- 2 files changed, 340 insertions(+), 34 deletions(-) create mode 100644 scripts/ml/list_pending_gpu_jobs.py diff --git a/scripts/ml/list_pending_gpu_jobs.py b/scripts/ml/list_pending_gpu_jobs.py new file mode 100644 index 00000000..79aadc80 --- /dev/null +++ b/scripts/ml/list_pending_gpu_jobs.py @@ -0,0 +1,138 @@ +"""List this user's Kubernetes jobs that are waiting for a GPU. + +The GPU request is read from the pod spec created by ``kube_jobs.submit_job``. +This script is read-only. + +Usage: + uv run python scripts/ml/list_pending_gpu_jobs.py + uv run python scripts/ml/list_pending_gpu_jobs.py --prefix mammaprint-sweep- + uv run python scripts/ml/list_pending_gpu_jobs.py --user kissmi +""" + +import argparse +from collections import Counter +from collections.abc import Iterator, Mapping + +from kube_jobs.constants import CLUSTER, JOB_NAMESPACE +from kubernetes import client +from kubernetes.config import load_kube_config + + +DEFAULT_USER = "kissmi" +GPU_PRODUCTS = { + "NVIDIA-A100-80GB-PCIe": "A100", + "NVIDIA-A40": "A40", + "NVIDIA-H100-NVL": "H100", + "Tesla-P100-SXM2-16GB": "Tesla P100", +} + + +def balanced_gpu_iterator(waiting_counts: Mapping[str, int]) -> Iterator[str]: + """Yield GPUs forever, first bringing their waiting counts to the same level. + + Ties follow the mapping's insertion order. The input mapping is not modified. + Include every GPU that may be selected, using a count of zero when its queue is + empty. + """ + if not waiting_counts: + raise ValueError("At least one GPU is required.") + if any(count < 0 for count in waiting_counts.values()): + raise ValueError("Waiting counts cannot be negative.") + + counts = dict(waiting_counts) + while True: + lowest_count = min(counts.values()) + for gpu in counts: + if counts[gpu] == lowest_count: + counts[gpu] += 1 + yield gpu + + +def _gpu_name(pod: client.V1Pod) -> str: + limits = { + resource: quantity + for container in pod.spec.containers + for resource, quantity in (container.resources.limits or {}).items() + } + + for resource in sorted(limits): + if resource.startswith("nvidia.com/mig-"): + return resource.removeprefix("nvidia.com/") + + if "nvidia.com/gpu" in limits: + selectors = pod.spec.node_selector or {} + product = selectors.get("nvidia.com/gpu.product", "GPU") + return GPU_PRODUCTS.get(product, product) + + return "none" + + +def _pending_gpu_jobs(user: str, prefix: str | None) -> list[tuple[str, str]]: + load_kube_config(context=CLUSTER) + jobs = ( + client.BatchV1Api() + .list_namespaced_job( + namespace=JOB_NAMESPACE, + label_selector=f"created_by={user}", + ) + .items + ) + job_names_by_uid = { + job.metadata.uid: job.metadata.name + for job in jobs + if job.metadata is not None + and job.metadata.uid is not None + and (prefix is None or job.metadata.name.startswith(prefix)) + } + + pending = [] + pods = client.CoreV1Api().list_namespaced_pod(namespace=JOB_NAMESPACE).items + for pod in pods: + if pod.status is None or pod.status.phase != "Pending": + continue + owner_uids = { + owner.uid for owner in (pod.metadata.owner_references or []) if owner.uid + } + job_uid = next((uid for uid in owner_uids if uid in job_names_by_uid), None) + if job_uid is not None: + pending.append((_gpu_name(pod), job_names_by_uid[job_uid])) + + return sorted(pending) + + +def pending_gpu_counts(user: str, prefix: str | None = None) -> Counter[str]: + return Counter(gpu for gpu, _ in _pending_gpu_jobs(user, prefix)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--user", + default=DEFAULT_USER, + help=f"Value of the created_by job label (default: {DEFAULT_USER}).", + ) + parser.add_argument( + "--prefix", + help="Only show job names starting with this prefix.", + ) + args = parser.parse_args() + + pending = _pending_gpu_jobs(args.user, args.prefix) + if not pending: + print("No pending jobs found.") + return 0 + + counts = Counter(gpu for gpu, _ in pending) + print( + "Waiting by GPU: " + + ", ".join(f"{gpu}={count}" for gpu, count in sorted(counts.items())) + ) + print(f"\n{'GPU':<16} JOB") + print("-" * 80) + for gpu, job_name in pending: + print(f"{gpu:<16} {job_name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py index 533a81f0..668d051f 100644 --- a/scripts/ml/run_train_sweep.py +++ b/scripts/ml/run_train_sweep.py @@ -1,10 +1,20 @@ import itertools import os import time +from collections import deque +from collections.abc import Mapping +from heapq import heapify, heappop, heappush +from typing import TYPE_CHECKING, Any from kube_jobs import storage, submit_job +if TYPE_CHECKING or __package__: + from scripts.ml.list_pending_gpu_jobs import pending_gpu_counts +else: + from list_pending_gpu_jobs import pending_gpu_counts + + # Each objective points to the matching training experiment. Edit these values, # together with the dataset/model lists below, to define the Cartesian sweep. EXPERIMENTS = { @@ -20,19 +30,65 @@ "eor5": "epithel_or_cancer_5", "ean5": "epithel_and_cancer_5", } -LEVELS = ( - # 2, - 3, 4, 5) +LEVELS = (2, 3, 4, 5) AGGREGATORS = ("mean", "max", "attention", "transformer") HEADS = ("mlp", "linear") SEEDS = tuple(range(20)) -GPUS = [ +GPUS = ( + "A40", + "H100", "mig-2g.20gb", - # "H100", - # "A40", + "mig-1g.10gb", +) +BASIC_RESOURCES = {"cpu": 8, "memory": "16Gi"} + +# GPU rules limit which entries from GPUS a matching job may use. Jobs without +# a matching rule may use the complete pool. All assignments share one set of +# projected waiting counts, including assignments restricted to one GPU type. +GPU_RULES: list[dict[str, Any]] = [ + { + "match": { + "level": (2, 3), + "aggregator": ("transformer", "spatial_transformer"), + }, + "only": ("H100",), + }, + { + "match": { + "level": 4, + "aggregator": ("transformer", "spatial_transformer"), + }, + "exclude_gpus": ("mig-1g.10gb",), + }, +] + +# Override host resources independently of GPU placement. Rules are applied in +# order, so later matches override only the fields they specify. +RESOURCE_RULES: list[dict[str, Any]] = [ + { + "match": { + "level": (2, 3), + "aggregator": ("mean", "max"), + }, + "resources": {"memory": "20Gi"}, + }, + { + "match": { + "level": (2, 3), + "aggregator": "attention", + }, + "resources": {"memory": "40Gi"}, + }, + { + "match": { + "level": (2, 3), + "aggregator": ("transformer", "spatial_transformer"), + }, + "resources": {"memory": "56Gi"}, + }, ] GIT_REF = "feat/tiling-values" -SUBMISSION_INTERVAL_SECONDS = 10 +SUBMISSION_INTERVAL_SECONDS = 2 # * 60 USERNAME = "kissmi" IMAGE = "cerit.io/rationai/base:2.0.6" POD_MLFLOW_TRACKING_URI = os.getenv( @@ -45,6 +101,9 @@ MLFLOW_EXPERIMENT = "MammaPrint" JOB_NAME_PREFIX = "mammaprint-sweep-" +SweepJob = tuple[int, str, str, int, str, str, str, str, int] +ResolvedJob = tuple[SweepJob, dict[str, Any]] + def _job_name( dataset_key: str, @@ -57,7 +116,7 @@ def _job_name( return f"{JOB_NAME_PREFIX}{dataset_key}-l{level}-{head}-{aggregator}-{objective}-s{seed}" -def _sweep_jobs() -> list[tuple[int, str, str, int, str, str, str, str, int]]: +def _sweep_jobs() -> list[SweepJob]: combinations = itertools.product( SEEDS, DATASETS.items(), @@ -89,16 +148,100 @@ def _sweep_jobs() -> list[tuple[int, str, str, int, str, str, str, str, int]]: ] -def _gpu_for_job(job_index: int) -> str: - configurations_per_seed = ( - len(DATASETS) - * len(LEVELS) - * len(EXPERIMENTS) - * len(AGGREGATORS) - * len(HEADS) - ) - seed_index, configuration_index = divmod(job_index, configurations_per_seed) - return GPUS[(configuration_index + seed_index) % len(GPUS)] +def _matches(actual: object, expected: object) -> bool: + if isinstance(expected, (list, tuple, set, frozenset)): + return actual in expected + return actual == expected + + +def _job_matches(job: dict[str, object], conditions: dict[str, object]) -> bool: + return all(_matches(job.get(key), value) for key, value in conditions.items()) + + +def _eligible_gpus(job: dict[str, object]) -> tuple[str, ...]: + eligible = list(GPUS) + for rule in GPU_RULES: + if not _job_matches(job, rule.get("match", {})): + continue + if "only" in rule: + allowed = set(rule["only"]) + eligible = [gpu for gpu in eligible if gpu in allowed] + excluded = set(rule.get("exclude_gpus", ())) + eligible = [gpu for gpu in eligible if gpu not in excluded] + + if not eligible: + raise ValueError(f"No configured GPU can run job {job!r}.") + return tuple(eligible) + + +def _assign_gpu( + job: dict[str, object], projected_waiting_counts: dict[str, int] +) -> str: + """Choose the least-loaded eligible GPU, using GPUS order for ties.""" + eligible = _eligible_gpus(job) + gpu = min(eligible, key=lambda candidate: projected_waiting_counts[candidate]) + projected_waiting_counts[gpu] += 1 + return gpu + + +def _resources_for_job( + projected_waiting_counts: dict[str, int], + dataset_key: str, + dataset_config: str, + level: int, + objective: str, + experiment: str, + aggregator: str, + head: str, + seed: int, +) -> dict[str, Any]: + job = { + "dataset_key": dataset_key, + "dataset_config": dataset_config, + "level": level, + "objective": objective, + "experiment": experiment, + "aggregator": aggregator, + "head": head, + "seed": seed, + } + resources: dict[str, Any] = dict(BASIC_RESOURCES) + + for rule in RESOURCE_RULES: + matches = _job_matches(job, rule.get("match", {})) + excluded = "exclude" in rule and _job_matches(job, rule["exclude"]) + if matches and not excluded: + resources.update(rule["resources"]) + + resources["gpu"] = _assign_gpu(job, projected_waiting_counts) + return resources + + +def _balanced_job_order( + resolved_jobs: list[ResolvedJob], waiting_counts: Mapping[str, int] +) -> list[ResolvedJob]: + """Interleave GPU queues by their projected waiting-job counts.""" + queues: dict[str, deque[ResolvedJob]] = {} + first_job_index: dict[str, int] = {} + for list_index, resolved_job in enumerate(resolved_jobs): + gpu = resolved_job[1]["gpu"] + if not isinstance(gpu, str): + raise TypeError(f"Resolved GPU must be a string, got {gpu!r}.") + queues.setdefault(gpu, deque()).append(resolved_job) + first_job_index.setdefault(gpu, list_index) + + gpu_heap = [ + (waiting_counts.get(gpu, 0), first_job_index[gpu], gpu) for gpu in queues + ] + heapify(gpu_heap) + + ordered_jobs = [] + while gpu_heap: + projected_count, tie_breaker, gpu = heappop(gpu_heap) + ordered_jobs.append(queues[gpu].popleft()) + if queues[gpu]: + heappush(gpu_heap, (projected_count + 1, tie_breaker, gpu)) + return ordered_jobs def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: @@ -164,18 +307,41 @@ def main() -> None: f"Sweep contains {len(jobs)} jobs: {len(existing_names)} already submitted, " f"{len(pending_jobs)} pending." ) + waiting_counts = pending_gpu_counts(USERNAME) + projected_waiting_counts = {gpu: waiting_counts[gpu] for gpu in GPUS} + print(f"GPU waiting counts: {projected_waiting_counts}") - for pending_index, ( - job_index, - dataset_key, - dataset_config, - level, - objective, - experiment, - aggregator, - head, - seed, - ) in enumerate(pending_jobs): + resolved_jobs = [ + ( + job, + _resources_for_job( + projected_waiting_counts, + job[1], + job[2], + job[3], + job[4], + job[5], + job[6], + job[7], + job[8], + ), + ) + for job in pending_jobs + ] + scheduled_jobs = _balanced_job_order(resolved_jobs, waiting_counts) + + for pending_index, (job, resources) in enumerate(scheduled_jobs): + ( + _, + dataset_key, + dataset_config, + level, + objective, + experiment, + aggregator, + head, + seed, + ) = job if pending_index > 0: print( f"Waiting {SUBMISSION_INTERVAL_SECONDS / 60} minutes before submitting the next job." @@ -183,15 +349,17 @@ def main() -> None: time.sleep(SUBMISSION_INTERVAL_SECONDS) name = _job_name(dataset_key, level, head, aggregator, objective, seed) - gpu = _gpu_for_job(job_index) - print(f"Submitting {name} to {gpu} ({pending_index + 1}/{len(pending_jobs)}).") + print( + f"Submitting {name} with {resources} " + f"({pending_index + 1}/{len(pending_jobs)})." + ) submit_job( job_name=name, username=USERNAME, image=IMAGE, - cpu=10, - memory="36Gi", - gpu=gpu, + cpu=resources["cpu"], + memory=resources["memory"], + gpu=resources["gpu"], public=False, script=[ "git clone https://github.com/rationAI/mammaprint workdir", From 63f28e7ee5c5f614df932dd917fe281f89dbbca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Thu, 24 Sep 2026 19:15:26 +0200 Subject: [PATCH 77/79] tmp: ignore h100 --- scripts/ml/run_train_sweep.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py index 668d051f..ce34651b 100644 --- a/scripts/ml/run_train_sweep.py +++ b/scripts/ml/run_train_sweep.py @@ -331,6 +331,9 @@ def main() -> None: scheduled_jobs = _balanced_job_order(resolved_jobs, waiting_counts) for pending_index, (job, resources) in enumerate(scheduled_jobs): + if resources["gpu"] == "H100": + continue + ( _, dataset_key, From 2253088ea2edf4c1e3f6fb29f1128ccfa05993f0 Mon Sep 17 00:00:00 2001 From: kissmi Date: Sun, 27 Sep 2026 20:03:38 +0200 Subject: [PATCH 78/79] feat: cap sweep submissions without delay --- scripts/ml/run_train_sweep.py | 43 ++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py index ce34651b..2a825e53 100644 --- a/scripts/ml/run_train_sweep.py +++ b/scripts/ml/run_train_sweep.py @@ -1,6 +1,6 @@ +import argparse import itertools import os -import time from collections import deque from collections.abc import Mapping from heapq import heapify, heappop, heappush @@ -88,7 +88,6 @@ }, ] GIT_REF = "feat/tiling-values" -SUBMISSION_INTERVAL_SECONDS = 2 # * 60 USERNAME = "kissmi" IMAGE = "cerit.io/rationai/base:2.0.6" POD_MLFLOW_TRACKING_URI = os.getenv( @@ -244,6 +243,18 @@ def _balanced_job_order( return ordered_jobs +def _jobs_to_submit( + scheduled_jobs: list[ResolvedJob], max_jobs: int | None +) -> list[ResolvedJob]: + """Return the scheduled non-H100 jobs, capped when requested.""" + eligible = [ + resolved_job + for resolved_job in scheduled_jobs + if resolved_job[1]["gpu"] != "H100" + ] + return eligible if max_jobs is None else eligible[:max_jobs] + + def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: """Fetch sweep runs that already exist, including failed/running runs.""" from mlflow import MlflowClient @@ -288,6 +299,16 @@ def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--max-jobs", + type=int, + help="Submit at most this many non-H100 jobs from the automatic sweep order.", + ) + args = parser.parse_args() + if args.max_jobs is not None and args.max_jobs <= 0: + parser.error("--max-jobs must be a positive integer") + jobs = _sweep_jobs() expected_names = { _job_name(dataset_key, level, head, aggregator, objective, seed) @@ -329,11 +350,13 @@ def main() -> None: for job in pending_jobs ] scheduled_jobs = _balanced_job_order(resolved_jobs, waiting_counts) + jobs_to_submit = _jobs_to_submit(scheduled_jobs, args.max_jobs) + print( + f"Submitting {len(jobs_to_submit)} of {len(pending_jobs)} pending jobs" + + (f" (limit {args.max_jobs})." if args.max_jobs is not None else ".") + ) - for pending_index, (job, resources) in enumerate(scheduled_jobs): - if resources["gpu"] == "H100": - continue - + for pending_index, (job, resources) in enumerate(jobs_to_submit): ( _, dataset_key, @@ -345,16 +368,10 @@ def main() -> None: head, seed, ) = job - if pending_index > 0: - print( - f"Waiting {SUBMISSION_INTERVAL_SECONDS / 60} minutes before submitting the next job." - ) - time.sleep(SUBMISSION_INTERVAL_SECONDS) - name = _job_name(dataset_key, level, head, aggregator, objective, seed) print( f"Submitting {name} with {resources} " - f"({pending_index + 1}/{len(pending_jobs)})." + f"({pending_index + 1}/{len(jobs_to_submit)})." ) submit_job( job_name=name, From e86923d6223a5b88ad9efcc9bd3da9db31fbbfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Ki=C5=A1=C5=A1?= <550439@mail.muni.cz> Date: Tue, 6 Oct 2026 18:00:25 +0200 Subject: [PATCH 79/79] feat: refill sweep GPU queues every five minutes --- scripts/ml/run_train_sweep.py | 273 +++++++++++++++++++++------------- tests/test_run_train_sweep.py | 194 ++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 103 deletions(-) create mode 100644 tests/test_run_train_sweep.py diff --git a/scripts/ml/run_train_sweep.py b/scripts/ml/run_train_sweep.py index 2a825e53..6853f994 100644 --- a/scripts/ml/run_train_sweep.py +++ b/scripts/ml/run_train_sweep.py @@ -1,6 +1,9 @@ +"""Refill GPU waiting queues every five minutes until the sweep is submitted.""" + import argparse import itertools import os +import time from collections import deque from collections.abc import Mapping from heapq import heapify, heappop, heappush @@ -40,6 +43,8 @@ "mig-2g.20gb", "mig-1g.10gb", ) +POLL_INTERVAL_SECONDS = 5 * 60 +WAITING_JOB_TARGETS = {gpu: 1 if gpu == "H100" else 2 for gpu in GPUS} BASIC_RESOURCES = {"cpu": 8, "memory": "16Gi"} # GPU rules limit which entries from GPUS a matching job may use. Jobs without @@ -174,10 +179,20 @@ def _eligible_gpus(job: dict[str, object]) -> tuple[str, ...]: def _assign_gpu( - job: dict[str, object], projected_waiting_counts: dict[str, int] -) -> str: - """Choose the least-loaded eligible GPU, using GPUS order for ties.""" + job: dict[str, object], + projected_waiting_counts: dict[str, int], + waiting_targets: Mapping[str, int] | None = None, +) -> str | None: + """Choose an eligible GPU with room in its queue; use GPUS order for ties.""" eligible = _eligible_gpus(job) + if waiting_targets is not None: + eligible = tuple( + gpu + for gpu in eligible + if projected_waiting_counts[gpu] < waiting_targets[gpu] + ) + if not eligible: + return None gpu = min(eligible, key=lambda candidate: projected_waiting_counts[candidate]) projected_waiting_counts[gpu] += 1 return gpu @@ -193,7 +208,9 @@ def _resources_for_job( aggregator: str, head: str, seed: int, -) -> dict[str, Any]: + *, + waiting_targets: Mapping[str, int] | None = None, +) -> dict[str, Any] | None: job = { "dataset_key": dataset_key, "dataset_config": dataset_config, @@ -212,7 +229,10 @@ def _resources_for_job( if matches and not excluded: resources.update(rule["resources"]) - resources["gpu"] = _assign_gpu(job, projected_waiting_counts) + gpu = _assign_gpu(job, projected_waiting_counts, waiting_targets) + if gpu is None: + return None + resources["gpu"] = gpu return resources @@ -244,15 +264,31 @@ def _balanced_job_order( def _jobs_to_submit( - scheduled_jobs: list[ResolvedJob], max_jobs: int | None -) -> list[ResolvedJob]: - """Return the scheduled non-H100 jobs, capped when requested.""" - eligible = [ - resolved_job - for resolved_job in scheduled_jobs - if resolved_job[1]["gpu"] != "H100" - ] - return eligible if max_jobs is None else eligible[:max_jobs] + pending_jobs: list[SweepJob], + waiting_counts: Mapping[str, int], + max_jobs: int | None, +) -> tuple[list[ResolvedJob], int]: + """Fill only vacant waiting slots, assigning jobs afresh on every poll.""" + projected_counts = {gpu: waiting_counts.get(gpu, 0) for gpu in GPUS} + resolved_jobs: list[ResolvedJob] = [] + deferred_count = 0 + for job in pending_jobs: + try: + resources = _resources_for_job( + projected_counts, + *job[1:], + waiting_targets=WAITING_JOB_TARGETS, + ) + except ValueError as error: + if not str(error).startswith("No configured GPU can run job "): + raise + deferred_count += 1 + continue + if resources is not None: + resolved_jobs.append((job, resources)) + if max_jobs is not None and len(resolved_jobs) >= max_jobs: + break + return _balanced_job_order(resolved_jobs, waiting_counts), deferred_count def _existing_mlflow_run_names(expected_names: set[str]) -> set[str]: @@ -298,110 +334,141 @@ def _existing_kubernetes_job_names(expected_names: set[str]) -> set[str]: return existing +def _submit_sweep_job(job: SweepJob, resources: dict[str, Any]) -> None: + ( + _, + dataset_key, + dataset_config, + level, + objective, + experiment, + aggregator, + head, + seed, + ) = job + name = _job_name(dataset_key, level, head, aggregator, objective, seed) + print(f"Submitting {name} with {resources}.") + submit_job( + job_name=name, + username=USERNAME, + image=IMAGE, + cpu=resources["cpu"], + memory=resources["memory"], + gpu=resources["gpu"], + public=False, + script=[ + "git clone https://github.com/rationAI/mammaprint workdir", + "cd workdir", + f"git checkout {GIT_REF}", + f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", + # "export HF_TOKEN=", + "uv sync --frozen", + f""" + uv run -m ml.train +experiment={experiment} \ + data/embedded={dataset_config}/l{level} \ + ml/aggregator={aggregator} \ + ml/head={head} \ + seed={seed} \ + metadata.run_name={name} \ + +logger.tags.data_variant={dataset_config} \ + +logger.tags.level=level_{level} \ + +logger.tags.sweep_seed=seed_{seed} \ + """, + ], + storage=[storage.secure.DATA, storage.secure.PROJECTS], + ) + # kube_jobs catches creation errors internally, so a normal return alone + # does not mean the job was created. + if name not in _existing_kubernetes_job_names({name}): + raise RuntimeError(f"Kubernetes did not confirm submission of {name}") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--max-jobs", type=int, - help="Submit at most this many non-H100 jobs from the automatic sweep order.", + help="Submit at most this many jobs in total, then exit.", + ) + parser.add_argument( + "--poll-interval-seconds", + type=int, + default=POLL_INTERVAL_SECONDS, + help="Seconds between queue refills (default: 300).", ) args = parser.parse_args() if args.max_jobs is not None and args.max_jobs <= 0: parser.error("--max-jobs must be a positive integer") + if args.poll_interval_seconds <= 0: + parser.error("--poll-interval-seconds must be a positive integer") jobs = _sweep_jobs() expected_names = { _job_name(dataset_key, level, head, aggregator, objective, seed) for _, dataset_key, _, level, objective, _, aggregator, head, seed in jobs } - kubernetes_names = _existing_kubernetes_job_names(expected_names) - mlflow_names = _existing_mlflow_run_names(expected_names) - existing_names = mlflow_names | kubernetes_names - pending_jobs = [ - job - for job in jobs - if _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) - not in existing_names - ] - - print( - f"Sweep contains {len(jobs)} jobs: {len(existing_names)} already submitted, " - f"{len(pending_jobs)} pending." - ) - waiting_counts = pending_gpu_counts(USERNAME) - projected_waiting_counts = {gpu: waiting_counts[gpu] for gpu in GPUS} - print(f"GPU waiting counts: {projected_waiting_counts}") - - resolved_jobs = [ - ( - job, - _resources_for_job( - projected_waiting_counts, - job[1], - job[2], - job[3], - job[4], - job[5], - job[6], - job[7], - job[8], - ), - ) - for job in pending_jobs - ] - scheduled_jobs = _balanced_job_order(resolved_jobs, waiting_counts) - jobs_to_submit = _jobs_to_submit(scheduled_jobs, args.max_jobs) - print( - f"Submitting {len(jobs_to_submit)} of {len(pending_jobs)} pending jobs" - + (f" (limit {args.max_jobs})." if args.max_jobs is not None else ".") - ) - - for pending_index, (job, resources) in enumerate(jobs_to_submit): - ( - _, - dataset_key, - dataset_config, - level, - objective, - experiment, - aggregator, - head, - seed, - ) = job - name = _job_name(dataset_key, level, head, aggregator, objective, seed) - print( - f"Submitting {name} with {resources} " - f"({pending_index + 1}/{len(jobs_to_submit)})." - ) - submit_job( - job_name=name, - username=USERNAME, - image=IMAGE, - cpu=resources["cpu"], - memory=resources["memory"], - gpu=resources["gpu"], - public=False, - script=[ - "git clone https://github.com/rationAI/mammaprint workdir", - "cd workdir", - f"git checkout {GIT_REF}", - f"export MLFLOW_TRACKING_URI={POD_MLFLOW_TRACKING_URI}", - # "export HF_TOKEN=", - "uv sync --frozen", - f""" - uv run -m ml.train +experiment={experiment} \ - data/embedded={dataset_config}/l{level} \ - ml/aggregator={aggregator} \ - ml/head={head} \ - seed={seed} \ - metadata.run_name={name} \ - +logger.tags.data_variant={dataset_config} \ - +logger.tags.level=level_{level} \ - +logger.tags.sweep_seed=seed_{seed} \ - """, - ], - storage=[storage.secure.DATA, storage.secure.PROJECTS], - ) + submitted_names: set[str] = set() + print(f"GPU waiting targets: {WAITING_JOB_TARGETS}") + try: + while True: + try: + kubernetes_names = _existing_kubernetes_job_names(expected_names) + mlflow_names = _existing_mlflow_run_names(expected_names) + existing_names = mlflow_names | kubernetes_names | submitted_names + pending_jobs = [ + job + for job in jobs + if _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + not in existing_names + ] + print( + f"Sweep contains {len(jobs)} jobs: " + f"{len(existing_names)} already submitted, " + f"{len(pending_jobs)} left to submit." + ) + if not pending_jobs: + print("All sweep jobs have been submitted.") + return + + waiting_counts = pending_gpu_counts(USERNAME) + print( + "GPU waiting counts: " + f"{ {gpu: waiting_counts.get(gpu, 0) for gpu in GPUS} }" + ) + remaining_limit = ( + None + if args.max_jobs is None + else args.max_jobs - len(submitted_names) + ) + jobs_to_submit, deferred_count = _jobs_to_submit( + pending_jobs, waiting_counts, remaining_limit + ) + print(f"Submitting {len(jobs_to_submit)} job(s) to refill the queues.") + if deferred_count: + print( + f"Deferred {deferred_count} job(s) with no eligible " + "configured GPU." + ) + if deferred_count == len(pending_jobs): + print("No remaining jobs can use the configured GPU pool.") + return + + for job, resources in jobs_to_submit: + _submit_sweep_job(job, resources) + submitted_names.add( + _job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + ) + if args.max_jobs is not None and len(submitted_names) >= args.max_jobs: + print(f"Reached the submission limit ({args.max_jobs}).") + return + except Exception as error: + # Refresh cluster state before retrying an uncertain submission. + print(f"Queue refill failed: {error}. Will retry after the interval.") + + print(f"Sleeping for {args.poll_interval_seconds} seconds.", flush=True) + time.sleep(args.poll_interval_seconds) + except KeyboardInterrupt: + print("\nSweep queue refiller stopped.") if __name__ == "__main__": diff --git a/tests/test_run_train_sweep.py b/tests/test_run_train_sweep.py new file mode 100644 index 00000000..05f25fbf --- /dev/null +++ b/tests/test_run_train_sweep.py @@ -0,0 +1,194 @@ +import io +import unittest +from collections import Counter +from contextlib import redirect_stdout +from unittest.mock import patch + +from scripts.ml import run_train_sweep as sweep + + +def job_name(job: sweep.SweepJob) -> str: + return sweep._job_name(job[1], job[3], job[7], job[6], job[4], job[8]) + + +class QueueSelectionTests(unittest.TestCase): + def setUp(self) -> None: + self.jobs = sweep._sweep_jobs() + + def test_empty_queues_fill_to_each_target(self) -> None: + selected, deferred = sweep._jobs_to_submit(self.jobs, {}, None) + self.assertEqual(deferred, 0) + self.assertEqual( + Counter(resources["gpu"] for _, resources in selected), + {"A40": 2, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + ) + self.assertEqual(len({job_name(job) for job, _ in selected}), 7) + + def test_existing_queues_only_fill_deficits(self) -> None: + waiting = {"A40": 1, "H100": 3, "mig-2g.20gb": 2, "mig-1g.10gb": 0} + selected, _ = sweep._jobs_to_submit(self.jobs, waiting, None) + self.assertEqual( + Counter(resources["gpu"] for _, resources in selected), + {"A40": 1, "mig-1g.10gb": 2}, + ) + self.assertEqual(waiting["A40"], 1) + + def test_full_queues_submit_nothing(self) -> None: + selected, _ = sweep._jobs_to_submit(self.jobs, sweep.WAITING_JOB_TARGETS, None) + self.assertEqual(selected, []) + + def test_restricted_jobs_wait_for_h100_capacity(self) -> None: + restricted = [ + job for job in self.jobs if job[3] in (2, 3) and job[6] == "transformer" + ] + selected, _ = sweep._jobs_to_submit(restricted, {"H100": 1}, None) + self.assertEqual(selected, []) + selected, _ = sweep._jobs_to_submit(restricted, {}, None) + self.assertEqual(len(selected), 1) + self.assertEqual(selected[0][1]["gpu"], "H100") + self.assertEqual(selected[0][1]["memory"], "56Gi") + + def test_level_four_transformers_do_not_use_mig10(self) -> None: + restricted = [ + job for job in self.jobs if job[3] == 4 and job[6] == "transformer" + ] + selected, _ = sweep._jobs_to_submit( + restricted, {"A40": 2, "H100": 1, "mig-2g.20gb": 2}, None + ) + self.assertEqual(selected, []) + + def test_unconfigured_h100_jobs_are_deferred(self) -> None: + restricted = [ + job for job in self.jobs if job[3] in (2, 3) and job[6] == "transformer" + ] + with patch.object(sweep, "GPUS", ("A40",)): + selected, deferred = sweep._jobs_to_submit(restricted, {}, None) + self.assertEqual(selected, []) + self.assertEqual(deferred, len(restricted)) + + def test_submission_limit_caps_the_refill(self) -> None: + selected, _ = sweep._jobs_to_submit(self.jobs, {}, 3) + self.assertEqual(len(selected), 3) + + def test_resource_and_gpu_policies_for_entire_grid(self) -> None: + self.assertEqual(len(self.jobs), 11_520) + counts = dict.fromkeys(sweep.GPUS, 0) + for job in self.jobs: + resources = sweep._resources_for_job(counts, *job[1:]) + self.assertIsNotNone(resources) + assert resources is not None + self.assertEqual(resources["cpu"], 8) + if job[3] in (2, 3): + if job[6] == "transformer": + self.assertEqual(resources["gpu"], "H100") + self.assertEqual(resources["memory"], "56Gi") + elif job[6] == "attention": + self.assertEqual(resources["memory"], "40Gi") + else: + self.assertEqual(resources["memory"], "20Gi") + else: + self.assertEqual(resources["memory"], "16Gi") + if job[3] == 4 and job[6] == "transformer": + self.assertNotEqual(resources["gpu"], "mig-1g.10gb") + + +class RefillLoopTests(unittest.TestCase): + def setUp(self) -> None: + self.jobs = sweep._sweep_jobs()[:3] + self.stdout = io.StringIO() + self.enterContext(redirect_stdout(self.stdout)) + self.jobs_mock = self.enterContext( + patch.object(sweep, "_sweep_jobs", return_value=self.jobs) + ) + self.kubernetes = self.enterContext( + patch.object(sweep, "_existing_kubernetes_job_names", return_value=set()) + ) + self.mlflow = self.enterContext( + patch.object(sweep, "_existing_mlflow_run_names", return_value=set()) + ) + self.waiting = self.enterContext(patch.object(sweep, "pending_gpu_counts")) + self.submit = self.enterContext(patch.object(sweep, "_submit_sweep_job")) + self.sleep = self.enterContext(patch.object(sweep.time, "sleep")) + + def run_main(self, *arguments: str) -> None: + with patch("sys.argv", ["run_train_sweep.py", *arguments]): + sweep.main() + + def test_refills_again_and_avoids_duplicates_before_mlflow_appears(self) -> None: + self.waiting.side_effect = [ + {"A40": 1, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + {"A40": 0, "H100": 1, "mig-2g.20gb": 2, "mig-1g.10gb": 2}, + ] + self.run_main("--max-jobs", "3") + submitted = [call.args[0] for call in self.submit.call_args_list] + self.assertEqual(submitted, self.jobs) + self.assertEqual(self.kubernetes.call_count, 2) + self.assertEqual(self.mlflow.call_count, 2) + self.sleep.assert_called_once_with(300) + + def test_refreshes_external_submissions_and_stops_when_complete(self) -> None: + self.waiting.return_value = sweep.WAITING_JOB_TARGETS.copy() + self.kubernetes.side_effect = [set(), {job_name(job) for job in self.jobs}] + self.run_main("--poll-interval-seconds", "17") + self.submit.assert_not_called() + self.sleep.assert_called_once_with(17) + + def test_lookup_failure_waits_before_retrying(self) -> None: + self.mlflow.side_effect = [RuntimeError("MLflow unavailable"), set()] + self.waiting.return_value = {} + self.run_main("--max-jobs", "1") + self.assertEqual(self.submit.call_count, 1) + self.sleep.assert_called_once_with(300) + self.assertIn("Queue refill failed", self.stdout.getvalue()) + + def test_partial_submission_failure_refreshes_before_retrying(self) -> None: + self.waiting.return_value = {} + self.submit.side_effect = [None, RuntimeError("connection lost"), None] + # The failed call actually created the job; the next poll must skip it. + self.kubernetes.side_effect = [set(), {job_name(self.jobs[1])}] + self.run_main("--max-jobs", "2") + self.assertEqual( + [call.args[0] for call in self.submit.call_args_list], self.jobs + ) + self.sleep.assert_called_once_with(300) + + def test_interrupt_stops_cleanly(self) -> None: + self.waiting.return_value = sweep.WAITING_JOB_TARGETS.copy() + self.sleep.side_effect = KeyboardInterrupt + self.run_main() + self.submit.assert_not_called() + self.assertIn("Sweep queue refiller stopped", self.stdout.getvalue()) + + +class SubmissionConfirmationTests(unittest.TestCase): + def test_unconfirmed_submission_raises(self) -> None: + job = sweep._sweep_jobs()[0] + with ( + patch.object(sweep, "submit_job") as submit, + patch.object(sweep, "_existing_kubernetes_job_names", return_value=set()), + redirect_stdout(io.StringIO()), + self.assertRaisesRegex(RuntimeError, "did not confirm"), + ): + sweep._submit_sweep_job(job, {"gpu": "A40", "cpu": 8, "memory": "20Gi"}) + submit.assert_called_once() + + def test_confirmed_submission_preserves_training_parameters(self) -> None: + job = sweep._sweep_jobs()[0] + with ( + patch.object(sweep, "submit_job") as submit, + patch.object( + sweep, "_existing_kubernetes_job_names", return_value={job_name(job)} + ) as lookup, + redirect_stdout(io.StringIO()), + ): + sweep._submit_sweep_job(job, {"gpu": "A40", "cpu": 8, "memory": "20Gi"}) + lookup.assert_called_once_with({job_name(job)}) + self.assertEqual(submit.call_args.kwargs["job_name"], job_name(job)) + self.assertEqual(submit.call_args.kwargs["gpu"], "A40") + self.assertIn( + "data/embedded=tissue_only/l2", submit.call_args.kwargs["script"][-1] + ) + + +if __name__ == "__main__": + unittest.main()