diff --git a/docs/examples/basic-functionalities/group-aware-splitting.md b/docs/examples/basic-functionalities/group-aware-splitting.md index 6db2eda2..118b9081 100644 --- a/docs/examples/basic-functionalities/group-aware-splitting.md +++ b/docs/examples/basic-functionalities/group-aware-splitting.md @@ -26,7 +26,9 @@ from ratiopath.model_selection import train_test_split X = np.array(["tile_1", "tile_2", "tile_3", "tile_4", "tile_5", "tile_6"]) y = np.array([0, 0, 1, 1, 0, 1]) -groups = np.array(["patient_a", "patient_a", "patient_b", "patient_b", "patient_c", "patient_c"]) +groups = np.array( + ["patient_a", "patient_a", "patient_b", "patient_b", "patient_c", "patient_c"] +) X_train, X_test, y_train, y_test = train_test_split( X, diff --git a/docs/examples/pipelines/prepare-a-training-dataset-with-leakage-safe-splits.md b/docs/examples/pipelines/prepare-a-training-dataset-with-leakage-safe-splits.md index 80d8cee1..677fb3f6 100644 --- a/docs/examples/pipelines/prepare-a-training-dataset-with-leakage-safe-splits.md +++ b/docs/examples/pipelines/prepare-a-training-dataset-with-leakage-safe-splits.md @@ -64,7 +64,9 @@ def expand_tiles(row: dict[str, Any]) -> list[dict[str, Any]]: def build_tiles(split_slides): - tiles = split_slides.flat_map(expand_tiles).repartition(target_num_rows_per_block=128) + tiles = split_slides.flat_map(expand_tiles).repartition( + target_num_rows_per_block=128 + ) tiles = tiles.with_column( "tile", read_slide_tiles( diff --git a/docs/learn/get-started/quick-start/annotations.md b/docs/learn/get-started/quick-start/annotations.md index be161228..2db30af1 100644 --- a/docs/learn/get-started/quick-start/annotations.md +++ b/docs/learn/get-started/quick-start/annotations.md @@ -37,12 +37,14 @@ For each tile, define its region of interest (ROI) as a polygon. Then, calculate ```python from shapely import Polygon -roi = Polygon([ - (0, 0), - (row["tile_extent_x"], 0), - (row["tile_extent_x"], row["tile_extent_y"]), - (0, row["tile_extent_y"]), -]) +roi = Polygon( + [ + (0, 0), + (row["tile_extent_x"], 0), + (row["tile_extent_x"], row["tile_extent_y"]), + (0, row["tile_extent_y"]), + ] +) ``` --- @@ -54,26 +56,31 @@ Use the `tile_annotations` function to compute which annotation polygons overlap ```python from ratiopath.tiling import tile_annotations + def tiling_with_annotations(row: dict[str, Any]) -> list[dict[str, Any]]: annotation_path = row["path"].replace(".mrxs", ".xml") parser = ASAPParser(annotation_path) annotations = list(parser.get_polygons(name="...", part_of_group="...")) - roi = Polygon([ - (0, 0), - (row["tile_extent_x"], 0), - (row["tile_extent_x"], row["tile_extent_y"]), - (0, row["tile_extent_y"]), - ]) - - coordinates = np.array(list( - grid_tiles( - slide_extent=(row["extent_x"], row["extent_y"]), - tile_extent=(row["tile_extent_x"], row["tile_extent_y"]), - stride=(row["stride_x"], row["stride_y"]), - last="keep", + roi = Polygon( + [ + (0, 0), + (row["tile_extent_x"], 0), + (row["tile_extent_x"], row["tile_extent_y"]), + (0, row["tile_extent_y"]), + ] + ) + + coordinates = np.array( + list( + grid_tiles( + slide_extent=(row["extent_x"], row["extent_y"]), + tile_extent=(row["tile_extent_x"], row["tile_extent_y"]), + stride=(row["stride_x"], row["stride_y"]), + last="keep", + ) ) - )) + ) return [ { "tile_x": coordinates[i, 0], diff --git a/docs/learn/get-started/quick-start/overlays.md b/docs/learn/get-started/quick-start/overlays.md index 56fd7227..95bad02c 100644 --- a/docs/learn/get-started/quick-start/overlays.md +++ b/docs/learn/get-started/quick-start/overlays.md @@ -36,12 +36,14 @@ You must augment the tile dataset to include a column with the file path to the slides = ... tiles = ... + def add_overlay_path(batch: dict) -> dict: """Adds the overlay path for each tile in the batch.""" # Example: Replace the WSI extension with the mask file extension batch["tissue_mask_path"] = batch["path"].str.replace(".mrxs", "_tissue_mask.tiff") return batch + tiles = tiles.map_batches(add_overlay_path) ``` @@ -54,7 +56,7 @@ from shapely.geometry import box # Assuming the tile size is 512x512 pixels -roi = box(128, 128, 384, 384) # A box from (128,128) to (384,384) +roi = box(128, 128, 384, 384) # A box from (128,128) to (384,384) ``` ### 2.3 Extracting Overlay Patches @@ -117,9 +119,10 @@ You can easily post-process this to extract coverage of a specific class (e.g., def extract_foreground_coverage(tile: dict) -> dict: """Extracts the foreground coverage (value 255) from the overlap dictionary.""" # Use .get(255, 0.0) to safely retrieve the value, defaulting to 0.0 if not present - tile["tissue_coverage"] = tile["tissue_mask_overlap"].get('255', 0.0) + tile["tissue_coverage"] = tile["tissue_mask_overlap"].get("255", 0.0) return tile + tiles_with_tissue_coverage = tiles_with_overlap.map(extract_foreground_coverage).filter( lambda tile: tile["tissue_coverage"] >= 0.5 # Keep tiles with at least 50% tissue ) diff --git a/docs/learn/get-started/quick-start/tiled-dataset.md b/docs/learn/get-started/quick-start/tiled-dataset.md index 5e7a5f7a..2cc6c7dd 100644 --- a/docs/learn/get-started/quick-start/tiled-dataset.md +++ b/docs/learn/get-started/quick-start/tiled-dataset.md @@ -59,7 +59,6 @@ class TileDataset(Dataset): # Read the region dynamically from the slide with OpenSlide(self.slide_path) as slide: return slide.read_tile(x, y, self.extent_x, self.extent_y, self.level) - ``` Notice that our `TileDataset` expects a `tiles` object containing the metadata. If we pass a standard Pandas DataFrame here, our RAM will quickly max out as we scale up to thousands of slides. @@ -102,9 +101,13 @@ class SlideDataset(ConcatDataset[TileDataset]): tiles_parquet_path: str, ) -> None: # 'train' is the default split name for Hugging Face datasets. - self.slides_dataset = load_dataset("parquet", data_files=slides_parquet_path, split="train") + self.slides_dataset = load_dataset( + "parquet", data_files=slides_parquet_path, split="train" + ) # Sort by slide_id for much faster filtering - self.tiles_dataset = load_dataset("parquet", data_files=tiles_parquet_path, split="train").sort("slide_id") + self.tiles_dataset = load_dataset( + "parquet", data_files=tiles_parquet_path, split="train" + ).sort("slide_id") self._slide_id_to_indices = self._build_tile_index(self.tiles_dataset) @@ -121,14 +124,13 @@ class SlideDataset(ConcatDataset[TileDataset]): super().__init__(datasets) - @staticmethod def _build_tile_index(tiles: HFDataset) -> dict[str, range]: """Creates a fast lookup table for slide indices. This function builds a mapping from `slide_id` to the range of indices in the `tiles` dataset that correspond to that slide. It assumes the dataset is sorted. - + Args: tiles: A dataset containing a `slide_id` column, sorted by `slide_id`. @@ -162,12 +164,11 @@ class SlideDataset(ConcatDataset[TileDataset]): def filter_tiles_by_slide(self, slide_id: str) -> HFDataset: """Returns a view of the dataset using a slice or indices. - This uses the precomputed `_slide_id_to_indices` mapping to efficiently + This uses the precomputed `_slide_id_to_indices` mapping to efficiently retrieve the relevant tiles without copying data. """ tile_range = self._slide_id_to_indices.get(slide_id, range(0)) return self.tiles_dataset.select(tile_range) - ``` ### Using the Dataset @@ -184,10 +185,5 @@ full_dataset = SlideDataset( ) # Load with multiprocessing for high throughput -dataloader = DataLoader( - full_dataset, - batch_size=64, - shuffle=True, - num_workers=8 -) +dataloader = DataLoader(full_dataset, batch_size=64, shuffle=True, num_workers=8) ``` \ No newline at end of file diff --git a/docs/learn/get-started/quick-start/tiling.md b/docs/learn/get-started/quick-start/tiling.md index 446e0c86..b0a13b87 100644 --- a/docs/learn/get-started/quick-start/tiling.md +++ b/docs/learn/get-started/quick-start/tiling.md @@ -185,6 +185,7 @@ First, define a `tiling` function. This function takes a slide row (which contai from typing import Any from ratiopath.tiling import grid_tiles + def tiling(row: dict[str, Any]) -> list[dict[str, Any]]: return [ { @@ -204,6 +205,7 @@ def tiling(row: dict[str, Any]) -> list[dict[str, Any]]: ) ] + tiles = slides.flat_map(tiling, num_cpus=0.2, memory=128 * 1024**2) ``` diff --git a/docs/reference/masks/mask_builders.md b/docs/reference/masks/mask_builders.md index bf89ce6e..f34bd9af 100644 --- a/docs/reference/masks/mask_builders.md +++ b/docs/reference/masks/mask_builders.md @@ -78,7 +78,9 @@ mask_builder = MaskBuilder( ) # Process tiles -for tiles, xs, ys in generate_tiles_from_slide(slide, LEVEL, tile_extents, tile_strides): +for tiles, xs, ys in generate_tiles_from_slide( + slide, LEVEL, tile_extents, tile_strides +): features = model.predict(tiles) # features shape: (B, 1) coords_batch = np.stack([ys, xs], axis=1) # shape: (B, 2) mask_builder.update_batch(features, coords_batch) diff --git a/ratiopath/masks/mask_builders/storage.py b/ratiopath/masks/mask_builders/storage.py index b452cd5d..81bdba86 100644 --- a/ratiopath/masks/mask_builders/storage.py +++ b/ratiopath/masks/mask_builders/storage.py @@ -73,13 +73,13 @@ def close(self) -> None: if hasattr(self, "_mmap") and self._mmap is not None: try: self._mmap.close() - except Exception as e: + except Exception as e: # noqa: BLE001 - best-effort cleanup, also runs from __del__ logger.warning(f"Failed to close memmap: {e}") if self._tempfile is not None: try: Path(self._tempfile).unlink(missing_ok=True) - except Exception as e: + except Exception as e: # noqa: BLE001 - best-effort cleanup, also runs from __del__ logger.warning(f"Failed to delete memmap file {self._tempfile}: {e}") self._tempfile = None diff --git a/ratiopath/model_selection/split.py b/ratiopath/model_selection/split.py index 0774e0e5..c66eae70 100644 --- a/ratiopath/model_selection/split.py +++ b/ratiopath/model_selection/split.py @@ -76,9 +76,9 @@ def __init__( self, n_splits: Int = 5, *, - test_size: None | Float = None, - train_size: None | Float = None, - random_state: np.random.RandomState | None | Int = None, + test_size: Float | None = None, + train_size: Float | None = None, + random_state: np.random.RandomState | Int | None = None, ) -> None: super().__init__( n_splits=n_splits, @@ -175,12 +175,12 @@ def split( ) def train_test_split( *arrays: Any, - test_size: None | Float = None, - train_size: None | Float = None, - random_state: np.random.RandomState | None | Int = None, + test_size: Float | None = None, + train_size: Float | None = None, + random_state: np.random.RandomState | Int | None = None, shuffle: bool = True, - stratify: None | ArrayLike = None, - groups: None | ArrayLike = None, + stratify: ArrayLike | None = None, + groups: ArrayLike | None = None, ) -> list: """Split arrays or matrices into random train and test subsets. diff --git a/ratiopath/torch/data/openslide_tile_reader.py b/ratiopath/torch/data/openslide_tile_reader.py index dd3b6ef1..e70ae552 100644 --- a/ratiopath/torch/data/openslide_tile_reader.py +++ b/ratiopath/torch/data/openslide_tile_reader.py @@ -17,7 +17,7 @@ def __init__( path: str | Path, level: int | str | None = None, resolution: float | tuple[float, float] | None = None, - background: None | tuple[int, int, int] = (255, 255, 255), + background: tuple[int, int, int] | None = (255, 255, 255), ) -> None: assert (level is None) != (resolution is None), ( "Either level or resolution must be provided"