Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/examples/basic-functionalities/group-aware-splitting.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ from ratiopath.model_selection import train_test_split

X = np.array(["tile_1", "tile_2", "tile_3", "tile_4", "tile_5", "tile_6"])
y = np.array([0, 0, 1, 1, 0, 1])
groups = np.array(["patient_a", "patient_a", "patient_b", "patient_b", "patient_c", "patient_c"])
groups = np.array(
["patient_a", "patient_a", "patient_b", "patient_b", "patient_c", "patient_c"]
)

X_train, X_test, y_train, y_test = train_test_split(
X,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,9 @@ def expand_tiles(row: dict[str, Any]) -> list[dict[str, Any]]:


def build_tiles(split_slides):
tiles = split_slides.flat_map(expand_tiles).repartition(target_num_rows_per_block=128)
tiles = split_slides.flat_map(expand_tiles).repartition(
target_num_rows_per_block=128
)
tiles = tiles.with_column(
"tile",
read_slide_tiles(
Expand Down
47 changes: 27 additions & 20 deletions docs/learn/get-started/quick-start/annotations.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@ For each tile, define its region of interest (ROI) as a polygon. Then, calculate
```python
from shapely import Polygon

roi = Polygon([
(0, 0),
(row["tile_extent_x"], 0),
(row["tile_extent_x"], row["tile_extent_y"]),
(0, row["tile_extent_y"]),
])
roi = Polygon(
[
(0, 0),
(row["tile_extent_x"], 0),
(row["tile_extent_x"], row["tile_extent_y"]),
(0, row["tile_extent_y"]),
]
)
```

---
Expand All @@ -54,26 +56,31 @@ Use the `tile_annotations` function to compute which annotation polygons overlap
```python
from ratiopath.tiling import tile_annotations


def tiling_with_annotations(row: dict[str, Any]) -> list[dict[str, Any]]:
annotation_path = row["path"].replace(".mrxs", ".xml")
parser = ASAPParser(annotation_path)
annotations = list(parser.get_polygons(name="...", part_of_group="..."))

roi = Polygon([
(0, 0),
(row["tile_extent_x"], 0),
(row["tile_extent_x"], row["tile_extent_y"]),
(0, row["tile_extent_y"]),
])

coordinates = np.array(list(
grid_tiles(
slide_extent=(row["extent_x"], row["extent_y"]),
tile_extent=(row["tile_extent_x"], row["tile_extent_y"]),
stride=(row["stride_x"], row["stride_y"]),
last="keep",
roi = Polygon(
[
(0, 0),
(row["tile_extent_x"], 0),
(row["tile_extent_x"], row["tile_extent_y"]),
(0, row["tile_extent_y"]),
]
)

coordinates = np.array(
list(
grid_tiles(
slide_extent=(row["extent_x"], row["extent_y"]),
tile_extent=(row["tile_extent_x"], row["tile_extent_y"]),
stride=(row["stride_x"], row["stride_y"]),
last="keep",
)
)
))
)
return [
{
"tile_x": coordinates[i, 0],
Expand Down
7 changes: 5 additions & 2 deletions docs/learn/get-started/quick-start/overlays.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,14 @@ You must augment the tile dataset to include a column with the file path to the
slides = ...
tiles = ...


def add_overlay_path(batch: dict) -> dict:
"""Adds the overlay path for each tile in the batch."""
# Example: Replace the WSI extension with the mask file extension
batch["tissue_mask_path"] = batch["path"].str.replace(".mrxs", "_tissue_mask.tiff")
return batch


tiles = tiles.map_batches(add_overlay_path)
```

Expand All @@ -54,7 +56,7 @@ from shapely.geometry import box


# Assuming the tile size is 512x512 pixels
roi = box(128, 128, 384, 384) # A box from (128,128) to (384,384)
roi = box(128, 128, 384, 384) # A box from (128,128) to (384,384)
```

### 2.3 Extracting Overlay Patches
Expand Down Expand Up @@ -117,9 +119,10 @@ You can easily post-process this to extract coverage of a specific class (e.g.,
def extract_foreground_coverage(tile: dict) -> dict:
"""Extracts the foreground coverage (value 255) from the overlap dictionary."""
# Use .get(255, 0.0) to safely retrieve the value, defaulting to 0.0 if not present
tile["tissue_coverage"] = tile["tissue_mask_overlap"].get('255', 0.0)
tile["tissue_coverage"] = tile["tissue_mask_overlap"].get("255", 0.0)
return tile


tiles_with_tissue_coverage = tiles_with_overlap.map(extract_foreground_coverage).filter(
lambda tile: tile["tissue_coverage"] >= 0.5 # Keep tiles with at least 50% tissue
)
Expand Down
22 changes: 9 additions & 13 deletions docs/learn/get-started/quick-start/tiled-dataset.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ class TileDataset(Dataset):
# Read the region dynamically from the slide
with OpenSlide(self.slide_path) as slide:
return slide.read_tile(x, y, self.extent_x, self.extent_y, self.level)

```

Notice that our `TileDataset` expects a `tiles` object containing the metadata. If we pass a standard Pandas DataFrame here, our RAM will quickly max out as we scale up to thousands of slides.
Expand Down Expand Up @@ -102,9 +101,13 @@ class SlideDataset(ConcatDataset[TileDataset]):
tiles_parquet_path: str,
) -> None:
# 'train' is the default split name for Hugging Face datasets.
self.slides_dataset = load_dataset("parquet", data_files=slides_parquet_path, split="train")
self.slides_dataset = load_dataset(
"parquet", data_files=slides_parquet_path, split="train"
)
# Sort by slide_id for much faster filtering
self.tiles_dataset = load_dataset("parquet", data_files=tiles_parquet_path, split="train").sort("slide_id")
self.tiles_dataset = load_dataset(
"parquet", data_files=tiles_parquet_path, split="train"
).sort("slide_id")

self._slide_id_to_indices = self._build_tile_index(self.tiles_dataset)

Expand All @@ -121,14 +124,13 @@ class SlideDataset(ConcatDataset[TileDataset]):

super().__init__(datasets)


@staticmethod
def _build_tile_index(tiles: HFDataset) -> dict[str, range]:
"""Creates a fast lookup table for slide indices.

This function builds a mapping from `slide_id` to the range of indices in the
`tiles` dataset that correspond to that slide. It assumes the dataset is sorted.

Args:
tiles: A dataset containing a `slide_id` column, sorted by `slide_id`.

Expand Down Expand Up @@ -162,12 +164,11 @@ class SlideDataset(ConcatDataset[TileDataset]):
def filter_tiles_by_slide(self, slide_id: str) -> HFDataset:
"""Returns a view of the dataset using a slice or indices.

This uses the precomputed `_slide_id_to_indices` mapping to efficiently
This uses the precomputed `_slide_id_to_indices` mapping to efficiently
retrieve the relevant tiles without copying data.
"""
tile_range = self._slide_id_to_indices.get(slide_id, range(0))
return self.tiles_dataset.select(tile_range)

```

### Using the Dataset
Expand All @@ -184,10 +185,5 @@ full_dataset = SlideDataset(
)

# Load with multiprocessing for high throughput
dataloader = DataLoader(
full_dataset,
batch_size=64,
shuffle=True,
num_workers=8
)
dataloader = DataLoader(full_dataset, batch_size=64, shuffle=True, num_workers=8)
```
2 changes: 2 additions & 0 deletions docs/learn/get-started/quick-start/tiling.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ First, define a `tiling` function. This function takes a slide row (which contai
from typing import Any
from ratiopath.tiling import grid_tiles


def tiling(row: dict[str, Any]) -> list[dict[str, Any]]:
return [
{
Expand All @@ -204,6 +205,7 @@ def tiling(row: dict[str, Any]) -> list[dict[str, Any]]:
)
]


tiles = slides.flat_map(tiling, num_cpus=0.2, memory=128 * 1024**2)
```

Expand Down
4 changes: 3 additions & 1 deletion docs/reference/masks/mask_builders.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,9 @@ mask_builder = MaskBuilder(
)

# Process tiles
for tiles, xs, ys in generate_tiles_from_slide(slide, LEVEL, tile_extents, tile_strides):
for tiles, xs, ys in generate_tiles_from_slide(
slide, LEVEL, tile_extents, tile_strides
):
features = model.predict(tiles) # features shape: (B, 1)
coords_batch = np.stack([ys, xs], axis=1) # shape: (B, 2)
mask_builder.update_batch(features, coords_batch)
Expand Down
4 changes: 2 additions & 2 deletions ratiopath/masks/mask_builders/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,13 @@ def close(self) -> None:
if hasattr(self, "_mmap") and self._mmap is not None:
try:
self._mmap.close()
except Exception as e:
except Exception as e: # noqa: BLE001 - best-effort cleanup, also runs from __del__
logger.warning(f"Failed to close memmap: {e}")

if self._tempfile is not None:
try:
Path(self._tempfile).unlink(missing_ok=True)
except Exception as e:
except Exception as e: # noqa: BLE001 - best-effort cleanup, also runs from __del__
logger.warning(f"Failed to delete memmap file {self._tempfile}: {e}")
self._tempfile = None

Expand Down
16 changes: 8 additions & 8 deletions ratiopath/model_selection/split.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ def __init__(
self,
n_splits: Int = 5,
*,
test_size: None | Float = None,
train_size: None | Float = None,
random_state: np.random.RandomState | None | Int = None,
test_size: Float | None = None,
train_size: Float | None = None,
random_state: np.random.RandomState | Int | None = None,
) -> None:
super().__init__(
n_splits=n_splits,
Expand Down Expand Up @@ -175,12 +175,12 @@ def split(
)
def train_test_split(
*arrays: Any,
test_size: None | Float = None,
train_size: None | Float = None,
random_state: np.random.RandomState | None | Int = None,
test_size: Float | None = None,
train_size: Float | None = None,
random_state: np.random.RandomState | Int | None = None,
shuffle: bool = True,
stratify: None | ArrayLike = None,
groups: None | ArrayLike = None,
stratify: ArrayLike | None = None,
groups: ArrayLike | None = None,
) -> list:
"""Split arrays or matrices into random train and test subsets.

Expand Down
2 changes: 1 addition & 1 deletion ratiopath/torch/data/openslide_tile_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def __init__(
path: str | Path,
level: int | str | None = None,
resolution: float | tuple[float, float] | None = None,
background: None | tuple[int, int, int] = (255, 255, 255),
background: tuple[int, int, int] | None = (255, 255, 255),
) -> None:
assert (level is None) != (resolution is None), (
"Either level or resolution must be provided"
Expand Down
Loading