Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
``read_api.read_mcap`` constructs it when ``DataContext.use_datasource_v2`` is
set. Listing reads each file's summary, drops files that cannot match the
selection, and emits one row per chunk (``MCAPSummaryIndexer``), weighted by
the memory reading it is estimated to need. ``OnlineBinPacker`` groups the
chunks into read tasks of about ``RAY_DATA_MCAP_BIN_PACKING_BYTES`` of that
estimate (128 MiB by default). When the read keeps order, ``OrderedFileGrouper``
first joins each file's chunks so that one task reads the file (``mcap_file_cuts``).
``TaskSummaryPartitioner`` adds to each task the summary records it needs.
the memory reading it is estimated to need. Read tasks hold about
``RAY_DATA_MCAP_BIN_PACKING_BYTES`` of that estimate (128 MiB by default).
``OrderedFileCutter`` cuts a large file into tasks that leave in order, and
``OnlineBinPacker`` packs small files whole into shared tasks
(``mcap_file_cuts``). ``TaskSummaryPartitioner`` adds to each task the summary
records it needs.
``MCAPScanner`` holds the optimizer's pushdowns and builds the ``MCAPReader``,
which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type``
(``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes.
Expand Down Expand Up @@ -38,7 +39,7 @@
infer_data_type,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_file_cuts import (
OrderedFileGrouper,
OrderedFileCutter,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_file_indexer import (
MCAPSummaryIndexer,
Expand Down Expand Up @@ -175,9 +176,10 @@ def _get_file_indexer(self) -> FileIndexer:

def get_file_partitioner(self, **kwargs):
# Each listing row is one chunk weighted by its estimated read memory,
# so chunks are packed into read tasks by that estimate. When the read
# keeps order, each file is kept in one task, and files keep their
# listing order (``mcap_file_cuts``).
# so read tasks are sized by that estimate. A large file is cut into
# tasks that leave in order, and small files are packed whole. When the
# read keeps order, files also keep their listing order
# (``mcap_file_cuts`` says when a file is read whole).
#
# Packing is per listing shard, not global, so listing can run as many
# parallel tasks. Each chunk still lands in exactly one read task; the
Expand All @@ -191,9 +193,12 @@ def get_file_partitioner(self, **kwargs):
max_shared_open_bins=_BIN_PACKING_MAX_SHARED_OPEN_BINS,
requires_global_input=False,
)
return TaskSummaryPartitioner(
OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES)
cutter = OrderedFileCutter(
packer,
budget=_BIN_PACKING_BYTES,
log_time_order=self._log_time_order,
)
return TaskSummaryPartitioner(cutter)

@property
@override
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,43 @@
"""Read-task grouping that keeps each MCAP file in order.
"""Read-task cuts that keep each MCAP file in order.

The listing hands the partitioner all of a file's chunk rows in one manifest
(``MCAPSummaryIndexer``). ``OnlineBinPacker`` packs those rows chunk by chunk,
so a file can be split across tasks, and the tasks can finish or be sent in any
order.
(``MCAPSummaryIndexer``). ``OrderedFileCutter`` decides where a file may be cut
before ``OnlineBinPacker`` packs anything:

- A light file, whose rows weigh no more than one read task's budget, goes to
the packer as one run of all its chunks, so it is never split.
- A heavy file is cut here, and its tasks leave at once, in order. A task
closes before the chunk that would take it past the budget.
- A file the cutter cannot place in time goes to the packer unchanged. Those
are a file listed whole and an unreadable file, whose rows carry no
``ChunkIndex`` record.

When the read keeps its blocks in order
(``DataContext.execution_options.preserve_order``), ``OrderedFileGrouper``
packs whole files into tasks itself, in listing order, so files come out in the
order they were listed. Each file is read by one task, which merges its chunks
by log time (``mcap_chunks``). A file heavier than a read task's budget then
gets a task of its own and holds that much memory, as the legacy reader does.
Otherwise the rows go to the packer as listed.
(``DataContext.execution_options.preserve_order``), the cutter packs light
files itself instead of the packer, in listing order, and a heavy file's tasks
leave only after the files listed before it. Files then come out in the order
they were listed.

With ``log_time_order``, a heavy file's chunks are taken in order of their
first log time. When the read also keeps its blocks in order, a heavy file is
not cut: it is packed whole like a light file and gets a task of its own. That
task merges the file's chunks by log time (``mcap_chunks``), so the file comes
out in log-time order from start to end.

Without ``log_time_order``, chunks are taken in file order.
"""

from collections import deque
from typing import Deque, Dict, Iterator, List
from dataclasses import dataclass
from typing import Deque, Dict, Iterator, List, Optional, Sequence

import pyarrow as pa

from ray.data._internal.datasource_v2.common.online_bin_packer import _bin_items
from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import (
CHUNK_INDEX_COLUMN,
chunk_start_time,
)
from ray.data._internal.datasource_v2.interfaces.file_manifest import (
FileChunk,
FileManifest,
Expand All @@ -27,28 +47,59 @@
)


class OrderedFileGrouper(FilePartitioner):
"""Keeps each file in one read task, in listing order, when the read keeps
order.
@dataclass(frozen=True)
class _Chunk:
"""One chunk row of a file: its listing run and its first message's log time."""

run: FileChunk
start_time: int

@property
def offset(self) -> int:
(offset,) = self.run.unit_ids
return offset

``packer`` is an ``OnlineBinPacker`` with tasks of ``budget`` bytes, which
packs the rows of a read that does not keep order. See the module docstring.

@dataclass(frozen=True)
class _FileRows:
"""One file's listing rows. ``chunks`` is empty when they cannot be placed in time."""

path: str
size: int
rows: FileManifest
chunks: List[_Chunk]


class OrderedFileCutter(FilePartitioner):
"""Cuts heavy files into read tasks and packs light files whole.

``packer`` is an ``OnlineBinPacker`` with bins of ``budget`` bytes. It packs
light files and whole-file rows of a read that does not keep order, and
heavy files' tasks bypass it. See the module docstring for when a heavy
file is cut.
"""

def __init__(self, packer: FilePartitioner, *, budget: int):
def __init__(self, packer: FilePartitioner, *, budget: int, log_time_order: bool):
self._packer = packer
self._budget = budget
self._log_time_order = log_time_order
# Whether the read keeps its blocks in order, and whether it reads each
# file whole to keep log-time order; set per input from the read.
self._keeps_order = False
self._whole_files = False
# Heavy files' tasks, and every file when the read keeps order, in
# order, handed out ahead of the packer's bins.
self._ordered = _TasksInListingOrder(budget)

@property
def requires_global_input(self) -> bool:
return self._packer.requires_global_input

def add_input(self, input_manifest: FileManifest) -> None:
if not _read_keeps_order():
self._packer.add_input(input_manifest)
return
self._keeps_order = _read_keeps_order()
self._whole_files = self._log_time_order and self._keeps_order
for file_rows in _rows_by_file(input_manifest):
self._ordered.add(_whole_file(file_rows))
self._place(file_rows)

def has_partition(self) -> bool:
return bool(self._ordered.tasks) or self._packer.has_partition()
Expand All @@ -62,6 +113,51 @@ def finalize(self) -> None:
self._ordered.seal()
self._packer.finalize()

def _place(self, file_rows: _FileRows) -> None:
"""Pack one file with others, or cut it into tasks of its own."""
if not file_rows.chunks:
self._pack(file_rows.rows)
return
chunks = self._in_read_order(file_rows.chunks)
weight = sum(chunk.run.size_bytes for chunk in chunks)
if self._whole_files or weight <= self._budget:
Comment thread
marwan116 marked this conversation as resolved.
Resolved
self._pack(_task_manifest(file_rows, chunks))
Comment thread
marwan116 marked this conversation as resolved.
Resolved
return
self._ordered.seal()
for task in self._cut(chunks):
self._ordered.tasks.append(_task_manifest(file_rows, task))

Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
def _pack(self, rows: FileManifest) -> None:
"""Pack ``rows`` with other files, in listing order if the read keeps order."""
if self._keeps_order:
self._ordered.add(rows)
else:
self._packer.add_input(rows)

def _in_read_order(self, chunks: List[_Chunk]) -> List[_Chunk]:
"""``chunks`` in the order the reader yields their messages."""
if self._log_time_order:
return sorted(chunks, key=lambda c: (c.start_time, c.offset))
return sorted(chunks, key=lambda c: c.offset)

def _cut(self, chunks: Sequence[_Chunk]) -> Iterator[Sequence[_Chunk]]:
"""Split ``chunks`` into consecutive tasks."""
start = 0
while start < len(chunks):
end = self._task_end(chunks, start)
yield chunks[start:end]
start = end

def _task_end(self, chunks: Sequence[_Chunk], start: int) -> int:
"""The exclusive end of the task that begins at ``chunks[start]``: the
chunk that would take it past the budget, or the file's end."""
size = chunks[start].run.size_bytes
for i in range(start + 1, len(chunks)):
size += chunks[i].run.size_bytes
if size > self._budget:
return i
return len(chunks)


class _TasksInListingOrder:
"""Packs files into read tasks without reordering them.
Expand Down Expand Up @@ -118,38 +214,60 @@ def _read_keeps_order() -> bool:
return DataContext.get_current().execution_options.preserve_order


def _rows_by_file(manifest: FileManifest) -> Iterator[FileManifest]:
def _rows_by_file(manifest: FileManifest) -> Iterator[_FileRows]:
"""The manifest's rows grouped by file, in listing order.

A listing manifest holds one listed file, so a path listed twice arrives as
two manifests and its copies stay apart.
"""
block = manifest.as_block()
assert isinstance(block, pa.Table)
records: List[Optional[bytes]] = block.column(CHUNK_INDEX_COLUMN).to_pylist()
rows_by_path: Dict[str, List[int]] = {}
for i, path in enumerate(manifest.paths):
rows_by_path.setdefault(str(path), []).append(i)
block = manifest.as_block()
for rows in rows_by_path.values():
yield FileManifest(block.take(rows))
for path, rows in rows_by_path.items():
yield _FileRows(
path=path,
size=int(manifest.file_sizes[rows[0]]),
rows=FileManifest(block.take(rows)),
chunks=_chunks(manifest, records, rows),
)


def _chunks(
manifest: FileManifest, records: List[Optional[bytes]], rows: List[int]
) -> List[_Chunk]:
"""The chunks that ``rows`` name, or none when a row cannot be placed in time.

A row is placed in time when it carries its chunk's ``ChunkIndex`` record
(see the module docstring).
"""
chunks = []
for i in rows:
metadata = manifest.file_chunk_metadatas[i]
start_time = chunk_start_time(records[i])
if metadata is None or start_time is None:
return []
chunks.append(_Chunk(FileChunk.from_metadata(metadata), start_time))
return chunks


def _whole_file(file_rows: FileManifest) -> FileManifest:
"""One row naming every chunk of the file, or the rows unchanged.
def _task_manifest(file_rows: _FileRows, chunks: Sequence[_Chunk]) -> FileManifest:
"""One row naming ``chunks`` of the file, as the packer emits its rows.

A file listed whole, or an unreadable file, has a row that names no chunk.
Such rows already cover the whole file, so they pass through as listed.
The packer keeps a run of several chunks whole, since MCAP's packer splits
no run (``split_coalesced`` is off).
"""
metadatas = file_rows.file_chunk_metadatas
if any(md is None or len(md["unit_ids"]) == 0 for md in metadatas):
return file_rows
runs = [FileChunk.from_metadata(md) for md in metadatas]
runs = [chunk.run for chunk in chunks]
run = FileChunk(
unit_ids=tuple(sorted(u for r in runs for u in r.unit_ids)),
unit_ids=tuple(sorted(chunk.offset for chunk in chunks)),
num_rows=sum(r.num_rows for r in runs),
size_bytes=sum(r.size_bytes for r in runs),
fully_matched=all(r.fully_matched for r in runs),
)
return FileManifest.construct_manifest(
paths=[file_rows.paths[0]],
sizes=[file_rows.file_sizes[0]],
paths=[file_rows.path],
sizes=[file_rows.size],
chunk_metadatas=[run.to_metadata()],
)
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

``MCAPSummaryIndexer`` walks the paths like ``NonSamplingFileIndexer``, then
reads each file's summary and emits one manifest row per chunk that can hold a
selected message. The partitioner (``OnlineBinPacker``) groups those rows into
read tasks by the memory each chunk is estimated to need (``chunk_runs``). A
large recording is then read by many tasks, and a small one shares a task with
its neighbours. A file whose summary shows it cannot match the selection is
dropped before any payload is read.
selected message. The partitioner groups those rows into read tasks by the
memory each chunk is estimated to need (``chunk_runs``). A large recording is
then read by many tasks, and a small one shares a task with its neighbours
(``mcap_file_cuts``). A file whose summary shows it cannot match the selection
is dropped before any payload is read.

A file whose summary is missing, has no chunk index or lists no channels is
listed as one whole-file row.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
file's records, stored once per file. For a chunk row those are the file's
schema and channel records. A whole-file row carries the raw summary section,
or no records when the file has no summary. An unreadable file carries null.
2. ``TaskSummaryPartitioner`` packs the rows with ``OnlineBinPacker``, which
keeps only paths, sizes and chunk metadata. To each packed row it adds the
2. ``TaskSummaryPartitioner`` groups the rows into read tasks with
``OrderedFileCutter`` and ``OnlineBinPacker`` (``mcap_file_cuts``), which
keep only paths, sizes and chunk metadata. To each packed row it adds the
file's records followed by the index records of the chunks the row names,
compressed with zstd (``TASK_SUMMARY_COLUMN``).

Expand Down Expand Up @@ -44,6 +45,7 @@

# A ``ChunkIndex`` record starts with the opcode and the record length, then the
# chunk's two log times (8 bytes each) and its start offset (8).
_START_TIME_AT = _RECORD_PREFIX_BYTES
_CHUNK_START_OFFSET_AT = _RECORD_PREFIX_BYTES + 16


Expand Down Expand Up @@ -76,6 +78,20 @@ def split_section(section: memoryview) -> SectionRecords:
)


def chunk_start_time(record: Optional[bytes]) -> Optional[int]:
"""The first log time in a chunk, read from its ``ChunkIndex`` record.

``None`` when ``record`` is not a ``ChunkIndex`` record, such as the empty or
missing record of a listing row that names no chunk.
"""
from mcap.opcode import Opcode

at = _START_TIME_AT
if not record or record[0] != Opcode.CHUNK_INDEX or len(record) < at + 8:
return None
return int.from_bytes(record[at : at + 8], "little")


def with_listing_records(
manifest: FileManifest,
file_records: Optional[bytes],
Expand Down Expand Up @@ -139,9 +155,10 @@ class _HeldFile:
class TaskSummaryPartitioner(FilePartitioner):
"""Packs listing rows with ``packer`` and adds each row's summary records.

``packer`` is an ``OnlineBinPacker``. It drops the listing's record
columns, so this partitioner holds each file's records until its last row
leaves in a packed task. What it holds follows the packer's open bins.
``packer`` groups the rows into read tasks (``OrderedFileCutter``). It
drops the listing's record columns, so this partitioner holds each file's
records until its last row leaves in a packed task. What it holds follows
the packer's open bins.
"""

def __init__(self, packer: FilePartitioner):
Expand Down
Loading
Loading