Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
"""The reading of selected messages out of one MCAP file.

An indexed file is read chunk by chunk, in file order or merged by log time. A
file without a chunk index is scanned from the start, and sorted in memory when
ordered by log time. A channel or schema that only an earlier chunk declares is
read back from that chunk.
An indexed file is read chunk by chunk, in file order or merged by log time.
Chunks that are read one after another and lie close together in the file are
fetched in one ranged read (``mcap_io.read_chunks``). A file without a chunk
index is scanned from the start, and sorted in memory when ordered by log time.
A channel or schema that only an earlier chunk declares is read back from that
chunk.
"""

import heapq
Expand All @@ -21,15 +23,19 @@
Tuple,
)

from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_chunk
from ray.data._internal.datasource_v2.formats.mcap.mcap_io import (
ChunkReads,
read_chunk,
read_chunks,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection
from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import (
message_row_id,
unindexed_message_row_id,
)

if TYPE_CHECKING:
from mcap.records import Channel, ChunkIndex, Message, Schema
from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema
from mcap.summary import Summary

# One selected message as the reader sees it: its schema (``None`` for a
Expand All @@ -43,6 +49,11 @@ def _log_time(message: _Scanned) -> int:
return message[2].log_time


def _merge_order(chunk_index: "ChunkIndex") -> Tuple[int, int]:
"""The order in which the log-time merge expands chunks."""
return chunk_index.message_start_time, chunk_index.chunk_start_offset


@dataclass
class _Declared:
"""Channel and schema records known while reading one file.
Expand Down Expand Up @@ -81,12 +92,19 @@ class SelectedMessageReader:
"""Reads the messages a selection keeps out of an open MCAP file.

With ``log_time_order``, an indexed file's messages come out in ascending
``log_time`` order rather than in file order.
``log_time`` order rather than in file order. ``chunk_reads`` decides which
chunks share a ranged read.
"""

def __init__(self, selection: MCAPSelection, log_time_order: bool):
def __init__(
self,
selection: MCAPSelection,
log_time_order: bool,
chunk_reads: ChunkReads = ChunkReads(),
):
self._selection = selection
self._log_time_order = log_time_order
self._chunk_reads = chunk_reads

def iter_chunks(
self,
Expand Down Expand Up @@ -118,9 +136,9 @@ def iter_chunks(
file_summary or (lambda: summary),
)
if not self._log_time_order:
for chunk_index in chunk_indexes:
yield from self._read_chunk(
f, path, summary, chunk_index, selected, declared
for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads):
yield from self._chunk_messages(
f, path, summary, chunk_index, chunk, selected, declared
)
return
yield from self._merge_chunks_by_log_time(
Expand Down Expand Up @@ -166,18 +184,32 @@ def _merge_chunks_by_log_time(
the top, so only the chunks that overlap in time are held at once. A
message is yielded once every chunk that could precede it is expanded.
"""
# Heap entries: (log time, kind, chunk offset, index in chunk, item).
# ``kind`` 0 is a chunk index, 1 a message, so on a tied log time a
# chunk is expanded before a message is yielded.
# Heap entries: (log time, kind, chunk offset, index in chunk, message).
# ``kind`` 0 is a chunk, 1 a message, so on a tied log time a chunk is
# expanded before a message is yielded.
heap: List[Tuple[int, int, int, int, Any]] = [
(c.message_start_time, 0, c.chunk_start_offset, 0, c) for c in chunk_indexes
(c.message_start_time, 0, c.chunk_start_offset, 0, None)
for c in chunk_indexes
]
heapq.heapify(heap)
# Chunks leave the heap in ``_merge_order``, so they are read in it.
chunks = read_chunks(
f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads
)
while heap:
_, kind, offset, _, item = heapq.heappop(heap)
if kind == 0:
for index, selected_message in self._read_chunk(
f, path, summary, item, selected, declared, with_index=True
chunk_index, chunk = next(chunks)
assert chunk_index.chunk_start_offset == offset
for index, selected_message in self._chunk_messages(
f,
path,
summary,
chunk_index,
chunk,
selected,
declared,
with_index=True,
):
heapq.heappush(
heap,
Expand All @@ -192,12 +224,13 @@ def _merge_chunks_by_log_time(
else:
yield item

def _read_chunk(
def _chunk_messages(
self,
f: Any,
path: str,
summary: "Summary",
chunk_index: "ChunkIndex",
chunk: "Chunk",
selected: Set[int],
declared: "_Declared",
with_index: bool = False,
Expand All @@ -212,7 +245,6 @@ def _read_chunk(
from mcap.records import Channel, Message, Schema
from mcap.stream_reader import breakup_chunk

chunk = read_chunk(f, chunk_index)
index = -1
for record in breakup_chunk(chunk):
if isinstance(record, Message):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@
decode_payload,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection
from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summaries
from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import (
parse_listed_summary,
read_summaries,
)
from ray.data._internal.tensor_extensions.arrow import convert_to_pyarrow_array
from ray.data.context import DataContext
from ray.util.debug import log_once
Expand Down Expand Up @@ -90,6 +93,7 @@ def infer_data_type(
filesystem: "FileSystem",
sampled_paths: Sequence[str],
list_paths: Callable[[], Iterable[str]],
summary_budget_bytes: int,
) -> Optional[pa.DataType]:
"""Type of ``data`` when it holds decoded JSON; ``None`` when it is ``binary``.

Expand All @@ -102,11 +106,12 @@ def infer_data_type(
sampled_paths: The files sampled for the schema, in listing order.
list_paths: Lists the read's files in listing order, with its extension
filter. Called only when no sampled file types ``data``.
summary_budget_bytes: Most bytes of summary sections held at once.

Returns:
The type of the sampled JSON values, or ``None`` for ``binary``.
"""
inspector = _FileInspector(selection, filesystem)
inspector = _FileInspector(selection, filesystem, summary_budget_bytes)
signature = inspector.inspect_sample(sampled_paths)
if signature.has_non_json or signature.value_type is not None:
return signature.value_type
Expand All @@ -130,9 +135,15 @@ class _FileInspector:
Scans at most ``_MAX_SCANNED_FILES`` files whole.
"""

def __init__(self, selection: MCAPSelection, filesystem: "FileSystem"):
def __init__(
self,
selection: MCAPSelection,
filesystem: "FileSystem",
summary_budget_bytes: int,
):
self._selection = selection
self._filesystem = filesystem
self._summary_budget_bytes = summary_budget_bytes
self._scans_left = _MAX_SCANNED_FILES

def inspect_sample(self, paths: Sequence[str]) -> _DataSignature:
Expand All @@ -145,7 +156,10 @@ def inspect_sample(self, paths: Sequence[str]) -> _DataSignature:
the read task that owns it reports the error.
"""
value_type: Optional[pa.DataType] = None
for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True):
summaries = _summaries(
self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True
)
for path, summary in summaries:
signature = self._inspect(
path, summary, want_type=value_type is None, may_scan=True
)
Expand All @@ -161,7 +175,10 @@ def walk(self, paths: List[str]) -> _DataSignature:
Only files whose summary settles the selection are inspected, so no file
is scanned whole. Files whose summary cannot be read are skipped.
"""
for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True):
summaries = _summaries(
self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True
)
for path, summary in summaries:
signature = self._inspect(path, summary, want_type=True, may_scan=False)
if signature.has_non_json or signature.value_type is not None:
return signature
Expand Down Expand Up @@ -333,16 +350,21 @@ def _readable_records(records: Iterable[Any], path: str) -> Iterator[Any]:


def _summaries(
filesystem: "FileSystem", paths: Iterable[str], *, skip_unreadable: bool
filesystem: "FileSystem",
paths: Iterable[str],
budget_bytes: int,
*,
skip_unreadable: bool,
) -> Iterator[Tuple[str, Optional["Summary"]]]:
"""Read the summaries of ``paths`` several at a time, yielding them in order.

With ``skip_unreadable``, a file whose summary cannot be read is skipped.
Otherwise its error is raised.
"""
for path, summary_read in read_summaries(filesystem, paths, lambda path: path):
sections = read_summaries(filesystem, paths, lambda path: path, budget_bytes)
for path, section_read in sections:
try:
listed = summary_read.result()
listed = parse_listed_summary(section_read.result(), path)
except Exception as exc: # noqa: BLE001 - not an MCAP file
if not skip_unreadable:
raise
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@
from ray.data.datasource.file_based_datasource import FileShuffleConfig

# Chunks are packed into read tasks of about this many bytes of estimated read
# memory (``mcap_summary.chunk_run``).
# memory (``mcap_summary.chunk_runs``). A listing task holds at most this many
# bytes of summary sections at once.
_BIN_PACKING_BYTES = env_integer("RAY_DATA_MCAP_BIN_PACKING_BYTES", 128 * MiB)
# How many partly filled read tasks the packer keeps open at once.
_BIN_PACKING_MAX_SHARED_OPEN_BINS = env_integer(
Expand Down Expand Up @@ -164,6 +165,7 @@ def _get_file_indexer(self) -> FileIndexer:
selection=self._selection,
ignore_missing_paths=self._ignore_missing_paths,
read_task_bytes=_BIN_PACKING_BYTES,
log_time_order=self._log_time_order,
row_layout=MessageRowLayout(
include_metadata=self._include_metadata,
include_row_id=self._include_row_id,
Expand Down Expand Up @@ -238,6 +240,7 @@ def infer_schema(self, sample: Optional[FileManifest]) -> pa.Schema:
self._filesystem,
sample.paths.tolist(),
self._listed_paths,
summary_budget_bytes=_BIN_PACKING_BYTES,
)
if len(sample) > 0
else None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
``MCAPSummaryIndexer`` walks the paths like ``NonSamplingFileIndexer``, then
reads each file's summary and emits one manifest row per chunk that can hold a
selected message. The partitioner (``OnlineBinPacker``) groups those rows into
read tasks by the memory each chunk is estimated to need (``chunk_run``). A
read tasks by the memory each chunk is estimated to need (``chunk_runs``). A
large recording is then read by many tasks, and a small one shares a task with
its neighbours. A file whose summary shows it cannot match the selection is
dropped before any payload is read.
Expand Down Expand Up @@ -43,8 +43,10 @@
from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import (
_DEFAULT_SUMMARY_IO_CONCURRENCY,
ListedSummary,
chunk_run,
ReadMemory,
chunk_runs,
chunk_unit_id,
parse_listed_summary,
read_summaries,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import (
Expand Down Expand Up @@ -76,7 +78,7 @@ class MCAPSummaryIndexer(NonSamplingFileIndexer):
Inherits directory traversal, extension and partition pruning, file shuffle
and whole-file checkpoint exclusion from :class:`NonSamplingFileIndexer`.
Overrides :meth:`list_files` to read each listed file's summary and turn it
into chunk rows (``mcap_summary.chunk_run``). Grouping rows into read tasks
into chunk rows (``mcap_summary.chunk_runs``). Grouping rows into read tasks
is the partitioner's job.
"""

Expand All @@ -86,6 +88,7 @@ def __init__(
selection: MCAPSelection,
ignore_missing_paths: bool,
read_task_bytes: int,
log_time_order: bool = True,
row_layout: MessageRowLayout = MessageRowLayout(),
skip_paths: Optional[Iterable[str]] = None,
num_workers: Optional[int] = None,
Expand All @@ -100,8 +103,10 @@ def __init__(
)
self._selection = selection
# The partitioner's budget per read task, which the row of an unreadable
# file weighs (see the module docstring).
# file weighs (see the module docstring), and which bounds the summary
# sections held at once.
self._read_task_bytes = read_task_bytes
self._read_memory = ReadMemory(read_task_bytes, log_time_order)
# The columns a row carries, which size each chunk's listing row.
self._row_layout = row_layout
self._io_concurrency = (
Expand Down Expand Up @@ -140,9 +145,9 @@ def list_files(
)
assert filesystem is not None # ``list_file_infos`` raised otherwise
excluded = excluded_read_unit_ids or frozenset()
for file_info, summary_read in self._read_summaries(file_infos, filesystem):
for file_info, section_read in self._read_summaries(file_infos, filesystem):
try:
listed = summary_read.result()
listed = parse_listed_summary(section_read.result(), file_info.path)
except Exception as e: # noqa: BLE001 - the file's own read task fails
manifest = self._unreadable_file_manifest(file_info, e)
else:
Expand All @@ -152,14 +157,18 @@ def list_files(

def _read_summaries(
self, file_infos: Iterable[FileInfo], filesystem: "FileSystem"
) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]:
"""Start summary reads concurrently, yielding them in listing order.
) -> Iterator[Tuple[FileInfo, "Future[Optional[memoryview]]"]]:
"""Fetch summary sections concurrently, yielding them in listing order.

Listing order decides how the partitioner groups chunks into tasks, so
it must not depend on which read finishes first.
"""
return read_summaries(
filesystem, file_infos, lambda info: info.path, self._io_concurrency
filesystem,
file_infos,
lambda info: info.path,
self._read_task_bytes,
self._io_concurrency,
)

def _unreadable_file_manifest(
Expand Down Expand Up @@ -222,19 +231,25 @@ def _manifest_for_file(
logger.debug("Skipping %s: no chunk can match the selection", path)
return None

total_uncompressed = sum(c.uncompressed_size for c in summary.chunk_indexes)
row_bytes = self._row_layout.row_bytes(
path, (summary.channels[c] for c in selected), summary.schemas
)
runs = [
chunk_run(chunk_index, summary.statistics, total_uncompressed, row_bytes)
chunk_indexes = [
chunk_index
for chunk_index in summary.chunk_indexes
if self._selection.chunk_may_match(chunk_index, selected)
and chunk_unit_id(path, chunk_index.chunk_start_offset)
not in excluded_read_unit_ids
]
if not runs:
if not chunk_indexes:
return None
channels = [summary.channels[c] for c in selected]
row_bytes = self._row_layout.row_bytes(path, channels, summary.schemas)
runs = chunk_runs(
path,
summary,
chunk_indexes,
row_bytes,
self._read_memory,
payload_factor=self._row_layout.payload_factor(channels),
)
manifest = FileManifest.construct_manifest(
paths=[path] * len(runs),
sizes=[file_info.size] * len(runs),
Expand Down
Loading
Loading