diff --git a/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py b/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py index 69a013c43b85..d69e663b9092 100644 --- a/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py +++ b/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py @@ -324,13 +324,14 @@ class OnlineBinPacker(FilePartitioner): :meth:`has_partition` / :meth:`next_partition` as they become available; call :meth:`finalize` once all input is added to flush the still-open bins. - By default it packs globally, with one pool of open bins keyed by file, so - it needs every listing row (``requires_global_input=True``). With - ``requires_global_input=False`` each listing task packs only its own shard - of files, and ``partition_files`` flushes the open bins when the task ends. - Listing then runs as parallel tasks, at the cost of at most one under-filled - bin per shard. A file's rows never cross shards, because shards split the - path list. + By default it packs globally, so it needs every listing row + (``requires_global_input=True``). Setting ``requires_global_input=False`` + lets listing run as parallel tasks: the listing planner + (``plan_list_files_op``) splits the paths passed to ``read_*`` evenly across + up to ``DEFAULT_MAX_NUM_LIST_FILES_TASKS`` (200) tasks, and a folder counts + as one path. It still uses one task when the read shuffles files. The cost + is that bins may not fill as well, since each task packs only its own + files, which may lead to more read tasks. """ def __init__( diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 9e41b52775a4..95f7ed8d4d74 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -11,6 +11,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Dict, Iterable, Iterator, @@ -49,11 +50,14 @@ class _Declared: Starts as the summary's records and grows with those met inside chunks, so a channel declared only in an earlier chunk serves the later ones. ``scanned`` holds the offsets of the chunks already read back for records. + ``file_summary`` returns the file's whole summary, whose chunk indexes say + where to read back. """ channels: Dict[int, "Channel"] schemas: Dict[int, "Schema"] scanned: Set[int] + file_summary: Callable[[], "Summary"] def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None: @@ -90,11 +94,14 @@ def iter_chunks( path: str, summary: "Summary", offsets: Optional[Set[int]], + file_summary: Optional[Callable[[], "Summary"]] = None, ) -> Iterator[_Selected]: """Yield the selected messages of the owned chunks of an indexed file. Chunks are read in file order, or merged by log time with - ``log_time_order``. + ``log_time_order``. ``summary`` may index only the owned chunks, as a + read task's summary does (``mcap_task_summary``). Then ``file_summary`` + returns the whole summary, which reading back a declaration needs. """ selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -104,7 +111,12 @@ def iter_chunks( for c in self.candidate_chunks(summary, selected) if offsets is None or c.chunk_start_offset in offsets ] - declared = _Declared(dict(summary.channels), dict(summary.schemas), set()) + declared = _Declared( + dict(summary.channels), + dict(summary.schemas), + set(), + file_summary or (lambda: summary), + ) if not self._log_time_order: for chunk_index in chunk_indexes: yield from self._read_chunk( @@ -206,7 +218,7 @@ def _read_chunk( if isinstance(record, Message): index += 1 channel, schema = self._channel_and_schema( - f, path, summary, chunk_index, declared, record + f, path, chunk_index, declared, record ) if not self._channel_is_selected(channel, schema, summary, selected): continue @@ -224,7 +236,6 @@ def _channel_and_schema( self, f: Any, path: str, - summary: "Summary", chunk_index: "ChunkIndex", declared: _Declared, message: "Message", @@ -236,7 +247,7 @@ def _channel_and_schema( channel = declared.channels.get(message.channel_id) if channel is None: self._find_in_earlier_chunks( - f, summary, chunk_index, declared, channel_id=message.channel_id + f, chunk_index, declared, channel_id=message.channel_id ) channel = declared.channels.get(message.channel_id) if channel is None: @@ -250,7 +261,7 @@ def _channel_and_schema( # (``repeat_schemas=False``). ``message_types`` and the schema # columns need it. self._find_in_earlier_chunks( - f, summary, chunk_index, declared, schema_id=channel.schema_id + f, chunk_index, declared, schema_id=channel.schema_id ) schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None return channel, schema @@ -281,7 +292,6 @@ def _channel_is_selected( def _find_in_earlier_chunks( self, f: Any, - summary: "Summary", chunk_index: "ChunkIndex", declared: "_Declared", *, @@ -299,7 +309,7 @@ def _find_in_earlier_chunks( earlier = sorted( ( c - for c in summary.chunk_indexes + for c in declared.file_summary().chunk_indexes if c.chunk_start_offset < chunk_index.chunk_start_offset and c.chunk_start_offset not in declared.scanned ), diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py index d2e5331fb331..0bd9f56aa680 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py @@ -342,13 +342,13 @@ def _summaries( """ for path, summary_read in read_summaries(filesystem, paths, lambda path: path): try: - summary = summary_read.result() + listed = summary_read.result() except Exception as exc: # noqa: BLE001 - not an MCAP file if not skip_unreadable: raise logger.debug("Skipping %s for the schema: %s", path, exc) continue - yield path, summary + yield path, None if listed is None else listed.summary def _paths_past_sample(listed: Iterable[str], sampled: Set[str]) -> List[str]: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index e1b9561dd768..bfa699e0f8b0 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -7,6 +7,7 @@ chunks into read tasks of about ``RAY_DATA_MCAP_BIN_PACKING_BYTES`` of that estimate (128 MiB by default). When the read keeps order, ``OrderedFileGrouper`` first joins each file's chunks so that one task reads the file (``mcap_file_cuts``). +``TaskSummaryPartitioner`` adds to each task the summary records it needs. ``MCAPScanner`` holds the optimizer's pushdowns and builds the ``MCAPReader``, which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type`` (``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes. @@ -51,6 +52,9 @@ TimeRange, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + TaskSummaryPartitioner, +) from ray.data._internal.datasource_v2.interfaces.datasource_v2 import ( DatasourceCategory, FileDataSourceV2, @@ -175,7 +179,7 @@ def get_file_partitioner(self, **kwargs): # # Packing is per listing shard, not global, so listing can run as many # parallel tasks. Each chunk still lands in exactly one read task; the - # cost is at most one under-filled task per shard. + # cost is that bins may not fill as well, which may mean more read tasks. from ray.data._internal.datasource_v2.common.online_bin_packer import ( OnlineBinPacker, ) @@ -185,7 +189,9 @@ def get_file_partitioner(self, **kwargs): max_shared_open_bins=_BIN_PACKING_MAX_SHARED_OPEN_BINS, requires_global_input=False, ) - return OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES) + return TaskSummaryPartitioner( + OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES) + ) @property @override diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py index e481d98f4df6..536fd1f16554 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py @@ -119,7 +119,11 @@ def _read_keeps_order() -> bool: def _rows_by_file(manifest: FileManifest) -> Iterator[FileManifest]: - """The manifest's rows grouped by file, in listing order.""" + """The manifest's rows grouped by file, in listing order. + + A listing manifest holds one listed file, so a path listed twice arrives as + two manifests and its copies stay apart. + """ rows_by_path: Dict[str, List[int]] = {} for i, path in enumerate(manifest.paths): rows_by_path.setdefault(str(path), []).append(i) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index 7763f775a18e..58310e0544c2 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -11,6 +11,9 @@ A file whose summary is missing, has no chunk index or lists no channels is listed as one whole-file row. +Each row also carries the summary records its read task will need, so read +tasks do not read summaries (``mcap_task_summary``). + A file whose summary cannot be read, such as a recording cut short, does not fail the listing. It is listed as one row that names no chunk, which the reader takes as the whole file, and that weighs a whole read task, so the packer puts @@ -39,10 +42,15 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( _DEFAULT_SUMMARY_IO_CONCURRENCY, + ListedSummary, chunk_run, chunk_unit_id, read_summaries, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + split_section, + with_listing_records, +) from ray.data._internal.datasource_v2.interfaces.file_indexer import FileInfo from ray.data._internal.datasource_v2.interfaces.file_manifest import ( FileChunk, @@ -52,7 +60,6 @@ if TYPE_CHECKING: from concurrent.futures import Future - from mcap.summary import Summary from pyarrow.fs import FileSystem from ray.data._internal.datasource_v2.interfaces.file_pruner import FilePruner @@ -135,17 +142,17 @@ def list_files( excluded = excluded_read_unit_ids or frozenset() for file_info, summary_read in self._read_summaries(file_infos, filesystem): try: - summary = summary_read.result() + listed = summary_read.result() except Exception as e: # noqa: BLE001 - the file's own read task fails manifest = self._unreadable_file_manifest(file_info, e) else: - manifest = self._manifest_for_file(file_info, summary, excluded) + manifest = self._manifest_for_file(file_info, listed, excluded) if manifest is not None: yield manifest def _read_summaries( self, file_infos: Iterable[FileInfo], filesystem: "FileSystem" - ) -> Iterator[Tuple[FileInfo, "Future[Optional[Summary]]"]]: + ) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]: """Start summary reads concurrently, yielding them in listing order. Listing order decides how the partitioner groups chunks into tasks, so @@ -172,16 +179,18 @@ def _unreadable_file_manifest( size_bytes=self._read_task_bytes, fully_matched=False, ) - return FileManifest.construct_manifest( + manifest = FileManifest.construct_manifest( paths=[file_info.path], sizes=[file_info.size], chunk_metadatas=[whole_file.to_metadata()], ) + # No records: the read task reads the summary itself and fails on it. + return with_listing_records(manifest, None, [None]) def _manifest_for_file( self, file_info: FileInfo, - summary: Optional["Summary"], + listed: Optional[ListedSummary], excluded_read_unit_ids: AbstractSet[str], ) -> Optional[FileManifest]: """The listing rows of one file, or ``None`` when it cannot match. @@ -193,13 +202,16 @@ def _manifest_for_file( """ path = file_info.path assert file_info.size is not None + summary = listed.summary if listed is not None else None if summary is None or not summary.chunk_indexes or not summary.channels: # Without a summary, a chunk index or channel records, nothing says - # which chunk holds what. - return FileManifest.construct_manifest( + # which chunk holds what. The read task gets the whole summary. + manifest = FileManifest.construct_manifest( paths=[path], sizes=[file_info.size], chunk_metadatas=[None] ) + records = b"" if listed is None else bytes(listed.section) + return with_listing_records(manifest, records, [None]) selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -223,8 +235,15 @@ def _manifest_for_file( ] if not runs: return None - return FileManifest.construct_manifest( + manifest = FileManifest.construct_manifest( paths=[path] * len(runs), sizes=[file_info.size] * len(runs), chunk_metadatas=[run.to_metadata() for run in runs], ) + assert listed is not None + records = split_section(listed.section) + return with_listing_records( + manifest, + records.declarations, + [records.chunk_indexes[run.unit_ids[0]] for run in runs], + ) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index ceb02087cb0d..ace3f48a94e9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -5,8 +5,11 @@ requests per chunk it indexes. Here the bytes are fetched first and parsed in memory: -- ``read_summary_from`` checks both magic bytes and the footer, then parses the - summary section, all in one to three ranged reads. +- ``read_summary_section_from`` checks both magic bytes and the footer, then + returns the summary section, all in one to three ranged reads. + ``read_summary_from`` also parses it. +- ``summary_records`` splits a section into its raw records, and + ``parse_summary_records`` parses records copied out of a section. - ``read_chunk`` fetches one chunk record in one ranged read. - ``buffered`` wraps a file for a linear scan, so it reads in large blocks. @@ -14,7 +17,7 @@ """ import io -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Iterator, Optional, Tuple, Union from ray.data._internal.util import KiB @@ -30,7 +33,6 @@ _MAGIC = b"\x89MCAP0\r\n" # A footer record is the opcode, an 8-byte length and 20 bytes of fields: # ``summary_start`` (8), ``summary_offset_start`` (8) and ``summary_crc`` (4). -_FOOTER_OPCODE = 0x02 _FOOTER_BYTES = 1 + 8 + 20 _RECORD_PREFIX_BYTES = 1 + 8 # opcode and record length @@ -46,6 +48,16 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: recording. Errors reading ``f`` propagate unchanged, so callers can retry them. """ + section = read_summary_section_from(f, path) + return None if section is None else _parse_summary(section, path) + + +def read_summary_section_from(f: Any, path: str) -> Optional[memoryview]: + """Return the raw summary section of the open MCAP file ``f``, or ``None``. + + The section runs from the first summary record through the footer. Raises + as ``read_summary_from`` does, except that the records are not parsed. + """ size = f.seek(0, io.SEEK_END) tail_start = max(0, size - _SUMMARY_TAIL_BYTES) tail = _read_range(f, tail_start, size) @@ -62,7 +74,35 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: tail = _read_range(f, summary_start, tail_start) + tail tail_start = summary_start end = len(tail) - len(_MAGIC) - return _parse_summary(memoryview(tail)[summary_start - tail_start : end], path) + return memoryview(tail)[summary_start - tail_start : end] + + +def summary_records(section: memoryview) -> Iterator[Tuple[int, memoryview]]: + """Yield ``(opcode, record)`` for each record of ``section`` before its footer. + + ``record`` is the whole raw record: opcode, length and body. The framing is + not checked here, so split only a section that parsed. + """ + from mcap.opcode import Opcode + + position = 0 + while position < len(section): + opcode = section[position] + if opcode == Opcode.FOOTER: + return + length = int.from_bytes(section[position + 1 : position + 9], "little") + end = position + _RECORD_PREFIX_BYTES + length + yield opcode, section[position:end] + position = end + + +def parse_summary_records(records: Union[bytes, memoryview], path: str) -> "Summary": + """Parse a summary section of ``path``, or records copied out of one. + + Copied records are whole raw records, as ``summary_records`` yields them, + and end without a footer. Raises as ``read_summary_from`` does. + """ + return _parse_summary(memoryview(records), path) def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": @@ -96,13 +136,15 @@ def _summary_start(tail: bytes, size: int, path: str) -> int: ``tail`` is the last bytes of the file, which is ``size`` bytes long: they must end with the footer record and then the magic bytes. """ + from mcap.opcode import Opcode + footer_start = len(tail) - len(_MAGIC) - _FOOTER_BYTES footer = tail[max(0, footer_start) : len(tail) - len(_MAGIC)] length = int.from_bytes(footer[1:9], "little") if ( footer_start < 0 or not tail.endswith(_MAGIC) - or footer[0] != _FOOTER_OPCODE + or footer[0] != Opcode.FOOTER or length != _FOOTER_BYTES - _RECORD_PREFIX_BYTES ): raise ValueError( @@ -119,7 +161,7 @@ def _summary_start(tail: bytes, size: int, path: str) -> int: def _parse_summary(section: memoryview, path: str) -> "Summary": - """Collect the records of a summary section, which runs up to the footer. + """Collect the records of a summary section, up to its footer or its end. Raises ``ValueError`` naming ``path`` when the section does not parse. """ @@ -147,7 +189,10 @@ def _collect_summary_records(section: memoryview) -> "Summary": from mcap.summary import Summary summary = Summary() - for record in StreamReader(io.BytesIO(section), skip_magic=True).records: + if not section: + return summary + stream = io.BytesIO(section) + for record in StreamReader(stream, skip_magic=True).records: if isinstance(record, Footer): break if isinstance(record, Statistics): @@ -162,6 +207,10 @@ def _collect_summary_records(section: memoryview) -> "Summary": summary.attachment_indexes.append(record) elif isinstance(record, MetadataIndex): summary.metadata_indexes.append(record) + # Records copied out of a section end without a footer. The reader has + # consumed exactly the records yielded so far. + if stream.tell() == len(section): + break return summary diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index d09e6efc1560..07aea2d965c7 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -1,17 +1,18 @@ """``MCAPReader``, which reads one read task's chunks into message rows. ``MCAPReader.read`` receives a ``FileManifest`` whose rows name files and, per -file, the byte offsets of the chunks the task owns. For each file it seeks to -those chunks, decompresses them, keeps the selected messages and yields Arrow -tables of about ``target_block_size`` bytes. With ``log_time_order``, a file's -chunks are merged by log time as they are read, so memory holds only the -chunks that overlap in time. +file, the byte offsets of the chunks the task owns and the summary records it +needs (``mcap_task_summary``). For each file it seeks to those chunks, +decompresses them, keeps the selected messages and yields Arrow tables of about +``target_block_size`` bytes. With ``log_time_order``, a file's chunks are merged +by log time as they are read, so memory holds only the chunks that overlap in +time. A row has the columns of ``message_schema``. Its optional ``row_id`` is a deterministic name for the message, built from the file path, the chunk's byte offset and the message's position in the chunk. -This module groups the manifest by file and finishes every table with its +This module reads the manifest row by row and finishes every table with its partition and synthesized columns. The other modules read and build the rows: - ``mcap_chunks`` reads the selected messages of a file. @@ -20,8 +21,9 @@ import itertools import logging +from dataclasses import dataclass from functools import partial -from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple +from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Set import pyarrow as pa from pyarrow.fs import FileSystem, LocalFileSystem @@ -33,6 +35,7 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + parse_summary_records, read_summary_from, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( @@ -40,6 +43,9 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import chunk_unit_id +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + task_summaries, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data._internal.datasource_v2.interfaces.read_units import ReadUnit from ray.data._internal.datasource_v2.interfaces.reader import Reader @@ -53,6 +59,9 @@ from ray.util.annotations import DeveloperAPI from ray.util.debug import log_once +if TYPE_CHECKING: + from mcap.summary import Summary + logger = logging.getLogger(__name__) @@ -117,9 +126,9 @@ def __init__( def read(self, input_split: FileManifest) -> Iterator[pa.Table]: """Read the files and chunks named by ``input_split``. - All rows that name one file are read together. Files come out in - manifest order. With ``limit``, reading stops at that many rows: no - later message is added to a table and no later chunk or file is read. + Each row is read on its own, in manifest order, so a path listed twice + is read twice. With ``limit``, reading stops at that many rows: no later + message is added to a table and no later chunk or file is read. """ from ray.data.context import DataContext @@ -127,12 +136,12 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: return retried_io_errors = DataContext.get_current().retried_io_errors remaining = self._limit - for path, offsets in _owned_chunks(input_split): + for owned in _owned_files(input_split): if remaining is not None and remaining <= 0: return tables = iterate_with_retry( - partial(self._read_file, path, offsets, remaining), - f"read MCAP file {path}", + partial(self._read_file, owned, remaining), + f"read MCAP file {owned.path}", match=retried_io_errors, ) for table in tables: @@ -143,27 +152,39 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: # -- one file ---------------------------------------------------------- def _read_file( - self, path: str, offsets: Optional[Set[int]], limit: Optional[int] + self, owned: "_OwnedFile", limit: Optional[int] ) -> Iterator[pa.Table]: - """Yield the tables of one file, limited to the chunks at ``offsets``. + """Yield the tables of one file, limited to the chunks it owns. - ``offsets`` is ``None`` for a whole-file row, which means every chunk. A - file without a chunk index is scanned linearly, through a read buffer of - ``DataContext.streaming_read_buffer_size`` bytes. At most ``limit`` rows - are yielded. + ``owned.offsets`` is ``None`` for a whole-file row, which means every + chunk. A file without a chunk index is scanned linearly, through a read + buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most + ``limit`` rows are yielded. """ from ray.data.context import DataContext + path, offsets = owned.path, owned.offsets filesystem = self._filesystem or LocalFileSystem() with filesystem.open_input_file(path) as f: - summary = read_summary_from(f, path) + if owned.summary_records is None: + # The listing could not read this summary. Reading it here fails + # the task with the file's own error. + summary = read_summary_from(f, path) + file_summary = None + else: + # The listing's copy, so the summary is not read. A chunk row's + # copy indexes only the owned chunks. + summary = parse_summary_records(owned.summary_records, path) + file_summary = None if offsets is None else _FileSummary(f, path) if summary is None or not summary.chunk_indexes: buffer_size = DataContext.get_current().streaming_read_buffer_size messages = self._message_reader.iter_unindexed( buffered(f, buffer_size), path, limit ) else: - messages = self._message_reader.iter_chunks(f, path, summary, offsets) + messages = self._message_reader.iter_chunks( + f, path, summary, offsets, file_summary + ) yield from self._tables(itertools.islice(messages, limit), path, offsets) # -- rows to tables ---------------------------------------------------- @@ -177,7 +198,7 @@ def _tables( first message is the next one needed. """ # Every table reports one read unit: the file for a whole-file read, else - # the task's first chunk. Listing excludes read units chunk by chunk + # the row's first chunk. Listing excludes read units chunk by chunk # (``excluded_read_unit_ids``), so handing this unit back would skip only # that first chunk. Nothing hands units back yet: a ``row_id`` checkpoint # filters rows by id instead. @@ -301,23 +322,44 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array -def _owned_chunks(manifest: FileManifest) -> List[Tuple[str, Optional[Set[int]]]]: - """Group a manifest's rows by file: ``(path, chunk offsets or None)``. +@dataclass(frozen=True) +class _OwnedFile: + """What one manifest row reads of one file.""" + + path: str + # Byte offsets of the owned chunks; ``None`` for the whole file. + offsets: Optional[Set[int]] + # The summary records the row needs (``mcap_task_summary``); ``None`` for a + # file whose summary the listing could not read. + summary_records: Optional[bytes] + - A row with chunk metadata contributes its ``unit_ids``, the chunk byte - offsets. A row that names no chunk means the whole file (see - ``mcap_file_indexer``), which wins over any offsets listed for the same path. +def _owned_files(manifest: FileManifest) -> List[_OwnedFile]: + """One ``_OwnedFile`` per manifest row, in manifest order. + + A row with chunk metadata owns its ``unit_ids``, the chunk byte offsets. A + row that names no chunk means the whole file (see ``mcap_file_indexer``). """ - owned: Dict[str, Optional[Set[int]]] = {} - for path, metadata in zip(manifest.paths, manifest.file_chunk_metadatas): - path = str(path) + owned = [] + rows = zip(manifest.paths, manifest.file_chunk_metadatas, task_summaries(manifest)) + for path, metadata, records in rows: unit_ids = None if metadata is None else metadata.get("unit_ids") - if unit_ids is None or len(unit_ids) == 0: - owned[path] = None - continue - offsets = {int(i) for i in unit_ids} - current = owned.get(path, offsets) - if current is not None: - current.update(offsets) - owned[path] = current - return list(owned.items()) + whole_file = unit_ids is None or len(unit_ids) == 0 + offsets = None if whole_file else {int(i) for i in unit_ids} + owned.append(_OwnedFile(str(path), offsets, records)) + return owned + + +class _FileSummary: + """The whole summary of an open file, read on the first call.""" + + def __init__(self, f: Any, path: str): + self._f = f + self._path = path + self._summary: Optional["Summary"] = None + + def __call__(self) -> "Summary": + if self._summary is None: + self._summary = read_summary_from(self._f, self._path) + assert self._summary is not None, "a file with chunk rows has a summary" + return self._summary diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index 858c57fc2e95..ad906b036f40 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -5,7 +5,8 @@ log-time bounds, channels) and a ``Statistics`` record (message counts, time bounds). Reading it takes one to three ranged reads (``mcap_io``), so a read can be planned without reading any message. ``read_summaries`` reads the summaries -of many files concurrently, with retries. +of many files concurrently, with retries, and keeps each raw section for the +listing to hand on to read tasks (``ListedSummary``). Format specification: https://mcap.dev/spec """ @@ -13,6 +14,7 @@ import math from collections import deque from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Callable, @@ -28,7 +30,11 @@ from pyarrow.fs import FileSystem from ray._common.utils import env_integer -from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_summary_from +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + parse_summary_records, + read_summary_from, + read_summary_section_from, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileChunk from ray.data._internal.util import call_with_retry @@ -55,6 +61,18 @@ ) +@dataclass(frozen=True) +class ListedSummary: + """A file's summary as ``read_summaries`` reads it: parsed, and as raw records. + + ``section`` is the raw summary section, from which the listing copies the + records each read task needs into the manifest (``mcap_task_summary``). + """ + + summary: "Summary" + section: memoryview + + def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: """Return the summary of the MCAP file at ``path``, or ``None`` if it has none. @@ -64,12 +82,21 @@ def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: return read_summary_from(f, path) +def read_listed_summary(filesystem: FileSystem, path: str) -> Optional[ListedSummary]: + """Like ``read_summary``, but also keep the raw summary section.""" + with filesystem.open_input_file(path) as f: + section = read_summary_section_from(f, path) + if section is None: + return None + return ListedSummary(parse_summary_records(section, path), section) + + def _read_summary_with_retry( filesystem: "FileSystem", path: str, retried_io_errors: List[str] -) -> Optional["Summary"]: +) -> Optional[ListedSummary]: """Read one file's summary, retrying errors that match ``retried_io_errors``.""" return call_with_retry( - lambda: read_summary(filesystem, path), + lambda: read_listed_summary(filesystem, path), description=f"read MCAP summary for {path}", match=retried_io_errors, ) @@ -80,7 +107,7 @@ def read_summaries( items: Iterable[_T], path: Callable[[_T], str], io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, -) -> Iterator[Tuple[_T, "Future[Optional[Summary]]"]]: +) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: """Read the summaries of ``items`` several at a time, yielding them in order. Each item comes with its pending read, so the caller decides what a failed diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py new file mode 100644 index 000000000000..5ace2ad34e10 --- /dev/null +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py @@ -0,0 +1,218 @@ +"""The summary records each read task needs, carried to it in its manifest. + +A read task needs, per file, the schema and channel records and the index +records of the chunks it owns. The listing has read every summary already, so +it hands those records on, and a read task neither requests nor parses the +whole summary. They stay MCAP records, in two steps: + +1. ``MCAPSummaryIndexer`` adds two columns to its listing rows + (``with_listing_records``): each chunk row's ``ChunkIndex`` record, and the + file's records, stored once per file. For a chunk row those are the file's + schema and channel records. A whole-file row carries the raw summary section, + or no records when the file has no summary. An unreadable file carries null. +2. ``TaskSummaryPartitioner`` packs the rows with ``OnlineBinPacker``, which + keeps only paths, sizes and chunk metadata. To each packed row it adds the + file's records followed by the index records of the chunks the row names, + compressed with zstd (``TASK_SUMMARY_COLUMN``). + +``MCAPReader`` parses that column (``task_summaries``). A null, carried by a +file the listing could not read, makes the reader read the summary from the +file, which fails the task with the file's own error. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import pyarrow as pa + +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + _RECORD_PREFIX_BYTES, + summary_records, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +from ray.data._internal.datasource_v2.interfaces.file_partitioner import ( + FilePartitioner, +) + +# Listing columns: the file's records, once per file, and each row's chunk. +FILE_RECORDS_COLUMN = "__mcap_file_records" +CHUNK_INDEX_COLUMN = "__mcap_chunk_index" +# Packed column: the records one read task needs for one file. +TASK_SUMMARY_COLUMN = "__mcap_task_summary" + +_FILE_RECORDS_TYPE = pa.dictionary(pa.int32(), pa.binary()) + +# A ``ChunkIndex`` record starts with the opcode and the record length, then the +# chunk's two log times (8 bytes each) and its start offset (8). +_CHUNK_START_OFFSET_AT = _RECORD_PREFIX_BYTES + 16 + + +@dataclass(frozen=True) +class SectionRecords: + """The raw records of a summary section that chunk rows carry.""" + + # The schema and channel records. + declarations: bytes + # Each ``ChunkIndex`` record, by its chunk's start offset. + chunk_indexes: Mapping[int, bytes] + + +def split_section(section: memoryview) -> SectionRecords: + """Split a summary section into the records that chunk rows carry.""" + from mcap.opcode import Opcode + + declarations: List[memoryview] = [] + chunk_indexes: Dict[int, bytes] = {} + for opcode, record in summary_records(section): + if opcode in (Opcode.SCHEMA, Opcode.CHANNEL): + declarations.append(record) + elif opcode == Opcode.CHUNK_INDEX: + at = _CHUNK_START_OFFSET_AT + offset = int.from_bytes(record[at : at + 8], "little") + chunk_indexes[offset] = bytes(record) + return SectionRecords( + declarations=b"".join(declarations), + chunk_indexes=chunk_indexes, + ) + + +def with_listing_records( + manifest: FileManifest, + file_records: Optional[bytes], + chunk_index_records: Sequence[Optional[bytes]], +) -> FileManifest: + """One file's listing rows, plus the columns that carry its records. + + ``file_records`` is stored once, however many rows the file has. + ``chunk_index_records`` has one entry per row, ``None`` for a whole-file row. + """ + num_rows = len(manifest) + assert len(chunk_index_records) == num_rows + if file_records is None: + repeated = pa.nulls(num_rows, type=_FILE_RECORDS_TYPE) + else: + repeated = pa.DictionaryArray.from_arrays( + pa.array([0] * num_rows, type=pa.int32()), + pa.array([file_records], type=pa.binary()), + ) + block = _table(manifest).append_column(FILE_RECORDS_COLUMN, repeated) + block = block.append_column( + CHUNK_INDEX_COLUMN, pa.array(chunk_index_records, type=pa.binary()) + ) + return FileManifest(block) + + +def task_summaries(manifest: FileManifest) -> List[Optional[bytes]]: + """Per row of a packed manifest, the summary records its read task needs. + + ``None`` for a file whose summary the listing could not read. + """ + import zstandard + + block = _table(manifest) + assert TASK_SUMMARY_COLUMN in block.column_names, ( + f"An MCAP read task's manifest has no {TASK_SUMMARY_COLUMN!r} column. " + "Pack the listing rows with TaskSummaryPartitioner." + ) + return [ + None if records is None else zstandard.decompress(records) + for records in block.column(TASK_SUMMARY_COLUMN).to_pylist() + ] + + +def _table(manifest: FileManifest) -> pa.Table: + """The Arrow table behind ``manifest``.""" + block = manifest.as_block() + assert isinstance(block, pa.Table) + return block + + +@dataclass +class _HeldFile: + """The records of one file whose listing rows are not all packed yet.""" + + file_records: Optional[bytes] + chunk_indexes: Dict[int, bytes] = field(default_factory=dict) + unpacked_rows: int = 0 + + +class TaskSummaryPartitioner(FilePartitioner): + """Packs listing rows with ``packer`` and adds each row's summary records. + + ``packer`` is an ``OnlineBinPacker``. It drops the listing's record + columns, so this partitioner holds each file's records until its last row + leaves in a packed task. What it holds follows the packer's open bins. + """ + + def __init__(self, packer: FilePartitioner): + self._packer = packer + self._held: Dict[str, _HeldFile] = {} + + @property + def requires_global_input(self) -> bool: + return self._packer.requires_global_input + + def add_input(self, input_manifest: FileManifest) -> None: + self._hold_records(input_manifest) + self._packer.add_input(input_manifest) + + def has_partition(self) -> bool: + return self._packer.has_partition() + + def next_partition(self) -> FileManifest: + manifest = self._packer.next_partition() + records = [ + self._task_summary(str(path), metadata) + for path, metadata in zip(manifest.paths, manifest.file_chunk_metadatas) + ] + block = _table(manifest).append_column( + TASK_SUMMARY_COLUMN, pa.array(records, type=pa.binary()) + ) + return FileManifest(block) + + def finalize(self) -> None: + self._packer.finalize() + + def _hold_records(self, manifest: FileManifest) -> None: + """Keep the records of ``manifest``'s rows until they are packed.""" + block = _table(manifest) + assert FILE_RECORDS_COLUMN in block.column_names, ( + f"MCAP listing rows have no {FILE_RECORDS_COLUMN!r} column. List " + "the files with MCAPSummaryIndexer." + ) + file_records = block.column(FILE_RECORDS_COLUMN) + chunk_indexes = block.column(CHUNK_INDEX_COLUMN).to_pylist() + rows = zip(manifest.paths, manifest.file_chunk_metadatas, chunk_indexes) + for i, (path, metadata, chunk_index) in enumerate(rows): + # A path listed twice shares one entry: its copies carry the same + # records, and every row of both counts toward the release. + held = self._held.get(str(path)) + if held is None: + # A dictionary value: one copy per file, not per row. + held = _HeldFile(file_records[i].as_py()) + self._held[str(path)] = held + held.unpacked_rows += 1 + if chunk_index is not None: + (offset,) = metadata["unit_ids"] + held.chunk_indexes[int(offset)] = chunk_index + + def _task_summary( + self, path: str, metadata: Optional[Mapping[str, Any]] + ) -> Optional[bytes]: + """The records of one packed row, releasing them once the file is packed. + + A packed row unites the listing rows of one file in one task: one per + chunk it names, or one whole-file row. + """ + held = self._held[path] + units = list(metadata["unit_ids"]) if metadata is not None else [] + held.unpacked_rows -= max(1, len(units)) + if held.unpacked_rows <= 0: + del self._held[path] + if held.file_records is None: + return None + import zstandard + + offsets = sorted({int(u) for u in units}) + records = held.file_records + b"".join(held.chunk_indexes[o] for o in offsets) + return zstandard.compress(records) diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index c30dc5206a5b..b108437f0ac6 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -9,17 +9,26 @@ import json import shutil from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import pyarrow as pa from pyarrow.fs import LocalFileSystem -from ray.data._internal.datasource_v2.common.listing_utils import sample_files +from ray.data._internal.datasource_v2.common.listing_utils import ( + partition_files, + sample_files, +) +from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summary +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + TASK_SUMMARY_COLUMN, + TaskSummaryPartitioner, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest BASE_TIME = 1_000_000_000 # log time of the first message, in nanoseconds STEP = 1_000_000 # 1 ms between consecutive messages @@ -196,6 +205,44 @@ def list_manifests(datasource, **kwargs): ) +def pack(datasource, paths: Optional[List[str]] = None, **kwargs) -> List[FileManifest]: + """List ``paths`` (the datasource's by default) and pack the rows into read + tasks, as one ``ListFiles`` task does. ``kwargs`` go to the listing.""" + indexer = datasource._get_file_indexer() + partitioner = datasource.get_file_partitioner() + assert partitioner is not None + listed = indexer.list_files( + pa.array(datasource.paths if paths is None else paths), + filesystem=datasource.filesystem, + **kwargs, + ) + return [ + FileManifest(block) + for block in partition_files( + [manifest.as_block() for manifest in listed], + None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) + partitioner, + ) + ] + + +# A read-task budget no test file reaches, so packing never splits a listing. +_ONE_TASK_BYTES = 2**62 + + +def as_read_task(*listings: FileManifest) -> FileManifest: + """Pack listing rows into one read task: one row per listed file, with the + summary records the reader needs.""" + packer = OnlineBinPacker(_ONE_TASK_BYTES, requires_global_input=False) + blocks = [listing.as_block() for listing in listings] + (task,) = partition_files( + blocks, + None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) + TaskSummaryPartitioner(packer), + ) + return FileManifest(task) + + def infer_schema(datasource) -> pa.Schema: """Plan the read schema from the files planning samples.""" indexer = datasource._get_file_indexer() @@ -209,10 +256,18 @@ def scanner_for(datasource): def read_all(datasource, *manifests, scanner=None) -> pa.Table: - """Read ``manifests`` into one table, with ``scanner`` or the planned one.""" + """Read ``manifests`` into one table, with ``scanner`` or the planned one. + + A packed read task is read as given. A listing manifest, which carries no + task summaries, is first packed into one read task (``as_read_task``). + """ scanner = scanner or scanner_for(datasource) reader = scanner.create_reader() - tables = [table for manifest in manifests for table in reader.read(manifest)] + tasks = [ + m if TASK_SUMMARY_COLUMN in m.as_block().column_names else as_read_task(m) + for m in manifests + ] + tables = [table for task in tasks for table in reader.read(task)] assert tables, "expected at least one table" return pa.concat_tables(tables) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index a271b48a57e5..2f7d3d6d2ea4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -13,6 +13,7 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + parse_summary_records, read_chunk, read_summary_from, ) @@ -143,6 +144,15 @@ def test_small_file_summary_takes_one_read_and_a_large_one_three(tmp_path): assert reads == {small: 1, large: 3} +def test_no_records_parse_to_an_empty_summary(): + """Empty records, as a read task gets for a file without a summary, parse to + a summary with nothing in it.""" + summary = parse_summary_records(b"", "file.mcap") + + for field in SUMMARY_FIELDS: + assert not getattr(summary, field), field + + def test_file_without_summary_section_has_no_summary(tmp_path): """A footer whose ``summary_start`` is 0 means no summary: ``None``.""" path = os.path.join(tmp_path, "no_summary.mcap") diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index 8cc0878c7cd2..dc1b9167964f 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -11,7 +11,6 @@ import pyarrow as pa import pytest -from ray.data._internal.datasource_v2.common.listing_utils import partition_files from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( @@ -31,6 +30,7 @@ CHUNKED_FILE_MESSAGES, STEP, list_manifests, + pack, read_all, round_robin_messages, scanner_for, @@ -53,26 +53,6 @@ def unit_ids(chunk_metadata: Any) -> List[int]: return [int(i) for i in chunk_metadata["unit_ids"]] -def pack(datasource: MCAPDatasourceV2, shard: List[str]) -> List[FileManifest]: - """List ``shard`` and pack its rows into read tasks, as one ``ListFiles`` - task does.""" - indexer = datasource._get_file_indexer() - partitioner = datasource.get_file_partitioner() - assert partitioner is not None - blocks = [ - m.as_block() - for m in indexer.list_files(pa.array(shard), filesystem=datasource.filesystem) - ] - return [ - FileManifest(block) - for block in partition_files( - blocks, - None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) - partitioner, - ) - ] - - def chunk_units(tasks: Iterable[FileManifest]) -> List[Tuple[str, int]]: """Every ``(path, chunk offset)`` the tasks read.""" return [ @@ -219,17 +199,7 @@ def test_file_without_index_is_one_whole_file_row(tmp_path): def test_summary_without_channels_is_one_whole_file_row(chunked_file, monkeypatch): """A file whose summary lists no channels is listed whole and still read.""" - from ray.data._internal.datasource_v2.formats.mcap import mcap_summary - - real_read_summary = mcap_summary.read_summary - - def without_channels(filesystem, path): - summary = real_read_summary(filesystem, path) - assert summary is not None - summary.channels = {} - return summary - - monkeypatch.setattr(mcap_summary, "read_summary", without_channels) + without_summary_records(monkeypatch, "channels") datasource = MCAPDatasourceV2([chunked_file], topics=["/a"]) (manifest,) = list_manifests(datasource) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index e799d049b603..8f726de23121 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -33,6 +33,7 @@ SHARED_SCHEMA, STEP, SeekLog, + as_read_task, chunks_read, infer_schema, list_manifests, @@ -377,23 +378,23 @@ def test_reader_cuts_tables_at_target_block_size(chunked_file): """A 1-byte target block size yields a table per message, and no target yields one table.""" datasource = MCAPDatasourceV2([chunked_file]) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) scanner = scanner_for(datasource) one_byte = dataclasses.replace(scanner, target_block_size=1) - tables = list(one_byte.create_reader().read(manifest)) + tables = list(one_byte.create_reader().read(task)) assert len(tables) == CHUNKED_FILE_MESSAGES assert all(t.num_rows == 1 for t in tables) unbounded = dataclasses.replace(scanner, target_block_size=None) - assert len(list(unbounded.create_reader().read(manifest))) == 1 + assert len(list(unbounded.create_reader().read(task))) == 1 def test_scanner_prunes_columns_and_pushes_limit(chunked_file): """A pruned, limited scanner reads only its columns and rows, and an empty projection still yields every row.""" datasource = MCAPDatasourceV2([chunked_file], include_paths=True) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) scanner = scanner_for(datasource) assert isinstance(scanner, MCAPScanner) @@ -401,13 +402,13 @@ def test_scanner_prunes_columns_and_pushes_limit(chunked_file): assert pruned.read_schema().names == ["log_time", "path"] assert pruned.pushed_limit() == 4 - table = pa.concat_tables(list(pruned.create_reader().read(manifest))) + table = pa.concat_tables(list(pruned.create_reader().read(task))) assert table.column_names == ["log_time", "path"] assert table.num_rows == 4 # An empty projection, as in ``count()``, keeps the row count through a stub column. empty = scanner.prune_columns([]) - table = pa.concat_tables(list(empty.create_reader().read(manifest))) + table = pa.concat_tables(list(empty.create_reader().read(task))) assert table.num_rows == CHUNKED_FILE_MESSAGES assert empty.read_schema().names == [] assert empty.prune_columns(["topic"]).read_schema().names == [] @@ -556,11 +557,11 @@ def test_reader_cuts_tables_at_the_target_counting_every_column(tmp_path): path = os.path.join(tmp_path, "mixed.mcap") write_mixed_channels(path, num_messages=400) datasource = MCAPDatasourceV2([path], include_row_id=True, include_paths=True) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) target = 8 * 1024 scanner = dataclasses.replace(scanner_for(datasource), target_block_size=target) - tables = list(scanner.create_reader().read(manifest)) + tables = list(scanner.create_reader().read(task)) assert len(tables) > 1 # One row of about 350 bytes past the target, at most. diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py new file mode 100644 index 000000000000..afacea99fe1c --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py @@ -0,0 +1,366 @@ +"""Unit tests for carrying each read task's summary records in its manifest +(``mcap_task_summary``). + +The listing, the partitioner and the reader are called directly, with no Ray +cluster. A read task must read no summary and return the same rows as a task +that reads the summary from the file. +""" + +import dataclasses +import importlib.util +import os +from collections import defaultdict +from typing import Dict, List, Tuple + +import pyarrow as pa +import pytest +from pyarrow.fs import LocalFileSystem + +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_datasource_v2, + mcap_reader, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + parse_summary_records, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_options import TimeRange +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + FILE_RECORDS_COLUMN, + TASK_SUMMARY_COLUMN, + task_summaries, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +from ray.data.context import DataContext +from ray.data.tests.unit.datasource_v2.mcap_testing import ( + BASE_TIME, + CHUNKED_FILE_MESSAGES, + STEP, + list_manifests, + pack, + round_robin_messages, + scanner_for, + summary_of, + without_summary_records, + write_cut_short, + write_mcap, + write_mcap_without_summary_schemas, + write_mixed_channels, +) + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +# A read-task budget smaller than any chunk: every chunk gets a task of its own. +ONE_CHUNK_PER_TASK = 1 + + +class ReadLog: + """A local filesystem that records ``(offset, bytes)`` of every read, per path.""" + + def __init__(self): + self._filesystem = LocalFileSystem() + self.reads: Dict[str, List[Tuple[int, int]]] = defaultdict(list) + + def open_input_file(self, path): + return _LoggedFile(self._filesystem.open_input_file(path), self.reads[path]) + + +class _LoggedFile: + def __init__(self, f, reads: List[Tuple[int, int]]): + self._f = f + self._reads = reads + + def read(self, nbytes=-1): + offset = self._f.tell() + data = self._f.read(nbytes) + if data: + self._reads.append((offset, len(data))) + return data + + def readinto(self, buffer): + offset = self._f.tell() + size = self._f.readinto(buffer) + if size: + self._reads.append((offset, size)) + return size + + def __getattr__(self, name): + return getattr(self._f, name) + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self._f.close() + + +def set_task_budget(monkeypatch, bin_bytes) -> None: + """Pack read tasks of ``bin_bytes`` estimated bytes; ``None`` keeps the default.""" + if bin_bytes is not None: + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", bin_bytes) + + +def table_of(manifest: FileManifest) -> pa.Table: + """The Arrow table behind ``manifest``.""" + block = manifest.as_block() + assert isinstance(block, pa.Table) + return block + + +def without_task_summaries(task: FileManifest) -> FileManifest: + """``task`` with null records, so the reader reads each summary from the file.""" + block = table_of(task) + nulls = pa.nulls(len(task), type=block.schema.field(TASK_SUMMARY_COLUMN).type) + index = block.column_names.index(TASK_SUMMARY_COLUMN) + return FileManifest(block.set_column(index, TASK_SUMMARY_COLUMN, nulls)) + + +def read_tasks(scanner, tasks: List[FileManifest]) -> pa.Table: + """Read every task, in order, into one table.""" + tables = [table for task in tasks for table in scanner.create_reader().read(task)] + return pa.concat_tables(tables) + + +def _write_overlapping(path): + # Messages out of log-time order across chunks, so a log-time merge + # interleaves the chunks of a task. + times = [10, 30, 20, 50, 40, 60, 15, 35] + write_mcap( + path, + [ + {"topic": "/a", "data": {"t": t}, "log_time": BASE_TIME + t * STEP} + for t in times + ], + chunk_size=60, + ) + + +def _write_unindexed(path): + write_mcap(path, round_robin_messages(9), index=False) + + +def _write_round_robin(path): + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + + +WRITERS = { + "one_message_chunks": _write_round_robin, + "overlapping_chunks": _write_overlapping, + "schemas_only_in_chunks": write_mcap_without_summary_schemas, + "mixed_channels": write_mixed_channels, + "unindexed": _write_unindexed, +} +READ_OPTIONS = { + "default": {}, + "topics_and_time": { + "topics": ["/a", "/c"], + "time_range": TimeRange(BASE_TIME + 2 * STEP, BASE_TIME + 30 * STEP), + }, +} + + +def chunk_records(path: str) -> List[Tuple[int, int]]: + """``(offset, bytes)`` of every chunk record of ``path``, in file order.""" + return [ + (c.chunk_start_offset, c.chunk_length) for c in summary_of(path).chunk_indexes + ] + + +def whole_file(path: str) -> List[Tuple[int, int]]: + """``(offset, bytes)`` of one read of all of ``path``.""" + return [(0, os.path.getsize(path))] + + +# Each file's writer, and the reads its tasks make with a task per chunk. +TASK_READS = { + "chunked": (_write_round_robin, chunk_records), + # A file without a chunk index is scanned from its start, in one buffered read. + "unindexed": (_write_unindexed, whole_file), +} + + +@pytest.mark.parametrize( + "writer, expected_reads", TASK_READS.values(), ids=TASK_READS.keys() +) +def test_read_tasks_read_only_their_own_chunks( + tmp_path, monkeypatch, writer, expected_reads +): + """With a task per chunk, each task makes one read: its chunk record, or the + whole file when it has no index. Nothing of the summary or the magic bytes + is read.""" + path = os.path.join(tmp_path, "file.mcap") + writer(path) + datasource = MCAPDatasourceV2([path]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + read_log = ReadLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) + + table = read_tasks(scanner, tasks) + + assert sorted(read_log.reads[path]) == expected_reads(path) + assert len(tasks) == len(expected_reads(path)) + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +@pytest.mark.parametrize("bin_bytes", [ONE_CHUNK_PER_TASK, None]) +@pytest.mark.parametrize("options", READ_OPTIONS.values(), ids=READ_OPTIONS.keys()) +@pytest.mark.parametrize("writer", WRITERS.values(), ids=WRITERS.keys()) +def test_rows_match_reading_the_summary_from_the_file( + tmp_path, monkeypatch, writer, options, bin_bytes +): + """Tasks that parse their summary records return the same tables as the same + tasks reading each file's summary from the file.""" + path = os.path.join(tmp_path, "file.mcap") + writer(path) + datasource = MCAPDatasourceV2([path], **options) + set_task_budget(monkeypatch, bin_bytes) + tasks = pack(datasource) + scanner = scanner_for(datasource) + + with_records = [list(scanner.create_reader().read(task)) for task in tasks] + from_file = [ + list(scanner.create_reader().read(without_task_summaries(task))) + for task in tasks + ] + + assert with_records == from_file + + +def test_whole_file_rows_carry_the_whole_summary(chunked_file, monkeypatch): + """A file listed whole, here because its summary lists no channels, gets + every record of its summary and reads every chunk.""" + without_summary_records(monkeypatch, "channels") + datasource = MCAPDatasourceV2([chunked_file]) + (task,) = pack(datasource) + + (records,) = task_summaries(task) + table = read_tasks(scanner_for(datasource), [task]) + + assert task.file_chunk_metadatas[0] is None and records is not None + assert parse_summary_records(records, chunked_file).chunk_indexes == ( + summary_of(chunked_file).chunk_indexes + ) + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +def test_declarations_missing_from_the_summary_are_read_back(tmp_path, monkeypatch): + """Schemas declared only in chunk 0: a task owning a later chunk reads the + file's summary once to find chunk 0, which declares them. The task owning + chunk 0 reads no summary.""" + path = os.path.join(tmp_path, "file.mcap") + write_mcap_without_summary_schemas(path) + datasource = MCAPDatasourceV2([path]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + summary_reads = [] + real_read = mcap_reader.read_summary_from + + def counted_read(f, file_path): + summary_reads.append(file_path) + return real_read(f, file_path) + + monkeypatch.setattr(mcap_reader, "read_summary_from", counted_read) + scanner = scanner_for(datasource) + + for task in tasks: + read_tasks(scanner, [task]) + + # Chunk 0 declares both schemas: its task needs no read-back. + assert len(summary_reads) == len(tasks) - 1 + + +def test_unreadable_file_still_fails_its_task_naming_it(tmp_path, chunked_file): + """A file whose summary could not be listed carries no records, so its task + reads the summary itself and fails naming the file.""" + cut_short = write_cut_short(os.path.join(tmp_path, "cut_short.mcap"), chunked_file) + datasource = MCAPDatasourceV2([chunked_file, cut_short]) + tasks = pack(datasource) + scanner = scanner_for(datasource) + bad = [t for t in tasks if cut_short in [str(p) for p in t.paths]] + + (task,) = bad + + assert task_summaries(task) == [None] + with pytest.raises(ValueError, match="cut_short.mcap"): + read_tasks(scanner, [task]) + + +@pytest.mark.parametrize("keep_order", [False, True]) +@pytest.mark.parametrize("bin_bytes", [ONE_CHUNK_PER_TASK, None]) +def test_a_path_listed_twice_is_read_twice( + chunked_file, monkeypatch, bin_bytes, keep_order +): + """A path listed twice is read twice, as Parquet reads it, whether its copies + share a read task or not. Every packed row carries its own records.""" + options = DataContext.get_current().execution_options + monkeypatch.setattr(options, "preserve_order", keep_order) + set_task_budget(monkeypatch, bin_bytes) + datasource = MCAPDatasourceV2([chunked_file, chunked_file]) + tasks = pack(datasource) + + table = read_tasks(scanner_for(datasource), tasks) + + seqs = [data["seq"] for data in table.column("data").to_pylist()] + one_copy = list(range(CHUNKED_FILE_MESSAGES)) + assert all(records is not None for t in tasks for records in task_summaries(t)) + if keep_order: + assert seqs == one_copy + one_copy + else: + assert sorted(seqs) == sorted(one_copy + one_copy) + + +# Chunks of each file in the partitioner test: one per message. +CHUNKS_PER_FILE = 4 +# Files held after each file is added: with a task per chunk every row is packed +# at once; with the default budget all three files wait in one open task. +HELD = { + "task_per_chunk": (ONE_CHUNK_PER_TASK, [0, 0, 0]), + "one_task": (None, [1, 2, 3]), +} + + +@pytest.mark.parametrize("bin_bytes, expected", HELD.values(), ids=HELD.keys()) +def test_partitioner_releases_a_file_once_its_rows_are_packed( + tmp_path, monkeypatch, bin_bytes, expected +): + """The listing stores a file's records once for all its rows, and the + partitioner holds them only until the file's last row leaves in a packed + task, so it holds nothing once packing is done.""" + paths = [] + for i in range(3): + path = os.path.join(tmp_path, f"f{i}.mcap") + write_mcap(path, round_robin_messages(CHUNKS_PER_FILE)) + paths.append(path) + set_task_budget(monkeypatch, bin_bytes) + datasource = MCAPDatasourceV2(paths) + partitioner = datasource.get_file_partitioner() + record_indices = [] + held_after_each_file = [] + + for manifest in list_manifests(datasource): + file_records = table_of(manifest).column(FILE_RECORDS_COLUMN) + record_indices.append(file_records.combine_chunks().indices.to_pylist()) + partitioner.add_input(manifest) + while partitioner.has_partition(): + partitioner.next_partition() + held_after_each_file.append(len(partitioner._held)) + partitioner.finalize() + while partitioner.has_partition(): + partitioner.next_partition() + + # Every row points at the one stored copy of its file's records. + assert record_indices == [[0] * CHUNKS_PER_FILE] * len(paths) + assert held_after_each_file == expected + assert partitioner._held == {} + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__]))