From 76f4b6ef8c7177412a43ebce299e040c9369cf77 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 08:57:13 -1000 Subject: [PATCH 1/5] [Data] OnlineBinPacker: correct the docstring on per-shard packing Problem: - The docstring promised at most one under-filled bin per listing shard. A shard's flush can leave several. - It did not say when packing per shard pays off. Change: - The docstring says that listing then runs as parallel tasks: the listing planner splits the paths passed to read_* evenly across up to DEFAULT_MAX_NUM_LIST_FILES_TASKS (200) tasks, and a folder counts as one path. It still uses one task when the read shuffles files. The cost is that bins may not fill as well, which may mean more read tasks. - The same claim in MCAPDatasourceV2.get_file_partitioner's comment is fixed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/common/online_bin_packer.py | 15 ++++++++------- .../formats/mcap/mcap_datasource_v2.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py b/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py index 69a013c43b85..d69e663b9092 100644 --- a/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py +++ b/python/ray/data/_internal/datasource_v2/common/online_bin_packer.py @@ -324,13 +324,14 @@ class OnlineBinPacker(FilePartitioner): :meth:`has_partition` / :meth:`next_partition` as they become available; call :meth:`finalize` once all input is added to flush the still-open bins. - By default it packs globally, with one pool of open bins keyed by file, so - it needs every listing row (``requires_global_input=True``). With - ``requires_global_input=False`` each listing task packs only its own shard - of files, and ``partition_files`` flushes the open bins when the task ends. - Listing then runs as parallel tasks, at the cost of at most one under-filled - bin per shard. A file's rows never cross shards, because shards split the - path list. + By default it packs globally, so it needs every listing row + (``requires_global_input=True``). Setting ``requires_global_input=False`` + lets listing run as parallel tasks: the listing planner + (``plan_list_files_op``) splits the paths passed to ``read_*`` evenly across + up to ``DEFAULT_MAX_NUM_LIST_FILES_TASKS`` (200) tasks, and a folder counts + as one path. It still uses one task when the read shuffles files. The cost + is that bins may not fill as well, since each task packs only its own + files, which may lead to more read tasks. """ def __init__( diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index e1b9561dd768..8ae3864bd0d3 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -175,7 +175,7 @@ def get_file_partitioner(self, **kwargs): # # Packing is per listing shard, not global, so listing can run as many # parallel tasks. Each chunk still lands in exactly one read task; the - # cost is at most one under-filled task per shard. + # cost is that bins may not fill as well, which may mean more read tasks. from ray.data._internal.datasource_v2.common.online_bin_packer import ( OnlineBinPacker, ) From 480d7a6394417ade03161aa4421bcb794a22334d Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 08:58:49 -1000 Subject: [PATCH 2/5] [Data] read_mcap tests: share one pack helper (no behaviour change) Problem: - Packing listing rows into read tasks, as a ListFiles task does, was a helper of the listing tests alone. The reader and task summary tests need the same steps. Change: - pack(datasource, paths=None, **kwargs) moves to mcap_testing.py. It lists the datasource's own paths by default and passes kwargs, such as preserve_order, to the listing. - Test code only. The listing tests call it as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../tests/unit/datasource_v2/mcap_testing.py | 29 +++++++++++++++++-- .../unit/datasource_v2/test_mcap_listing.py | 22 +------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index c30dc5206a5b..45924549b9a7 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -9,17 +9,21 @@ import json import shutil from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import pyarrow as pa from pyarrow.fs import LocalFileSystem -from ray.data._internal.datasource_v2.common.listing_utils import sample_files +from ray.data._internal.datasource_v2.common.listing_utils import ( + partition_files, + sample_files, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summary +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest BASE_TIME = 1_000_000_000 # log time of the first message, in nanoseconds STEP = 1_000_000 # 1 ms between consecutive messages @@ -196,6 +200,27 @@ def list_manifests(datasource, **kwargs): ) +def pack(datasource, paths: Optional[List[str]] = None, **kwargs) -> List[FileManifest]: + """List ``paths`` (the datasource's by default) and pack the rows into read + tasks, as one ``ListFiles`` task does. ``kwargs`` go to the listing.""" + indexer = datasource._get_file_indexer() + partitioner = datasource.get_file_partitioner() + assert partitioner is not None + listed = indexer.list_files( + pa.array(datasource.paths if paths is None else paths), + filesystem=datasource.filesystem, + **kwargs, + ) + return [ + FileManifest(block) + for block in partition_files( + [manifest.as_block() for manifest in listed], + None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) + partitioner, + ) + ] + + def infer_schema(datasource) -> pa.Schema: """Plan the read schema from the files planning samples.""" indexer = datasource._get_file_indexer() diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index 8cc0878c7cd2..44d6e5a6d661 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -11,7 +11,6 @@ import pyarrow as pa import pytest -from ray.data._internal.datasource_v2.common.listing_utils import partition_files from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( @@ -31,6 +30,7 @@ CHUNKED_FILE_MESSAGES, STEP, list_manifests, + pack, read_all, round_robin_messages, scanner_for, @@ -53,26 +53,6 @@ def unit_ids(chunk_metadata: Any) -> List[int]: return [int(i) for i in chunk_metadata["unit_ids"]] -def pack(datasource: MCAPDatasourceV2, shard: List[str]) -> List[FileManifest]: - """List ``shard`` and pack its rows into read tasks, as one ``ListFiles`` - task does.""" - indexer = datasource._get_file_indexer() - partitioner = datasource.get_file_partitioner() - assert partitioner is not None - blocks = [ - m.as_block() - for m in indexer.list_files(pa.array(shard), filesystem=datasource.filesystem) - ] - return [ - FileManifest(block) - for block in partition_files( - blocks, - None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) - partitioner, - ) - ] - - def chunk_units(tasks: Iterable[FileManifest]) -> List[Tuple[str, int]]: """Every ``(path, chunk offset)`` the tasks read.""" return [ From 73bff171d147514065a5122d2bba1f5d855610e5 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 08:59:43 -1000 Subject: [PATCH 3/5] [Data] read_mcap V2: carry read tasks' summary records in the manifest Problem: - The listing reads every file's summary, and then each read task read its files' summaries again, one to three requests per file, to use a small part of them. Change: - Listing rows carry the summary records their read tasks need: each chunk row its ChunkIndex record, and each file its schema and channel records, stored once (mcap_task_summary). - TaskSummaryPartitioner wraps the bin packer. To each packed row it adds the file's records and the index records of the chunks the row names, compressed with zstd. A carried ChunkIndex record leaves out its message index offsets, which the listing has already applied. - The reader parses those records instead of reading the summary. A task that reads back a declaration from an earlier chunk reads the whole summary once. A file named by two rows of a task, or a manifest without the records, reads the summary from the file. - A whole-file row carries every record, or none for a file without a summary. An unreadable file carries null, so its task still reads the summary and fails naming the file. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_chunks.py | 26 +- .../formats/mcap/mcap_datasource_v2.py | 8 +- .../formats/mcap/mcap_file_indexer.py | 43 +- .../datasource_v2/formats/mcap/mcap_io.py | 58 ++- .../datasource_v2/formats/mcap/mcap_reader.py | 103 +++-- .../formats/mcap/mcap_summary.py | 68 ++- .../formats/mcap/mcap_task_summary.py | 235 ++++++++++ .../tests/unit/datasource_v2/test_mcap_io.py | 34 ++ .../unit/datasource_v2/test_mcap_listing.py | 12 +- .../datasource_v2/test_mcap_task_summary.py | 415 ++++++++++++++++++ 10 files changed, 933 insertions(+), 69 deletions(-) create mode 100644 python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py create mode 100644 python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 9e41b52775a4..95f7ed8d4d74 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -11,6 +11,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Dict, Iterable, Iterator, @@ -49,11 +50,14 @@ class _Declared: Starts as the summary's records and grows with those met inside chunks, so a channel declared only in an earlier chunk serves the later ones. ``scanned`` holds the offsets of the chunks already read back for records. + ``file_summary`` returns the file's whole summary, whose chunk indexes say + where to read back. """ channels: Dict[int, "Channel"] schemas: Dict[int, "Schema"] scanned: Set[int] + file_summary: Callable[[], "Summary"] def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None: @@ -90,11 +94,14 @@ def iter_chunks( path: str, summary: "Summary", offsets: Optional[Set[int]], + file_summary: Optional[Callable[[], "Summary"]] = None, ) -> Iterator[_Selected]: """Yield the selected messages of the owned chunks of an indexed file. Chunks are read in file order, or merged by log time with - ``log_time_order``. + ``log_time_order``. ``summary`` may index only the owned chunks, as a + read task's summary does (``mcap_task_summary``). Then ``file_summary`` + returns the whole summary, which reading back a declaration needs. """ selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -104,7 +111,12 @@ def iter_chunks( for c in self.candidate_chunks(summary, selected) if offsets is None or c.chunk_start_offset in offsets ] - declared = _Declared(dict(summary.channels), dict(summary.schemas), set()) + declared = _Declared( + dict(summary.channels), + dict(summary.schemas), + set(), + file_summary or (lambda: summary), + ) if not self._log_time_order: for chunk_index in chunk_indexes: yield from self._read_chunk( @@ -206,7 +218,7 @@ def _read_chunk( if isinstance(record, Message): index += 1 channel, schema = self._channel_and_schema( - f, path, summary, chunk_index, declared, record + f, path, chunk_index, declared, record ) if not self._channel_is_selected(channel, schema, summary, selected): continue @@ -224,7 +236,6 @@ def _channel_and_schema( self, f: Any, path: str, - summary: "Summary", chunk_index: "ChunkIndex", declared: _Declared, message: "Message", @@ -236,7 +247,7 @@ def _channel_and_schema( channel = declared.channels.get(message.channel_id) if channel is None: self._find_in_earlier_chunks( - f, summary, chunk_index, declared, channel_id=message.channel_id + f, chunk_index, declared, channel_id=message.channel_id ) channel = declared.channels.get(message.channel_id) if channel is None: @@ -250,7 +261,7 @@ def _channel_and_schema( # (``repeat_schemas=False``). ``message_types`` and the schema # columns need it. self._find_in_earlier_chunks( - f, summary, chunk_index, declared, schema_id=channel.schema_id + f, chunk_index, declared, schema_id=channel.schema_id ) schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None return channel, schema @@ -281,7 +292,6 @@ def _channel_is_selected( def _find_in_earlier_chunks( self, f: Any, - summary: "Summary", chunk_index: "ChunkIndex", declared: "_Declared", *, @@ -299,7 +309,7 @@ def _find_in_earlier_chunks( earlier = sorted( ( c - for c in summary.chunk_indexes + for c in declared.file_summary().chunk_indexes if c.chunk_start_offset < chunk_index.chunk_start_offset and c.chunk_start_offset not in declared.scanned ), diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index 8ae3864bd0d3..bfa699e0f8b0 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -7,6 +7,7 @@ chunks into read tasks of about ``RAY_DATA_MCAP_BIN_PACKING_BYTES`` of that estimate (128 MiB by default). When the read keeps order, ``OrderedFileGrouper`` first joins each file's chunks so that one task reads the file (``mcap_file_cuts``). +``TaskSummaryPartitioner`` adds to each task the summary records it needs. ``MCAPScanner`` holds the optimizer's pushdowns and builds the ``MCAPReader``, which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type`` (``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes. @@ -51,6 +52,9 @@ TimeRange, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + TaskSummaryPartitioner, +) from ray.data._internal.datasource_v2.interfaces.datasource_v2 import ( DatasourceCategory, FileDataSourceV2, @@ -185,7 +189,9 @@ def get_file_partitioner(self, **kwargs): max_shared_open_bins=_BIN_PACKING_MAX_SHARED_OPEN_BINS, requires_global_input=False, ) - return OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES) + return TaskSummaryPartitioner( + OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES) + ) @property @override diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index 7763f775a18e..31c873fbc851 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -11,6 +11,9 @@ A file whose summary is missing, has no chunk index or lists no channels is listed as one whole-file row. +Each row also carries the summary records its read task will need, so read +tasks do not read summaries (``mcap_task_summary``). + A file whose summary cannot be read, such as a recording cut short, does not fail the listing. It is listed as one row that names no chunk, which the reader takes as the whole file, and that weighs a whole read task, so the packer puts @@ -39,9 +42,14 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( _DEFAULT_SUMMARY_IO_CONCURRENCY, + ListedSummary, chunk_run, chunk_unit_id, - read_summaries, + read_listed_summaries, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + split_section, + with_listing_records, ) from ray.data._internal.datasource_v2.interfaces.file_indexer import FileInfo from ray.data._internal.datasource_v2.interfaces.file_manifest import ( @@ -52,7 +60,6 @@ if TYPE_CHECKING: from concurrent.futures import Future - from mcap.summary import Summary from pyarrow.fs import FileSystem from ray.data._internal.datasource_v2.interfaces.file_pruner import FilePruner @@ -135,23 +142,23 @@ def list_files( excluded = excluded_read_unit_ids or frozenset() for file_info, summary_read in self._read_summaries(file_infos, filesystem): try: - summary = summary_read.result() + listed = summary_read.result() except Exception as e: # noqa: BLE001 - the file's own read task fails manifest = self._unreadable_file_manifest(file_info, e) else: - manifest = self._manifest_for_file(file_info, summary, excluded) + manifest = self._manifest_for_file(file_info, listed, excluded) if manifest is not None: yield manifest def _read_summaries( self, file_infos: Iterable[FileInfo], filesystem: "FileSystem" - ) -> Iterator[Tuple[FileInfo, "Future[Optional[Summary]]"]]: + ) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]: """Start summary reads concurrently, yielding them in listing order. Listing order decides how the partitioner groups chunks into tasks, so it must not depend on which read finishes first. """ - return read_summaries( + return read_listed_summaries( filesystem, file_infos, lambda info: info.path, self._io_concurrency ) @@ -172,16 +179,18 @@ def _unreadable_file_manifest( size_bytes=self._read_task_bytes, fully_matched=False, ) - return FileManifest.construct_manifest( + manifest = FileManifest.construct_manifest( paths=[file_info.path], sizes=[file_info.size], chunk_metadatas=[whole_file.to_metadata()], ) + # No records: the read task reads the summary itself and fails on it. + return with_listing_records(manifest, None, [None]) def _manifest_for_file( self, file_info: FileInfo, - summary: Optional["Summary"], + listed: Optional[ListedSummary], excluded_read_unit_ids: AbstractSet[str], ) -> Optional[FileManifest]: """The listing rows of one file, or ``None`` when it cannot match. @@ -193,13 +202,18 @@ def _manifest_for_file( """ path = file_info.path assert file_info.size is not None + summary = listed.summary if listed is not None else None if summary is None or not summary.chunk_indexes or not summary.channels: # Without a summary, a chunk index or channel records, nothing says - # which chunk holds what. - return FileManifest.construct_manifest( + # which chunk holds what. The read task gets the whole summary. + manifest = FileManifest.construct_manifest( paths=[path], sizes=[file_info.size], chunk_metadatas=[None] ) + records = ( + b"" if listed is None else split_section(listed.section).everything + ) + return with_listing_records(manifest, records, [None]) selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -223,8 +237,15 @@ def _manifest_for_file( ] if not runs: return None - return FileManifest.construct_manifest( + manifest = FileManifest.construct_manifest( paths=[path] * len(runs), sizes=[file_info.size] * len(runs), chunk_metadatas=[run.to_metadata() for run in runs], ) + assert listed is not None + records = split_section(listed.section) + return with_listing_records( + manifest, + records.declarations, + [records.chunk_indexes[run.unit_ids[0]] for run in runs], + ) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index ceb02087cb0d..da1e666a773c 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -5,8 +5,11 @@ requests per chunk it indexes. Here the bytes are fetched first and parsed in memory: -- ``read_summary_from`` checks both magic bytes and the footer, then parses the - summary section, all in one to three ranged reads. +- ``read_summary_section_from`` checks both magic bytes and the footer, then + returns the summary section, all in one to three ranged reads. + ``read_summary_from`` also parses it. +- ``summary_records`` splits a section into its raw records, and + ``parse_summary_records`` parses records copied out of a section. - ``read_chunk`` fetches one chunk record in one ranged read. - ``buffered`` wraps a file for a linear scan, so it reads in large blocks. @@ -14,7 +17,7 @@ """ import io -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Iterator, Optional, Tuple, Union from ray.data._internal.util import KiB @@ -46,6 +49,16 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: recording. Errors reading ``f`` propagate unchanged, so callers can retry them. """ + section = read_summary_section_from(f, path) + return None if section is None else _parse_summary(section, path) + + +def read_summary_section_from(f: Any, path: str) -> Optional[memoryview]: + """Return the raw summary section of the open MCAP file ``f``, or ``None``. + + The section runs from the first summary record through the footer. Raises + as ``read_summary_from`` does, except that the records are not parsed. + """ size = f.seek(0, io.SEEK_END) tail_start = max(0, size - _SUMMARY_TAIL_BYTES) tail = _read_range(f, tail_start, size) @@ -62,7 +75,33 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: tail = _read_range(f, summary_start, tail_start) + tail tail_start = summary_start end = len(tail) - len(_MAGIC) - return _parse_summary(memoryview(tail)[summary_start - tail_start : end], path) + return memoryview(tail)[summary_start - tail_start : end] + + +def summary_records(section: memoryview) -> Iterator[Tuple[int, memoryview]]: + """Yield ``(opcode, record)`` for each record of ``section`` before its footer. + + ``record`` is the whole raw record: opcode, length and body. The framing is + not checked here, so split only a section that parsed. + """ + position = 0 + while position < len(section): + opcode = section[position] + if opcode == _FOOTER_OPCODE: + return + length = int.from_bytes(section[position + 1 : position + 9], "little") + end = position + _RECORD_PREFIX_BYTES + length + yield opcode, section[position:end] + position = end + + +def parse_summary_records(records: Union[bytes, memoryview], path: str) -> "Summary": + """Parse a summary section of ``path``, or records copied out of one. + + Copied records are whole raw records, as ``summary_records`` yields them, + and end without a footer. Raises as ``read_summary_from`` does. + """ + return _parse_summary(memoryview(records), path) def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": @@ -119,7 +158,7 @@ def _summary_start(tail: bytes, size: int, path: str) -> int: def _parse_summary(section: memoryview, path: str) -> "Summary": - """Collect the records of a summary section, which runs up to the footer. + """Collect the records of a summary section, up to its footer or its end. Raises ``ValueError`` naming ``path`` when the section does not parse. """ @@ -147,7 +186,10 @@ def _collect_summary_records(section: memoryview) -> "Summary": from mcap.summary import Summary summary = Summary() - for record in StreamReader(io.BytesIO(section), skip_magic=True).records: + if not section: + return summary + stream = io.BytesIO(section) + for record in StreamReader(stream, skip_magic=True).records: if isinstance(record, Footer): break if isinstance(record, Statistics): @@ -162,6 +204,10 @@ def _collect_summary_records(section: memoryview) -> "Summary": summary.attachment_indexes.append(record) elif isinstance(record, MetadataIndex): summary.metadata_indexes.append(record) + # Records copied out of a section end without a footer. The reader has + # consumed exactly the records yielded so far. + if stream.tell() == len(section): + break return summary diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index d09e6efc1560..13fecc8b5e68 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -1,11 +1,12 @@ """``MCAPReader``, which reads one read task's chunks into message rows. ``MCAPReader.read`` receives a ``FileManifest`` whose rows name files and, per -file, the byte offsets of the chunks the task owns. For each file it seeks to -those chunks, decompresses them, keeps the selected messages and yields Arrow -tables of about ``target_block_size`` bytes. With ``log_time_order``, a file's -chunks are merged by log time as they are read, so memory holds only the -chunks that overlap in time. +file, the byte offsets of the chunks the task owns and the summary records it +needs (``mcap_task_summary``). For each file it seeks to those chunks, +decompresses them, keeps the selected messages and yields Arrow tables of about +``target_block_size`` bytes. With ``log_time_order``, a file's chunks are merged +by log time as they are read, so memory holds only the chunks that overlap in +time. A row has the columns of ``message_schema``. Its optional ``row_id`` is a deterministic name for the message, built from the file path, the chunk's byte @@ -20,8 +21,9 @@ import itertools import logging +from dataclasses import dataclass from functools import partial -from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence, Set import pyarrow as pa from pyarrow.fs import FileSystem, LocalFileSystem @@ -33,6 +35,7 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + parse_summary_records, read_summary_from, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( @@ -40,6 +43,9 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import chunk_unit_id +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + task_summaries, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data._internal.datasource_v2.interfaces.read_units import ReadUnit from ray.data._internal.datasource_v2.interfaces.reader import Reader @@ -53,6 +59,9 @@ from ray.util.annotations import DeveloperAPI from ray.util.debug import log_once +if TYPE_CHECKING: + from mcap.summary import Summary + logger = logging.getLogger(__name__) @@ -127,12 +136,12 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: return retried_io_errors = DataContext.get_current().retried_io_errors remaining = self._limit - for path, offsets in _owned_chunks(input_split): + for owned in _owned_files(input_split): if remaining is not None and remaining <= 0: return tables = iterate_with_retry( - partial(self._read_file, path, offsets, remaining), - f"read MCAP file {path}", + partial(self._read_file, owned, remaining), + f"read MCAP file {owned.path}", match=retried_io_errors, ) for table in tables: @@ -143,27 +152,37 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: # -- one file ---------------------------------------------------------- def _read_file( - self, path: str, offsets: Optional[Set[int]], limit: Optional[int] + self, owned: "_OwnedFile", limit: Optional[int] ) -> Iterator[pa.Table]: - """Yield the tables of one file, limited to the chunks at ``offsets``. + """Yield the tables of one file, limited to the chunks it owns. - ``offsets`` is ``None`` for a whole-file row, which means every chunk. A - file without a chunk index is scanned linearly, through a read buffer of - ``DataContext.streaming_read_buffer_size`` bytes. At most ``limit`` rows - are yielded. + ``owned.offsets`` is ``None`` for a whole-file row, which means every + chunk. A file without a chunk index is scanned linearly, through a read + buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most + ``limit`` rows are yielded. """ from ray.data.context import DataContext + path, offsets = owned.path, owned.offsets filesystem = self._filesystem or LocalFileSystem() with filesystem.open_input_file(path) as f: - summary = read_summary_from(f, path) + if owned.summary_records is None: + summary = read_summary_from(f, path) + file_summary = None + else: + # The listing's copy, so the summary is not read. A chunk row's + # copy indexes only the owned chunks. + summary = parse_summary_records(owned.summary_records, path) + file_summary = None if offsets is None else _FileSummary(f, path) if summary is None or not summary.chunk_indexes: buffer_size = DataContext.get_current().streaming_read_buffer_size messages = self._message_reader.iter_unindexed( buffered(f, buffer_size), path, limit ) else: - messages = self._message_reader.iter_chunks(f, path, summary, offsets) + messages = self._message_reader.iter_chunks( + f, path, summary, offsets, file_summary + ) yield from self._tables(itertools.islice(messages, limit), path, offsets) # -- rows to tables ---------------------------------------------------- @@ -301,23 +320,57 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array -def _owned_chunks(manifest: FileManifest) -> List[Tuple[str, Optional[Set[int]]]]: - """Group a manifest's rows by file: ``(path, chunk offsets or None)``. +@dataclass(frozen=True) +class _OwnedFile: + """What one read task reads of one file.""" + + path: str + # Byte offsets of the owned chunks; ``None`` for the whole file. + offsets: Optional[Set[int]] + # The summary records the task needs (``mcap_task_summary``); ``None`` to + # read the summary from the file. + summary_records: Optional[bytes] + + +def _owned_files(manifest: FileManifest) -> List[_OwnedFile]: + """Group a manifest's rows by file, in manifest order. A row with chunk metadata contributes its ``unit_ids``, the chunk byte offsets. A row that names no chunk means the whole file (see ``mcap_file_indexer``), which wins over any offsets listed for the same path. + A file named by more than one row reads its summary from the file. """ - owned: Dict[str, Optional[Set[int]]] = {} - for path, metadata in zip(manifest.paths, manifest.file_chunk_metadatas): + offsets_by_path: Dict[str, Optional[Set[int]]] = {} + records_by_path: Dict[str, Optional[bytes]] = {} + rows = zip(manifest.paths, manifest.file_chunk_metadatas, task_summaries(manifest)) + for path, metadata, records in rows: path = str(path) + records_by_path[path] = None if path in records_by_path else records unit_ids = None if metadata is None else metadata.get("unit_ids") if unit_ids is None or len(unit_ids) == 0: - owned[path] = None + offsets_by_path[path] = None continue offsets = {int(i) for i in unit_ids} - current = owned.get(path, offsets) + current = offsets_by_path.get(path, offsets) if current is not None: current.update(offsets) - owned[path] = current - return list(owned.items()) + offsets_by_path[path] = current + return [ + _OwnedFile(path, offsets, records_by_path[path]) + for path, offsets in offsets_by_path.items() + ] + + +class _FileSummary: + """The whole summary of an open file, read on the first call.""" + + def __init__(self, f: Any, path: str): + self._f = f + self._path = path + self._summary: Optional["Summary"] = None + + def __call__(self) -> "Summary": + if self._summary is None: + self._summary = read_summary_from(self._f, self._path) + assert self._summary is not None, "a file with chunk rows has a summary" + return self._summary diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index 858c57fc2e95..9abb607f4296 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -5,7 +5,8 @@ log-time bounds, channels) and a ``Statistics`` record (message counts, time bounds). Reading it takes one to three ranged reads (``mcap_io``), so a read can be planned without reading any message. ``read_summaries`` reads the summaries -of many files concurrently, with retries. +of many files concurrently, with retries. The listing reads them with +``read_listed_summaries``, which keeps each raw section for the read tasks. Format specification: https://mcap.dev/spec """ @@ -13,6 +14,7 @@ import math from collections import deque from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Callable, @@ -28,7 +30,11 @@ from pyarrow.fs import FileSystem from ray._common.utils import env_integer -from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_summary_from +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + parse_summary_records, + read_summary_from, + read_summary_section_from, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileChunk from ray.data._internal.util import call_with_retry @@ -37,6 +43,7 @@ from mcap.summary import Summary _T = TypeVar("_T") +_R = TypeVar("_R") # Peak memory of reading a chunk, as a multiple of the bytes its rows take as # tables: the table being built plus the chunk being decompressed. Reads of one @@ -55,6 +62,18 @@ ) +@dataclass(frozen=True) +class ListedSummary: + """A file's summary as the listing reads it: parsed, and as raw records. + + ``section`` is the raw summary section, from which the listing copies the + records each read task needs into the manifest (``mcap_task_summary``). + """ + + summary: "Summary" + section: memoryview + + def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: """Return the summary of the MCAP file at ``path``, or ``None`` if it has none. @@ -64,12 +83,24 @@ def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: return read_summary_from(f, path) -def _read_summary_with_retry( - filesystem: "FileSystem", path: str, retried_io_errors: List[str] -) -> Optional["Summary"]: +def read_listed_summary(filesystem: FileSystem, path: str) -> Optional[ListedSummary]: + """Like ``read_summary``, but also keep the raw summary section.""" + with filesystem.open_input_file(path) as f: + section = read_summary_section_from(f, path) + if section is None: + return None + return ListedSummary(parse_summary_records(section, path), section) + + +def _read_with_retry( + read: Callable[["FileSystem", str], _R], + filesystem: "FileSystem", + path: str, + retried_io_errors: List[str], +) -> _R: """Read one file's summary, retrying errors that match ``retried_io_errors``.""" return call_with_retry( - lambda: read_summary(filesystem, path), + lambda: read(filesystem, path), description=f"read MCAP summary for {path}", match=retried_io_errors, ) @@ -87,6 +118,29 @@ def read_summaries( read means. A bounded window of in-flight reads keeps memory flat for a long listing. """ + return _read_concurrently(read_summary, filesystem, items, path, io_concurrency) + + +def read_listed_summaries( + filesystem: "FileSystem", + items: Iterable[_T], + path: Callable[[_T], str], + io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, +) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: + """``read_summaries``, keeping each raw section (``read_listed_summary``).""" + return _read_concurrently( + read_listed_summary, filesystem, items, path, io_concurrency + ) + + +def _read_concurrently( + read: Callable[["FileSystem", str], _R], + filesystem: "FileSystem", + items: Iterable[_T], + path: Callable[[_T], str], + io_concurrency: int, +) -> Iterator[Tuple[_T, "Future[_R]"]]: + """Run ``read`` on the path of each item, in a bounded window, in order.""" from ray.data.context import DataContext retried_io_errors = DataContext.get_current().retried_io_errors @@ -95,7 +149,7 @@ def read_summaries( with ThreadPoolExecutor(max_workers=max(1, io_concurrency)) as pool: for item in items: summary_read = pool.submit( - _read_summary_with_retry, filesystem, path(item), retried_io_errors + _read_with_retry, read, filesystem, path(item), retried_io_errors ) pending.append((item, summary_read)) if len(pending) >= window: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py new file mode 100644 index 000000000000..50a9ad019bda --- /dev/null +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py @@ -0,0 +1,235 @@ +"""The summary records each read task needs, carried to it in its manifest. + +A read task needs, per file, the schema and channel records and the index +records of the chunks it owns. The listing has read every summary already, so +it hands those records on, and a read task neither requests nor parses the +whole summary. They stay MCAP records, in two steps: + +1. ``MCAPSummaryIndexer`` adds two columns to its listing rows + (``with_listing_records``): each chunk row's ``ChunkIndex`` record, and the + file's records, stored once per file. For a chunk row those are the file's + schema and channel records. A whole-file row carries every record of the + summary, or none when the file has no summary. An unreadable file carries + null. +2. ``TaskSummaryPartitioner`` packs the rows with ``OnlineBinPacker``, which + keeps only paths, sizes and chunk metadata. To each packed row it adds the + file's records followed by the index records of the chunks the row names, + compressed with zstd (``TASK_SUMMARY_COLUMN``). + +A carried ``ChunkIndex`` record leaves out its ``message_index_offsets``. The +reader uses them only to pick the chunks that may hold a selected channel, and +the listing has already picked the task's chunks with the same selection. + +``MCAPReader`` parses that column (``task_summaries``). A null, or a manifest +packed some other way, makes the reader read the summary from the file. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import pyarrow as pa + +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import summary_records +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +from ray.data._internal.datasource_v2.interfaces.file_partitioner import ( + FilePartitioner, +) + +# Listing columns: the file's records, once per file, and each row's chunk. +FILE_RECORDS_COLUMN = "__mcap_file_records" +CHUNK_INDEX_COLUMN = "__mcap_chunk_index" +# Packed column: the records one read task needs for one file. +TASK_SUMMARY_COLUMN = "__mcap_task_summary" + +_FILE_RECORDS_TYPE = pa.dictionary(pa.int32(), pa.binary()) + +_SCHEMA_OPCODE = 0x03 +_CHANNEL_OPCODE = 0x04 +_CHUNK_INDEX_OPCODE = 0x08 +# A ``ChunkIndex`` record: the opcode (1 byte) and the record length (8), the +# chunk's two log times (8 each), its start offset (8) and length (8), then +# ``message_index_offsets`` as a byte count (4) and 10 bytes per channel. +_RECORD_PREFIX_BYTES = 1 + 8 +_CHUNK_START_OFFSET_AT = _RECORD_PREFIX_BYTES + 16 +_MESSAGE_INDEX_OFFSETS_AT = _RECORD_PREFIX_BYTES + 32 + + +@dataclass(frozen=True) +class SectionRecords: + """The raw records of one summary section that read tasks may need.""" + + # Every record before the footer, for a file read whole. + everything: bytes + # The schema and channel records. + declarations: bytes + # Each ``ChunkIndex`` record without its message index offsets, by its + # chunk's start offset. + chunk_indexes: Mapping[int, bytes] + + +def split_section(section: memoryview) -> SectionRecords: + """Split a summary section into the records read tasks may need.""" + everything: List[memoryview] = [] + declarations: List[memoryview] = [] + chunk_indexes: Dict[int, bytes] = {} + for opcode, record in summary_records(section): + everything.append(record) + if opcode in (_SCHEMA_OPCODE, _CHANNEL_OPCODE): + declarations.append(record) + elif opcode == _CHUNK_INDEX_OPCODE: + at = _CHUNK_START_OFFSET_AT + offset = int.from_bytes(record[at : at + 8], "little") + chunk_indexes[offset] = _without_message_index_offsets(record) + return SectionRecords( + everything=b"".join(everything), + declarations=b"".join(declarations), + chunk_indexes=chunk_indexes, + ) + + +def _without_message_index_offsets(record: memoryview) -> bytes: + """A ``ChunkIndex`` record with its ``message_index_offsets`` left empty.""" + at = _MESSAGE_INDEX_OFFSETS_AT + map_bytes = int.from_bytes(record[at : at + 4], "little") + body = b"".join( + [record[_RECORD_PREFIX_BYTES:at], bytes(4), record[at + 4 + map_bytes :]] + ) + return b"".join([record[:1], len(body).to_bytes(8, "little"), body]) + + +def with_listing_records( + manifest: FileManifest, + file_records: Optional[bytes], + chunk_index_records: Sequence[Optional[bytes]], +) -> FileManifest: + """One file's listing rows, plus the columns that carry its records. + + ``file_records`` is stored once, however many rows the file has. + ``chunk_index_records`` has one entry per row, ``None`` for a whole-file row. + """ + num_rows = len(manifest) + assert len(chunk_index_records) == num_rows + if file_records is None: + repeated = pa.nulls(num_rows, type=_FILE_RECORDS_TYPE) + else: + repeated = pa.DictionaryArray.from_arrays( + pa.array([0] * num_rows, type=pa.int32()), + pa.array([file_records], type=pa.binary()), + ) + block = _table(manifest).append_column(FILE_RECORDS_COLUMN, repeated) + block = block.append_column( + CHUNK_INDEX_COLUMN, pa.array(chunk_index_records, type=pa.binary()) + ) + return FileManifest(block) + + +def task_summaries(manifest: FileManifest) -> List[Optional[bytes]]: + """Per row of a packed manifest, the summary records its read task needs. + + ``None`` where the row has none, so the reader reads the file's summary. + """ + import zstandard + + block = _table(manifest) + if TASK_SUMMARY_COLUMN not in block.column_names: + return [None] * len(manifest) + return [ + None if records is None else zstandard.decompress(records) + for records in block.column(TASK_SUMMARY_COLUMN).to_pylist() + ] + + +def _table(manifest: FileManifest) -> pa.Table: + """The Arrow table behind ``manifest``.""" + block = manifest.as_block() + assert isinstance(block, pa.Table) + return block + + +@dataclass +class _HeldFile: + """The records of one file whose listing rows are not all packed yet.""" + + file_records: Optional[bytes] + chunk_indexes: Dict[int, bytes] = field(default_factory=dict) + unpacked_rows: int = 0 + + +class TaskSummaryPartitioner(FilePartitioner): + """Packs listing rows with ``packer`` and adds each row's summary records. + + ``packer`` is an ``OnlineBinPacker``. It drops the listing's record + columns, so this partitioner holds each file's records until its last row + leaves in a packed task. What it holds follows the packer's open bins. + """ + + def __init__(self, packer: FilePartitioner): + self._packer = packer + self._held: Dict[str, _HeldFile] = {} + + @property + def requires_global_input(self) -> bool: + return self._packer.requires_global_input + + def add_input(self, input_manifest: FileManifest) -> None: + self._hold_records(input_manifest) + self._packer.add_input(input_manifest) + + def has_partition(self) -> bool: + return self._packer.has_partition() + + def next_partition(self) -> FileManifest: + manifest = self._packer.next_partition() + records = [ + self._task_summary(str(path), metadata) + for path, metadata in zip(manifest.paths, manifest.file_chunk_metadatas) + ] + block = _table(manifest).append_column( + TASK_SUMMARY_COLUMN, pa.array(records, type=pa.binary()) + ) + return FileManifest(block) + + def finalize(self) -> None: + self._packer.finalize() + + def _hold_records(self, manifest: FileManifest) -> None: + """Keep the records of ``manifest``'s rows until they are packed.""" + block = _table(manifest) + if FILE_RECORDS_COLUMN not in block.column_names: + return + file_records = block.column(FILE_RECORDS_COLUMN) + chunk_indexes = block.column(CHUNK_INDEX_COLUMN).to_pylist() + rows = zip(manifest.paths, manifest.file_chunk_metadatas, chunk_indexes) + for i, (path, metadata, chunk_index) in enumerate(rows): + held = self._held.get(str(path)) + if held is None: + # A dictionary value: one copy per file, not per row. + held = _HeldFile(file_records[i].as_py()) + self._held[str(path)] = held + held.unpacked_rows += 1 + if chunk_index is not None: + (offset,) = metadata["unit_ids"] + held.chunk_indexes[int(offset)] = chunk_index + + def _task_summary( + self, path: str, metadata: Optional[Mapping[str, Any]] + ) -> Optional[bytes]: + """The records of one packed row, releasing them once the file is packed. + + A packed row unites the listing rows of one file in one task: one per + chunk it names, or one whole-file row. + """ + held = self._held.get(path) + if held is None: + return None + units = list(metadata["unit_ids"]) if metadata is not None else [] + held.unpacked_rows -= max(1, len(units)) + if held.unpacked_rows <= 0: + del self._held[path] + if held.file_records is None: + return None + import zstandard + + offsets = sorted({int(u) for u in units}) + records = held.file_records + b"".join(held.chunk_indexes[o] for o in offsets) + return zstandard.compress(records) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index a271b48a57e5..278bad9b19b2 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -13,8 +13,11 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + parse_summary_records, read_chunk, read_summary_from, + read_summary_section_from, + summary_records, ) from ray.data.tests.unit.datasource_v2.mcap_testing import ( CHUNKED_FILE_MESSAGES, @@ -33,6 +36,7 @@ # index of about a hundred bytes per message. MANY_CHUNKS = 3000 FOOTER_AND_MAGIC = 37 # footer record (29 bytes) and the 8 magic bytes +CHUNK_INDEX_OPCODE = 0x08 SUMMARY_FIELDS = ( "statistics", "schemas", @@ -143,6 +147,36 @@ def test_small_file_summary_takes_one_read_and_a_large_one_three(tmp_path): assert reads == {small: 1, large: 3} +@pytest.mark.parametrize("writer", WRITERS.values(), ids=WRITERS.keys()) +def test_records_copied_out_of_a_section_parse_as_the_section_does(tmp_path, writer): + """A section's records, copied without the footer, parse to the file's + summary; a run of them parses to just those records.""" + path = os.path.join(tmp_path, "file.mcap") + writer(path) + section = read_summary_section_from(CountingFile(path), path) + expected = read_summary_from(CountingFile(path), path) + assert section is not None and expected is not None + + records = list(summary_records(section)) + copied = parse_summary_records(b"".join(bytes(r) for _, r in records), path) + last_chunk_index = [bytes(r) for op, r in records if op == CHUNK_INDEX_OPCODE][-1:] + alone = parse_summary_records(b"".join(last_chunk_index), path) + + for field in SUMMARY_FIELDS: + assert getattr(copied, field) == getattr(expected, field), field + assert alone.chunk_indexes == expected.chunk_indexes[-1:] + assert not alone.channels and not alone.schemas + + +def test_no_records_parse_to_an_empty_summary(): + """Empty records, as a read task gets for a file without a summary, parse to + a summary with nothing in it.""" + summary = parse_summary_records(b"", "file.mcap") + + for field in SUMMARY_FIELDS: + assert not getattr(summary, field), field + + def test_file_without_summary_section_has_no_summary(tmp_path): """A footer whose ``summary_start`` is 0 means no summary: ``None``.""" path = os.path.join(tmp_path, "no_summary.mcap") diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index 44d6e5a6d661..dc1b9167964f 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -199,17 +199,7 @@ def test_file_without_index_is_one_whole_file_row(tmp_path): def test_summary_without_channels_is_one_whole_file_row(chunked_file, monkeypatch): """A file whose summary lists no channels is listed whole and still read.""" - from ray.data._internal.datasource_v2.formats.mcap import mcap_summary - - real_read_summary = mcap_summary.read_summary - - def without_channels(filesystem, path): - summary = real_read_summary(filesystem, path) - assert summary is not None - summary.channels = {} - return summary - - monkeypatch.setattr(mcap_summary, "read_summary", without_channels) + without_summary_records(monkeypatch, "channels") datasource = MCAPDatasourceV2([chunked_file], topics=["/a"]) (manifest,) = list_manifests(datasource) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py new file mode 100644 index 000000000000..086863b3d5e6 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py @@ -0,0 +1,415 @@ +"""Unit tests for carrying each read task's summary records in its manifest +(``mcap_task_summary``). + +The listing, the partitioner and the reader are called directly, with no Ray +cluster. A read task must read no summary and return the same rows as a task +that reads the summary from the file. +""" + +import dataclasses +import importlib.util +import os +from collections import defaultdict +from typing import Dict, List, Tuple + +import pyarrow as pa +import pytest +from pyarrow.fs import LocalFileSystem + +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_datasource_v2, + mcap_reader, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + parse_summary_records, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_options import TimeRange +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + CHUNK_INDEX_COLUMN, + FILE_RECORDS_COLUMN, + TASK_SUMMARY_COLUMN, + task_summaries, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +from ray.data.tests.unit.datasource_v2.mcap_testing import ( + BASE_TIME, + CHUNKED_FILE_MESSAGES, + STEP, + list_manifests, + pack, + round_robin_messages, + scanner_for, + summary_of, + without_summary_records, + write_cut_short, + write_mcap, + write_mcap_without_summary_schemas, + write_mixed_channels, +) + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +# A read-task budget smaller than any chunk: every chunk gets a task of its own. +ONE_CHUNK_PER_TASK = 1 + + +class ReadLog: + """A local filesystem that records ``(offset, bytes)`` of every read, per path.""" + + def __init__(self): + self._filesystem = LocalFileSystem() + self.reads: Dict[str, List[Tuple[int, int]]] = defaultdict(list) + + def open_input_file(self, path): + return _LoggedFile(self._filesystem.open_input_file(path), self.reads[path]) + + +class _LoggedFile: + def __init__(self, f, reads: List[Tuple[int, int]]): + self._f = f + self._reads = reads + + def read(self, nbytes=-1): + offset = self._f.tell() + data = self._f.read(nbytes) + if data: + self._reads.append((offset, len(data))) + return data + + def readinto(self, buffer): + offset = self._f.tell() + size = self._f.readinto(buffer) + if size: + self._reads.append((offset, size)) + return size + + def __getattr__(self, name): + return getattr(self._f, name) + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self._f.close() + + +def set_task_budget(monkeypatch, bin_bytes) -> None: + """Pack read tasks of ``bin_bytes`` estimated bytes; ``None`` keeps the default.""" + if bin_bytes is not None: + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", bin_bytes) + + +def table_of(manifest: FileManifest) -> pa.Table: + """The Arrow table behind ``manifest``.""" + block = manifest.as_block() + assert isinstance(block, pa.Table) + return block + + +def task_summary_of(task: FileManifest): + """The summary records of a one-row task.""" + (records,) = task_summaries(task) + return records + + +def without_message_index_offsets(chunk_indexes): + """``chunk_indexes`` as a task carries them: without message index offsets.""" + return [dataclasses.replace(c, message_index_offsets={}) for c in chunk_indexes] + + +def without_task_summaries(task: FileManifest) -> FileManifest: + """``task`` as a reader that reads each summary from the file sees it.""" + return FileManifest(table_of(task).drop([TASK_SUMMARY_COLUMN])) + + +def read_tasks(scanner, tasks: List[FileManifest]) -> pa.Table: + """Read every task, in order, into one table.""" + tables = [table for task in tasks for table in scanner.create_reader().read(task)] + return pa.concat_tables(tables) + + +def test_listing_rows_carry_their_chunk_and_the_files_declarations(chunked_file): + """Each listing row carries its chunk's index record, and the file's schema + and channel records are stored once for all its rows.""" + datasource = MCAPDatasourceV2([chunked_file]) + (manifest,) = list_manifests(datasource) + block = table_of(manifest) + summary = summary_of(chunked_file) + + file_records = block.column(FILE_RECORDS_COLUMN).combine_chunks() + (declarations,) = file_records.dictionary.to_pylist() + chunk_records = block.column(CHUNK_INDEX_COLUMN).to_pylist() + declared = parse_summary_records(declarations, chunked_file) + + assert file_records.indices.to_pylist() == [0] * CHUNKED_FILE_MESSAGES + assert declared.channels == summary.channels + assert declared.schemas == summary.schemas + assert not declared.chunk_indexes + assert [ + parse_summary_records(r, chunked_file).chunk_indexes[0] for r in chunk_records + ] == without_message_index_offsets(summary.chunk_indexes) + + +def test_read_tasks_read_only_their_own_chunks(chunked_file, monkeypatch): + """With a task per chunk, each task makes one read: its chunk record. Nothing + of the summary or the magic bytes is read.""" + datasource = MCAPDatasourceV2([chunked_file]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + read_log = ReadLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) + + table = read_tasks(scanner, tasks) + + chunk_records = [ + (c.chunk_start_offset, c.chunk_length) + for c in summary_of(chunked_file).chunk_indexes + ] + assert len(tasks) == CHUNKED_FILE_MESSAGES + assert sorted(read_log.reads[chunked_file]) == chunk_records + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +def test_task_summary_holds_the_declarations_and_the_owned_chunk_indexes( + chunked_file, monkeypatch +): + """A packed row's records parse to the file's channels and schemas and the + index records of exactly the chunks the row names, without their message + index offsets.""" + datasource = MCAPDatasourceV2([chunked_file]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + summary = summary_of(chunked_file) + index_by_offset = { + c.chunk_start_offset: c + for c in without_message_index_offsets(summary.chunk_indexes) + } + + for task in tasks: + (metadata,) = task.file_chunk_metadatas + records = task_summary_of(task) + assert records is not None + task_summary = parse_summary_records(records, chunked_file) + + assert task_summary.channels == summary.channels + assert task_summary.schemas == summary.schemas + assert task_summary.chunk_indexes == [ + index_by_offset[int(offset)] for offset in metadata["unit_ids"] + ] + + +def _write_overlapping(path): + # Messages out of log-time order across chunks, so a log-time merge + # interleaves the chunks of a task. + times = [10, 30, 20, 50, 40, 60, 15, 35] + write_mcap( + path, + [ + {"topic": "/a", "data": {"t": t}, "log_time": BASE_TIME + t * STEP} + for t in times + ], + chunk_size=60, + ) + + +def _write_unindexed(path): + write_mcap(path, round_robin_messages(9), index=False) + + +def _write_round_robin(path): + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + + +WRITERS = { + "one_message_chunks": _write_round_robin, + "overlapping_chunks": _write_overlapping, + "schemas_only_in_chunks": write_mcap_without_summary_schemas, + "mixed_channels": write_mixed_channels, + "unindexed": _write_unindexed, +} +READ_OPTIONS = { + "default": {}, + "topics_and_time": { + "topics": ["/a", "/c"], + "time_range": TimeRange(BASE_TIME + 2 * STEP, BASE_TIME + 30 * STEP), + }, + "message_types": {"message_types": ["other_schema"]}, + "file_order_with_row_id": {"log_time_order": False, "include_row_id": True}, +} + + +@pytest.mark.parametrize("bin_bytes", [ONE_CHUNK_PER_TASK, None]) +@pytest.mark.parametrize("options", READ_OPTIONS.values(), ids=READ_OPTIONS.keys()) +@pytest.mark.parametrize("writer", WRITERS.values(), ids=WRITERS.keys()) +def test_rows_match_reading_the_summary_from_the_file( + tmp_path, monkeypatch, writer, options, bin_bytes +): + """Tasks that parse their summary records return the same tables as the same + tasks reading each file's summary from the file.""" + path = os.path.join(tmp_path, "file.mcap") + writer(path) + datasource = MCAPDatasourceV2([path], **options) + set_task_budget(monkeypatch, bin_bytes) + tasks = pack(datasource) + scanner = scanner_for(datasource) + + with_records = [list(scanner.create_reader().read(task)) for task in tasks] + from_file = [ + list(scanner.create_reader().read(without_task_summaries(task))) + for task in tasks + ] + + assert with_records == from_file + + +def test_whole_file_rows_carry_the_whole_summary(chunked_file, monkeypatch): + """A file listed whole, here because its summary lists no channels, gets + every record of its summary and reads every chunk.""" + without_summary_records(monkeypatch, "channels") + datasource = MCAPDatasourceV2([chunked_file]) + (task,) = pack(datasource) + + records = task_summary_of(task) + table = read_tasks(scanner_for(datasource), [task]) + + assert task.file_chunk_metadatas[0] is None and records is not None + assert parse_summary_records(records, chunked_file).chunk_indexes == ( + summary_of(chunked_file).chunk_indexes + ) + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +def test_declarations_missing_from_the_summary_are_read_back(tmp_path, monkeypatch): + """Schemas declared only in chunk 0: a task owning a later chunk reads the + file's summary once to find the earlier chunks, and reads chunk 0 back. The + task owning chunk 0 reads no summary.""" + path = os.path.join(tmp_path, "file.mcap") + write_mcap_without_summary_schemas(path) + datasource = MCAPDatasourceV2([path]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + summary_reads = [] + real_read = mcap_reader.read_summary_from + + def counted_read(f, file_path): + summary_reads.append(file_path) + return real_read(f, file_path) + + monkeypatch.setattr(mcap_reader, "read_summary_from", counted_read) + scanner = scanner_for(datasource) + + per_task = [read_tasks(scanner, [task]) for task in tasks] + reads_per_task = len(summary_reads) + + assert [t.column("topic").to_pylist() for t in per_task] == [ + ["/a"], + ["/b"], + ["/a"], + ["/b"], + ["/a"], + ["/b"], + ] + assert [t.column("schema_name").to_pylist()[0] for t in per_task] == [ + "test_schema", + "other_schema", + ] * 3 + # Chunk 0 declares both schemas: its task needs no read-back. + assert reads_per_task == len(tasks) - 1 + + +def test_unindexed_file_is_scanned_without_a_summary_read(tmp_path, monkeypatch): + """A file without a summary is scanned from its start; its task reads neither + its end nor its magic bytes first.""" + path = os.path.join(tmp_path, "file.mcap") + _write_unindexed(path) + datasource = MCAPDatasourceV2([path]) + (task,) = pack(datasource) + read_log = ReadLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) + + table = read_tasks(scanner, [task]) + + # One buffered read covers the small file. + assert read_log.reads[path] == [(0, os.path.getsize(path))] + assert table.num_rows == 9 + + +def test_unreadable_file_still_fails_its_task_naming_it(tmp_path, chunked_file): + """A file whose summary could not be listed carries no records, so its task + reads the summary itself and fails naming the file.""" + cut_short = write_cut_short(os.path.join(tmp_path, "cut_short.mcap"), chunked_file) + datasource = MCAPDatasourceV2([chunked_file, cut_short]) + tasks = pack(datasource) + scanner = scanner_for(datasource) + bad = [t for t in tasks if cut_short in [str(p) for p in t.paths]] + + (task,) = bad + + assert task_summary_of(task) is None + with pytest.raises(ValueError, match="cut_short.mcap"): + read_tasks(scanner, [task]) + + +def test_a_file_named_by_two_rows_reads_its_summary_from_the_file(chunked_file): + """Two rows naming one file in one task, as a path listed twice can give, make + the reader read that file's summary instead of joining the rows' records.""" + datasource = MCAPDatasourceV2([chunked_file]) + (task,) = pack(datasource) + block = table_of(task) + twice = FileManifest(pa.concat_tables([block, block])) + + (owned,) = mcap_reader._owned_files(twice) + + assert owned.summary_records is None + assert owned.offsets == { + c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes + } + + +# Files held after each file is added: with a task per chunk every row is packed +# at once; with the default budget all three files wait in one open task. +HELD = { + "task_per_chunk": (ONE_CHUNK_PER_TASK, [0, 0, 0]), + "one_task": (None, [1, 2, 3]), +} + + +@pytest.mark.parametrize("bin_bytes, expected", HELD.values(), ids=HELD.keys()) +def test_partitioner_releases_a_file_once_its_rows_are_packed( + tmp_path, monkeypatch, bin_bytes, expected +): + """The partitioner holds a file's records only until its last row leaves in a + packed task, so it holds nothing once packing is done.""" + paths = [] + for i in range(3): + path = os.path.join(tmp_path, f"f{i}.mcap") + write_mcap(path, round_robin_messages(4)) + paths.append(path) + set_task_budget(monkeypatch, bin_bytes) + datasource = MCAPDatasourceV2(paths) + partitioner = datasource.get_file_partitioner() + held_after_each_file = [] + + for manifest in list_manifests(datasource): + partitioner.add_input(manifest) + while partitioner.has_partition(): + partitioner.next_partition() + held_after_each_file.append(len(partitioner._held)) + partitioner.finalize() + while partitioner.has_partition(): + partitioner.next_partition() + + assert held_after_each_file == expected + assert partitioner._held == {} + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) From dc87f27faf57e99ac6c5609d0e87e05da25808f7 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 6 Nov 2026 23:33:51 -0800 Subject: [PATCH 4/5] [Data] read_mcap: read each manifest row on its own; require summaries Problem: - The reader merged the manifest rows of one path. A path listed twice was read once when its copies shared a read task and twice when they did not, so the row count depended on the task budget. - The packer keeps one row per listed copy, and Parquet reads a repeated path twice. - Three fallbacks could turn off the summary handoff unnoticed: a missing task summary column, a path named by two rows, and a listing without record columns all made read tasks read summaries again. Change: - The reader reads each manifest row on its own, in manifest order. - task_summaries asserts the task summary column, and the partitioner asserts the listing record columns. A null record only marks a file the listing could not read. - Tests pack listing rows into a read task (as_read_task) instead of reading listing manifests directly. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_file_cuts.py | 6 ++- .../datasource_v2/formats/mcap/mcap_reader.py | 49 +++++++------------ .../formats/mcap/mcap_task_summary.py | 25 ++++++---- .../tests/unit/datasource_v2/mcap_testing.py | 34 ++++++++++++- .../unit/datasource_v2/test_mcap_reader.py | 17 ++++--- .../datasource_v2/test_mcap_task_summary.py | 40 +++++++++------ 6 files changed, 106 insertions(+), 65 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py index e481d98f4df6..536fd1f16554 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_cuts.py @@ -119,7 +119,11 @@ def _read_keeps_order() -> bool: def _rows_by_file(manifest: FileManifest) -> Iterator[FileManifest]: - """The manifest's rows grouped by file, in listing order.""" + """The manifest's rows grouped by file, in listing order. + + A listing manifest holds one listed file, so a path listed twice arrives as + two manifests and its copies stay apart. + """ rows_by_path: Dict[str, List[int]] = {} for i, path in enumerate(manifest.paths): rows_by_path.setdefault(str(path), []).append(i) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 13fecc8b5e68..07aea2d965c7 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -12,7 +12,7 @@ deterministic name for the message, built from the file path, the chunk's byte offset and the message's position in the chunk. -This module groups the manifest by file and finishes every table with its +This module reads the manifest row by row and finishes every table with its partition and synthesized columns. The other modules read and build the rows: - ``mcap_chunks`` reads the selected messages of a file. @@ -23,7 +23,7 @@ import logging from dataclasses import dataclass from functools import partial -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Sequence, Set +from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Set import pyarrow as pa from pyarrow.fs import FileSystem, LocalFileSystem @@ -126,9 +126,9 @@ def __init__( def read(self, input_split: FileManifest) -> Iterator[pa.Table]: """Read the files and chunks named by ``input_split``. - All rows that name one file are read together. Files come out in - manifest order. With ``limit``, reading stops at that many rows: no - later message is added to a table and no later chunk or file is read. + Each row is read on its own, in manifest order, so a path listed twice + is read twice. With ``limit``, reading stops at that many rows: no later + message is added to a table and no later chunk or file is read. """ from ray.data.context import DataContext @@ -167,6 +167,8 @@ def _read_file( filesystem = self._filesystem or LocalFileSystem() with filesystem.open_input_file(path) as f: if owned.summary_records is None: + # The listing could not read this summary. Reading it here fails + # the task with the file's own error. summary = read_summary_from(f, path) file_summary = None else: @@ -196,7 +198,7 @@ def _tables( first message is the next one needed. """ # Every table reports one read unit: the file for a whole-file read, else - # the task's first chunk. Listing excludes read units chunk by chunk + # the row's first chunk. Listing excludes read units chunk by chunk # (``excluded_read_unit_ids``), so handing this unit back would skip only # that first chunk. Nothing hands units back yet: a ``row_id`` checkpoint # filters rows by id instead. @@ -322,43 +324,30 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr @dataclass(frozen=True) class _OwnedFile: - """What one read task reads of one file.""" + """What one manifest row reads of one file.""" path: str # Byte offsets of the owned chunks; ``None`` for the whole file. offsets: Optional[Set[int]] - # The summary records the task needs (``mcap_task_summary``); ``None`` to - # read the summary from the file. + # The summary records the row needs (``mcap_task_summary``); ``None`` for a + # file whose summary the listing could not read. summary_records: Optional[bytes] def _owned_files(manifest: FileManifest) -> List[_OwnedFile]: - """Group a manifest's rows by file, in manifest order. + """One ``_OwnedFile`` per manifest row, in manifest order. - A row with chunk metadata contributes its ``unit_ids``, the chunk byte - offsets. A row that names no chunk means the whole file (see - ``mcap_file_indexer``), which wins over any offsets listed for the same path. - A file named by more than one row reads its summary from the file. + A row with chunk metadata owns its ``unit_ids``, the chunk byte offsets. A + row that names no chunk means the whole file (see ``mcap_file_indexer``). """ - offsets_by_path: Dict[str, Optional[Set[int]]] = {} - records_by_path: Dict[str, Optional[bytes]] = {} + owned = [] rows = zip(manifest.paths, manifest.file_chunk_metadatas, task_summaries(manifest)) for path, metadata, records in rows: - path = str(path) - records_by_path[path] = None if path in records_by_path else records unit_ids = None if metadata is None else metadata.get("unit_ids") - if unit_ids is None or len(unit_ids) == 0: - offsets_by_path[path] = None - continue - offsets = {int(i) for i in unit_ids} - current = offsets_by_path.get(path, offsets) - if current is not None: - current.update(offsets) - offsets_by_path[path] = current - return [ - _OwnedFile(path, offsets, records_by_path[path]) - for path, offsets in offsets_by_path.items() - ] + whole_file = unit_ids is None or len(unit_ids) == 0 + offsets = None if whole_file else {int(i) for i in unit_ids} + owned.append(_OwnedFile(str(path), offsets, records)) + return owned class _FileSummary: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py index 50a9ad019bda..a11f174014f0 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py @@ -20,8 +20,9 @@ reader uses them only to pick the chunks that may hold a selected channel, and the listing has already picked the task's chunks with the same selection. -``MCAPReader`` parses that column (``task_summaries``). A null, or a manifest -packed some other way, makes the reader read the summary from the file. +``MCAPReader`` parses that column (``task_summaries``). A null, carried by a +file the listing could not read, makes the reader read the summary from the +file, which fails the task with the file's own error. """ from dataclasses import dataclass, field @@ -126,13 +127,15 @@ def with_listing_records( def task_summaries(manifest: FileManifest) -> List[Optional[bytes]]: """Per row of a packed manifest, the summary records its read task needs. - ``None`` where the row has none, so the reader reads the file's summary. + ``None`` for a file whose summary the listing could not read. """ import zstandard block = _table(manifest) - if TASK_SUMMARY_COLUMN not in block.column_names: - return [None] * len(manifest) + assert TASK_SUMMARY_COLUMN in block.column_names, ( + f"An MCAP read task's manifest has no {TASK_SUMMARY_COLUMN!r} column. " + "Pack the listing rows with TaskSummaryPartitioner." + ) return [ None if records is None else zstandard.decompress(records) for records in block.column(TASK_SUMMARY_COLUMN).to_pylist() @@ -195,12 +198,16 @@ def finalize(self) -> None: def _hold_records(self, manifest: FileManifest) -> None: """Keep the records of ``manifest``'s rows until they are packed.""" block = _table(manifest) - if FILE_RECORDS_COLUMN not in block.column_names: - return + assert FILE_RECORDS_COLUMN in block.column_names, ( + f"MCAP listing rows have no {FILE_RECORDS_COLUMN!r} column. List " + "the files with MCAPSummaryIndexer." + ) file_records = block.column(FILE_RECORDS_COLUMN) chunk_indexes = block.column(CHUNK_INDEX_COLUMN).to_pylist() rows = zip(manifest.paths, manifest.file_chunk_metadatas, chunk_indexes) for i, (path, metadata, chunk_index) in enumerate(rows): + # A path listed twice shares one entry: its copies carry the same + # records, and every row of both counts toward the release. held = self._held.get(str(path)) if held is None: # A dictionary value: one copy per file, not per row. @@ -219,9 +226,7 @@ def _task_summary( A packed row unites the listing rows of one file in one task: one per chunk it names, or one whole-file row. """ - held = self._held.get(path) - if held is None: - return None + held = self._held[path] units = list(metadata["unit_ids"]) if metadata is not None else [] held.unpacked_rows -= max(1, len(units)) if held.unpacked_rows <= 0: diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 45924549b9a7..b108437f0ac6 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -18,11 +18,16 @@ partition_files, sample_files, ) +from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summary +from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( + TASK_SUMMARY_COLUMN, + TaskSummaryPartitioner, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest BASE_TIME = 1_000_000_000 # log time of the first message, in nanoseconds @@ -221,6 +226,23 @@ def pack(datasource, paths: Optional[List[str]] = None, **kwargs) -> List[FileMa ] +# A read-task budget no test file reaches, so packing never splits a listing. +_ONE_TASK_BYTES = 2**62 + + +def as_read_task(*listings: FileManifest) -> FileManifest: + """Pack listing rows into one read task: one row per listed file, with the + summary records the reader needs.""" + packer = OnlineBinPacker(_ONE_TASK_BYTES, requires_global_input=False) + blocks = [listing.as_block() for listing in listings] + (task,) = partition_files( + blocks, + None, # pyrefly: ignore[bad-argument-type] (TaskContext unused) + TaskSummaryPartitioner(packer), + ) + return FileManifest(task) + + def infer_schema(datasource) -> pa.Schema: """Plan the read schema from the files planning samples.""" indexer = datasource._get_file_indexer() @@ -234,10 +256,18 @@ def scanner_for(datasource): def read_all(datasource, *manifests, scanner=None) -> pa.Table: - """Read ``manifests`` into one table, with ``scanner`` or the planned one.""" + """Read ``manifests`` into one table, with ``scanner`` or the planned one. + + A packed read task is read as given. A listing manifest, which carries no + task summaries, is first packed into one read task (``as_read_task``). + """ scanner = scanner or scanner_for(datasource) reader = scanner.create_reader() - tables = [table for manifest in manifests for table in reader.read(manifest)] + tasks = [ + m if TASK_SUMMARY_COLUMN in m.as_block().column_names else as_read_task(m) + for m in manifests + ] + tables = [table for task in tasks for table in reader.read(task)] assert tables, "expected at least one table" return pa.concat_tables(tables) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index e799d049b603..8f726de23121 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -33,6 +33,7 @@ SHARED_SCHEMA, STEP, SeekLog, + as_read_task, chunks_read, infer_schema, list_manifests, @@ -377,23 +378,23 @@ def test_reader_cuts_tables_at_target_block_size(chunked_file): """A 1-byte target block size yields a table per message, and no target yields one table.""" datasource = MCAPDatasourceV2([chunked_file]) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) scanner = scanner_for(datasource) one_byte = dataclasses.replace(scanner, target_block_size=1) - tables = list(one_byte.create_reader().read(manifest)) + tables = list(one_byte.create_reader().read(task)) assert len(tables) == CHUNKED_FILE_MESSAGES assert all(t.num_rows == 1 for t in tables) unbounded = dataclasses.replace(scanner, target_block_size=None) - assert len(list(unbounded.create_reader().read(manifest))) == 1 + assert len(list(unbounded.create_reader().read(task))) == 1 def test_scanner_prunes_columns_and_pushes_limit(chunked_file): """A pruned, limited scanner reads only its columns and rows, and an empty projection still yields every row.""" datasource = MCAPDatasourceV2([chunked_file], include_paths=True) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) scanner = scanner_for(datasource) assert isinstance(scanner, MCAPScanner) @@ -401,13 +402,13 @@ def test_scanner_prunes_columns_and_pushes_limit(chunked_file): assert pruned.read_schema().names == ["log_time", "path"] assert pruned.pushed_limit() == 4 - table = pa.concat_tables(list(pruned.create_reader().read(manifest))) + table = pa.concat_tables(list(pruned.create_reader().read(task))) assert table.column_names == ["log_time", "path"] assert table.num_rows == 4 # An empty projection, as in ``count()``, keeps the row count through a stub column. empty = scanner.prune_columns([]) - table = pa.concat_tables(list(empty.create_reader().read(manifest))) + table = pa.concat_tables(list(empty.create_reader().read(task))) assert table.num_rows == CHUNKED_FILE_MESSAGES assert empty.read_schema().names == [] assert empty.prune_columns(["topic"]).read_schema().names == [] @@ -556,11 +557,11 @@ def test_reader_cuts_tables_at_the_target_counting_every_column(tmp_path): path = os.path.join(tmp_path, "mixed.mcap") write_mixed_channels(path, num_messages=400) datasource = MCAPDatasourceV2([path], include_row_id=True, include_paths=True) - (manifest,) = list_manifests(datasource) + task = as_read_task(*list_manifests(datasource)) target = 8 * 1024 scanner = dataclasses.replace(scanner_for(datasource), target_block_size=target) - tables = list(scanner.create_reader().read(manifest)) + tables = list(scanner.create_reader().read(task)) assert len(tables) > 1 # One row of about 350 bytes past the target, at most. diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py index 086863b3d5e6..1560a7b177d6 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py @@ -34,6 +34,7 @@ task_summaries, ) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +from ray.data.context import DataContext from ray.data.tests.unit.datasource_v2.mcap_testing import ( BASE_TIME, CHUNKED_FILE_MESSAGES, @@ -124,8 +125,11 @@ def without_message_index_offsets(chunk_indexes): def without_task_summaries(task: FileManifest) -> FileManifest: - """``task`` as a reader that reads each summary from the file sees it.""" - return FileManifest(table_of(task).drop([TASK_SUMMARY_COLUMN])) + """``task`` with null records, so the reader reads each summary from the file.""" + block = table_of(task) + nulls = pa.nulls(len(task), type=block.schema.field(TASK_SUMMARY_COLUMN).type) + index = block.column_names.index(TASK_SUMMARY_COLUMN) + return FileManifest(block.set_column(index, TASK_SUMMARY_COLUMN, nulls)) def read_tasks(scanner, tasks: List[FileManifest]) -> pa.Table: @@ -356,20 +360,28 @@ def test_unreadable_file_still_fails_its_task_naming_it(tmp_path, chunked_file): read_tasks(scanner, [task]) -def test_a_file_named_by_two_rows_reads_its_summary_from_the_file(chunked_file): - """Two rows naming one file in one task, as a path listed twice can give, make - the reader read that file's summary instead of joining the rows' records.""" - datasource = MCAPDatasourceV2([chunked_file]) - (task,) = pack(datasource) - block = table_of(task) - twice = FileManifest(pa.concat_tables([block, block])) +@pytest.mark.parametrize("keep_order", [False, True]) +@pytest.mark.parametrize("bin_bytes", [ONE_CHUNK_PER_TASK, None]) +def test_a_path_listed_twice_is_read_twice( + chunked_file, monkeypatch, bin_bytes, keep_order +): + """A path listed twice is read twice, as Parquet reads it, whether its copies + share a read task or not. Every packed row carries its own records.""" + options = DataContext.get_current().execution_options + monkeypatch.setattr(options, "preserve_order", keep_order) + set_task_budget(monkeypatch, bin_bytes) + datasource = MCAPDatasourceV2([chunked_file, chunked_file]) + tasks = pack(datasource) - (owned,) = mcap_reader._owned_files(twice) + table = read_tasks(scanner_for(datasource), tasks) - assert owned.summary_records is None - assert owned.offsets == { - c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes - } + seqs = [data["seq"] for data in table.column("data").to_pylist()] + one_copy = list(range(CHUNKED_FILE_MESSAGES)) + assert all(records is not None for t in tasks for records in task_summaries(t)) + if keep_order: + assert seqs == one_copy + one_copy + else: + assert sorted(seqs) == sorted(one_copy + one_copy) # Files held after each file is added: with a task per chunk every row is packed From 7d43c9511d47cbeace51c8573b76423d11e95990 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 8 Nov 2026 13:36:46 -0800 Subject: [PATCH 5/5] [Data] read_mcap: carry summary records as written; trim duplicate tests Problem: - The listing copied every record of a whole-file row's summary, and rewrote each ChunkIndex record without its message index offsets. Both need hand-written byte offsets for little gain. - Opcodes were written out as numbers that mcap already names. - read_summaries and read_listed_summaries read the same summaries. - Several tests checked what other tests already cover. Change: - A whole-file row carries the raw summary section. A chunk row carries each ChunkIndex record unchanged. - Opcodes come from mcap.opcode.Opcode, imported inside the functions. - read_summaries returns ListedSummary for every caller. - Cut or merge duplicate tests. Keep one check that a file's records are stored once, the unreadable-file test, and the empty-records test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_data_column.py | 4 +- .../formats/mcap/mcap_file_indexer.py | 8 +- .../datasource_v2/formats/mcap/mcap_io.py | 9 +- .../formats/mcap/mcap_summary.py | 45 +---- .../formats/mcap/mcap_task_summary.py | 54 ++--- .../tests/unit/datasource_v2/test_mcap_io.py | 24 --- .../datasource_v2/test_mcap_task_summary.py | 185 ++++++------------ 7 files changed, 98 insertions(+), 231 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py index d2e5331fb331..0bd9f56aa680 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py @@ -342,13 +342,13 @@ def _summaries( """ for path, summary_read in read_summaries(filesystem, paths, lambda path: path): try: - summary = summary_read.result() + listed = summary_read.result() except Exception as exc: # noqa: BLE001 - not an MCAP file if not skip_unreadable: raise logger.debug("Skipping %s for the schema: %s", path, exc) continue - yield path, summary + yield path, None if listed is None else listed.summary def _paths_past_sample(listed: Iterable[str], sampled: Set[str]) -> List[str]: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index 31c873fbc851..58310e0544c2 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -45,7 +45,7 @@ ListedSummary, chunk_run, chunk_unit_id, - read_listed_summaries, + read_summaries, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( split_section, @@ -158,7 +158,7 @@ def _read_summaries( Listing order decides how the partitioner groups chunks into tasks, so it must not depend on which read finishes first. """ - return read_listed_summaries( + return read_summaries( filesystem, file_infos, lambda info: info.path, self._io_concurrency ) @@ -210,9 +210,7 @@ def _manifest_for_file( manifest = FileManifest.construct_manifest( paths=[path], sizes=[file_info.size], chunk_metadatas=[None] ) - records = ( - b"" if listed is None else split_section(listed.section).everything - ) + records = b"" if listed is None else bytes(listed.section) return with_listing_records(manifest, records, [None]) selected = self._selection.selected_channel_ids( diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index da1e666a773c..ace3f48a94e9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -33,7 +33,6 @@ _MAGIC = b"\x89MCAP0\r\n" # A footer record is the opcode, an 8-byte length and 20 bytes of fields: # ``summary_start`` (8), ``summary_offset_start`` (8) and ``summary_crc`` (4). -_FOOTER_OPCODE = 0x02 _FOOTER_BYTES = 1 + 8 + 20 _RECORD_PREFIX_BYTES = 1 + 8 # opcode and record length @@ -84,10 +83,12 @@ def summary_records(section: memoryview) -> Iterator[Tuple[int, memoryview]]: ``record`` is the whole raw record: opcode, length and body. The framing is not checked here, so split only a section that parsed. """ + from mcap.opcode import Opcode + position = 0 while position < len(section): opcode = section[position] - if opcode == _FOOTER_OPCODE: + if opcode == Opcode.FOOTER: return length = int.from_bytes(section[position + 1 : position + 9], "little") end = position + _RECORD_PREFIX_BYTES + length @@ -135,13 +136,15 @@ def _summary_start(tail: bytes, size: int, path: str) -> int: ``tail`` is the last bytes of the file, which is ``size`` bytes long: they must end with the footer record and then the magic bytes. """ + from mcap.opcode import Opcode + footer_start = len(tail) - len(_MAGIC) - _FOOTER_BYTES footer = tail[max(0, footer_start) : len(tail) - len(_MAGIC)] length = int.from_bytes(footer[1:9], "little") if ( footer_start < 0 or not tail.endswith(_MAGIC) - or footer[0] != _FOOTER_OPCODE + or footer[0] != Opcode.FOOTER or length != _FOOTER_BYTES - _RECORD_PREFIX_BYTES ): raise ValueError( diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index 9abb607f4296..ad906b036f40 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -5,8 +5,8 @@ log-time bounds, channels) and a ``Statistics`` record (message counts, time bounds). Reading it takes one to three ranged reads (``mcap_io``), so a read can be planned without reading any message. ``read_summaries`` reads the summaries -of many files concurrently, with retries. The listing reads them with -``read_listed_summaries``, which keeps each raw section for the read tasks. +of many files concurrently, with retries, and keeps each raw section for the +listing to hand on to read tasks (``ListedSummary``). Format specification: https://mcap.dev/spec """ @@ -43,7 +43,6 @@ from mcap.summary import Summary _T = TypeVar("_T") -_R = TypeVar("_R") # Peak memory of reading a chunk, as a multiple of the bytes its rows take as # tables: the table being built plus the chunk being decompressed. Reads of one @@ -64,7 +63,7 @@ @dataclass(frozen=True) class ListedSummary: - """A file's summary as the listing reads it: parsed, and as raw records. + """A file's summary as ``read_summaries`` reads it: parsed, and as raw records. ``section`` is the raw summary section, from which the listing copies the records each read task needs into the manifest (``mcap_task_summary``). @@ -92,15 +91,12 @@ def read_listed_summary(filesystem: FileSystem, path: str) -> Optional[ListedSum return ListedSummary(parse_summary_records(section, path), section) -def _read_with_retry( - read: Callable[["FileSystem", str], _R], - filesystem: "FileSystem", - path: str, - retried_io_errors: List[str], -) -> _R: +def _read_summary_with_retry( + filesystem: "FileSystem", path: str, retried_io_errors: List[str] +) -> Optional[ListedSummary]: """Read one file's summary, retrying errors that match ``retried_io_errors``.""" return call_with_retry( - lambda: read(filesystem, path), + lambda: read_listed_summary(filesystem, path), description=f"read MCAP summary for {path}", match=retried_io_errors, ) @@ -111,36 +107,13 @@ def read_summaries( items: Iterable[_T], path: Callable[[_T], str], io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, -) -> Iterator[Tuple[_T, "Future[Optional[Summary]]"]]: +) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: """Read the summaries of ``items`` several at a time, yielding them in order. Each item comes with its pending read, so the caller decides what a failed read means. A bounded window of in-flight reads keeps memory flat for a long listing. """ - return _read_concurrently(read_summary, filesystem, items, path, io_concurrency) - - -def read_listed_summaries( - filesystem: "FileSystem", - items: Iterable[_T], - path: Callable[[_T], str], - io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, -) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: - """``read_summaries``, keeping each raw section (``read_listed_summary``).""" - return _read_concurrently( - read_listed_summary, filesystem, items, path, io_concurrency - ) - - -def _read_concurrently( - read: Callable[["FileSystem", str], _R], - filesystem: "FileSystem", - items: Iterable[_T], - path: Callable[[_T], str], - io_concurrency: int, -) -> Iterator[Tuple[_T, "Future[_R]"]]: - """Run ``read`` on the path of each item, in a bounded window, in order.""" from ray.data.context import DataContext retried_io_errors = DataContext.get_current().retried_io_errors @@ -149,7 +122,7 @@ def _read_concurrently( with ThreadPoolExecutor(max_workers=max(1, io_concurrency)) as pool: for item in items: summary_read = pool.submit( - _read_with_retry, read, filesystem, path(item), retried_io_errors + _read_summary_with_retry, filesystem, path(item), retried_io_errors ) pending.append((item, summary_read)) if len(pending) >= window: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py index a11f174014f0..5ace2ad34e10 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_task_summary.py @@ -8,18 +8,13 @@ 1. ``MCAPSummaryIndexer`` adds two columns to its listing rows (``with_listing_records``): each chunk row's ``ChunkIndex`` record, and the file's records, stored once per file. For a chunk row those are the file's - schema and channel records. A whole-file row carries every record of the - summary, or none when the file has no summary. An unreadable file carries - null. + schema and channel records. A whole-file row carries the raw summary section, + or no records when the file has no summary. An unreadable file carries null. 2. ``TaskSummaryPartitioner`` packs the rows with ``OnlineBinPacker``, which keeps only paths, sizes and chunk metadata. To each packed row it adds the file's records followed by the index records of the chunks the row names, compressed with zstd (``TASK_SUMMARY_COLUMN``). -A carried ``ChunkIndex`` record leaves out its ``message_index_offsets``. The -reader uses them only to pick the chunks that may hold a selected channel, and -the listing has already picked the task's chunks with the same selection. - ``MCAPReader`` parses that column (``task_summaries``). A null, carried by a file the listing could not read, makes the reader read the summary from the file, which fails the task with the file's own error. @@ -30,7 +25,10 @@ import pyarrow as pa -from ray.data._internal.datasource_v2.formats.mcap.mcap_io import summary_records +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + _RECORD_PREFIX_BYTES, + summary_records, +) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data._internal.datasource_v2.interfaces.file_partitioner import ( FilePartitioner, @@ -44,60 +42,40 @@ _FILE_RECORDS_TYPE = pa.dictionary(pa.int32(), pa.binary()) -_SCHEMA_OPCODE = 0x03 -_CHANNEL_OPCODE = 0x04 -_CHUNK_INDEX_OPCODE = 0x08 -# A ``ChunkIndex`` record: the opcode (1 byte) and the record length (8), the -# chunk's two log times (8 each), its start offset (8) and length (8), then -# ``message_index_offsets`` as a byte count (4) and 10 bytes per channel. -_RECORD_PREFIX_BYTES = 1 + 8 +# A ``ChunkIndex`` record starts with the opcode and the record length, then the +# chunk's two log times (8 bytes each) and its start offset (8). _CHUNK_START_OFFSET_AT = _RECORD_PREFIX_BYTES + 16 -_MESSAGE_INDEX_OFFSETS_AT = _RECORD_PREFIX_BYTES + 32 @dataclass(frozen=True) class SectionRecords: - """The raw records of one summary section that read tasks may need.""" + """The raw records of a summary section that chunk rows carry.""" - # Every record before the footer, for a file read whole. - everything: bytes # The schema and channel records. declarations: bytes - # Each ``ChunkIndex`` record without its message index offsets, by its - # chunk's start offset. + # Each ``ChunkIndex`` record, by its chunk's start offset. chunk_indexes: Mapping[int, bytes] def split_section(section: memoryview) -> SectionRecords: - """Split a summary section into the records read tasks may need.""" - everything: List[memoryview] = [] + """Split a summary section into the records that chunk rows carry.""" + from mcap.opcode import Opcode + declarations: List[memoryview] = [] chunk_indexes: Dict[int, bytes] = {} for opcode, record in summary_records(section): - everything.append(record) - if opcode in (_SCHEMA_OPCODE, _CHANNEL_OPCODE): + if opcode in (Opcode.SCHEMA, Opcode.CHANNEL): declarations.append(record) - elif opcode == _CHUNK_INDEX_OPCODE: + elif opcode == Opcode.CHUNK_INDEX: at = _CHUNK_START_OFFSET_AT offset = int.from_bytes(record[at : at + 8], "little") - chunk_indexes[offset] = _without_message_index_offsets(record) + chunk_indexes[offset] = bytes(record) return SectionRecords( - everything=b"".join(everything), declarations=b"".join(declarations), chunk_indexes=chunk_indexes, ) -def _without_message_index_offsets(record: memoryview) -> bytes: - """A ``ChunkIndex`` record with its ``message_index_offsets`` left empty.""" - at = _MESSAGE_INDEX_OFFSETS_AT - map_bytes = int.from_bytes(record[at : at + 4], "little") - body = b"".join( - [record[_RECORD_PREFIX_BYTES:at], bytes(4), record[at + 4 + map_bytes :]] - ) - return b"".join([record[:1], len(body).to_bytes(8, "little"), body]) - - def with_listing_records( manifest: FileManifest, file_records: Optional[bytes], diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index 278bad9b19b2..2f7d3d6d2ea4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -16,8 +16,6 @@ parse_summary_records, read_chunk, read_summary_from, - read_summary_section_from, - summary_records, ) from ray.data.tests.unit.datasource_v2.mcap_testing import ( CHUNKED_FILE_MESSAGES, @@ -36,7 +34,6 @@ # index of about a hundred bytes per message. MANY_CHUNKS = 3000 FOOTER_AND_MAGIC = 37 # footer record (29 bytes) and the 8 magic bytes -CHUNK_INDEX_OPCODE = 0x08 SUMMARY_FIELDS = ( "statistics", "schemas", @@ -147,27 +144,6 @@ def test_small_file_summary_takes_one_read_and_a_large_one_three(tmp_path): assert reads == {small: 1, large: 3} -@pytest.mark.parametrize("writer", WRITERS.values(), ids=WRITERS.keys()) -def test_records_copied_out_of_a_section_parse_as_the_section_does(tmp_path, writer): - """A section's records, copied without the footer, parse to the file's - summary; a run of them parses to just those records.""" - path = os.path.join(tmp_path, "file.mcap") - writer(path) - section = read_summary_section_from(CountingFile(path), path) - expected = read_summary_from(CountingFile(path), path) - assert section is not None and expected is not None - - records = list(summary_records(section)) - copied = parse_summary_records(b"".join(bytes(r) for _, r in records), path) - last_chunk_index = [bytes(r) for op, r in records if op == CHUNK_INDEX_OPCODE][-1:] - alone = parse_summary_records(b"".join(last_chunk_index), path) - - for field in SUMMARY_FIELDS: - assert getattr(copied, field) == getattr(expected, field), field - assert alone.chunk_indexes == expected.chunk_indexes[-1:] - assert not alone.channels and not alone.schemas - - def test_no_records_parse_to_an_empty_summary(): """Empty records, as a read task gets for a file without a summary, parse to a summary with nothing in it.""" diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py index 1560a7b177d6..afacea99fe1c 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_task_summary.py @@ -28,7 +28,6 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import TimeRange from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( - CHUNK_INDEX_COLUMN, FILE_RECORDS_COLUMN, TASK_SUMMARY_COLUMN, task_summaries, @@ -113,17 +112,6 @@ def table_of(manifest: FileManifest) -> pa.Table: return block -def task_summary_of(task: FileManifest): - """The summary records of a one-row task.""" - (records,) = task_summaries(task) - return records - - -def without_message_index_offsets(chunk_indexes): - """``chunk_indexes`` as a task carries them: without message index offsets.""" - return [dataclasses.replace(c, message_index_offsets={}) for c in chunk_indexes] - - def without_task_summaries(task: FileManifest) -> FileManifest: """``task`` with null records, so the reader reads each summary from the file.""" block = table_of(task) @@ -138,76 +126,6 @@ def read_tasks(scanner, tasks: List[FileManifest]) -> pa.Table: return pa.concat_tables(tables) -def test_listing_rows_carry_their_chunk_and_the_files_declarations(chunked_file): - """Each listing row carries its chunk's index record, and the file's schema - and channel records are stored once for all its rows.""" - datasource = MCAPDatasourceV2([chunked_file]) - (manifest,) = list_manifests(datasource) - block = table_of(manifest) - summary = summary_of(chunked_file) - - file_records = block.column(FILE_RECORDS_COLUMN).combine_chunks() - (declarations,) = file_records.dictionary.to_pylist() - chunk_records = block.column(CHUNK_INDEX_COLUMN).to_pylist() - declared = parse_summary_records(declarations, chunked_file) - - assert file_records.indices.to_pylist() == [0] * CHUNKED_FILE_MESSAGES - assert declared.channels == summary.channels - assert declared.schemas == summary.schemas - assert not declared.chunk_indexes - assert [ - parse_summary_records(r, chunked_file).chunk_indexes[0] for r in chunk_records - ] == without_message_index_offsets(summary.chunk_indexes) - - -def test_read_tasks_read_only_their_own_chunks(chunked_file, monkeypatch): - """With a task per chunk, each task makes one read: its chunk record. Nothing - of the summary or the magic bytes is read.""" - datasource = MCAPDatasourceV2([chunked_file]) - set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) - tasks = pack(datasource) - read_log = ReadLog() - scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) - - table = read_tasks(scanner, tasks) - - chunk_records = [ - (c.chunk_start_offset, c.chunk_length) - for c in summary_of(chunked_file).chunk_indexes - ] - assert len(tasks) == CHUNKED_FILE_MESSAGES - assert sorted(read_log.reads[chunked_file]) == chunk_records - assert table.num_rows == CHUNKED_FILE_MESSAGES - - -def test_task_summary_holds_the_declarations_and_the_owned_chunk_indexes( - chunked_file, monkeypatch -): - """A packed row's records parse to the file's channels and schemas and the - index records of exactly the chunks the row names, without their message - index offsets.""" - datasource = MCAPDatasourceV2([chunked_file]) - set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) - tasks = pack(datasource) - summary = summary_of(chunked_file) - index_by_offset = { - c.chunk_start_offset: c - for c in without_message_index_offsets(summary.chunk_indexes) - } - - for task in tasks: - (metadata,) = task.file_chunk_metadatas - records = task_summary_of(task) - assert records is not None - task_summary = parse_summary_records(records, chunked_file) - - assert task_summary.channels == summary.channels - assert task_summary.schemas == summary.schemas - assert task_summary.chunk_indexes == [ - index_by_offset[int(offset)] for offset in metadata["unit_ids"] - ] - - def _write_overlapping(path): # Messages out of log-time order across chunks, so a log-time merge # interleaves the chunks of a task. @@ -243,11 +161,53 @@ def _write_round_robin(path): "topics": ["/a", "/c"], "time_range": TimeRange(BASE_TIME + 2 * STEP, BASE_TIME + 30 * STEP), }, - "message_types": {"message_types": ["other_schema"]}, - "file_order_with_row_id": {"log_time_order": False, "include_row_id": True}, } +def chunk_records(path: str) -> List[Tuple[int, int]]: + """``(offset, bytes)`` of every chunk record of ``path``, in file order.""" + return [ + (c.chunk_start_offset, c.chunk_length) for c in summary_of(path).chunk_indexes + ] + + +def whole_file(path: str) -> List[Tuple[int, int]]: + """``(offset, bytes)`` of one read of all of ``path``.""" + return [(0, os.path.getsize(path))] + + +# Each file's writer, and the reads its tasks make with a task per chunk. +TASK_READS = { + "chunked": (_write_round_robin, chunk_records), + # A file without a chunk index is scanned from its start, in one buffered read. + "unindexed": (_write_unindexed, whole_file), +} + + +@pytest.mark.parametrize( + "writer, expected_reads", TASK_READS.values(), ids=TASK_READS.keys() +) +def test_read_tasks_read_only_their_own_chunks( + tmp_path, monkeypatch, writer, expected_reads +): + """With a task per chunk, each task makes one read: its chunk record, or the + whole file when it has no index. Nothing of the summary or the magic bytes + is read.""" + path = os.path.join(tmp_path, "file.mcap") + writer(path) + datasource = MCAPDatasourceV2([path]) + set_task_budget(monkeypatch, ONE_CHUNK_PER_TASK) + tasks = pack(datasource) + read_log = ReadLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) + + table = read_tasks(scanner, tasks) + + assert sorted(read_log.reads[path]) == expected_reads(path) + assert len(tasks) == len(expected_reads(path)) + assert table.num_rows == CHUNKED_FILE_MESSAGES + + @pytest.mark.parametrize("bin_bytes", [ONE_CHUNK_PER_TASK, None]) @pytest.mark.parametrize("options", READ_OPTIONS.values(), ids=READ_OPTIONS.keys()) @pytest.mark.parametrize("writer", WRITERS.values(), ids=WRITERS.keys()) @@ -279,7 +239,7 @@ def test_whole_file_rows_carry_the_whole_summary(chunked_file, monkeypatch): datasource = MCAPDatasourceV2([chunked_file]) (task,) = pack(datasource) - records = task_summary_of(task) + (records,) = task_summaries(task) table = read_tasks(scanner_for(datasource), [task]) assert task.file_chunk_metadatas[0] is None and records is not None @@ -291,8 +251,8 @@ def test_whole_file_rows_carry_the_whole_summary(chunked_file, monkeypatch): def test_declarations_missing_from_the_summary_are_read_back(tmp_path, monkeypatch): """Schemas declared only in chunk 0: a task owning a later chunk reads the - file's summary once to find the earlier chunks, and reads chunk 0 back. The - task owning chunk 0 reads no summary.""" + file's summary once to find chunk 0, which declares them. The task owning + chunk 0 reads no summary.""" path = os.path.join(tmp_path, "file.mcap") write_mcap_without_summary_schemas(path) datasource = MCAPDatasourceV2([path]) @@ -308,40 +268,11 @@ def counted_read(f, file_path): monkeypatch.setattr(mcap_reader, "read_summary_from", counted_read) scanner = scanner_for(datasource) - per_task = [read_tasks(scanner, [task]) for task in tasks] - reads_per_task = len(summary_reads) + for task in tasks: + read_tasks(scanner, [task]) - assert [t.column("topic").to_pylist() for t in per_task] == [ - ["/a"], - ["/b"], - ["/a"], - ["/b"], - ["/a"], - ["/b"], - ] - assert [t.column("schema_name").to_pylist()[0] for t in per_task] == [ - "test_schema", - "other_schema", - ] * 3 # Chunk 0 declares both schemas: its task needs no read-back. - assert reads_per_task == len(tasks) - 1 - - -def test_unindexed_file_is_scanned_without_a_summary_read(tmp_path, monkeypatch): - """A file without a summary is scanned from its start; its task reads neither - its end nor its magic bytes first.""" - path = os.path.join(tmp_path, "file.mcap") - _write_unindexed(path) - datasource = MCAPDatasourceV2([path]) - (task,) = pack(datasource) - read_log = ReadLog() - scanner = dataclasses.replace(scanner_for(datasource), filesystem=read_log) - - table = read_tasks(scanner, [task]) - - # One buffered read covers the small file. - assert read_log.reads[path] == [(0, os.path.getsize(path))] - assert table.num_rows == 9 + assert len(summary_reads) == len(tasks) - 1 def test_unreadable_file_still_fails_its_task_naming_it(tmp_path, chunked_file): @@ -355,7 +286,7 @@ def test_unreadable_file_still_fails_its_task_naming_it(tmp_path, chunked_file): (task,) = bad - assert task_summary_of(task) is None + assert task_summaries(task) == [None] with pytest.raises(ValueError, match="cut_short.mcap"): read_tasks(scanner, [task]) @@ -384,6 +315,8 @@ def test_a_path_listed_twice_is_read_twice( assert sorted(seqs) == sorted(one_copy + one_copy) +# Chunks of each file in the partitioner test: one per message. +CHUNKS_PER_FILE = 4 # Files held after each file is added: with a task per chunk every row is packed # at once; with the default budget all three files wait in one open task. HELD = { @@ -396,19 +329,23 @@ def test_a_path_listed_twice_is_read_twice( def test_partitioner_releases_a_file_once_its_rows_are_packed( tmp_path, monkeypatch, bin_bytes, expected ): - """The partitioner holds a file's records only until its last row leaves in a - packed task, so it holds nothing once packing is done.""" + """The listing stores a file's records once for all its rows, and the + partitioner holds them only until the file's last row leaves in a packed + task, so it holds nothing once packing is done.""" paths = [] for i in range(3): path = os.path.join(tmp_path, f"f{i}.mcap") - write_mcap(path, round_robin_messages(4)) + write_mcap(path, round_robin_messages(CHUNKS_PER_FILE)) paths.append(path) set_task_budget(monkeypatch, bin_bytes) datasource = MCAPDatasourceV2(paths) partitioner = datasource.get_file_partitioner() + record_indices = [] held_after_each_file = [] for manifest in list_manifests(datasource): + file_records = table_of(manifest).column(FILE_RECORDS_COLUMN) + record_indices.append(file_records.combine_chunks().indices.to_pylist()) partitioner.add_input(manifest) while partitioner.has_partition(): partitioner.next_partition() @@ -417,6 +354,8 @@ def test_partitioner_releases_a_file_once_its_rows_are_packed( while partitioner.has_partition(): partitioner.next_partition() + # Every row points at the one stored copy of its file's records. + assert record_indices == [[0] * CHUNKS_PER_FILE] * len(paths) assert held_after_each_file == expected assert partitioner._held == {}