From 269dc4a7051b5a6341266c0a55c48f94a2a2f148 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 09:00:49 -1000 Subject: [PATCH 01/10] [Data] read_mcap V2: read neighbouring chunks in one ranged request Problem: - A read task fetched each chunk in its own request. On object storage each request waits for a round trip, so a task of many chunks spent most of its time waiting. Change: - mcap_io.read_chunks groups chunks, in the order the reader needs them, into ranged reads (ChunkReads). A chunk joins the read before it when it starts after that read, within a gap of 1 MiB, and the read stays within 16 MiB. - A file's first read spans at most 4 MiB, and each later one may span twice as much, so a read that stops early, as under a limit, fetches little. - A read is fetched when its first chunk is needed, and its bytes are released once its last chunk is parsed. - Both the file-order path and the log-time merge read this way. The merge plans its reads in the order it expands chunks. - RAY_DATA_MCAP_CHUNK_READ_BYTES, RAY_DATA_MCAP_CHUNK_READ_GAP_BYTES and RAY_DATA_MCAP_CHUNK_READ_FIRST_BYTES set the limits. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_chunks.py | 70 +++++++--- .../datasource_v2/formats/mcap/mcap_io.py | 129 +++++++++++++++++- .../tests/unit/datasource_v2/mcap_testing.py | 21 ++- .../tests/unit/datasource_v2/test_mcap_io.py | 127 +++++++++++++++++ .../unit/datasource_v2/test_mcap_reader.py | 105 +++++++++++++- 5 files changed, 421 insertions(+), 31 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 95f7ed8d4d74..ecc07d1e559b 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -1,9 +1,11 @@ """The reading of selected messages out of one MCAP file. -An indexed file is read chunk by chunk, in file order or merged by log time. A -file without a chunk index is scanned from the start, and sorted in memory when -ordered by log time. A channel or schema that only an earlier chunk declares is -read back from that chunk. +An indexed file is read chunk by chunk, in file order or merged by log time. +Chunks that are read one after another and lie close together in the file are +fetched in one ranged read (``mcap_io.read_chunks``). A file without a chunk +index is scanned from the start, and sorted in memory when ordered by log time. +A channel or schema that only an earlier chunk declares is read back from that +chunk. """ import heapq @@ -21,7 +23,11 @@ Tuple, ) -from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_chunk +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, + read_chunk, + read_chunks, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( message_row_id, @@ -29,7 +35,7 @@ ) if TYPE_CHECKING: - from mcap.records import Channel, ChunkIndex, Message, Schema + from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema from mcap.summary import Summary # One selected message as the reader sees it: its schema (``None`` for a @@ -43,6 +49,11 @@ def _log_time(message: _Scanned) -> int: return message[2].log_time +def _merge_order(chunk_index: "ChunkIndex") -> Tuple[int, int]: + """The order in which the log-time merge expands chunks.""" + return chunk_index.message_start_time, chunk_index.chunk_start_offset + + @dataclass class _Declared: """Channel and schema records known while reading one file. @@ -81,12 +92,19 @@ class SelectedMessageReader: """Reads the messages a selection keeps out of an open MCAP file. With ``log_time_order``, an indexed file's messages come out in ascending - ``log_time`` order rather than in file order. + ``log_time`` order rather than in file order. ``chunk_reads`` decides which + chunks share a ranged read. """ - def __init__(self, selection: MCAPSelection, log_time_order: bool): + def __init__( + self, + selection: MCAPSelection, + log_time_order: bool, + chunk_reads: ChunkReads = ChunkReads(), + ): self._selection = selection self._log_time_order = log_time_order + self._chunk_reads = chunk_reads def iter_chunks( self, @@ -118,9 +136,9 @@ def iter_chunks( file_summary or (lambda: summary), ) if not self._log_time_order: - for chunk_index in chunk_indexes: - yield from self._read_chunk( - f, path, summary, chunk_index, selected, declared + for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads): + yield from self._chunk_messages( + f, path, summary, chunk_index, chunk, selected, declared ) return yield from self._merge_chunks_by_log_time( @@ -166,18 +184,32 @@ def _merge_chunks_by_log_time( the top, so only the chunks that overlap in time are held at once. A message is yielded once every chunk that could precede it is expanded. """ - # Heap entries: (log time, kind, chunk offset, index in chunk, item). - # ``kind`` 0 is a chunk index, 1 a message, so on a tied log time a - # chunk is expanded before a message is yielded. + # Heap entries: (log time, kind, chunk offset, index in chunk, message). + # ``kind`` 0 is a chunk, 1 a message, so on a tied log time a chunk is + # expanded before a message is yielded. heap: List[Tuple[int, int, int, int, Any]] = [ - (c.message_start_time, 0, c.chunk_start_offset, 0, c) for c in chunk_indexes + (c.message_start_time, 0, c.chunk_start_offset, 0, None) + for c in chunk_indexes ] heapq.heapify(heap) + # Chunks leave the heap in ``_merge_order``, so they are read in it. + chunks = read_chunks( + f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads + ) while heap: _, kind, offset, _, item = heapq.heappop(heap) if kind == 0: - for index, selected_message in self._read_chunk( - f, path, summary, item, selected, declared, with_index=True + chunk_index, chunk = next(chunks) + assert chunk_index.chunk_start_offset == offset + for index, selected_message in self._chunk_messages( + f, + path, + summary, + chunk_index, + chunk, + selected, + declared, + with_index=True, ): heapq.heappush( heap, @@ -192,12 +224,13 @@ def _merge_chunks_by_log_time( else: yield item - def _read_chunk( + def _chunk_messages( self, f: Any, path: str, summary: "Summary", chunk_index: "ChunkIndex", + chunk: "Chunk", selected: Set[int], declared: "_Declared", with_index: bool = False, @@ -212,7 +245,6 @@ def _read_chunk( from mcap.records import Channel, Message, Schema from mcap.stream_reader import breakup_chunk - chunk = read_chunk(f, chunk_index) index = -1 for record in breakup_chunk(chunk): if isinstance(record, Message): diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index ace3f48a94e9..ab3d5e1fa173 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -10,16 +10,19 @@ ``read_summary_from`` also parses it. - ``summary_records`` splits a section into its raw records, and ``parse_summary_records`` parses records copied out of a section. -- ``read_chunk`` fetches one chunk record in one ranged read. +- ``read_chunks`` fetches chunk records that lie close together in one ranged + read, as ``ChunkReads`` allows; ``read_chunk`` fetches one chunk record. - ``buffered`` wraps a file for a linear scan, so it reads in large blocks. Format specification: https://mcap.dev/spec """ import io -from typing import TYPE_CHECKING, Any, Iterator, Optional, Tuple, Union +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Iterable, Iterator, List, Optional, Tuple, Union -from ray.data._internal.util import KiB +from ray._common.utils import env_integer +from ray.data._internal.util import KiB, MiB if TYPE_CHECKING: from mcap.records import Chunk, ChunkIndex @@ -30,6 +33,16 @@ # one more read. _SUMMARY_TAIL_BYTES = 64 * KiB +# Defaults of ``ChunkReads``. A request to object storage costs a fixed delay +# (16-32 ms on S3) plus its bytes over the connection's bandwidth (about +# 64 MiB/s). Reading through a gap costs less than the request it saves while +# the gap is under delay x bandwidth: 1 MiB at 16 ms. A 16 MiB read spends about +# 90% of its time moving bytes. A first read of 4 MiB keeps a read that stops +# early, such as under a row limit, from fetching far more than it parses. +_CHUNK_READ_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_BYTES", 16 * MiB) +_CHUNK_READ_GAP_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_GAP_BYTES", 1 * MiB) +_CHUNK_READ_FIRST_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_FIRST_BYTES", 4 * MiB) + _MAGIC = b"\x89MCAP0\r\n" # A footer record is the opcode, an 8-byte length and 20 bytes of fields: # ``summary_start`` (8), ``summary_offset_start`` (8) and ``summary_crc`` (4). @@ -37,6 +50,84 @@ _RECORD_PREFIX_BYTES = 1 + 8 # opcode and record length +@dataclass(frozen=True) +class ChunkReads: + """How ``read_chunks`` groups chunk records into ranged reads. + + Attributes: + max_bytes: Most bytes one read spans, gaps included. A chunk record + longer than this is read alone, and 0 reads every chunk alone. + A read task holds one read's bytes at a time. + max_gap_bytes: Longest gap between two chunk records that one read + spans; the gap's bytes are read and dropped. Neighbouring chunks + are apart by their message index records (16 bytes per message), + and a chunk the read does not need is a gap as large as itself. + first_bytes: Most bytes the first read of a file spans. Each later + read may span twice as many as the one before, up to + ``max_bytes``, so a read that stops early fetches little. + """ + + max_bytes: int = _CHUNK_READ_BYTES + max_gap_bytes: int = _CHUNK_READ_GAP_BYTES + first_bytes: int = _CHUNK_READ_FIRST_BYTES + + +@dataclass(frozen=True) +class RangedRead: + """Bytes ``[start, end)`` of a file, which hold the ``chunk_indexes``' records.""" + + start: int + end: int + chunk_indexes: Tuple["ChunkIndex", ...] + + +def plan_chunk_reads( + chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads +) -> List[RangedRead]: + """Group chunks, taken in the order given, into ranged reads. + + A chunk joins the read before it if it starts at or after that read's end, + at most ``max_gap_bytes`` past it, and the read then spans at most its + size limit: ``first_bytes`` for the first read, then twice the previous + limit, up to ``max_bytes``. Otherwise it starts a new read. + """ + planned: List[RangedRead] = [] + group: List["ChunkIndex"] = [] + limit = min(reads.first_bytes, reads.max_bytes) + for chunk_index in chunk_indexes: + if group and not _extends(group, chunk_index, reads.max_gap_bytes, limit): + planned.append(_ranged_read(group)) + group = [] + limit = min(2 * limit, reads.max_bytes) + group.append(chunk_index) + if group: + planned.append(_ranged_read(group)) + return planned + + +def read_chunks( + f: Any, chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads +) -> Iterator[Tuple["ChunkIndex", "Chunk"]]: + """Yield the chunk record of each of ``chunk_indexes``, in that order. + + Each read of ``plan_chunk_reads`` is made when its first chunk is needed, + and its bytes are released once its last chunk is parsed. + """ + for ranged_read in plan_chunk_reads(chunk_indexes, reads): + # ``BytesIO`` shares the bytes rather than copying them; parsing a + # chunk copies only the chunk's records. + stream = io.BytesIO(_read_range(f, ranged_read.start, ranged_read.end)) + last = len(ranged_read.chunk_indexes) - 1 + for i, chunk_index in enumerate(ranged_read.chunk_indexes): + position = chunk_index.chunk_start_offset - ranged_read.start + chunk = _parse_chunk(stream, position) + if i == last: + # Free the bytes now, not when the next read replaces them, so + # two reads are never held at once. + stream.close() + yield chunk_index, chunk + + def read_summary_from(f: Any, path: str) -> Optional["Summary"]: """Return the summary of the open MCAP file ``f``, or ``None`` if it has none. @@ -107,15 +198,41 @@ def parse_summary_records(records: Union[bytes, memoryview], path: str) -> "Summ def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": """Read the chunk record that ``chunk_index`` points to, in one ranged read.""" + start = chunk_index.chunk_start_offset + stream = io.BytesIO(_read_range(f, start, start + chunk_index.chunk_length)) + return _parse_chunk(stream, 0) + + +def _parse_chunk(stream: io.BytesIO, position: int) -> "Chunk": + """Parse the chunk record that starts at ``position`` of ``stream``.""" from mcap.data_stream import ReadDataStream from mcap.records import Chunk - start = chunk_index.chunk_start_offset - stream = io.BytesIO(_read_range(f, start, start + chunk_index.chunk_length)) - stream.seek(_RECORD_PREFIX_BYTES) + stream.seek(position + _RECORD_PREFIX_BYTES) return Chunk.read(ReadDataStream(stream)) +def _record_end(chunk_index: "ChunkIndex") -> int: + """The offset just past the chunk record that ``chunk_index`` points to.""" + return chunk_index.chunk_start_offset + chunk_index.chunk_length + + +def _extends( + group: List["ChunkIndex"], chunk_index: "ChunkIndex", max_gap: int, limit: int +) -> bool: + """Whether ``chunk_index`` can join the read of ``group`` (``plan_chunk_reads``).""" + end = _record_end(group[-1]) + return ( + end <= chunk_index.chunk_start_offset <= end + max_gap + and _record_end(chunk_index) - group[0].chunk_start_offset <= limit + ) + + +def _ranged_read(group: List["ChunkIndex"]) -> RangedRead: + """The read that spans the chunk records of ``group``, in file order.""" + return RangedRead(group[0].chunk_start_offset, _record_end(group[-1]), tuple(group)) + + def buffered(f: Any, buffer_size: int) -> io.BufferedReader: """``f`` behind a read buffer of ``buffer_size`` bytes, for a linear scan. diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index b108437f0ac6..65f6a134851a 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -393,9 +393,26 @@ def __exit__(self, *exc_info): def chunks_read(seek_log: SeekLog, path: str) -> List[int]: - """Start offsets of the chunks of ``path`` that were read. + """Start offsets of the reads of chunk records in ``path``, in order. - A chunk record is read in one read from its start offset. + Each read starts at a chunk record and fetches that chunk, or a run of + chunks fetched together (``mcap_io.read_chunks``). """ starts = {c.chunk_start_offset for c in summary_of(path).chunk_indexes} return [offset for offset in seek_log.seeks[path] if offset in starts] + + +def record_parsed_chunks(monkeypatch) -> List[int]: + """Record the offset of every chunk the reader parses out of its reads.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + + real_read_chunks = mcap_chunks.read_chunks + offsets: List[int] = [] + + def read_chunks(f, chunk_indexes, reads): + for chunk_index, chunk in real_read_chunks(f, chunk_indexes, reads): + offsets.append(chunk_index.chunk_start_offset) + yield chunk_index, chunk + + monkeypatch.setattr(mcap_chunks, "read_chunks", read_chunks) + return offsets diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index 2f7d3d6d2ea4..f1b481b01ff7 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -12,9 +12,12 @@ from pyarrow.fs import LocalFileSystem from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, buffered, parse_summary_records, + plan_chunk_reads, read_chunk, + read_chunks, read_summary_from, ) from ray.data.tests.unit.datasource_v2.mcap_testing import ( @@ -231,6 +234,130 @@ def test_chunk_is_read_in_one_read(tmp_path): assert f.reads == 1 +def chunk_at(offset: int, length: int = 100): + """The ``ChunkIndex`` of a chunk record ``length`` bytes long at ``offset``.""" + from mcap.records import ChunkIndex + + return ChunkIndex( + chunk_length=length, + chunk_start_offset=offset, + compression="zstd", + compressed_size=length, + message_end_time=0, + message_index_length=0, + message_index_offsets={}, + message_start_time=0, + uncompressed_size=length, + ) + + +# Chunk records of 100 B, given by offset, in the order they are read. +PLANS = { + "neighbours_within_the_gap": ( + [0, 110, 220], + ChunkReads(max_bytes=1000, max_gap_bytes=10, first_bytes=1000), + [[0, 110, 220]], + ), + "gap_past_the_limit": ( + [0, 111], + ChunkReads(max_bytes=1000, max_gap_bytes=10, first_bytes=1000), + [[0], [111]], + ), + "span_past_max_bytes": ( + [0, 100, 200], + ChunkReads(max_bytes=250, max_gap_bytes=0, first_bytes=250), + [[0, 100], [200]], + ), + "earlier_in_the_file": ( + [200, 0, 100], + ChunkReads(max_bytes=1000, max_gap_bytes=1000, first_bytes=1000), + [[200], [0, 100]], + ), + "zero_reads_each_chunk_alone": ( + [0, 100, 200], + ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), + [[0], [100], [200]], + ), + # The limit doubles per read: 100, 200, 400, then max_bytes (500). + "first_reads_grow": ( + list(range(0, 1500, 100)), + ChunkReads(max_bytes=500, max_gap_bytes=0, first_bytes=100), + [ + [0], + [100, 200], + [300, 400, 500, 600], + [700, 800, 900, 1000, 1100], + [1200, 1300, 1400], + ], + ), +} + + +@pytest.mark.parametrize("offsets, reads, expected", PLANS.values(), ids=PLANS.keys()) +def test_plan_groups_chunks_within_the_limits(offsets, reads, expected): + """A chunk joins the read before it only if it lies after that read, within + the gap limit, and keeps the read within its size limit.""" + planned = plan_chunk_reads([chunk_at(offset) for offset in offsets], reads) + + assert [[c.chunk_start_offset for c in r.chunk_indexes] for r in planned] == ( + expected + ) + for ranged_read in planned: + assert ranged_read.start == ranged_read.chunk_indexes[0].chunk_start_offset + assert ranged_read.end == ranged_read.chunk_indexes[-1].chunk_start_offset + 100 + + +def test_a_chunk_longer_than_max_bytes_is_read_alone(): + """A chunk record longer than ``max_bytes`` is one read of its own.""" + chunks = [chunk_at(0, length=500), chunk_at(500), chunk_at(600)] + + planned = plan_chunk_reads(chunks, ChunkReads(max_bytes=250, max_gap_bytes=0)) + + assert [len(r.chunk_indexes) for r in planned] == [1, 2] + + +@pytest.mark.parametrize( + "reads, expected_reads", + [(ChunkReads(), 1), (ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), 9)], + ids=["neighbours_together", "one_per_chunk"], +) +def test_read_chunks_returns_what_read_chunk_returns(tmp_path, reads, expected_reads): + """``read_chunks`` yields the chunks ``read_chunk`` reads one by one, in the + order asked, fetching all nine in one read unless told to read each alone.""" + path = os.path.join(tmp_path, "chunked.mcap") + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + chunk_indexes = sorted(summary.chunk_indexes, key=lambda c: c.chunk_start_offset) + expected = [read_chunk(CountingFile(path), c) for c in chunk_indexes] + + f = CountingFile(path) + chunks = list(read_chunks(f, chunk_indexes, reads)) + + assert [chunk_index for chunk_index, _ in chunks] == chunk_indexes + assert [chunk for _, chunk in chunks] == expected + assert f.reads == expected_reads + + +def test_read_chunks_fetches_a_read_when_its_first_chunk_is_needed(tmp_path): + """Nothing is fetched until a chunk is asked for, and each read only when + its first chunk is.""" + path = os.path.join(tmp_path, "chunked.mcap") + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + one_per_chunk = ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0) + f = CountingFile(path) + + chunks = read_chunks(f, summary.chunk_indexes, one_per_chunk) + reads = [f.reads] + for _ in range(3): + next(chunks) + reads.append(f.reads) + + assert reads == [0, 1, 2, 3] + + def test_buffered_scan_reads_in_large_blocks(tmp_path): """A linear scan through ``buffered`` yields the same records with one read per buffer's worth of the file, and closing the buffer leaves the file open.""" diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 8f726de23121..0726a813290f 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -13,9 +13,13 @@ import pytest from ray.data._internal.datasource_v2.formats.mcap import mcap_message_rows +from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( + SelectedMessageReader, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( MCAPDatasourceV2, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ChunkReads from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( _MessageTableBuilder, message_schema, @@ -39,6 +43,7 @@ list_manifests, mixed_encoding_messages, read_all, + record_parsed_chunks, round_robin_messages, scanner_for, selected_messages, @@ -316,13 +321,17 @@ def test_file_without_a_chunk_index_is_sorted_by_its_task( @pytest.mark.parametrize("log_time_order", [True, False]) -def test_limit_reads_only_the_chunks_it_needs(chunked_file, log_time_order): - """A limit of two reads the two chunks holding the first two messages and no - other chunk, in log-time and in file order.""" +def test_limit_parses_only_the_chunks_it_needs( + chunked_file, log_time_order, monkeypatch +): + """A limit of two parses the two chunks holding the first two messages and no + other chunk, in log-time and in file order. The small file's chunks are + fetched in one ranged read.""" datasource = MCAPDatasourceV2([chunked_file], log_time_order=log_time_order) (manifest,) = list_manifests(datasource) seek_log = SeekLog() scanner = dataclasses.replace(scanner_for(datasource), filesystem=seek_log) + parsed = record_parsed_chunks(monkeypatch) table = read_all(datasource, manifest, scanner=scanner.push_limit(2)) @@ -330,7 +339,95 @@ def test_limit_reads_only_the_chunks_it_needs(chunked_file, log_time_order): offsets = sorted( c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes ) - assert chunks_read(seek_log, chunked_file) == offsets[:2] + assert parsed == offsets[:2] + assert chunks_read(seek_log, chunked_file) == offsets[:1] + + +def selected_rows(path, chunk_reads, topics=None, log_time_order=True, f=None): + """``(log_time, row_id)`` of each message the reader selects from ``path``.""" + reader = SelectedMessageReader( + MCAPSelection.create(topics, None, None), log_time_order, chunk_reads + ) + stream = f if f is not None else open(path, "rb") + with stream: + messages = reader.iter_chunks(stream, path, summary_of(path), None) + return [(message.log_time, row_id) for _, _, message, row_id in messages] + + +# One chunk per read; the defaults; reads too small for most runs; one read +# spanning the whole file, gaps and all. +CHUNK_READS = { + "one_per_chunk": ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), + "default": ChunkReads(), + "small": ChunkReads(max_bytes=400, max_gap_bytes=100, first_bytes=100), + "whole_file": ChunkReads(max_bytes=1 << 30, max_gap_bytes=1 << 30), +} + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("topics", [None, ["/a"]]) +@pytest.mark.parametrize("chunk_reads", CHUNK_READS.values(), ids=CHUNK_READS.keys()) +def test_rows_do_not_depend_on_how_chunks_are_fetched( + chunked_file, log_time_order, topics, chunk_reads +): + """The selected messages, their order and their row ids are the same whether + each chunk is fetched alone or with its neighbours, with or without the gaps + a topic filter leaves.""" + expected = selected_rows( + chunked_file, CHUNK_READS["one_per_chunk"], topics, log_time_order + ) + + rows = selected_rows(chunked_file, chunk_reads, topics, log_time_order) + + assert rows == expected + assert len(rows) == (3 if topics else CHUNKED_FILE_MESSAGES) + + +@pytest.mark.parametrize( + "max_gap_bytes, expected_reads", + # /a is in chunks 0, 3 and 6: a 1 KiB gap spans the two chunks in between. + [(0, [0, 3, 6]), (1024, [0])], +) +def test_a_gap_up_to_the_limit_is_read_through( + chunked_file, max_gap_bytes, expected_reads +): + """Chunks a topic filter selects share one read when the chunks between them + fit in ``max_gap_bytes``, and are read one by one otherwise.""" + seek_log = SeekLog() + chunk_reads = ChunkReads(max_bytes=1 << 20, max_gap_bytes=max_gap_bytes) + + selected_rows( + chunked_file, + chunk_reads, + topics=["/a"], + f=seek_log.open_input_file(chunked_file), + ) + + offsets = sorted( + c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes + ) + assert chunks_read(seek_log, chunked_file) == [offsets[i] for i in expected_reads] + + +def test_log_time_merge_fetches_chunks_in_the_order_it_expands_them(tmp_path): + """The merge expands chunks by first log time. A chunk that comes later in + time but earlier in the file starts a new read, and the rows stay in + log-time order.""" + # One message per chunk; chunk 1 holds the earliest time. + path = os.path.join(tmp_path, "out_of_order.mcap") + messages = [ + {"topic": "/a", "data": {"seq": seq}, "log_time": log_time} + for seq, log_time in enumerate((20, 10, 30)) + ] + write_mcap(path, messages) + offsets = sorted(c.chunk_start_offset for c in summary_of(path).chunk_indexes) + seek_log = SeekLog() + + rows = selected_rows(path, ChunkReads(), f=seek_log.open_input_file(path)) + + assert [log_time for log_time, _ in rows] == [10, 20, 30] + # Chunk 1 is read alone; chunk 0 then starts a read that spans chunk 2. + assert chunks_read(seek_log, path) == [offsets[1], offsets[0]] def test_limit_never_decodes_the_messages_past_it(tmp_path): From efffb6580c44b356341186ef4950e5a9b036e209 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 09:04:15 -1000 Subject: [PATCH 02/10] [Data] read_mcap V2: read 32 listing summaries at once by default Problem: - One listing task reads every summary under its path, 16 at a time. Each summary read waits on request latency, so on a folder of many small files that wait was the slow part of the read. Change: - RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY defaults to 32. Up to twice that many read summaries are held at once. Planning reads summaries with the same default. - A test checks that the listing rows come out in the same order with one summary read at a time or 32, so the packer forms the same read tasks. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_summary.py | 7 +++--- .../unit/datasource_v2/test_mcap_listing.py | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index ad906b036f40..7d197a27d1be 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -54,10 +54,11 @@ _MESSAGE_INDEX_HEADER_BYTES = 15 _MESSAGE_INDEX_ENTRY_BYTES = 16 -# Summaries read concurrently within one listing task. A summary read is a few -# small ranged requests, so it is latency-bound on remote storage. +# Summaries read concurrently within one listing task. A summary read is an +# open and one to three small ranged reads, so a listing waits on request +# latency. Up to twice this many read summaries are held at once. _DEFAULT_SUMMARY_IO_CONCURRENCY = env_integer( - "RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", 16 + "RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", 32 ) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index dc1b9167964f..963def20ab38 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -428,6 +428,30 @@ def test_packer_packs_per_listing_shard(tmp_path): assert len(per_shard) <= len(whole) + 1 +def listed_rows(datasource: MCAPDatasourceV2) -> List[Tuple[str, List[int], int]]: + """Every listing row in listing order: its path, chunk offsets and weight.""" + return [ + (str(path), unit_ids(md), int(md["size_bytes"])) + for manifest in list_manifests(datasource) + for path, md in zip(manifest.paths, manifest.file_chunk_metadatas) + ] + + +def test_listing_order_does_not_depend_on_summary_concurrency(tmp_path, monkeypatch): + """Listing 30 files with one summary read at a time or 32 at once gives the + same rows in the same order, so the packer forms the same read tasks.""" + for i in range(30): + write_mcap(os.path.join(tmp_path, f"f{i:02d}.mcap"), round_robin_messages(4)) + + monkeypatch.setenv("RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", "1") + one_at_a_time = listed_rows(MCAPDatasourceV2([str(tmp_path)])) + monkeypatch.setenv("RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", "32") + many_at_once = listed_rows(MCAPDatasourceV2([str(tmp_path)])) + + assert len(one_at_a_time) == 30 * 4 + assert many_at_once == one_at_a_time + + if __name__ == "__main__": import sys From 57475664d60ab9d74fb6b8a5e5a0f20d4f9771d6 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 09:04:55 -1000 Subject: [PATCH 03/10] [Data] read_mcap V2: count the ranged read in the peak-factor comment Problem: - The comment on _READ_PEAK_FACTOR named only the table being built and the chunk being decompressed. A read task also holds the ranged read that fetched the chunk, so one table's read peaks a little higher. Change: - The comment names the ranged read and gives the measured range. The factor stays 1.2. - Comment only, no behaviour change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../_internal/datasource_v2/formats/mcap/mcap_summary.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index 7d197a27d1be..a7eec4a5843c 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -45,8 +45,9 @@ _T = TypeVar("_T") # Peak memory of reading a chunk, as a multiple of the bytes its rows take as -# tables: the table being built plus the chunk being decompressed. Reads of one -# 128 MiB bin peaked at 1.08 to 1.16 times their tables. +# tables: the table being built, the chunk being decompressed and the ranged +# read that fetched it (``mcap_io.ChunkReads``). Reads of one 128 MiB table +# peaked at 1.14 to 1.20 times their tables. _READ_PEAK_FACTOR = 1.2 # A ``MessageIndex`` record after a chunk: opcode (1), length (8), channel id From 811692acde173958cf5656ddd04efc6337a5c91e Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 09:05:43 -1000 Subject: [PATCH 04/10] [Data] read_mcap V2: warn that override_num_blocks is ignored Problem: - The V2 datasource sizes read tasks by bytes, so it ignores override_num_blocks without saying so. Change: - read_mcap logs a warning when override_num_blocks (or parallelism) is set on the V2 path, and the docstring says so. - test_read_mcap_override_num_blocks checks the warning on V2 and its absence on the legacy path. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/read_api.py | 13 ++++++++-- python/ray/data/tests/datasource/test_mcap.py | 24 ++++++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index a3d90bee1d42..b873ea835fae 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3562,7 +3562,9 @@ def read_mcap( override_num_blocks: Override the number of output blocks from all read tasks. By default, the number of output blocks is dynamically decided based on input data size and available resources. You shouldn't manually set this - value in most cases. + value in most cases. The V2 datasource ignores it and logs a warning: it + sizes each read task by bytes instead, about 128 MiB by default + (``RAY_DATA_MCAP_BIN_PACKING_BYTES``). label_selector: Labels required on the node where each read task runs. fallback_strategy: Alternative label requirements that Ray tries in order if @@ -3608,6 +3610,13 @@ def read_mcap( ): # A checkpoint keyed on ``row_id`` needs the column in the read output. include_row_id = True + num_blocks = _get_num_output_blocks(parallelism, override_num_blocks) + if num_blocks != -1: + logger.warning( + "`read_mcap` ignores `override_num_blocks` with the V2 datasource. " + "It sizes each read task by bytes instead, about 128 MiB by " + "default (RAY_DATA_MCAP_BIN_PACKING_BYTES)." + ) datasource_v2 = MCAPDatasourceV2( paths=paths if isinstance(paths, list) else [paths], topics=topics, @@ -3626,7 +3635,7 @@ def read_mcap( ) return _read_datasource_v2( datasource_v2, - parallelism=_get_num_output_blocks(parallelism, override_num_blocks), + parallelism=num_blocks, num_cpus=num_cpus, num_gpus=num_gpus, memory=memory, diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 1c6a659e4486..a73c9f101673 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -1,5 +1,6 @@ import importlib.util import json +import logging import os import re @@ -316,9 +317,12 @@ def test_read_mcap_nonexistent_file(ray_start_regular_shared): @pytest.mark.parametrize("override_num_blocks", [1, 2]) def test_read_mcap_override_num_blocks( - ray_start_regular_shared, tmp_path, override_num_blocks + ray_start_regular_shared, tmp_path, override_num_blocks, datasource_v2, caplog ): - """Test override_num_blocks parameter.""" + """A read with ``override_num_blocks`` still returns every row. + + The V2 datasource sizes read tasks by bytes, so it ignores the setting and warns. + """ path = os.path.join(tmp_path, "blocks_test.mcap") messages = [ { @@ -330,12 +334,20 @@ def test_read_mcap_override_num_blocks( ] create_test_mcap_file(path, messages) - ds = ray.data.read_mcap(path, override_num_blocks=override_num_blocks) + # Ray's loggers do not propagate to the root logger caplog listens on. + read_api_logger = logging.getLogger("ray.data.read_api") + read_api_logger.addHandler(caplog.handler) + try: + ds = ray.data.read_mcap(path, override_num_blocks=override_num_blocks) + finally: + read_api_logger.removeHandler(caplog.handler) - # Should still read all the data assert ds.count() == 3 - rows = ds.take_all() - assert len(rows) == 3 + assert len(ds.take_all()) == 3 + warned = any( + "ignores `override_num_blocks`" in record.message for record in caplog.records + ) + assert warned == datasource_v2 def test_read_mcap_file_extensions(ray_start_regular_shared, tmp_path): From c864fac6aa4c89e15cd87e659395c0b2461b408d Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 08:28:34 -1000 Subject: [PATCH 05/10] [Data] read_mcap: refuse generated checkpoint ids Problem: - CheckpointConfig(generated_id_column="row_id") also reports id_column == "row_id". read_mcap then turns on its own text row_id, where checkpointing expects the struct ids only read_parquet builds. - With any other generated name, the write fails later with a misleading "ID column ... is absent" error. - On the V1 legacy path, CheckpointConfig(id_column="row_id") neither turned row_id on nor raised, so the read had no ids for the checkpoint. Change: - read_mcap raises InvalidCheckpointingConfig for a generated id column, on both read paths, and points to CheckpointConfig(id_column="row_id"). - row_id is turned on only for a user id_column of "row_id", on both read paths. The V1 legacy path then raises the NotImplementedError it raises for include_row_id=True. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/read_api.py | 35 ++++++++++------ python/ray/data/tests/datasource/test_mcap.py | 42 +++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index b873ea835fae..fb1025d47881 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3535,7 +3535,9 @@ def read_mcap( ``id_column`` of a :class:`~ray.data.checkpoint.CheckpointConfig`. Turned on automatically when the current checkpoint config's ``id_column`` is ``row_id``. Requires the V2 datasource - (``DataContext.use_datasource_v2``). + (``DataContext.use_datasource_v2``). ``read_mcap`` refuses + ``CheckpointConfig(generated_id_column=...)``, which only + ``read_parquet`` supports. filesystem: The PyArrow filesystem implementation to read from. parallelism: This argument is deprecated. Use ``override_num_blocks`` argument. num_cpus: The number of CPUs to reserve for each parallel read worker. @@ -3595,21 +3597,28 @@ def read_mcap( time_range = TimeRange(start_time=time_range[0], end_time=time_range[1]) ctx = DataContext.get_current() + checkpoint_config = ctx.checkpoint_config + if checkpoint_config is not None and checkpoint_config.has_generated_id_column: + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + raise InvalidCheckpointingConfig( + "`CheckpointConfig(generated_id_column=...)` works only with " + "`read_parquet`. For `read_mcap`, use " + '`CheckpointConfig(id_column="row_id")`. Its `row_id` column gives ' + "each message an id that stays the same on every run." + ) + from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( + ROW_ID_COLUMN, + ) + + if checkpoint_config is not None and checkpoint_config.id_column == ROW_ID_COLUMN: + # A checkpoint keyed on ``row_id`` needs the column in the read output. + include_row_id = True if ctx.use_datasource_v2: from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( MCAPDatasourceV2, ) - from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( - ROW_ID_COLUMN, - ) - checkpoint_config = ctx.checkpoint_config - if ( - checkpoint_config is not None - and checkpoint_config.id_column == ROW_ID_COLUMN - ): - # A checkpoint keyed on ``row_id`` needs the column in the read output. - include_row_id = True num_blocks = _get_num_output_blocks(parallelism, override_num_blocks) if num_blocks != -1: logger.warning( @@ -3652,8 +3661,8 @@ def read_mcap( if not log_time_order or include_row_id: raise NotImplementedError( - "`log_time_order=False` and `include_row_id` on `read_mcap` require " - "the V2 datasource. Enable it with " + "`log_time_order=False`, `include_row_id` and a checkpoint keyed on " + "`row_id` on `read_mcap` require the V2 datasource. Enable it with " "`ray.data.DataContext.get_current().use_datasource_v2 = True` " "(or set RAY_DATA_USE_DATASOURCE_V2=1)." ) diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index a73c9f101673..5f75df2cc949 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -660,6 +660,48 @@ def test_read_mcap_checkpoint_config_turns_on_row_id( assert "row_id" in ray.data.read_mcap(simple_mcap_file).schema().names +def test_read_mcap_v1_refuses_a_row_id_checkpoint( + ray_start_regular_shared, simple_mcap_file, datasource_v2 +): + """A checkpoint keyed on ``row_id`` needs the column, which only the V2 + datasource builds. On the V1 legacy design ``read_mcap`` refuses it at once, + instead of reading without the ids the checkpoint needs.""" + from ray.data.checkpoint import CheckpointConfig + + if datasource_v2: + pytest.skip("V2 turns row_id on") + ctx = ray.data.DataContext.get_current() + ctx.checkpoint_config = CheckpointConfig( + id_column="row_id", checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" + ) + + with pytest.raises(NotImplementedError, match="checkpoint keyed on `row_id`"): + ray.data.read_mcap(simple_mcap_file) + + +@pytest.mark.parametrize("name", ["row_id", "uid"]) +def test_read_mcap_refuses_generated_id_column( + ray_start_regular_shared, simple_mcap_file, name +): + """A checkpoint config that asks for generated ids fails at ``read_mcap``. + + Only ``read_parquet`` builds generated ids. The name ``row_id`` must not turn + on MCAP's own ``row_id`` column instead. + """ + from ray.data.checkpoint import CheckpointConfig + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + ctx = ray.data.DataContext.get_current() + ctx.checkpoint_config = CheckpointConfig( + generated_id_column=name, checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" + ) + + with pytest.raises( + InvalidCheckpointingConfig, match="works only with `read_parquet`" + ): + ray.data.read_mcap(simple_mcap_file) + + def test_read_mcap_v2_schema_matches_rows( ray_start_regular_shared, multi_topic_mcap_file, datasource_v2 ): From f58c8e81cfb5350577843eb5d4e22356a59612ff Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 11 Oct 2026 08:28:34 -1000 Subject: [PATCH 06/10] [Data] read_mcap: document the V2 datasource Problem: - read_mcap switches to the V2 datasource by default, but its docstring did not say so, or how its rows differ. - The use_datasource_v2 docstring did not name MCAP. Change: - The read_mcap docstring lists the three differences from the legacy reader, how to switch back, and an example that stores schema_data as plain bytes. - The use_datasource_v2 docstring names MCAP. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/context.py | 8 ++++---- python/ray/data/read_api.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 77f90993d929..20f7c3f3a744 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -718,10 +718,10 @@ class DataContext: min_parallelism: This setting is deprecated. Use ``read_op_min_num_blocks`` instead. read_op_min_num_blocks: Minimum number of read output blocks for a dataset. - use_datasource_v2: When True, ``ray.data.read_parquet()`` and - ``ray.data.read_orc()`` route through the DataSourceV2 pipeline. - Defaults to True; override with ``RAY_DATA_USE_DATASOURCE_V2`` - (``0`` for V1, ``1`` for V2). + use_datasource_v2: When True, ``ray.data.read_parquet()``, + ``ray.data.read_orc()`` and ``ray.data.read_mcap()`` route through + the DataSourceV2 pipeline. Defaults to True; override with + ``RAY_DATA_USE_DATASOURCE_V2`` (``0`` for V1, ``1`` for V2). small_shuffle_chunk_size: Target uncompressed size in bytes of the row-group runs that ``FileShuffleConfig(small_chunks_shuffle=True)`` shuffles. Contiguous row groups are coalesced up to this size; a single row group diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index fb1025d47881..f23f58bb1e2f 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3451,6 +3451,26 @@ def read_mcap( the selected messages of the first chunk that holds one. A field that first appears in a later chunk or file is missing from that schema. + By default, ``read_mcap`` reads through the V2 datasource + (``DataContext.use_datasource_v2``). Its rows differ from the legacy reader's + in three ways: + + - ``schema_data`` and the new ``channel_metadata`` column are + dictionary-encoded, so a block stores each channel's values once. pandas + shows them as category columns. Cast them to ``binary`` and ``string`` + before you group by them or write blocks from several read tasks into one + Arrow IPC file. Cast ``schema_data`` before you mix this output with the + legacy reader's in one Parquet folder. + - A large file is split across read tasks. Its messages keep ``log_time`` + order across the whole file only with + ``DataContext.execution_options.preserve_order``, and the files of one + directory can come out in a different order than listed. + - ``override_num_blocks`` has no effect. + + To use the legacy reader, set ``DataContext.get_current().use_datasource_v2`` + to ``False`` or set ``RAY_DATA_USE_DATASOURCE_V2=0``. This also moves + ``read_parquet`` back to its legacy reader. + Examples: :noindex: @@ -3494,6 +3514,14 @@ def read_mcap( ... include_paths=True # doctest: +SKIP ... ) # doctest: +SKIP + Store ``schema_data`` as plain bytes, as the legacy reader does. + + >>> import pyarrow as pa # doctest: +SKIP + >>> def plain_schema_data(t: pa.Table) -> pa.Table: # doctest: +SKIP + ... i = t.schema.get_field_index("schema_data") # doctest: +SKIP + ... return t.set_column(i, "schema_data", t["schema_data"].cast(pa.binary())) # doctest: +SKIP + >>> ds = ds.map_batches(plain_schema_data, batch_format="pyarrow") # doctest: +SKIP + Args: paths: A single file or directory, or a list of file or directory paths. A list of paths can contain both files and directories. From 2af168c77fdc8e7e9115e390dc989baf2be51745 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 6 Nov 2026 23:28:12 -0800 Subject: [PATCH 07/10] [Data] read_mcap V2: name the argument the read ignores Problem: - The V2 warning names override_num_blocks even when the caller passed parallelism. - It hard-codes 128 MiB instead of the configured budget. Change: - Name the argument the caller gave. - Print the budget from RAY_DATA_MCAP_BIN_PACKING_BYTES. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/read_api.py | 11 ++++++--- python/ray/data/tests/datasource/test_mcap.py | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index f23f58bb1e2f..749a713af79e 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3644,15 +3644,20 @@ def read_mcap( include_row_id = True if ctx.use_datasource_v2: from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + _BIN_PACKING_BYTES, MCAPDatasourceV2, ) num_blocks = _get_num_output_blocks(parallelism, override_num_blocks) if num_blocks != -1: + given = ( + "parallelism" if override_num_blocks is None else "override_num_blocks" + ) logger.warning( - "`read_mcap` ignores `override_num_blocks` with the V2 datasource. " - "It sizes each read task by bytes instead, about 128 MiB by " - "default (RAY_DATA_MCAP_BIN_PACKING_BYTES)." + f"`read_mcap` ignores `{given}` with the V2 datasource. It sizes " + f"each read task by bytes instead, about " + f"{_BIN_PACKING_BYTES / 2**20:g} MiB " + "(RAY_DATA_MCAP_BIN_PACKING_BYTES)." ) datasource_v2 = MCAPDatasourceV2( paths=paths if isinstance(paths, list) else [paths], diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 5f75df2cc949..cd80766cd2ba 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -350,6 +350,30 @@ def test_read_mcap_override_num_blocks( assert warned == datasource_v2 +def test_read_mcap_warning_names_parallelism( + ray_start_regular_shared, tmp_path, datasource_v2, caplog +): + """The V2 datasource's warning names ``parallelism`` when that is the argument + given, rather than ``override_num_blocks``. + """ + path = os.path.join(tmp_path, "parallelism_test.mcap") + create_test_mcap_file( + path, [{"topic": "/test", "data": {"seq": 0}, "log_time": 1000000000}] + ) + + read_api_logger = logging.getLogger("ray.data.read_api") + read_api_logger.addHandler(caplog.handler) + try: + ray.data.read_mcap(path, parallelism=2) + finally: + read_api_logger.removeHandler(caplog.handler) + + messages = [record.message for record in caplog.records] + warned = any("ignores `parallelism`" in m for m in messages) + assert warned == datasource_v2 + assert not any("ignores `override_num_blocks`" in m for m in messages) + + def test_read_mcap_file_extensions(ray_start_regular_shared, tmp_path): """Test file extension filtering.""" # Create MCAP file From 5dd1bc1109d204ecf07792ca46c808540f509d56 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 8 Nov 2026 13:17:51 -0800 Subject: [PATCH 08/10] [Data] read_mcap: trim duplicate ranged-read and warning tests Problem: - The chunk-fetch test ran four read configs, but one is its own baseline and two make the same single read on the test file. - The parallelism warning had its own test beside the override_num_blocks test, which varied only the value. - The generated-id test ran two names that hit the same raise. Change: - Compare one chunk per read with small reads only. - Parametrize the override_num_blocks test over the argument name and drop the separate parallelism test. - Check the generated-id raise with the name row_id only. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/tests/datasource/test_mcap.py | 53 ++++++------------- .../unit/datasource_v2/test_mcap_reader.py | 25 ++++----- 2 files changed, 26 insertions(+), 52 deletions(-) diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index cd80766cd2ba..5801da65beea 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -3,6 +3,7 @@ import logging import os import re +from typing import Any import pyarrow as pa import pytest @@ -315,13 +316,15 @@ def test_read_mcap_nonexistent_file(ray_start_regular_shared): ds.materialize() # Force execution -@pytest.mark.parametrize("override_num_blocks", [1, 2]) +@pytest.mark.parametrize("argument", ["override_num_blocks", "parallelism"]) def test_read_mcap_override_num_blocks( - ray_start_regular_shared, tmp_path, override_num_blocks, datasource_v2, caplog + ray_start_regular_shared, tmp_path, argument, datasource_v2, caplog ): - """A read with ``override_num_blocks`` still returns every row. + """A read given ``override_num_blocks`` or ``parallelism`` still returns every + row. - The V2 datasource sizes read tasks by bytes, so it ignores the setting and warns. + The V2 datasource sizes read tasks by bytes, so it ignores the argument and + warns, naming the argument given. """ path = os.path.join(tmp_path, "blocks_test.mcap") messages = [ @@ -333,45 +336,24 @@ def test_read_mcap_override_num_blocks( for i in range(3) ] create_test_mcap_file(path, messages) + read_args: dict[str, Any] = {argument: 2} + other = ( + "parallelism" if argument == "override_num_blocks" else "override_num_blocks" + ) # Ray's loggers do not propagate to the root logger caplog listens on. read_api_logger = logging.getLogger("ray.data.read_api") read_api_logger.addHandler(caplog.handler) try: - ds = ray.data.read_mcap(path, override_num_blocks=override_num_blocks) + ds = ray.data.read_mcap(path, **read_args) finally: read_api_logger.removeHandler(caplog.handler) assert ds.count() == 3 assert len(ds.take_all()) == 3 - warned = any( - "ignores `override_num_blocks`" in record.message for record in caplog.records - ) - assert warned == datasource_v2 - - -def test_read_mcap_warning_names_parallelism( - ray_start_regular_shared, tmp_path, datasource_v2, caplog -): - """The V2 datasource's warning names ``parallelism`` when that is the argument - given, rather than ``override_num_blocks``. - """ - path = os.path.join(tmp_path, "parallelism_test.mcap") - create_test_mcap_file( - path, [{"topic": "/test", "data": {"seq": 0}, "log_time": 1000000000}] - ) - - read_api_logger = logging.getLogger("ray.data.read_api") - read_api_logger.addHandler(caplog.handler) - try: - ray.data.read_mcap(path, parallelism=2) - finally: - read_api_logger.removeHandler(caplog.handler) - - messages = [record.message for record in caplog.records] - warned = any("ignores `parallelism`" in m for m in messages) - assert warned == datasource_v2 - assert not any("ignores `override_num_blocks`" in m for m in messages) + warnings = [record.message for record in caplog.records] + assert any(f"ignores `{argument}`" in m for m in warnings) == datasource_v2 + assert not any(f"ignores `{other}`" in m for m in warnings) def test_read_mcap_file_extensions(ray_start_regular_shared, tmp_path): @@ -703,9 +685,8 @@ def test_read_mcap_v1_refuses_a_row_id_checkpoint( ray.data.read_mcap(simple_mcap_file) -@pytest.mark.parametrize("name", ["row_id", "uid"]) def test_read_mcap_refuses_generated_id_column( - ray_start_regular_shared, simple_mcap_file, name + ray_start_regular_shared, simple_mcap_file ): """A checkpoint config that asks for generated ids fails at ``read_mcap``. @@ -717,7 +698,7 @@ def test_read_mcap_refuses_generated_id_column( ctx = ray.data.DataContext.get_current() ctx.checkpoint_config = CheckpointConfig( - generated_id_column=name, checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" + generated_id_column="row_id", checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" ) with pytest.raises( diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 0726a813290f..c90fd47d3dbf 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -354,30 +354,23 @@ def selected_rows(path, chunk_reads, topics=None, log_time_order=True, f=None): return [(message.log_time, row_id) for _, _, message, row_id in messages] -# One chunk per read; the defaults; reads too small for most runs; one read -# spanning the whole file, gaps and all. -CHUNK_READS = { - "one_per_chunk": ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), - "default": ChunkReads(), - "small": ChunkReads(max_bytes=400, max_gap_bytes=100, first_bytes=100), - "whole_file": ChunkReads(max_bytes=1 << 30, max_gap_bytes=1 << 30), -} +ONE_CHUNK_PER_READ = ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0) +# Small enough that the ``chunked_file`` fixture takes several reads, some of +# them spanning more than one chunk. +SMALL_READS = ChunkReads(max_bytes=400, max_gap_bytes=100, first_bytes=100) @pytest.mark.parametrize("log_time_order", [True, False]) @pytest.mark.parametrize("topics", [None, ["/a"]]) -@pytest.mark.parametrize("chunk_reads", CHUNK_READS.values(), ids=CHUNK_READS.keys()) def test_rows_do_not_depend_on_how_chunks_are_fetched( - chunked_file, log_time_order, topics, chunk_reads + chunked_file, log_time_order, topics ): """The selected messages, their order and their row ids are the same whether - each chunk is fetched alone or with its neighbours, with or without the gaps - a topic filter leaves.""" - expected = selected_rows( - chunked_file, CHUNK_READS["one_per_chunk"], topics, log_time_order - ) + each chunk is fetched alone or with its neighbours, with or without a topic + filter.""" + expected = selected_rows(chunked_file, ONE_CHUNK_PER_READ, topics, log_time_order) - rows = selected_rows(chunked_file, chunk_reads, topics, log_time_order) + rows = selected_rows(chunked_file, SMALL_READS, topics, log_time_order) assert rows == expected assert len(rows) == (3 if topics else CHUNKED_FILE_MESSAGES) From c51dcca7a9fa79b2c3ec9953ef5ecc4b74986b52 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 8 Nov 2026 18:28:21 -0800 Subject: [PATCH 09/10] [Data] read_mcap: bound listing by bytes, estimate peak from chunks Problem: - Listing threads read and parsed summaries, and up to twice the summary concurrency of parsed summaries were held at once. Parsed summaries are much larger than their bytes, and nothing bounded their size. Parsing holds the GIL, so the threads gained nothing from it. - A read task's peak was estimated as 1.2 times its table. That misses the chunk being decompressed and parsed, the log-time heap and the ranged read, so a task of large chunks or small messages peaked at up to 2.6 times its estimate. Change: - Summary reads are split in two: a tail read, whose footer gives the section's exact length, and a section read. Listing threads only fetch bytes, and the caller parses each section in order. - A section is fetched once it fits the read task budget (RAY_DATA_MCAP_BIN_PACKING_BYTES) with the sections held. With nothing held, the next one is fetched whatever its size. A failed read is still that file's own error. - A read task's peak is estimated as its table, plus the largest working set of one chunk, plus one ranged read. Each chunk weighs its table scaled so that a full task leaves room for the rest. - New tests pin the estimate's constants by measurement, check the estimate against a measured read, and check the byte budget of summary reads. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_data_column.py | 38 ++- .../formats/mcap/mcap_datasource_v2.py | 5 +- .../formats/mcap/mcap_file_indexer.py | 41 ++- .../datasource_v2/formats/mcap/mcap_io.py | 73 +++- .../formats/mcap/mcap_summary.py | 323 ++++++++++++++---- .../tests/unit/datasource_v2/mcap_testing.py | 4 +- .../datasource_v2/test_mcap_data_column.py | 7 +- .../unit/datasource_v2/test_mcap_listing.py | 118 ++++++- .../datasource_v2/test_mcap_read_memory.py | 204 +++++++++++ 9 files changed, 691 insertions(+), 122 deletions(-) create mode 100644 python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py index 0bd9f56aa680..4ab167d62616 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py @@ -47,7 +47,10 @@ decode_payload, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection -from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summaries +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + parse_listed_summary, + read_summaries, +) from ray.data._internal.tensor_extensions.arrow import convert_to_pyarrow_array from ray.data.context import DataContext from ray.util.debug import log_once @@ -90,6 +93,7 @@ def infer_data_type( filesystem: "FileSystem", sampled_paths: Sequence[str], list_paths: Callable[[], Iterable[str]], + summary_budget_bytes: int, ) -> Optional[pa.DataType]: """Type of ``data`` when it holds decoded JSON; ``None`` when it is ``binary``. @@ -102,11 +106,12 @@ def infer_data_type( sampled_paths: The files sampled for the schema, in listing order. list_paths: Lists the read's files in listing order, with its extension filter. Called only when no sampled file types ``data``. + summary_budget_bytes: Most bytes of summary sections held at once. Returns: The type of the sampled JSON values, or ``None`` for ``binary``. """ - inspector = _FileInspector(selection, filesystem) + inspector = _FileInspector(selection, filesystem, summary_budget_bytes) signature = inspector.inspect_sample(sampled_paths) if signature.has_non_json or signature.value_type is not None: return signature.value_type @@ -130,9 +135,15 @@ class _FileInspector: Scans at most ``_MAX_SCANNED_FILES`` files whole. """ - def __init__(self, selection: MCAPSelection, filesystem: "FileSystem"): + def __init__( + self, + selection: MCAPSelection, + filesystem: "FileSystem", + summary_budget_bytes: int, + ): self._selection = selection self._filesystem = filesystem + self._summary_budget_bytes = summary_budget_bytes self._scans_left = _MAX_SCANNED_FILES def inspect_sample(self, paths: Sequence[str]) -> _DataSignature: @@ -145,7 +156,10 @@ def inspect_sample(self, paths: Sequence[str]) -> _DataSignature: the read task that owns it reports the error. """ value_type: Optional[pa.DataType] = None - for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True): + summaries = _summaries( + self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True + ) + for path, summary in summaries: signature = self._inspect( path, summary, want_type=value_type is None, may_scan=True ) @@ -161,7 +175,10 @@ def walk(self, paths: List[str]) -> _DataSignature: Only files whose summary settles the selection are inspected, so no file is scanned whole. Files whose summary cannot be read are skipped. """ - for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True): + summaries = _summaries( + self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True + ) + for path, summary in summaries: signature = self._inspect(path, summary, want_type=True, may_scan=False) if signature.has_non_json or signature.value_type is not None: return signature @@ -333,16 +350,21 @@ def _readable_records(records: Iterable[Any], path: str) -> Iterator[Any]: def _summaries( - filesystem: "FileSystem", paths: Iterable[str], *, skip_unreadable: bool + filesystem: "FileSystem", + paths: Iterable[str], + budget_bytes: int, + *, + skip_unreadable: bool, ) -> Iterator[Tuple[str, Optional["Summary"]]]: """Read the summaries of ``paths`` several at a time, yielding them in order. With ``skip_unreadable``, a file whose summary cannot be read is skipped. Otherwise its error is raised. """ - for path, summary_read in read_summaries(filesystem, paths, lambda path: path): + sections = read_summaries(filesystem, paths, lambda path: path, budget_bytes) + for path, section_read in sections: try: - listed = summary_read.result() + listed = parse_listed_summary(section_read.result(), path) except Exception as exc: # noqa: BLE001 - not an MCAP file if not skip_unreadable: raise diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index bfa699e0f8b0..c9d7fda5adc2 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -81,7 +81,8 @@ from ray.data.datasource.file_based_datasource import FileShuffleConfig # Chunks are packed into read tasks of about this many bytes of estimated read -# memory (``mcap_summary.chunk_run``). +# memory (``mcap_summary.chunk_runs``). A listing task holds at most this many +# bytes of summary sections at once. _BIN_PACKING_BYTES = env_integer("RAY_DATA_MCAP_BIN_PACKING_BYTES", 128 * MiB) # How many partly filled read tasks the packer keeps open at once. _BIN_PACKING_MAX_SHARED_OPEN_BINS = env_integer( @@ -164,6 +165,7 @@ def _get_file_indexer(self) -> FileIndexer: selection=self._selection, ignore_missing_paths=self._ignore_missing_paths, read_task_bytes=_BIN_PACKING_BYTES, + log_time_order=self._log_time_order, row_layout=MessageRowLayout( include_metadata=self._include_metadata, include_row_id=self._include_row_id, @@ -238,6 +240,7 @@ def infer_schema(self, sample: Optional[FileManifest]) -> pa.Schema: self._filesystem, sample.paths.tolist(), self._listed_paths, + summary_budget_bytes=_BIN_PACKING_BYTES, ) if len(sample) > 0 else None diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index 58310e0544c2..a0e0fee42ccc 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -3,7 +3,7 @@ ``MCAPSummaryIndexer`` walks the paths like ``NonSamplingFileIndexer``, then reads each file's summary and emits one manifest row per chunk that can hold a selected message. The partitioner (``OnlineBinPacker``) groups those rows into -read tasks by the memory each chunk is estimated to need (``chunk_run``). A +read tasks by the memory each chunk is estimated to need (``chunk_runs``). A large recording is then read by many tasks, and a small one shares a task with its neighbours. A file whose summary shows it cannot match the selection is dropped before any payload is read. @@ -43,8 +43,10 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( _DEFAULT_SUMMARY_IO_CONCURRENCY, ListedSummary, - chunk_run, + ReadMemory, + chunk_runs, chunk_unit_id, + parse_listed_summary, read_summaries, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( @@ -76,7 +78,7 @@ class MCAPSummaryIndexer(NonSamplingFileIndexer): Inherits directory traversal, extension and partition pruning, file shuffle and whole-file checkpoint exclusion from :class:`NonSamplingFileIndexer`. Overrides :meth:`list_files` to read each listed file's summary and turn it - into chunk rows (``mcap_summary.chunk_run``). Grouping rows into read tasks + into chunk rows (``mcap_summary.chunk_runs``). Grouping rows into read tasks is the partitioner's job. """ @@ -86,6 +88,7 @@ def __init__( selection: MCAPSelection, ignore_missing_paths: bool, read_task_bytes: int, + log_time_order: bool = True, row_layout: MessageRowLayout = MessageRowLayout(), skip_paths: Optional[Iterable[str]] = None, num_workers: Optional[int] = None, @@ -100,8 +103,10 @@ def __init__( ) self._selection = selection # The partitioner's budget per read task, which the row of an unreadable - # file weighs (see the module docstring). + # file weighs (see the module docstring), and which bounds the summary + # sections held at once. self._read_task_bytes = read_task_bytes + self._read_memory = ReadMemory(read_task_bytes, log_time_order) # The columns a row carries, which size each chunk's listing row. self._row_layout = row_layout self._io_concurrency = ( @@ -140,9 +145,9 @@ def list_files( ) assert filesystem is not None # ``list_file_infos`` raised otherwise excluded = excluded_read_unit_ids or frozenset() - for file_info, summary_read in self._read_summaries(file_infos, filesystem): + for file_info, section_read in self._read_summaries(file_infos, filesystem): try: - listed = summary_read.result() + listed = parse_listed_summary(section_read.result(), file_info.path) except Exception as e: # noqa: BLE001 - the file's own read task fails manifest = self._unreadable_file_manifest(file_info, e) else: @@ -152,14 +157,18 @@ def list_files( def _read_summaries( self, file_infos: Iterable[FileInfo], filesystem: "FileSystem" - ) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]: - """Start summary reads concurrently, yielding them in listing order. + ) -> Iterator[Tuple[FileInfo, "Future[Optional[memoryview]]"]]: + """Fetch summary sections concurrently, yielding them in listing order. Listing order decides how the partitioner groups chunks into tasks, so it must not depend on which read finishes first. """ return read_summaries( - filesystem, file_infos, lambda info: info.path, self._io_concurrency + filesystem, + file_infos, + lambda info: info.path, + self._read_task_bytes, + self._io_concurrency, ) def _unreadable_file_manifest( @@ -222,19 +231,19 @@ def _manifest_for_file( logger.debug("Skipping %s: no chunk can match the selection", path) return None - total_uncompressed = sum(c.uncompressed_size for c in summary.chunk_indexes) - row_bytes = self._row_layout.row_bytes( - path, (summary.channels[c] for c in selected), summary.schemas - ) - runs = [ - chunk_run(chunk_index, summary.statistics, total_uncompressed, row_bytes) + chunk_indexes = [ + chunk_index for chunk_index in summary.chunk_indexes if self._selection.chunk_may_match(chunk_index, selected) and chunk_unit_id(path, chunk_index.chunk_start_offset) not in excluded_read_unit_ids ] - if not runs: + if not chunk_indexes: return None + row_bytes = self._row_layout.row_bytes( + path, (summary.channels[c] for c in selected), summary.schemas + ) + runs = chunk_runs(path, summary, chunk_indexes, row_bytes, self._read_memory) manifest = FileManifest.construct_manifest( paths=[path] * len(runs), sizes=[file_info.size] * len(runs), diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index ab3d5e1fa173..4440f7fee115 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -5,9 +5,9 @@ requests per chunk it indexes. Here the bytes are fetched first and parsed in memory: -- ``read_summary_section_from`` checks both magic bytes and the footer, then - returns the summary section, all in one to three ranged reads. - ``read_summary_from`` also parses it. +- ``read_tail`` checks the magic bytes and footer, which give the summary + section's exact length; ``read_section`` returns the section, reading more + only when it starts before the tail. ``read_summary_from`` also parses it. - ``summary_records`` splits a section into its raw records, and ``parse_summary_records`` parses records copied out of a section. - ``read_chunks`` fetches chunk records that lie close together in one ranged @@ -139,33 +139,70 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: recording. Errors reading ``f`` propagate unchanged, so callers can retry them. """ - section = read_summary_section_from(f, path) + section = read_section(f, read_tail(f, path)) return None if section is None else _parse_summary(section, path) -def read_summary_section_from(f: Any, path: str) -> Optional[memoryview]: - """Return the raw summary section of the open MCAP file ``f``, or ``None``. +@dataclass(frozen=True) +class SummaryTail: + """The checked last bytes of an MCAP file, which locate its summary section. + + Attributes: + path: The file the bytes come from. + start: Offset of ``data`` in the file. + data: The file's last ``_SUMMARY_TAIL_BYTES`` bytes, or all of a smaller file. + summary_start: Offset of the summary section; 0 when there is none. + """ + + path: str + start: int + data: bytes + summary_start: int + + @property + def section_bytes(self) -> int: + """The summary section's exact length, footer included; 0 if it has none.""" + if self.summary_start == 0: + return 0 + return self.start + len(self.data) - len(_MAGIC) - self.summary_start - The section runs from the first summary record through the footer. Raises - as ``read_summary_from`` does, except that the records are not parsed. + @property + def holds_section(self) -> bool: + """Whether ``data`` holds the whole summary section, or there is none.""" + return self.summary_start == 0 or self.summary_start >= self.start + + +def read_tail(f: Any, path: str) -> SummaryTail: + """Read the tail of the open MCAP file ``f``, checking magic bytes and footer. + + Raises as ``read_summary_from`` does, except that no record is parsed. """ size = f.seek(0, io.SEEK_END) - tail_start = max(0, size - _SUMMARY_TAIL_BYTES) - tail = _read_range(f, tail_start, size) - head = tail if tail_start == 0 else _read_range(f, 0, len(_MAGIC)) + start = max(0, size - _SUMMARY_TAIL_BYTES) + data = _read_range(f, start, size) + head = data if start == 0 else _read_range(f, 0, len(_MAGIC)) if not head.startswith(_MAGIC): raise ValueError( f"{path!r} is not an MCAP file: it does not start with the MCAP " "magic bytes." ) - summary_start = _summary_start(tail, size, path) - if summary_start == 0: + return SummaryTail(path, start, data, _summary_start(data, size, path)) + + +def read_section(f: Any, tail: SummaryTail) -> Optional[memoryview]: + """Return the raw summary section that ``tail`` locates, or ``None``. + + The section runs through the footer. ``f`` is read only for the part + before the tail, and may be ``None`` when ``tail.holds_section``. + """ + if tail.summary_start == 0: return None - if summary_start < tail_start: - tail = _read_range(f, summary_start, tail_start) + tail - tail_start = summary_start - end = len(tail) - len(_MAGIC) - return memoryview(tail)[summary_start - tail_start : end] + data, start = tail.data, tail.start + if not tail.holds_section: + data = _read_range(f, tail.summary_start, start) + data + start = tail.summary_start + end = len(data) - len(_MAGIC) + return memoryview(data)[tail.summary_start - start : end] def summary_records(section: memoryview) -> Iterator[Tuple[int, memoryview]]: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index a7eec4a5843c..5e18c15ea6b4 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -4,25 +4,29 @@ the schemas and channels, one ``ChunkIndex`` per chunk (byte offset, sizes, log-time bounds, channels) and a ``Statistics`` record (message counts, time bounds). Reading it takes one to three ranged reads (``mcap_io``), so a read can -be planned without reading any message. ``read_summaries`` reads the summaries -of many files concurrently, with retries, and keeps each raw section for the -listing to hand on to read tasks (``ListedSummary``). +be planned without reading any message. ``read_summaries`` fetches many files' +sections concurrently within a byte budget, and ``chunk_runs`` weighs each chunk +by the peak memory of the read task that reads it. Format specification: https://mcap.dev/spec """ +import itertools import math from collections import deque from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from typing import ( TYPE_CHECKING, + Any, Callable, Deque, + Generic, Iterable, Iterator, List, Optional, + Sequence, Tuple, TypeVar, ) @@ -31,9 +35,12 @@ from ray._common.utils import env_integer from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, + SummaryTail, parse_summary_records, + read_section, read_summary_from, - read_summary_section_from, + read_tail, ) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileChunk from ray.data._internal.util import call_with_retry @@ -44,20 +51,27 @@ _T = TypeVar("_T") -# Peak memory of reading a chunk, as a multiple of the bytes its rows take as -# tables: the table being built, the chunk being decompressed and the ranged -# read that fetched it (``mcap_io.ChunkReads``). Reads of one 128 MiB table -# peaked at 1.14 to 1.20 times their tables. -_READ_PEAK_FACTOR = 1.2 +# Terms of a read task's peak memory (``chunk_runs``). The parsed sizes are +# upper bounds for CPython 3.10 to 3.12 with mcap 1.x. +# A message record's bytes besides its payload: opcode (1), length (8), +# channel id (2), sequence (4), and log and publish times (16). +_MESSAGE_RECORD_HEADER_BYTES = 31 +# Bytes one parsed ``mcap.records.Message`` holds beyond its payload. +_PARSED_MESSAGE_BYTES = 295 +# Bytes one entry of the log-time merge's heap holds, beyond the path in its +# ``row_id`` (``mcap_chunks``). +_HEAP_ENTRY_BYTES = 250 +# Most spare capacity of a ``bytearray`` grown by ``+=``, as a share of its bytes. +_BUFFER_GROWTH = 1 / 8 # A ``MessageIndex`` record after a chunk: opcode (1), length (8), channel id # (2) and entries length (4), then a log time and an offset (16) per message. _MESSAGE_INDEX_HEADER_BYTES = 15 _MESSAGE_INDEX_ENTRY_BYTES = 16 -# Summaries read concurrently within one listing task. A summary read is an -# open and one to three small ranged reads, so a listing waits on request -# latency. Up to twice this many read summaries are held at once. +# Summary tail reads in flight within one listing task. A read is an open and +# one or two small ranged reads, so a listing waits on request latency. Up to +# twice this many fetched sections are held at once (``read_summaries``). _DEFAULT_SUMMARY_IO_CONCURRENCY = env_integer( "RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", 32 ) @@ -65,7 +79,7 @@ @dataclass(frozen=True) class ListedSummary: - """A file's summary as ``read_summaries`` reads it: parsed, and as raw records. + """A file's summary as the listing reads it: parsed, and as raw records. ``section`` is the raw summary section, from which the listing copies the records each read task needs into the manifest (``mcap_task_summary``). @@ -75,6 +89,15 @@ class ListedSummary: section: memoryview +def parse_listed_summary( + section: Optional[memoryview], path: str +) -> Optional[ListedSummary]: + """Parse a section that ``read_summaries`` fetched; ``None`` stays ``None``.""" + if section is None: + return None + return ListedSummary(parse_summary_records(section, path), section) + + def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: """Return the summary of the MCAP file at ``path``, or ``None`` if it has none. @@ -84,53 +107,142 @@ def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: return read_summary_from(f, path) -def read_listed_summary(filesystem: FileSystem, path: str) -> Optional[ListedSummary]: - """Like ``read_summary``, but also keep the raw summary section.""" - with filesystem.open_input_file(path) as f: - section = read_summary_section_from(f, path) - if section is None: - return None - return ListedSummary(parse_summary_records(section, path), section) - - -def _read_summary_with_retry( - filesystem: "FileSystem", path: str, retried_io_errors: List[str] -) -> Optional[ListedSummary]: - """Read one file's summary, retrying errors that match ``retried_io_errors``.""" - return call_with_retry( - lambda: read_listed_summary(filesystem, path), - description=f"read MCAP summary for {path}", - match=retried_io_errors, - ) - - def read_summaries( filesystem: "FileSystem", items: Iterable[_T], path: Callable[[_T], str], + budget_bytes: int, io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, -) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: - """Read the summaries of ``items`` several at a time, yielding them in order. - - Each item comes with its pending read, so the caller decides what a failed - read means. A bounded window of in-flight reads keeps memory flat for a long - listing. +) -> Iterator[Tuple[_T, "Future[Optional[memoryview]]"]]: + """Fetch the summary sections of ``items`` concurrently, yielding them in order. + + Each item comes with its pending section, ``None`` for a file without one. + Threads only fetch; the caller parses (``parse_listed_summary``), since + parsing holds the GIL. Sections are fetched in file order while they fit + ``budget_bytes``, and the next file always fits when nothing is held. A + section counts against the budget until the caller asks for the next item. """ from ray.data.context import DataContext retried_io_errors = DataContext.get_current().retried_io_errors - window = max(1, io_concurrency * 2) - pending: Deque[Tuple[_T, Future]] = deque() + queue = iter(items) with ThreadPoolExecutor(max_workers=max(1, io_concurrency)) as pool: - for item in items: - summary_read = pool.submit( - _read_summary_with_retry, filesystem, path(item), retried_io_errors + window = _SectionWindow( + pool, filesystem, retried_io_errors, budget_bytes, io_concurrency + ) + while True: + window.start_tails(queue, path) + window.admit_sections() + fetched = window.next_section() + if fetched is None: + return + yield fetched.item, fetched.read + window.release(fetched) + + +@dataclass(frozen=True) +class _Fetch(Generic[_T]): + """One item's pending read: its tail, or its section of ``nbytes`` bytes.""" + + item: _T + read: "Future[Any]" + nbytes: int = 0 + + +class _SectionWindow(Generic[_T]): + """The tails and sections ``read_summaries`` holds, in file order.""" + + def __init__( + self, + pool: ThreadPoolExecutor, + filesystem: "FileSystem", + retried_io_errors: List[str], + budget_bytes: int, + io_concurrency: int, + ): + self._pool = pool + self._filesystem = filesystem + self._retried = retried_io_errors + self._budget = budget_bytes + self._concurrency = max(1, io_concurrency) + self._tails: Deque[_Fetch[_T]] = deque() + self._sections: Deque[_Fetch[_T]] = deque() + # Bytes of the sections admitted and not yet released. + self._held = 0 + + def start_tails(self, queue: Iterator[_T], path: Callable[[_T], str]) -> None: + """Keep ``io_concurrency`` tail reads in flight.""" + for item in itertools.islice(queue, self._concurrency - len(self._tails)): + read = self._pool.submit( + _read_tail, self._filesystem, path(item), self._retried ) - pending.append((item, summary_read)) - if len(pending) >= window: - yield pending.popleft() - while pending: - yield pending.popleft() + self._tails.append(_Fetch(item, read)) + + def admit_sections(self) -> None: + """Start section reads in file order while their sections fit the budget.""" + while self._tails and len(self._sections) < 2 * self._concurrency: + tail = self._tails[0] + if self._sections and not tail.read.done(): + return # Hand out a fetched section rather than wait. + nbytes = _section_bytes(tail.read) + if self._sections and self._held + nbytes > self._budget: + return + self._tails.popleft() + self._held += nbytes + read = self._pool.submit( + _read_section, self._filesystem, tail.read, self._retried + ) + self._sections.append(_Fetch(tail.item, read, nbytes)) + + def next_section(self) -> Optional[_Fetch[_T]]: + """The next admitted section in file order, or ``None`` when all are out.""" + return self._sections.popleft() if self._sections else None + + def release(self, fetched: _Fetch[_T]) -> None: + """Stop counting a section the caller is done with.""" + self._held -= fetched.nbytes + + +def _read_tail( + filesystem: "FileSystem", path: str, retried_io_errors: List[str] +) -> SummaryTail: + """Read one file's tail, retrying errors that match ``retried_io_errors``.""" + + def read() -> SummaryTail: + with filesystem.open_input_file(path) as f: + return read_tail(f, path) + + return call_with_retry( + read, description=f"read MCAP summary for {path}", match=retried_io_errors + ) + + +def _read_section( + filesystem: "FileSystem", + tail_read: "Future[SummaryTail]", + retried_io_errors: List[str], +) -> Optional[memoryview]: + """Read the section a finished tail read locates, or raise the tail's error.""" + tail = tail_read.result() + if tail.holds_section: + return read_section(None, tail) + + def read() -> Optional[memoryview]: + with filesystem.open_input_file(tail.path) as f: + return read_section(f, tail) + + return call_with_retry( + read, + description=f"read MCAP summary for {tail.path}", + match=retried_io_errors, + ) + + +def _section_bytes(tail_read: "Future[SummaryTail]") -> int: + """The length of the section a tail read locates; 0 when the read failed.""" + if tail_read.exception() is not None: + return 0 + return tail_read.result().section_bytes def chunk_unit_id(path: str, chunk_start_offset: int) -> str: @@ -187,26 +299,99 @@ def estimate_chunk_rows( return max(1, round(statistics.message_count * share)) -def chunk_run( - chunk_index: "ChunkIndex", - statistics: Optional["Statistics"], - total_uncompressed_size: int, +@dataclass(frozen=True) +class ReadMemory: + """How the listing sizes a read task's memory, to weigh its chunks. + + Attributes: + task_bytes: The partitioner's budget per read task. + log_time_order: Whether the read merges chunks by log time. + ranged_read_bytes: Most bytes one ranged read spans. + """ + + task_bytes: int + log_time_order: bool + ranged_read_bytes: int = ChunkReads().max_bytes + + +def chunk_runs( + path: str, + summary: "Summary", + chunk_indexes: Sequence["ChunkIndex"], row_bytes: int, -) -> FileChunk: - """Build the listing row for one chunk. - - ``size_bytes`` estimates the peak memory of reading the chunk, which is - what the partitioner budgets on: ``_READ_PEAK_FACTOR`` times the bytes its - rows take as tables. Those are the chunk's uncompressed records plus - ``row_bytes`` per message for the columns a row adds (see - ``MessageRowLayout.row_bytes``). A chunk that records no size counts as one - byte. + memory: ReadMemory, +) -> List[FileChunk]: + """Build the listing rows of one file's ``chunk_indexes``, weighted for packing. + + A task's peak is its table plus W + R (``read_overhead``). A chunk's share + of the table is its uncompressed records plus ``row_bytes`` per message, + plus ``_BUFFER_GROWTH``. Its row weighs that share times + ``task_bytes / (task_bytes - W - R)``, so a full task leaves room for W + R, + or a whole task when W + R fills one. """ - num_rows = estimate_chunk_rows(chunk_index, statistics, total_uncompressed_size) - table_bytes = chunk_index.uncompressed_size + num_rows * row_bytes - return FileChunk( - unit_ids=(chunk_index.chunk_start_offset,), - num_rows=num_rows, - size_bytes=max(math.ceil(_READ_PEAK_FACTOR * table_bytes), 1), - fully_matched=False, + total_uncompressed = sum(c.uncompressed_size for c in summary.chunk_indexes) + counts = [ + estimate_chunk_rows(c, summary.statistics, total_uncompressed) + for c in chunk_indexes + ] + room = memory.task_bytes - read_overhead(path, chunk_indexes, counts, memory) + runs = [] + for chunk_index, num_rows in zip(chunk_indexes, counts): + table = (chunk_index.uncompressed_size + num_rows * row_bytes) * ( + 1 + _BUFFER_GROWTH + ) + weight = table * memory.task_bytes / room if room > 0 else memory.task_bytes + runs.append( + FileChunk( + unit_ids=(chunk_index.chunk_start_offset,), + num_rows=num_rows, + size_bytes=max(math.ceil(weight), 1), + fully_matched=False, + ) + ) + return runs + + +def read_overhead( + path: str, + chunk_indexes: Sequence["ChunkIndex"], + counts: Sequence[int], + memory: ReadMemory, +) -> int: + """W + R of ``chunk_runs``: the largest chunk working set, plus one ranged + read when the chunks share reads.""" + working_set = max( + _working_set(c, n, path, memory.log_time_order) + for c, n in zip(chunk_indexes, counts) + ) + return working_set + _ranged_read(chunk_indexes, memory.ranged_read_bytes) + + +def _working_set( + chunk_index: "ChunkIndex", num_messages: int, path: str, log_time_order: bool +) -> int: + """Most bytes one chunk holds while it is read, its rows aside. + + The chunk record is held throughout, with either its span of the ranged + read or its decompressed and parsed messages, plus a heap entry each when + merging by log time. + """ + payload = max( + 0, chunk_index.uncompressed_size - num_messages * _MESSAGE_RECORD_HEADER_BYTES ) + decompressed = chunk_index.uncompressed_size if chunk_index.compression else 0 + per_message = _PARSED_MESSAGE_BYTES + if log_time_order: + per_message += _HEAP_ENTRY_BYTES + len(path) + parsed = decompressed + payload + num_messages * per_message + return chunk_index.compressed_size + max(chunk_index.chunk_length, parsed) + + +def _ranged_read(chunk_indexes: Sequence["ChunkIndex"], max_bytes: int) -> int: + """Most bytes a ranged read holds while chunks shorter than ``max_bytes`` + share it, within the chunks' extent (``mcap_io.plan_chunk_reads``).""" + if sum(c.chunk_length < max_bytes for c in chunk_indexes) < 2: + return 0 + start = min(c.chunk_start_offset for c in chunk_indexes) + end = max(c.chunk_start_offset + c.chunk_length for c in chunk_indexes) + return min(max_bytes, end - start) diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 65f6a134851a..3e4abd8fe0d2 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -44,7 +44,8 @@ def write_mcap( index=True, encodings=None, ): - """Write ``messages`` (dicts with topic, log_time, data) to ``path``. + """Write ``messages`` (dicts with topic, log_time, data, and optionally + publish_time and sequence) to ``path``. The default ``chunk_size=1`` closes a chunk after every message, so N messages make N chunks, each a possible split point. ``encodings`` @@ -84,6 +85,7 @@ def write_mcap( log_time=message["log_time"], publish_time=message.get("publish_time", message["log_time"]), data=data, + sequence=message.get("sequence", 0), ) writer.finish() diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py index 578a15a8da2a..58a1f018ca1e 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py @@ -51,6 +51,7 @@ ) ONLY_A = MCAPSelection.create(["/a"], None, None) +SUMMARY_BUDGET_BYTES = 128 * 2**20 # the default read task budget def write_channel( @@ -111,6 +112,7 @@ def infer( filesystem or LocalFileSystem(), paths[:sampled], lambda: iter(paths), + SUMMARY_BUDGET_BYTES, ) @@ -229,9 +231,10 @@ def no_listing(): raise AssertionError("listed although the sample types data") sampled = paths[4:] - assert infer_data_type(ONLY_A, LocalFileSystem(), sampled, no_listing) == struct_of( - "walked" + planned = infer_data_type( + ONLY_A, LocalFileSystem(), sampled, no_listing, SUMMARY_BUDGET_BYTES ) + assert planned == struct_of("walked") def test_unreadable_sampled_file_is_skipped(tmp_path): diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index 963def20ab38..ca02d83a12e4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -12,17 +12,22 @@ import pytest from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker -from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_datasource_v2, + mcap_summary, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( MCAPDatasourceV2, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import SummaryTail from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( ROW_ID_COLUMN, TimeRange, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( - _READ_PEAK_FACTOR, + _BUFFER_GROWTH, chunk_unit_id, + read_summaries, ) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data.tests.unit.datasource_v2.mcap_testing import ( @@ -333,13 +338,15 @@ def test_unreadable_file_gets_a_read_task_of_its_own(tmp_path): [(False, False, False), (True, False, False), (True, True, True)], ) def test_listed_sizes_bound_the_table_a_read_builds( - tmp_path, include_metadata, include_row_id, include_paths + tmp_path, monkeypatch, include_metadata, include_row_id, include_paths ): - """A chunk's listed size is ``_READ_PEAK_FACTOR`` times a bound on the bytes - its rows take as a table, and the bound is tight: a read of the file builds - a table between 80% and 100% of it, whichever columns the rows carry.""" + """Under a budget far above the working set, a chunk's listed size is a + bound on the bytes its rows take as a table, plus ``_BUFFER_GROWTH``. The + bound is tight: a read of the file builds a table between 80% and 100% of + it, whichever columns the rows carry.""" path = os.path.join(tmp_path, "mixed.mcap") write_mixed_channels(path, num_messages=400, payload_bytes=1000) + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", 2**62) datasource = MCAPDatasourceV2( [path], include_metadata=include_metadata, @@ -351,7 +358,7 @@ def test_listed_sizes_bound_the_table_a_read_builds( table = read_all(datasource, manifest) listed = sum(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) - bound = listed / _READ_PEAK_FACTOR + bound = listed / (1 + _BUFFER_GROWTH) assert 0.8 * bound <= table.nbytes <= bound @@ -452,6 +459,103 @@ def test_listing_order_does_not_depend_on_summary_concurrency(tmp_path, monkeypa assert many_at_once == one_at_a_time +IO_CONCURRENCY = 2 +SECTION_BUDGET = 100 # bytes of summary sections held at once + + +def fake_tail(path: str, section_bytes: int, *, in_tail: bool = True) -> SummaryTail: + """A tail whose section is ``section_bytes`` long and needs no further read + when ``in_tail``; otherwise reading the rest opens the file.""" + data = bytes(1 + section_bytes + 8) # a byte before the section, magic after + # A section that starts before the tail's first byte needs another read. + return SummaryTail(path, start=0 if in_tail else 2, data=data, summary_start=1) + + +class FailingFileSystem: + """A filesystem whose files cannot be opened, so a section read past the + tail fails.""" + + def open_input_file(self, path): + raise OSError(f"cannot open {path}") + + +@dataclasses.dataclass +class WindowLog: + """What ``read_summaries`` holds after each admission: bytes, sections, tails.""" + + held: List[int] = dataclasses.field(default_factory=list) + sections: List[int] = dataclasses.field(default_factory=list) + tails: List[int] = dataclasses.field(default_factory=list) + + +def fetch_sections(monkeypatch, tails, filesystem=None) -> Tuple[List[Any], WindowLog]: + """Run ``read_summaries`` over ``tails`` (a path to tail or error mapping), + returning each item's section or error, in order, and the window's log.""" + log = WindowLog() + + def read_tail(filesystem, path, retried_io_errors): + if isinstance(tails[path], Exception): + raise tails[path] + return tails[path] + + real_admit = mcap_summary._SectionWindow.admit_sections + + def admit_sections(window): + real_admit(window) + log.held.append(window._held) + log.sections.append(len(window._sections)) + log.tails.append(len(window._tails)) + + monkeypatch.setattr(mcap_summary, "_read_tail", read_tail) + monkeypatch.setattr(mcap_summary._SectionWindow, "admit_sections", admit_sections) + fetched = [] + for path, section_read in read_summaries( + filesystem, list(tails), lambda p: p, SECTION_BUDGET, IO_CONCURRENCY + ): + error = section_read.exception() + section = None if error else section_read.result() + fetched.append((path, error or len(section or b""))) + return fetched, log + + +@pytest.mark.parametrize( + "sizes", + [[30, 50, 20, 250, 10, 40, 60, 0], [1] * 20], + ids=["by_bytes", "by_count"], +) +def test_read_summaries_bounds_held_sections(monkeypatch, sizes): + """Sections come back in file order. The sections held fit the byte budget, + except an oversize one held alone, and there are at most twice + ``io_concurrency`` of them, with at most ``io_concurrency`` tails ahead.""" + tails = {f"f{i}": fake_tail(f"f{i}", n) for i, n in enumerate(sizes)} + + fetched, log = fetch_sections(monkeypatch, tails) + + assert fetched == [(f"f{i}", n) for i, n in enumerate(sizes)] + for held, sections in zip(log.held, log.sections): + assert held <= SECTION_BUDGET or sections == 1 + assert max(log.sections) <= 2 * IO_CONCURRENCY + assert max(log.tails) <= IO_CONCURRENCY + + +def test_read_summaries_hands_a_failed_read_to_its_file(monkeypatch): + """A failed tail read and a failed section read each come back as that + file's error, in its place, and the files after them are still read.""" + tails = { + "ok_0": fake_tail("ok_0", 10), + "bad_tail": OSError("tail"), + "bad_section": fake_tail("bad_section", 10, in_tail=False), + "ok_1": fake_tail("ok_1", 20), + } + + fetched, _ = fetch_sections(monkeypatch, tails, FailingFileSystem()) + + assert [path for path, _ in fetched] == list(tails) + assert fetched[0] == ("ok_0", 10) and fetched[3] == ("ok_1", 20) + assert str(fetched[1][1]) == "tail" + assert str(fetched[2][1]) == "cannot open bad_section" + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py new file mode 100644 index 000000000000..fd63597df6a3 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py @@ -0,0 +1,204 @@ +"""Unit tests for the estimate of a read task's peak memory (``chunk_runs``). + +Each constant is pinned by measuring the objects it stands for, and the +estimate is checked against the measured peak of a real read. +""" + +import gc +import importlib.util +import os +import random +import sys +import tracemalloc +from collections import deque + +import pyarrow as pa +import pytest + +from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 +from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( + SelectedMessageReader, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_chunk +from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + _BUFFER_GROWTH, + _HEAP_ENTRY_BYTES, + _MESSAGE_RECORD_HEADER_BYTES, + _PARSED_MESSAGE_BYTES, + ReadMemory, + read_overhead, +) +from ray.data.tests.unit.datasource_v2.mcap_testing import ( + as_read_task, + list_manifests, + pack, + scanner_for, + summary_of, + write_mcap, +) + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +PAYLOAD_BYTES = 48 # small messages, where per-message overhead dominates +NUM_MESSAGES = 20_000 +# Log times in nanoseconds since the epoch, which Python stores in larger ints +# than small numbers. +EPOCH_NS = 1_700_000_000 * 10**9 +SMALL_CHUNKS = 64 << 10 # chunk size that splits the file into many chunks +ONE_CHUNK = 1 << 30 + + +def write_small_messages(path: str, chunk_size: int) -> str: + """Write ``NUM_MESSAGES`` payloads of ``PAYLOAD_BYTES`` on one channel, half + random so that zstd compresses them partly, timed and numbered as a recorder + writes them.""" + rng = random.Random(0) + half = PAYLOAD_BYTES // 2 + messages = [ + { + "topic": "/a", + "log_time": EPOCH_NS + i * 1000, + "sequence": i, + "data": rng.randbytes(half) + bytes(half), + } + for i in range(NUM_MESSAGES) + ] + write_mcap(path, messages, chunk_size=chunk_size, message_encoding="cdr") + return path + + +def traced(run): + """``(kept, peak)``: bytes ``run()`` leaves allocated, and its peak.""" + gc.collect() + tracemalloc.start() + before = tracemalloc.get_traced_memory()[0] + result = run() + kept, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + del result + return kept - before, peak - before + + +def listed_overhead(path: str, manifest, log_time_order: bool) -> int: + """W + R of the listing of every chunk of ``path``.""" + counts = [int(md["num_rows"]) for md in manifest.file_chunk_metadatas] + memory = ReadMemory(mcap_datasource_v2._BIN_PACKING_BYTES, log_time_order) + return read_overhead(path, summary_of(path).chunk_indexes, counts, memory) + + +def test_message_record_header_is_31_bytes(): + """A message record is its payload plus the header the estimate subtracts.""" + from mcap.data_stream import RecordBuilder + from mcap.records import Message + + builder = RecordBuilder() + message = Message(channel_id=1, log_time=2, data=b"x", publish_time=3, sequence=4) + message.write(builder) + + assert len(builder.end()) - 1 == _MESSAGE_RECORD_HEADER_BYTES + + +def test_parsed_message_and_heap_entry_sizes_are_bounded(tmp_path): + """A parsed message and a log-time heap entry take at most their constants + beyond their payload and path, and at least 80% of them.""" + from mcap.stream_reader import breakup_chunk + + path = write_small_messages(os.path.join(tmp_path, "one.mcap"), ONE_CHUNK) + summary = summary_of(path) + with open(path, "rb") as f: + chunk = read_chunk(f, summary.chunk_indexes[0]) + + parsed, _ = traced(lambda: breakup_chunk(chunk)) + # The unordered read peaks at the end of parsing, with the decompressed + # records. The ordered read peaks with every message in its heap, after + # those records are freed. + peaks = {} + for ordered in (True, False): + reader = SelectedMessageReader(MCAPSelection(), log_time_order=ordered) + with open(path, "rb") as f: + messages = reader.iter_chunks(f, path, summary, None) + peaks[ordered] = traced(lambda: deque(messages, maxlen=0))[1] + heap = peaks[True] - peaks[False] + summary.chunk_indexes[0].uncompressed_size + + message_overhead = parsed / NUM_MESSAGES - PAYLOAD_BYTES + heap_entry = heap / NUM_MESSAGES - len(path) + assert 0.8 * _PARSED_MESSAGE_BYTES <= message_overhead <= _PARSED_MESSAGE_BYTES + assert 0.8 * _HEAP_ENTRY_BYTES <= heap_entry <= _HEAP_ENTRY_BYTES + + +def test_bytearray_spare_capacity_is_an_eighth(): + """A ``bytearray`` grown by ``+=``, as the table builder grows its payload + buffer, holds up to ``_BUFFER_GROWTH`` of its bytes in spare capacity.""" + empty = sys.getsizeof(bytearray()) + buffer, worst = bytearray(), 0.0 + + while len(buffer) < 4 << 20: + buffer += b"y" * PAYLOAD_BYTES + if len(buffer) >= 1 << 20: # past the fixed slack of small sizes + worst = max(worst, (sys.getsizeof(buffer) - empty) / len(buffer)) + + assert 1 + _BUFFER_GROWTH - 0.01 <= worst <= 1 + _BUFFER_GROWTH + 1e-4 + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("chunk_size", [ONE_CHUNK, SMALL_CHUNKS]) +def test_estimate_bounds_the_measured_peak_of_a_read( + tmp_path, log_time_order, chunk_size +): + """The estimate T + W + R for a task that reads the whole file is at least + the measured peak of reading it, and less than twice that.""" + path = write_small_messages(os.path.join(tmp_path, "small.mcap"), chunk_size) + datasource = MCAPDatasourceV2([path], log_time_order=log_time_order) + (manifest,) = list_manifests(datasource) + task = as_read_task(manifest) + reader = scanner_for(datasource).create_reader() + deque(reader.read(task), maxlen=0) # Warm imports outside the measurement. + pool = pa.default_memory_pool() + arrow_before = max(pool.bytes_allocated(), pool.max_memory()) + + _, python_peak = traced(lambda: deque(reader.read(task), maxlen=0)) + + measured = python_peak + max(0, pool.max_memory() - arrow_before) + # Rows weigh T * budget / (budget - W - R), which gives back T. + budget = mcap_datasource_v2._BIN_PACKING_BYTES + overhead = listed_overhead(path, manifest, log_time_order) + weights = sum(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) + estimate = weights * (budget - overhead) / budget + overhead + assert measured <= estimate < 2 * measured + + +def test_packer_leaves_room_for_the_working_set(tmp_path, monkeypatch): + """With a budget of W + R plus 3.5 chunks' tables, four chunks would fit by + their tables alone. Weighed with room for W + R, a fourth full chunk does + not fit, and every task stays within the budget.""" + path = write_small_messages(os.path.join(tmp_path, "small.mcap"), SMALL_CHUNKS) + datasource = MCAPDatasourceV2([path], log_time_order=False) + (manifest,) = list_manifests(datasource) + overhead = listed_overhead(path, manifest, log_time_order=False) + heaviest = max(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) + table = heaviest * (1 - overhead / mcap_datasource_v2._BIN_PACKING_BYTES) + budget = int(overhead + 3.5 * table) + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", budget) + + (relisted,) = list_manifests(datasource) + tasks = pack(datasource) + + weights = [int(md["size_bytes"]) for md in relisted.file_chunk_metadatas] + assert 4 * table < budget < 4 * max(weights) + task_weights = [ + sum(int(md["size_bytes"]) for md in task.file_chunk_metadatas) for task in tasks + ] + assert max(task_weights) <= budget + chunks = [sum(len(md["unit_ids"]) for md in t.file_chunk_metadatas) for t in tasks] + assert sum(chunks) == len(summary_of(path).chunk_indexes) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) From 4cd6af53bbb729a48922f019d4fa146f7b17f614 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Tue, 10 Nov 2026 17:48:37 -0800 Subject: [PATCH 10/10] [Data] read_mcap: count decoded JSON in the read-memory estimate Problem: - The estimate counts a payload's raw bytes, but a JSON payload is decoded into Python values that hold several times as many bytes until its table is built. A read of small JSON messages peaks above the estimate. Change: - When every selected channel of a file is JSON-encoded, each payload byte counts the reader's own bytes-per-payload-byte for decoded JSON. - The peak test also reads JSON payloads. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_file_indexer.py | 12 +++++-- .../formats/mcap/mcap_message_rows.py | 13 ++++++++ .../formats/mcap/mcap_summary.py | 10 ++++-- .../datasource_v2/test_mcap_read_memory.py | 32 +++++++++++++------ 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index a0e0fee42ccc..4544355c89ec 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -240,10 +240,16 @@ def _manifest_for_file( ] if not chunk_indexes: return None - row_bytes = self._row_layout.row_bytes( - path, (summary.channels[c] for c in selected), summary.schemas + channels = [summary.channels[c] for c in selected] + row_bytes = self._row_layout.row_bytes(path, channels, summary.schemas) + runs = chunk_runs( + path, + summary, + chunk_indexes, + row_bytes, + self._read_memory, + payload_factor=self._row_layout.payload_factor(channels), ) - runs = chunk_runs(path, summary, chunk_indexes, row_bytes, self._read_memory) manifest = FileManifest.construct_manifest( paths=[path] * len(runs), sizes=[file_info.size] * len(runs), diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py index ef2651215864..fd3994aa7a83 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py @@ -232,6 +232,19 @@ def row_bytes( per_row += len(path.encode()) + _INT32_BYTES return per_row + @staticmethod + def payload_factor(channels: Iterable["Channel"]) -> int: + """Bytes a row of one of ``channels`` holds per payload byte. + + ``data`` holds decoded JSON only when every selected channel is + JSON-encoded (``mcap_data_column``). Decoded JSON is held as Python + objects until its table is built, so it counts + ``_DECODED_JSON_BYTES_PER_PAYLOAD_BYTE``. + """ + if {c.message_encoding for c in channels} == {"json"}: + return _DECODED_JSON_BYTES_PER_PAYLOAD_BYTE + return 1 + def _int_array(values: array, type_: pa.DataType) -> pa.Array: """Wrap an ``array.array`` of int32 or int64 values as an Arrow array, no copy.""" diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index 5e18c15ea6b4..90bc800f0f7d 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -320,12 +320,14 @@ def chunk_runs( chunk_indexes: Sequence["ChunkIndex"], row_bytes: int, memory: ReadMemory, + payload_factor: int = 1, ) -> List[FileChunk]: """Build the listing rows of one file's ``chunk_indexes``, weighted for packing. A task's peak is its table plus W + R (``read_overhead``). A chunk's share of the table is its uncompressed records plus ``row_bytes`` per message, - plus ``_BUFFER_GROWTH``. Its row weighs that share times + plus ``_BUFFER_GROWTH``. Each payload byte counts ``payload_factor`` bytes, + for payloads the reader decodes into larger values. Its row weighs that share times ``task_bytes / (task_bytes - W - R)``, so a full task leaves room for W + R, or a whole task when W + R fills one. """ @@ -337,9 +339,11 @@ def chunk_runs( room = memory.task_bytes - read_overhead(path, chunk_indexes, counts, memory) runs = [] for chunk_index, num_rows in zip(chunk_indexes, counts): - table = (chunk_index.uncompressed_size + num_rows * row_bytes) * ( - 1 + _BUFFER_GROWTH + payload = max( + 0, chunk_index.uncompressed_size - num_rows * _MESSAGE_RECORD_HEADER_BYTES ) + records = chunk_index.uncompressed_size + (payload_factor - 1) * payload + table = (records + num_rows * row_bytes) * (1 + _BUFFER_GROWTH) weight = table * memory.task_bytes / room if room > 0 else memory.task_bytes runs.append( FileChunk( diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py index fd63597df6a3..4de4625b31a3 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py @@ -6,6 +6,7 @@ import gc import importlib.util +import json import os import random import sys @@ -55,22 +56,31 @@ ONE_CHUNK = 1 << 30 -def write_small_messages(path: str, chunk_size: int) -> str: - """Write ``NUM_MESSAGES`` payloads of ``PAYLOAD_BYTES`` on one channel, half - random so that zstd compresses them partly, timed and numbered as a recorder - writes them.""" +def write_small_messages(path: str, chunk_size: int, encoding: str = "cdr") -> str: + """Write ``NUM_MESSAGES`` small payloads on one channel, timed and numbered + as a recorder writes them. + + A CDR payload is ``PAYLOAD_BYTES``, half random so that zstd compresses it + partly. A JSON payload is an object of two numbers, which the reader decodes. + """ rng = random.Random(0) half = PAYLOAD_BYTES // 2 + + def payload(i: int) -> bytes: + if encoding == "json": + return json.dumps({"x": rng.random(), "i": i}).encode() + return rng.randbytes(half) + bytes(half) + messages = [ { "topic": "/a", "log_time": EPOCH_NS + i * 1000, "sequence": i, - "data": rng.randbytes(half) + bytes(half), + "data": payload(i), } for i in range(NUM_MESSAGES) ] - write_mcap(path, messages, chunk_size=chunk_size, message_encoding="cdr") + write_mcap(path, messages, chunk_size=chunk_size, message_encoding=encoding) return path @@ -147,14 +157,18 @@ def test_bytearray_spare_capacity_is_an_eighth(): assert 1 + _BUFFER_GROWTH - 0.01 <= worst <= 1 + _BUFFER_GROWTH + 1e-4 +@pytest.mark.parametrize("encoding", ["cdr", "json"]) @pytest.mark.parametrize("log_time_order", [True, False]) @pytest.mark.parametrize("chunk_size", [ONE_CHUNK, SMALL_CHUNKS]) def test_estimate_bounds_the_measured_peak_of_a_read( - tmp_path, log_time_order, chunk_size + tmp_path, log_time_order, chunk_size, encoding ): """The estimate T + W + R for a task that reads the whole file is at least - the measured peak of reading it, and less than twice that.""" - path = write_small_messages(os.path.join(tmp_path, "small.mcap"), chunk_size) + the measured peak of reading it, and less than twice that. JSON payloads + are decoded into Python values, which hold several times their bytes.""" + path = write_small_messages( + os.path.join(tmp_path, "small.mcap"), chunk_size, encoding + ) datasource = MCAPDatasourceV2([path], log_time_order=log_time_order) (manifest,) = list_manifests(datasource) task = as_read_task(manifest)