diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 95f7ed8d4d74..ecc07d1e559b 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -1,9 +1,11 @@ """The reading of selected messages out of one MCAP file. -An indexed file is read chunk by chunk, in file order or merged by log time. A -file without a chunk index is scanned from the start, and sorted in memory when -ordered by log time. A channel or schema that only an earlier chunk declares is -read back from that chunk. +An indexed file is read chunk by chunk, in file order or merged by log time. +Chunks that are read one after another and lie close together in the file are +fetched in one ranged read (``mcap_io.read_chunks``). A file without a chunk +index is scanned from the start, and sorted in memory when ordered by log time. +A channel or schema that only an earlier chunk declares is read back from that +chunk. """ import heapq @@ -21,7 +23,11 @@ Tuple, ) -from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_chunk +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, + read_chunk, + read_chunks, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( message_row_id, @@ -29,7 +35,7 @@ ) if TYPE_CHECKING: - from mcap.records import Channel, ChunkIndex, Message, Schema + from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema from mcap.summary import Summary # One selected message as the reader sees it: its schema (``None`` for a @@ -43,6 +49,11 @@ def _log_time(message: _Scanned) -> int: return message[2].log_time +def _merge_order(chunk_index: "ChunkIndex") -> Tuple[int, int]: + """The order in which the log-time merge expands chunks.""" + return chunk_index.message_start_time, chunk_index.chunk_start_offset + + @dataclass class _Declared: """Channel and schema records known while reading one file. @@ -81,12 +92,19 @@ class SelectedMessageReader: """Reads the messages a selection keeps out of an open MCAP file. With ``log_time_order``, an indexed file's messages come out in ascending - ``log_time`` order rather than in file order. + ``log_time`` order rather than in file order. ``chunk_reads`` decides which + chunks share a ranged read. """ - def __init__(self, selection: MCAPSelection, log_time_order: bool): + def __init__( + self, + selection: MCAPSelection, + log_time_order: bool, + chunk_reads: ChunkReads = ChunkReads(), + ): self._selection = selection self._log_time_order = log_time_order + self._chunk_reads = chunk_reads def iter_chunks( self, @@ -118,9 +136,9 @@ def iter_chunks( file_summary or (lambda: summary), ) if not self._log_time_order: - for chunk_index in chunk_indexes: - yield from self._read_chunk( - f, path, summary, chunk_index, selected, declared + for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads): + yield from self._chunk_messages( + f, path, summary, chunk_index, chunk, selected, declared ) return yield from self._merge_chunks_by_log_time( @@ -166,18 +184,32 @@ def _merge_chunks_by_log_time( the top, so only the chunks that overlap in time are held at once. A message is yielded once every chunk that could precede it is expanded. """ - # Heap entries: (log time, kind, chunk offset, index in chunk, item). - # ``kind`` 0 is a chunk index, 1 a message, so on a tied log time a - # chunk is expanded before a message is yielded. + # Heap entries: (log time, kind, chunk offset, index in chunk, message). + # ``kind`` 0 is a chunk, 1 a message, so on a tied log time a chunk is + # expanded before a message is yielded. heap: List[Tuple[int, int, int, int, Any]] = [ - (c.message_start_time, 0, c.chunk_start_offset, 0, c) for c in chunk_indexes + (c.message_start_time, 0, c.chunk_start_offset, 0, None) + for c in chunk_indexes ] heapq.heapify(heap) + # Chunks leave the heap in ``_merge_order``, so they are read in it. + chunks = read_chunks( + f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads + ) while heap: _, kind, offset, _, item = heapq.heappop(heap) if kind == 0: - for index, selected_message in self._read_chunk( - f, path, summary, item, selected, declared, with_index=True + chunk_index, chunk = next(chunks) + assert chunk_index.chunk_start_offset == offset + for index, selected_message in self._chunk_messages( + f, + path, + summary, + chunk_index, + chunk, + selected, + declared, + with_index=True, ): heapq.heappush( heap, @@ -192,12 +224,13 @@ def _merge_chunks_by_log_time( else: yield item - def _read_chunk( + def _chunk_messages( self, f: Any, path: str, summary: "Summary", chunk_index: "ChunkIndex", + chunk: "Chunk", selected: Set[int], declared: "_Declared", with_index: bool = False, @@ -212,7 +245,6 @@ def _read_chunk( from mcap.records import Channel, Message, Schema from mcap.stream_reader import breakup_chunk - chunk = read_chunk(f, chunk_index) index = -1 for record in breakup_chunk(chunk): if isinstance(record, Message): diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py index 0bd9f56aa680..4ab167d62616 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_data_column.py @@ -47,7 +47,10 @@ decode_payload, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection -from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summaries +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + parse_listed_summary, + read_summaries, +) from ray.data._internal.tensor_extensions.arrow import convert_to_pyarrow_array from ray.data.context import DataContext from ray.util.debug import log_once @@ -90,6 +93,7 @@ def infer_data_type( filesystem: "FileSystem", sampled_paths: Sequence[str], list_paths: Callable[[], Iterable[str]], + summary_budget_bytes: int, ) -> Optional[pa.DataType]: """Type of ``data`` when it holds decoded JSON; ``None`` when it is ``binary``. @@ -102,11 +106,12 @@ def infer_data_type( sampled_paths: The files sampled for the schema, in listing order. list_paths: Lists the read's files in listing order, with its extension filter. Called only when no sampled file types ``data``. + summary_budget_bytes: Most bytes of summary sections held at once. Returns: The type of the sampled JSON values, or ``None`` for ``binary``. """ - inspector = _FileInspector(selection, filesystem) + inspector = _FileInspector(selection, filesystem, summary_budget_bytes) signature = inspector.inspect_sample(sampled_paths) if signature.has_non_json or signature.value_type is not None: return signature.value_type @@ -130,9 +135,15 @@ class _FileInspector: Scans at most ``_MAX_SCANNED_FILES`` files whole. """ - def __init__(self, selection: MCAPSelection, filesystem: "FileSystem"): + def __init__( + self, + selection: MCAPSelection, + filesystem: "FileSystem", + summary_budget_bytes: int, + ): self._selection = selection self._filesystem = filesystem + self._summary_budget_bytes = summary_budget_bytes self._scans_left = _MAX_SCANNED_FILES def inspect_sample(self, paths: Sequence[str]) -> _DataSignature: @@ -145,7 +156,10 @@ def inspect_sample(self, paths: Sequence[str]) -> _DataSignature: the read task that owns it reports the error. """ value_type: Optional[pa.DataType] = None - for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True): + summaries = _summaries( + self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True + ) + for path, summary in summaries: signature = self._inspect( path, summary, want_type=value_type is None, may_scan=True ) @@ -161,7 +175,10 @@ def walk(self, paths: List[str]) -> _DataSignature: Only files whose summary settles the selection are inspected, so no file is scanned whole. Files whose summary cannot be read are skipped. """ - for path, summary in _summaries(self._filesystem, paths, skip_unreadable=True): + summaries = _summaries( + self._filesystem, paths, self._summary_budget_bytes, skip_unreadable=True + ) + for path, summary in summaries: signature = self._inspect(path, summary, want_type=True, may_scan=False) if signature.has_non_json or signature.value_type is not None: return signature @@ -333,16 +350,21 @@ def _readable_records(records: Iterable[Any], path: str) -> Iterator[Any]: def _summaries( - filesystem: "FileSystem", paths: Iterable[str], *, skip_unreadable: bool + filesystem: "FileSystem", + paths: Iterable[str], + budget_bytes: int, + *, + skip_unreadable: bool, ) -> Iterator[Tuple[str, Optional["Summary"]]]: """Read the summaries of ``paths`` several at a time, yielding them in order. With ``skip_unreadable``, a file whose summary cannot be read is skipped. Otherwise its error is raised. """ - for path, summary_read in read_summaries(filesystem, paths, lambda path: path): + sections = read_summaries(filesystem, paths, lambda path: path, budget_bytes) + for path, section_read in sections: try: - listed = summary_read.result() + listed = parse_listed_summary(section_read.result(), path) except Exception as exc: # noqa: BLE001 - not an MCAP file if not skip_unreadable: raise diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index bfa699e0f8b0..c9d7fda5adc2 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -81,7 +81,8 @@ from ray.data.datasource.file_based_datasource import FileShuffleConfig # Chunks are packed into read tasks of about this many bytes of estimated read -# memory (``mcap_summary.chunk_run``). +# memory (``mcap_summary.chunk_runs``). A listing task holds at most this many +# bytes of summary sections at once. _BIN_PACKING_BYTES = env_integer("RAY_DATA_MCAP_BIN_PACKING_BYTES", 128 * MiB) # How many partly filled read tasks the packer keeps open at once. _BIN_PACKING_MAX_SHARED_OPEN_BINS = env_integer( @@ -164,6 +165,7 @@ def _get_file_indexer(self) -> FileIndexer: selection=self._selection, ignore_missing_paths=self._ignore_missing_paths, read_task_bytes=_BIN_PACKING_BYTES, + log_time_order=self._log_time_order, row_layout=MessageRowLayout( include_metadata=self._include_metadata, include_row_id=self._include_row_id, @@ -238,6 +240,7 @@ def infer_schema(self, sample: Optional[FileManifest]) -> pa.Schema: self._filesystem, sample.paths.tolist(), self._listed_paths, + summary_budget_bytes=_BIN_PACKING_BYTES, ) if len(sample) > 0 else None diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py index 58310e0544c2..4544355c89ec 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_file_indexer.py @@ -3,7 +3,7 @@ ``MCAPSummaryIndexer`` walks the paths like ``NonSamplingFileIndexer``, then reads each file's summary and emits one manifest row per chunk that can hold a selected message. The partitioner (``OnlineBinPacker``) groups those rows into -read tasks by the memory each chunk is estimated to need (``chunk_run``). A +read tasks by the memory each chunk is estimated to need (``chunk_runs``). A large recording is then read by many tasks, and a small one shares a task with its neighbours. A file whose summary shows it cannot match the selection is dropped before any payload is read. @@ -43,8 +43,10 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( _DEFAULT_SUMMARY_IO_CONCURRENCY, ListedSummary, - chunk_run, + ReadMemory, + chunk_runs, chunk_unit_id, + parse_listed_summary, read_summaries, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( @@ -76,7 +78,7 @@ class MCAPSummaryIndexer(NonSamplingFileIndexer): Inherits directory traversal, extension and partition pruning, file shuffle and whole-file checkpoint exclusion from :class:`NonSamplingFileIndexer`. Overrides :meth:`list_files` to read each listed file's summary and turn it - into chunk rows (``mcap_summary.chunk_run``). Grouping rows into read tasks + into chunk rows (``mcap_summary.chunk_runs``). Grouping rows into read tasks is the partitioner's job. """ @@ -86,6 +88,7 @@ def __init__( selection: MCAPSelection, ignore_missing_paths: bool, read_task_bytes: int, + log_time_order: bool = True, row_layout: MessageRowLayout = MessageRowLayout(), skip_paths: Optional[Iterable[str]] = None, num_workers: Optional[int] = None, @@ -100,8 +103,10 @@ def __init__( ) self._selection = selection # The partitioner's budget per read task, which the row of an unreadable - # file weighs (see the module docstring). + # file weighs (see the module docstring), and which bounds the summary + # sections held at once. self._read_task_bytes = read_task_bytes + self._read_memory = ReadMemory(read_task_bytes, log_time_order) # The columns a row carries, which size each chunk's listing row. self._row_layout = row_layout self._io_concurrency = ( @@ -140,9 +145,9 @@ def list_files( ) assert filesystem is not None # ``list_file_infos`` raised otherwise excluded = excluded_read_unit_ids or frozenset() - for file_info, summary_read in self._read_summaries(file_infos, filesystem): + for file_info, section_read in self._read_summaries(file_infos, filesystem): try: - listed = summary_read.result() + listed = parse_listed_summary(section_read.result(), file_info.path) except Exception as e: # noqa: BLE001 - the file's own read task fails manifest = self._unreadable_file_manifest(file_info, e) else: @@ -152,14 +157,18 @@ def list_files( def _read_summaries( self, file_infos: Iterable[FileInfo], filesystem: "FileSystem" - ) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]: - """Start summary reads concurrently, yielding them in listing order. + ) -> Iterator[Tuple[FileInfo, "Future[Optional[memoryview]]"]]: + """Fetch summary sections concurrently, yielding them in listing order. Listing order decides how the partitioner groups chunks into tasks, so it must not depend on which read finishes first. """ return read_summaries( - filesystem, file_infos, lambda info: info.path, self._io_concurrency + filesystem, + file_infos, + lambda info: info.path, + self._read_task_bytes, + self._io_concurrency, ) def _unreadable_file_manifest( @@ -222,19 +231,25 @@ def _manifest_for_file( logger.debug("Skipping %s: no chunk can match the selection", path) return None - total_uncompressed = sum(c.uncompressed_size for c in summary.chunk_indexes) - row_bytes = self._row_layout.row_bytes( - path, (summary.channels[c] for c in selected), summary.schemas - ) - runs = [ - chunk_run(chunk_index, summary.statistics, total_uncompressed, row_bytes) + chunk_indexes = [ + chunk_index for chunk_index in summary.chunk_indexes if self._selection.chunk_may_match(chunk_index, selected) and chunk_unit_id(path, chunk_index.chunk_start_offset) not in excluded_read_unit_ids ] - if not runs: + if not chunk_indexes: return None + channels = [summary.channels[c] for c in selected] + row_bytes = self._row_layout.row_bytes(path, channels, summary.schemas) + runs = chunk_runs( + path, + summary, + chunk_indexes, + row_bytes, + self._read_memory, + payload_factor=self._row_layout.payload_factor(channels), + ) manifest = FileManifest.construct_manifest( paths=[path] * len(runs), sizes=[file_info.size] * len(runs), diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index ace3f48a94e9..4440f7fee115 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -5,21 +5,24 @@ requests per chunk it indexes. Here the bytes are fetched first and parsed in memory: -- ``read_summary_section_from`` checks both magic bytes and the footer, then - returns the summary section, all in one to three ranged reads. - ``read_summary_from`` also parses it. +- ``read_tail`` checks the magic bytes and footer, which give the summary + section's exact length; ``read_section`` returns the section, reading more + only when it starts before the tail. ``read_summary_from`` also parses it. - ``summary_records`` splits a section into its raw records, and ``parse_summary_records`` parses records copied out of a section. -- ``read_chunk`` fetches one chunk record in one ranged read. +- ``read_chunks`` fetches chunk records that lie close together in one ranged + read, as ``ChunkReads`` allows; ``read_chunk`` fetches one chunk record. - ``buffered`` wraps a file for a linear scan, so it reads in large blocks. Format specification: https://mcap.dev/spec """ import io -from typing import TYPE_CHECKING, Any, Iterator, Optional, Tuple, Union +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Iterable, Iterator, List, Optional, Tuple, Union -from ray.data._internal.util import KiB +from ray._common.utils import env_integer +from ray.data._internal.util import KiB, MiB if TYPE_CHECKING: from mcap.records import Chunk, ChunkIndex @@ -30,6 +33,16 @@ # one more read. _SUMMARY_TAIL_BYTES = 64 * KiB +# Defaults of ``ChunkReads``. A request to object storage costs a fixed delay +# (16-32 ms on S3) plus its bytes over the connection's bandwidth (about +# 64 MiB/s). Reading through a gap costs less than the request it saves while +# the gap is under delay x bandwidth: 1 MiB at 16 ms. A 16 MiB read spends about +# 90% of its time moving bytes. A first read of 4 MiB keeps a read that stops +# early, such as under a row limit, from fetching far more than it parses. +_CHUNK_READ_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_BYTES", 16 * MiB) +_CHUNK_READ_GAP_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_GAP_BYTES", 1 * MiB) +_CHUNK_READ_FIRST_BYTES = env_integer("RAY_DATA_MCAP_CHUNK_READ_FIRST_BYTES", 4 * MiB) + _MAGIC = b"\x89MCAP0\r\n" # A footer record is the opcode, an 8-byte length and 20 bytes of fields: # ``summary_start`` (8), ``summary_offset_start`` (8) and ``summary_crc`` (4). @@ -37,6 +50,84 @@ _RECORD_PREFIX_BYTES = 1 + 8 # opcode and record length +@dataclass(frozen=True) +class ChunkReads: + """How ``read_chunks`` groups chunk records into ranged reads. + + Attributes: + max_bytes: Most bytes one read spans, gaps included. A chunk record + longer than this is read alone, and 0 reads every chunk alone. + A read task holds one read's bytes at a time. + max_gap_bytes: Longest gap between two chunk records that one read + spans; the gap's bytes are read and dropped. Neighbouring chunks + are apart by their message index records (16 bytes per message), + and a chunk the read does not need is a gap as large as itself. + first_bytes: Most bytes the first read of a file spans. Each later + read may span twice as many as the one before, up to + ``max_bytes``, so a read that stops early fetches little. + """ + + max_bytes: int = _CHUNK_READ_BYTES + max_gap_bytes: int = _CHUNK_READ_GAP_BYTES + first_bytes: int = _CHUNK_READ_FIRST_BYTES + + +@dataclass(frozen=True) +class RangedRead: + """Bytes ``[start, end)`` of a file, which hold the ``chunk_indexes``' records.""" + + start: int + end: int + chunk_indexes: Tuple["ChunkIndex", ...] + + +def plan_chunk_reads( + chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads +) -> List[RangedRead]: + """Group chunks, taken in the order given, into ranged reads. + + A chunk joins the read before it if it starts at or after that read's end, + at most ``max_gap_bytes`` past it, and the read then spans at most its + size limit: ``first_bytes`` for the first read, then twice the previous + limit, up to ``max_bytes``. Otherwise it starts a new read. + """ + planned: List[RangedRead] = [] + group: List["ChunkIndex"] = [] + limit = min(reads.first_bytes, reads.max_bytes) + for chunk_index in chunk_indexes: + if group and not _extends(group, chunk_index, reads.max_gap_bytes, limit): + planned.append(_ranged_read(group)) + group = [] + limit = min(2 * limit, reads.max_bytes) + group.append(chunk_index) + if group: + planned.append(_ranged_read(group)) + return planned + + +def read_chunks( + f: Any, chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads +) -> Iterator[Tuple["ChunkIndex", "Chunk"]]: + """Yield the chunk record of each of ``chunk_indexes``, in that order. + + Each read of ``plan_chunk_reads`` is made when its first chunk is needed, + and its bytes are released once its last chunk is parsed. + """ + for ranged_read in plan_chunk_reads(chunk_indexes, reads): + # ``BytesIO`` shares the bytes rather than copying them; parsing a + # chunk copies only the chunk's records. + stream = io.BytesIO(_read_range(f, ranged_read.start, ranged_read.end)) + last = len(ranged_read.chunk_indexes) - 1 + for i, chunk_index in enumerate(ranged_read.chunk_indexes): + position = chunk_index.chunk_start_offset - ranged_read.start + chunk = _parse_chunk(stream, position) + if i == last: + # Free the bytes now, not when the next read replaces them, so + # two reads are never held at once. + stream.close() + yield chunk_index, chunk + + def read_summary_from(f: Any, path: str) -> Optional["Summary"]: """Return the summary of the open MCAP file ``f``, or ``None`` if it has none. @@ -48,33 +139,70 @@ def read_summary_from(f: Any, path: str) -> Optional["Summary"]: recording. Errors reading ``f`` propagate unchanged, so callers can retry them. """ - section = read_summary_section_from(f, path) + section = read_section(f, read_tail(f, path)) return None if section is None else _parse_summary(section, path) -def read_summary_section_from(f: Any, path: str) -> Optional[memoryview]: - """Return the raw summary section of the open MCAP file ``f``, or ``None``. +@dataclass(frozen=True) +class SummaryTail: + """The checked last bytes of an MCAP file, which locate its summary section. - The section runs from the first summary record through the footer. Raises - as ``read_summary_from`` does, except that the records are not parsed. + Attributes: + path: The file the bytes come from. + start: Offset of ``data`` in the file. + data: The file's last ``_SUMMARY_TAIL_BYTES`` bytes, or all of a smaller file. + summary_start: Offset of the summary section; 0 when there is none. + """ + + path: str + start: int + data: bytes + summary_start: int + + @property + def section_bytes(self) -> int: + """The summary section's exact length, footer included; 0 if it has none.""" + if self.summary_start == 0: + return 0 + return self.start + len(self.data) - len(_MAGIC) - self.summary_start + + @property + def holds_section(self) -> bool: + """Whether ``data`` holds the whole summary section, or there is none.""" + return self.summary_start == 0 or self.summary_start >= self.start + + +def read_tail(f: Any, path: str) -> SummaryTail: + """Read the tail of the open MCAP file ``f``, checking magic bytes and footer. + + Raises as ``read_summary_from`` does, except that no record is parsed. """ size = f.seek(0, io.SEEK_END) - tail_start = max(0, size - _SUMMARY_TAIL_BYTES) - tail = _read_range(f, tail_start, size) - head = tail if tail_start == 0 else _read_range(f, 0, len(_MAGIC)) + start = max(0, size - _SUMMARY_TAIL_BYTES) + data = _read_range(f, start, size) + head = data if start == 0 else _read_range(f, 0, len(_MAGIC)) if not head.startswith(_MAGIC): raise ValueError( f"{path!r} is not an MCAP file: it does not start with the MCAP " "magic bytes." ) - summary_start = _summary_start(tail, size, path) - if summary_start == 0: + return SummaryTail(path, start, data, _summary_start(data, size, path)) + + +def read_section(f: Any, tail: SummaryTail) -> Optional[memoryview]: + """Return the raw summary section that ``tail`` locates, or ``None``. + + The section runs through the footer. ``f`` is read only for the part + before the tail, and may be ``None`` when ``tail.holds_section``. + """ + if tail.summary_start == 0: return None - if summary_start < tail_start: - tail = _read_range(f, summary_start, tail_start) + tail - tail_start = summary_start - end = len(tail) - len(_MAGIC) - return memoryview(tail)[summary_start - tail_start : end] + data, start = tail.data, tail.start + if not tail.holds_section: + data = _read_range(f, tail.summary_start, start) + data + start = tail.summary_start + end = len(data) - len(_MAGIC) + return memoryview(data)[tail.summary_start - start : end] def summary_records(section: memoryview) -> Iterator[Tuple[int, memoryview]]: @@ -107,15 +235,41 @@ def parse_summary_records(records: Union[bytes, memoryview], path: str) -> "Summ def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": """Read the chunk record that ``chunk_index`` points to, in one ranged read.""" + start = chunk_index.chunk_start_offset + stream = io.BytesIO(_read_range(f, start, start + chunk_index.chunk_length)) + return _parse_chunk(stream, 0) + + +def _parse_chunk(stream: io.BytesIO, position: int) -> "Chunk": + """Parse the chunk record that starts at ``position`` of ``stream``.""" from mcap.data_stream import ReadDataStream from mcap.records import Chunk - start = chunk_index.chunk_start_offset - stream = io.BytesIO(_read_range(f, start, start + chunk_index.chunk_length)) - stream.seek(_RECORD_PREFIX_BYTES) + stream.seek(position + _RECORD_PREFIX_BYTES) return Chunk.read(ReadDataStream(stream)) +def _record_end(chunk_index: "ChunkIndex") -> int: + """The offset just past the chunk record that ``chunk_index`` points to.""" + return chunk_index.chunk_start_offset + chunk_index.chunk_length + + +def _extends( + group: List["ChunkIndex"], chunk_index: "ChunkIndex", max_gap: int, limit: int +) -> bool: + """Whether ``chunk_index`` can join the read of ``group`` (``plan_chunk_reads``).""" + end = _record_end(group[-1]) + return ( + end <= chunk_index.chunk_start_offset <= end + max_gap + and _record_end(chunk_index) - group[0].chunk_start_offset <= limit + ) + + +def _ranged_read(group: List["ChunkIndex"]) -> RangedRead: + """The read that spans the chunk records of ``group``, in file order.""" + return RangedRead(group[0].chunk_start_offset, _record_end(group[-1]), tuple(group)) + + def buffered(f: Any, buffer_size: int) -> io.BufferedReader: """``f`` behind a read buffer of ``buffer_size`` bytes, for a linear scan. diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py index ef2651215864..fd3994aa7a83 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py @@ -232,6 +232,19 @@ def row_bytes( per_row += len(path.encode()) + _INT32_BYTES return per_row + @staticmethod + def payload_factor(channels: Iterable["Channel"]) -> int: + """Bytes a row of one of ``channels`` holds per payload byte. + + ``data`` holds decoded JSON only when every selected channel is + JSON-encoded (``mcap_data_column``). Decoded JSON is held as Python + objects until its table is built, so it counts + ``_DECODED_JSON_BYTES_PER_PAYLOAD_BYTE``. + """ + if {c.message_encoding for c in channels} == {"json"}: + return _DECODED_JSON_BYTES_PER_PAYLOAD_BYTE + return 1 + def _int_array(values: array, type_: pa.DataType) -> pa.Array: """Wrap an ``array.array`` of int32 or int64 values as an Arrow array, no copy.""" diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py index ad906b036f40..90bc800f0f7d 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_summary.py @@ -4,25 +4,29 @@ the schemas and channels, one ``ChunkIndex`` per chunk (byte offset, sizes, log-time bounds, channels) and a ``Statistics`` record (message counts, time bounds). Reading it takes one to three ranged reads (``mcap_io``), so a read can -be planned without reading any message. ``read_summaries`` reads the summaries -of many files concurrently, with retries, and keeps each raw section for the -listing to hand on to read tasks (``ListedSummary``). +be planned without reading any message. ``read_summaries`` fetches many files' +sections concurrently within a byte budget, and ``chunk_runs`` weighs each chunk +by the peak memory of the read task that reads it. Format specification: https://mcap.dev/spec """ +import itertools import math from collections import deque from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from typing import ( TYPE_CHECKING, + Any, Callable, Deque, + Generic, Iterable, Iterator, List, Optional, + Sequence, Tuple, TypeVar, ) @@ -31,9 +35,12 @@ from ray._common.utils import env_integer from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, + SummaryTail, parse_summary_records, + read_section, read_summary_from, - read_summary_section_from, + read_tail, ) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileChunk from ray.data._internal.util import call_with_retry @@ -44,26 +51,35 @@ _T = TypeVar("_T") -# Peak memory of reading a chunk, as a multiple of the bytes its rows take as -# tables: the table being built plus the chunk being decompressed. Reads of one -# 128 MiB bin peaked at 1.08 to 1.16 times their tables. -_READ_PEAK_FACTOR = 1.2 +# Terms of a read task's peak memory (``chunk_runs``). The parsed sizes are +# upper bounds for CPython 3.10 to 3.12 with mcap 1.x. +# A message record's bytes besides its payload: opcode (1), length (8), +# channel id (2), sequence (4), and log and publish times (16). +_MESSAGE_RECORD_HEADER_BYTES = 31 +# Bytes one parsed ``mcap.records.Message`` holds beyond its payload. +_PARSED_MESSAGE_BYTES = 295 +# Bytes one entry of the log-time merge's heap holds, beyond the path in its +# ``row_id`` (``mcap_chunks``). +_HEAP_ENTRY_BYTES = 250 +# Most spare capacity of a ``bytearray`` grown by ``+=``, as a share of its bytes. +_BUFFER_GROWTH = 1 / 8 # A ``MessageIndex`` record after a chunk: opcode (1), length (8), channel id # (2) and entries length (4), then a log time and an offset (16) per message. _MESSAGE_INDEX_HEADER_BYTES = 15 _MESSAGE_INDEX_ENTRY_BYTES = 16 -# Summaries read concurrently within one listing task. A summary read is a few -# small ranged requests, so it is latency-bound on remote storage. +# Summary tail reads in flight within one listing task. A read is an open and +# one or two small ranged reads, so a listing waits on request latency. Up to +# twice this many fetched sections are held at once (``read_summaries``). _DEFAULT_SUMMARY_IO_CONCURRENCY = env_integer( - "RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", 16 + "RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", 32 ) @dataclass(frozen=True) class ListedSummary: - """A file's summary as ``read_summaries`` reads it: parsed, and as raw records. + """A file's summary as the listing reads it: parsed, and as raw records. ``section`` is the raw summary section, from which the listing copies the records each read task needs into the manifest (``mcap_task_summary``). @@ -73,6 +89,15 @@ class ListedSummary: section: memoryview +def parse_listed_summary( + section: Optional[memoryview], path: str +) -> Optional[ListedSummary]: + """Parse a section that ``read_summaries`` fetched; ``None`` stays ``None``.""" + if section is None: + return None + return ListedSummary(parse_summary_records(section, path), section) + + def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: """Return the summary of the MCAP file at ``path``, or ``None`` if it has none. @@ -82,53 +107,142 @@ def read_summary(filesystem: FileSystem, path: str) -> Optional["Summary"]: return read_summary_from(f, path) -def read_listed_summary(filesystem: FileSystem, path: str) -> Optional[ListedSummary]: - """Like ``read_summary``, but also keep the raw summary section.""" - with filesystem.open_input_file(path) as f: - section = read_summary_section_from(f, path) - if section is None: - return None - return ListedSummary(parse_summary_records(section, path), section) - - -def _read_summary_with_retry( - filesystem: "FileSystem", path: str, retried_io_errors: List[str] -) -> Optional[ListedSummary]: - """Read one file's summary, retrying errors that match ``retried_io_errors``.""" - return call_with_retry( - lambda: read_listed_summary(filesystem, path), - description=f"read MCAP summary for {path}", - match=retried_io_errors, - ) - - def read_summaries( filesystem: "FileSystem", items: Iterable[_T], path: Callable[[_T], str], + budget_bytes: int, io_concurrency: int = _DEFAULT_SUMMARY_IO_CONCURRENCY, -) -> Iterator[Tuple[_T, "Future[Optional[ListedSummary]]"]]: - """Read the summaries of ``items`` several at a time, yielding them in order. - - Each item comes with its pending read, so the caller decides what a failed - read means. A bounded window of in-flight reads keeps memory flat for a long - listing. +) -> Iterator[Tuple[_T, "Future[Optional[memoryview]]"]]: + """Fetch the summary sections of ``items`` concurrently, yielding them in order. + + Each item comes with its pending section, ``None`` for a file without one. + Threads only fetch; the caller parses (``parse_listed_summary``), since + parsing holds the GIL. Sections are fetched in file order while they fit + ``budget_bytes``, and the next file always fits when nothing is held. A + section counts against the budget until the caller asks for the next item. """ from ray.data.context import DataContext retried_io_errors = DataContext.get_current().retried_io_errors - window = max(1, io_concurrency * 2) - pending: Deque[Tuple[_T, Future]] = deque() + queue = iter(items) with ThreadPoolExecutor(max_workers=max(1, io_concurrency)) as pool: - for item in items: - summary_read = pool.submit( - _read_summary_with_retry, filesystem, path(item), retried_io_errors + window = _SectionWindow( + pool, filesystem, retried_io_errors, budget_bytes, io_concurrency + ) + while True: + window.start_tails(queue, path) + window.admit_sections() + fetched = window.next_section() + if fetched is None: + return + yield fetched.item, fetched.read + window.release(fetched) + + +@dataclass(frozen=True) +class _Fetch(Generic[_T]): + """One item's pending read: its tail, or its section of ``nbytes`` bytes.""" + + item: _T + read: "Future[Any]" + nbytes: int = 0 + + +class _SectionWindow(Generic[_T]): + """The tails and sections ``read_summaries`` holds, in file order.""" + + def __init__( + self, + pool: ThreadPoolExecutor, + filesystem: "FileSystem", + retried_io_errors: List[str], + budget_bytes: int, + io_concurrency: int, + ): + self._pool = pool + self._filesystem = filesystem + self._retried = retried_io_errors + self._budget = budget_bytes + self._concurrency = max(1, io_concurrency) + self._tails: Deque[_Fetch[_T]] = deque() + self._sections: Deque[_Fetch[_T]] = deque() + # Bytes of the sections admitted and not yet released. + self._held = 0 + + def start_tails(self, queue: Iterator[_T], path: Callable[[_T], str]) -> None: + """Keep ``io_concurrency`` tail reads in flight.""" + for item in itertools.islice(queue, self._concurrency - len(self._tails)): + read = self._pool.submit( + _read_tail, self._filesystem, path(item), self._retried ) - pending.append((item, summary_read)) - if len(pending) >= window: - yield pending.popleft() - while pending: - yield pending.popleft() + self._tails.append(_Fetch(item, read)) + + def admit_sections(self) -> None: + """Start section reads in file order while their sections fit the budget.""" + while self._tails and len(self._sections) < 2 * self._concurrency: + tail = self._tails[0] + if self._sections and not tail.read.done(): + return # Hand out a fetched section rather than wait. + nbytes = _section_bytes(tail.read) + if self._sections and self._held + nbytes > self._budget: + return + self._tails.popleft() + self._held += nbytes + read = self._pool.submit( + _read_section, self._filesystem, tail.read, self._retried + ) + self._sections.append(_Fetch(tail.item, read, nbytes)) + + def next_section(self) -> Optional[_Fetch[_T]]: + """The next admitted section in file order, or ``None`` when all are out.""" + return self._sections.popleft() if self._sections else None + + def release(self, fetched: _Fetch[_T]) -> None: + """Stop counting a section the caller is done with.""" + self._held -= fetched.nbytes + + +def _read_tail( + filesystem: "FileSystem", path: str, retried_io_errors: List[str] +) -> SummaryTail: + """Read one file's tail, retrying errors that match ``retried_io_errors``.""" + + def read() -> SummaryTail: + with filesystem.open_input_file(path) as f: + return read_tail(f, path) + + return call_with_retry( + read, description=f"read MCAP summary for {path}", match=retried_io_errors + ) + + +def _read_section( + filesystem: "FileSystem", + tail_read: "Future[SummaryTail]", + retried_io_errors: List[str], +) -> Optional[memoryview]: + """Read the section a finished tail read locates, or raise the tail's error.""" + tail = tail_read.result() + if tail.holds_section: + return read_section(None, tail) + + def read() -> Optional[memoryview]: + with filesystem.open_input_file(tail.path) as f: + return read_section(f, tail) + + return call_with_retry( + read, + description=f"read MCAP summary for {tail.path}", + match=retried_io_errors, + ) + + +def _section_bytes(tail_read: "Future[SummaryTail]") -> int: + """The length of the section a tail read locates; 0 when the read failed.""" + if tail_read.exception() is not None: + return 0 + return tail_read.result().section_bytes def chunk_unit_id(path: str, chunk_start_offset: int) -> str: @@ -185,26 +299,103 @@ def estimate_chunk_rows( return max(1, round(statistics.message_count * share)) -def chunk_run( - chunk_index: "ChunkIndex", - statistics: Optional["Statistics"], - total_uncompressed_size: int, +@dataclass(frozen=True) +class ReadMemory: + """How the listing sizes a read task's memory, to weigh its chunks. + + Attributes: + task_bytes: The partitioner's budget per read task. + log_time_order: Whether the read merges chunks by log time. + ranged_read_bytes: Most bytes one ranged read spans. + """ + + task_bytes: int + log_time_order: bool + ranged_read_bytes: int = ChunkReads().max_bytes + + +def chunk_runs( + path: str, + summary: "Summary", + chunk_indexes: Sequence["ChunkIndex"], row_bytes: int, -) -> FileChunk: - """Build the listing row for one chunk. - - ``size_bytes`` estimates the peak memory of reading the chunk, which is - what the partitioner budgets on: ``_READ_PEAK_FACTOR`` times the bytes its - rows take as tables. Those are the chunk's uncompressed records plus - ``row_bytes`` per message for the columns a row adds (see - ``MessageRowLayout.row_bytes``). A chunk that records no size counts as one - byte. + memory: ReadMemory, + payload_factor: int = 1, +) -> List[FileChunk]: + """Build the listing rows of one file's ``chunk_indexes``, weighted for packing. + + A task's peak is its table plus W + R (``read_overhead``). A chunk's share + of the table is its uncompressed records plus ``row_bytes`` per message, + plus ``_BUFFER_GROWTH``. Each payload byte counts ``payload_factor`` bytes, + for payloads the reader decodes into larger values. Its row weighs that share times + ``task_bytes / (task_bytes - W - R)``, so a full task leaves room for W + R, + or a whole task when W + R fills one. """ - num_rows = estimate_chunk_rows(chunk_index, statistics, total_uncompressed_size) - table_bytes = chunk_index.uncompressed_size + num_rows * row_bytes - return FileChunk( - unit_ids=(chunk_index.chunk_start_offset,), - num_rows=num_rows, - size_bytes=max(math.ceil(_READ_PEAK_FACTOR * table_bytes), 1), - fully_matched=False, + total_uncompressed = sum(c.uncompressed_size for c in summary.chunk_indexes) + counts = [ + estimate_chunk_rows(c, summary.statistics, total_uncompressed) + for c in chunk_indexes + ] + room = memory.task_bytes - read_overhead(path, chunk_indexes, counts, memory) + runs = [] + for chunk_index, num_rows in zip(chunk_indexes, counts): + payload = max( + 0, chunk_index.uncompressed_size - num_rows * _MESSAGE_RECORD_HEADER_BYTES + ) + records = chunk_index.uncompressed_size + (payload_factor - 1) * payload + table = (records + num_rows * row_bytes) * (1 + _BUFFER_GROWTH) + weight = table * memory.task_bytes / room if room > 0 else memory.task_bytes + runs.append( + FileChunk( + unit_ids=(chunk_index.chunk_start_offset,), + num_rows=num_rows, + size_bytes=max(math.ceil(weight), 1), + fully_matched=False, + ) + ) + return runs + + +def read_overhead( + path: str, + chunk_indexes: Sequence["ChunkIndex"], + counts: Sequence[int], + memory: ReadMemory, +) -> int: + """W + R of ``chunk_runs``: the largest chunk working set, plus one ranged + read when the chunks share reads.""" + working_set = max( + _working_set(c, n, path, memory.log_time_order) + for c, n in zip(chunk_indexes, counts) + ) + return working_set + _ranged_read(chunk_indexes, memory.ranged_read_bytes) + + +def _working_set( + chunk_index: "ChunkIndex", num_messages: int, path: str, log_time_order: bool +) -> int: + """Most bytes one chunk holds while it is read, its rows aside. + + The chunk record is held throughout, with either its span of the ranged + read or its decompressed and parsed messages, plus a heap entry each when + merging by log time. + """ + payload = max( + 0, chunk_index.uncompressed_size - num_messages * _MESSAGE_RECORD_HEADER_BYTES ) + decompressed = chunk_index.uncompressed_size if chunk_index.compression else 0 + per_message = _PARSED_MESSAGE_BYTES + if log_time_order: + per_message += _HEAP_ENTRY_BYTES + len(path) + parsed = decompressed + payload + num_messages * per_message + return chunk_index.compressed_size + max(chunk_index.chunk_length, parsed) + + +def _ranged_read(chunk_indexes: Sequence["ChunkIndex"], max_bytes: int) -> int: + """Most bytes a ranged read holds while chunks shorter than ``max_bytes`` + share it, within the chunks' extent (``mcap_io.plan_chunk_reads``).""" + if sum(c.chunk_length < max_bytes for c in chunk_indexes) < 2: + return 0 + start = min(c.chunk_start_offset for c in chunk_indexes) + end = max(c.chunk_start_offset + c.chunk_length for c in chunk_indexes) + return min(max_bytes, end - start) diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 77f90993d929..20f7c3f3a744 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -718,10 +718,10 @@ class DataContext: min_parallelism: This setting is deprecated. Use ``read_op_min_num_blocks`` instead. read_op_min_num_blocks: Minimum number of read output blocks for a dataset. - use_datasource_v2: When True, ``ray.data.read_parquet()`` and - ``ray.data.read_orc()`` route through the DataSourceV2 pipeline. - Defaults to True; override with ``RAY_DATA_USE_DATASOURCE_V2`` - (``0`` for V1, ``1`` for V2). + use_datasource_v2: When True, ``ray.data.read_parquet()``, + ``ray.data.read_orc()`` and ``ray.data.read_mcap()`` route through + the DataSourceV2 pipeline. Defaults to True; override with + ``RAY_DATA_USE_DATASOURCE_V2`` (``0`` for V1, ``1`` for V2). small_shuffle_chunk_size: Target uncompressed size in bytes of the row-group runs that ``FileShuffleConfig(small_chunks_shuffle=True)`` shuffles. Contiguous row groups are coalesced up to this size; a single row group diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index a3d90bee1d42..749a713af79e 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3451,6 +3451,26 @@ def read_mcap( the selected messages of the first chunk that holds one. A field that first appears in a later chunk or file is missing from that schema. + By default, ``read_mcap`` reads through the V2 datasource + (``DataContext.use_datasource_v2``). Its rows differ from the legacy reader's + in three ways: + + - ``schema_data`` and the new ``channel_metadata`` column are + dictionary-encoded, so a block stores each channel's values once. pandas + shows them as category columns. Cast them to ``binary`` and ``string`` + before you group by them or write blocks from several read tasks into one + Arrow IPC file. Cast ``schema_data`` before you mix this output with the + legacy reader's in one Parquet folder. + - A large file is split across read tasks. Its messages keep ``log_time`` + order across the whole file only with + ``DataContext.execution_options.preserve_order``, and the files of one + directory can come out in a different order than listed. + - ``override_num_blocks`` has no effect. + + To use the legacy reader, set ``DataContext.get_current().use_datasource_v2`` + to ``False`` or set ``RAY_DATA_USE_DATASOURCE_V2=0``. This also moves + ``read_parquet`` back to its legacy reader. + Examples: :noindex: @@ -3494,6 +3514,14 @@ def read_mcap( ... include_paths=True # doctest: +SKIP ... ) # doctest: +SKIP + Store ``schema_data`` as plain bytes, as the legacy reader does. + + >>> import pyarrow as pa # doctest: +SKIP + >>> def plain_schema_data(t: pa.Table) -> pa.Table: # doctest: +SKIP + ... i = t.schema.get_field_index("schema_data") # doctest: +SKIP + ... return t.set_column(i, "schema_data", t["schema_data"].cast(pa.binary())) # doctest: +SKIP + >>> ds = ds.map_batches(plain_schema_data, batch_format="pyarrow") # doctest: +SKIP + Args: paths: A single file or directory, or a list of file or directory paths. A list of paths can contain both files and directories. @@ -3535,7 +3563,9 @@ def read_mcap( ``id_column`` of a :class:`~ray.data.checkpoint.CheckpointConfig`. Turned on automatically when the current checkpoint config's ``id_column`` is ``row_id``. Requires the V2 datasource - (``DataContext.use_datasource_v2``). + (``DataContext.use_datasource_v2``). ``read_mcap`` refuses + ``CheckpointConfig(generated_id_column=...)``, which only + ``read_parquet`` supports. filesystem: The PyArrow filesystem implementation to read from. parallelism: This argument is deprecated. Use ``override_num_blocks`` argument. num_cpus: The number of CPUs to reserve for each parallel read worker. @@ -3562,7 +3592,9 @@ def read_mcap( override_num_blocks: Override the number of output blocks from all read tasks. By default, the number of output blocks is dynamically decided based on input data size and available resources. You shouldn't manually set this - value in most cases. + value in most cases. The V2 datasource ignores it and logs a warning: it + sizes each read task by bytes instead, about 128 MiB by default + (``RAY_DATA_MCAP_BIN_PACKING_BYTES``). label_selector: Labels required on the node where each read task runs. fallback_strategy: Alternative label requirements that Ray tries in order if @@ -3593,21 +3625,40 @@ def read_mcap( time_range = TimeRange(start_time=time_range[0], end_time=time_range[1]) ctx = DataContext.get_current() + checkpoint_config = ctx.checkpoint_config + if checkpoint_config is not None and checkpoint_config.has_generated_id_column: + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + raise InvalidCheckpointingConfig( + "`CheckpointConfig(generated_id_column=...)` works only with " + "`read_parquet`. For `read_mcap`, use " + '`CheckpointConfig(id_column="row_id")`. Its `row_id` column gives ' + "each message an id that stays the same on every run." + ) + from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( + ROW_ID_COLUMN, + ) + + if checkpoint_config is not None and checkpoint_config.id_column == ROW_ID_COLUMN: + # A checkpoint keyed on ``row_id`` needs the column in the read output. + include_row_id = True if ctx.use_datasource_v2: from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + _BIN_PACKING_BYTES, MCAPDatasourceV2, ) - from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( - ROW_ID_COLUMN, - ) - checkpoint_config = ctx.checkpoint_config - if ( - checkpoint_config is not None - and checkpoint_config.id_column == ROW_ID_COLUMN - ): - # A checkpoint keyed on ``row_id`` needs the column in the read output. - include_row_id = True + num_blocks = _get_num_output_blocks(parallelism, override_num_blocks) + if num_blocks != -1: + given = ( + "parallelism" if override_num_blocks is None else "override_num_blocks" + ) + logger.warning( + f"`read_mcap` ignores `{given}` with the V2 datasource. It sizes " + f"each read task by bytes instead, about " + f"{_BIN_PACKING_BYTES / 2**20:g} MiB " + "(RAY_DATA_MCAP_BIN_PACKING_BYTES)." + ) datasource_v2 = MCAPDatasourceV2( paths=paths if isinstance(paths, list) else [paths], topics=topics, @@ -3626,7 +3677,7 @@ def read_mcap( ) return _read_datasource_v2( datasource_v2, - parallelism=_get_num_output_blocks(parallelism, override_num_blocks), + parallelism=num_blocks, num_cpus=num_cpus, num_gpus=num_gpus, memory=memory, @@ -3643,8 +3694,8 @@ def read_mcap( if not log_time_order or include_row_id: raise NotImplementedError( - "`log_time_order=False` and `include_row_id` on `read_mcap` require " - "the V2 datasource. Enable it with " + "`log_time_order=False`, `include_row_id` and a checkpoint keyed on " + "`row_id` on `read_mcap` require the V2 datasource. Enable it with " "`ray.data.DataContext.get_current().use_datasource_v2 = True` " "(or set RAY_DATA_USE_DATASOURCE_V2=1)." ) diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 1c6a659e4486..5801da65beea 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -1,7 +1,9 @@ import importlib.util import json +import logging import os import re +from typing import Any import pyarrow as pa import pytest @@ -314,11 +316,16 @@ def test_read_mcap_nonexistent_file(ray_start_regular_shared): ds.materialize() # Force execution -@pytest.mark.parametrize("override_num_blocks", [1, 2]) +@pytest.mark.parametrize("argument", ["override_num_blocks", "parallelism"]) def test_read_mcap_override_num_blocks( - ray_start_regular_shared, tmp_path, override_num_blocks + ray_start_regular_shared, tmp_path, argument, datasource_v2, caplog ): - """Test override_num_blocks parameter.""" + """A read given ``override_num_blocks`` or ``parallelism`` still returns every + row. + + The V2 datasource sizes read tasks by bytes, so it ignores the argument and + warns, naming the argument given. + """ path = os.path.join(tmp_path, "blocks_test.mcap") messages = [ { @@ -329,13 +336,24 @@ def test_read_mcap_override_num_blocks( for i in range(3) ] create_test_mcap_file(path, messages) + read_args: dict[str, Any] = {argument: 2} + other = ( + "parallelism" if argument == "override_num_blocks" else "override_num_blocks" + ) - ds = ray.data.read_mcap(path, override_num_blocks=override_num_blocks) + # Ray's loggers do not propagate to the root logger caplog listens on. + read_api_logger = logging.getLogger("ray.data.read_api") + read_api_logger.addHandler(caplog.handler) + try: + ds = ray.data.read_mcap(path, **read_args) + finally: + read_api_logger.removeHandler(caplog.handler) - # Should still read all the data assert ds.count() == 3 - rows = ds.take_all() - assert len(rows) == 3 + assert len(ds.take_all()) == 3 + warnings = [record.message for record in caplog.records] + assert any(f"ignores `{argument}`" in m for m in warnings) == datasource_v2 + assert not any(f"ignores `{other}`" in m for m in warnings) def test_read_mcap_file_extensions(ray_start_regular_shared, tmp_path): @@ -648,6 +666,47 @@ def test_read_mcap_checkpoint_config_turns_on_row_id( assert "row_id" in ray.data.read_mcap(simple_mcap_file).schema().names +def test_read_mcap_v1_refuses_a_row_id_checkpoint( + ray_start_regular_shared, simple_mcap_file, datasource_v2 +): + """A checkpoint keyed on ``row_id`` needs the column, which only the V2 + datasource builds. On the V1 legacy design ``read_mcap`` refuses it at once, + instead of reading without the ids the checkpoint needs.""" + from ray.data.checkpoint import CheckpointConfig + + if datasource_v2: + pytest.skip("V2 turns row_id on") + ctx = ray.data.DataContext.get_current() + ctx.checkpoint_config = CheckpointConfig( + id_column="row_id", checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" + ) + + with pytest.raises(NotImplementedError, match="checkpoint keyed on `row_id`"): + ray.data.read_mcap(simple_mcap_file) + + +def test_read_mcap_refuses_generated_id_column( + ray_start_regular_shared, simple_mcap_file +): + """A checkpoint config that asks for generated ids fails at ``read_mcap``. + + Only ``read_parquet`` builds generated ids. The name ``row_id`` must not turn + on MCAP's own ``row_id`` column instead. + """ + from ray.data.checkpoint import CheckpointConfig + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + ctx = ray.data.DataContext.get_current() + ctx.checkpoint_config = CheckpointConfig( + generated_id_column="row_id", checkpoint_path="/tmp/ray_data_mcap_ckpt_unused" + ) + + with pytest.raises( + InvalidCheckpointingConfig, match="works only with `read_parquet`" + ): + ray.data.read_mcap(simple_mcap_file) + + def test_read_mcap_v2_schema_matches_rows( ray_start_regular_shared, multi_topic_mcap_file, datasource_v2 ): diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index b108437f0ac6..3e4abd8fe0d2 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -44,7 +44,8 @@ def write_mcap( index=True, encodings=None, ): - """Write ``messages`` (dicts with topic, log_time, data) to ``path``. + """Write ``messages`` (dicts with topic, log_time, data, and optionally + publish_time and sequence) to ``path``. The default ``chunk_size=1`` closes a chunk after every message, so N messages make N chunks, each a possible split point. ``encodings`` @@ -84,6 +85,7 @@ def write_mcap( log_time=message["log_time"], publish_time=message.get("publish_time", message["log_time"]), data=data, + sequence=message.get("sequence", 0), ) writer.finish() @@ -393,9 +395,26 @@ def __exit__(self, *exc_info): def chunks_read(seek_log: SeekLog, path: str) -> List[int]: - """Start offsets of the chunks of ``path`` that were read. + """Start offsets of the reads of chunk records in ``path``, in order. - A chunk record is read in one read from its start offset. + Each read starts at a chunk record and fetches that chunk, or a run of + chunks fetched together (``mcap_io.read_chunks``). """ starts = {c.chunk_start_offset for c in summary_of(path).chunk_indexes} return [offset for offset in seek_log.seeks[path] if offset in starts] + + +def record_parsed_chunks(monkeypatch) -> List[int]: + """Record the offset of every chunk the reader parses out of its reads.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + + real_read_chunks = mcap_chunks.read_chunks + offsets: List[int] = [] + + def read_chunks(f, chunk_indexes, reads): + for chunk_index, chunk in real_read_chunks(f, chunk_indexes, reads): + offsets.append(chunk_index.chunk_start_offset) + yield chunk_index, chunk + + monkeypatch.setattr(mcap_chunks, "read_chunks", read_chunks) + return offsets diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py index 578a15a8da2a..58a1f018ca1e 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_data_column.py @@ -51,6 +51,7 @@ ) ONLY_A = MCAPSelection.create(["/a"], None, None) +SUMMARY_BUDGET_BYTES = 128 * 2**20 # the default read task budget def write_channel( @@ -111,6 +112,7 @@ def infer( filesystem or LocalFileSystem(), paths[:sampled], lambda: iter(paths), + SUMMARY_BUDGET_BYTES, ) @@ -229,9 +231,10 @@ def no_listing(): raise AssertionError("listed although the sample types data") sampled = paths[4:] - assert infer_data_type(ONLY_A, LocalFileSystem(), sampled, no_listing) == struct_of( - "walked" + planned = infer_data_type( + ONLY_A, LocalFileSystem(), sampled, no_listing, SUMMARY_BUDGET_BYTES ) + assert planned == struct_of("walked") def test_unreadable_sampled_file_is_skipped(tmp_path): diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index 2f7d3d6d2ea4..f1b481b01ff7 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -12,9 +12,12 @@ from pyarrow.fs import LocalFileSystem from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( + ChunkReads, buffered, parse_summary_records, + plan_chunk_reads, read_chunk, + read_chunks, read_summary_from, ) from ray.data.tests.unit.datasource_v2.mcap_testing import ( @@ -231,6 +234,130 @@ def test_chunk_is_read_in_one_read(tmp_path): assert f.reads == 1 +def chunk_at(offset: int, length: int = 100): + """The ``ChunkIndex`` of a chunk record ``length`` bytes long at ``offset``.""" + from mcap.records import ChunkIndex + + return ChunkIndex( + chunk_length=length, + chunk_start_offset=offset, + compression="zstd", + compressed_size=length, + message_end_time=0, + message_index_length=0, + message_index_offsets={}, + message_start_time=0, + uncompressed_size=length, + ) + + +# Chunk records of 100 B, given by offset, in the order they are read. +PLANS = { + "neighbours_within_the_gap": ( + [0, 110, 220], + ChunkReads(max_bytes=1000, max_gap_bytes=10, first_bytes=1000), + [[0, 110, 220]], + ), + "gap_past_the_limit": ( + [0, 111], + ChunkReads(max_bytes=1000, max_gap_bytes=10, first_bytes=1000), + [[0], [111]], + ), + "span_past_max_bytes": ( + [0, 100, 200], + ChunkReads(max_bytes=250, max_gap_bytes=0, first_bytes=250), + [[0, 100], [200]], + ), + "earlier_in_the_file": ( + [200, 0, 100], + ChunkReads(max_bytes=1000, max_gap_bytes=1000, first_bytes=1000), + [[200], [0, 100]], + ), + "zero_reads_each_chunk_alone": ( + [0, 100, 200], + ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), + [[0], [100], [200]], + ), + # The limit doubles per read: 100, 200, 400, then max_bytes (500). + "first_reads_grow": ( + list(range(0, 1500, 100)), + ChunkReads(max_bytes=500, max_gap_bytes=0, first_bytes=100), + [ + [0], + [100, 200], + [300, 400, 500, 600], + [700, 800, 900, 1000, 1100], + [1200, 1300, 1400], + ], + ), +} + + +@pytest.mark.parametrize("offsets, reads, expected", PLANS.values(), ids=PLANS.keys()) +def test_plan_groups_chunks_within_the_limits(offsets, reads, expected): + """A chunk joins the read before it only if it lies after that read, within + the gap limit, and keeps the read within its size limit.""" + planned = plan_chunk_reads([chunk_at(offset) for offset in offsets], reads) + + assert [[c.chunk_start_offset for c in r.chunk_indexes] for r in planned] == ( + expected + ) + for ranged_read in planned: + assert ranged_read.start == ranged_read.chunk_indexes[0].chunk_start_offset + assert ranged_read.end == ranged_read.chunk_indexes[-1].chunk_start_offset + 100 + + +def test_a_chunk_longer_than_max_bytes_is_read_alone(): + """A chunk record longer than ``max_bytes`` is one read of its own.""" + chunks = [chunk_at(0, length=500), chunk_at(500), chunk_at(600)] + + planned = plan_chunk_reads(chunks, ChunkReads(max_bytes=250, max_gap_bytes=0)) + + assert [len(r.chunk_indexes) for r in planned] == [1, 2] + + +@pytest.mark.parametrize( + "reads, expected_reads", + [(ChunkReads(), 1), (ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0), 9)], + ids=["neighbours_together", "one_per_chunk"], +) +def test_read_chunks_returns_what_read_chunk_returns(tmp_path, reads, expected_reads): + """``read_chunks`` yields the chunks ``read_chunk`` reads one by one, in the + order asked, fetching all nine in one read unless told to read each alone.""" + path = os.path.join(tmp_path, "chunked.mcap") + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + chunk_indexes = sorted(summary.chunk_indexes, key=lambda c: c.chunk_start_offset) + expected = [read_chunk(CountingFile(path), c) for c in chunk_indexes] + + f = CountingFile(path) + chunks = list(read_chunks(f, chunk_indexes, reads)) + + assert [chunk_index for chunk_index, _ in chunks] == chunk_indexes + assert [chunk for _, chunk in chunks] == expected + assert f.reads == expected_reads + + +def test_read_chunks_fetches_a_read_when_its_first_chunk_is_needed(tmp_path): + """Nothing is fetched until a chunk is asked for, and each read only when + its first chunk is.""" + path = os.path.join(tmp_path, "chunked.mcap") + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + one_per_chunk = ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0) + f = CountingFile(path) + + chunks = read_chunks(f, summary.chunk_indexes, one_per_chunk) + reads = [f.reads] + for _ in range(3): + next(chunks) + reads.append(f.reads) + + assert reads == [0, 1, 2, 3] + + def test_buffered_scan_reads_in_large_blocks(tmp_path): """A linear scan through ``buffered`` yields the same records with one read per buffer's worth of the file, and closing the buffer leaves the file open.""" diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py index dc1b9167964f..ca02d83a12e4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_listing.py @@ -12,17 +12,22 @@ import pytest from ray.data._internal.datasource_v2.common.online_bin_packer import OnlineBinPacker -from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_datasource_v2, + mcap_summary, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( MCAPDatasourceV2, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import SummaryTail from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( ROW_ID_COLUMN, TimeRange, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( - _READ_PEAK_FACTOR, + _BUFFER_GROWTH, chunk_unit_id, + read_summaries, ) from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data.tests.unit.datasource_v2.mcap_testing import ( @@ -333,13 +338,15 @@ def test_unreadable_file_gets_a_read_task_of_its_own(tmp_path): [(False, False, False), (True, False, False), (True, True, True)], ) def test_listed_sizes_bound_the_table_a_read_builds( - tmp_path, include_metadata, include_row_id, include_paths + tmp_path, monkeypatch, include_metadata, include_row_id, include_paths ): - """A chunk's listed size is ``_READ_PEAK_FACTOR`` times a bound on the bytes - its rows take as a table, and the bound is tight: a read of the file builds - a table between 80% and 100% of it, whichever columns the rows carry.""" + """Under a budget far above the working set, a chunk's listed size is a + bound on the bytes its rows take as a table, plus ``_BUFFER_GROWTH``. The + bound is tight: a read of the file builds a table between 80% and 100% of + it, whichever columns the rows carry.""" path = os.path.join(tmp_path, "mixed.mcap") write_mixed_channels(path, num_messages=400, payload_bytes=1000) + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", 2**62) datasource = MCAPDatasourceV2( [path], include_metadata=include_metadata, @@ -351,7 +358,7 @@ def test_listed_sizes_bound_the_table_a_read_builds( table = read_all(datasource, manifest) listed = sum(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) - bound = listed / _READ_PEAK_FACTOR + bound = listed / (1 + _BUFFER_GROWTH) assert 0.8 * bound <= table.nbytes <= bound @@ -428,6 +435,127 @@ def test_packer_packs_per_listing_shard(tmp_path): assert len(per_shard) <= len(whole) + 1 +def listed_rows(datasource: MCAPDatasourceV2) -> List[Tuple[str, List[int], int]]: + """Every listing row in listing order: its path, chunk offsets and weight.""" + return [ + (str(path), unit_ids(md), int(md["size_bytes"])) + for manifest in list_manifests(datasource) + for path, md in zip(manifest.paths, manifest.file_chunk_metadatas) + ] + + +def test_listing_order_does_not_depend_on_summary_concurrency(tmp_path, monkeypatch): + """Listing 30 files with one summary read at a time or 32 at once gives the + same rows in the same order, so the packer forms the same read tasks.""" + for i in range(30): + write_mcap(os.path.join(tmp_path, f"f{i:02d}.mcap"), round_robin_messages(4)) + + monkeypatch.setenv("RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", "1") + one_at_a_time = listed_rows(MCAPDatasourceV2([str(tmp_path)])) + monkeypatch.setenv("RAY_DATA_MCAP_SUMMARY_IO_CONCURRENCY", "32") + many_at_once = listed_rows(MCAPDatasourceV2([str(tmp_path)])) + + assert len(one_at_a_time) == 30 * 4 + assert many_at_once == one_at_a_time + + +IO_CONCURRENCY = 2 +SECTION_BUDGET = 100 # bytes of summary sections held at once + + +def fake_tail(path: str, section_bytes: int, *, in_tail: bool = True) -> SummaryTail: + """A tail whose section is ``section_bytes`` long and needs no further read + when ``in_tail``; otherwise reading the rest opens the file.""" + data = bytes(1 + section_bytes + 8) # a byte before the section, magic after + # A section that starts before the tail's first byte needs another read. + return SummaryTail(path, start=0 if in_tail else 2, data=data, summary_start=1) + + +class FailingFileSystem: + """A filesystem whose files cannot be opened, so a section read past the + tail fails.""" + + def open_input_file(self, path): + raise OSError(f"cannot open {path}") + + +@dataclasses.dataclass +class WindowLog: + """What ``read_summaries`` holds after each admission: bytes, sections, tails.""" + + held: List[int] = dataclasses.field(default_factory=list) + sections: List[int] = dataclasses.field(default_factory=list) + tails: List[int] = dataclasses.field(default_factory=list) + + +def fetch_sections(monkeypatch, tails, filesystem=None) -> Tuple[List[Any], WindowLog]: + """Run ``read_summaries`` over ``tails`` (a path to tail or error mapping), + returning each item's section or error, in order, and the window's log.""" + log = WindowLog() + + def read_tail(filesystem, path, retried_io_errors): + if isinstance(tails[path], Exception): + raise tails[path] + return tails[path] + + real_admit = mcap_summary._SectionWindow.admit_sections + + def admit_sections(window): + real_admit(window) + log.held.append(window._held) + log.sections.append(len(window._sections)) + log.tails.append(len(window._tails)) + + monkeypatch.setattr(mcap_summary, "_read_tail", read_tail) + monkeypatch.setattr(mcap_summary._SectionWindow, "admit_sections", admit_sections) + fetched = [] + for path, section_read in read_summaries( + filesystem, list(tails), lambda p: p, SECTION_BUDGET, IO_CONCURRENCY + ): + error = section_read.exception() + section = None if error else section_read.result() + fetched.append((path, error or len(section or b""))) + return fetched, log + + +@pytest.mark.parametrize( + "sizes", + [[30, 50, 20, 250, 10, 40, 60, 0], [1] * 20], + ids=["by_bytes", "by_count"], +) +def test_read_summaries_bounds_held_sections(monkeypatch, sizes): + """Sections come back in file order. The sections held fit the byte budget, + except an oversize one held alone, and there are at most twice + ``io_concurrency`` of them, with at most ``io_concurrency`` tails ahead.""" + tails = {f"f{i}": fake_tail(f"f{i}", n) for i, n in enumerate(sizes)} + + fetched, log = fetch_sections(monkeypatch, tails) + + assert fetched == [(f"f{i}", n) for i, n in enumerate(sizes)] + for held, sections in zip(log.held, log.sections): + assert held <= SECTION_BUDGET or sections == 1 + assert max(log.sections) <= 2 * IO_CONCURRENCY + assert max(log.tails) <= IO_CONCURRENCY + + +def test_read_summaries_hands_a_failed_read_to_its_file(monkeypatch): + """A failed tail read and a failed section read each come back as that + file's error, in its place, and the files after them are still read.""" + tails = { + "ok_0": fake_tail("ok_0", 10), + "bad_tail": OSError("tail"), + "bad_section": fake_tail("bad_section", 10, in_tail=False), + "ok_1": fake_tail("ok_1", 20), + } + + fetched, _ = fetch_sections(monkeypatch, tails, FailingFileSystem()) + + assert [path for path, _ in fetched] == list(tails) + assert fetched[0] == ("ok_0", 10) and fetched[3] == ("ok_1", 20) + assert str(fetched[1][1]) == "tail" + assert str(fetched[2][1]) == "cannot open bad_section" + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py new file mode 100644 index 000000000000..4de4625b31a3 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_read_memory.py @@ -0,0 +1,218 @@ +"""Unit tests for the estimate of a read task's peak memory (``chunk_runs``). + +Each constant is pinned by measuring the objects it stands for, and the +estimate is checked against the measured peak of a real read. +""" + +import gc +import importlib.util +import json +import os +import random +import sys +import tracemalloc +from collections import deque + +import pyarrow as pa +import pytest + +from ray.data._internal.datasource_v2.formats.mcap import mcap_datasource_v2 +from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( + SelectedMessageReader, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import read_chunk +from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + _BUFFER_GROWTH, + _HEAP_ENTRY_BYTES, + _MESSAGE_RECORD_HEADER_BYTES, + _PARSED_MESSAGE_BYTES, + ReadMemory, + read_overhead, +) +from ray.data.tests.unit.datasource_v2.mcap_testing import ( + as_read_task, + list_manifests, + pack, + scanner_for, + summary_of, + write_mcap, +) + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +PAYLOAD_BYTES = 48 # small messages, where per-message overhead dominates +NUM_MESSAGES = 20_000 +# Log times in nanoseconds since the epoch, which Python stores in larger ints +# than small numbers. +EPOCH_NS = 1_700_000_000 * 10**9 +SMALL_CHUNKS = 64 << 10 # chunk size that splits the file into many chunks +ONE_CHUNK = 1 << 30 + + +def write_small_messages(path: str, chunk_size: int, encoding: str = "cdr") -> str: + """Write ``NUM_MESSAGES`` small payloads on one channel, timed and numbered + as a recorder writes them. + + A CDR payload is ``PAYLOAD_BYTES``, half random so that zstd compresses it + partly. A JSON payload is an object of two numbers, which the reader decodes. + """ + rng = random.Random(0) + half = PAYLOAD_BYTES // 2 + + def payload(i: int) -> bytes: + if encoding == "json": + return json.dumps({"x": rng.random(), "i": i}).encode() + return rng.randbytes(half) + bytes(half) + + messages = [ + { + "topic": "/a", + "log_time": EPOCH_NS + i * 1000, + "sequence": i, + "data": payload(i), + } + for i in range(NUM_MESSAGES) + ] + write_mcap(path, messages, chunk_size=chunk_size, message_encoding=encoding) + return path + + +def traced(run): + """``(kept, peak)``: bytes ``run()`` leaves allocated, and its peak.""" + gc.collect() + tracemalloc.start() + before = tracemalloc.get_traced_memory()[0] + result = run() + kept, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + del result + return kept - before, peak - before + + +def listed_overhead(path: str, manifest, log_time_order: bool) -> int: + """W + R of the listing of every chunk of ``path``.""" + counts = [int(md["num_rows"]) for md in manifest.file_chunk_metadatas] + memory = ReadMemory(mcap_datasource_v2._BIN_PACKING_BYTES, log_time_order) + return read_overhead(path, summary_of(path).chunk_indexes, counts, memory) + + +def test_message_record_header_is_31_bytes(): + """A message record is its payload plus the header the estimate subtracts.""" + from mcap.data_stream import RecordBuilder + from mcap.records import Message + + builder = RecordBuilder() + message = Message(channel_id=1, log_time=2, data=b"x", publish_time=3, sequence=4) + message.write(builder) + + assert len(builder.end()) - 1 == _MESSAGE_RECORD_HEADER_BYTES + + +def test_parsed_message_and_heap_entry_sizes_are_bounded(tmp_path): + """A parsed message and a log-time heap entry take at most their constants + beyond their payload and path, and at least 80% of them.""" + from mcap.stream_reader import breakup_chunk + + path = write_small_messages(os.path.join(tmp_path, "one.mcap"), ONE_CHUNK) + summary = summary_of(path) + with open(path, "rb") as f: + chunk = read_chunk(f, summary.chunk_indexes[0]) + + parsed, _ = traced(lambda: breakup_chunk(chunk)) + # The unordered read peaks at the end of parsing, with the decompressed + # records. The ordered read peaks with every message in its heap, after + # those records are freed. + peaks = {} + for ordered in (True, False): + reader = SelectedMessageReader(MCAPSelection(), log_time_order=ordered) + with open(path, "rb") as f: + messages = reader.iter_chunks(f, path, summary, None) + peaks[ordered] = traced(lambda: deque(messages, maxlen=0))[1] + heap = peaks[True] - peaks[False] + summary.chunk_indexes[0].uncompressed_size + + message_overhead = parsed / NUM_MESSAGES - PAYLOAD_BYTES + heap_entry = heap / NUM_MESSAGES - len(path) + assert 0.8 * _PARSED_MESSAGE_BYTES <= message_overhead <= _PARSED_MESSAGE_BYTES + assert 0.8 * _HEAP_ENTRY_BYTES <= heap_entry <= _HEAP_ENTRY_BYTES + + +def test_bytearray_spare_capacity_is_an_eighth(): + """A ``bytearray`` grown by ``+=``, as the table builder grows its payload + buffer, holds up to ``_BUFFER_GROWTH`` of its bytes in spare capacity.""" + empty = sys.getsizeof(bytearray()) + buffer, worst = bytearray(), 0.0 + + while len(buffer) < 4 << 20: + buffer += b"y" * PAYLOAD_BYTES + if len(buffer) >= 1 << 20: # past the fixed slack of small sizes + worst = max(worst, (sys.getsizeof(buffer) - empty) / len(buffer)) + + assert 1 + _BUFFER_GROWTH - 0.01 <= worst <= 1 + _BUFFER_GROWTH + 1e-4 + + +@pytest.mark.parametrize("encoding", ["cdr", "json"]) +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("chunk_size", [ONE_CHUNK, SMALL_CHUNKS]) +def test_estimate_bounds_the_measured_peak_of_a_read( + tmp_path, log_time_order, chunk_size, encoding +): + """The estimate T + W + R for a task that reads the whole file is at least + the measured peak of reading it, and less than twice that. JSON payloads + are decoded into Python values, which hold several times their bytes.""" + path = write_small_messages( + os.path.join(tmp_path, "small.mcap"), chunk_size, encoding + ) + datasource = MCAPDatasourceV2([path], log_time_order=log_time_order) + (manifest,) = list_manifests(datasource) + task = as_read_task(manifest) + reader = scanner_for(datasource).create_reader() + deque(reader.read(task), maxlen=0) # Warm imports outside the measurement. + pool = pa.default_memory_pool() + arrow_before = max(pool.bytes_allocated(), pool.max_memory()) + + _, python_peak = traced(lambda: deque(reader.read(task), maxlen=0)) + + measured = python_peak + max(0, pool.max_memory() - arrow_before) + # Rows weigh T * budget / (budget - W - R), which gives back T. + budget = mcap_datasource_v2._BIN_PACKING_BYTES + overhead = listed_overhead(path, manifest, log_time_order) + weights = sum(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) + estimate = weights * (budget - overhead) / budget + overhead + assert measured <= estimate < 2 * measured + + +def test_packer_leaves_room_for_the_working_set(tmp_path, monkeypatch): + """With a budget of W + R plus 3.5 chunks' tables, four chunks would fit by + their tables alone. Weighed with room for W + R, a fourth full chunk does + not fit, and every task stays within the budget.""" + path = write_small_messages(os.path.join(tmp_path, "small.mcap"), SMALL_CHUNKS) + datasource = MCAPDatasourceV2([path], log_time_order=False) + (manifest,) = list_manifests(datasource) + overhead = listed_overhead(path, manifest, log_time_order=False) + heaviest = max(int(md["size_bytes"]) for md in manifest.file_chunk_metadatas) + table = heaviest * (1 - overhead / mcap_datasource_v2._BIN_PACKING_BYTES) + budget = int(overhead + 3.5 * table) + monkeypatch.setattr(mcap_datasource_v2, "_BIN_PACKING_BYTES", budget) + + (relisted,) = list_manifests(datasource) + tasks = pack(datasource) + + weights = [int(md["size_bytes"]) for md in relisted.file_chunk_metadatas] + assert 4 * table < budget < 4 * max(weights) + task_weights = [ + sum(int(md["size_bytes"]) for md in task.file_chunk_metadatas) for task in tasks + ] + assert max(task_weights) <= budget + chunks = [sum(len(md["unit_ids"]) for md in t.file_chunk_metadatas) for t in tasks] + assert sum(chunks) == len(summary_of(path).chunk_indexes) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 8f726de23121..c90fd47d3dbf 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -13,9 +13,13 @@ import pytest from ray.data._internal.datasource_v2.formats.mcap import mcap_message_rows +from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( + SelectedMessageReader, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( MCAPDatasourceV2, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ChunkReads from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( _MessageTableBuilder, message_schema, @@ -39,6 +43,7 @@ list_manifests, mixed_encoding_messages, read_all, + record_parsed_chunks, round_robin_messages, scanner_for, selected_messages, @@ -316,13 +321,17 @@ def test_file_without_a_chunk_index_is_sorted_by_its_task( @pytest.mark.parametrize("log_time_order", [True, False]) -def test_limit_reads_only_the_chunks_it_needs(chunked_file, log_time_order): - """A limit of two reads the two chunks holding the first two messages and no - other chunk, in log-time and in file order.""" +def test_limit_parses_only_the_chunks_it_needs( + chunked_file, log_time_order, monkeypatch +): + """A limit of two parses the two chunks holding the first two messages and no + other chunk, in log-time and in file order. The small file's chunks are + fetched in one ranged read.""" datasource = MCAPDatasourceV2([chunked_file], log_time_order=log_time_order) (manifest,) = list_manifests(datasource) seek_log = SeekLog() scanner = dataclasses.replace(scanner_for(datasource), filesystem=seek_log) + parsed = record_parsed_chunks(monkeypatch) table = read_all(datasource, manifest, scanner=scanner.push_limit(2)) @@ -330,7 +339,88 @@ def test_limit_reads_only_the_chunks_it_needs(chunked_file, log_time_order): offsets = sorted( c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes ) - assert chunks_read(seek_log, chunked_file) == offsets[:2] + assert parsed == offsets[:2] + assert chunks_read(seek_log, chunked_file) == offsets[:1] + + +def selected_rows(path, chunk_reads, topics=None, log_time_order=True, f=None): + """``(log_time, row_id)`` of each message the reader selects from ``path``.""" + reader = SelectedMessageReader( + MCAPSelection.create(topics, None, None), log_time_order, chunk_reads + ) + stream = f if f is not None else open(path, "rb") + with stream: + messages = reader.iter_chunks(stream, path, summary_of(path), None) + return [(message.log_time, row_id) for _, _, message, row_id in messages] + + +ONE_CHUNK_PER_READ = ChunkReads(max_bytes=0, max_gap_bytes=0, first_bytes=0) +# Small enough that the ``chunked_file`` fixture takes several reads, some of +# them spanning more than one chunk. +SMALL_READS = ChunkReads(max_bytes=400, max_gap_bytes=100, first_bytes=100) + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("topics", [None, ["/a"]]) +def test_rows_do_not_depend_on_how_chunks_are_fetched( + chunked_file, log_time_order, topics +): + """The selected messages, their order and their row ids are the same whether + each chunk is fetched alone or with its neighbours, with or without a topic + filter.""" + expected = selected_rows(chunked_file, ONE_CHUNK_PER_READ, topics, log_time_order) + + rows = selected_rows(chunked_file, SMALL_READS, topics, log_time_order) + + assert rows == expected + assert len(rows) == (3 if topics else CHUNKED_FILE_MESSAGES) + + +@pytest.mark.parametrize( + "max_gap_bytes, expected_reads", + # /a is in chunks 0, 3 and 6: a 1 KiB gap spans the two chunks in between. + [(0, [0, 3, 6]), (1024, [0])], +) +def test_a_gap_up_to_the_limit_is_read_through( + chunked_file, max_gap_bytes, expected_reads +): + """Chunks a topic filter selects share one read when the chunks between them + fit in ``max_gap_bytes``, and are read one by one otherwise.""" + seek_log = SeekLog() + chunk_reads = ChunkReads(max_bytes=1 << 20, max_gap_bytes=max_gap_bytes) + + selected_rows( + chunked_file, + chunk_reads, + topics=["/a"], + f=seek_log.open_input_file(chunked_file), + ) + + offsets = sorted( + c.chunk_start_offset for c in summary_of(chunked_file).chunk_indexes + ) + assert chunks_read(seek_log, chunked_file) == [offsets[i] for i in expected_reads] + + +def test_log_time_merge_fetches_chunks_in_the_order_it_expands_them(tmp_path): + """The merge expands chunks by first log time. A chunk that comes later in + time but earlier in the file starts a new read, and the rows stay in + log-time order.""" + # One message per chunk; chunk 1 holds the earliest time. + path = os.path.join(tmp_path, "out_of_order.mcap") + messages = [ + {"topic": "/a", "data": {"seq": seq}, "log_time": log_time} + for seq, log_time in enumerate((20, 10, 30)) + ] + write_mcap(path, messages) + offsets = sorted(c.chunk_start_offset for c in summary_of(path).chunk_indexes) + seek_log = SeekLog() + + rows = selected_rows(path, ChunkReads(), f=seek_log.open_input_file(path)) + + assert [log_time for log_time, _ in rows] == [10, 20, 30] + # Chunk 1 is read alone; chunk 0 then starts a read that spans chunk 2. + assert chunks_read(seek_log, path) == [offsets[1], offsets[0]] def test_limit_never_decodes_the_messages_past_it(tmp_path):