diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 043a788d6fe4..eb30cf6e876e 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -143,6 +143,7 @@ def iter_chunks( for c in self.candidate_chunks(summary, selected) if offsets is None or c.chunk_start_offset in offsets ] + time_bounds = (self._selection.start_time, self._selection.end_time) declared = _Declared( dict(summary.channels), dict(summary.schemas), @@ -153,17 +154,64 @@ def iter_chunks( chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt) for chunk_index, chunk in chunks: yield from self._decoded_messages( - f, path, summary, chunk_index, chunk, selected, declared, on_corrupt + f, + path, + summary, + chunk_index, + chunk, + selected, + declared, + time_bounds, + on_corrupt, ) return yield from self._merge_chunks_by_log_time( - f, path, summary, chunk_indexes, selected, declared, on_corrupt + f, path, summary, chunk_indexes, selected, declared, time_bounds, on_corrupt ) + def iter_chunk_messages( + self, + f: Any, + path: str, + summary: "Summary", + chunk_indexes: List["ChunkIndex"], + file_summary: Optional[Callable[[], "Summary"]] = None, + *, + selected: Optional[Set[int]] = None, + ) -> Iterator[Tuple["ChunkIndex", List[_Selected]]]: + """Yield each chunk with its selected messages, in file order. + + Unlike ``iter_chunks``, a chunk's messages come together and are not + cut to the time range, so a caller can keep or drop whole chunks. + Chunks close together in the file still share a ranged read. A chunk + that cannot be decoded raises. + """ + if selected is None: + selected = self._selection.selected_channel_ids( + summary.channels, summary.schemas + ) + declared = _Declared( + dict(summary.channels), + dict(summary.schemas), + set(), + file_summary or (lambda: summary), + ) + ordered = sorted(chunk_indexes, key=lambda c: c.chunk_start_offset) + for chunk_index, chunk in read_chunks(f, ordered, self._chunk_reads): + assert chunk is not None, "without on_corrupt, a bad chunk raises" + messages = self._chunk_messages( + f, path, summary, chunk_index, chunk, selected, declared, (None, None) + ) + yield chunk_index, list(messages) + def candidate_chunks( self, summary: "Summary", selected: Set[int] ) -> List["ChunkIndex"]: - """The file's chunks that may hold a selected message, in file order.""" + """The file's chunks that may hold a selected message, in file order. + + The indexer lists exactly these, so the indexer and the reader agree + on which chunks a selection reads. + """ # A summary that repeats no channel record says nothing about which # chunk holds a selected message. Every chunk in the time range is read, # and the channels declared inside them are filtered as they appear. @@ -191,6 +239,7 @@ def _merge_chunks_by_log_time( chunk_indexes: List["ChunkIndex"], selected: Set[int], declared: _Declared, + time_bounds: Tuple[Optional[int], Optional[int]], on_corrupt: Optional[OnCorrupt], ) -> Iterator[_Selected]: """Yield the selected messages of the chunks in log-time order. @@ -225,6 +274,7 @@ def _merge_chunks_by_log_time( chunk, selected, declared, + time_bounds, on_corrupt, with_index=True, ): @@ -250,6 +300,7 @@ def _decoded_messages( chunk: Optional["Chunk"], selected: Set[int], declared: "_Declared", + time_bounds: Tuple[Optional[int], Optional[int]], on_corrupt: Optional[OnCorrupt], with_index: bool = False, ) -> Iterator[Any]: @@ -265,7 +316,15 @@ def _decoded_messages( return try: yield from self._chunk_messages( - f, path, summary, chunk_index, chunk, selected, declared, with_index + f, + path, + summary, + chunk_index, + chunk, + selected, + declared, + time_bounds, + with_index, ) except OSError: raise @@ -283,6 +342,7 @@ def _chunk_messages( chunk: "Chunk", selected: Set[int], declared: "_Declared", + time_bounds: Tuple[Optional[int], Optional[int]], with_index: bool = False, ) -> Iterator[Any]: """Decompress one chunk and yield its selected messages in file order. @@ -295,6 +355,7 @@ def _chunk_messages( from mcap.records import Channel, Message, Schema from mcap.stream_reader import breakup_chunk + start_time, end_time = time_bounds index = -1 for record in breakup_chunk(chunk): if isinstance(record, Message): @@ -304,7 +365,9 @@ def _chunk_messages( ) if not self._channel_is_selected(channel, schema, summary, selected): continue - if not self._selection.in_time_range(record.log_time): + if start_time is not None and record.log_time < start_time: + continue + if end_time is not None and record.log_time >= end_time: continue row_id = message_row_id(path, chunk_index.chunk_start_offset, index) item = (schema, channel, record, row_id) @@ -405,7 +468,13 @@ def _find_in_earlier_chunks( break def iter_unindexed( - self, f: Any, path: str, limit: Optional[int] = None + self, + f: Any, + path: str, + limit: Optional[int] = None, + *, + time_bounds: Optional[Tuple[Optional[int], Optional[int]]] = None, + log_time_order: Optional[bool] = None, ) -> Iterator[_Selected]: """Yield the selected messages of a file without a chunk index. @@ -413,16 +482,27 @@ def iter_unindexed( can only be ordered whole, so its selected messages are held in memory and sorted by log time. The sort is stable: equal log times keep file order. With ``limit``, only the ``limit`` earliest are held. + + ``log_time_order`` overrides the reader's setting, and ``time_bounds`` + the selection's time range. Sample rows stream the whole file once and + index what they collect themselves. """ - scanned: Iterable[_Scanned] = self._scan_unindexed(f, path) - if self._log_time_order and limit is not None: + if log_time_order is None: + log_time_order = self._log_time_order + scanned: Iterable[_Scanned] = self._scan_unindexed(f, path, time_bounds) + if log_time_order and limit is not None: scanned = heapq.nsmallest(limit, scanned, key=_log_time) - elif self._log_time_order: + elif log_time_order: scanned = sorted(scanned, key=_log_time) for schema, channel, message, ordinal in scanned: yield schema, channel, message, unindexed_message_row_id(path, ordinal) - def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]: + def _scan_unindexed( + self, + f: Any, + path: str, + time_bounds: Optional[Tuple[Optional[int], Optional[int]]], + ) -> Iterator[_Scanned]: """Scan a file without a chunk index from the start, in file order. Each message comes with its position among all messages of the file, so @@ -431,6 +511,9 @@ def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]: from mcap.records import Channel, Message, Schema from mcap.stream_reader import StreamReader + if time_bounds is None: + time_bounds = (self._selection.start_time, self._selection.end_time) + start_time, end_time = time_bounds f.seek(0) schemas: Dict[int, "Schema"] = {} channels: Dict[int, "Channel"] = {} @@ -451,6 +534,8 @@ def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]: schema = schemas.get(channel.schema_id) if channel.schema_id else None if not self._selection.accepts_channel(channel, schema): continue - if not self._selection.in_time_range(record.log_time): + if start_time is not None and record.log_time < start_time: + continue + if end_time is not None and record.log_time >= end_time: continue yield schema, channel, record, ordinal diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py index dc2923384ec2..cd261aba2e72 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_datasource_v2.py @@ -13,6 +13,10 @@ which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type`` (``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes. +A sample read (``read_mcap(samples=...)``) lists nothing: the samples dataset +names the files, and ``read_api`` feeds it to the read tasks as their manifest. +Its rows hold each topic's steps around a sample's anchor (``mcap_samples``). + Format specification: https://mcap.dev/spec """ @@ -26,7 +30,9 @@ Iterator, List, Literal, + Mapping, Optional, + Tuple, Union, ) @@ -52,10 +58,18 @@ MCAPSelection, TimeRange, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_sample_rows import ( + DEFAULT_MAX_ROW_BYTES, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import ( + TopicPlan, + sample_row_schema, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( TaskSummaryPartitioner, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec from ray.data._internal.datasource_v2.interfaces.datasource_v2 import ( DatasourceCategory, FileDataSourceV2, @@ -98,7 +112,7 @@ def __init__( self, paths: List[str], *, - topics: Optional[Iterable[str]] = None, + topics: Optional[Union[Iterable[str], Mapping[str, TopicSpec]]] = None, time_range: Optional[TimeRange] = None, message_types: Optional[Iterable[str]] = None, include_metadata: bool = True, @@ -112,18 +126,36 @@ def __init__( ignore_missing_paths: bool = False, shuffle: Optional[Union[Literal["files"], "FileShuffleConfig"]] = None, read_task_memory: Optional[float] = None, + samples_schema: Optional[pa.Schema] = None, ): + """Set up a read of ``paths``, or, with ``samples_schema``, a sample read. + + A sample read takes ``topics`` as a mapping from each topic to its + ``TopicSpec``, and ``samples_schema`` is the samples dataset's schema. + Its files come with the samples, so ``paths`` is empty and the read + tasks resolve each file against ``filesystem``, or its URI. + """ super().__init__(name="MCAP", category=DatasourceCategory.FILE_BASED) _check_import(self, module="mcap", package="mcap") - - # Captured against the original paths: resolution below strips the - # ``local://`` scheme (see ``ParquetDatasourceV2``). - self._supports_distributed_reads = not _is_local_scheme(paths) - resolved_paths, resolved_filesystem = _resolve_paths_and_filesystem( - paths, filesystem - ) - self._paths: List[str] = resolved_paths - self._filesystem = resolved_filesystem + self._samples_schema = samples_schema + self._topic_plans: Tuple[TopicPlan, ...] = () + if samples_schema is not None: + assert isinstance(topics, Mapping), "a sample read maps topics to specs" + self._topic_plans = tuple( + TopicPlan.from_spec(topic, spec) for topic, spec in topics.items() + ) + self._supports_distributed_reads = True + self._paths: List[str] = [] + self._filesystem = filesystem + else: + # Captured against the original paths: resolution below strips the + # ``local://`` scheme (see ``ParquetDatasourceV2``). + self._supports_distributed_reads = not _is_local_scheme(paths) + resolved_paths, resolved_filesystem = _resolve_paths_and_filesystem( + paths, filesystem + ) + self._paths = resolved_paths + self._filesystem = resolved_filesystem self._selection = MCAPSelection.create(topics, time_range, message_types) self._include_metadata = include_metadata self._log_time_order = log_time_order @@ -238,8 +270,11 @@ def infer_schema(self, sample: Optional[FileManifest]) -> pa.Schema: values when every selected channel of the sampled files is JSON-encoded, and the raw payload bytes (``binary``) otherwise. The reader follows this one decision for every file, so a selection that mixes encodings keeps - every payload as bytes. + every payload as bytes. Sample rows hold the samples' own columns, then + each topic's (``sample_row_schema``). """ + if self._samples_schema is not None: + return sample_row_schema(self._samples_schema, self._topic_plans) assert sample is not None, "MCAP always receives a sample" data_type = ( infer_data_type( @@ -307,4 +342,8 @@ def create_scanner( synthesized_columns=self._synthesized_columns, shuffle=self._shuffle, target_block_size=DataContext.get_current().target_max_block_size, + max_row_bytes=env_integer( + "RAY_DATA_MCAP_MAX_ROW_BYTES", DEFAULT_MAX_ROW_BYTES + ), + topic_plans=self._topic_plans, ) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index bf5fe7c418d7..19b4a55e6493 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -1,4 +1,4 @@ -"""``MCAPReader``, which reads one read task's chunks into message rows. +"""``MCAPReader``, which reads one read task's chunks into rows. ``MCAPReader.read`` receives a ``FileManifest`` whose rows name files and, per file, the byte offsets of the chunks the task owns and the summary records it @@ -8,15 +8,18 @@ by log time as they are read, so memory holds only the chunks that overlap in time. -A row has the columns of ``message_schema``. Its optional ``row_id`` is a -deterministic name for the message, built from the file path, the chunk's byte -offset and the message's position in the chunk. +A message row has the columns of ``message_schema``. Its optional ``row_id`` +is a deterministic name for the message, built from the file path, the chunk's +byte offset and the message's position in the chunk. A sample read's manifest +rows are samples instead, and each becomes one row of each topic's steps. This module groups the manifest by file and finishes every table with its partition and synthesized columns. The other modules read and build the rows: - ``mcap_chunks`` reads the selected messages of a file. - ``mcap_message_rows`` builds message rows. +- ``mcap_sample_rows`` builds sample rows, with the step rules of + ``mcap_samples``. """ import itertools @@ -26,6 +29,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Dict, Iterator, List, @@ -52,6 +56,14 @@ _MessageTableBuilder, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection +from ray.data._internal.datasource_v2.formats.mcap.mcap_sample_rows import ( + DEFAULT_MAX_ROW_BYTES, + SampleFile, + SamplePaths, + SampleRows, + group_samples_by_file, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import TopicPlan from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import chunk_unit_id from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import ( task_summaries, @@ -99,6 +111,8 @@ def __init__( target_block_size: Optional[int] = None, schema: Optional[pa.Schema] = None, decode_json: bool = False, + max_row_bytes: int = DEFAULT_MAX_ROW_BYTES, + topic_plans: Sequence[TopicPlan] = (), ): """Initialize the reader. @@ -119,6 +133,11 @@ def __init__( schema: Dataset schema, used to type partition columns. decode_json: Whether ``data`` holds decoded JSON values rather than the payload bytes. + max_row_bytes: Largest payload a sample row may carry. A larger row + fails the read. + topic_plans: For a sample read, each topic's steps + (``mcap_samples``), in the order the row's columns take. Empty + for a read of message rows. """ self._message_reader = SelectedMessageReader(selection, log_time_order) self._decode_json = decode_json @@ -133,6 +152,17 @@ def __init__( self._synthesized_columns = tuple(synthesized_columns) self._target_block_size = target_block_size self._schema = schema + self._sample_rows: Optional[SampleRows] = None + if topic_plans: + self._sample_rows = SampleRows( + topic_plans, + self._message_reader, + self._project, + columns=self._columns, + target_block_size=target_block_size, + max_row_bytes=max_row_bytes, + ) + self._sample_paths = SamplePaths(filesystem) def read(self, input_split: FileManifest) -> Iterator[pa.Table]: """Read the files and chunks named by ``input_split``. @@ -140,6 +170,7 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: All rows that name one file are read together. Files come out in manifest order. With ``limit``, reading stops at that many rows: no later message is added to a table and no later chunk or file is read. + A sample row is built whole, so the table that reaches the limit is cut. When ``DataContext.max_errored_blocks`` lets errors be skipped, a corrupt chunk costs only itself: the reader skips it and reads the @@ -157,15 +188,15 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: read_past_failures = context.max_errored_blocks != 0 remaining = self._limit failures: List[Tuple[str, Exception]] = [] - for owned in _owned_files(input_split): + for path, read_file in self._file_reads(input_split): if remaining is not None and remaining <= 0: break # The chunks skipped as corrupt, by offset. Each attempt starts it # afresh, so it holds the last attempt's. corrupt: Optional[Dict[int, Exception]] = {} if read_past_failures else None tables = iterate_with_retry( - partial(self._read_file, owned, remaining, corrupt), - f"read MCAP file {owned.path}", + partial(read_file, remaining, corrupt), + f"read MCAP file {path}", match=retried_io_errors, ) try: @@ -176,15 +207,27 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: except Exception as e: # noqa: BLE001 - raised once the others are read if not read_past_failures: raise - failures.append((owned.path, e)) + failures.append((path, e)) else: if corrupt: - failures.append( - (owned.path, _corrupt_chunks_error(owned.path, corrupt)) - ) + failures.append((path, _corrupt_chunks_error(path, corrupt))) if failures: raise _read_error(failures) + def _file_reads( + self, manifest: FileManifest + ) -> List[Tuple[str, Callable[..., Iterator[pa.Table]]]]: + """Each file of the task, with the call that reads its tables.""" + if self._sample_rows is not None: + return [ + (file.path, partial(self._read_samples, file)) + for file in group_samples_by_file(manifest, self._sample_paths) + ] + return [ + (owned.path, partial(self._read_file, owned)) + for owned in _owned_files(manifest) + ] + # -- one file ---------------------------------------------------------- def _read_file( @@ -237,6 +280,31 @@ def _read_file( ) yield from self._tables(itertools.islice(messages, limit), path, offsets) + def _read_samples( + self, + file: SampleFile, + limit: Optional[int], + corrupt: Optional[Dict[int, Exception]] = None, + ) -> Iterator[pa.Table]: + """Yield the sample rows of one file, at most ``limit`` of them. + + A sample row must hold every step, so a corrupt chunk fails the file + whole, and ``corrupt`` stays empty. A read that keeps only the samples' + own columns opens no file. + """ + assert self._sample_rows is not None + if not self._sample_rows.reads_messages: + tables = self._sample_rows.tables_without_messages(file) + yield from _within_limit(tables, limit) + return + filesystem = file.filesystem or self._filesystem or LocalFileSystem() + with filesystem.open_input_file(file.path) as f: + summary = read_summary_from(f, file.path) + if summary is not None and not summary.chunk_indexes: + summary = None + tables = self._sample_rows.tables(f, file, summary) + yield from _within_limit(tables, limit) + # -- rows to tables ---------------------------------------------------- def _tables( @@ -299,9 +367,14 @@ def _finish( ) -> pa.Table: """Append partition and synthesized columns, then apply the projection.""" wanted = set(self._columns) if self._columns is not None else None - num_rows = table.num_rows table = self._append_partition_columns(table, path, wanted) table = self._append_synthesized_columns(table, unit, rows_before, wanted) + return self._project(table) + + def _project(self, table: pa.Table) -> pa.Table: + """Apply the projection, keeping the row count of a table left without + columns, and refuse pickled-object columns.""" + num_rows = table.num_rows if self._columns is not None: produced = set(table.column_names) table = table.select([c for c in self._columns if c in produced]) @@ -372,6 +445,25 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array +def _within_limit( + tables: Iterator[pa.Table], limit: Optional[int] +) -> Iterator[pa.Table]: + """``tables``, cut to hold at most ``limit`` rows in all. + + A sample row is built whole, so the table that reaches the limit is cut + here, and no later table is built. + """ + if limit is None: + yield from tables + return + for table in tables: + if table.num_rows >= limit: + yield table.slice(0, limit) + return + limit -= table.num_rows + yield table + + def _record_corrupt_chunk( corrupt: Dict[int, Exception], chunk_index: "ChunkIndex", error: Exception ) -> None: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_sample_rows.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_sample_rows.py new file mode 100644 index 000000000000..35662dcf6184 --- /dev/null +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_sample_rows.py @@ -0,0 +1,673 @@ +"""Sample rows: one row per sample, with each topic's steps around its anchor. + +``MCAPReader`` hands this module a task's samples one file at a time, in anchor +order (``read_mcap(samples=...)``). A ``ChunkTimeline`` reads the chunks the +steps need. Samples whose windows meet form a run, and a run's chunks are read +in one coalesced pass. A step that reaches further, such as one in a gap of the +recording, reads on chunk by chunk, nearest first. ``mcap_samples`` decides +which message fills each step, and ``SampleTableBuilder`` lays the rows out:: + + one file's samples, by anchor a1 a2 a3 a4 + their windows, all topics [-----] + [-----] + [-----] [-----] + runs [ run 1 ] [run 2] + chunks read c3 c4 c5 c6 c9 c10 + +A file without a chunk index is read whole, once per task. +""" + +import bisect +import itertools +import urllib.parse +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterator, + List, + Optional, + Sequence, + Set, + Tuple, +) + +import pyarrow as pa + +from ray.data._internal.arrow_block import _BATCH_SIZE_PRESERVING_STUB_COL_NAME +from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( + SelectedMessageReader, + _Selected, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_io import buffered +from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import ( + ANCHOR_COLUMN, + CLIP_END_COLUMN, + CLIP_START_COLUMN, + DATA_PREFIX, + SAMPLE_ID_COLUMN, + TIMES_PREFIX, + VALID_PREFIX, + SampleTimes, + SortedTimes, + Step, + TopicPlan, + resolve_steps, + topic_column_names, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import ( + FILE_CHUNK_METADATA_COLUMN_NAME, + FILE_SIZE_COLUMN_NAME, + PATH_COLUMN_NAME, + FileManifest, +) +from ray.data._internal.util import GiB +from ray.data.block import BlockAccessor + +if TYPE_CHECKING: + from mcap.records import ChunkIndex + from mcap.summary import Summary + from pyarrow.fs import FileSystem + +# A message as a row holds it: schema, channel, record. +_Entry = Tuple[Any, Any, Any] +# A timeline's sort key: log time, then where the message lies in the file. +_Key = Tuple[int, int, int] + +# The columns ``read_mcap`` adds to each sample for the read, which rows drop. +MANIFEST_COLUMNS = ( + PATH_COLUMN_NAME, + FILE_SIZE_COLUMN_NAME, + FILE_CHUNK_METADATA_COLUMN_NAME, +) + +# Largest payload a sample row may carry: a memory guardrail, not a format +# limit. A row is built in the worker and copied into the object store as one +# object, so a read needs about twice its size. ``RAY_DATA_MCAP_MAX_ROW_BYTES`` +# overrides it. +DEFAULT_MAX_ROW_BYTES = GiB + +# How many chunks a step that reaches past the chunks read so far reads at a +# time, nearest first. +_CHUNKS_PER_REACH = 4 +# The most samples one run holds, so a dense stretch of anchors still lets +# its chunks go now and then. +_MAX_RUN_SAMPLES = 256 + + +class _TopicView: + """One topic of a ``ChunkTimeline``, as ``resolve_steps`` asks for it.""" + + def __init__(self, timeline: "ChunkTimeline", topic: str): + self._timeline = timeline + self._topic = topic + + def all_in(self, low: int, high: int) -> List[Tuple[int, _Entry]]: + self._timeline.read_span(self._topic, low, high) + return self._timeline.held(self._topic).all_in(low, high) + + def last_in(self, low: int, high: int, count: int) -> List[Tuple[int, _Entry]]: + while True: + found = self._timeline.held(self._topic).last_in(low, high, count) + # Only a chunk reaching the earliest message kept could add one. + reach = found[0][0] if len(found) == count else low + if not self._timeline.read_nearest(self._topic, reach, high, last=True): + return found + + def first_in(self, low: int, high: int, count: int) -> List[Tuple[int, _Entry]]: + while True: + found = self._timeline.held(self._topic).first_in(low, high, count) + reach = found[-1][0] if len(found) == count else high + if not self._timeline.read_nearest(self._topic, low, reach, last=False): + return found + + +class ChunkTimeline: + """The sample topics' messages of one open file, read chunk by chunk. + + The summary's chunk indexes tell which chunks may hold a topic: those whose + message indexes name one of its channels, or every chunk when they name + none. A chunk is read once and kept until ``forget_before`` lets it go. + Each topic's messages are held sorted by log time, then by their place in + the file, so equal log times keep the order they were recorded in. + """ + + def __init__( + self, + f: Any, + path: str, + summary: "Summary", + file_summary: Callable[[], "Summary"], + reader: SelectedMessageReader, + topics: Sequence[str], + ): + self._f = f + self._path = path + self._summary = summary + self._file_summary = file_summary + self._reader = reader + self._topics = set(topics) + topic_ids: Dict[str, Set[int]] = {topic: set() for topic in topics} + for channel_id, channel in summary.channels.items(): + if channel.topic in topic_ids: + topic_ids[channel.topic].add(channel_id) + self._selected = set().union(*topic_ids.values()) + # Each topic's candidate chunks, by first log time. A summary that + # lists no channel cannot tell, so every chunk is a candidate. One that + # lists others but none of the topic's says the file lacks the topic. + self._candidates = { + topic: sorted( + ( + c + for c in summary.chunk_indexes + if not summary.channels + or ( + ids + and ( + not c.message_index_offsets + or not ids.isdisjoint(c.message_index_offsets) + ) + ) + ), + key=lambda c: (c.message_start_time, c.chunk_start_offset), + ) + for topic, ids in topic_ids.items() + } + # Chunk offset -> its index record and, per topic, its messages. + self._chunks: Dict[int, "ChunkIndex"] = {} + self._loaded: Dict[int, Dict[str, List[Tuple[_Key, _Entry]]]] = {} + self._held: Dict[str, SortedTimes] = {} + + def view(self, topic: str) -> _TopicView: + return _TopicView(self, topic) + + def held(self, topic: str) -> SortedTimes: + """The topic's messages in the chunks read so far.""" + held = self._held.get(topic) + if held is None: + keyed = sorted( + itertools.chain.from_iterable( + by_topic.get(topic, ()) for by_topic in self._loaded.values() + ), + key=lambda pair: pair[0], + ) + held = self._held[topic] = SortedTimes( + [(key[0], entry) for key, entry in keyed] + ) + return held + + def read_spans(self, spans: Dict[str, Tuple[int, int]]) -> None: + """Read, in one pass, every chunk not read yet that may hold one of the + topics in its span ``[low, high]``.""" + wanted: Dict[int, "ChunkIndex"] = {} + for topic, (low, high) in spans.items(): + for chunk in self._unread(topic, low, high): + wanted[chunk.chunk_start_offset] = chunk + self._read(list(wanted.values())) + + def read_span(self, topic: str, low: int, high: int) -> None: + """Read every chunk not read yet that may hold the topic in ``[low, high]``.""" + self._read(self._unread(topic, low, high)) + + def read_nearest(self, topic: str, low: int, high: int, *, last: bool) -> bool: + """Read a few chunks not read yet that may hold the topic in + ``[low, high]``: the latest ending with ``last``, else the earliest + starting. Returns whether any chunk was left to read.""" + unread = self._unread(topic, low, high) + if last: + unread.sort(key=lambda c: c.message_end_time, reverse=True) + self._read(unread[:_CHUNKS_PER_REACH]) + return bool(unread) + + def forget_before(self, time: int) -> None: + """Let go of the chunks whose messages all lie before ``time``.""" + gone = [ + offset + for offset, chunk in self._chunks.items() + if chunk.message_end_time < time + ] + for offset in gone: + del self._chunks[offset] + del self._loaded[offset] + if gone: + self._held.clear() + + def _unread(self, topic: str, low: int, high: int) -> List["ChunkIndex"]: + """The topic's candidate chunks not read yet that overlap ``[low, high]``.""" + candidates = self._candidates.get(topic, []) + # Candidates are sorted by first log time, so those starting after + # ``high`` are cut off at once. + end = bisect.bisect_right(candidates, high, key=lambda c: c.message_start_time) + return [ + c + for c in candidates[:end] + if c.message_end_time >= low and c.chunk_start_offset not in self._loaded + ] + + def _read(self, chunks: List["ChunkIndex"]) -> None: + if not chunks: + return + for chunk, messages in self._reader.iter_chunk_messages( + self._f, + self._path, + self._summary, + chunks, + self._file_summary, + selected=self._selected, + ): + offset = chunk.chunk_start_offset + by_topic: Dict[str, List[Tuple[_Key, _Entry]]] = {} + for position, (schema, channel, message, _) in enumerate(messages): + if channel.topic in self._topics: + key = (message.log_time, offset, position) + by_topic.setdefault(channel.topic, []).append( + (key, (schema, channel, message)) + ) + self._chunks[offset] = chunk + self._loaded[offset] = by_topic + self._held.clear() + + +class WholeFileTimeline: + """The sample topics' messages of a file without a chunk index, read whole.""" + + def __init__(self, messages: Iterator[_Selected], topics: Sequence[str]): + by_topic: Dict[str, List[Tuple[int, _Entry]]] = {topic: [] for topic in topics} + for schema, channel, message, _ in messages: + if channel.topic in by_topic: + by_topic[channel.topic].append( + (message.log_time, (schema, channel, message)) + ) + # The scan is in file order, and the sort is stable. + self._held = { + topic: SortedTimes(sorted(timed, key=lambda pair: pair[0])) + for topic, timed in by_topic.items() + } + + def view(self, topic: str) -> SortedTimes: + return self._held[topic] + + def read_spans(self, spans: Dict[str, Tuple[int, int]]) -> None: + pass + + def forget_before(self, time: int) -> None: + pass + + +@dataclass(frozen=True) +class _Sample: + """One sample of a task: its row in the file's manifest, and its times.""" + + index: int + times: SampleTimes + where: str + + +@dataclass(frozen=True) +class SampleFile: + """One file of a read task's samples. + + ``samples`` holds the file's manifest rows in anchor order, and + ``filesystem`` is the one its path resolved against. + """ + + path: str + samples: pa.Table + filesystem: Optional["FileSystem"] + + +def group_samples_by_file( + manifest: FileManifest, paths: "SamplePaths" +) -> List[SampleFile]: + """Group a samples manifest by file, each file's samples in anchor order. + + Files come in the order the manifest first names them, and samples with + equal anchors keep their manifest order. + """ + table = BlockAccessor.for_block(manifest.as_block()).to_arrow() + anchors = table.column(ANCHOR_COLUMN).to_pylist() + rows_by_path: Dict[str, List[int]] = {} + for index, path in enumerate(table.column(PATH_COLUMN_NAME).to_pylist()): + rows_by_path.setdefault(path, []).append(index) + files = [] + for path, rows in rows_by_path.items(): + rows.sort(key=lambda index: anchors[index]) + resolved, filesystem = paths.resolve(path) + files.append(SampleFile(resolved, table.take(rows), filesystem)) + return files + + +@dataclass(frozen=True) +class _Run: + """Samples of one file whose windows meet, and each topic's span over them.""" + + samples: List[_Sample] + spans: Dict[str, Tuple[int, int]] + + @property + def low(self) -> int: + return min(low for low, _ in self.spans.values()) + + +def _samples_of(file: SampleFile) -> List[_Sample]: + """The samples of one file, in anchor order as the manifest holds them.""" + table = file.samples + starts = table.column(CLIP_START_COLUMN).to_pylist() + ends = table.column(CLIP_END_COLUMN).to_pylist() + anchors = table.column(ANCHOR_COLUMN).to_pylist() + ids = ( + table.column(SAMPLE_ID_COLUMN).to_pylist() + if SAMPLE_ID_COLUMN in table.column_names + else None + ) + samples = [] + for i, (start, end, anchor) in enumerate(zip(starts, ends, anchors)): + if ids is not None: + where = f"Sample {ids[i]!r} of {file.path!r}" + else: + where = f"The sample anchored at {anchor} in {file.path!r}" + samples.append(_Sample(i, SampleTimes(start, end, anchor), where)) + return samples + + +def _runs(samples: List[_Sample], plans: Sequence[TopicPlan]) -> Iterator[_Run]: + """Group consecutive samples whose windows meet, over every topic. + + A run ends at ``_MAX_RUN_SAMPLES`` samples, so its chunks can be let go. + """ + run: List[_Sample] = [] + spans: Dict[str, Tuple[int, int]] = {} + high = 0 + for sample in samples: + reach = {plan.topic: plan.reach(sample.times) for plan in plans} + low = min(topic_low for topic_low, _ in reach.values()) + if run and (len(run) >= _MAX_RUN_SAMPLES or low > high): + yield _Run(run, spans) + run, spans = [], {} + if not run: + high = low + run.append(sample) + for topic, (topic_low, topic_high) in reach.items(): + old_low, old_high = spans.get(topic, (topic_low, topic_high)) + spans[topic] = (min(old_low, topic_low), max(old_high, topic_high)) + high = max(high, topic_high) + if run: + yield _Run(run, spans) + + +class SampleTableBuilder: + """Accumulates the sample rows of one file and builds Arrow tables of them. + + A row is the sample's own columns, taken from its manifest row, then the + wanted columns of each topic. Only the wanted columns are built. + """ + + def __init__( + self, + samples: pa.Table, + plans: Sequence[TopicPlan], + wanted: Optional[Set[str]], + ): + self._samples = samples.select( + [ + name + for name in samples.column_names + if name not in MANIFEST_COLUMNS and (wanted is None or name in wanted) + ] + ) + self._sample_row_bytes = self._samples.nbytes // max(self._samples.num_rows, 1) + self._plans = list(plans) + self._wanted = wanted + self._rows: List[int] = [] + self._columns = { + plan.topic: _TopicColumns(with_data=self._wants(DATA_PREFIX, plan.topic)) + for plan in plans + } + self.estimated_bytes = 0 + + @property + def num_rows(self) -> int: + return len(self._rows) + + def add_row(self, index: int, steps: Dict[str, List[Step[_Entry]]]) -> int: + """Add the sample at ``index`` of the manifest with its topics' steps. + + Returns the bytes of the payloads the row carries. + """ + self._rows.append(index) + added = 0 + for plan in self._plans: + added += self._columns[plan.topic].add(steps.get(plan.topic, [])) + self.estimated_bytes += ( + self._sample_row_bytes + + added + + sum(9 * len(topic_steps) + 16 for topic_steps in steps.values()) + ) + return added + + def build(self) -> pa.Table: + """The rows added so far, as one table; the builder starts over. + + A table without columns keeps its row count in a stub column, as + ``MCAPReader`` projections do. + """ + if self._samples.num_columns: + table = self._samples.take(self._rows) + else: + table = pa.table( + {_BATCH_SIZE_PRESERVING_STUB_COL_NAME: pa.nulls(len(self._rows))} + ) + for plan in self._plans: + columns = self._columns[plan.topic] + for name, array in columns.finish(lambda p: self._wants(p, plan.topic)): + table = table.append_column(f"{name}{plan.topic}", array) + if table.num_columns > 1 and not self._samples.num_columns: + table = table.drop([_BATCH_SIZE_PRESERVING_STUB_COL_NAME]) + self._rows = [] + self.estimated_bytes = 0 + return table + + def _wants(self, prefix: str, topic: str) -> bool: + return self._wanted is None or f"{prefix}{topic}" in self._wanted + + +class _TopicColumns: + """One topic's list columns while rows are added: an entry per step. + + ``with_data`` keeps the steps' payloads, for the ``data:`` column. + """ + + def __init__(self, *, with_data: bool): + self._with_data = with_data + self._start() + + def _start(self) -> None: + self.offsets: List[int] = [0] + self.payloads: List[bytes] = [] + self.step_times: List[int] = [] + self.step_valid: List[bool] = [] + + def add(self, steps: List[Step[_Entry]]) -> int: + """Add one row's steps; return the bytes of their payloads.""" + added = 0 + for step in steps: + if self._with_data: + payload = b"" if step.message is None else step.message[2].data + self.payloads.append(payload) + added += len(payload) + self.step_times.append(step.time) + self.step_valid.append(step.valid) + self.offsets.append(self.offsets[-1] + len(steps)) + return added + + def finish(self, wants: Callable[[str], bool]) -> List[Tuple[str, pa.Array]]: + """The wanted columns of the rows added so far; the columns start over.""" + offsets = pa.array(self.offsets, pa.int32()) + columns = [] + for prefix, values in ( + (DATA_PREFIX, lambda: pa.array(self.payloads, pa.large_binary())), + (TIMES_PREFIX, lambda: pa.array(self.step_times, pa.int64())), + (VALID_PREFIX, lambda: pa.array(self.step_valid, pa.bool_())), + ): + if wants(prefix): + column = values() + columns.append( + ( + prefix, + pa.ListArray.from_arrays( + offsets, column, type=pa.list_(column.type) + ), + ) + ) + self._start() + return columns + + +class SampleRows: + """Builds the sample rows of one read task, one file at a time. + + ``plans`` are the topics of ``read_mcap(topics=...)``, in order. Only the + topics with a column left after projection are read. A row whose payloads + pass ``max_row_bytes`` fails the read. + """ + + def __init__( + self, + plans: Sequence[TopicPlan], + message_reader: SelectedMessageReader, + finish: Callable[[pa.Table], pa.Table], + *, + columns: Optional[Sequence[str]], + target_block_size: Optional[int], + max_row_bytes: int, + ): + self._wanted = set(columns) if columns is not None else None + self._plans = [ + plan + for plan in plans + if self._wanted is None + or not self._wanted.isdisjoint(topic_column_names(plan.topic)) + ] + self._message_reader = message_reader + self._finish = finish + self._target_block_size = target_block_size + self._max_row_bytes = max_row_bytes + + @property + def reads_messages(self) -> bool: + """Whether a row needs any message: not when the projection keeps only + the samples' own columns, as ``count()`` does.""" + return bool(self._plans) + + def tables_without_messages(self, file: SampleFile) -> Iterator[pa.Table]: + """The rows of a read that keeps only the samples' own columns.""" + builder = SampleTableBuilder(file.samples, [], self._wanted) + for index in range(file.samples.num_rows): + builder.add_row(index, {}) + yield self._finish(builder.build()) + + def tables( + self, + f: Any, + file: SampleFile, + summary: Optional["Summary"], + ) -> Iterator[pa.Table]: + """Yield the rows of the samples of one open file, in anchor order. + + ``summary`` is the file's whole summary, or ``None`` when the file has + no chunk index. The chunks of a run of samples are read together, and + let go of once the runs to come start after them. + """ + timeline = self._timeline(f, file.path, summary) + builder = SampleTableBuilder(file.samples, self._plans, self._wanted) + runs = list(_runs(_samples_of(file), self._plans)) + for i, run in enumerate(runs): + timeline.read_spans(run.spans) + for sample in run.samples: + yield from self._add_row(builder, sample, self._steps(timeline, sample)) + if i + 1 < len(runs): + timeline.forget_before(runs[i + 1].low) + if builder.num_rows > 0: + yield self._finish(builder.build()) + + def _add_row( + self, + builder: SampleTableBuilder, + sample: _Sample, + steps: Dict[str, List[Step[_Entry]]], + ) -> Iterator[pa.Table]: + """Add one sample's row, and yield the table once it is full.""" + if builder.add_row(sample.index, steps) > self._max_row_bytes: + raise ValueError( + f"{sample.where}: its payloads would exceed {self._max_row_bytes} " + "bytes (RAY_DATA_MCAP_MAX_ROW_BYTES). Shorten its windows." + ) + if ( + self._target_block_size is not None + and builder.estimated_bytes >= self._target_block_size + ): + yield self._finish(builder.build()) + + def _timeline( + self, + f: Any, + path: str, + summary: Optional["Summary"], + ) -> Any: + """The file's topics, read chunk by chunk, or whole without a chunk index.""" + topics = [plan.topic for plan in self._plans] + if summary is None: + from ray.data.context import DataContext + + buffer_size = DataContext.get_current().streaming_read_buffer_size + messages = self._message_reader.iter_unindexed( + buffered(f, buffer_size), + path, + time_bounds=(None, None), + log_time_order=False, + ) + return WholeFileTimeline(messages, topics) + whole = summary + return ChunkTimeline( + f, path, summary, lambda: whole, self._message_reader, topics + ) + + def _steps(self, timeline: Any, sample: _Sample) -> Dict[str, List[Step[_Entry]]]: + """Each topic's steps for one sample.""" + return { + plan.topic: resolve_steps( + plan, timeline.view(plan.topic), sample.times, sample.where + ) + for plan in self._plans + } + + +class SamplePaths: + """Resolves the paths that samples name, as the read tasks open them. + + With ``filesystem`` every path resolves against it. Without, each path's + filesystem comes from its URI, created once per scheme and authority, such + as one per S3 bucket. + """ + + def __init__(self, filesystem: Optional["FileSystem"]): + self._filesystem = filesystem + self._by_root: Dict[Tuple[str, str], "FileSystem"] = {} + + def resolve(self, path: str) -> Tuple[str, "FileSystem"]: + from ray.data.datasource.path_util import _resolve_paths_and_filesystem + + if self._filesystem is not None: + (resolved,), filesystem = _resolve_paths_and_filesystem( + [path], self._filesystem + ) + return resolved, filesystem + parsed = urllib.parse.urlparse(path) + root = (parsed.scheme, parsed.netloc) + (resolved,), filesystem = _resolve_paths_and_filesystem( + [path], self._by_root.get(root) + ) + self._by_root.setdefault(root, filesystem) + return resolved, filesystem diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_samples.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_samples.py new file mode 100644 index 000000000000..54f2b40d80b7 --- /dev/null +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_samples.py @@ -0,0 +1,597 @@ +"""Training samples: which message fills each step of a sample row. + +``read_mcap(samples=...)`` reads one row per sample. A sample names its file, +a clip ``[clip_start, clip_end)`` and an ``anchor`` inside the clip, in +nanoseconds of log time. Each topic's ``TopicSpec`` places its steps before or +after the anchor (``mcap_topic_spec``). This module turns the two into the +message that fills each step, and lays out a sample row's columns. It does no I/O: +``resolve_steps`` asks a ``TopicTimes`` for the topic's messages, and the +reader answers from the chunks it reads (``mcap_sample_rows``). + +Time steps (``steps.before``, ``steps.after``) are fixed times counted from the +anchor. Each step takes the message logged nearest to it inside the clip, the +earlier one on a tie (``match.nearest``). A before-window takes only messages +logged up to the anchor, and an after-window only messages logged after it, so +no step looks past the anchor from its side (m is a message):: + + clip [---------------------------------------------) + messages m m m m m m m m m m m m + anchor A + steps.before(0.3, fps=10) s0 s1 s2 + steps.after(0.2, fps=10) s0 s1 + +A step with no such message, such as one past the clip's edge, is padded: +``"edge"`` repeats the nearest real step, ``"zeros"`` leaves it empty, and +``"raise"`` fails the read. +""" + +import bisect +from dataclasses import dataclass +from fractions import Fraction +from typing import ( + TYPE_CHECKING, + Callable, + Generic, + List, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, +) + +import pyarrow as pa + +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import ( + TimeSteps, + TopicSpec, +) +from ray.data._internal.datasource_v2.interfaces.file_manifest import ( + FILE_CHUNK_METADATA_COLUMN_NAME, + FILE_SIZE_COLUMN_NAME, + PATH_COLUMN_NAME, +) + +if TYPE_CHECKING: + import numpy as np + +# The columns every samples dataset has. ``files`` lists a sample's recordings, +# one for now. The three times are log times in nanoseconds; ``clip_end`` is +# exclusive and the anchor lies inside the clip. +FILES_COLUMN = "files" +CLIP_START_COLUMN = "clip_start" +CLIP_END_COLUMN = "clip_end" +ANCHOR_COLUMN = "anchor" +SAMPLE_COLUMNS: Tuple[str, ...] = ( + FILES_COLUMN, + CLIP_START_COLUMN, + CLIP_END_COLUMN, + ANCHOR_COLUMN, +) +# An optional column whose value names a sample in error messages. +SAMPLE_ID_COLUMN = "sample_id" + +# A sample row's columns per topic. +DATA_PREFIX = "data:" +TIMES_PREFIX = "times:" +VALID_PREFIX = "valid:" +TOPIC_PREFIXES: Tuple[str, ...] = (DATA_PREFIX, TIMES_PREFIX, VALID_PREFIX) + +_NS_PER_S = 1_000_000_000 + +M = TypeVar("M") +M_co = TypeVar("M_co", covariant=True) + + +class TopicTimes(Protocol[M_co]): + """One topic's messages in one recording, by log time. + + ``resolve_steps`` asks for the messages it needs through these methods. + Each returns ``(log_time, message)`` pairs in log-time order, with messages + of equal log time in the order they were recorded. Bounds are inclusive. + """ + + def last_in(self, low: int, high: int, count: int) -> List[Tuple[int, M_co]]: + """The last ``count`` messages logged in ``[low, high]``, or all of them + if there are fewer.""" + ... + + def first_in(self, low: int, high: int, count: int) -> List[Tuple[int, M_co]]: + """The first ``count`` messages logged in ``[low, high]``, or all of + them if there are fewer.""" + ... + + def all_in(self, low: int, high: int) -> List[Tuple[int, M_co]]: + """Every message logged in ``[low, high]``.""" + ... + + +class SortedTimes(Generic[M]): + """A ``TopicTimes`` over messages held in a list, sorted by log time.""" + + def __init__(self, timed: Sequence[Tuple[int, M]]): + self._timed = list(timed) + self._times = [time for time, _ in self._timed] + + def _span(self, low: int, high: int) -> Tuple[int, int]: + return ( + bisect.bisect_left(self._times, low), + bisect.bisect_right(self._times, high), + ) + + def last_in(self, low: int, high: int, count: int) -> List[Tuple[int, M]]: + start, end = self._span(low, high) + return self._timed[max(start, end - count) : end] + + def first_in(self, low: int, high: int, count: int) -> List[Tuple[int, M]]: + start, end = self._span(low, high) + return self._timed[start : min(end, start + count)] + + def all_in(self, low: int, high: int) -> List[Tuple[int, M]]: + start, end = self._span(low, high) + return self._timed[start:end] + + +@dataclass(frozen=True) +class SampleTimes: + """A sample's clip ``[clip_start, clip_end)`` and its anchor, in log-time ns.""" + + clip_start: int + clip_end: int + anchor: int + + @property + def clip_last(self) -> int: + """The clip's last nanosecond: ``clip_end`` is exclusive.""" + return self.clip_end - 1 + + +@dataclass(frozen=True) +class TopicPlan: + """One topic's ``TopicSpec`` in the terms its steps are resolved in. + + Attributes: + topic: The topic name. + before: Whether the window ends at the anchor rather than starting + after it. + pad: How a step with no message is filled (``TopicSpec.pad``). + num_steps: The window's fixed number of steps, or ``None`` for + ``steps.every``, whose length varies. + fps: Steps per second of time steps, or ``None`` for message steps. + span_ns: The horizon of ``steps.every``, in nanoseconds. + """ + + topic: str + before: bool + pad: str + num_steps: Optional[int] + fps: Optional[Fraction] = None + span_ns: Optional[int] = None + + @classmethod + def from_spec(cls, topic: str, spec: TopicSpec) -> "TopicPlan": + steps = spec.steps + before = steps.side == "before" + if isinstance(steps, TimeSteps): + return cls(topic, before, spec.pad, num_steps=steps.count, fps=steps.fps) + if steps.count is not None: + return cls(topic, before, spec.pad, num_steps=steps.count) + return cls(topic, before, spec.pad, num_steps=None, span_ns=steps.span_ns) + + def step_offset_ns(self, k: int) -> int: + """How far step ``k`` lies from the anchor, rounded to the nanosecond.""" + assert self.fps is not None + return round(Fraction(k * _NS_PER_S) / self.fps) + + def step_times(self, anchor: int) -> List[int]: + """The log time of each step of a window with ``fps``, in order. + + A before-window's last step is the anchor. An after-window's first step + is one step after it. + """ + assert self.num_steps is not None and self.fps is not None + n = self.num_steps + if self.before: + return [anchor - self.step_offset_ns(n - 1 - i) for i in range(n)] + return [anchor + self.step_offset_ns(i + 1) for i in range(n)] + + def reach(self, sample: SampleTimes) -> Tuple[int, int]: + """The span of log time inside the clip that the window's steps fall in. + + For time steps it runs from the first step to the last, and for + ``steps.every`` over its horizon. ``steps.last`` and ``steps.next`` + reach as far back or on as the recording needs, so only the anchor's + side of them is known here. The reader reads the chunks over + this span first, and reads on as the steps need. + """ + anchor = sample.anchor + if self.before: + if self.fps is not None: + assert self.num_steps is not None + low = anchor - self.step_offset_ns(self.num_steps - 1) + elif self.span_ns is not None: + low = anchor - self.span_ns + 1 + else: + low = anchor + return max(low, sample.clip_start), anchor + if self.fps is not None: + assert self.num_steps is not None + high = anchor + self.step_offset_ns(self.num_steps) + elif self.span_ns is not None: + high = anchor + self.span_ns + else: + high = anchor + 1 + return min(anchor + 1, sample.clip_last), min(high, sample.clip_last) + + +@dataclass(frozen=True) +class Step(Generic[M]): + """One step of a topic's window in a sample row. + + Attributes: + message: What fills the step: the message the step takes, the message + ``"edge"`` padding repeats, or ``None`` for an empty step. + time: The log time of ``message``. An empty step has its own time with + ``fps``, and otherwise the clip's edge on the window's side. + valid: Whether the step took a message of its own. + """ + + message: Optional[M] + time: int + valid: bool + + +@dataclass(frozen=True) +class _Slot(Generic[M]): + """A step before padding: its time, and the message it took, if any. + + ``outside`` marks a step that lies outside the clip. + """ + + time: int + taken: Optional[Tuple[int, M]] + outside: bool = False + + +def resolve_steps( + plan: TopicPlan, times: TopicTimes[M], sample: SampleTimes, where: str +) -> List[Step[M]]: + """The steps of one topic's window for one sample, padded. + + ``where`` names the sample in the error that ``pad="raise"`` raises when a + step has no message. ``steps.every`` has no fixed steps: it holds every + message within its horizon and inside the clip. + """ + if plan.num_steps is None: + return _variable_steps(plan, times, sample, where) + if plan.fps is not None: + slots = _timed_slots(plan, times, sample) + else: + slots = _counted_slots(plan, times, sample) + return _padded(plan, slots, times, sample, where) + + +def _timed_slots( + plan: TopicPlan, times: TopicTimes[M], sample: SampleTimes +) -> List[_Slot[M]]: + """The steps of a window with ``fps``, each with the message it takes. + + Each step inside the clip takes the message logged nearest to it, the + earlier one on a tie, from the window's side of the anchor. Log times + jitter, so a step meant to fall on a frame lands just before or after it; + the nearest message is still that frame. The messages are asked for once: + the nearest before the first step, every one up to the last step, and + the nearest after it. + """ + step_times = plan.step_times(sample.anchor) + if plan.before: + low, high = sample.clip_start, sample.anchor + else: + low, high = sample.anchor + 1, sample.clip_last + inside = [t for t in step_times if sample.clip_start <= t <= sample.clip_last] + timed: List[Tuple[int, M]] = [] + if inside: + first, last = inside[0], inside[-1] + timed = times.last_in(low, first, 1) + if first < last: + timed += times.all_in(first + 1, last) + if last < high: + timed += times.first_in(last + 1, high, 1) + known = [time for time, _ in timed] + slots: List[_Slot[M]] = [] + for t in step_times: + if not sample.clip_start <= t <= sample.clip_last: + slots.append(_Slot(t, None, outside=True)) + else: + slots.append(_Slot(t, _nearest(timed, known, t))) + return slots + + +def _nearest( + timed: List[Tuple[int, M]], known: List[int], t: int +) -> Optional[Tuple[int, M]]: + """The message of ``timed`` logged nearest to ``t``, the earlier on a tie. + + ``known`` holds the log times of ``timed``, in order. + """ + after = bisect.bisect_right(known, t) + candidates = [i for i in (after - 1, after) if 0 <= i < len(known)] + if not candidates: + return None + return timed[min(candidates, key=lambda i: (abs(known[i] - t), known[i]))] + + +def _counted_slots( + plan: TopicPlan, times: TopicTimes[M], sample: SampleTimes +) -> List[_Slot[M]]: + """The steps of ``steps.last`` or ``steps.next``: that many recorded + messages next to the anchor, inside the clip. + + A before-window takes the last messages logged at or before the anchor, + an after-window the first logged after it. The steps the clip cannot fill + lie past its last message on that side, and take the clip's edge as their + time. + """ + assert plan.num_steps is not None + count = plan.num_steps + if plan.before: + taken = times.last_in(sample.clip_start, sample.anchor, count) + missing = [_Slot(sample.clip_start, None) for _ in range(count - len(taken))] + return missing + [_Slot(time, (time, m)) for time, m in taken] + taken = times.first_in(sample.anchor + 1, sample.clip_last, count) + missing = [_Slot(sample.clip_last, None) for _ in range(count - len(taken))] + return [_Slot(time, (time, m)) for time, m in taken] + missing + + +def _padded( + plan: TopicPlan, + slots: List[_Slot[M]], + times: TopicTimes[M], + sample: SampleTimes, + where: str, +) -> List[Step[M]]: + """The steps, with those that took no message filled as ``plan.pad`` says. + + ``"edge"`` repeats the nearest step that took a message, the earlier one + on a tie. A window where no step did repeats the clip's first message on + the topic for a before-window, and its last for an after-window. A clip + with no message on the topic leaves the steps empty. + """ + real = [i for i, slot in enumerate(slots) if slot.taken is not None] + if len(real) < len(slots) and plan.pad == "raise": + raise ValueError(_unfilled_message(plan, slots, sample, where)) + edge: Optional[Tuple[int, M]] = None + if len(real) < len(slots) and not real and plan.pad == "edge": + edge = _clip_edge(plan, times, sample) + steps: List[Step[M]] = [] + for i, slot in enumerate(slots): + if slot.taken is not None: + steps.append(Step(slot.taken[1], slot.taken[0], True)) + continue + repeated = edge + if plan.pad == "edge" and real: + nearest = min(real, key=lambda j: (abs(j - i), j)) + repeated = slots[nearest].taken + if repeated is None: + steps.append(Step(None, slot.time, False)) + else: + steps.append(Step(repeated[1], repeated[0], False)) + return steps + + +def _clip_edge( + plan: TopicPlan, times: TopicTimes[M], sample: SampleTimes +) -> Optional[Tuple[int, M]]: + """The message ``"edge"`` repeats in a window where no step took one: the + clip's first on the topic for a before-window, its last for an after-window.""" + if plan.before: + found = times.first_in(sample.clip_start, sample.clip_last, 1) + else: + found = times.last_in(sample.clip_start, sample.clip_last, 1) + return found[0] if found else None + + +def _variable_steps( + plan: TopicPlan, times: TopicTimes[M], sample: SampleTimes, where: str +) -> List[Step[M]]: + """Every message of ``steps.every``'s window, inside the clip. + + A before-window holds ``(anchor - span, anchor]``, an after-window + ``(anchor, anchor + span]``. One that reaches past the clip fails the read + with ``pad="raise"``, and is cut at the clip's edge otherwise. + """ + assert plan.span_ns is not None + anchor = sample.anchor + if plan.before: + low, high = anchor - plan.span_ns + 1, anchor + past = sample.clip_start - low + else: + low, high = anchor + 1, anchor + plan.span_ns + past = high - sample.clip_last + if past > 0 and plan.pad == "raise": + clip = f"the clip [{sample.clip_start}, {sample.clip_end})" + reach = f"before {clip} starts" if plan.before else f"past the end of {clip}" + raise ValueError( + f"{where}: the window of topic {plan.topic!r} reaches " + f"{past / _NS_PER_S:g} s {reach}, and its pad is 'raise'. Use " + "pad='edge' or pad='zeros' to cut it at the clip's edge." + ) + low, high = max(low, sample.clip_start), min(high, sample.clip_last) + if low > high: + return [] + return [Step(m, time, True) for time, m in times.all_in(low, high)] + + +def _unfilled_message( + plan: TopicPlan, slots: Sequence[_Slot], sample: SampleTimes, where: str +) -> str: + """The error for a window with steps that took no message, under ``"raise"``.""" + empty = [slot for slot in slots if slot.taken is None] + outside = [slot.time for slot in empty if slot.outside] + if outside: + if plan.before: + past = sample.clip_start - min(outside) + reason = ( + f"The window reaches {past / _NS_PER_S:g} s before the clip starts." + ) + else: + past = max(outside) - sample.clip_last + reason = f"The window reaches {past / _NS_PER_S:g} s past the clip's end." + elif plan.fps is None: + side = "up to" if plan.before else "after" + found = len(slots) - len(empty) + reason = f"The clip holds only {found} messages on the topic {side} the anchor." + else: + side = "up to" if plan.before else "after" + reason = f"No message on the topic is logged in the clip {side} the anchor." + return ( + f"{where}: {len(empty)} of {len(slots)} steps of topic {plan.topic!r} have " + f"no message in the clip [{sample.clip_start}, {sample.clip_end}), and its " + f"pad is 'raise'. {reason} Use pad='edge' or pad='zeros' to pad such steps." + ) + + +# -- the samples dataset and the row layout --------------------------------- + + +def sample_manifest(batch: pa.Table) -> pa.Table: + """A batch of samples as manifest rows for the read tasks. + + Each sample keeps its columns and gains the manifest's: its file as the + path, with no size and no chunks. Fails on a sample that lists no file or + several, or whose anchor lies outside its clip. + """ + import numpy as np + + files = batch.column(FILES_COLUMN).combine_chunks() + counts = files.value_lengths().fill_null(0).to_numpy(zero_copy_only=False) + _check(batch, counts == 1, _files_problem) + paths = files.flatten() + _check(batch, ~paths.is_null().to_numpy(zero_copy_only=False), _files_problem) + start, end, anchor = ( + batch.column(name) + for name in (CLIP_START_COLUMN, CLIP_END_COLUMN, ANCHOR_COLUMN) + ) + known = ~np.logical_or.reduce( + [c.is_null().to_numpy(zero_copy_only=False) for c in (start, end, anchor)] + ) + _check(batch, known, _clip_problem) + start, end, anchor = (c.to_numpy() for c in (start, end, anchor)) + _check(batch, (start < end) & (start <= anchor) & (anchor < end), _clip_problem) + num_rows = batch.num_rows + return ( + batch.append_column(PATH_COLUMN_NAME, paths) + .append_column(FILE_SIZE_COLUMN_NAME, pa.nulls(num_rows, pa.int64())) + .append_column(FILE_CHUNK_METADATA_COLUMN_NAME, pa.nulls(num_rows)) + ) + + +def _check( + batch: pa.Table, ok: "np.ndarray", problem: Callable[[dict], Exception] +) -> None: + """Raise ``problem(row)`` for the first sample of ``batch`` that is not ``ok``.""" + if ok.all(): + return + first = int(ok.argmin()) + raise problem(batch.slice(first, 1).to_pylist()[0]) + + +def _describe(row: dict) -> str: + """How an error names a sample: by its id, else by its files and anchor.""" + if row.get(SAMPLE_ID_COLUMN) is not None: + return f"Sample {row[SAMPLE_ID_COLUMN]!r}" + return f"The sample with files {row[FILES_COLUMN]} and anchor {row[ANCHOR_COLUMN]}" + + +def _files_problem(row: dict) -> Exception: + files = row[FILES_COLUMN] or [] + if len(files) > 1: + return NotImplementedError( + f"{_describe(row)} lists {len(files)} files. A sample that spans " + "several files is not supported yet: give each sample one file." + ) + return ValueError(f"{_describe(row)} lists no file: give each sample one.") + + +def _clip_problem(row: dict) -> Exception: + return ValueError( + f"{_describe(row)} has clip [{row[CLIP_START_COLUMN]}, " + f"{row[CLIP_END_COLUMN]}) and anchor {row[ANCHOR_COLUMN]}. A clip must " + "not be empty, and the anchor must lie inside it: clip_start <= anchor < " + "clip_end, in nanoseconds of log time." + ) + + +def check_samples_schema(schema: pa.Schema, topics: Sequence[str]) -> None: + """Check that a samples dataset has the columns a sample read needs. + + ``files`` must be a list of strings, and the clip and anchor columns + integers. No column may take the name of a column the read adds. + """ + names = set(schema.names) + missing = [name for name in SAMPLE_COLUMNS if name not in names] + if missing: + raise ValueError( + f"The samples dataset lacks the columns {missing}. It needs " + f"{list(SAMPLE_COLUMNS)}: the sample's files, its clip [clip_start, " + "clip_end) and its anchor, in nanoseconds of log time." + ) + files_type = schema.field(FILES_COLUMN).type + if not ( + (pa.types.is_list(files_type) or pa.types.is_large_list(files_type)) + and ( + pa.types.is_string(files_type.value_type) + or pa.types.is_large_string(files_type.value_type) + ) + ): + raise ValueError( + f"The samples column {FILES_COLUMN!r} must be a list of paths, got " + f"{files_type}" + ) + for name in (CLIP_START_COLUMN, CLIP_END_COLUMN, ANCHOR_COLUMN): + if not pa.types.is_integer(schema.field(name).type): + raise ValueError( + f"The samples column {name!r} must hold integer nanoseconds of log " + f"time, got {schema.field(name).type}" + ) + added = {name for topic in topics for name in topic_column_names(topic)} + added |= {PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME, FILE_CHUNK_METADATA_COLUMN_NAME} + taken = sorted(names & added) + if taken: + raise ValueError( + f"The samples dataset has columns {taken} that the read adds. Rename " + "or drop them." + ) + + +def topic_column_names(topic: str) -> List[str]: + """Every name a sample row may give one topic's columns.""" + return [f"{prefix}{topic}" for prefix in TOPIC_PREFIXES] + + +def topic_fields(plan: TopicPlan) -> List[pa.Field]: + """A topic's columns in a sample row, one list entry per step. + + ``data:`` holds each step's payload, empty for a step that + ``"zeros"`` pads. ``times:`` holds each step's log time and + ``valid:`` whether the step took a message of its own. + """ + topic = plan.topic + return [ + pa.field(f"{DATA_PREFIX}{topic}", pa.list_(pa.large_binary())), + pa.field(f"{TIMES_PREFIX}{topic}", pa.list_(pa.int64())), + pa.field(f"{VALID_PREFIX}{topic}", pa.list_(pa.bool_())), + ] + + +def sample_row_schema( + samples_schema: pa.Schema, plans: Sequence[TopicPlan] +) -> pa.Schema: + """The schema of sample rows: the samples' own columns, then each topic's. + + Topics come in the order ``read_mcap(topics=...)`` gives them. + """ + fields = list(samples_schema) + for plan in plans: + fields += topic_fields(plan) + return pa.schema(fields) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_scanner.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_scanner.py index 90fe7f814a0d..d046d20eb54c 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_scanner.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_scanner.py @@ -10,6 +10,10 @@ from ray.data._internal.datasource_v2.common.file_scanner import FileScanner from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_reader import MCAPReader +from ray.data._internal.datasource_v2.formats.mcap.mcap_sample_rows import ( + DEFAULT_MAX_ROW_BYTES, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import TopicPlan from ray.data._internal.datasource_v2.interfaces.pushdown import ( SupportsColumnPruning, SupportsLimitPushdown, @@ -32,7 +36,8 @@ class MCAPScanner(FileScanner, SupportsColumnPruning, SupportsLimitPushdown): The planned ``schema`` fixes what ``data`` holds in every block: decoded JSON values when every selected channel of the sampled files is - JSON-encoded, and the payload bytes otherwise. + JSON-encoded, and the payload bytes otherwise. A sample read carries each + topic's steps in ``topic_plans``. """ schema: pa.Schema @@ -45,6 +50,8 @@ class MCAPScanner(FileScanner, SupportsColumnPruning, SupportsLimitPushdown): limit: Optional[int] = None synthesized_columns: Tuple[SynthesizedColumn, ...] = () target_block_size: Optional[int] = None + max_row_bytes: int = DEFAULT_MAX_ROW_BYTES + topic_plans: Tuple[TopicPlan, ...] = () def read_schema(self) -> pa.Schema: """Return the dataset schema after column pruning. @@ -104,4 +111,6 @@ def create_reader(self) -> MCAPReader: target_block_size=self.target_block_size, schema=self.schema, decode_json=self.decodes_json, + max_row_bytes=self.max_row_bytes, + topic_plans=self.topic_plans, ) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_topic_spec.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_topic_spec.py new file mode 100644 index 000000000000..1bf940eec9e1 --- /dev/null +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_topic_spec.py @@ -0,0 +1,209 @@ +"""``TopicSpec``: how ``read_mcap(samples=...)`` reads one topic of a sample. + +A spec has two slots. ``steps`` says where the topic's steps fall, from the +sample row alone. ``match`` says which message fills each step. ``pad`` says +what a step that no message fills holds. ``ray.data.datasource.mcap.steps`` and +``ray.data.datasource.mcap.match`` export the built-ins below:: + + TopicSpec(steps.before(1, fps=10)) time steps, nearest message + TopicSpec(steps.after(2, fps=2), pad="zeros") time steps after the anchor + TopicSpec(steps.last(4)) the last 4 messages + TopicSpec(steps.every(before_s=1)) every message of the last 1 s + +Time steps (``before``, ``after``) lie ``1/fps`` seconds apart and are filled by +``match``. Message steps (``last``, ``next``, ``every``) are recorded messages +themselves, so they take no ``match``. ``mcap_samples`` resolves the steps. +""" + +import math +import numbers +from dataclasses import dataclass +from fractions import Fraction +from typing import Literal, Optional, Tuple, Union + +from ray.util.annotations import PublicAPI + +_NS_PER_S = 1_000_000_000 + +Side = Literal["before", "after"] +# How ``read_mcap(samples=...)`` fills a step that no message fills. +PAD_MODES: Tuple[str, ...] = ("edge", "zeros", "raise") + + +def exact_number(value: Union[float, Fraction], name: str) -> Fraction: + """``value`` as a fraction. A float is read as the decimal it prints as.""" + if isinstance(value, bool) or not isinstance(value, numbers.Real): + raise ValueError( + f"{name} must be a positive number or a Fraction, got {value!r}" + ) + if isinstance(value, numbers.Rational): + return Fraction(value.numerator, value.denominator) + if not math.isfinite(value): + raise ValueError(f"{name} must be finite, got {value!r}") + return Fraction(str(value)) + + +def _positive_count(value: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer, got {value!r}") + return value + + +def _positive_seconds(value: Union[float, Fraction], name: str) -> Fraction: + seconds = exact_number(value, name) + if seconds <= 0: + raise ValueError(f"{name} must be positive, got {value!r}") + if round(seconds * _NS_PER_S) < 1: + raise ValueError(f"{name} must be at least one nanosecond, got {value!r}") + return seconds + + +@PublicAPI(stability="alpha") +@dataclass(frozen=True) +class TimeSteps: + """Steps ``1/fps`` seconds apart on one side of the anchor. + + Build it with ``steps.before`` or ``steps.after``. A before-window's last + step is the anchor; an after-window's first step is one step after it. + + Attributes: + side: ``"before"`` or ``"after"`` the anchor. + count: The number of steps. + fps: Steps per second, as an exact fraction. + """ + + side: Side + count: int + fps: Fraction + + +@PublicAPI(stability="alpha") +@dataclass(frozen=True) +class MessageSteps: + """Steps that are the topic's recorded messages next to the anchor. + + Build it with ``steps.last``, ``steps.next`` or ``steps.every``. Exactly one + of ``count`` and ``span_ns`` is set. + + Attributes: + side: ``"before"`` (up to and including the anchor) or ``"after"`` it. + count: That many messages, or ``None``. + span_ns: Every message within this many nanoseconds, or ``None``. The + number of steps then varies from row to row. + """ + + side: Side + count: Optional[int] = None + span_ns: Optional[int] = None + + +Steps = Union[TimeSteps, MessageSteps] + + +@PublicAPI(stability="alpha") +@dataclass(frozen=True) +class Nearest: + """Each step takes the message logged nearest to it, the earlier on a tie. + + A step before the anchor takes only messages logged up to the anchor, and a + step after it only messages logged after it, so no step looks past the + anchor from its side. Build it with ``match.nearest``. + """ + + +Match = Nearest + + +def time_steps( + side: Side, + seconds: Optional[Union[float, Fraction]], + frames: Optional[int], + fps: Union[float, Fraction], +) -> TimeSteps: + """The ``TimeSteps`` that ``steps.before`` and ``steps.after`` build.""" + name = "steps.before" if side == "before" else "steps.after" + if (seconds is None) == (frames is None): + raise ValueError(f"{name} takes seconds or frames=, exactly one of them") + rate = exact_number(fps, "fps") + if rate <= 0: + raise ValueError(f"fps must be a positive number, got {fps!r}") + if rate > _NS_PER_S: + raise ValueError( + f"fps must leave at least one nanosecond per step, got {fps!r}" + ) + if frames is not None: + return TimeSteps(side, _positive_count(frames, "frames"), rate) + assert seconds is not None + count = _positive_seconds(seconds, "seconds") * rate + if count.denominator != 1: + raise ValueError( + f"{name}({seconds!r}, fps={fps!r}) is {float(count):g} steps. Give a " + "whole number of steps, or a step count with frames=." + ) + return TimeSteps(side, int(count), rate) + + +def message_count_steps(side: Side, count: int) -> MessageSteps: + """The ``MessageSteps`` that ``steps.last`` and ``steps.next`` build.""" + return MessageSteps(side, count=_positive_count(count, "n")) + + +def message_span_steps( + before_s: Optional[Union[float, Fraction]], + after_s: Optional[Union[float, Fraction]], +) -> MessageSteps: + """The ``MessageSteps`` that ``steps.every`` builds.""" + if (before_s is None) == (after_s is None): + raise ValueError("steps.every takes before_s= or after_s=, exactly one of them") + side: Side = "before" if before_s is not None else "after" + seconds = _positive_seconds( + before_s if before_s is not None else after_s, # type: ignore[arg-type] + "before_s" if before_s is not None else "after_s", + ) + return MessageSteps(side, span_ns=round(seconds * _NS_PER_S)) + + +@PublicAPI(stability="alpha") +@dataclass(frozen=True) +class TopicSpec: + """How ``read_mcap(samples=...)`` reads one topic of each sample. + + Args: + steps: Where the topic's steps fall: one of the built-ins of + ``ray.data.datasource.mcap.steps``. + match: Which message fills each time step: a built-in of + ``ray.data.datasource.mcap.match``. Defaults to ``match.nearest()``. + Message steps (``steps.last``, ``steps.next``, ``steps.every``) are + messages already and refuse a ``match``. + pad: What a step that no message fills holds, such as one past the + clip's edge. ``"edge"`` repeats the nearest real step, or the clip's + first or last message on the topic when the window has none. + ``"zeros"`` gives an empty payload. ``"raise"`` fails the read, + naming the sample. ``valid:`` marks a padded step ``False``. + A window of every message has no fixed steps to pad: ``"raise"`` + fails the read when it reaches past the clip, and the other two cut + it there. + """ + + steps: Steps + match: Optional[Match] = None + pad: Literal["edge", "zeros", "raise"] = "edge" + + def __post_init__(self): + if not isinstance(self.steps, (TimeSteps, MessageSteps)): + raise ValueError( + "TopicSpec needs steps from ray.data.datasource.mcap.steps, got " + f"{self.steps!r}" + ) + if self.match is not None and not isinstance(self.match, Nearest): + raise ValueError( + "TopicSpec needs match from ray.data.datasource.mcap.match, got " + f"{self.match!r}" + ) + if self.match is not None and isinstance(self.steps, MessageSteps): + raise ValueError( + "steps.last, steps.next and steps.every take recorded messages " + "as their steps, so they take no match" + ) + if self.pad not in PAD_MODES: + raise ValueError(f"pad must be one of {list(PAD_MODES)}, got {self.pad!r}") diff --git a/python/ray/data/_internal/logical/operators/read_operator.py b/python/ray/data/_internal/logical/operators/read_operator.py index a1a91029cb31..6480d7f320e4 100644 --- a/python/ray/data/_internal/logical/operators/read_operator.py +++ b/python/ray/data/_internal/logical/operators/read_operator.py @@ -269,6 +269,10 @@ class ReadFiles( # here). Applied in ``plan_read_files_op.do_read`` after each # table is read. block_udf: Optional[Callable[[Block], Block]] = None + # Whether each output row comes from one input row and keeps its columns, + # as in ``read_mcap(samples=...)``. A checkpoint then drops finished input + # rows before the read rather than finished rows after it. + checkpoint_on_input: bool = False input_dependencies: List[LogicalOperator] = field(repr=False, kw_only=True) can_modify_num_rows: bool = field(init=False, default=True) min_rows_per_bundled_input: Optional[int] = field(init=False, default=None) diff --git a/python/ray/data/_internal/planner/checkpoint/plan_read_files_op.py b/python/ray/data/_internal/planner/checkpoint/plan_read_files_op.py index 19fcc1bb5089..7cc5b792dc47 100644 --- a/python/ray/data/_internal/planner/checkpoint/plan_read_files_op.py +++ b/python/ray/data/_internal/planner/checkpoint/plan_read_files_op.py @@ -39,8 +39,16 @@ def plan_read_files_op_with_checkpoint_filter( Defers all wrapping behavior to V1's :func:`create_checkpoint_filter_op` so the not-found short-circuit, ``IdColumnCheckpointManager.load_checkpoint`` invocation, actor-pool sizing, and ``supports_fusion=False`` placement stay - in one place across the V1 and V2 read paths. + in one place across the V1 and V2 read paths. With + ``op.checkpoint_on_input``, the filter goes on the read's input instead, so + finished rows are dropped before anything is read for them. """ + if op.checkpoint_on_input: + (child,) = physical_children + filtered = create_checkpoint_filter_op( + child, data_context, data_file_dir, data_file_filesystem + ) + return plan_read_files_op(op, [filtered], data_context) physical_read_op = plan_read_files_op(op, physical_children, data_context) return create_checkpoint_filter_op( physical_read_op, data_context, data_file_dir, data_file_filesystem diff --git a/python/ray/data/_internal/planner/plan_read_files_op.py b/python/ray/data/_internal/planner/plan_read_files_op.py index 7bce8d93f558..683d9af67ac7 100644 --- a/python/ray/data/_internal/planner/plan_read_files_op.py +++ b/python/ray/data/_internal/planner/plan_read_files_op.py @@ -33,7 +33,7 @@ ) from ray.data._internal.logical.operators import ReadFiles from ray.data._internal.output_buffer import OutputBlockSizeOption -from ray.data.block import Block +from ray.data.block import Block, BlockAccessor from ray.data.context import DataContext logger = logging.getLogger(__name__) @@ -61,6 +61,10 @@ def do_read(blocks: Iterable[Block], _: TaskContext) -> Iterable[Block]: # ``prune_input_split`` is an identity by default; ``FileScanner`` # overrides it to drop files failing a pushed-down partition predicate. for block in blocks: + # A block can arrive empty and without the manifest's columns, + # such as when a checkpoint filter upstream dropped all its rows. + if BlockAccessor.for_block(block).num_rows() == 0: + continue manifest = scanner.prune_input_split(FileManifest(block)) if len(manifest) == 0: continue diff --git a/python/ray/data/_internal/planner/planner.py b/python/ray/data/_internal/planner/planner.py index 29033a2e445c..1bd73d3ce9f3 100644 --- a/python/ray/data/_internal/planner/planner.py +++ b/python/ray/data/_internal/planner/planner.py @@ -1,6 +1,16 @@ import warnings from functools import partial -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Type, TypeVar +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, +) if TYPE_CHECKING: import pyarrow.fs @@ -280,6 +290,10 @@ class Planner: def __init__(self): self._supports_checkpointing = False self._plan_fns_for_checkpointing = {} + # The reads that get a checkpoint filter, by ``id``: on each path from + # the sink, the first one. A read further up, such as one that lists + # the inputs of another read, has rows without the id column. + self._checkpoint_filter_ops: Set[int] = set() def plan( self, logical_plan: LogicalPlan @@ -294,6 +308,7 @@ def plan( logical_plan ): self._supports_checkpointing = True + self._checkpoint_filter_ops = self._nearest_reads(logical_plan.dag) data_file_dir, data_file_fs = self._get_data_file_info(logical_plan) checkpoint_callback = self._create_checkpoint_callback( @@ -345,7 +360,10 @@ def plan( return physical_plan, callbacks def get_plan_fn(self, logical_op: LogicalOperator) -> PlanLogicalOpFn: - if self._supports_checkpointing: + upstream_read = isinstance(logical_op, self._CHECKPOINT_FILTER_OPS) and ( + id(logical_op) not in self._checkpoint_filter_ops + ) + if self._supports_checkpointing and not upstream_read: assert self._plan_fns_for_checkpointing plan_fn = find_plan_fn(logical_op, self._plan_fns_for_checkpointing) if plan_fn is not None: @@ -462,6 +480,18 @@ def _get_plan_fns_for_checkpointing( } return plan_fns + def _nearest_reads(self, root: LogicalOperator) -> Set[int]: + """The ``id`` of the first checkpoint-filter op on each path from ``root``.""" + found: Set[int] = set() + pending = [root] + while pending: + op = pending.pop() + if isinstance(op, self._CHECKPOINT_FILTER_OPS): + found.add(id(op)) + else: + pending.extend(op.input_dependencies) + return found + def _check_supports_checkpointing(self, logical_plan: LogicalPlan) -> bool: """Check if the logical plan supports checkpointing. diff --git a/python/ray/data/datasource/__init__.py b/python/ray/data/datasource/__init__.py index ce36ec6f3a31..73dbf356f717 100644 --- a/python/ray/data/datasource/__init__.py +++ b/python/ray/data/datasource/__init__.py @@ -10,6 +10,7 @@ from ray.data._internal.datasource.turbopuffer_datasink import ( TurbopufferDatasink, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec from ray.data._internal.savemode import SaveMode from ray.data.datasource.datasink import ( Datasink, @@ -75,6 +76,7 @@ "BlockBasedFileDatasink", "_S3FileSystemWrapper", "TimeRange", + "TopicSpec", "WriteResult", "WriteReturnType", "SaveMode", diff --git a/python/ray/data/datasource/mcap/__init__.py b/python/ray/data/datasource/mcap/__init__.py new file mode 100644 index 000000000000..ad423dadaec0 --- /dev/null +++ b/python/ray/data/datasource/mcap/__init__.py @@ -0,0 +1,14 @@ +"""Building blocks of ``read_mcap(samples=...)``'s ``TopicSpec``. + +``steps`` holds the rules for where a topic's steps fall, and ``match`` the +rules for which message fills each step:: + + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import match, steps + + TopicSpec(steps.before(1, fps=5), match.nearest()) +""" + +from ray.data.datasource.mcap import match, steps + +__all__ = ["match", "steps"] diff --git a/python/ray/data/datasource/mcap/match.py b/python/ray/data/datasource/mcap/match.py new file mode 100644 index 000000000000..182c1f5977a9 --- /dev/null +++ b/python/ray/data/datasource/mcap/match.py @@ -0,0 +1,14 @@ +"""Which message fills each time step of a topic, for ``TopicSpec``.""" + +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import Nearest +from ray.util.annotations import PublicAPI + + +@PublicAPI(stability="alpha") +def nearest() -> Nearest: + """Each step takes the message logged nearest to it, the earlier on a tie. + + A step never takes a message from the other side of the anchor. This is + ``TopicSpec``'s default. + """ + return Nearest() diff --git a/python/ray/data/datasource/mcap/steps.py b/python/ray/data/datasource/mcap/steps.py new file mode 100644 index 000000000000..fb0d5f246dba --- /dev/null +++ b/python/ray/data/datasource/mcap/steps.py @@ -0,0 +1,99 @@ +"""Where a topic's steps fall around a sample's anchor, for ``TopicSpec``. + +``before`` and ``after`` place steps at fixed times, ``1/fps`` seconds apart, +which ``match`` fills. ``last``, ``next`` and ``every`` take the topic's +recorded messages themselves as the steps. +""" + +from fractions import Fraction +from typing import Optional, Union + +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import ( + MessageSteps, + TimeSteps, + message_count_steps, + message_span_steps, + time_steps, +) +from ray.util.annotations import PublicAPI + +Number = Union[float, Fraction] + + +@PublicAPI(stability="alpha") +def before( + seconds: Optional[Number] = None, + *, + frames: Optional[int] = None, + fps: Number, +) -> TimeSteps: + """Steps ``1/fps`` seconds apart up to the anchor, the last one on it. + + Args: + seconds: The window's length, a whole number of steps at ``fps``. + frames: The window's length as a number of steps, in place of + ``seconds``. + fps: Steps per second. Accepts a ``fractions.Fraction``, such as + ``Fraction(30000, 1001)``, and reads a ``float`` as the decimal it + prints as. + + Returns: + The steps, for ``TopicSpec(steps=...)``. + + Examples: + >>> from ray.data.datasource.mcap import steps + >>> steps.before(1, fps=10).count + 10 + """ + return time_steps("before", seconds, frames, fps) + + +@PublicAPI(stability="alpha") +def after( + seconds: Optional[Number] = None, + *, + frames: Optional[int] = None, + fps: Number, +) -> TimeSteps: + """Steps ``1/fps`` seconds apart after the anchor, the first one step after it. + + Args: + seconds: The window's length, a whole number of steps at ``fps``. + frames: The window's length as a number of steps, in place of + ``seconds``. + fps: Steps per second, as for :func:`before`. + + Returns: + The steps, for ``TopicSpec(steps=...)``. + + Examples: + >>> from ray.data.datasource.mcap import steps + >>> steps.after(2, fps=2).count + 4 + """ + return time_steps("after", seconds, frames, fps) + + +@PublicAPI(stability="alpha") +def last(n: int) -> MessageSteps: + """The last ``n`` messages logged up to and including the anchor, one step + each, at whatever rate they were logged.""" + return message_count_steps("before", n) + + +@PublicAPI(stability="alpha") +def next(n: int) -> MessageSteps: # noqa: A001 - mirrors last(n) + """The first ``n`` messages logged after the anchor, one step each.""" + return message_count_steps("after", n) + + +@PublicAPI(stability="alpha") +def every( + *, before_s: Optional[Number] = None, after_s: Optional[Number] = None +) -> MessageSteps: + """Every message logged within ``before_s`` seconds up to the anchor, or + within ``after_s`` seconds after it, one step each. + + The number of steps varies from row to row. + """ + return message_span_steps(before_s, after_s) diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index 02422d188728..b107aa035b60 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -71,6 +71,7 @@ from ray.data._internal.datasource.video_datasource import VideoDatasource from ray.data._internal.datasource.webdataset_datasource import WebDatasetDatasource from ray.data._internal.datasource.zarrv2_datasource import ZarrV2Datasource +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder from ray.data._internal.logical.interfaces import LogicalPlan from ray.data._internal.logical.operators import ( @@ -3244,11 +3245,144 @@ def read_tfrecords( ) +def _reject_options_of_listed_reads(**options: Any) -> None: + """Fail when a sample read is given an option that only a read of paths has. + + An option counts as given when it differs from ``read_mcap``'s default. + """ + defaults = { + "log_time_order": True, + "include_row_id": False, + "include_paths": False, + "ignore_missing_paths": False, + } + given = sorted( + name for name, value in options.items() if value != defaults.get(name) + ) + if given: + raise ValueError( + "A sample read (samples=...) takes its files and times from the " + f"samples, so it does not take {given}." + ) + + +def _read_mcap_samples( + samples: Dataset, + *, + topics: Any, + time_source: str, + filesystem: Optional["pyarrow.fs.FileSystem"], + num_cpus: Optional[float], + num_gpus: Optional[float], + memory: Optional[float], + concurrency: Optional[int], + label_selector: Optional[Dict[str, str]], + fallback_strategy: Optional[List[Dict[str, Any]]], + max_calls: Optional[int], + resources: Optional[Dict[str, float]], + accelerator_type: Optional[str], + runtime_env: Optional[Dict[str, Any]], + ray_remote_args: Optional[Dict[str, Any]], +) -> Dataset: + """Read one row per sample of ``samples``, as ``read_mcap(samples=...)``. + + The samples dataset feeds the read tasks in place of a listing. A map turns + each sample into a manifest row that names its file + (``mcap_samples.sample_manifest``), and the rows are cut into blocks of + ``RAY_DATA_MCAP_SAMPLES_PER_TASK`` samples, one read task each:: + + samples -> map to manifest rows -> blocks of N samples -> ReadFiles + """ + import pyarrow as pa + + from ray._common.utils import env_integer + from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, + ) + from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import ( + check_samples_schema, + sample_manifest, + ) + + ctx = DataContext.get_current() + if not ctx.use_datasource_v2: + raise NotImplementedError( + "read_mcap(samples=...) requires the V2 datasource. Enable it with " + "`ray.data.DataContext.get_current().use_datasource_v2 = True` " + "(or set RAY_DATA_USE_DATASOURCE_V2=1)." + ) + if time_source == "header": + raise NotImplementedError( + "time_source='header' is not supported yet: give the clips and " + "anchors in log time." + ) + if time_source != "log": + raise ValueError(f"time_source must be 'log' or 'header', got {time_source!r}") + if not isinstance(topics, collections.abc.Mapping) or not topics: + raise ValueError( + "A sample read needs topics={topic: TopicSpec(...)}: where each " + "topic's steps fall around a sample's anchor." + ) + not_specs = sorted( + t for t, spec in topics.items() if not isinstance(spec, TopicSpec) + ) + if not_specs: + raise ValueError(f"topics must map each topic to a TopicSpec: {not_specs}") + schema = samples.schema() + if schema is None: + raise ValueError("The samples dataset has no schema: is it empty?") + samples_schema = schema.base_schema + if not isinstance(samples_schema, pa.Schema): + samples_schema = samples.take_batch(1, batch_format="pyarrow").schema + check_samples_schema(samples_schema, list(topics)) + + datasource = MCAPDatasourceV2( + [], + topics=dict(topics), + filesystem=filesystem, + samples_schema=samples_schema, + ) + row_schema = datasource.infer_schema(None) + scanner = datasource.create_scanner(schema=row_schema, filesystem=filesystem) + samples_per_task = env_integer("RAY_DATA_MCAP_SAMPLES_PER_TASK", 64) + manifest = samples.map_batches( + sample_manifest, batch_format="pyarrow", zero_copy_batch=True + ).repartition(target_num_rows_per_block=samples_per_task, strict=True) + read_op = ReadFiles( + datasource_name=datasource.name, + scanner=scanner, + schema=row_schema, + parallelism=-1, + ray_remote_args=_resolve_read_remote_args( + datasource, + ray_remote_args=ray_remote_args, + num_cpus=num_cpus, + num_gpus=num_gpus, + memory=memory, + label_selector=label_selector, + fallback_strategy=fallback_strategy, + max_calls=max_calls, + resources=resources, + accelerator_type=accelerator_type, + runtime_env=runtime_env, + ctx=ctx, + ), + compute=get_compute_strategy_for_read_api(None, concurrency), + # Each row comes from one sample and keeps its columns, so a checkpoint + # skips finished samples before their files are read. + checkpoint_on_input=True, + input_dependencies=[manifest._logical_plan.dag], + ) + return Dataset._from_parent(manifest, LogicalPlan(read_op, manifest.context)) + + @PublicAPI(stability="alpha") def read_mcap( - paths: Union[str, List[str]], + paths: Optional[Union[str, List[str]]] = None, *, - topics: Optional[Union[List[str], Set[str]]] = None, + samples: Optional[Dataset] = None, + topics: Optional[Union[List[str], Set[str], Dict[str, TopicSpec]]] = None, + time_source: Literal["log", "header"] = "log", time_range: Optional[Union[Tuple[int, int], TimeRange]] = None, message_types: Optional[Union[List[str], Set[str]]] = None, include_metadata: bool = True, @@ -3356,6 +3490,21 @@ def read_mcap( ... include_paths=True # doctest: +SKIP ... ) # doctest: +SKIP + Read one training sample per row: each topic's steps around the + sample's anchor, with the samples' own columns passed through. + + >>> from ray.data.datasource import TopicSpec # doctest: +SKIP + >>> from ray.data.datasource.mcap import steps # doctest: +SKIP + >>> samples = ray.data.read_parquet("s3://bucket/samples/") # doctest: +SKIP + >>> ds = ray.data.read_mcap( # doctest: +SKIP + ... samples=samples, # doctest: +SKIP + ... topics={ # doctest: +SKIP + ... "/cam/head": TopicSpec(steps.before(0.5, fps=10)), # doctest: +SKIP + ... "/joint_states": TopicSpec(steps.before(1, fps=50), pad="zeros"), # doctest: +SKIP + ... "/action": TopicSpec(steps.after(1, fps=5), pad="raise"), # doctest: +SKIP + ... }, # doctest: +SKIP + ... ) # doctest: +SKIP + Store ``schema_data`` as plain bytes, as the legacy reader does. >>> import pyarrow as pa # doctest: +SKIP @@ -3366,9 +3515,29 @@ def read_mcap( Args: paths: A single file or directory, or a list of file or directory paths. - A list of paths can contain both files and directories. + A list of paths can contain both files and directories. Leave it out + when ``samples`` names the files. + samples: A :class:`~ray.data.Dataset` of training samples to read, one + output row each, in place of ``paths``. Each sample has ``files``, a + list of one MCAP path, its clip ``clip_start`` and ``clip_end``, and an + ``anchor`` inside the clip: integer nanoseconds of log time, the clip's + end exclusive. ``topics`` then maps each topic to a + :class:`~ray.data.datasource.TopicSpec`: where the topic's steps fall + around the anchor, and which message fills each. A row keeps the + sample's columns and + adds, per topic, ``data:`` with each step's payload, + ``times:`` with each step's log time and ``valid:``, + which is ``False`` for a padded step. A ``sample_id`` column names + a sample in errors. Read tasks take about 64 samples each + (``RAY_DATA_MCAP_SAMPLES_PER_TASK``) and read each file's samples in + anchor order, so a samples dataset sorted by file and anchor shares + chunks between samples. Requires the V2 datasource. topics: Optional list or set of topic names to include. If specified, only - messages from these topics will be read. + messages from these topics will be read. With ``samples``, a dict from + each topic to its :class:`~ray.data.datasource.TopicSpec`, in the order + the row's columns take. + time_source: The clock of a sample's clip and anchor. Only ``"log"``, the + messages' ``log_time``, is supported so far. time_range: Optional time range for filtering messages by timestamp. Can be either a tuple of (start_time, end_time) in nanoseconds (for backwards compatibility) or a TimeRange object. Both values must be non-negative and start_time < end_time. @@ -3457,6 +3626,76 @@ def read_mcap( :class:`~ray.data.Dataset` producing records read from the specified MCAP files. """ _validate_shuffle_arg(shuffle) + checkpoint_config = DataContext.get_current().checkpoint_config + if checkpoint_config is not None and checkpoint_config.has_generated_id_column: + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + raise InvalidCheckpointingConfig( + "`CheckpointConfig(generated_id_column=...)` works only with " + "`read_parquet`. For `read_mcap`, use " + '`CheckpointConfig(id_column="row_id")`. Its `row_id` column gives ' + "each message an id that stays the same on every run." + ) + from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( + ROW_ID_COLUMN, + ) + + if samples is not None: + if ( + checkpoint_config is not None + and checkpoint_config.id_column == ROW_ID_COLUMN + ): + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + + raise InvalidCheckpointingConfig( + "A sample read's rows have no `row_id`. Key the checkpoint on a " + 'column of the samples instead, such as `id_column="sample_id"`.' + ) + _reject_options_of_listed_reads( + paths=paths, + time_range=time_range, + message_types=message_types, + log_time_order=log_time_order, + include_row_id=include_row_id, + partition_filter=partition_filter, + partitioning=partitioning, + include_paths=include_paths, + ignore_missing_paths=ignore_missing_paths, + shuffle=shuffle, + file_extensions=file_extensions, + ) + return _read_mcap_samples( + samples, + topics=topics, + time_source=time_source, + filesystem=filesystem, + num_cpus=num_cpus, + num_gpus=num_gpus, + memory=memory, + concurrency=concurrency, + label_selector=label_selector, + fallback_strategy=fallback_strategy, + max_calls=max_calls, + resources=resources, + accelerator_type=accelerator_type, + runtime_env=runtime_env, + ray_remote_args=ray_remote_args, + ) + if paths is None: + raise ValueError( + "read_mcap needs the paths to read, or a dataset of samples to read " + "(samples=...)." + ) + if isinstance(topics, collections.abc.Mapping): + raise ValueError( + "topics maps each topic to a TopicSpec only in a sample read: pass " + "samples=... too, or list the topics to read." + ) + if time_source != "log": + raise ValueError( + f"time_source={time_source!r} applies only to a sample read " + "(samples=...)." + ) if file_extensions is None: file_extensions = ["mcap"] @@ -3471,20 +3710,6 @@ def read_mcap( time_range = TimeRange(start_time=time_range[0], end_time=time_range[1]) ctx = DataContext.get_current() - checkpoint_config = ctx.checkpoint_config - if checkpoint_config is not None and checkpoint_config.has_generated_id_column: - from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig - - raise InvalidCheckpointingConfig( - "`CheckpointConfig(generated_id_column=...)` works only with " - "`read_parquet`. For `read_mcap`, use " - '`CheckpointConfig(id_column="row_id")`. Its `row_id` column gives ' - "each message an id that stays the same on every run." - ) - from ray.data._internal.datasource_v2.formats.mcap.mcap_options import ( - ROW_ID_COLUMN, - ) - if checkpoint_config is not None and checkpoint_config.id_column == ROW_ID_COLUMN: # A checkpoint keyed on ``row_id`` needs the column in the read output. include_row_id = True diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 199babf03eab..3b236477a1d8 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -3,6 +3,7 @@ import logging import os import re +from typing import Any, Dict import pyarrow as pa import pytest @@ -871,6 +872,196 @@ def test_read_mcap_lists_files_in_parallel( assert int(match.group(1)) == 3 +def _write_video_recording(path, seconds=3): + """A 10 fps camera with a keyframe every ten frames (synthetic H.264 Annex-B) + and a 50 Hz IMU, in small chunks.""" + from mcap.writer import CompressionType, Writer + + start = b"\x00\x00\x00\x01" + keyframe = ( + ( + start + + bytes([0x67, 0x42, 0x00, 0x1F]) + + start + + bytes([0x68, 0xCE, 0x38, 0x80]) + ) + + start + + bytes([0x65, 0x88, 0x84, 0x00]) + ) + pframe = start + bytes([0x41, 0x9A, 0x02, 0x04]) + with open(path, "wb") as stream: + writer = Writer(stream, chunk_size=600, compression=CompressionType.ZSTD) + writer.start(profile="ros2", library="ray-test") + cam_schema_id = writer.register_schema( + name="foxglove_msgs/msg/CompressedVideo", encoding="ros2msg", data=b"" + ) + imu_schema = writer.register_schema( + name="sensor_msgs/msg/Imu", encoding="ros2msg", data=b"" + ) + cam = writer.register_channel( + schema_id=cam_schema_id, topic="/cam", message_encoding="cdr" + ) + imu = writer.register_channel( + schema_id=imu_schema, topic="/imu", message_encoding="cdr" + ) + messages = [] + for frame in range(10 * seconds): + payload = (keyframe if frame % 10 == 0 else pframe) + frame.to_bytes( + 4, "big" + ) + messages.append((frame * 100_000_000, cam, payload)) + for i in range(50 * seconds): + messages.append((i * 20_000_000, imu, i.to_bytes(8, "big"))) + for log_time, channel_id, payload in sorted(messages): + writer.add_message( + channel_id=channel_id, + log_time=log_time, + publish_time=log_time, + data=payload, + ) + writer.finish() + + +def _sample_rows(path, anchors): + """One sample per anchor in ``path``, clipped to [0.5 s, 2.5 s).""" + return [ + { + "sample_id": f"s{i}", + "files": [path], + "clip_start": 500_000_000, + "clip_end": 2_500_000_000, + "anchor": anchor, + } + for i, anchor in enumerate(anchors) + ] + + +def test_read_mcap_samples(ray_start_regular_shared, tmp_path, datasource_v2): + """A sample read gives one row per sample: its own columns, then each topic's + steps. Projection, ``count`` and ``limit`` apply to it. The legacy datasource + refuses it.""" + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import steps + + path = os.path.join(tmp_path, "video.mcap") + _write_video_recording(path) + samples = ray.data.from_items( + _sample_rows(path, [1_000_000_000, 1_500_000_000, 2_400_000_000]) + ) + topics = { + "/cam": TopicSpec(steps.before(0.3, fps=10)), + "/imu": TopicSpec(steps.after(frames=3, fps=20), pad="zeros"), + } + + if not datasource_v2: + with pytest.raises(NotImplementedError, match="V2 datasource"): + ray.data.read_mcap(samples=samples, topics=topics) + return + ds = ray.data.read_mcap(samples=samples, topics=topics) + rows = sorted(ds.take_all(), key=lambda row: row["anchor"]) + + assert ds.schema().names == [ + "sample_id", + "files", + "clip_start", + "clip_end", + "anchor", + "data:/cam", + "times:/cam", + "valid:/cam", + "data:/imu", + "times:/imu", + "valid:/imu", + ] + cam = [[int.from_bytes(d[-4:], "big") for d in row["data:/cam"]] for row in rows] + assert cam == [[8, 9, 10], [13, 14, 15], [22, 23, 24]] + # The last sample's steps at 2.5 and 2.55 s lie past its clip, so are empty. + assert [row["valid:/imu"] for row in rows] == [[True] * 3] * 2 + [ + [True, False, False] + ] + assert rows[2]["data:/imu"][1:] == [b"", b""] + assert ds.count() == 3 + assert ds.limit(2).count() == 2 + assert sorted( + r["sample_id"] for r in ds.select_columns(["sample_id"]).take_all() + ) == [ + "s0", + "s1", + "s2", + ] + + +@pytest.mark.parametrize( + "kwargs, error, match", + [ + ({"paths": "x.mcap"}, ValueError, r"does not take \['paths'\]"), + ({"time_range": (0, 1)}, ValueError, "time_range"), + ({"topics": ["/cam"]}, ValueError, r"topics=\{topic: TopicSpec"), + ({"topics": {"/cam": 0.5}}, ValueError, "map each topic to a TopicSpec"), + ({"time_source": "header"}, NotImplementedError, "header"), + ({"time_source": "wall"}, ValueError, "time_source must be"), + ], +) +def test_read_mcap_samples_rejects_options( + ray_start_regular_shared, tmp_path, datasource_v2, kwargs, error, match +): + """A sample read refuses the options of a read of paths, and needs a + ``TopicSpec`` per topic and log time.""" + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import steps + + if not datasource_v2: + pytest.skip("validated on the V2 path") + samples = ray.data.from_items(_sample_rows("x.mcap", [1_000_000_000])) + options: Dict[str, Any] = {"topics": {"/cam": TopicSpec(steps.last(1))}} + options.update(kwargs) + + with pytest.raises(error, match=match): + ray.data.read_mcap(samples=samples, **options) + + +def test_read_mcap_samples_needs_its_columns( + ray_start_regular_shared, simple_mcap_file, datasource_v2 +): + """The samples need their four columns, and ``topics`` takes specs only + with samples.""" + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import steps + + if not datasource_v2: + pytest.skip("validated on the V2 path") + topics = {"/cam": TopicSpec(steps.last(1))} + no_anchor = ray.data.from_items( + [{"files": ["x.mcap"], "clip_start": 0, "clip_end": 10}] + ) + + with pytest.raises(ValueError, match=r"lacks the columns \['anchor'\]"): + ray.data.read_mcap(samples=no_anchor, topics=topics) + with pytest.raises(ValueError, match="only in a sample read"): + ray.data.read_mcap(simple_mcap_file, topics=topics) + with pytest.raises(ValueError, match="needs the paths to read"): + ray.data.read_mcap(topics=["/cam"]) + + +def test_read_mcap_samples_refuses_a_row_id_checkpoint( + ray_start_regular_shared, restore_data_context, datasource_v2 +): + """Sample rows have no ``row_id``, so a checkpoint keyed on it is refused up + front rather than when the first row is written.""" + from ray.data.checkpoint import CheckpointConfig + from ray.data.checkpoint.interfaces import InvalidCheckpointingConfig + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import steps + + ray.data.DataContext.get_current().checkpoint_config = CheckpointConfig( + id_column="row_id", checkpoint_path="/tmp/unused" + ) + samples = ray.data.from_items(_sample_rows("x.mcap", [1_000_000_000])) + + with pytest.raises(InvalidCheckpointingConfig, match='id_column="sample_id"'): + ray.data.read_mcap(samples=samples, topics={"/cam": TopicSpec(steps.last(1))}) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/datasource/test_mcap_resume.py b/python/ray/data/tests/datasource/test_mcap_resume.py new file mode 100644 index 000000000000..466f7b9ccc90 --- /dev/null +++ b/python/ray/data/tests/datasource/test_mcap_resume.py @@ -0,0 +1,123 @@ +"""Resuming ``read_mcap(samples=...)`` from a checkpoint keyed on a sample column. + +A checkpoint filter needs actors of its own, so these tests run on a cluster +with more CPUs than ``test_mcap.py``'s. +""" + +import importlib.util +import os + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +import ray +from ray.data.tests.conftest import * # noqa +from ray.data.tests.unit.datasource_v2.mcap_testing import write_recording +from ray.tests.conftest import * # noqa + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +SECOND = 1_000_000_000 +ANCHORS = [1 * SECOND, 3 * SECOND // 2, 2 * SECOND, 5 * SECOND // 2] + + +def samples_of(path, anchors): + """One sample of ``path`` per anchor, clipped to [0.5 s, 3 s).""" + return [ + { + "sample_id": f"s{i}", + "files": [path], + "clip_start": SECOND // 2, + "clip_end": 3 * SECOND, + "anchor": anchor, + } + for i, anchor in enumerate(anchors) + ] + + +def write_twice(tmp_path, first, second, source): + """Write a sample read of ``first``, then of ``second``, to one sink with a + checkpoint on ``sample_id`` kept after each run, as a crashed run keeps it. + Returns the ``sample_id`` of every row in the sink. + + ``source`` is where each run's samples come from: ``"items"``, + ``"parquet"``, or ``"episodes"``, a Parquet listing fanned out into samples + by a ``flat_map`` as a labeling job builds them from a list of episodes. + The listing's rows have no ``sample_id``. + """ + from ray.data.checkpoint import CheckpointConfig + from ray.data.datasource import TopicSpec + from ray.data.datasource.mcap import steps + + out = str(tmp_path / "out") + for run, rows in enumerate((first, second)): + ray.data.DataContext.get_current().checkpoint_config = CheckpointConfig( + id_column="sample_id", + checkpoint_path=str(tmp_path / "ckpt"), + delete_checkpoint_on_success=False, + ) + listing = str(tmp_path / f"samples{run}.parquet") + if source == "episodes": + pq.write_table(pa.Table.from_pylist([{"samples": rows}]), listing) + samples = ray.data.read_parquet(listing).flat_map(lambda r: r["samples"]) + elif source == "parquet": + pq.write_table(pa.Table.from_pylist(rows), listing) + samples = ray.data.read_parquet(listing) + else: + samples = ray.data.from_items(rows) + ds = ray.data.read_mcap( + samples=samples, topics={"/imu": TopicSpec(steps.last(2))} + ) + # A map keeps the read's topic columns, so every sample's file is read. + ds = ds.map_batches(lambda b: {"sample_id": b["sample_id"]}) + ds.write_parquet(out) + return sorted(pq.read_table(out)["sample_id"].to_pylist()) + + +@pytest.fixture +def recording(tmp_path, restore_data_context): + """A five-second camera and IMU recording, read through the V2 datasource.""" + ray.data.DataContext.get_current().use_datasource_v2 = True + path = os.path.join(tmp_path, "run.mcap") + write_recording(path) + return path + + +@pytest.mark.parametrize("source", ["items", "parquet", "episodes"]) +def test_resume_skips_finished_samples_before_reading( + ray_start_10_cpus_shared, recording, tmp_path, source +): + """A rerun drops finished samples before their files are opened: in the + second run they name a file that does not exist, and the read still + succeeds. Each sample lands in the sink once.""" + rows = samples_of(recording, ANCHORS) + # The first run finishes half the samples. The second asks for all of + # them, with the finished half pointing at a missing file. + missing = [dict(r, files=[str(tmp_path / "missing.mcap")]) for r in rows[:2]] + + written = write_twice(tmp_path, rows[:2], missing + rows[2:], source) + + assert written == ["s0", "s1", "s2", "s3"] + + +@pytest.mark.parametrize("source", ["items", "parquet", "episodes"]) +def test_resume_with_every_sample_finished( + ray_start_10_cpus_shared, recording, tmp_path, source +): + """A rerun whose samples are all finished writes nothing, rather than + failing on the empty manifest blocks the checkpoint leaves.""" + rows = samples_of(recording, ANCHORS[:2]) + + written = write_twice(tmp_path, rows, rows, source) + + assert written == ["s0", "s1"] + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/unit/datasource_v2/conftest.py b/python/ray/data/tests/unit/datasource_v2/conftest.py index fe186961b4f0..4a088f0f0a6e 100644 --- a/python/ray/data/tests/unit/datasource_v2/conftest.py +++ b/python/ray/data/tests/unit/datasource_v2/conftest.py @@ -7,7 +7,9 @@ from ray.data.tests.unit.datasource_v2.mcap_testing import ( CHUNKED_FILE_MESSAGES, round_robin_messages, + summary_of, write_mcap, + write_recording, ) @@ -23,3 +25,21 @@ def chunked_file(tmp_path): path = os.path.join(tmp_path, "chunked.mcap") write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) return path + + +@pytest.fixture +def recording(tmp_path): + """Five seconds of a 10 fps camera and a 50 Hz IMU, in ~600-byte chunks. + + seconds 0.0 0.1 0.2 0.3 0.4 0.5 ... 4.9 + /cam f0 f1 f2 f3 f4 f5 ... f49 + /imu a sample every 20 ms, 0.00 to 4.98 s + + ``/cam`` frame ``k`` is logged at ``frame_time(k)``. More than 8 chunks let + nearby samples share some chunks and not others. + """ + path = os.path.join(tmp_path, "run.mcap") + write_recording(path) + summary = summary_of(path) + assert len(summary.chunk_indexes) > 8, len(summary.chunk_indexes) + return path diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 4d0f9a7bffcf..ce415193749d 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -21,10 +21,14 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import read_summary from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest +SECOND = 1_000_000_000 # one second in nanoseconds BASE_TIME = 1_000_000_000 # log time of the first message, in nanoseconds STEP = 1_000_000 # 1 ms between consecutive messages CHUNKED_FILE_MESSAGES = 9 # messages, one per chunk, in the ``chunked_file`` fixture @@ -398,3 +402,105 @@ def read_chunks(f, chunk_indexes, reads, on_corrupt=None): monkeypatch.setattr(mcap_chunks, "read_chunks", read_chunks) return offsets + + +def read_samples( + samples: List[Dict[str, Any]], + topics: Dict[str, Any], + *, + columns: Optional[List[str]] = None, + limit: Optional[int] = None, + filesystem=None, + **options, +) -> List[Dict[str, Any]]: + """Read ``samples`` as one read task of ``read_mcap(samples=...)`` does. + + Each sample is a dict with ``files``, ``clip_start``, ``clip_end`` and + ``anchor``, and any other columns. ``topics`` maps each topic to its + ``TopicSpec``. ``columns`` prunes the read and ``limit`` caps its rows, as + the optimizer's pushdowns do. ``options`` go to the datasource. + """ + from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import ( + sample_manifest, + ) + + table = pa.Table.from_pylist(samples) + datasource = MCAPDatasourceV2( + [], topics=topics, samples_schema=table.schema, filesystem=filesystem, **options + ) + scanner = datasource.create_scanner(datasource.infer_schema(None), filesystem) + if columns is not None: + scanner = scanner.prune_columns(columns) + if limit is not None: + scanner = scanner.push_limit(limit) + rows: List[Dict[str, Any]] = [] + for result in scanner.create_reader().read(FileManifest(sample_manifest(table))): + if columns != []: + assert result.schema.equals(scanner.read_schema()), result.schema + rows.extend(result.to_pylist()) + return rows + + +# -- the camera and IMU recording (the ``recording`` fixture) ----------------- + +CAM_FPS = 10 +IMU_HZ = 50 +DURATION_S = 5 + + +def frame_time(k: int) -> int: + """The log time of camera frame ``k``.""" + return k * SECOND // CAM_FPS + + +def write_recording(path, *, chunk_size=600, index=True): + """Five seconds of a 10 fps camera and a 50 Hz IMU, in small chunks. + + Camera frame ``k`` is logged at ``frame_time(k)`` and its payload ends with + ``k`` as four big-endian bytes. IMU sample ``i`` is ``i`` as eight bytes. + The ``recording`` fixture's docstring draws the timeline. + """ + from mcap.writer import CompressionType, IndexType, Writer + + with open(path, "wb") as stream: + writer = Writer( + stream, + chunk_size=chunk_size, + compression=CompressionType.ZSTD, + index_types=IndexType.ALL if index else IndexType.NONE, + use_chunking=index, + use_statistics=index, + use_summary_offsets=index, + ) + writer.start(profile="ros2", library="ray-test") + cam_schema = writer.register_schema( + name="test_msgs/msg/Frame", + encoding="ros2msg", + data=b"string format\nuint8[] data", + ) + imu_schema = writer.register_schema( + name="sensor_msgs/msg/Imu", encoding="ros2msg", data=b"float64 x" + ) + cam = writer.register_channel( + schema_id=cam_schema, topic="/cam", message_encoding="cdr" + ) + imu = writer.register_channel( + schema_id=imu_schema, topic="/imu", message_encoding="cdr" + ) + messages = [] + for frame in range(CAM_FPS * DURATION_S): + messages.append( + (frame_time(frame), cam, b"frame" + frame.to_bytes(4, "big")) + ) + for i in range(IMU_HZ * DURATION_S): + messages.append((i * SECOND // IMU_HZ, imu, i.to_bytes(8, "big"))) + messages.sort() + for log_time, channel_id, payload in messages: + writer.add_message( + channel_id=channel_id, + log_time=log_time, + publish_time=log_time, + data=payload, + sequence=0, + ) + writer.finish() diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_sample_rows.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_sample_rows.py new file mode 100644 index 000000000000..48169d97ac53 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_sample_rows.py @@ -0,0 +1,354 @@ +"""Unit tests for sample rows: reading each topic's steps around a sample's anchor. + +The tests drive the scanner and reader as a read task of +``read_mcap(samples=...)`` does, with no Ray cluster. Most read the +``recording`` fixture: a 10 fps camera whose payloads end with their frame +number, and a 50 Hz IMU whose payloads are their sample number. +""" + +import importlib.util +import json +import os + +import pytest + +from ray.data._internal.datasource_v2.formats.mcap.mcap_datasource_v2 import ( + MCAPDatasourceV2, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec +from ray.data.datasource.mcap import steps +from ray.data.tests.unit.datasource_v2.mcap_testing import ( + SECOND, + frame_time, + pack, + read_all, + read_samples, + record_parsed_chunks, + summary_of, + write_mcap, + write_recording, +) + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("mcap") is None, + reason="mcap module not available. Install with: pip install mcap", +) + +TENTH = SECOND // 10 +CLIP_START, CLIP_END = 1 * SECOND, 4 * SECOND +IMU_NS = SECOND // 50 # the IMU's sample period + + +def sample(path, anchor, sample_id="s", **columns): + """A sample of ``path`` with the default clip, anchored at ``anchor``.""" + return { + "sample_id": sample_id, + "files": [path], + "clip_start": CLIP_START, + "clip_end": CLIP_END, + "anchor": anchor, + **columns, + } + + +def frames(row, topic="/cam"): + """The camera frame each step holds, ``None`` for an empty step.""" + return [int.from_bytes(d[-4:], "big") if d else None for d in row[f"data:{topic}"]] + + +def imu_samples(row): + """The IMU sample each step holds, ``None`` for an empty step.""" + return [int.from_bytes(d, "big") if d else None for d in row["data:/imu"]] + + +def json_values(row, topic): + """The ``t`` value of each step's JSON payload, ``None`` for an empty step.""" + return [json.loads(d)["t"] if d else None for d in row[f"data:{topic}"]] + + +@pytest.fixture +def gap_file(tmp_path): + """Two JSON topics, one message per chunk, ``/a`` with a gap:: + + seconds 1.0 1.1 1.2 1.3 ... 2.9 3.0 + /a a a a a + /b b b b b ... b b + + Each message's ``t`` is its log time in tenths of a second. + """ + path = os.path.join(tmp_path, "gap.mcap") + messages = [] + for k in range(10, 31): + if k <= 12 or k == 30: + messages.append({"topic": "/a", "log_time": k * TENTH, "data": {"t": k}}) + messages.append({"topic": "/b", "log_time": k * TENTH, "data": {"t": k}}) + write_mcap(path, messages) + return path + + +def chunks_of(path, topic): + """The offsets of the chunks that hold ``topic``, one message per chunk.""" + summary = summary_of(path) + ids = {cid for cid, c in summary.channels.items() if c.topic == topic} + return { + c.chunk_start_offset + for c in summary.chunk_indexes + if not ids.isdisjoint(c.message_index_offsets) + } + + +def test_rows_hold_each_topics_steps(recording): + """Each row keeps its sample's columns, and per topic the payload, log time and + validity of every step: the camera's frames at 10 fps up to the anchor, and + the IMU's next two samples after it.""" + topics = { + "/cam": TopicSpec(steps.before(0.3, fps=10)), + "/imu": TopicSpec(steps.next(2)), + } + anchors = [frame_time(15), frame_time(25) + 5_000_000] + + rows = read_samples( + [sample(recording, a, f"s{i}", label=i) for i, a in enumerate(anchors)], topics + ) + + assert [(row["sample_id"], row["label"]) for row in rows] == [("s0", 0), ("s1", 1)] + assert [frames(row) for row in rows] == [[13, 14, 15], [23, 24, 25]] + assert rows[0]["times:/cam"] == [frame_time(k) for k in (13, 14, 15)] + assert rows[0]["valid:/cam"] == [True, True, True] + # The IMU logs every 20 ms, so the samples after 1.5 s are 76 and 77. + assert [imu_samples(row) for row in rows] == [[76, 77], [126, 127]] + assert rows[1]["times:/imu"] == [126 * IMU_NS, 127 * IMU_NS] + + +def test_files_in_manifest_order_samples_in_anchor_order(tmp_path): + """A task reads its files in the order the manifest first names them, and + each file's samples in anchor order.""" + a, b = (os.path.join(tmp_path, f"{name}.mcap") for name in "ab") + write_recording(a) + write_recording(b) + samples = [ + sample(b, 3 * SECOND, "b3"), + sample(a, 2 * SECOND, "a2"), + sample(b, frame_time(15), "b1.5"), + ] + + rows = read_samples(samples, {"/cam": TopicSpec(steps.before(frames=1, fps=10))}) + + assert [row["sample_id"] for row in rows] == ["b1.5", "b3", "a2"] + + +@pytest.mark.parametrize( + "pad, expected, valid", + [ + # Steps at 0.7, 0.8 and 0.9 s lie before the clip, which starts at 1.0 s. + ("edge", [10, 10, 10, 10, 11], [False, False, False, True, True]), + ("zeros", [None, None, None, 10, 11], [False, False, False, True, True]), + ], +) +def test_steps_before_the_clip_are_padded(recording, pad, expected, valid): + """Steps past the clip's edge are padded: ``edge`` repeats the first real + frame, and ``zeros`` leaves empty payloads at the steps' own times.""" + topics = {"/cam": TopicSpec(steps.before(0.5, fps=10), pad=pad)} + + (row,) = read_samples([sample(recording, frame_time(11))], topics) + + assert frames(row) == expected + assert row["valid:/cam"] == valid + if pad == "zeros": + assert row["times:/cam"][:3] == [frame_time(k) for k in (7, 8, 9)] + + +def test_raise_names_the_sample_and_its_file(recording): + """``pad="raise"`` fails the read on a sample whose window reaches past its + clip, naming the sample and its file.""" + topics = {"/cam": TopicSpec(steps.before(0.5, fps=10), pad="raise")} + + with pytest.raises(ValueError, match=r"Sample 'early' of .*run\.mcap.*3 of 5"): + read_samples([sample(recording, frame_time(11), "early")], topics) + + +def test_nearby_samples_share_their_chunks(recording, monkeypatch): + """Samples whose windows meet are read in one pass: no chunk is parsed + twice, and none outside their windows is parsed.""" + parsed = record_parsed_chunks(monkeypatch) + anchors = [frame_time(k) for k in range(15, 26)] + topics = { + "/cam": TopicSpec(steps.before(0.5, fps=10)), + "/imu": TopicSpec(steps.every(before_s=0.2)), + } + + rows = read_samples([sample(recording, a) for a in anchors], topics) + + assert [frames(row)[-1] for row in rows] == list(range(15, 26)) + assert len(parsed) == len(set(parsed)) + # The first window starts 0.4 s before the first anchor, the last ends at + # the last anchor. + low, high = anchors[0] - 4 * TENTH, anchors[-1] + near = { + c.chunk_start_offset + for c in summary_of(recording).chunk_indexes + if c.message_end_time >= low and c.message_start_time <= high + } + assert set(parsed) <= near + + +def test_a_step_in_a_gap_reads_back_to_the_last_message(gap_file, monkeypatch): + """A step in a gap takes the topic's last message before the gap, read + back chunk by chunk through the chunks that hold the topic alone.""" + parsed = record_parsed_chunks(monkeypatch) + + (row,) = read_samples( + [sample(gap_file, 25 * TENTH)], + {"/a": TopicSpec(steps.before(frames=2, fps=10))}, + ) + + assert json_values(row, "/a") == [12, 12] + assert row["valid:/a"] == [True, True] + # Only the chunks of /a before the gap are read, not those of /b. + assert set(parsed) <= chunks_of(gap_file, "/a") + + +def test_frame_count_without_fps_reads_as_far_as_it_needs(recording): + """``before_frames`` without ``fps`` takes that many recorded frames up to + the anchor, however many chunks back they lie.""" + (row,) = read_samples( + [sample(recording, frame_time(25))], {"/cam": TopicSpec(steps.last(12))} + ) + + assert frames(row) == list(range(14, 26)) + + +def test_file_without_a_chunk_index(tmp_path): + """A file without a chunk index is read whole and gives the same rows.""" + indexed, unindexed = (os.path.join(tmp_path, f"{n}.mcap") for n in ("i", "u")) + write_recording(indexed) + write_recording(unindexed, index=False) + topics = { + "/cam": TopicSpec(steps.before(0.3, fps=10)), + "/imu": TopicSpec(steps.every(after_s=0.1)), + } + + rows = { + path: read_samples([sample(path, frame_time(20) + 1)], topics)[0] + for path in (indexed, unindexed) + } + + for column in ("data:/cam", "times:/cam", "data:/imu", "valid:/imu"): + assert rows[indexed][column] == rows[unindexed][column] + + +def test_topic_missing_from_the_file(recording): + """A topic the file lacks leaves every step empty and invalid, at the clip's + edge for a frame count without ``fps``.""" + (row,) = read_samples( + [sample(recording, 2 * SECOND)], {"/nope": TopicSpec(steps.next(2))} + ) + + assert row["data:/nope"] == [b"", b""] + assert row["valid:/nope"] == [False, False] + assert row["times:/nope"] == [CLIP_END - 1, CLIP_END - 1] + + +def test_projection_reads_only_the_wanted_topics(gap_file, monkeypatch): + """A projection that keeps one topic's columns reads only that topic's + chunks and builds only its kept columns.""" + parsed = record_parsed_chunks(monkeypatch) + topics = { + "/a": TopicSpec(steps.before(frames=1, fps=10)), + "/b": TopicSpec(steps.before(frames=1, fps=10)), + } + + rows = read_samples( + [sample(gap_file, 11 * TENTH)], topics, columns=["valid:/b", "sample_id"] + ) + + assert rows == [{"valid:/b": [True], "sample_id": "s"}] + assert set(parsed) <= chunks_of(gap_file, "/b") + + +def test_projection_to_the_samples_columns_reads_no_file(tmp_path): + """A projection that keeps only the samples' own columns, as ``count()`` + does, needs no message: the files are never opened.""" + missing = os.path.join(tmp_path, "never-written.mcap") + samples = [sample(missing, 2 * SECOND, f"s{i}") for i in range(3)] + topics = {"/cam": TopicSpec(steps.last(1))} + + rows = read_samples(samples, topics, columns=["sample_id"]) + counted = read_samples(samples, topics, columns=[]) + + assert [row["sample_id"] for row in rows] == ["s0", "s1", "s2"] + assert len(counted) == 3 + + +def test_limit_stops_after_its_rows(recording): + """A pushed-down limit stops the task after that many rows.""" + samples = [sample(recording, frame_time(k)) for k in range(12, 20)] + + rows = read_samples(samples, {"/cam": TopicSpec(steps.last(1))}, limit=3) + + assert [frames(row) for row in rows] == [[12], [13], [14]] + + +def test_row_over_the_row_limit_fails(recording, monkeypatch): + """A row whose payloads pass ``RAY_DATA_MCAP_MAX_ROW_BYTES`` fails the read.""" + monkeypatch.setenv("RAY_DATA_MCAP_MAX_ROW_BYTES", "100") + + with pytest.raises(ValueError, match="RAY_DATA_MCAP_MAX_ROW_BYTES"): + read_samples( + [sample(recording, 3 * SECOND)], + {"/imu": TopicSpec(steps.every(before_s=1.0))}, + ) + + +def test_file_uris_resolve_in_the_task(recording): + """A sample may name its file by URI: the task resolves its filesystem.""" + (row,) = read_samples( + [sample(f"file://{recording}", frame_time(20))], + {"/cam": TopicSpec(steps.before(frames=2, fps=10))}, + ) + + assert frames(row) == [19, 20] + + +@pytest.mark.parametrize("window_ns", [SECOND // 2, 3 * SECOND // 10]) +def test_tiled_windows_hold_the_message_rows(recording, window_ns): + """Samples that tile the recording into back-to-back windows, each taking + every message of its window, hold exactly the message rows of the same file + grouped by window, in log-time order. + + This is the samples table a fixed-window read amounts to. + """ + topics = ["/cam", "/imu"] + statistics = summary_of(recording).statistics + assert statistics is not None + start, end = statistics.message_start_time, statistics.message_end_time + 1 + # The anchor sits on a window's last nanosecond, so ``steps.every`` over the + # window's length covers it exactly. + windows = [(t, min(t + window_ns, end)) for t in range(start, end, window_ns)] + samples = [ + {"files": [recording], "clip_start": a, "clip_end": b, "anchor": b - 1} + for a, b in windows + ] + datasource = MCAPDatasourceV2([recording], topics=topics, include_metadata=False) + + rows = read_samples( + samples, + {t: TopicSpec(steps.every(before_s=window_ns / SECOND)) for t in topics}, + ) + messages = read_all(datasource, *pack(datasource)).to_pylist() + + assert len(rows) == len(windows) + for (a, b), row in zip(windows, sorted(rows, key=lambda r: r["clip_start"])): + for t in topics: + expected = [ + m for m in messages if m["topic"] == t and a <= m["log_time"] < b + ] + assert row[f"times:{t}"] == [m["log_time"] for m in expected] + assert row[f"data:{t}"] == [m["data"] for m in expected] + assert all(row[f"valid:{t}"]) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_samples.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_samples.py new file mode 100644 index 000000000000..af88396ee159 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_samples.py @@ -0,0 +1,398 @@ +"""Unit tests for the step rules of ``read_mcap(samples=...)`` (``mcap_samples``). + +The rules need no recording: each test resolves a window over message times +held in a list, a 10 Hz topic logged at every tenth of a second. +""" + +from fractions import Fraction + +import pyarrow as pa +import pytest + +from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import ( + SampleTimes, + SortedTimes, + TopicPlan, + check_samples_schema, + resolve_steps, + sample_manifest, + sample_row_schema, +) +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec +from ray.data.datasource.mcap import steps + +SECOND = 1_000_000_000 +TENTH = SECOND // 10 # the topic's message period +# Clip [1.0 s, 4.0 s); the default anchor lies between two messages. +CLIP_START, CLIP_END = 1 * SECOND, 4 * SECOND +ANCHOR = 2 * SECOND + 5 * SECOND // 100 # 2.05 s +WHERE = "sample 's0'" + + +def tenths(*ks): + """The log times of the messages logged ``k`` tenths of a second in.""" + return [k * TENTH for k in ks] + + +def topic(times=None): + """A ``TopicTimes`` whose messages are their own log times, 10 Hz from 0 s to + 4.9 s unless ``times`` lists others.""" + if times is None: + times = tenths(*range(50)) + return SortedTimes([(t, t) for t in times]) + + +def resolve(spec, *, anchor=ANCHOR, times=None, clip=(CLIP_START, CLIP_END)): + """The steps of ``spec`` for one sample, as ``(time, valid)`` pairs and the + messages that fill them.""" + plan = TopicPlan.from_spec("/t", spec) + resolved = resolve_steps( + plan, topic(times), SampleTimes(clip[0], clip[1], anchor), WHERE + ) + return [(s.time, s.valid) for s in resolved], [s.message for s in resolved] + + +# -- TopicPlan ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "spec, before, num_steps, span_ns", + [ + (TopicSpec(steps.before(0.5, fps=10)), True, 5, None), + (TopicSpec(steps.after(1, fps=Fraction(5))), False, 5, None), + (TopicSpec(steps.before(frames=3, fps=10)), True, 3, None), + (TopicSpec(steps.next(4)), False, 4, None), + (TopicSpec(steps.every(before_s=0.25)), True, None, SECOND // 4), + ], +) +def test_topic_plan_from_spec(spec, before, num_steps, span_ns): + """Time steps and ``steps.last``/``steps.next`` have a fixed number of + steps; ``steps.every`` is a span whose number of steps varies.""" + plan = TopicPlan.from_spec("/t", spec) + + assert (plan.before, plan.num_steps, plan.span_ns) == (before, num_steps, span_ns) + + +def test_step_times_count_from_the_anchor(): + """A before-window's last step is the anchor, and an after-window's first + step is one step after it.""" + before = TopicPlan.from_spec("/t", TopicSpec(steps.before(0.3, fps=10))) + after = TopicPlan.from_spec("/t", TopicSpec(steps.after(0.2, fps=10))) + + assert before.step_times(ANCHOR) == [ANCHOR - 2 * TENTH, ANCHOR - TENTH, ANCHOR] + assert after.step_times(ANCHOR) == [ANCHOR + TENTH, ANCHOR + 2 * TENTH] + + +def test_steps_on_a_rounded_clock_take_their_own_frame(): + """Steps that fall on the frames of a 30 fps clock, which rounds each frame + to the nanosecond, take those frames and not their neighbours.""" + frames = [round(k * Fraction(SECOND, 30)) for k in range(120)] + anchor = frames[90] + + resolved, messages = resolve( + TopicSpec(steps.before(frames=30, fps=30)), anchor=anchor, times=frames + ) + + assert messages == frames[61:91] + assert all(valid for _, valid in resolved) + + +def test_steps_take_evenly_spaced_frames_despite_jitter(): + """With the anchor on a frame of a jittery 30 fps camera, 10 steps a + second take every third frame back from it: each step lands within the + jitter of its frame, and the nearest message is that frame.""" + # Frames 1 ms early or late in turn, as a recorder's log times are. From + # an early frame, every other step lands 2 ms before a late frame. + frames = [k * SECOND // 30 + (1 if k % 2 else -1) * 1_000_000 for k in range(120)] + + _, messages = resolve( + TopicSpec(steps.before(frames=5, fps=10)), anchor=frames[90], times=frames + ) + + assert messages == [frames[k] for k in (78, 81, 84, 87, 90)] + + +# -- windows with fps --------------------------------------------------------- + + +def test_steps_take_the_nearest_message_up_to_the_anchor(): + """Each step of a before-window takes the message logged nearest to it, + the earlier one on a tie, but never one logged after the anchor.""" + anchor = 20 * TENTH + 8 * SECOND // 100 # 2.08 s + + _, ties = resolve(TopicSpec(steps.before(0.3, fps=10))) + resolved, messages = resolve(TopicSpec(steps.before(0.3, fps=10)), anchor=anchor) + + # Steps at 1.85, 1.95 and 2.05 s lie halfway between messages. + assert ties == tenths(18, 19, 20) + # Steps at 1.88 and 1.98 s take the later message; the anchor's step takes + # 2.0 s, since 2.1 s lies past the anchor. + assert messages == tenths(19, 20, 20) + assert [valid for _, valid in resolved] == [True] * 3 + + +def test_after_window_takes_only_messages_after_the_anchor(): + """An after-window's steps take messages logged after the anchor: with the + anchor on a message, no step takes the anchor's message, however near.""" + anchor = 20 * TENTH + + _, messages = resolve(TopicSpec(steps.after(0.2, fps=10)), anchor=anchor) + _, far = resolve( + TopicSpec(steps.after(frames=2, fps=10)), + anchor=anchor, + times=tenths(15, 20, 35), + ) + _, none = resolve( + TopicSpec(steps.after(frames=2, fps=10), pad="zeros"), + anchor=anchor, + times=tenths(15, 20), + ) + + assert messages == tenths(21, 22) + # The only message after the anchor, 1.5 s later, is the nearest one. + assert far == tenths(35, 35) + assert none == [None, None] + + +def test_gap_takes_the_nearest_message_on_its_side_of_the_anchor(): + """A step inside a gap in the recording takes the nearest message on its + side of the anchor, however far, and its time shows how far.""" + times = tenths(10, 11, 12, 30, 31) + + resolved, messages = resolve(TopicSpec(steps.before(1.0, fps=5)), times=times) + + # Steps at 1.25, 1.45, 1.65, 1.85 and 2.05 s all fall in the gap, and the + # messages after it lie past the anchor. + assert messages == tenths(12, 12, 12, 12, 12) + assert [valid for _, valid in resolved] == [True] * 5 + + +@pytest.mark.parametrize( + "pad, messages, valid", + [ + # Steps at 0.85 and 0.95 s lie before the clip; 1.05 s onwards inside. + ("edge", tenths(10, 10, 10, 11, 12), [False, False, True, True, True]), + ("zeros", [None, None] + tenths(10, 11, 12), [False, False, True, True, True]), + ], +) +def test_steps_before_the_clip_are_padded(pad, messages, valid): + """Steps before the clip's start are padded: ``edge`` repeats the first + real step, and ``zeros`` leaves them empty at their own times.""" + anchor = 12 * TENTH + 5 * SECOND // 100 # 1.25 s + + resolved, got = resolve( + TopicSpec(steps.before(0.5, fps=10), pad=pad), anchor=anchor + ) + + assert got == messages + assert [v for _, v in resolved] == valid + if pad == "zeros": + # An empty step keeps its own time. + assert [t for t, _ in resolved[:2]] == [anchor - 4 * TENTH, anchor - 3 * TENTH] + + +def test_edge_repeats_the_nearest_real_step(): + """``edge`` fills a step outside the clip from the nearest step that took a + message, and the steps inside take the clip's one message.""" + # The clip starts at 1.05 s, after the message at 1.0 s, so its only + # message up to the anchor is the one at 1.4 s. + times = tenths(10, 14, 30) + clip = (10 * TENTH + 5 * SECOND // 100, CLIP_END) + + resolved, messages = resolve( + TopicSpec(steps.before(frames=6, fps=10)), + times=times, + clip=clip, + anchor=15 * TENTH, + ) + + # Steps at 1.0 s (outside the clip), 1.1, 1.2, 1.3, 1.4 and 1.5 s. + assert messages == tenths(14, 14, 14, 14, 14, 14) + assert [valid for _, valid in resolved] == [False] + [True] * 5 + + +def test_edge_without_real_steps_repeats_the_clips_edge_message(): + """A window where no step took a message repeats the clip's first message + for a before-window and its last for an after-window.""" + times = tenths(25, 26, 30) + + before_steps, before = resolve( + TopicSpec(steps.before(frames=2, fps=10)), times=times + ) + _, after = resolve( + TopicSpec(steps.after(frames=2, fps=10)), times=times, anchor=35 * TENTH + ) + _, none = resolve(TopicSpec(steps.before(frames=2, fps=10)), times=tenths(45)) + + assert before == tenths(25, 25) + assert [v for _, v in before_steps] == [False, False] + assert after == tenths(30, 30) + # No message on the topic inside the clip: the steps stay empty. + assert none == [None, None] + + +def test_raise_names_the_sample_and_the_overreach(): + """``raise`` fails on the first sample with an empty step, saying how far + past the clip the window reaches.""" + anchor = 11 * TENTH + + with pytest.raises(ValueError, match=r"sample 's0': 3 of 5 steps .* 0.3 s before"): + resolve(TopicSpec(steps.before(0.5, fps=10), pad="raise"), anchor=anchor) + with pytest.raises(ValueError, match="0.1 s past the clip's end"): + resolve( + TopicSpec(steps.after(frames=3, fps=10), pad="raise"), anchor=38 * TENTH + ) + with pytest.raises(ValueError, match="No message on the topic .* up to the anchor"): + # The clip's only message, at 2.5 s, lies past the anchor. + resolve( + TopicSpec(steps.before(frames=2, fps=10), pad="raise"), times=tenths(25) + ) + + +# -- frame counts and spans without fps --------------------------------------- + + +def test_frame_count_without_fps_takes_recorded_messages(): + """``before_frames`` without ``fps`` takes the messages logged up to the + anchor and ``after_frames`` those after it, whatever their spacing.""" + times = tenths(10, 13, 19, 20, 21, 27, 39) + + _, before = resolve(TopicSpec(steps.last(3)), times=times) + _, after = resolve(TopicSpec(steps.next(2)), times=times) + + assert before == tenths(13, 19, 20) + assert after == tenths(21, 27) + + +def test_frame_count_short_of_messages_pads_the_far_end(): + """A count the clip cannot fill pads its far end, at the clip's edge.""" + times = tenths(5, 12, 20, 38, 41) + + before_steps, before = resolve(TopicSpec(steps.last(4)), times=times) + after_steps, after = resolve(TopicSpec(steps.next(3), pad="zeros"), times=times) + + # The message at 0.5 s lies before the clip, so two steps are padded. + assert before == tenths(12, 12, 12, 20) + assert [v for _, v in before_steps] == [False, False, True, True] + assert after == tenths(38) + [None, None] + assert after_steps[1:] == [(CLIP_END - 1, False)] * 2 + + +def test_span_without_fps_takes_every_message_and_cuts_at_the_clip(): + """``before_s`` without ``fps`` holds every message in ``(anchor - span, + anchor]``. It is cut at the clip's edge, or fails with ``raise``.""" + _, inside = resolve(TopicSpec(steps.every(before_s=0.3))) + _, cut = resolve(TopicSpec(steps.every(before_s=2.0))) + + assert inside == tenths(18, 19, 20) + assert cut == tenths(*range(10, 21)) + with pytest.raises(ValueError, match="0.95 s before the clip"): + resolve(TopicSpec(steps.every(before_s=2.0), pad="raise")) + + +# -- the samples dataset and the row layout ----------------------------------- + +SAMPLES_SCHEMA = pa.schema( + [ + ("sample_id", pa.string()), + ("files", pa.list_(pa.string())), + ("clip_start", pa.int64()), + ("clip_end", pa.int64()), + ("anchor", pa.int64()), + ("instruction", pa.string()), + ] +) + + +@pytest.mark.parametrize( + "change, error", + [ + (lambda s: s.remove(s.get_field_index("anchor")), "lacks the columns"), + (lambda s: s.set(1, pa.field("files", pa.string())), "list of paths"), + (lambda s: s.set(2, pa.field("clip_start", pa.float64())), "integer"), + (lambda s: s.append(pa.field("valid:/cam", pa.bool_())), "the read adds"), + (lambda s: s.append(pa.field("__path", pa.string())), "the read adds"), + ], +) +def test_check_samples_schema(change, error): + """A samples dataset needs its four columns with their types, and no column + the read adds.""" + check_samples_schema(SAMPLES_SCHEMA, ["/cam"]) + + with pytest.raises(ValueError, match=error): + check_samples_schema(change(SAMPLES_SCHEMA), ["/cam"]) + + +def samples_batch(**changes): + """Two valid samples as a batch of the samples dataset, with ``changes`` + replacing whole columns.""" + columns = { + "sample_id": ["a", "b"], + "files": [["x.mcap"], ["y.mcap"]], + "clip_start": [0, 10], + "clip_end": [100, 20], + "anchor": [5, 15], + } + columns.update(changes) + return pa.table(columns) + + +def test_sample_manifest_names_each_samples_file(): + """Each sample becomes a manifest row: its file as the path, no size and no + chunks, with its own columns kept.""" + manifest = sample_manifest(samples_batch()) + + assert manifest.column("__path").to_pylist() == ["x.mcap", "y.mcap"] + assert manifest.column("__file_size").null_count == 2 + assert manifest.column("__file_chunk_metadata").null_count == 2 + assert manifest.column("sample_id").to_pylist() == ["a", "b"] + + +@pytest.mark.parametrize( + "changes, error, message", + [ + ({"files": [["x.mcap"], ["y.mcap", "z.mcap"]]}, NotImplementedError, "2 files"), + ({"files": [["x.mcap"], []]}, ValueError, "Sample 'b' lists no file"), + ( + {"anchor": [5, 20]}, + ValueError, + r"Sample 'b' has clip \[10, 20\) and anchor 20", + ), + ({"clip_end": [0, 20]}, ValueError, "Sample 'a' has clip"), + ({"clip_start": [0, None]}, ValueError, "Sample 'b' has clip"), + ], +) +def test_sample_manifest_rejects_bad_samples(changes, error, message): + """A sample must list exactly one file, and its anchor must lie inside a + clip that is not empty. The error names the first bad sample.""" + with pytest.raises(error, match=message): + sample_manifest(samples_batch(**changes)) + + +def test_sample_row_schema(): + """A row keeps the samples' columns, then each topic's in the order given: + the payloads, log times and validity of its steps.""" + plans = [ + TopicPlan.from_spec("/cam", TopicSpec(steps.before(frames=4, fps=10))), + TopicPlan.from_spec("/joints", TopicSpec(steps.every(before_s=0.5))), + ] + + schema = sample_row_schema(SAMPLES_SCHEMA, plans) + + assert schema.names == SAMPLES_SCHEMA.names + [ + "data:/cam", + "times:/cam", + "valid:/cam", + "data:/joints", + "times:/joints", + "valid:/joints", + ] + assert schema.field("data:/joints").type == pa.list_(pa.large_binary()) + assert schema.field("times:/cam").type == pa.list_(pa.int64()) + assert schema.field("valid:/joints").type == pa.list_(pa.bool_()) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_topic_spec.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_topic_spec.py new file mode 100644 index 000000000000..2c5a8d27d832 --- /dev/null +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_topic_spec.py @@ -0,0 +1,117 @@ +"""Unit tests for ``TopicSpec`` and its step and match built-ins (``mcap_topic_spec``).""" + +from fractions import Fraction +from typing import Any + +import pytest + +from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import ( + MessageSteps, + TimeSteps, + TopicSpec, +) +from ray.data.datasource.mcap import match, steps + +SECOND = 1_000_000_000 +# Values of the wrong type, which the built-ins and TopicSpec must refuse. +NOT_A_COUNT: Any = 2.0 +NOT_STEPS: Any = "before" +NOT_A_MATCH: Any = "nearest" +UNKNOWN_PAD: Any = "repeat" + + +@pytest.mark.parametrize( + "build, expected", + [ + (lambda: steps.before(1, fps=10), TimeSteps("before", 10, Fraction(10))), + (lambda: steps.after(2, fps=2), TimeSteps("after", 4, Fraction(2))), + ( + lambda: steps.before(frames=3, fps=Fraction(30000, 1001)), + TimeSteps("before", 3, Fraction(30000, 1001)), + ), + # A float fps is read as the decimal it prints as. + ( + lambda: steps.after(frames=2, fps=29.97), + TimeSteps("after", 2, Fraction(2997, 100)), + ), + (lambda: steps.last(4), MessageSteps("before", count=4)), + (lambda: steps.next(2), MessageSteps("after", count=2)), + ( + lambda: steps.every(before_s=0.25), + MessageSteps("before", span_ns=SECOND // 4), + ), + (lambda: steps.every(after_s=1), MessageSteps("after", span_ns=SECOND)), + ], +) +def test_steps_built_ins(build, expected): + """Each built-in gives the side, count or span and rate it names.""" + assert build() == expected + + +@pytest.mark.parametrize( + "build, error", + [ + (lambda: steps.before(fps=10), "exactly one of them"), + (lambda: steps.before(1, frames=2, fps=10), "exactly one of them"), + (lambda: steps.before(frames=0, fps=10), "positive integer"), + ( + lambda: steps.after(frames=NOT_A_COUNT, fps=10), + "positive integer", + ), + (lambda: steps.before(frames=True, fps=10), "positive integer"), + (lambda: steps.before(-0.5, fps=10), "positive"), + (lambda: steps.after(float("nan"), fps=10), "finite"), + (lambda: steps.before(0.5, fps=0), "positive"), + (lambda: steps.before(0.5, fps=15), "7.5 steps"), + (lambda: steps.last(0), "positive integer"), + (lambda: steps.every(), "exactly one of them"), + (lambda: steps.every(before_s=1, after_s=1), "exactly one of them"), + (lambda: steps.every(before_s=1e-12), "at least one nanosecond"), + ], +) +def test_steps_built_ins_reject(build, error): + """A built-in needs one positive horizon, and a whole number of steps when + the horizon is in seconds.""" + with pytest.raises(ValueError, match=error): + build() + + +def test_topic_spec_defaults(): + """A spec matches time steps to the nearest message and pads at the edge + unless told otherwise.""" + spec = TopicSpec(steps.before(1, fps=5)) + + assert (spec.match, spec.pad) == (None, "edge") + assert TopicSpec(steps.before(1, fps=5), match.nearest()).match == match.nearest() + + +@pytest.mark.parametrize( + "build, error", + [ + ( + lambda: TopicSpec(NOT_STEPS), + "needs steps", + ), + ( + lambda: TopicSpec(steps.before(1, fps=5), NOT_A_MATCH), + "needs match", + ), + (lambda: TopicSpec(steps.last(2), match.nearest()), "take no match"), + (lambda: TopicSpec(steps.every(before_s=1), match.nearest()), "take no match"), + ( + lambda: TopicSpec(steps.last(2), pad=UNKNOWN_PAD), + "pad must be one of", + ), + ], +) +def test_topic_spec_rejects(build, error): + """A spec takes built-in steps and match only, no match for message steps, + and a known pad mode.""" + with pytest.raises(ValueError, match=error): + build() + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__]))