Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 96 additions & 11 deletions python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ def iter_chunks(
for c in self.candidate_chunks(summary, selected)
if offsets is None or c.chunk_start_offset in offsets
]
time_bounds = (self._selection.start_time, self._selection.end_time)
declared = _Declared(
dict(summary.channels),
dict(summary.schemas),
Expand All @@ -153,17 +154,64 @@ def iter_chunks(
chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt)
for chunk_index, chunk in chunks:
yield from self._decoded_messages(
f, path, summary, chunk_index, chunk, selected, declared, on_corrupt
f,
path,
summary,
chunk_index,
chunk,
selected,
declared,
time_bounds,
on_corrupt,
)
return
yield from self._merge_chunks_by_log_time(
f, path, summary, chunk_indexes, selected, declared, on_corrupt
f, path, summary, chunk_indexes, selected, declared, time_bounds, on_corrupt
)

def iter_chunk_messages(
self,
f: Any,
path: str,
summary: "Summary",
chunk_indexes: List["ChunkIndex"],
file_summary: Optional[Callable[[], "Summary"]] = None,
*,
selected: Optional[Set[int]] = None,
) -> Iterator[Tuple["ChunkIndex", List[_Selected]]]:
"""Yield each chunk with its selected messages, in file order.

Unlike ``iter_chunks``, a chunk's messages come together and are not
cut to the time range, so a caller can keep or drop whole chunks.
Chunks close together in the file still share a ranged read. A chunk
that cannot be decoded raises.
"""
if selected is None:
selected = self._selection.selected_channel_ids(
summary.channels, summary.schemas
)
declared = _Declared(
dict(summary.channels),
dict(summary.schemas),
set(),
file_summary or (lambda: summary),
)
ordered = sorted(chunk_indexes, key=lambda c: c.chunk_start_offset)
for chunk_index, chunk in read_chunks(f, ordered, self._chunk_reads):
assert chunk is not None, "without on_corrupt, a bad chunk raises"
messages = self._chunk_messages(
f, path, summary, chunk_index, chunk, selected, declared, (None, None)
)
yield chunk_index, list(messages)

def candidate_chunks(
self, summary: "Summary", selected: Set[int]
) -> List["ChunkIndex"]:
"""The file's chunks that may hold a selected message, in file order."""
"""The file's chunks that may hold a selected message, in file order.

The indexer lists exactly these, so the indexer and the reader agree
on which chunks a selection reads.
"""
# A summary that repeats no channel record says nothing about which
# chunk holds a selected message. Every chunk in the time range is read,
# and the channels declared inside them are filtered as they appear.
Expand Down Expand Up @@ -191,6 +239,7 @@ def _merge_chunks_by_log_time(
chunk_indexes: List["ChunkIndex"],
selected: Set[int],
declared: _Declared,
time_bounds: Tuple[Optional[int], Optional[int]],
on_corrupt: Optional[OnCorrupt],
) -> Iterator[_Selected]:
"""Yield the selected messages of the chunks in log-time order.
Expand Down Expand Up @@ -225,6 +274,7 @@ def _merge_chunks_by_log_time(
chunk,
selected,
declared,
time_bounds,
on_corrupt,
with_index=True,
):
Expand All @@ -250,6 +300,7 @@ def _decoded_messages(
chunk: Optional["Chunk"],
selected: Set[int],
declared: "_Declared",
time_bounds: Tuple[Optional[int], Optional[int]],
on_corrupt: Optional[OnCorrupt],
with_index: bool = False,
) -> Iterator[Any]:
Expand All @@ -265,7 +316,15 @@ def _decoded_messages(
return
try:
yield from self._chunk_messages(
f, path, summary, chunk_index, chunk, selected, declared, with_index
f,
path,
summary,
chunk_index,
chunk,
selected,
declared,
time_bounds,
with_index,
)
except OSError:
raise
Expand All @@ -283,6 +342,7 @@ def _chunk_messages(
chunk: "Chunk",
selected: Set[int],
declared: "_Declared",
time_bounds: Tuple[Optional[int], Optional[int]],
with_index: bool = False,
) -> Iterator[Any]:
"""Decompress one chunk and yield its selected messages in file order.
Expand All @@ -295,6 +355,7 @@ def _chunk_messages(
from mcap.records import Channel, Message, Schema
from mcap.stream_reader import breakup_chunk

start_time, end_time = time_bounds
index = -1
for record in breakup_chunk(chunk):
if isinstance(record, Message):
Expand All @@ -304,7 +365,9 @@ def _chunk_messages(
)
if not self._channel_is_selected(channel, schema, summary, selected):
continue
if not self._selection.in_time_range(record.log_time):
if start_time is not None and record.log_time < start_time:
continue
if end_time is not None and record.log_time >= end_time:
continue
row_id = message_row_id(path, chunk_index.chunk_start_offset, index)
item = (schema, channel, record, row_id)
Expand Down Expand Up @@ -405,24 +468,41 @@ def _find_in_earlier_chunks(
break

def iter_unindexed(
self, f: Any, path: str, limit: Optional[int] = None
self,
f: Any,
path: str,
limit: Optional[int] = None,
*,
time_bounds: Optional[Tuple[Optional[int], Optional[int]]] = None,
log_time_order: Optional[bool] = None,
) -> Iterator[_Selected]:
"""Yield the selected messages of a file without a chunk index.

Without ``log_time_order``, they stream in file order. With it, the file
can only be ordered whole, so its selected messages are held in memory
and sorted by log time. The sort is stable: equal log times keep file
order. With ``limit``, only the ``limit`` earliest are held.

``log_time_order`` overrides the reader's setting, and ``time_bounds``
the selection's time range. Sample rows stream the whole file once and
index what they collect themselves.
"""
scanned: Iterable[_Scanned] = self._scan_unindexed(f, path)
if self._log_time_order and limit is not None:
if log_time_order is None:
log_time_order = self._log_time_order
scanned: Iterable[_Scanned] = self._scan_unindexed(f, path, time_bounds)
if log_time_order and limit is not None:
scanned = heapq.nsmallest(limit, scanned, key=_log_time)
elif self._log_time_order:
elif log_time_order:
scanned = sorted(scanned, key=_log_time)
for schema, channel, message, ordinal in scanned:
yield schema, channel, message, unindexed_message_row_id(path, ordinal)

def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]:
def _scan_unindexed(
self,
f: Any,
path: str,
time_bounds: Optional[Tuple[Optional[int], Optional[int]]],
) -> Iterator[_Scanned]:
"""Scan a file without a chunk index from the start, in file order.

Each message comes with its position among all messages of the file, so
Expand All @@ -431,6 +511,9 @@ def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]:
from mcap.records import Channel, Message, Schema
from mcap.stream_reader import StreamReader

if time_bounds is None:
time_bounds = (self._selection.start_time, self._selection.end_time)
start_time, end_time = time_bounds
f.seek(0)
schemas: Dict[int, "Schema"] = {}
channels: Dict[int, "Channel"] = {}
Expand All @@ -451,6 +534,8 @@ def _scan_unindexed(self, f: Any, path: str) -> Iterator[_Scanned]:
schema = schemas.get(channel.schema_id) if channel.schema_id else None
if not self._selection.accepts_channel(channel, schema):
continue
if not self._selection.in_time_range(record.log_time):
if start_time is not None and record.log_time < start_time:
continue
if end_time is not None and record.log_time >= end_time:
continue
yield schema, channel, record, ordinal
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type``
(``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes.

A sample read (``read_mcap(samples=...)``) lists nothing: the samples dataset
names the files, and ``read_api`` feeds it to the read tasks as their manifest.
Its rows hold each topic's steps around a sample's anchor (``mcap_samples``).

Format specification: https://mcap.dev/spec
"""

Expand All @@ -26,7 +30,9 @@
Iterator,
List,
Literal,
Mapping,
Optional,
Tuple,
Union,
)

Expand All @@ -52,10 +58,18 @@
MCAPSelection,
TimeRange,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_sample_rows import (
DEFAULT_MAX_ROW_BYTES,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_samples import (
TopicPlan,
sample_row_schema,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner
from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import (
TaskSummaryPartitioner,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_topic_spec import TopicSpec
from ray.data._internal.datasource_v2.interfaces.datasource_v2 import (
DatasourceCategory,
FileDataSourceV2,
Expand Down Expand Up @@ -98,7 +112,7 @@ def __init__(
self,
paths: List[str],
*,
topics: Optional[Iterable[str]] = None,
topics: Optional[Union[Iterable[str], Mapping[str, TopicSpec]]] = None,
time_range: Optional[TimeRange] = None,
message_types: Optional[Iterable[str]] = None,
include_metadata: bool = True,
Expand All @@ -112,18 +126,36 @@ def __init__(
ignore_missing_paths: bool = False,
shuffle: Optional[Union[Literal["files"], "FileShuffleConfig"]] = None,
read_task_memory: Optional[float] = None,
samples_schema: Optional[pa.Schema] = None,
):
"""Set up a read of ``paths``, or, with ``samples_schema``, a sample read.

A sample read takes ``topics`` as a mapping from each topic to its
``TopicSpec``, and ``samples_schema`` is the samples dataset's schema.
Its files come with the samples, so ``paths`` is empty and the read
tasks resolve each file against ``filesystem``, or its URI.
"""
super().__init__(name="MCAP", category=DatasourceCategory.FILE_BASED)
_check_import(self, module="mcap", package="mcap")

# Captured against the original paths: resolution below strips the
# ``local://`` scheme (see ``ParquetDatasourceV2``).
self._supports_distributed_reads = not _is_local_scheme(paths)
resolved_paths, resolved_filesystem = _resolve_paths_and_filesystem(
paths, filesystem
)
self._paths: List[str] = resolved_paths
self._filesystem = resolved_filesystem
self._samples_schema = samples_schema
self._topic_plans: Tuple[TopicPlan, ...] = ()
if samples_schema is not None:
assert isinstance(topics, Mapping), "a sample read maps topics to specs"
self._topic_plans = tuple(
TopicPlan.from_spec(topic, spec) for topic, spec in topics.items()
)
self._supports_distributed_reads = True
self._paths: List[str] = []
self._filesystem = filesystem
else:
# Captured against the original paths: resolution below strips the
# ``local://`` scheme (see ``ParquetDatasourceV2``).
self._supports_distributed_reads = not _is_local_scheme(paths)
resolved_paths, resolved_filesystem = _resolve_paths_and_filesystem(
paths, filesystem
)
self._paths = resolved_paths
self._filesystem = resolved_filesystem
self._selection = MCAPSelection.create(topics, time_range, message_types)
self._include_metadata = include_metadata
self._log_time_order = log_time_order
Expand Down Expand Up @@ -238,8 +270,11 @@ def infer_schema(self, sample: Optional[FileManifest]) -> pa.Schema:
values when every selected channel of the sampled files is JSON-encoded,
and the raw payload bytes (``binary``) otherwise. The reader follows this
one decision for every file, so a selection that mixes encodings keeps
every payload as bytes.
every payload as bytes. Sample rows hold the samples' own columns, then
each topic's (``sample_row_schema``).
"""
if self._samples_schema is not None:
return sample_row_schema(self._samples_schema, self._topic_plans)
assert sample is not None, "MCAP always receives a sample"
data_type = (
infer_data_type(
Expand Down Expand Up @@ -307,4 +342,8 @@ def create_scanner(
synthesized_columns=self._synthesized_columns,
shuffle=self._shuffle,
target_block_size=DataContext.get_current().target_max_block_size,
max_row_bytes=env_integer(
"RAY_DATA_MCAP_MAX_ROW_BYTES", DEFAULT_MAX_ROW_BYTES
),
topic_plans=self._topic_plans,
)
Loading