Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/data/execution-configurations.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ ctx.verbose_stats_logs = True

Many {class}`~ray.data.DataContext` options are for advanced use cases or debugging, and you usually don't need to modify them. The following are some of the most important options:

* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. Ray Data drops the data in the failed blocks. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors.
* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. A failed map task keeps the rows it produced before the error, and Ray Data drops the rest of the task's input. While Ray Data or Ray can still rerun the task, through `retried_map_errors` or the task's `max_retries` and `retry_exceptions`, a failed attempt emits no rows, so the rerun produces each row once. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors.
* `retried_io_errors`: A list of error message patterns that trigger a retry when reading or writing files. Ray Data matches each pattern first as literal text and then as a regular expression. Use it to handle transient errors from remote storage systems. By default, Ray Data retries on common transient AWS S3 errors. This option replaces the deprecated `write_file_retry_on_errors`.
* `verbose_stats_logs`: Whether stats logs are verbose. Verbose logs include fields such as `extra_metrics` in the stats output, which are otherwise excluded. Off by default.
* `log_internal_stack_trace`: Whether to write the full internal stack frames from Ray Data and Ray Core to the Ray Data log file when logging a user-code error. Ray Data always omits these internal frames from `stdout`, and by default also omits them from the log file. Set this to `True` to include them in the log file. Off by default.
Expand Down
158 changes: 131 additions & 27 deletions python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"""

import heapq
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import (
TYPE_CHECKING,
Any,
Expand All @@ -25,6 +25,8 @@

from ray.data._internal.datasource_v2.formats.mcap.mcap_io import (
ChunkReads,
OnCorrupt,
is_corrupt_data_error,
read_chunk,
read_chunks,
)
Expand All @@ -35,7 +37,7 @@
)

if TYPE_CHECKING:
from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema
from mcap.records import Channel, Chunk, ChunkIndex, McapRecord, Message, Schema
from mcap.summary import Summary

# One selected message as the reader sees it: its schema (``None`` for a
Expand All @@ -62,26 +64,48 @@ class _Declared:
a channel declared only in an earlier chunk serves the later ones.
``scanned`` holds the offsets of the chunks already read back for records.
``file_summary`` returns the file's whole summary, whose chunk indexes say
where to read back.
where to read back. With ``skip_corrupt``, a chunk read back that does not
decode is skipped and its offset kept in ``undecodable``.
"""

channels: Dict[int, "Channel"]
schemas: Dict[int, "Schema"]
scanned: Set[int]
file_summary: Callable[[], "Summary"]
skip_corrupt: bool = False
undecodable: List[int] = field(default_factory=list)

def undecodable_note(self) -> str:
"""A sentence naming the chunks read back that did not decode, if any."""
if not self.undecodable:
return ""
offsets = ", ".join(str(offset) for offset in self.undecodable)
if len(self.undecodable) == 1:
return f" The chunk at byte offset {offsets}, read back for it, is corrupt."
return f" The chunks at byte offsets {offsets}, read back for it, are corrupt."


def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None:
"""Add the channel and schema records of one chunk to ``declared``.

A record already in ``declared`` is not replaced. The chunk is marked as
scanned.
scanned. With ``declared.skip_corrupt``, a chunk that does not decode adds
nothing and is noted in ``declared.undecodable``, so the walk goes on to the
next earlier chunk. It is not reported as skipped: the task that owns it
does that. Any other error, and every error without ``skip_corrupt``, raises.
"""
from mcap.records import Channel, Schema
from mcap.stream_reader import breakup_chunk

declared.scanned.add(chunk_index.chunk_start_offset)
for record in breakup_chunk(read_chunk(f, chunk_index)):
try:
records = breakup_chunk(read_chunk(f, chunk_index))
except Exception as e:
if not declared.skip_corrupt or not is_corrupt_data_error(e):
raise
declared.undecodable.append(chunk_index.chunk_start_offset)
return
for record in records:
if isinstance(record, Channel):
declared.channels.setdefault(record.id, record)
elif isinstance(record, Schema):
Expand Down Expand Up @@ -113,13 +137,18 @@ def iter_chunks(
summary: "Summary",
offsets: Optional[Set[int]],
file_summary: Optional[Callable[[], "Summary"]] = None,
on_corrupt: Optional[OnCorrupt] = None,
) -> Iterator[_Selected]:
"""Yield the selected messages of the owned chunks of an indexed file.

Chunks are read in file order, or merged by log time with
``log_time_order``. ``summary`` may index only the owned chunks, as a
read task's summary does (``mcap_task_summary``). Then ``file_summary``
returns the whole summary, which reading back a declaration needs.

With ``on_corrupt``, a chunk that cannot be decoded is handed to it and
skipped, so the file's other chunks are still read. An error reading
``f`` still raises.
"""
selected = self._selection.selected_channel_ids(
summary.channels, summary.schemas
Expand All @@ -134,15 +163,17 @@ def iter_chunks(
dict(summary.schemas),
set(),
file_summary or (lambda: summary),
skip_corrupt=on_corrupt is not None,
)
if not self._log_time_order:
for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads):
yield from self._chunk_messages(
f, path, summary, chunk_index, chunk, selected, declared
chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt)
for chunk_index, chunk in chunks:
yield from self._decoded_messages(
f, path, summary, chunk_index, chunk, selected, declared, on_corrupt
)
return
yield from self._merge_chunks_by_log_time(
f, path, summary, chunk_indexes, selected, declared
f, path, summary, chunk_indexes, selected, declared, on_corrupt
)

def candidate_chunks(
Expand Down Expand Up @@ -176,6 +207,7 @@ def _merge_chunks_by_log_time(
chunk_indexes: List["ChunkIndex"],
selected: Set[int],
declared: _Declared,
on_corrupt: Optional[OnCorrupt],
) -> Iterator[_Selected]:
"""Yield the selected messages of the chunks in log-time order.

Expand All @@ -194,21 +226,22 @@ def _merge_chunks_by_log_time(
heapq.heapify(heap)
# Chunks leave the heap in ``_merge_order``, so they are read in it.
chunks = read_chunks(
f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads
f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads, on_corrupt
)
while heap:
_, kind, offset, _, item = heapq.heappop(heap)
if kind == 0:
chunk_index, chunk = next(chunks)
assert chunk_index.chunk_start_offset == offset
for index, selected_message in self._chunk_messages(
for index, selected_message in self._decoded_messages(
f,
path,
summary,
chunk_index,
chunk,
selected,
declared,
on_corrupt,
with_index=True,
):
heapq.heappush(
Expand All @@ -224,38 +257,75 @@ def _merge_chunks_by_log_time(
else:
yield item

def _decoded_messages(
self,
f: Any,
path: str,
summary: "Summary",
chunk_index: "ChunkIndex",
chunk: Optional["Chunk"],
selected: Set[int],
declared: "_Declared",
on_corrupt: Optional[OnCorrupt],
with_index: bool = False,
) -> Iterator[Any]:
"""Decode ``chunk`` and yield ``_chunk_messages`` of its records.

``chunk`` is ``None`` when its record did not parse, which
``read_chunks`` has already handed to ``on_corrupt``. A chunk that does
not decompress or parse (``is_corrupt_data_error``) goes to
``on_corrupt`` and yields nothing. Without ``on_corrupt`` it raises, as
does any error after the chunk is decoded.
"""
from mcap.stream_reader import breakup_chunk

if chunk is None:
return
try:
records = breakup_chunk(chunk)
except Exception as e:
if on_corrupt is None or not is_corrupt_data_error(e):
raise
on_corrupt(chunk_index, e)
Comment thread
marwan116 marked this conversation as resolved.
Resolved
return
yield from self._chunk_messages(
f, path, summary, chunk_index, records, selected, declared, with_index
)

def _chunk_messages(
self,
f: Any,
path: str,
summary: "Summary",
chunk_index: "ChunkIndex",
chunk: "Chunk",
records: List["McapRecord"],
selected: Set[int],
declared: "_Declared",
with_index: bool = False,
) -> Iterator[Any]:
"""Decompress one chunk and yield its selected messages in file order.
"""Yield the selected messages of one decoded chunk, in file order.

Channel and schema records inside the chunk take precedence over the
summary's. ``row_id`` counts every message record of the chunk, so it
does not depend on the selection. With ``with_index``, each message is
yielded as ``(index in chunk, message)``.
"""
from mcap.records import Channel, Message, Schema
from mcap.stream_reader import breakup_chunk

index = -1
for record in breakup_chunk(chunk):
for record in records:
if isinstance(record, Message):
index += 1
channel, schema = self._channel_and_schema(
f, path, chunk_index, declared, record
)
if not self._channel_is_selected(channel, schema, summary, selected):
channel = self._declared_channel(f, path, chunk_index, declared, record)
# Only a message that may be read needs its schema, so one the
# topic or time filters drop never fails on a missing schema.
if self._rejected_without_schema(channel, summary, selected):
continue
if not self._selection.in_time_range(record.log_time):
continue
schema = self._declared_schema(f, path, chunk_index, declared, channel)
if not self._channel_is_selected(channel, schema, summary, selected):
continue
row_id = message_row_id(path, chunk_index.chunk_start_offset, index)
item = (schema, channel, record, row_id)
yield (index, item) if with_index else item
Expand All @@ -264,17 +334,17 @@ def _chunk_messages(
elif isinstance(record, Schema):
declared.schemas[record.id] = record

def _channel_and_schema(
def _declared_channel(
self,
f: Any,
path: str,
chunk_index: "ChunkIndex",
declared: _Declared,
message: "Message",
) -> Tuple["Channel", Optional["Schema"]]:
"""Look up the channel and schema of ``message``.
) -> "Channel":
"""Look up the channel of ``message``.

A record not met yet is read back from the earlier chunks of the file.
A channel not met yet is read back from the earlier chunks of the file.
"""
channel = declared.channels.get(message.channel_id)
if channel is None:
Expand All @@ -286,17 +356,51 @@ def _channel_and_schema(
raise ValueError(
f"MCAP file {path!r} has a message on channel "
f"{message.channel_id}, which neither the summary nor any "
"chunk up to this one declares."
f"chunk up to this one declares.{declared.undecodable_note()}"
)
if channel.schema_id and channel.schema_id not in declared.schemas:
return channel

def _declared_schema(
self,
f: Any,
path: str,
chunk_index: "ChunkIndex",
declared: _Declared,
channel: "Channel",
) -> Optional["Schema"]:
"""Look up the schema of ``channel``, or ``None`` if it has none.

A schema not met yet is read back from the earlier chunks of the file.
"""
if not channel.schema_id:
return None
if channel.schema_id not in declared.schemas:
# The schema record may be only in an earlier chunk
# (``repeat_schemas=False``). ``message_types`` and the schema
# columns need it.
self._find_in_earlier_chunks(
f, chunk_index, declared, schema_id=channel.schema_id
)
schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None
return channel, schema
schema = declared.schemas.get(channel.schema_id)
if schema is None:
raise ValueError(
f"MCAP file {path!r} has channel {channel.topic!r} with schema "
f"{channel.schema_id}, which neither the summary nor any chunk "
f"up to this one declares.{declared.undecodable_note()}"
)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
return schema

def _rejected_without_schema(
self, channel: "Channel", summary: "Summary", selected: Set[int]
) -> bool:
"""Whether messages on ``channel`` fail the selection whatever its schema.

That is the case for a channel the summary lists but ``selected``
leaves out, and for a channel whose topic is not selected.
"""
if channel.id in summary.channels and channel.id not in selected:
return True
return not self._selection.accepts_channel(channel, None)

def _channel_is_selected(
self,
Expand Down
Loading
Loading