diff --git a/doc/source/data/execution-configurations.md b/doc/source/data/execution-configurations.md index 914ea80b2de6..6ce104615269 100644 --- a/doc/source/data/execution-configurations.md +++ b/doc/source/data/execution-configurations.md @@ -60,7 +60,7 @@ ctx.verbose_stats_logs = True Many {class}`~ray.data.DataContext` options are for advanced use cases or debugging, and you usually don't need to modify them. The following are some of the most important options: -* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. Ray Data drops the data in the failed blocks. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors. +* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. A failed map task keeps the rows it produced before the error, and Ray Data drops the rest of the task's input. While Ray Data or Ray can still rerun the task, through `retried_map_errors` or the task's `max_retries` and `retry_exceptions`, a failed attempt emits no rows, so the rerun produces each row once. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors. * `retried_io_errors`: A list of error message patterns that trigger a retry when reading or writing files. Ray Data matches each pattern first as literal text and then as a regular expression. Use it to handle transient errors from remote storage systems. By default, Ray Data retries on common transient AWS S3 errors. This option replaces the deprecated `write_file_retry_on_errors`. * `verbose_stats_logs`: Whether stats logs are verbose. Verbose logs include fields such as `extra_metrics` in the stats output, which are otherwise excluded. Off by default. * `log_internal_stack_trace`: Whether to write the full internal stack frames from Ray Data and Ray Core to the Ray Data log file when logging a user-code error. Ray Data always omits these internal frames from `stdout`, and by default also omits them from the log file. Set this to `True` to include them in the log file. Off by default. diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index ecc07d1e559b..fc7b7380c656 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -9,7 +9,7 @@ """ import heapq -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, Any, @@ -25,6 +25,8 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( ChunkReads, + OnCorrupt, + is_corrupt_data_error, read_chunk, read_chunks, ) @@ -35,7 +37,7 @@ ) if TYPE_CHECKING: - from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema + from mcap.records import Channel, Chunk, ChunkIndex, McapRecord, Message, Schema from mcap.summary import Summary # One selected message as the reader sees it: its schema (``None`` for a @@ -62,26 +64,48 @@ class _Declared: a channel declared only in an earlier chunk serves the later ones. ``scanned`` holds the offsets of the chunks already read back for records. ``file_summary`` returns the file's whole summary, whose chunk indexes say - where to read back. + where to read back. With ``skip_corrupt``, a chunk read back that does not + decode is skipped and its offset kept in ``undecodable``. """ channels: Dict[int, "Channel"] schemas: Dict[int, "Schema"] scanned: Set[int] file_summary: Callable[[], "Summary"] + skip_corrupt: bool = False + undecodable: List[int] = field(default_factory=list) + + def undecodable_note(self) -> str: + """A sentence naming the chunks read back that did not decode, if any.""" + if not self.undecodable: + return "" + offsets = ", ".join(str(offset) for offset in self.undecodable) + if len(self.undecodable) == 1: + return f" The chunk at byte offset {offsets}, read back for it, is corrupt." + return f" The chunks at byte offsets {offsets}, read back for it, are corrupt." def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None: """Add the channel and schema records of one chunk to ``declared``. A record already in ``declared`` is not replaced. The chunk is marked as - scanned. + scanned. With ``declared.skip_corrupt``, a chunk that does not decode adds + nothing and is noted in ``declared.undecodable``, so the walk goes on to the + next earlier chunk. It is not reported as skipped: the task that owns it + does that. Any other error, and every error without ``skip_corrupt``, raises. """ from mcap.records import Channel, Schema from mcap.stream_reader import breakup_chunk declared.scanned.add(chunk_index.chunk_start_offset) - for record in breakup_chunk(read_chunk(f, chunk_index)): + try: + records = breakup_chunk(read_chunk(f, chunk_index)) + except Exception as e: + if not declared.skip_corrupt or not is_corrupt_data_error(e): + raise + declared.undecodable.append(chunk_index.chunk_start_offset) + return + for record in records: if isinstance(record, Channel): declared.channels.setdefault(record.id, record) elif isinstance(record, Schema): @@ -113,6 +137,7 @@ def iter_chunks( summary: "Summary", offsets: Optional[Set[int]], file_summary: Optional[Callable[[], "Summary"]] = None, + on_corrupt: Optional[OnCorrupt] = None, ) -> Iterator[_Selected]: """Yield the selected messages of the owned chunks of an indexed file. @@ -120,6 +145,10 @@ def iter_chunks( ``log_time_order``. ``summary`` may index only the owned chunks, as a read task's summary does (``mcap_task_summary``). Then ``file_summary`` returns the whole summary, which reading back a declaration needs. + + With ``on_corrupt``, a chunk that cannot be decoded is handed to it and + skipped, so the file's other chunks are still read. An error reading + ``f`` still raises. """ selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -134,15 +163,17 @@ def iter_chunks( dict(summary.schemas), set(), file_summary or (lambda: summary), + skip_corrupt=on_corrupt is not None, ) if not self._log_time_order: - for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads): - yield from self._chunk_messages( - f, path, summary, chunk_index, chunk, selected, declared + chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt) + for chunk_index, chunk in chunks: + yield from self._decoded_messages( + f, path, summary, chunk_index, chunk, selected, declared, on_corrupt ) return yield from self._merge_chunks_by_log_time( - f, path, summary, chunk_indexes, selected, declared + f, path, summary, chunk_indexes, selected, declared, on_corrupt ) def candidate_chunks( @@ -176,6 +207,7 @@ def _merge_chunks_by_log_time( chunk_indexes: List["ChunkIndex"], selected: Set[int], declared: _Declared, + on_corrupt: Optional[OnCorrupt], ) -> Iterator[_Selected]: """Yield the selected messages of the chunks in log-time order. @@ -194,14 +226,14 @@ def _merge_chunks_by_log_time( heapq.heapify(heap) # Chunks leave the heap in ``_merge_order``, so they are read in it. chunks = read_chunks( - f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads + f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads, on_corrupt ) while heap: _, kind, offset, _, item = heapq.heappop(heap) if kind == 0: chunk_index, chunk = next(chunks) assert chunk_index.chunk_start_offset == offset - for index, selected_message in self._chunk_messages( + for index, selected_message in self._decoded_messages( f, path, summary, @@ -209,6 +241,7 @@ def _merge_chunks_by_log_time( chunk, selected, declared, + on_corrupt, with_index=True, ): heapq.heappush( @@ -224,18 +257,53 @@ def _merge_chunks_by_log_time( else: yield item + def _decoded_messages( + self, + f: Any, + path: str, + summary: "Summary", + chunk_index: "ChunkIndex", + chunk: Optional["Chunk"], + selected: Set[int], + declared: "_Declared", + on_corrupt: Optional[OnCorrupt], + with_index: bool = False, + ) -> Iterator[Any]: + """Decode ``chunk`` and yield ``_chunk_messages`` of its records. + + ``chunk`` is ``None`` when its record did not parse, which + ``read_chunks`` has already handed to ``on_corrupt``. A chunk that does + not decompress or parse (``is_corrupt_data_error``) goes to + ``on_corrupt`` and yields nothing. Without ``on_corrupt`` it raises, as + does any error after the chunk is decoded. + """ + from mcap.stream_reader import breakup_chunk + + if chunk is None: + return + try: + records = breakup_chunk(chunk) + except Exception as e: + if on_corrupt is None or not is_corrupt_data_error(e): + raise + on_corrupt(chunk_index, e) + return + yield from self._chunk_messages( + f, path, summary, chunk_index, records, selected, declared, with_index + ) + def _chunk_messages( self, f: Any, path: str, summary: "Summary", chunk_index: "ChunkIndex", - chunk: "Chunk", + records: List["McapRecord"], selected: Set[int], declared: "_Declared", with_index: bool = False, ) -> Iterator[Any]: - """Decompress one chunk and yield its selected messages in file order. + """Yield the selected messages of one decoded chunk, in file order. Channel and schema records inside the chunk take precedence over the summary's. ``row_id`` counts every message record of the chunk, so it @@ -243,19 +311,21 @@ def _chunk_messages( yielded as ``(index in chunk, message)``. """ from mcap.records import Channel, Message, Schema - from mcap.stream_reader import breakup_chunk index = -1 - for record in breakup_chunk(chunk): + for record in records: if isinstance(record, Message): index += 1 - channel, schema = self._channel_and_schema( - f, path, chunk_index, declared, record - ) - if not self._channel_is_selected(channel, schema, summary, selected): + channel = self._declared_channel(f, path, chunk_index, declared, record) + # Only a message that may be read needs its schema, so one the + # topic or time filters drop never fails on a missing schema. + if self._rejected_without_schema(channel, summary, selected): continue if not self._selection.in_time_range(record.log_time): continue + schema = self._declared_schema(f, path, chunk_index, declared, channel) + if not self._channel_is_selected(channel, schema, summary, selected): + continue row_id = message_row_id(path, chunk_index.chunk_start_offset, index) item = (schema, channel, record, row_id) yield (index, item) if with_index else item @@ -264,17 +334,17 @@ def _chunk_messages( elif isinstance(record, Schema): declared.schemas[record.id] = record - def _channel_and_schema( + def _declared_channel( self, f: Any, path: str, chunk_index: "ChunkIndex", declared: _Declared, message: "Message", - ) -> Tuple["Channel", Optional["Schema"]]: - """Look up the channel and schema of ``message``. + ) -> "Channel": + """Look up the channel of ``message``. - A record not met yet is read back from the earlier chunks of the file. + A channel not met yet is read back from the earlier chunks of the file. """ channel = declared.channels.get(message.channel_id) if channel is None: @@ -286,17 +356,51 @@ def _channel_and_schema( raise ValueError( f"MCAP file {path!r} has a message on channel " f"{message.channel_id}, which neither the summary nor any " - "chunk up to this one declares." + f"chunk up to this one declares.{declared.undecodable_note()}" ) - if channel.schema_id and channel.schema_id not in declared.schemas: + return channel + + def _declared_schema( + self, + f: Any, + path: str, + chunk_index: "ChunkIndex", + declared: _Declared, + channel: "Channel", + ) -> Optional["Schema"]: + """Look up the schema of ``channel``, or ``None`` if it has none. + + A schema not met yet is read back from the earlier chunks of the file. + """ + if not channel.schema_id: + return None + if channel.schema_id not in declared.schemas: # The schema record may be only in an earlier chunk # (``repeat_schemas=False``). ``message_types`` and the schema # columns need it. self._find_in_earlier_chunks( f, chunk_index, declared, schema_id=channel.schema_id ) - schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None - return channel, schema + schema = declared.schemas.get(channel.schema_id) + if schema is None: + raise ValueError( + f"MCAP file {path!r} has channel {channel.topic!r} with schema " + f"{channel.schema_id}, which neither the summary nor any chunk " + f"up to this one declares.{declared.undecodable_note()}" + ) + return schema + + def _rejected_without_schema( + self, channel: "Channel", summary: "Summary", selected: Set[int] + ) -> bool: + """Whether messages on ``channel`` fail the selection whatever its schema. + + That is the case for a channel the summary lists but ``selected`` + leaves out, and for a channel whose topic is not selected. + """ + if channel.id in summary.channels and channel.id not in selected: + return True + return not self._selection.accepts_channel(channel, None) def _channel_is_selected( self, diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index 4440f7fee115..37f48fba22e9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -19,7 +19,17 @@ import io from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterable, Iterator, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Iterable, + Iterator, + List, + Optional, + Tuple, + Union, +) from ray._common.utils import env_integer from ray.data._internal.util import KiB, MiB @@ -105,13 +115,50 @@ def plan_chunk_reads( return planned +# Takes a chunk that cannot be decoded and the error, in a read that skips +# corrupt chunks (``SelectedMessageReader.iter_chunks``). +OnCorrupt = Callable[["ChunkIndex", Exception], None] + + +def is_corrupt_data_error(error: BaseException) -> bool: + """Whether ``error`` says MCAP bytes do not decode, rather than that a read + failed or the reader has a bug. + + Parsing a malformed record raises an mcap ``McapError``, ``struct.error`` or + a ``ValueError`` such as ``UnicodeDecodeError``. Decompressing raises + ``zstandard.ZstdError``, or for lz4 a ``RuntimeError`` whose message starts + with ``LZ4F_``. An ``OSError`` and a missing codec + (``UnsupportedCompressionError``) are not corruption. + """ + import struct + + from mcap.exceptions import McapError, UnsupportedCompressionError + + if isinstance(error, (OSError, UnsupportedCompressionError)): + return False + if isinstance(error, (McapError, struct.error, ValueError)): + return True + if type(error) is RuntimeError: + return str(error).startswith("LZ4F_") + try: + import zstandard + except ImportError: + return False + return isinstance(error, zstandard.ZstdError) + + def read_chunks( - f: Any, chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads -) -> Iterator[Tuple["ChunkIndex", "Chunk"]]: + f: Any, + chunk_indexes: Iterable["ChunkIndex"], + reads: ChunkReads, + on_corrupt: Optional[OnCorrupt] = None, +) -> Iterator[Tuple["ChunkIndex", Optional["Chunk"]]]: """Yield the chunk record of each of ``chunk_indexes``, in that order. Each read of ``plan_chunk_reads`` is made when its first chunk is needed, - and its bytes are released once its last chunk is parsed. + and its bytes are released once its last chunk is parsed. With + ``on_corrupt``, a chunk record that does not parse is handed to it and + yielded as ``None``; an error reading ``f`` still raises. """ for ranged_read in plan_chunk_reads(chunk_indexes, reads): # ``BytesIO`` shares the bytes rather than copying them; parsing a @@ -120,7 +167,7 @@ def read_chunks( last = len(ranged_read.chunk_indexes) - 1 for i, chunk_index in enumerate(ranged_read.chunk_indexes): position = chunk_index.chunk_start_offset - ranged_read.start - chunk = _parse_chunk(stream, position) + chunk = _parse_chunk_or_report(stream, position, chunk_index, on_corrupt) if i == last: # Free the bytes now, not when the next read replaces them, so # two reads are never held at once. @@ -240,6 +287,24 @@ def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": return _parse_chunk(stream, 0) +def _parse_chunk_or_report( + stream: io.BytesIO, + position: int, + chunk_index: "ChunkIndex", + on_corrupt: Optional[OnCorrupt], +) -> Optional["Chunk"]: + """``_parse_chunk``, or ``None`` after handing a corrupt record to + ``on_corrupt``. Without ``on_corrupt``, or for an error that is not + corruption (``is_corrupt_data_error``), the error raises.""" + try: + return _parse_chunk(stream, position) + except Exception as e: + if on_corrupt is None or not is_corrupt_data_error(e): + raise + on_corrupt(chunk_index, e) + return None + + def _parse_chunk(stream: io.BytesIO, position: int) -> "Chunk": """Parse the chunk record that starts at ``position`` of ``stream``.""" from mcap.data_stream import ReadDataStream diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py index fd3994aa7a83..85c4f22979fc 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py @@ -128,14 +128,18 @@ def message_schema( return pa.schema(fields) +class InvalidJsonPayload(ValueError): + """A message on a JSON-encoded channel whose payload is not valid JSON.""" + + def decode_payload(channel: "Channel", data: bytes, where: str) -> Any: """Decode a JSON payload into Python values. The reader calls this only when ``data`` was planned as decoded JSON, and planning calls it to type that column. Arrow has no column type for a mix of - bytes and decoded values, so a channel of another encoding, or a payload - that is not valid JSON, raises a ``ValueError`` that names ``where``: a - ``row_id`` or a file. + bytes and decoded values, so a channel of another encoding raises a + ``ValueError``, and a payload that is not valid JSON an + ``InvalidJsonPayload``. Both name ``where``: a ``row_id`` or a file. """ if channel.message_encoding != "json": raise ValueError( @@ -147,7 +151,7 @@ def decode_payload(channel: "Channel", data: bytes, where: str) -> Any: try: return json.loads(data.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError) as e: - raise ValueError( + raise InvalidJsonPayload( f"{where}: message on JSON-encoded topic {channel.topic!r} is not valid " f"JSON: {e}" ) from e @@ -383,7 +387,15 @@ def reset(self) -> None: ) def add(self, selected: _Selected) -> None: + """Add one message as a row. + + The payload is decoded before anything is added, so a payload that + raises ``InvalidJsonPayload`` leaves the builder as it was. + """ schema, channel, message, row_id = selected + value = None + if self._json_values is not None: + value = decode_payload(channel, message.data, row_id) slot = self._slot(channel, schema) self._slot_ids.append(slot) self.num_rows += 1 @@ -394,7 +406,7 @@ def add(self, selected: _Selected) -> None: self._payloads.append(message.data) self.estimated_bytes += len(message.data) elif self._json_values is not None: - self._json_values.append(decode_payload(channel, message.data, row_id)) + self._json_values.append(value) payload_bytes = len(message.data) self.estimated_bytes += _DECODED_JSON_BYTES_PER_PAYLOAD_BYTE * payload_bytes if self._row_ids is not None: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 07aea2d965c7..6c76ed0825d9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -19,11 +19,19 @@ - ``mcap_message_rows`` builds message rows. """ -import itertools import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from functools import partial -from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Set +from typing import ( + TYPE_CHECKING, + Any, + Iterator, + List, + Optional, + Sequence, + Set, + Tuple, +) import pyarrow as pa from pyarrow.fs import FileSystem, LocalFileSystem @@ -35,10 +43,12 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + is_corrupt_data_error, parse_summary_records, read_summary_from, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( + InvalidJsonPayload, _MessageTableBuilder, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection @@ -60,6 +70,7 @@ from ray.util.debug import log_once if TYPE_CHECKING: + from mcap.records import ChunkIndex from mcap.summary import Summary logger = logging.getLogger(__name__) @@ -129,40 +140,95 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: Each row is read on its own, in manifest order, so a path listed twice is read twice. With ``limit``, reading stops at that many rows: no later message is added to a table and no later chunk or file is read. + + When ``DataContext.max_errored_blocks`` lets errors be skipped, a + corrupt chunk costs only itself: the reader skips it and reads the + file's other chunks. A message whose payload is not valid JSON likewise + costs only its own row. A file whose other data does not decode keeps + the rows before the failure and does not stop the task's other files. + Once they are read, one error names every file, chunk and message + skipped, so the task counts once against the budget. Any other error, + such as a failed read, raises at once, so the task's retry settings see + it. Without skipping, the first failure fails the task at once. """ from ray.data.context import DataContext if len(input_split) == 0: return - retried_io_errors = DataContext.get_current().retried_io_errors + skip_corrupt = DataContext.get_current().max_errored_blocks != 0 remaining = self._limit + failures: List[Tuple[str, Exception]] = [] for owned in _owned_files(input_split): if remaining is not None and remaining <= 0: - return - tables = iterate_with_retry( - partial(self._read_file, owned, remaining), - f"read MCAP file {owned.path}", - match=retried_io_errors, - ) - for table in tables: + break + for table in self._read_owned_file( + owned, remaining, skip_corrupt, failures + ): if remaining is not None: remaining -= table.num_rows yield table + if failures: + raise _read_error(failures) + + def _read_owned_file( + self, + owned: "_OwnedFile", + limit: Optional[int], + skip_corrupt: bool, + failures: List[Tuple[str, Exception]], + ) -> Iterator[pa.Table]: + """Yield the tables of one file, retrying as ``retried_io_errors`` allows. + + With ``skip_corrupt``, the file's corrupt data is appended to + ``failures`` rather than raised: its skipped chunks and messages, or + its own error when it does not decode as a whole + (``is_corrupt_data_error``). + """ + from ray.data.context import DataContext + + # What the read skips. Each attempt starts it afresh, so it holds the + # last attempt's. + skipped = _SkippedData() if skip_corrupt else None + tables = iterate_with_retry( + partial(self._read_file, owned, limit, skipped), + f"read MCAP file {owned.path}", + match=DataContext.get_current().retried_io_errors, + ) + try: + yield from tables + except Exception as e: + if not skip_corrupt or not is_corrupt_data_error(e): + raise + failures.append((owned.path, e)) + return + if skipped is not None and (skipped.chunks.offsets or skipped.num_messages): + failures.append((owned.path, _skipped_data_error(owned.path, skipped))) # -- one file ---------------------------------------------------------- def _read_file( - self, owned: "_OwnedFile", limit: Optional[int] + self, + owned: "_OwnedFile", + limit: Optional[int], + skipped: Optional["_SkippedData"] = None, ) -> Iterator[pa.Table]: """Yield the tables of one file, limited to the chunks it owns. ``owned.offsets`` is ``None`` for a whole-file row, which means every chunk. A file without a chunk index is scanned linearly, through a read buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most - ``limit`` rows are yielded. + ``limit`` rows are built. With ``skipped``, a chunk that cannot be + decoded and a message that is not valid JSON are recorded there and + skipped, and a skipped message does not count against ``limit``. A + scanned file has no chunk boundaries to skip to, so a corrupt chunk + still fails it whole. A retry reads the file from its start, so + ``skipped`` is cleared first. """ from ray.data.context import DataContext + if skipped is not None: + skipped.clear() + path, offsets = owned.path, owned.offsets filesystem = self._filesystem or LocalFileSystem() with filesystem.open_input_file(path) as f: @@ -178,24 +244,43 @@ def _read_file( file_summary = None if offsets is None else _FileSummary(f, path) if summary is None or not summary.chunk_indexes: buffer_size = DataContext.get_current().streaming_read_buffer_size + # A skipped message leaves its place in the limit to the next + # one, so a read that can skip holds every selected message. messages = self._message_reader.iter_unindexed( - buffered(f, buffer_size), path, limit + buffered(f, buffer_size), path, limit if skipped is None else None ) else: messages = self._message_reader.iter_chunks( - f, path, summary, offsets, file_summary + f, + path, + summary, + offsets, + file_summary, + on_corrupt=None + if skipped is None + else partial(_record_corrupt_chunk, skipped.chunks), ) - yield from self._tables(itertools.islice(messages, limit), path, offsets) + yield from self._tables(messages, path, offsets, limit, skipped) # -- rows to tables ---------------------------------------------------- def _tables( - self, messages: Iterator[_Selected], path: str, offsets: Optional[Set[int]] + self, + messages: Iterator[_Selected], + path: str, + offsets: Optional[Set[int]], + limit: Optional[int], + skipped: Optional["_SkippedData"] = None, ) -> Iterator[pa.Table]: """Build tables of about ``target_block_size`` bytes from the messages. Messages are pulled one at a time, so a chunk is read only once its - first message is the next one needed. + first message is the next one needed, and no message is pulled once + ``limit`` rows are built. With ``skipped``, a message that is not valid + JSON is recorded there instead of becoming a row. An error that fails + the rest of the file (``is_corrupt_data_error``) first yields the rows + built before it, so what the file keeps does not depend on the block + size. """ # Every table reports one read unit: the file for a whole-file read, else # the row's first chunk. Listing excludes read units chunk by chunk @@ -215,20 +300,36 @@ def _tables( extra_row_bytes=self._appended_row_bytes(path, unit, wanted), ) rows_before = 0 - for selected in messages: - builder.add(selected) - if ( - self._target_block_size is not None - and builder.estimated_bytes >= self._target_block_size - ): - num_rows = builder.num_rows - # ``build`` empties the builder, so the yielded table is the - # only copy of its rows. + try: + for selected in messages: + if not _add_message(builder, selected, skipped): + continue + if limit is not None and rows_before + builder.num_rows >= limit: + break + if self._block_is_full(builder): + num_rows = builder.num_rows + # ``build`` empties the builder, so the yielded table is the + # only copy of its rows. + yield self._finish(builder.build(), path, unit, rows_before) + rows_before += num_rows + except Exception as e: + if skipped is None or not is_corrupt_data_error(e): + raise + # The file's bytes fail the same way on every read, so a retried + # read yields these same tables and the retry skips them all. + if builder.num_rows > 0: yield self._finish(builder.build(), path, unit, rows_before) - rows_before += num_rows + raise if builder.num_rows > 0: yield self._finish(builder.build(), path, unit, rows_before) + def _block_is_full(self, builder: _MessageTableBuilder) -> bool: + """Whether ``builder`` holds ``target_block_size`` bytes or more.""" + return ( + self._target_block_size is not None + and builder.estimated_bytes >= self._target_block_size + ) + def _appended_row_bytes( self, path: str, unit: ReadUnit, wanted: Optional[Set[str]] ) -> int: @@ -322,6 +423,146 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array +@dataclass(frozen=True) +class _SkippedMessage: + """A message whose payload is not valid JSON, and the error it raised.""" + + row_id: str + error: Exception + + +@dataclass +class _CorruptOffsets: + """The byte offsets of what a read skipped as corrupt, and one error. + + Only the error at the lowest offset is kept. An error's traceback holds the + bytes that failed to parse, so keeping every error would hold every + corrupt chunk until the file is done. + """ + + offsets: Set[int] = field(default_factory=set) + lowest: Optional[int] = None + error: Optional[Exception] = None + + def add(self, offset: int, error: Exception) -> None: + """Record ``offset``, keeping ``error`` if ``offset`` is the lowest.""" + if self.lowest is None or offset < self.lowest: + self.lowest = offset + self.error = error + self.offsets.add(offset) + + +@dataclass +class _SkippedData: + """What a read of one file skipped. + + Every corrupt chunk is named by its offset, and one error is kept for them + all. Messages that are not valid JSON are counted and only the first is + kept. + """ + + chunks: _CorruptOffsets = field(default_factory=_CorruptOffsets) + num_messages: int = 0 + first_message: Optional[_SkippedMessage] = None + + def record_message(self, row_id: str, error: Exception) -> None: + """Count a skipped message, keeping it if it is the first.""" + if self.first_message is None: + self.first_message = _SkippedMessage(row_id, error) + self.num_messages += 1 + + def clear(self) -> None: + """Forget what was skipped, for a read that starts the file over.""" + self.chunks = _CorruptOffsets() + self.num_messages = 0 + self.first_message = None + + +def _add_message( + builder: _MessageTableBuilder, + selected: _Selected, + skipped: Optional[_SkippedData], +) -> bool: + """Add one message to ``builder`` and return whether it became a row. + + With ``skipped``, a message whose payload is not valid JSON is recorded + there rather than raised. + """ + try: + builder.add(selected) + except InvalidJsonPayload as e: + if skipped is None: + raise + _, _, _, row_id = selected + skipped.record_message(row_id, e) + return False + return True + + +def _record_corrupt_chunk( + corrupt: _CorruptOffsets, chunk_index: "ChunkIndex", error: Exception +) -> None: + """Record a chunk a read skips as corrupt.""" + corrupt.add(chunk_index.chunk_start_offset, error) + + +def _skipped_data_error(path: str, skipped: _SkippedData) -> ValueError: + """The error for the chunks and messages of ``path`` that a read skipped. + + It names every chunk and the first message, and quotes the first error: + the first chunk's, or the first message's when no chunk was skipped. + """ + offsets = sorted(skipped.chunks.offsets) + parts = [] + if offsets: + where = ( + "chunk at byte offset" if len(offsets) == 1 else "chunks at byte offsets" + ) + listed = ", ".join(str(offset) for offset in offsets) + parts.append(f"{len(offsets)} corrupt {where} {listed}") + message = skipped.first_message + if message is not None: + count = skipped.num_messages + what = "message that is" if count == 1 else "messages that are" + which = "row id" if count == 1 else "first: row id" + parts.append(f"{count} {what} not valid JSON ({which} {message.row_id})") + if offsets: + first = skipped.chunks.error + assert first is not None + else: + assert message is not None + first = message.error + error = ValueError( + f"Skipped {' and '.join(parts)} of MCAP file {path!r}: " + f"{type(first).__name__}: {_first_line(first)}" + ) + error.__cause__ = first + return error + + +def _read_error(failures: List[Tuple[str, Exception]]) -> Exception: + """The error for the files whose data a task found corrupt. + + A single failure is its own error, whose message already names the file. + Several become one ``ValueError`` naming each file, caused by the first. + Every failure is corruption, so no rerun would read any of them. + """ + if len(failures) == 1: + return failures[0][1] + lines = [f"{path}: {type(e).__name__}: {_first_line(e)}" for path, e in failures] + error = ValueError( + f"{len(failures)} MCAP files of this read task could not be read in full:\n" + + "\n".join(lines) + ) + error.__cause__ = failures[0][1] + return error + + +def _first_line(error: Exception) -> str: + """The first non-empty line of ``error``'s message.""" + return next((line for line in str(error).splitlines() if line.strip()), "") + + @dataclass(frozen=True) class _OwnedFile: """What one manifest row reads of one file.""" diff --git a/python/ray/data/_internal/execution/interfaces/task_context.py b/python/ray/data/_internal/execution/interfaces/task_context.py index 35ce6506ea84..db23642eb275 100644 --- a/python/ray/data/_internal/execution/interfaces/task_context.py +++ b/python/ray/data/_internal/execution/interfaces/task_context.py @@ -1,7 +1,18 @@ import contextlib +import dataclasses import threading from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Iterator, Optional +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + Optional, + Sequence, + Tuple, + Type, + Union, +) if TYPE_CHECKING: from ray.data._internal.execution.operators.map_transformer import MapTransformer @@ -11,6 +22,78 @@ _thread_local = threading.local() +@dataclass(frozen=True) +class RerunPolicy: + """The errors on which a map task runs again after it raises. + + A rerun produces the task's blocks from the start, so a task emits the rows + it buffered before an error the job skips only when no rerun follows. The + default reruns on any error: a task emits partial output only when its + operator states how Ray retries it. + """ + + # Whether Ray reruns the task on any error, and otherwise the exception + # types it reruns on, matched against the error and its causes. + ray_reruns_any_error: bool = True + ray_retried_types: Tuple[Type[BaseException], ...] = () + + # How many more attempts ``_map_task`` makes, and the ``retried_map_errors`` + # patterns it makes them on. ``None`` matches any error, as for + # ``iterate_with_retry``. + map_attempts_left: int = 0 + map_retried_patterns: Optional[Tuple[str, ...]] = () + + @classmethod + def from_retry_exceptions( + cls, retry_exceptions: Union[bool, Sequence[Type[BaseException]]] + ) -> "RerunPolicy": + """The policy of a task Ray reruns on ``retry_exceptions``. + + ``retry_exceptions`` takes the form of the remote arg: ``True`` for any + error, ``False`` for none, or a list of exception types. + """ + if isinstance(retry_exceptions, bool): + return cls(ray_reruns_any_error=retry_exceptions) + return cls( + ray_reruns_any_error=False, ray_retried_types=tuple(retry_exceptions) + ) + + def with_map_retries( + self, attempts_left: int, patterns: Optional[Sequence[str]] + ) -> "RerunPolicy": + """This policy for an attempt of ``_map_task`` with ``attempts_left``.""" + return dataclasses.replace( + self, + map_attempts_left=attempts_left, + map_retried_patterns=None if patterns is None else tuple(patterns), + ) + + def may_rerun(self, error: Exception) -> bool: + """Whether Ray or ``_map_task`` runs the task again after ``error``.""" + return self._ray_reruns_on(error) or self._map_task_reruns_on(error) + + def _ray_reruns_on(self, error: Exception) -> bool: + if self.ray_reruns_any_error: + return True + # A transform may wrap the error, so its causes count too. + cause: Optional[BaseException] = error + while cause is not None: + if isinstance(cause, self.ray_retried_types): + return True + cause = cause.__cause__ + return False + + def _map_task_reruns_on(self, error: Exception) -> bool: + # Matched as ``_map_task`` matches it: the message with its causes. + from ray._common.retry import format_exception + from ray.data._internal.util import is_retryable_error + + if self.map_attempts_left <= 0: + return False + error_str = format_exception(error, include_cause=True) + return is_retryable_error(error_str, self.map_retried_patterns) + + @dataclass class TaskContext: """This describes the information of a task running block transform.""" @@ -47,6 +130,10 @@ class TaskContext: # Override of the target max-block-size for the task target_max_block_size_override: Optional[int] = None + # The errors on which this task runs again. A map operator sets Ray's part + # and ``_map_task`` its own part for each attempt. + rerun_policy: RerunPolicy = field(default_factory=RerunPolicy) + # Additional keyword arguments passed to the task. kwargs: Dict[str, Any] = field(default_factory=dict) diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 62063abf256c..ef1a51da718c 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -57,6 +57,7 @@ TaskContext, actor_resource_dict, ) +from ray.data._internal.execution.interfaces.task_context import RerunPolicy from ray.data._internal.execution.node_trackers.actor_location import ( ActorLocationTracker, get_or_create_actor_location_tracker, @@ -118,6 +119,23 @@ def get_map_worker_cls_name(op_name: str) -> str: return f"MapWorker({op_name})" +def _actor_task_rerun_policy( + task_args: Dict[str, Any], actor_args: Dict[str, Any] +) -> RerunPolicy: + """The errors on which Ray reruns an actor task. + + Ray reruns an actor task only while ``max_task_retries`` allows it. The task + option overrides the actor's, and Ray defaults it to 0, as it stays when + ``max_restarts=0``. + """ + max_task_retries = task_args.get( + "max_task_retries", actor_args.get("max_task_retries", 0) + ) + if max_task_retries == 0: + return RerunPolicy.from_retry_exceptions(False) + return RerunPolicy.from_retry_exceptions(task_args.get("retry_exceptions", False)) + + class ActorPoolMapOperator(MapOperator, ReportsExtraResourceUsage): """A MapOperator implementation that executes tasks on an actor pool. @@ -505,12 +523,15 @@ def _try_schedule_tasks_internal(self) -> int: input_blocks = [entry.ref for entry in bundle.blocks] self._actor_pool.on_task_submitted(actor) + actor_task_args = dict(self._ray_actor_task_remote_args) ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, + rerun_policy=_actor_task_rerun_policy( + actor_task_args, self._ray_remote_args + ), ) - actor_task_args = dict(self._ray_actor_task_remote_args) extra_labels = actor_task_args.pop("_labels", None) or {} gen = actor.submit.options( num_returns="streaming", diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index 4d27e34b47dd..44481cd0210f 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -986,7 +986,24 @@ def _map_task( # `max_concurrent_calls_per_actor > 1` several run at once. clock = TransformClock() + match retry_on: + case True: + to_match = None + case False: + to_match = [] + case _: + to_match = retry_on + max_attempts = data_context.max_map_retries + 1 + attempts_started = 0 + def transform_iter_factory(): + nonlocal attempts_started + # Tell the transform whether this attempt can still be rerun, so it + # knows whether to emit its buffered rows before an error. + attempts_started += 1 + ctx.rerun_policy = ctx.rerun_policy.with_map_retries( + max_attempts - attempts_started, to_match + ) # Clear any per-task custom stats before each attempt (the reporter # is reused across retries of this task), so a prior attempt's stats # can't leak into this one. A producing transform repopulates it @@ -1003,19 +1020,11 @@ def transform_iter_factory(): clock=clock, ) - match retry_on: - case True: - to_match = None - case False: - to_match = [] - case _: - to_match = retry_on - block_iter = iterate_with_retry( transform_iter_factory, description="apply UDF transform", match=to_match, - max_attempts=data_context.max_map_retries + 1, + max_attempts=max_attempts, unwrap_cause=True, ) diff --git a/python/ray/data/_internal/execution/operators/map_transformer.py b/python/ray/data/_internal/execution/operators/map_transformer.py index bf23bf405007..0bcbe0077f8f 100644 --- a/python/ray/data/_internal/execution/operators/map_transformer.py +++ b/python/ray/data/_internal/execution/operators/map_transformer.py @@ -19,7 +19,10 @@ from ray._common.utils import env_integer from ray.data._internal.block_batching.block_batching import batch_blocks -from ray.data._internal.execution.interfaces.task_context import TaskContext +from ray.data._internal.execution.interfaces.task_context import ( + RerunPolicy, + TaskContext, +) from ray.data._internal.output_buffer import BlockOutputBuffer, OutputBlockSizeOption from ray.data.block import ( BatchFormat, @@ -763,6 +766,13 @@ def __init__( self._target_batch_size_bytes = target_batch_size_bytes def _pre_process(self, blocks: Iterable[Block]) -> Iterable[MapTransformFnData]: + # An upstream stage of a fused task may raise an error the task skips. + # The batcher then sees the end of its input, so the rows it holds come + # out as a short batch before the error is raised. + input_blocks = _InputEndingAtSkippedError(blocks) + return _ErrorRaisedAtEnd(self._batches(input_blocks), input_blocks) + + def _batches(self, blocks: Iterable[Block]) -> Iterable[MapTransformFnData]: # TODO make batch-udf zero-copy by default if self._batch_size == "auto": batch_size, blocks = _compute_auto_batch_size( @@ -836,6 +846,10 @@ class _BlockShapingIterator(Iterator[Block]): Unlike a generator, local variables in __next__ go out of scope when the method returns, avoiding holding references to yielded values. + + When the results raise an error the job skips, the rows already buffered + come out as a last block before the error, so the skipped task keeps the + output it produced (see ``_should_emit_buffered_rows``). """ def __init__( @@ -847,6 +861,8 @@ def __init__( self._results_iter = iter(results) self._buffer = BlockOutputBuffer(output_block_size_option) self._finalized = False + # The error the results raised, held until the buffered rows are out. + self._error: Optional[Exception] = None if input_type == MapTransformFnDataType.Block: self._append_buffer = self._buffer.add_block @@ -867,6 +883,9 @@ def __next__(self) -> Block: # If finalized, no more data elif self._finalized: + if self._error is not None: + error, self._error = self._error, None + raise error raise StopIteration try: @@ -876,6 +895,79 @@ def __next__(self) -> Block: except StopIteration: self._buffer.finalize() self._finalized = True + except Exception as e: + if not self._buffer.has_buffered_rows(): + raise + if not _should_emit_buffered_rows(e): + raise + self._error = e + self._buffer.finalize() + self._finalized = True + + +def _should_emit_buffered_rows(error: Exception) -> bool: + """Whether a map task that raised ``error`` first emits its buffered rows. + + Only when the job skips the error (``DataContext.max_errored_blocks`` is not + 0) and the task does not run again. A rerun produces its blocks from the + start and skips as many as were already emitted, so a last block cut short + at the error would hide rows of the rerun's block in its place. The task's + ``RerunPolicy`` says when it runs again. + """ + from ray.data.context import DataContext + + if DataContext.get_current().max_errored_blocks == 0: + return False + task_context = TaskContext.get_current() + policy = task_context.rerun_policy if task_context else RerunPolicy() + return not policy.may_rerun(error) + + +class _InputEndingAtSkippedError(Iterator[Any]): + """Input that ends where an upstream stage raises an error the task skips. + + The error is held, so a stage that buffers this input, such as a batcher, + hands on the rows it holds as at the end of its input. + ``_ErrorRaisedAtEnd`` then raises the held error. + """ + + def __init__(self, items: Iterable[Any]): + self._items = iter(items) + self.error: Optional[Exception] = None + + def __iter__(self) -> "_InputEndingAtSkippedError": + return self + + def __next__(self) -> Any: + try: + return next(self._items) + except StopIteration: + raise + except Exception as e: + if not _should_emit_buffered_rows(e): + raise + self.error = e + raise StopIteration + + +class _ErrorRaisedAtEnd(Iterator[Any]): + """Passes ``items`` through, then raises the error ``source`` held, if any.""" + + def __init__(self, items: Iterable[Any], source: _InputEndingAtSkippedError): + self._items = iter(items) + self._source = source + + def __iter__(self) -> "_ErrorRaisedAtEnd": + return self + + def __next__(self) -> Any: + try: + return next(self._items) + except StopIteration: + if self._source.error is None: + raise + error, self._source.error = self._source.error, None + raise error class _RowBasedIterator(Iterator[Row]): diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index 0ec9cffed5b8..e7bf7c44e33d 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -19,12 +19,16 @@ TaskContext, task_resource_dict, ) +from ray.data._internal.execution.interfaces.task_context import RerunPolicy from ray.data._internal.execution.operators.map_operator import ( MapOperator, _map_task, ) from ray.data._internal.execution.operators.map_transformer import MapTransformer -from ray.data._internal.remote_fn import cached_remote_fn +from ray.data._internal.remote_fn import ( + _add_system_error_to_retry_exceptions, + cached_remote_fn, +) from ray.data._internal.utils.cached_ray_internals import ( get_local_ongoing_lineage_reconstruction_tasks, ) @@ -184,13 +188,13 @@ def _try_schedule_task(self, bundle: RefBundle, strict: bool): # Notify first input for deferred initialization (e.g., Iceberg schema evolution). self._notify_first_input(bundle) # Submit the task as a normal Ray task. + dynamic_ray_remote_args = self._get_dynamic_ray_remote_args(input_bundle=bundle) ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, + rerun_policy=_task_rerun_policy(dynamic_ray_remote_args), ) - - dynamic_ray_remote_args = self._get_dynamic_ray_remote_args(input_bundle=bundle) dynamic_ray_remote_args["name"] = self.name logical_usage = ExecutionResources.from_resource_dict(dynamic_ray_remote_args) resource_request = task_resource_dict(dynamic_ray_remote_args) @@ -332,3 +336,19 @@ def all_inputs_done(self): "to increase the number of concurrent tasks by configuring " "`override_num_blocks` earlier in the pipeline." ) + + +def _task_rerun_policy(ray_remote_args: Dict[str, Any]) -> RerunPolicy: + """The errors on which Ray reruns a task with ``ray_remote_args``. + + ``cached_remote_fn`` defaults ``max_retries`` to -1 (unlimited); with 0, Ray + reruns on nothing. Otherwise it also reruns on ``RaySystemError``, which + ``cached_remote_fn`` adds to ``retry_exceptions``. + """ + if ray_remote_args.get("max_retries", -1) == 0: + return RerunPolicy.from_retry_exceptions(False) + retried = ray_remote_args.get("retry_exceptions", False) + # A copy, since adding ``RaySystemError`` appends to a list in place. + args = {"retry_exceptions": list(retried) if isinstance(retried, list) else retried} + _add_system_error_to_retry_exceptions(args) + return RerunPolicy.from_retry_exceptions(args["retry_exceptions"]) diff --git a/python/ray/data/_internal/output_buffer.py b/python/ray/data/_internal/output_buffer.py index 17a4e536d792..d347407844cb 100644 --- a/python/ray/data/_internal/output_buffer.py +++ b/python/ray/data/_internal/output_buffer.py @@ -111,6 +111,10 @@ def finalize(self) -> None: assert not self._finalized self._finalized = True + def has_buffered_rows(self) -> bool: + """Whether rows added since the last output block are waiting.""" + return self._buffer.num_rows() > 0 + def _exceeded_buffer_row_limit(self) -> bool: if self._output_block_size_option.disable_block_shaping: return False diff --git a/python/ray/data/_internal/util.py b/python/ray/data/_internal/util.py index 1aadb7e9ec8a..e4072a2cc81d 100644 --- a/python/ray/data/_internal/util.py +++ b/python/ray/data/_internal/util.py @@ -23,6 +23,7 @@ Iterator, List, Optional, + Sequence, Tuple, TypeVar, Union, @@ -1657,6 +1658,15 @@ def _annotate_exception_with_retry_context( return exc +def is_retryable_error(error_str: str, match: Optional[Sequence[str]]) -> bool: + """Whether ``iterate_with_retry`` retries an error formatted as ``error_str``. + + ``match`` holds patterns, each checked as a substring, then as a regex. + ``None`` retries any error. + """ + return match is None or any(matches_error(pattern, error_str) for pattern in match) + + def iterate_with_retry( iterable_factory: Callable[[], Iterable], description: str, @@ -1707,10 +1717,7 @@ def iterate_with_retry( return except Exception as e: error_str = format_exception(e, include_cause=unwrap_cause) - is_retryable = match is None or any( - matches_error(pattern, error_str) for pattern in match - ) - if is_retryable and attempt + 1 < max_attempts: + if is_retryable_error(error_str, match) and attempt + 1 < max_attempts: # Retry with binary expoential backoff with random jitter. backoff = min((2 ** (attempt + 1)), max_backoff_s) * random.random() total_backoff_s += backoff diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 20f7c3f3a744..f5338ec0f88d 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -860,9 +860,12 @@ class DataContext: max_errored_blocks: Max number of blocks that are allowed to have errors, unlimited if negative. This option allows application-level exceptions in block processing tasks. These exceptions may be caused by UDFs (e.g., due to - corrupted data samples) or IO errors. Data in the failed blocks are dropped. - This option can be useful to prevent a long-running job from failing due to - a small number of bad blocks. + corrupted data samples) or IO errors. A failed map task keeps the rows it + produced before the error and drops the rest of its input. While the + task can still be rerun (``retried_map_errors``, or Ray's ``max_retries`` + with ``retry_exceptions``), a failed attempt emits no rows, so the rerun + produces each row once. This option can be useful to prevent a + long-running job from failing due to a small number of bad blocks. log_internal_stack_trace: Whether to write the full Ray Data/Ray Core internal code stack frames to the Ray Data log file when logging a user-code error. These internal frames are always omitted from diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index efa89eb64515..3e64d1e8b7aa 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -775,6 +775,108 @@ def test_read_mcap_max_errored_blocks_skips_a_corrupt_recording( assert "2.mcap" in str(error.value) +def write_folder_with_a_corrupt_chunk(tmp_path) -> str: + """Write ``a.mcap``, ``b.mcap`` and ``c.mcap`` (three messages each) into one + folder, with ``b.mcap``'s only chunk zeroed: its summary reads, its messages + do not. Returns the folder.""" + from ray.data.tests.unit.datasource_v2.mcap_testing import write_corrupt_chunk + + folder = os.path.join(tmp_path, "drive") + os.makedirs(folder) + for name in ("a", "c"): + create_test_mcap_file( + os.path.join(folder, f"{name}.mcap"), + [{"topic": "/imu", "data": {"i": i}, "log_time": i} for i in range(3)], + ) + a = os.path.join(folder, "a.mcap") + write_corrupt_chunk(os.path.join(folder, "b.mcap"), a, 0) + return folder + + +def test_read_mcap_max_errored_blocks_keeps_files_sharing_a_task_with_a_corrupt_chunk( + ray_start_regular_shared, tmp_path, datasource_v2 +): + """With ``max_errored_blocks=1``, a recording whose only chunk is corrupt is + skipped and every other file is read, including files in the same read task.""" + folder = write_folder_with_a_corrupt_chunk(tmp_path) + ray.data.DataContext.get_current().max_errored_blocks = 1 + + rows = ray.data.read_mcap(folder, include_paths=True).take_all() + + # V2 packs the three small files into one read task; legacy splits them + # over two tasks on this one-CPU cluster, a.mcap with b.mcap. + files = sorted(os.path.basename(row["path"]) for row in rows) + assert files == ["a.mcap"] * 3 + ["c.mcap"] * 3 + + +def test_read_mcap_max_errored_blocks_loses_only_the_rows_of_corrupt_chunks( + ray_start_regular_shared, tmp_path, datasource_v2 +): + """With ``max_errored_blocks=1``, V2 loses only the rows of a file's two + corrupt chunks. Their read task raises once for both, so one allowed error is + enough.""" + if not datasource_v2: + pytest.skip("the legacy reader stops at a file's first corrupt chunk") + from ray.data.tests.unit.datasource_v2.mcap_testing import ( + CHUNKED_FILE_MESSAGES, + round_robin_messages, + write_corrupt_chunk, + write_mcap, + ) + + folder = os.path.join(tmp_path, "drive") + os.makedirs(folder) + good = os.path.join(folder, "a.mcap") + # One message per chunk; b.mcap is a.mcap with chunks 2 and 6 zeroed. + write_mcap(good, round_robin_messages(CHUNKED_FILE_MESSAGES)) + once = write_corrupt_chunk(os.path.join(tmp_path, "once.mcap"), good, 2) + write_corrupt_chunk(os.path.join(folder, "b.mcap"), once, 6) + ray.data.DataContext.get_current().max_errored_blocks = 1 + + rows = ray.data.read_mcap(folder, include_paths=True).take_all() + + seqs = sorted((os.path.basename(row["path"]), row["data"]["seq"]) for row in rows) + expected = [("a.mcap", i) for i in range(CHUNKED_FILE_MESSAGES)] + expected += [("b.mcap", i) for i in range(CHUNKED_FILE_MESSAGES) if i not in (2, 6)] + assert seqs == expected + + +def test_read_mcap_max_errored_blocks_loses_only_a_message_that_is_not_json( + ray_start_regular_shared, tmp_path, datasource_v2 +): + """With ``max_errored_blocks=1``, V2 loses only the row of a message whose + payload is not valid JSON and keeps the rest of its file. With + ``max_errored_blocks=0``, the read fails naming the message's ``row_id``.""" + if not datasource_v2: + pytest.skip("the legacy reader keeps a payload that is not JSON as bytes") + from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + message_row_id, + ) + from ray.data.tests.unit.datasource_v2.mcap_testing import ( + CHUNKED_FILE_MESSAGES, + summary_of, + write_invalid_json, + ) + + bad = 4 + # One message per chunk, so ``bad`` is the only message of chunk ``bad``. + path = write_invalid_json(os.path.join(tmp_path, "bad.mcap"), bad={bad}) + bad_offset = summary_of(path).chunk_indexes[bad].chunk_start_offset + ctx = ray.data.DataContext.get_current() + + ctx.max_errored_blocks = 1 + rows = ray.data.read_mcap(path).take_all() + + seqs = sorted(row["data"]["seq"] for row in rows) + assert seqs == [i for i in range(CHUNKED_FILE_MESSAGES) if i != bad] + + ctx.max_errored_blocks = 0 + with pytest.raises(Exception) as error: + ray.data.read_mcap(path).take_all() + + assert message_row_id(path, bad_offset, 0) in str(error.value) + + def test_read_mcap_lists_files_in_parallel( ray_start_regular_shared, tmp_path, datasource_v2 ): diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index e7f520fa1490..03085ba79fb9 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -38,6 +38,8 @@ from ray.data._internal.execution.interfaces.ref_bundle import BlockEntry, RefBundle from ray.data._internal.execution.interfaces.task_context import TaskContext from ray.data._internal.execution.operators.actor_pool_map_operator import ( + ActorPoolMapOperator, + _actor_task_rerun_policy, _ActorPool, _MapWorker, ) @@ -1975,6 +1977,34 @@ def test_merge_ray_remote_args_op_wins_on_collision(restore_data_context): assert merged["label_selector"] == {"subcluster": "val", "node": "X"} +@pytest.mark.parametrize( + "actor_args, reruns", + [ + # By default the operator gives actors unlimited task retries. + ({}, True), + # With ``max_restarts=0`` it leaves ``max_task_retries`` at Ray's 0. + ({"max_restarts": 0}, False), + ({"max_task_retries": 0}, False), + ], +) +def test_actor_task_rerun_policy_follows_max_task_retries( + restore_data_context, actor_args, reruns +): + """An actor task counts as rerun on ``actor_task_retry_on_errors`` only when + its actor's ``max_task_retries`` lets Ray rerun it, so a task that won't + rerun emits its buffered rows before an error the job skips.""" + context = DataContext.get_current() + context.actor_task_retry_on_errors = [ValueError] + actor_args = ActorPoolMapOperator._apply_default_remote_args(actor_args, context) + task_args = ActorPoolMapOperator._apply_default_actor_task_remote_args( + None, context + ) + + policy = _actor_task_rerun_policy(task_args, actor_args) + + assert policy.may_rerun(ValueError("bad input")) is reruns + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_map_transformer.py b/python/ray/data/tests/test_map_transformer.py index 3ac104a57ac2..34acb1f7c67e 100644 --- a/python/ray/data/tests/test_map_transformer.py +++ b/python/ray/data/tests/test_map_transformer.py @@ -2,21 +2,33 @@ import time import weakref from collections import deque +from dataclasses import dataclass +from typing import Type import pandas as pd +import pyarrow as pa import pytest -from ray.data._internal.execution.interfaces.task_context import TaskContext +from ray.data._internal.execution.interfaces.task_context import ( + RerunPolicy, + TaskContext, +) from ray.data._internal.execution.operators.map_transformer import ( BatchMapTransformFn, + BlockMapTransformFn, MapTransformer, TransformClock, ) +from ray.data._internal.execution.operators.task_pool_map_operator import ( + _task_rerun_policy, +) from ray.data._internal.output_buffer import OutputBlockSizeOption from ray.data._internal.planner.plan_udf_map_op import ( _generate_transform_fn_for_map_batches, ) from ray.data.block import BlockAccessor, DataBatch +from ray.data.context import DataContext +from ray.exceptions import RaySystemError def _create_chained_transformer(udf, n): @@ -293,6 +305,193 @@ def make_input_blocks(): ) +def _failing_block_transformer(tables_before_error, error_type=ValueError): + """A one-stage transformer whose function yields ``tables_before_error`` + one-row tables, then raises ``error_type("bad input")``. Its output blocks + are 1 MiB, so the tables share one buffered block.""" + + def fn(blocks, ctx): + for i in range(tables_before_error): + yield pa.table({"x": [i]}) + raise error_type("bad input") + + return MapTransformer( + [ + BlockMapTransformFn( + fn, + output_block_size_option=OutputBlockSizeOption.of( + target_max_block_size=1024 * 1024 + ), + ) + ] + ) + + +# A task that Ray does not rerun, for the tests of ``_map_task``'s own retries. +NO_RAY_RERUN = RerunPolicy(ray_reruns_any_error=False) + + +def _outputs(transformer, rerun_policy=NO_RAY_RERUN): + """The transformer's output blocks for one one-row input block, in a task + that runs again as ``rerun_policy`` says. + + A generator, so the task context is current while the blocks are drawn, as + ``_map_task`` sets it. + """ + ctx = TaskContext(task_idx=0, op_name="test", rerun_policy=rerun_policy) + with TaskContext.current(ctx): + yield from transformer.apply_transform( + iter([pa.table({"x": [0]})]), ctx, clock=TransformClock() + ) + + +@pytest.fixture +def skip_errors(monkeypatch): + """Let the job skip an errored block (``max_errored_blocks=1``); returns the + patched ``DataContext``.""" + context = DataContext.get_current() + monkeypatch.setattr(context, "max_errored_blocks", 1) + return context + + +@dataclass(frozen=True) +class BufferedRowsCase: + """A map task that raises after buffering rows 0 and 1, and whether it emits + them as a last block before the error.""" + + rerun_policy: RerunPolicy + emits_rows: bool + error_type: Type[Exception] = ValueError + # The job's ``DataContext`` settings; 1 lets the job skip the error. + max_errored_blocks: int = 1 + actor_task_retry_on_errors: bool = False + + +@pytest.mark.parametrize( + "case", + [ + pytest.param( + BufferedRowsCase(NO_RAY_RERUN, emits_rows=True), id="skipped-error" + ), + pytest.param( + BufferedRowsCase(NO_RAY_RERUN, emits_rows=False, max_errored_blocks=0), + id="error-fails-the-job", + ), + # ``None`` patterns retry any error, as ``retried_map_errors=True`` sets them. + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=None), + emits_rows=False, + ), + id="map-task-retries-any-error", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries( + attempts_left=1, patterns=["connection reset"] + ), + emits_rows=True, + ), + id="no-map-retry-pattern-matches", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["bad input"]), + emits_rows=False, + ), + id="a-map-retry-pattern-matches", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=0, patterns=["bad input"]), + emits_rows=True, + ), + id="last-map-attempt", + ), + pytest.param( + BufferedRowsCase( + RerunPolicy.from_retry_exceptions([ValueError]), emits_rows=False + ), + id="ray-retries-the-error", + ), + pytest.param( + BufferedRowsCase( + RerunPolicy.from_retry_exceptions([KeyError]), emits_rows=True + ), + id="ray-retries-other-errors", + ), + # Ray Data's default for a task: unlimited retries, ``RaySystemError`` + # among them. With ``max_retries=0`` Ray reruns on nothing. + pytest.param( + BufferedRowsCase( + _task_rerun_policy({}), emits_rows=False, error_type=RaySystemError + ), + id="system-error-ray-retries", + ), + pytest.param( + BufferedRowsCase( + _task_rerun_policy({"max_retries": 0}), + emits_rows=True, + error_type=RaySystemError, + ), + id="system-error-no-ray-retries", + ), + # ``actor_task_retry_on_errors`` reaches actor tasks only, through their + # ``retry_exceptions``. + pytest.param( + BufferedRowsCase( + _task_rerun_policy({}), emits_rows=True, actor_task_retry_on_errors=True + ), + id="task-with-actor-retry-setting", + ), + ], +) +def test_a_failing_task_emits_its_buffered_rows_only_without_a_rerun(case, monkeypatch): + """A map task that raises after buffering rows emits them as one last block + before the error only when the job skips the error and neither Ray nor + ``_map_task`` runs the task again.""" + context = DataContext.get_current() + monkeypatch.setattr(context, "max_errored_blocks", case.max_errored_blocks) + monkeypatch.setattr( + context, "actor_task_retry_on_errors", case.actor_task_retry_on_errors + ) + transformer = _failing_block_transformer( + tables_before_error=2, error_type=case.error_type + ) + + out = _outputs(transformer, case.rerun_policy) + + if case.emits_rows: + rows = BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() + assert rows == [0, 1] + with pytest.raises(case.error_type, match="bad input"): + next(out) + + +def test_a_task_without_retry_settings_raises_without_a_partial_block(skip_errors): + """A task whose ``TaskContext`` does not state its retry settings may be + rerun, so its buffered rows are not emitted even when the job skips errors.""" + ctx = TaskContext(task_idx=0, op_name="test") + transformer = _failing_block_transformer(tables_before_error=2) + + with TaskContext.current(ctx): + out = transformer.apply_transform( + iter([pa.table({"x": [0]})]), ctx, clock=TransformClock() + ) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_a_transform_failing_before_any_output_emits_no_block(skip_errors): + """A transform that raises before producing a row raises at once, without + an empty block.""" + out = _outputs(_failing_block_transformer(tables_before_error=0)) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_streaming_executor_errored_blocks.py b/python/ray/data/tests/test_streaming_executor_errored_blocks.py index 1412ac087e58..cbf1a0729234 100644 --- a/python/ray/data/tests/test_streaming_executor_errored_blocks.py +++ b/python/ray/data/tests/test_streaming_executor_errored_blocks.py @@ -52,6 +52,106 @@ def map_func(row): assert stats.extra_metrics["num_tasks_failed"] == num_errored_blocks +def test_map_retry_after_partial_output_returns_every_row_once(restore_data_context): + """A task that fails once after producing rows and is retried + (``retried_map_errors``) returns every row exactly once, also when the job + skips errors.""" + ctx = ray.data.DataContext.get_current() + ctx.retried_map_errors = ["transient"] + ctx.max_map_retries = 1 + ctx.max_errored_blocks = 1 + failed = [] + + def flaky(batch): + # Fails once, at the third batch, after two batches are buffered. + if batch["id"][0] == 4 and not failed: + failed.append(True) + raise ValueError("transient") + return batch + + ds = ray.data.range(10, override_num_blocks=1).map_batches(flaky, batch_size=2) + + assert sorted(row["id"] for row in ds.take_all()) == list(range(10)) + + +def test_ray_task_retry_after_partial_output_returns_every_row_once( + restore_data_context, tmp_path +): + """A task that fails once after producing rows and that Ray reruns + (``retry_exceptions``) returns every row exactly once, also when the job + skips errors.""" + ctx = ray.data.DataContext.get_current() + ctx.max_errored_blocks = 1 + # Ray reruns the task in a fresh worker, so "failed once" lives in a file. + failed = tmp_path / "failed" + + def flaky(batch): + # Fails once, at the third batch, after two batches are buffered. + if batch["id"][0] == 4 and not failed.exists(): + failed.touch() + raise ValueError("transient") + return batch + + ds = ray.data.range(10, override_num_blocks=1).map_batches( + flaky, batch_size=2, retry_exceptions=[ValueError] + ) + + assert sorted(row["id"] for row in ds.take_all()) == list(range(10)) + + +NUM_ROWS = 10 # rows of the one-block datasets below, one read task +FAILING_ROW = 7 # the row whose UDF call raises + + +def fail_at_failing_row(batch): + """A batch UDF that raises on the batch holding ``FAILING_ROW``.""" + if FAILING_ROW in batch["id"]: + raise ValueError(f"bad row {FAILING_ROW}") + return batch + + +def fail_row_at_failing_row(row): + """A row UDF that raises on ``FAILING_ROW``.""" + if row["id"] == FAILING_ROW: + raise ValueError(f"bad row {FAILING_ROW}") + return row + + +def test_a_skipped_error_keeps_the_rows_a_task_produced_before_it(restore_data_context): + """A one-task ``map_batches`` that fails at the batch holding row 7 keeps the + rows of the batches before it when the job skips the error.""" + ctx = ray.data.DataContext.get_current() + # Any limit other than 0 skips the error. -1 (no limit) also catches a check + # that treats only a positive limit as skipping. + ctx.max_errored_blocks = -1 + + ds = ray.data.range(NUM_ROWS, override_num_blocks=1).map_batches( + fail_at_failing_row, batch_size=2 + ) + rows = sorted(row["id"] for row in ds.take_all()) + + # Batches of two: [6, 7] fails, so the batches before it, rows 0 to 5, stay. + assert rows == list(range(6)) + + +def test_a_skipped_error_keeps_the_rows_a_fused_batcher_holds(restore_data_context): + """In one fused task, ``map`` fails at row 7 while the downstream + ``map_batches`` batcher holds rows 4 to 6 of an unfinished batch of four. + The batcher hands them on as a short batch, so every row before the error + is kept.""" + ctx = ray.data.DataContext.get_current() + ctx.max_errored_blocks = 1 + + ds = ( + ray.data.range(NUM_ROWS, override_num_blocks=1) + .map(fail_row_at_failing_row) + .map_batches(lambda batch: batch, batch_size=4) + ) + rows = sorted(row["id"] for row in ds.take_all()) + + assert rows == list(range(FAILING_ROW)) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 3e4abd8fe0d2..a7edbdb28b47 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -9,7 +9,7 @@ import json import shutil from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional import pyarrow as pa from pyarrow.fs import LocalFileSystem @@ -111,7 +111,9 @@ def mixed_encoding_messages(n=6): return messages -def write_mcap_without_summary_schemas(path): +def write_mcap_without_summary_schemas( + path, b_declared_in_chunk_1=False, message_indexes=True +): """Write two JSON channels with two schemas, one message per chunk. The schema records are written only inside the chunks @@ -120,6 +122,10 @@ def write_mcap_without_summary_schemas(path): chunk 0 1 2 3 4 5 topic /a /b /a /b /a /b + + Both schemas are declared in chunk 0, or with ``b_declared_in_chunk_1``, + ``other_schema`` only in chunk 1. Without ``message_indexes``, the chunk + indexes do not say which channels a chunk holds, so a read opens every chunk. """ from mcap.writer import CompressionType, IndexType, Writer @@ -128,20 +134,28 @@ def write_mcap_without_summary_schemas(path): stream, chunk_size=1, compression=CompressionType.ZSTD, - index_types=IndexType.ALL, + index_types=IndexType.ALL if message_indexes else IndexType.CHUNK, repeat_schemas=False, ) writer.start(profile="", library="ray-test") channels = {} - for topic, name in (("/a", "test_schema"), ("/b", "other_schema")): + + def declare(topic, name): schema_id = writer.register_schema( name=name, encoding="jsonschema", data=b'{"type": "object"}' ) channels[topic] = writer.register_channel( schema_id=schema_id, topic=topic, message_encoding="json" ) + + declare("/a", "test_schema") + if not b_declared_in_chunk_1: + declare("/b", "other_schema") for i in range(6): topic = "/a" if i % 2 == 0 else "/b" + if topic not in channels: + # Chunk 0 closed after its message, so this lands in chunk 1. + declare(topic, "other_schema") writer.add_message( channel_id=channels[topic], log_time=BASE_TIME + i * STEP, @@ -151,6 +165,38 @@ def write_mcap_without_summary_schemas(path): writer.finish() +UNDECLARED_CHANNEL = 99 # the channel id ``write_undeclared_channel`` never declares + + +def write_undeclared_channel(path): + """Write one chunk that holds a message on ``/a``, then a message on channel + ``UNDECLARED_CHANNEL``, which no record of the file declares. + + The chunk decodes, but the reader cannot place its second message. + """ + from mcap.writer import CompressionType, IndexType, Writer + + with open(path, "wb") as stream: + writer = Writer( + stream, compression=CompressionType.ZSTD, index_types=IndexType.ALL + ) + writer.start(profile="", library="ray-test") + schema_id = writer.register_schema( + name="test_schema", encoding="jsonschema", data=b'{"type": "object"}' + ) + channel_id = writer.register_channel( + schema_id=schema_id, topic="/a", message_encoding="json" + ) + for i, channel in enumerate([channel_id, UNDECLARED_CHANNEL]): + writer.add_message( + channel_id=channel, + log_time=BASE_TIME + i * STEP, + publish_time=BASE_TIME + i * STEP, + data=json.dumps({"seq": i}).encode(), + ) + writer.finish() + + MIXED_TOPICS = ("/a", "/b", "/raw", "/ΓΌ") # channels of ``write_mixed_channels`` SHARED_SCHEMA = b"float64 x\n" * 30 # the definition /a and /b both use @@ -316,6 +362,22 @@ def write_corrupt_chunk(path: str, source: str, index: int) -> str: return path +NOT_JSON = b"not json" # a payload on a JSON-encoded topic that does not decode + + +def write_invalid_json(path: str, bad: Iterable[int], **options: Any) -> str: + """Write ``CHUNKED_FILE_MESSAGES`` ``round_robin_messages`` to ``path``, with + the payloads of the messages whose ``seq`` is in ``bad`` set to ``NOT_JSON``. + + ``options`` go to ``write_mcap``; by default each message is its own chunk. + """ + messages = round_robin_messages(CHUNKED_FILE_MESSAGES) + for seq in bad: + messages[seq]["data"] = NOT_JSON + write_mcap(path, messages, **options) + return path + + # Where a chunk record's compression name starts: after the opcode (1 byte), # the record length (8), the message start and end times (8 each), the # uncompressed size (8), its CRC (4) and the name's length (4). @@ -411,8 +473,9 @@ def record_parsed_chunks(monkeypatch) -> List[int]: real_read_chunks = mcap_chunks.read_chunks offsets: List[int] = [] - def read_chunks(f, chunk_indexes, reads): - for chunk_index, chunk in real_read_chunks(f, chunk_indexes, reads): + def read_chunks(f, chunk_indexes, reads, on_corrupt=None): + chunks = real_read_chunks(f, chunk_indexes, reads, on_corrupt) + for chunk_index, chunk in chunks: offsets.append(chunk_index.chunk_start_offset) yield chunk_index, chunk diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index f1b481b01ff7..e59c479ace69 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -23,6 +23,7 @@ from ray.data.tests.unit.datasource_v2.mcap_testing import ( CHUNKED_FILE_MESSAGES, round_robin_messages, + write_corrupt_chunk_header, write_cut_short, write_mcap, write_mcap_without_summary_schemas, @@ -358,6 +359,32 @@ def test_read_chunks_fetches_a_read_when_its_first_chunk_is_needed(tmp_path): assert reads == [0, 1, 2, 3] +def test_read_chunks_hands_a_record_that_does_not_parse_to_on_corrupt(tmp_path): + """With ``on_corrupt``, a chunk record that does not parse is handed to it + and yielded as ``None``, and the chunks around it in the same read are still + parsed. Without ``on_corrupt``, the error raises.""" + source = os.path.join(tmp_path, "chunked.mcap") + write_mcap(source, round_robin_messages(CHUNKED_FILE_MESSAGES)) + path = write_corrupt_chunk_header(os.path.join(tmp_path, "bad.mcap"), source, 4) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + chunk_indexes = sorted(summary.chunk_indexes, key=lambda c: c.chunk_start_offset) + expected = [read_chunk(CountingFile(path), c) for c in chunk_indexes[:4]] + expected += [None] + [read_chunk(CountingFile(path), c) for c in chunk_indexes[5:]] + reported = [] + + f = CountingFile(path) + chunks = list( + read_chunks(f, chunk_indexes, ChunkReads(), lambda c, e: reported.append(c)) + ) + + assert [chunk for _, chunk in chunks] == expected + assert reported == [chunk_indexes[4]] + assert f.reads == 1 + with pytest.raises(UnicodeDecodeError): + list(read_chunks(CountingFile(path), chunk_indexes, ChunkReads())) + + def test_buffered_scan_reads_in_large_blocks(tmp_path): """A linear scan through ``buffered`` yields the same records with one read per buffer's worth of the file, and closing the buffer leaves the file open.""" diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index c90fd47d3dbf..17bfa93f7ba4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -8,11 +8,18 @@ import importlib.util import json import os +import random +import time +import tracemalloc +from functools import partial import pyarrow as pa import pytest -from ray.data._internal.datasource_v2.formats.mcap import mcap_message_rows +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_message_rows, + mcap_reader, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) @@ -29,6 +36,7 @@ MCAPSelection, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import message_row_id from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data.tests.unit.datasource_v2.mcap_testing import ( BASE_TIME, @@ -36,21 +44,28 @@ MIXED_TOPICS, SHARED_SCHEMA, STEP, + UNDECLARED_CHANNEL, SeekLog, as_read_task, chunks_read, infer_schema, list_manifests, mixed_encoding_messages, + pack, read_all, record_parsed_chunks, round_robin_messages, scanner_for, selected_messages, summary_of, + write_corrupt_chunk, + write_corrupt_chunk_header, write_cut_short, + write_invalid_json, write_mcap, + write_mcap_without_summary_schemas, write_mixed_channels, + write_undeclared_channel, ) pytestmark = pytest.mark.skipif( @@ -128,6 +143,557 @@ def test_reading_a_file_cut_short_fails_naming_it(tmp_path, chunked_file): read_all(datasource, manifest) +CORRUPT_CHUNK = 4 # the chunk, and so the message, a bad file loses + + +def write_task_files(tmp_path, bad, corrupt=write_corrupt_chunk): + """Write ``a.mcap``, ``b.mcap`` and ``c.mcap`` (nine messages, one per chunk) + and corrupt chunk ``CORRUPT_CHUNK`` of the files named in ``bad`` with + ``corrupt``; return the three paths.""" + paths = [os.path.join(tmp_path, f"{name}.mcap") for name in "abc"] + for path in paths: + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + if os.path.basename(path) in bad: + corrupt_in_place(path, CORRUPT_CHUNK, corrupt) + return paths + + +def corrupt_in_place(path, index, corrupt=write_corrupt_chunk): + """Corrupt chunk ``index`` of the MCAP file ``path`` with ``corrupt``.""" + corrupt(path + ".tmp", path, index) + os.replace(path + ".tmp", path) + + +def read_one_task(paths, max_errored_blocks=1, **options): + """Read ``paths`` as one read task under ``max_errored_blocks``, with the + datasource ``options``; return the tables it yields and its error.""" + datasource = MCAPDatasourceV2(paths, include_paths=True, **options) + return read_task_with(scanner_for(datasource), datasource, max_errored_blocks) + + +def read_task_with(scanner, datasource, max_errored_blocks=1): + """Read the files of ``datasource`` as one read task with ``scanner``, under + ``max_errored_blocks``; return the tables it yields and its error.""" + from ray.data.context import DataContext + + DataContext.get_current().max_errored_blocks = max_errored_blocks + # preserve_order keeps the files in the order given; the listing threads + # otherwise finish in any order. + task = as_read_task(*list_manifests(datasource, preserve_order=True)) + tables = [] + with pytest.raises(Exception) as error: + for table in scanner.create_reader().read(task): + tables.append(table) + return tables, error.value + + +def rows_of(tables): + """The ``(path, seq)`` of each row of ``tables``, in order.""" + return [ + (row["path"], row["data"]["seq"]) + for table in tables + for row in table.select(["path", "data"]).to_pylist() + ] + + +def file_rows(path, skipped=()): + """The ``(path, seq)`` rows of a file of ``round_robin_messages``, in order, + without the messages of the ``skipped`` chunks (one message per chunk).""" + return [(path, seq) for seq in range(CHUNKED_FILE_MESSAGES) if seq not in skipped] + + +def chunk_offset(path, index): + """The byte offset of chunk ``index`` of the MCAP file ``path``.""" + return summary_of(path).chunk_indexes[index].chunk_start_offset + + +@pytest.fixture +def restore_max_errored_blocks(): + """Put ``DataContext.max_errored_blocks`` back after the test.""" + from ray.data.context import DataContext + + saved = DataContext.get_current().max_errored_blocks + yield + DataContext.get_current().max_errored_blocks = saved + + +CORRUPTIONS = {"payload": write_corrupt_chunk, "header": write_corrupt_chunk_header} + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("corrupt", CORRUPTIONS.values(), ids=CORRUPTIONS.keys()) +def test_a_corrupt_chunk_costs_only_its_own_rows( + tmp_path, corrupt, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a task whose middle file has a corrupt chunk + yields every other row of the three files, then raises naming that file and + chunk. That holds for a payload that does not decompress and a record that + does not parse, in log-time and in file order.""" + a, b, c = write_task_files(tmp_path, bad={"b.mcap"}, corrupt=corrupt) + + tables, error = read_one_task([a, b, c], log_time_order=log_time_order) + + skipped = {CORRUPT_CHUNK} + assert rows_of(tables) == file_rows(a) + file_rows(b, skipped) + file_rows(c) + assert isinstance(error, ValueError) + assert str(error).startswith( + f"Skipped 1 corrupt chunk at byte offset {chunk_offset(b, CORRUPT_CHUNK)} " + f"of MCAP file {b!r}: " + ) + + +CORRUPT_CHUNKS = 16 # chunks of a file whose every chunk is corrupt +CHUNK_BYTES = 1 << 20 # each chunk's payload, random so it does not compress + + +def test_a_file_of_corrupt_chunks_keeps_one_error( + tmp_path, monkeypatch, restore_max_errored_blocks +): + """A file whose every chunk is corrupt loses all its rows, and one error + names every chunk. The read keeps one chunk's error rather than one per + chunk, so its peak does not grow with the number of corrupt chunks.""" + path = os.path.join(tmp_path, "b.mcap") + rng = random.Random(0) + write_mcap( + path, + [ + { + "topic": "/a", + "log_time": BASE_TIME + i, + "data": rng.randbytes(CHUNK_BYTES), + } + for i in range(CORRUPT_CHUNKS) + ], + message_encoding="cdr", + ) + offsets = [chunk_offset(path, i) for i in range(CORRUPT_CHUNKS)] + for index in range(CORRUPT_CHUNKS): + corrupt_in_place(path, index) + # One chunk per read, so the read's own buffers hold one chunk at a time. + monkeypatch.setattr( + mcap_reader, + "SelectedMessageReader", + partial(SelectedMessageReader, chunk_reads=ONE_CHUNK_PER_READ), + ) + + tracemalloc.start() + try: + tables, error = read_one_task([path]) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert rows_of(tables) == [] + assert str(error).startswith( + f"Skipped {CORRUPT_CHUNKS} corrupt chunks at byte offsets " + f"{', '.join(map(str, offsets))} of MCAP file {path!r}: ZstdError: " + ) + # The read's buffers hold a few chunks at once. An error's traceback holds + # its chunk's bytes, so keeping every error would hold all of them. + assert peak < CORRUPT_CHUNKS // 2 * CHUNK_BYTES + + +BAD_JSON = 4 # the message whose payload is not valid JSON + + +def message_row_id_of(path, seq, one_chunk): + """The ``row_id`` of message ``seq`` of a file of ``write_invalid_json``, + written with one chunk per message or with every message in one chunk.""" + if one_chunk: + return message_row_id(path, chunk_offset(path, 0), seq) + return message_row_id(path, chunk_offset(path, seq), 0) + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("one_chunk", [False, True], ids=["own_chunk", "mid_chunk"]) +def test_a_message_that_is_not_json_costs_only_its_own_row( + tmp_path, one_chunk, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a message that is not valid JSON in the middle + file costs only its own row, whether it fills a chunk or sits between good + messages of one chunk. The kept rows' columns line up, and the task then + raises naming the message.""" + a, b, c = write_task_files(tmp_path, bad=set()) + # A chunk size past the file's size puts every message in one chunk. + write_invalid_json(b, bad={BAD_JSON}, chunk_size=1 << 20 if one_chunk else 1) + + tables, error = read_one_task( + [a, b, c], log_time_order=log_time_order, include_row_id=True + ) + + assert rows_of(tables) == file_rows(a) + file_rows(b, {BAD_JSON}) + file_rows(c) + rows = [row for table in tables for row in table.to_pylist() if row["path"] == b] + seqs = [row["data"]["seq"] for row in rows] + assert [row["log_time"] for row in rows] == [BASE_TIME + s * STEP for s in seqs] + expected_ids = [message_row_id_of(b, s, one_chunk) for s in seqs] + assert [row[ROW_ID_COLUMN] for row in rows] == expected_ids + row_id = message_row_id_of(b, BAD_JSON, one_chunk) + assert str(error).startswith( + f"Skipped 1 message that is not valid JSON (row id {row_id}) of MCAP file " + f"{b!r}: InvalidJsonPayload: {row_id}: message on JSON-encoded topic" + ) + + +def test_one_error_names_the_corrupt_chunks_and_the_messages_that_are_not_json( + tmp_path, restore_max_errored_blocks +): + """A file with a corrupt chunk and two messages that are not valid JSON loses + only those three rows, and one error names the chunk, counts the messages + and names the first of them, with the chunk's error.""" + a, b, c = write_task_files(tmp_path, bad=set()) + write_invalid_json(b, bad={5, 6}) + corrupt_in_place(b, 2) + + tables, error = read_one_task([a, b, c]) + + assert rows_of(tables) == file_rows(a) + file_rows(b, {2, 5, 6}) + file_rows(c) + first_row_id = message_row_id_of(b, 5, False) + assert str(error).startswith( + f"Skipped 1 corrupt chunk at byte offset {chunk_offset(b, 2)} and " + f"2 messages that are not valid JSON (first: row id {first_row_id}) of " + f"MCAP file {b!r}: ZstdError: " + ) + + +LIMIT = 4 # rows a limited read asks for + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("index", [True, False], ids=["indexed", "unindexed"]) +def test_a_skipped_message_does_not_count_against_the_limit( + tmp_path, index, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a limited read of a file whose first two + messages are not valid JSON still yields ``LIMIT`` rows, the next ones in + the file, with and without a chunk index, in log-time and in file order.""" + path = write_invalid_json(os.path.join(tmp_path, "b.mcap"), bad={0, 1}, index=index) + datasource = MCAPDatasourceV2( + [path], include_paths=True, log_time_order=log_time_order + ) + + tables, error = read_task_with( + scanner_for(datasource).push_limit(LIMIT), datasource + ) + + assert rows_of(tables) == [(path, seq) for seq in range(2, 2 + LIMIT)] + assert str(error).startswith("Skipped 2 messages that are not valid JSON") + + +LATE_CDR = 5 # the message on a CDR-encoded topic, after five JSON messages + + +@pytest.mark.parametrize( + "target_block_size", [None, 1], ids=["one_block", "block_per_row"] +) +def test_a_file_that_fails_mid_way_keeps_the_rows_before_the_failure( + tmp_path, target_block_size, restore_max_errored_blocks +): + """When errors can be skipped, a file that message ``LATE_CDR`` fails keeps + the rows before it, whether they share one block or fill one each. The + task then raises the file's error.""" + planned_on = os.path.join(tmp_path, "json.mcap") + write_mcap(planned_on, round_robin_messages(3)) + path = os.path.join(tmp_path, "late_cdr.mcap") + messages = round_robin_messages(LATE_CDR + 1) + messages[LATE_CDR].update(topic="/cdr", data=b"\x00\x01") + write_mcap(path, messages, encodings={"/cdr": "cdr"}) + # Planned on a JSON-only file, so the read decodes JSON and the CDR + # message fails it. + planned = MCAPDatasourceV2([planned_on], include_paths=True) + scanner = dataclasses.replace( + scanner_for(planned), target_block_size=target_block_size + ) + + tables, error = read_task_with(scanner, MCAPDatasourceV2([path])) + + assert rows_of(tables) == [(path, seq) for seq in range(LATE_CDR)] + assert "topic '/cdr' is 'cdr'-encoded" in str(error) + + +def test_a_task_names_every_file_it_could_not_read_in_full( + tmp_path, restore_max_errored_blocks +): + """Two files with a corrupt chunk fail their task once, with one error that + names both, after every other row of the three files is read.""" + a, b, c = write_task_files(tmp_path, bad={"a.mcap", "c.mcap"}) + + tables, error = read_one_task([a, b, c], max_errored_blocks=-1) + + skipped = {CORRUPT_CHUNK} + expected = file_rows(a, skipped) + file_rows(b) + file_rows(c, skipped) + assert rows_of(tables) == expected + assert isinstance(error, ValueError) + assert "2 MCAP files of this read task could not be read in full" in str(error) + assert a in str(error) and c in str(error) + + +def test_a_task_fails_at_its_first_corrupt_chunk_when_errors_are_not_skipped( + tmp_path, restore_max_errored_blocks +): + """With ``max_errored_blocks=0`` the read fails anyway, so the task raises at + the corrupt chunk without reading the file after it.""" + a, b, c = write_task_files(tmp_path, bad={"b.mcap"}) + + tables, error = read_one_task([a, b, c], max_errored_blocks=0) + + # b's rows before the corrupt chunk were still being built into a table. + assert rows_of(tables) == file_rows(a) + assert type(error).__name__ == "ZstdError" + + +def test_an_error_reading_the_file_is_not_skipped_as_a_corrupt_chunk( + tmp_path, monkeypatch +): + """An ``OSError`` while a chunk is decoded, here reading back an earlier + chunk for a schema, raises even when corrupt chunks are skipped, so the + file's read is retried rather than the chunk dropped.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + + path = os.path.join(tmp_path, "schemas_in_chunks.mcap") + write_mcap_without_summary_schemas(path) + summary = summary_of(path) + # Chunks 2 to 5 declare no schema, so reading them reads chunk 0 back. + owned = {c.chunk_start_offset for c in summary.chunk_indexes[2:]} + + def read_chunk(f, chunk_index): + raise OSError("connection reset") + + monkeypatch.setattr(mcap_chunks, "read_chunk", read_chunk) + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + reported = [] + + with open(path, "rb") as f, pytest.raises(OSError, match="connection reset"): + for _ in reader.iter_chunks( + f, path, summary, owned, on_corrupt=lambda c, e: reported.append(c) + ): + pass + + assert reported == [] + + +def test_a_corrupt_chunk_read_back_for_a_schema_costs_only_itself(tmp_path): + """A task reading back for a schema skips an earlier corrupt chunk and finds + the schema in the next one, so its own chunk is not reported as corrupt.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean, b_declared_in_chunk_1=True) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + summary = summary_of(path) + # The task owns the /b chunks 3 and 5. Chunk 3 reads back through chunk 0, + # which is corrupt, to chunk 1, which declares other_schema. + owned = {summary.chunk_indexes[i].chunk_start_offset for i in (3, 5)} + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + reported = [] + + with open(path, "rb") as f: + selected = list( + reader.iter_chunks( + f, path, summary, owned, on_corrupt=lambda c, e: reported.append(c) + ) + ) + + assert [channel.topic for _, channel, _, _ in selected] == ["/b", "/b"] + assert all(schema and schema.name == "other_schema" for schema, *_ in selected) + assert reported == [] + + +def read_all_chunks(path, owned_from, on_corrupt): + """The selected messages of chunks ``owned_from`` onward of the MCAP file + ``path``, read with ``on_corrupt`` as one task reads them.""" + summary = summary_of(path) + owned = {c.chunk_start_offset for c in summary.chunk_indexes[owned_from:]} + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + with open(path, "rb") as f: + return list(reader.iter_chunks(f, path, summary, owned, on_corrupt=on_corrupt)) + + +def test_a_schema_only_a_corrupt_chunk_declares_fails_an_unskipped_read(tmp_path): + """Without ``on_corrupt``, a task that reads back for a schema only a corrupt + chunk declares raises that chunk's decoding error rather than yield rows + without their schema.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + + # Chunk 0 alone declares both schemas; the task owns chunks 2 to 5. + with pytest.raises(Exception) as error: + read_all_chunks(path, owned_from=2, on_corrupt=None) + + assert type(error.value).__name__ == "ZstdError" + + +def test_a_schema_only_a_corrupt_chunk_declares_fails_a_skipping_read(tmp_path): + """With ``on_corrupt``, a task whose schema only a corrupt earlier chunk + declares raises an error naming that chunk. Neither chunk is reported as + corrupt by this task: the owner of the corrupt one reports it.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + reported = [] + + with pytest.raises(ValueError) as error: + read_all_chunks(path, owned_from=2, on_corrupt=lambda c, e: reported.append(c)) + + assert "neither the summary nor any chunk up to this one declares" in str( + error.value + ) + assert ( + f"chunk at byte offset {chunk_offset(path, 0)}, read back for it, is corrupt" + in str(error.value) + ) + assert reported == [] + + +UNDECODABLE_SCHEMA_CHUNK = 1 # the only chunk that declares ``/b``'s schema + + +def test_an_unselected_topic_whose_schema_is_corrupt_costs_no_rows( + tmp_path, restore_max_errored_blocks +): + """When errors are skipped, a read of ``/a`` whose file declares the + unselected ``/b``'s schema only in a corrupt chunk yields every ``/a`` row. + It reports that chunk as skipped, not the missing schema.""" + clean = os.path.join(tmp_path, "clean.mcap") + # Without message indexes, the read opens the /b chunks 3 and 5 too. + write_mcap_without_summary_schemas( + clean, b_declared_in_chunk_1=True, message_indexes=False + ) + path = write_corrupt_chunk( + os.path.join(tmp_path, "bad.mcap"), clean, UNDECODABLE_SCHEMA_CHUNK + ) + + tables, error = read_one_task([path], topics=["/a"]) + + assert rows_of(tables) == [(path, seq) for seq in (0, 2, 4)] + assert str(error).startswith( + "Skipped 1 corrupt chunk at byte offset " + f"{chunk_offset(path, UNDECODABLE_SCHEMA_CHUNK)} of MCAP file {path!r}: " + ) + + +def test_a_message_on_an_undeclared_channel_is_not_skipped_as_corrupt(tmp_path): + """A chunk that decodes but holds a message on a channel nothing declares + raises that error even when corrupt chunks are skipped, rather than being + reported as a corrupt chunk.""" + path = os.path.join(tmp_path, "undeclared.mcap") + write_undeclared_channel(path) + reported = [] + + with pytest.raises(ValueError, match=f"on channel {UNDECLARED_CHANNEL}, which"): + read_all_chunks(path, owned_from=0, on_corrupt=lambda c, e: reported.append(c)) + + assert reported == [] + + +def test_an_error_reading_a_file_raises_before_skipped_corrupt_chunks( + tmp_path, monkeypatch, restore_max_errored_blocks +): + """When errors are skipped, a task whose first file has a corrupt chunk and + whose second file fails to read raises the read's ``OSError`` at once, so + the task's retry settings see it, rather than one error for both files.""" + from ray.data._internal.datasource_v2.formats.mcap.mcap_reader import MCAPReader + + a, b, c = write_task_files(tmp_path, bad={"a.mcap"}) + real_read_file = MCAPReader._read_file + + def read_file(self, owned, *args): + if owned.path == b: + raise OSError("disk unplugged") + return real_read_file(self, owned, *args) + + monkeypatch.setattr(MCAPReader, "_read_file", read_file) + + tables, error = read_one_task([a, b, c]) + + assert isinstance(error, OSError) + assert rows_of(tables) == file_rows(a, skipped={CORRUPT_CHUNK}) + + +def test_a_retried_read_reports_only_its_own_corrupt_chunks( + chunked_file, monkeypatch, restore_max_errored_blocks +): + """A file read that is retried starts with no corrupt chunks recorded, so a + chunk the failed attempt recorded but the retry reads cleanly raises no + error.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + from ray.data.context import DataContext + + context = DataContext.get_current() + context.max_errored_blocks = 1 + monkeypatch.setattr(context, "retried_io_errors", ["connection reset"]) + # Skip the retry's backoff wait. + monkeypatch.setattr(time, "sleep", lambda seconds: None) + real_read_chunks = mcap_chunks.read_chunks + attempts = [] + + def read_chunks(f, chunk_indexes, reads, on_corrupt=None): + # The first attempt records its first chunk as corrupt, then fails. + attempts.append(1) + if len(attempts) == 1: + assert on_corrupt is not None + on_corrupt(list(chunk_indexes)[0], ValueError("stale")) + raise OSError("connection reset") + yield from real_read_chunks(f, chunk_indexes, reads, on_corrupt) + + monkeypatch.setattr(mcap_chunks, "read_chunks", read_chunks) + datasource = MCAPDatasourceV2([chunked_file]) + + table = read_all(datasource, *list_manifests(datasource)) + + assert len(attempts) == 2 + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +# Small files that share one read task, and which of them have a corrupt chunk: +# the first and the last are among the five. +TASK_FILES = 8 +BAD_FILES = {1: [3], 5: [0, 2, 3, 5, 7]} + + +@pytest.mark.parametrize("num_bad", BAD_FILES.keys()) +def test_a_corrupt_chunk_in_a_shared_read_costs_only_its_own_rows( + tmp_path, num_bad, restore_max_errored_blocks +): + """A corrupt chunk fetched in one ranged read with its neighbours costs only + its own rows: the bad files and the task's other files keep every other + row, and the error names each bad file, with one and with five bad files of + eight.""" + from ray.data.context import DataContext + + paths = [os.path.join(tmp_path, f"f{i}.mcap") for i in range(TASK_FILES)] + for path in paths: + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + bad = [paths[i] for i in BAD_FILES[num_bad]] + for path in bad: + corrupt_in_place(path, CORRUPT_CHUNK) + # One allowed error is enough: the task raises once for all its bad files. + DataContext.get_current().max_errored_blocks = 1 + datasource = MCAPDatasourceV2(paths, include_paths=True) + # The eight small files share one read task, which carries their summaries. + (task,) = pack(datasource, preserve_order=True) + seek_log = SeekLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=seek_log) + + tables = [] + with pytest.raises(Exception) as error: + for table in scanner.create_reader().read(task): + tables.append(table) + + # Each file, bad ones too, is one read from its first chunk: the corrupt + # chunk came in the same read as its neighbours, and no summary was read. + for path in paths: + assert seek_log.seeks[path] == [chunk_offset(path, 0)] + in_task_order = [str(p) for p in dict.fromkeys(task.paths.tolist())] + good = [path for path in in_task_order if path not in bad] + expected = [ + row + for path in in_task_order + for row in file_rows(path, () if path in good else {CORRUPT_CHUNK}) + ] + assert rows_of(tables) == expected + assert all(path in str(error.value) for path in bad) + assert not any(path in str(error.value) for path in good) + + def test_non_json_channel_under_a_decoded_plan_fails(tmp_path): """A read planned as decoded JSON fails on a non-JSON file the sample missed.""" json_path = os.path.join(tmp_path, "json.mcap")