From 140d26070865f7ee0ab3f4f505d7f3a74f19121a Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Tue, 20 Oct 2026 05:55:11 -0700 Subject: [PATCH 01/13] [Data] Emit a map task's buffered rows before an error the job skips Problem: - A map task buffers its output until a block reaches its target size. When the task raised, its buffered rows were dropped, even when max_errored_blocks let the job skip the error. A read task that hit one corrupt file lost the rows of the files it had read before it. Change: - When the job skips errors (max_errored_blocks is not 0) and the task cannot run again, _BlockShapingIterator emits the buffered rows as a last block, then raises the error. - A task that may run again raises without a partial block: on an error that retried_map_errors matches while _map_task has attempts left, on a RaySystemError, or on an error the task's retry_exceptions names (or one it wraps). An actor task's retry_exceptions come from actor_task_retry_on_errors. A rerun skips as many blocks as were already emitted, so a partial block would hide rows of the rerun. - TaskContext carries the task's Ray retry_exceptions (False when max_retries is 0), and _map_task sets each attempt's retries left and retried_map_errors patterns on it. - BlockOutputBuffer.has_buffered_rows says whether rows are waiting. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../execution/interfaces/task_context.py | 12 +- .../operators/actor_pool_map_operator.py | 3 +- .../execution/operators/map_operator.py | 26 ++- .../execution/operators/map_transformer.py | 77 ++++++++ .../operators/task_pool_map_operator.py | 17 +- python/ray/data/_internal/output_buffer.py | 4 + python/ray/data/tests/test_map_transformer.py | 182 ++++++++++++++++++ .../test_streaming_executor_errored_blocks.py | 47 +++++ 8 files changed, 354 insertions(+), 14 deletions(-) diff --git a/python/ray/data/_internal/execution/interfaces/task_context.py b/python/ray/data/_internal/execution/interfaces/task_context.py index 35ce6506ea84..810165ffd354 100644 --- a/python/ray/data/_internal/execution/interfaces/task_context.py +++ b/python/ray/data/_internal/execution/interfaces/task_context.py @@ -1,7 +1,7 @@ import contextlib import threading from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Iterator, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union if TYPE_CHECKING: from ray.data._internal.execution.operators.map_transformer import MapTransformer @@ -47,6 +47,16 @@ class TaskContext: # Override of the target max-block-size for the task target_max_block_size_override: Optional[int] = None + # The exceptions Ray reruns this task on: its ``retry_exceptions`` remote + # arg, or ``False`` when ``max_retries`` is 0. ``True`` means any exception. + ray_retried_exceptions: Union[bool, List[type]] = False + + # Set by ``_map_task`` for each attempt: how many more times it reruns the + # transform, and the ``retried_map_errors`` patterns it reruns on (``None`` + # means any error). + map_retries_left: int = 0 + map_retried_errors: Optional[List[str]] = field(default_factory=list) + # Additional keyword arguments passed to the task. kwargs: Dict[str, Any] = field(default_factory=dict) diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 62063abf256c..6b922ecbb210 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -505,12 +505,13 @@ def _try_schedule_tasks_internal(self) -> int: input_blocks = [entry.ref for entry in bundle.blocks] self._actor_pool.on_task_submitted(actor) + actor_task_args = dict(self._ray_actor_task_remote_args) ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, + ray_retried_exceptions=actor_task_args.get("retry_exceptions", False), ) - actor_task_args = dict(self._ray_actor_task_remote_args) extra_labels = actor_task_args.pop("_labels", None) or {} gen = actor.submit.options( num_returns="streaming", diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index 4d27e34b47dd..c528a2bce89d 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -986,7 +986,23 @@ def _map_task( # `max_concurrent_calls_per_actor > 1` several run at once. clock = TransformClock() + match retry_on: + case True: + to_match = None + case False: + to_match = [] + case _: + to_match = retry_on + max_attempts = data_context.max_map_retries + 1 + attempts_started = 0 + def transform_iter_factory(): + nonlocal attempts_started + # Tell the transform whether this attempt can still be rerun, so it + # knows whether to emit its buffered rows before an error. + attempts_started += 1 + ctx.map_retries_left = max_attempts - attempts_started + ctx.map_retried_errors = to_match # Clear any per-task custom stats before each attempt (the reporter # is reused across retries of this task), so a prior attempt's stats # can't leak into this one. A producing transform repopulates it @@ -1003,19 +1019,11 @@ def transform_iter_factory(): clock=clock, ) - match retry_on: - case True: - to_match = None - case False: - to_match = [] - case _: - to_match = retry_on - block_iter = iterate_with_retry( transform_iter_factory, description="apply UDF transform", match=to_match, - max_attempts=data_context.max_map_retries + 1, + max_attempts=max_attempts, unwrap_cause=True, ) diff --git a/python/ray/data/_internal/execution/operators/map_transformer.py b/python/ray/data/_internal/execution/operators/map_transformer.py index bf23bf405007..bf5ec6f6d55c 100644 --- a/python/ray/data/_internal/execution/operators/map_transformer.py +++ b/python/ray/data/_internal/execution/operators/map_transformer.py @@ -836,6 +836,10 @@ class _BlockShapingIterator(Iterator[Block]): Unlike a generator, local variables in __next__ go out of scope when the method returns, avoiding holding references to yielded values. + + When the results raise an error the job skips, the rows already buffered + come out as a last block before the error, so the skipped task keeps the + output it produced (see ``_emit_rows_before_error``). """ def __init__( @@ -847,6 +851,8 @@ def __init__( self._results_iter = iter(results) self._buffer = BlockOutputBuffer(output_block_size_option) self._finalized = False + # The error the results raised, held until the buffered rows are out. + self._error: Optional[Exception] = None if input_type == MapTransformFnDataType.Block: self._append_buffer = self._buffer.add_block @@ -867,6 +873,9 @@ def __next__(self) -> Block: # If finalized, no more data elif self._finalized: + if self._error is not None: + error, self._error = self._error, None + raise error raise StopIteration try: @@ -876,6 +885,74 @@ def __next__(self) -> Block: except StopIteration: self._buffer.finalize() self._finalized = True + except Exception as e: + if not self._buffer.has_buffered_rows(): + raise + if not _emit_rows_before_error(e): + raise + self._error = e + self._buffer.finalize() + self._finalized = True + + +def _emit_rows_before_error(error: Exception) -> bool: + """Whether a map task that raised ``error`` first emits its buffered rows. + + Only when the job skips the error (``DataContext.max_errored_blocks`` is not + 0) and the task cannot run again. A rerun produces its blocks from the start + and skips as many as were already emitted, so a last block cut short at the + error would hide rows of the rerun's block in its place. ``_map_task`` + reruns on an error that ``retried_map_errors`` matches while it has attempts + left. Ray reruns a task on a ``RaySystemError`` and on an error its + ``retry_exceptions`` names, which for an actor task come from + ``actor_task_retry_on_errors``. + """ + import ray.exceptions + from ray.data.context import DataContext + + may_rerun = ( + _map_task_reruns_on(error) + or isinstance(error, ray.exceptions.RaySystemError) + or _ray_reruns_on(error) + ) + return DataContext.get_current().max_errored_blocks != 0 and not may_rerun + + +def _ray_reruns_on(error: Exception) -> bool: + """Whether Ray reruns the running task on ``error``. + + Checks ``error`` and its causes against the task's ``retry_exceptions``, so + an error a transform wraps still counts. + """ + task_context = TaskContext.get_current() + retried = task_context.ray_retried_exceptions if task_context else False + if isinstance(retried, bool): + return retried + cause: Optional[BaseException] = error + while cause is not None: + if isinstance(cause, tuple(retried)): + return True + cause = cause.__cause__ + return False + + +def _map_task_reruns_on(error: Exception) -> bool: + """Whether ``_map_task`` reruns the running attempt on ``error``. + + It does while attempts are left and the error's message, with its causes, + matches a ``retried_map_errors`` pattern, as ``iterate_with_retry`` checks + it. ``None`` patterns match any error. + """ + from ray._common.retry import format_exception, matches_error + + task_context = TaskContext.get_current() + if task_context is None or task_context.map_retries_left <= 0: + return False + patterns = task_context.map_retried_errors + if patterns is None: + return True + text = format_exception(error, include_cause=True) + return any(matches_error(pattern, text) for pattern in patterns) class _RowBasedIterator(Iterator[Row]): diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index 0ec9cffed5b8..e33b13fd0923 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -1,6 +1,6 @@ import copy import warnings -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union if TYPE_CHECKING: import pyarrow as pa @@ -184,13 +184,13 @@ def _try_schedule_task(self, bundle: RefBundle, strict: bool): # Notify first input for deferred initialization (e.g., Iceberg schema evolution). self._notify_first_input(bundle) # Submit the task as a normal Ray task. + dynamic_ray_remote_args = self._get_dynamic_ray_remote_args(input_bundle=bundle) ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, + ray_retried_exceptions=_ray_retried_exceptions(dynamic_ray_remote_args), ) - - dynamic_ray_remote_args = self._get_dynamic_ray_remote_args(input_bundle=bundle) dynamic_ray_remote_args["name"] = self.name logical_usage = ExecutionResources.from_resource_dict(dynamic_ray_remote_args) resource_request = task_resource_dict(dynamic_ray_remote_args) @@ -332,3 +332,14 @@ def all_inputs_done(self): "to increase the number of concurrent tasks by configuring " "`override_num_blocks` earlier in the pipeline." ) + + +def _ray_retried_exceptions(ray_remote_args: Dict[str, Any]) -> Union[bool, List[type]]: + """The exceptions Ray reruns a task with ``ray_remote_args`` on. + + ``cached_remote_fn`` defaults ``max_retries`` to -1 (unlimited); with 0, Ray + reruns on nothing. + """ + if ray_remote_args.get("max_retries", -1) == 0: + return False + return ray_remote_args.get("retry_exceptions", False) diff --git a/python/ray/data/_internal/output_buffer.py b/python/ray/data/_internal/output_buffer.py index 17a4e536d792..d347407844cb 100644 --- a/python/ray/data/_internal/output_buffer.py +++ b/python/ray/data/_internal/output_buffer.py @@ -111,6 +111,10 @@ def finalize(self) -> None: assert not self._finalized self._finalized = True + def has_buffered_rows(self) -> bool: + """Whether rows added since the last output block are waiting.""" + return self._buffer.num_rows() > 0 + def _exceeded_buffer_row_limit(self) -> bool: if self._output_block_size_option.disable_block_shaping: return False diff --git a/python/ray/data/tests/test_map_transformer.py b/python/ray/data/tests/test_map_transformer.py index 3ac104a57ac2..649cc1873e96 100644 --- a/python/ray/data/tests/test_map_transformer.py +++ b/python/ray/data/tests/test_map_transformer.py @@ -4,11 +4,13 @@ from collections import deque import pandas as pd +import pyarrow as pa import pytest from ray.data._internal.execution.interfaces.task_context import TaskContext from ray.data._internal.execution.operators.map_transformer import ( BatchMapTransformFn, + BlockMapTransformFn, MapTransformer, TransformClock, ) @@ -293,6 +295,186 @@ def make_input_blocks(): ) +def _failing_block_transformer(tables_before_error): + """A one-stage transformer whose function yields ``tables_before_error`` + one-row tables, then raises ``ValueError("bad input")``. Its output blocks + are 1 MiB, so the tables share one buffered block.""" + + def fn(blocks, ctx): + for i in range(tables_before_error): + yield pa.table({"x": [i]}) + raise ValueError("bad input") + + return MapTransformer( + [ + BlockMapTransformFn( + fn, + output_block_size_option=OutputBlockSizeOption.of( + target_max_block_size=1024 * 1024 + ), + ) + ] + ) + + +def _outputs(transformer, **retry_settings): + """The transformer's output blocks for one one-row input block, in a task + whose ``TaskContext`` carries ``retry_settings`` (how ``_map_task`` and Ray + rerun it). + + A generator, so the task context is current while the blocks are drawn, as + ``_map_task`` sets it. + """ + ctx = TaskContext(task_idx=0, op_name="test", **retry_settings) + with TaskContext.current(ctx): + yield from transformer.apply_transform( + iter([pa.table({"x": [0]})]), ctx, clock=TransformClock() + ) + + +@pytest.fixture +def skip_errors(monkeypatch): + """Let the job skip an errored block (``max_errored_blocks=1``); returns the + patched ``DataContext``.""" + from ray.data.context import DataContext + + context = DataContext.get_current() + monkeypatch.setattr(context, "max_errored_blocks", 1) + return context + + +def test_a_skipped_error_emits_the_buffered_rows_first(skip_errors): + """When the job skips errors, rows a transform produced before raising come + out as one last block, then the error is raised, so the task keeps them.""" + out = _outputs(_failing_block_transformer(tables_before_error=2)) + + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_an_error_that_fails_the_job_emits_no_partial_block(): + """With ``max_errored_blocks=0``, the default, the job fails anyway, so the + transform raises without emitting its buffered rows.""" + out = _outputs(_failing_block_transformer(tables_before_error=2)) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_a_transform_failing_before_any_output_emits_no_block(skip_errors): + """A transform that raises before producing a row raises at once, without + an empty block.""" + out = _outputs(_failing_block_transformer(tables_before_error=0)) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_a_transform_that_may_rerun_raises_without_a_partial_block(skip_errors): + """When ``_map_task`` may rerun the transform (``retried_map_errors``), the + buffered rows are not emitted: the rerun skips as many blocks as were + emitted, so a partial block would hide rows of the rerun's first block.""" + # ``None`` patterns retry any error, as ``retried_map_errors=True`` sets them. + out = _outputs( + _failing_block_transformer(tables_before_error=2), + map_retries_left=1, + map_retried_errors=None, + ) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_an_error_no_retry_pattern_matches_emits_the_buffered_rows_first( + skip_errors, +): + """``_map_task`` retries only errors that match ``retried_map_errors``, so an + error that matches none won't rerun the transform, and its buffered rows come + out first.""" + out = _outputs( + _failing_block_transformer(tables_before_error=2), + map_retries_left=1, + map_retried_errors=["connection reset"], + ) + + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_an_error_a_retry_pattern_matches_raises_without_a_partial_block( + skip_errors, +): + """An error that matches a ``retried_map_errors`` pattern may rerun the + transform, so its buffered rows are not emitted.""" + out = _outputs( + _failing_block_transformer(tables_before_error=2), + map_retries_left=1, + map_retried_errors=["bad input"], + ) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_an_error_ray_reruns_the_task_on_raises_without_a_partial_block( + skip_errors, +): + """Ray reruns a task on an error its ``retry_exceptions`` names, so the + buffered rows are not emitted, as for ``retried_map_errors``.""" + out = _outputs( + _failing_block_transformer(tables_before_error=2), + ray_retried_exceptions=[ValueError], + ) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_an_error_ray_does_not_rerun_the_task_on_emits_the_buffered_rows_first( + skip_errors, +): + """An error that ``retry_exceptions`` does not name won't rerun the task, + so its buffered rows come out first.""" + out = _outputs( + _failing_block_transformer(tables_before_error=2), + ray_retried_exceptions=[KeyError], + ) + + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_the_last_map_attempt_emits_the_buffered_rows_first(skip_errors): + """On its last attempt ``_map_task`` won't rerun the transform even for an + error a pattern matches, so the buffered rows come out first.""" + out = _outputs( + _failing_block_transformer(tables_before_error=2), + map_retries_left=0, + map_retried_errors=["bad input"], + ) + + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] + with pytest.raises(ValueError, match="bad input"): + next(out) + + +def test_actor_retry_settings_do_not_stop_a_task_pool_map_from_emitting( + skip_errors, monkeypatch +): + """``actor_task_retry_on_errors`` applies to actor tasks only, through their + ``retry_exceptions``. A task that Ray won't rerun still emits its buffered + rows when the setting is on.""" + monkeypatch.setattr(skip_errors, "actor_task_retry_on_errors", True) + out = _outputs(_failing_block_transformer(tables_before_error=2)) + + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] + with pytest.raises(ValueError, match="bad input"): + next(out) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_streaming_executor_errored_blocks.py b/python/ray/data/tests/test_streaming_executor_errored_blocks.py index 1412ac087e58..ed6560ea177f 100644 --- a/python/ray/data/tests/test_streaming_executor_errored_blocks.py +++ b/python/ray/data/tests/test_streaming_executor_errored_blocks.py @@ -52,6 +52,53 @@ def map_func(row): assert stats.extra_metrics["num_tasks_failed"] == num_errored_blocks +def test_map_retry_after_partial_output_returns_every_row_once(restore_data_context): + """A task that fails once after producing rows and is retried + (``retried_map_errors``) returns every row exactly once, also when the job + skips errors.""" + ctx = ray.data.DataContext.get_current() + ctx.retried_map_errors = ["transient"] + ctx.max_map_retries = 1 + ctx.max_errored_blocks = 1 + failed = [] + + def flaky(batch): + # Fails once, at the third batch, after two batches are buffered. + if batch["id"][0] == 4 and not failed: + failed.append(True) + raise ValueError("transient") + return batch + + ds = ray.data.range(10, override_num_blocks=1).map_batches(flaky, batch_size=2) + + assert sorted(row["id"] for row in ds.take_all()) == list(range(10)) + + +def test_ray_task_retry_after_partial_output_returns_every_row_once( + restore_data_context, tmp_path +): + """A task that fails once after producing rows and that Ray reruns + (``retry_exceptions``) returns every row exactly once, also when the job + skips errors.""" + ctx = ray.data.DataContext.get_current() + ctx.max_errored_blocks = 1 + # Ray reruns the task in a fresh worker, so "failed once" lives in a file. + failed = tmp_path / "failed" + + def flaky(batch): + # Fails once, at the third batch, after two batches are buffered. + if batch["id"][0] == 4 and not failed.exists(): + failed.touch() + raise ValueError("transient") + return batch + + ds = ray.data.range(10, override_num_blocks=1).map_batches( + flaky, batch_size=2, retry_exceptions=[ValueError] + ) + + assert sorted(row["id"] for row in ds.take_all()) == list(range(10)) + + if __name__ == "__main__": import sys From 14660c224a3772d0e4166e73952ac63802c342f8 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Tue, 20 Oct 2026 05:55:12 -0700 Subject: [PATCH 02/13] [Data] read_mcap V2: read past a corrupt chunk when errors are skipped Problem: - The listing cannot isolate a file whose summary reads but whose chunk is corrupt. That chunk failed the read task holding it: the reader stopped at the bad file, and the small files sharing the task lost their rows too. Change: - When max_errored_blocks lets errors be skipped, the reader skips a corrupt chunk and reads the rest of its file. Each chunk is compressed on its own and found through the chunk index, so one bad chunk costs only its own rows. - A chunk read back for a declaration that does not decode is skipped, and the look-back goes on to the next earlier chunk, so the chunk being read is not blamed for it. - A retried file read starts with no corrupt chunks recorded. - A file that fails as a whole does not stop the task's other files. A file without a chunk index still fails whole, and an error reading the file still raises, so the read retry applies. - Once the task's files are read, it raises one error naming every file and chunk that failed, so the task counts once against max_errored_blocks. With max_errored_blocks=0 the first failure still fails the task at once. - With the map task's flush of buffered rows, the job keeps every other row, also when a map is fused into the read. - New tests: a chunk whose payload does not decompress and one whose record does not parse, in both read orders, also when fetched in one ranged read with its neighbours. End to end, a file with two corrupt chunks loses only their rows under max_errored_blocks=1. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_chunks.py | 66 +++- .../datasource_v2/formats/mcap/mcap_io.py | 47 ++- .../datasource_v2/formats/mcap/mcap_reader.py | 117 ++++++- python/ray/data/tests/datasource/test_mcap.py | 75 +++++ .../tests/unit/datasource_v2/mcap_testing.py | 20 +- .../tests/unit/datasource_v2/test_mcap_io.py | 27 ++ .../unit/datasource_v2/test_mcap_reader.py | 287 ++++++++++++++++++ 7 files changed, 611 insertions(+), 28 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index ecc07d1e559b..043a788d6fe4 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -25,6 +25,7 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( ChunkReads, + OnCorrupt, read_chunk, read_chunks, ) @@ -75,13 +76,21 @@ def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) - """Add the channel and schema records of one chunk to ``declared``. A record already in ``declared`` is not replaced. The chunk is marked as - scanned. + scanned. A chunk that does not decode adds nothing, so the walk goes on to + the next earlier chunk; the task that owns it reports it. An error reading + ``f`` still raises. """ from mcap.records import Channel, Schema from mcap.stream_reader import breakup_chunk declared.scanned.add(chunk_index.chunk_start_offset) - for record in breakup_chunk(read_chunk(f, chunk_index)): + try: + records = list(breakup_chunk(read_chunk(f, chunk_index))) + except OSError: + raise + except Exception: # noqa: BLE001 - a corrupt chunk declares nothing here + return + for record in records: if isinstance(record, Channel): declared.channels.setdefault(record.id, record) elif isinstance(record, Schema): @@ -113,6 +122,7 @@ def iter_chunks( summary: "Summary", offsets: Optional[Set[int]], file_summary: Optional[Callable[[], "Summary"]] = None, + on_corrupt: Optional[OnCorrupt] = None, ) -> Iterator[_Selected]: """Yield the selected messages of the owned chunks of an indexed file. @@ -120,6 +130,10 @@ def iter_chunks( ``log_time_order``. ``summary`` may index only the owned chunks, as a read task's summary does (``mcap_task_summary``). Then ``file_summary`` returns the whole summary, which reading back a declaration needs. + + With ``on_corrupt``, a chunk that cannot be decoded is handed to it and + skipped, so the file's other chunks are still read. An error reading + ``f`` still raises. """ selected = self._selection.selected_channel_ids( summary.channels, summary.schemas @@ -136,13 +150,14 @@ def iter_chunks( file_summary or (lambda: summary), ) if not self._log_time_order: - for chunk_index, chunk in read_chunks(f, chunk_indexes, self._chunk_reads): - yield from self._chunk_messages( - f, path, summary, chunk_index, chunk, selected, declared + chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt) + for chunk_index, chunk in chunks: + yield from self._decoded_messages( + f, path, summary, chunk_index, chunk, selected, declared, on_corrupt ) return yield from self._merge_chunks_by_log_time( - f, path, summary, chunk_indexes, selected, declared + f, path, summary, chunk_indexes, selected, declared, on_corrupt ) def candidate_chunks( @@ -176,6 +191,7 @@ def _merge_chunks_by_log_time( chunk_indexes: List["ChunkIndex"], selected: Set[int], declared: _Declared, + on_corrupt: Optional[OnCorrupt], ) -> Iterator[_Selected]: """Yield the selected messages of the chunks in log-time order. @@ -194,14 +210,14 @@ def _merge_chunks_by_log_time( heapq.heapify(heap) # Chunks leave the heap in ``_merge_order``, so they are read in it. chunks = read_chunks( - f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads + f, sorted(chunk_indexes, key=_merge_order), self._chunk_reads, on_corrupt ) while heap: _, kind, offset, _, item = heapq.heappop(heap) if kind == 0: chunk_index, chunk = next(chunks) assert chunk_index.chunk_start_offset == offset - for index, selected_message in self._chunk_messages( + for index, selected_message in self._decoded_messages( f, path, summary, @@ -209,6 +225,7 @@ def _merge_chunks_by_log_time( chunk, selected, declared, + on_corrupt, with_index=True, ): heapq.heappush( @@ -224,6 +241,39 @@ def _merge_chunks_by_log_time( else: yield item + def _decoded_messages( + self, + f: Any, + path: str, + summary: "Summary", + chunk_index: "ChunkIndex", + chunk: Optional["Chunk"], + selected: Set[int], + declared: "_Declared", + on_corrupt: Optional[OnCorrupt], + with_index: bool = False, + ) -> Iterator[Any]: + """``_chunk_messages``, or the messages before a decoding error. + + ``chunk`` is ``None`` when its record did not parse, which + ``read_chunks`` has already handed to ``on_corrupt``. A chunk that fails + while it is decompressed or parsed stops there and goes to + ``on_corrupt``. Without ``on_corrupt``, or for an error reading ``f``, + the error raises. + """ + if chunk is None: + return + try: + yield from self._chunk_messages( + f, path, summary, chunk_index, chunk, selected, declared, with_index + ) + except OSError: + raise + except Exception as e: # noqa: BLE001 - a corrupt chunk costs only itself + if on_corrupt is None: + raise + on_corrupt(chunk_index, e) + def _chunk_messages( self, f: Any, diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index 4440f7fee115..cd225f9ca981 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -19,7 +19,17 @@ import io from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterable, Iterator, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Iterable, + Iterator, + List, + Optional, + Tuple, + Union, +) from ray._common.utils import env_integer from ray.data._internal.util import KiB, MiB @@ -105,13 +115,23 @@ def plan_chunk_reads( return planned +# Takes a chunk that cannot be decoded and the error, in a read that skips +# corrupt chunks (``SelectedMessageReader.iter_chunks``). +OnCorrupt = Callable[["ChunkIndex", Exception], None] + + def read_chunks( - f: Any, chunk_indexes: Iterable["ChunkIndex"], reads: ChunkReads -) -> Iterator[Tuple["ChunkIndex", "Chunk"]]: + f: Any, + chunk_indexes: Iterable["ChunkIndex"], + reads: ChunkReads, + on_corrupt: Optional[OnCorrupt] = None, +) -> Iterator[Tuple["ChunkIndex", Optional["Chunk"]]]: """Yield the chunk record of each of ``chunk_indexes``, in that order. Each read of ``plan_chunk_reads`` is made when its first chunk is needed, - and its bytes are released once its last chunk is parsed. + and its bytes are released once its last chunk is parsed. With + ``on_corrupt``, a chunk record that does not parse is handed to it and + yielded as ``None``; an error reading ``f`` still raises. """ for ranged_read in plan_chunk_reads(chunk_indexes, reads): # ``BytesIO`` shares the bytes rather than copying them; parsing a @@ -120,7 +140,7 @@ def read_chunks( last = len(ranged_read.chunk_indexes) - 1 for i, chunk_index in enumerate(ranged_read.chunk_indexes): position = chunk_index.chunk_start_offset - ranged_read.start - chunk = _parse_chunk(stream, position) + chunk = _parse_chunk_or_report(stream, position, chunk_index, on_corrupt) if i == last: # Free the bytes now, not when the next read replaces them, so # two reads are never held at once. @@ -240,6 +260,23 @@ def read_chunk(f: Any, chunk_index: "ChunkIndex") -> "Chunk": return _parse_chunk(stream, 0) +def _parse_chunk_or_report( + stream: io.BytesIO, + position: int, + chunk_index: "ChunkIndex", + on_corrupt: Optional[OnCorrupt], +) -> Optional["Chunk"]: + """``_parse_chunk``, or ``None`` after handing a corrupt record to + ``on_corrupt``. Without ``on_corrupt`` the error raises.""" + try: + return _parse_chunk(stream, position) + except Exception as e: # noqa: BLE001 - any parse failure is a corrupt record + if on_corrupt is None: + raise + on_corrupt(chunk_index, e) + return None + + def _parse_chunk(stream: io.BytesIO, position: int) -> "Chunk": """Parse the chunk record that starts at ``position`` of ``stream``.""" from mcap.data_stream import ReadDataStream diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 07aea2d965c7..6984dfc0632b 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -23,7 +23,17 @@ import logging from dataclasses import dataclass from functools import partial -from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Set +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Set, + Tuple, +) import pyarrow as pa from pyarrow.fs import FileSystem, LocalFileSystem @@ -60,6 +70,7 @@ from ray.util.debug import log_once if TYPE_CHECKING: + from mcap.records import ChunkIndex from mcap.summary import Summary logger = logging.getLogger(__name__) @@ -129,40 +140,74 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: Each row is read on its own, in manifest order, so a path listed twice is read twice. With ``limit``, reading stops at that many rows: no later message is added to a table and no later chunk or file is read. + + When ``DataContext.max_errored_blocks`` lets errors be skipped, a + corrupt chunk costs only itself: the reader skips it and reads the + file's other chunks. A file that fails as a whole does not stop the + task's other files. Once they are read, one error names every file and + chunk that failed, so the task counts once against the budget. + Otherwise the first failure fails the task at once. """ from ray.data.context import DataContext if len(input_split) == 0: return - retried_io_errors = DataContext.get_current().retried_io_errors + context = DataContext.get_current() + retried_io_errors = context.retried_io_errors + read_past_failures = context.max_errored_blocks != 0 remaining = self._limit + failures: List[Tuple[str, Exception]] = [] for owned in _owned_files(input_split): if remaining is not None and remaining <= 0: - return + break + # The chunks skipped as corrupt, by offset. Each attempt starts it + # afresh, so it holds the last attempt's. + corrupt: Optional[Dict[int, Exception]] = {} if read_past_failures else None tables = iterate_with_retry( - partial(self._read_file, owned, remaining), + partial(self._read_file, owned, remaining, corrupt), f"read MCAP file {owned.path}", match=retried_io_errors, ) - for table in tables: - if remaining is not None: - remaining -= table.num_rows - yield table + try: + for table in tables: + if remaining is not None: + remaining -= table.num_rows + yield table + except Exception as e: # noqa: BLE001 - raised once the others are read + if not read_past_failures: + raise + failures.append((owned.path, e)) + else: + if corrupt: + failures.append( + (owned.path, _corrupt_chunks_error(owned.path, corrupt)) + ) + if failures: + raise _read_error(failures) # -- one file ---------------------------------------------------------- def _read_file( - self, owned: "_OwnedFile", limit: Optional[int] + self, + owned: "_OwnedFile", + limit: Optional[int], + corrupt: Optional[Dict[int, Exception]] = None, ) -> Iterator[pa.Table]: """Yield the tables of one file, limited to the chunks it owns. ``owned.offsets`` is ``None`` for a whole-file row, which means every chunk. A file without a chunk index is scanned linearly, through a read buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most - ``limit`` rows are yielded. + ``limit`` rows are yielded. With ``corrupt``, a chunk that cannot be + decoded is recorded there by offset and skipped; a scanned file has no + chunk boundaries to skip to, so it still fails whole. A retry reads the + file from its start, so ``corrupt`` is cleared first. """ from ray.data.context import DataContext + if corrupt is not None: + corrupt.clear() + path, offsets = owned.path, owned.offsets filesystem = self._filesystem or LocalFileSystem() with filesystem.open_input_file(path) as f: @@ -183,7 +228,14 @@ def _read_file( ) else: messages = self._message_reader.iter_chunks( - f, path, summary, offsets, file_summary + f, + path, + summary, + offsets, + file_summary, + on_corrupt=None + if corrupt is None + else partial(_record_corrupt_chunk, corrupt), ) yield from self._tables(itertools.islice(messages, limit), path, offsets) @@ -322,6 +374,49 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array +def _record_corrupt_chunk( + corrupt: Dict[int, Exception], chunk_index: "ChunkIndex", error: Exception +) -> None: + """Keep the first error of each chunk a read skips as corrupt.""" + corrupt.setdefault(chunk_index.chunk_start_offset, error) + + +def _corrupt_chunks_error(path: str, corrupt: Dict[int, Exception]) -> ValueError: + """The error for the chunks of ``path`` that a read skipped as corrupt.""" + offsets = sorted(corrupt) + first = corrupt[offsets[0]] + where = "chunk at byte offset" if len(offsets) == 1 else "chunks at byte offsets" + error = ValueError( + f"Skipped {len(offsets)} corrupt {where} " + f"{', '.join(str(offset) for offset in offsets)} of MCAP file {path!r}: " + f"{type(first).__name__}: {_first_line(first)}" + ) + error.__cause__ = first + return error + + +def _read_error(failures: List[Tuple[str, Exception]]) -> Exception: + """The error for the files a task could not read in full. + + A single failure is its own error, whose message already names the file. + Several become one ``ValueError`` naming each file, caused by the first. + """ + if len(failures) == 1: + return failures[0][1] + lines = [f"{path}: {type(e).__name__}: {_first_line(e)}" for path, e in failures] + error = ValueError( + f"{len(failures)} MCAP files of this read task could not be read in full:\n" + + "\n".join(lines) + ) + error.__cause__ = failures[0][1] + return error + + +def _first_line(error: Exception) -> str: + """The first non-empty line of ``error``'s message.""" + return next((line for line in str(error).splitlines() if line.strip()), "") + + @dataclass(frozen=True) class _OwnedFile: """What one manifest row reads of one file.""" diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index efa89eb64515..39e3b83c436f 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -775,6 +775,81 @@ def test_read_mcap_max_errored_blocks_skips_a_corrupt_recording( assert "2.mcap" in str(error.value) +def write_folder_with_a_corrupt_chunk(tmp_path) -> str: + """Write ``a.mcap``, ``b.mcap`` and ``c.mcap`` (three messages each) into one + folder, with ``b.mcap``'s only chunk zeroed: its summary reads, its messages + do not. Returns the folder.""" + from ray.data.tests.unit.datasource_v2.mcap_testing import write_corrupt_chunk + + folder = os.path.join(tmp_path, "drive") + os.makedirs(folder) + for name in ("a", "c"): + create_test_mcap_file( + os.path.join(folder, f"{name}.mcap"), + [{"topic": "/imu", "data": {"i": i}, "log_time": i} for i in range(3)], + ) + a = os.path.join(folder, "a.mcap") + write_corrupt_chunk(os.path.join(folder, "b.mcap"), a, 0) + return folder + + +@pytest.mark.parametrize("fused_map", [False, True]) +def test_read_mcap_max_errored_blocks_keeps_files_sharing_a_task_with_a_corrupt_chunk( + ray_start_regular_shared, tmp_path, datasource_v2, fused_map +): + """With ``max_errored_blocks=1``, a recording whose only chunk is corrupt is + skipped and every other file is read, including files in the same read task, + also when a map is fused into the read.""" + folder = write_folder_with_a_corrupt_chunk(tmp_path) + ray.data.DataContext.get_current().max_errored_blocks = 1 + ds = ray.data.read_mcap(folder, include_paths=True) + if fused_map: + ds = ds.map_batches(lambda batch: batch) + + rows = ds.take_all() + + # V2 packs the three small files into one read task; legacy splits them + # over two tasks on this one-CPU cluster, a.mcap with b.mcap. + files = sorted(os.path.basename(row["path"]) for row in rows) + assert files == ["a.mcap"] * 3 + ["c.mcap"] * 3 + + +@pytest.mark.parametrize("fused_map", [False, True]) +def test_read_mcap_max_errored_blocks_loses_only_the_rows_of_corrupt_chunks( + ray_start_regular_shared, tmp_path, datasource_v2, fused_map +): + """With ``max_errored_blocks=1``, V2 loses only the rows of a file's two + corrupt chunks, also when a map is fused into the read. Their read task + raises once for both, so one allowed error is enough.""" + if not datasource_v2: + pytest.skip("the legacy reader stops at a file's first corrupt chunk") + from ray.data.tests.unit.datasource_v2.mcap_testing import ( + CHUNKED_FILE_MESSAGES, + round_robin_messages, + write_corrupt_chunk, + write_mcap, + ) + + folder = os.path.join(tmp_path, "drive") + os.makedirs(folder) + good = os.path.join(folder, "a.mcap") + # One message per chunk; b.mcap is a.mcap with chunks 2 and 6 zeroed. + write_mcap(good, round_robin_messages(CHUNKED_FILE_MESSAGES)) + once = write_corrupt_chunk(os.path.join(tmp_path, "once.mcap"), good, 2) + write_corrupt_chunk(os.path.join(folder, "b.mcap"), once, 6) + ray.data.DataContext.get_current().max_errored_blocks = 1 + ds = ray.data.read_mcap(folder, include_paths=True) + if fused_map: + ds = ds.map_batches(lambda batch: batch) + + rows = ds.take_all() + + seqs = sorted((os.path.basename(row["path"]), row["data"]["seq"]) for row in rows) + expected = [("a.mcap", i) for i in range(CHUNKED_FILE_MESSAGES)] + expected += [("b.mcap", i) for i in range(CHUNKED_FILE_MESSAGES) if i not in (2, 6)] + assert seqs == expected + + def test_read_mcap_lists_files_in_parallel( ray_start_regular_shared, tmp_path, datasource_v2 ): diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 3e4abd8fe0d2..a3bdd6950288 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -111,7 +111,7 @@ def mixed_encoding_messages(n=6): return messages -def write_mcap_without_summary_schemas(path): +def write_mcap_without_summary_schemas(path, b_declared_in_chunk_1=False): """Write two JSON channels with two schemas, one message per chunk. The schema records are written only inside the chunks @@ -120,6 +120,9 @@ def write_mcap_without_summary_schemas(path): chunk 0 1 2 3 4 5 topic /a /b /a /b /a /b + + Both schemas are declared in chunk 0, or with ``b_declared_in_chunk_1``, + ``other_schema`` only in chunk 1. """ from mcap.writer import CompressionType, IndexType, Writer @@ -133,15 +136,23 @@ def write_mcap_without_summary_schemas(path): ) writer.start(profile="", library="ray-test") channels = {} - for topic, name in (("/a", "test_schema"), ("/b", "other_schema")): + + def declare(topic, name): schema_id = writer.register_schema( name=name, encoding="jsonschema", data=b'{"type": "object"}' ) channels[topic] = writer.register_channel( schema_id=schema_id, topic=topic, message_encoding="json" ) + + declare("/a", "test_schema") + if not b_declared_in_chunk_1: + declare("/b", "other_schema") for i in range(6): topic = "/a" if i % 2 == 0 else "/b" + if topic not in channels: + # Chunk 0 closed after its message, so this lands in chunk 1. + declare(topic, "other_schema") writer.add_message( channel_id=channels[topic], log_time=BASE_TIME + i * STEP, @@ -411,8 +422,9 @@ def record_parsed_chunks(monkeypatch) -> List[int]: real_read_chunks = mcap_chunks.read_chunks offsets: List[int] = [] - def read_chunks(f, chunk_indexes, reads): - for chunk_index, chunk in real_read_chunks(f, chunk_indexes, reads): + def read_chunks(f, chunk_indexes, reads, on_corrupt=None): + chunks = real_read_chunks(f, chunk_indexes, reads, on_corrupt) + for chunk_index, chunk in chunks: offsets.append(chunk_index.chunk_start_offset) yield chunk_index, chunk diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py index f1b481b01ff7..e59c479ace69 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_io.py @@ -23,6 +23,7 @@ from ray.data.tests.unit.datasource_v2.mcap_testing import ( CHUNKED_FILE_MESSAGES, round_robin_messages, + write_corrupt_chunk_header, write_cut_short, write_mcap, write_mcap_without_summary_schemas, @@ -358,6 +359,32 @@ def test_read_chunks_fetches_a_read_when_its_first_chunk_is_needed(tmp_path): assert reads == [0, 1, 2, 3] +def test_read_chunks_hands_a_record_that_does_not_parse_to_on_corrupt(tmp_path): + """With ``on_corrupt``, a chunk record that does not parse is handed to it + and yielded as ``None``, and the chunks around it in the same read are still + parsed. Without ``on_corrupt``, the error raises.""" + source = os.path.join(tmp_path, "chunked.mcap") + write_mcap(source, round_robin_messages(CHUNKED_FILE_MESSAGES)) + path = write_corrupt_chunk_header(os.path.join(tmp_path, "bad.mcap"), source, 4) + summary = read_summary_from(CountingFile(path), path) + assert summary is not None + chunk_indexes = sorted(summary.chunk_indexes, key=lambda c: c.chunk_start_offset) + expected = [read_chunk(CountingFile(path), c) for c in chunk_indexes[:4]] + expected += [None] + [read_chunk(CountingFile(path), c) for c in chunk_indexes[5:]] + reported = [] + + f = CountingFile(path) + chunks = list( + read_chunks(f, chunk_indexes, ChunkReads(), lambda c, e: reported.append(c)) + ) + + assert [chunk for _, chunk in chunks] == expected + assert reported == [chunk_indexes[4]] + assert f.reads == 1 + with pytest.raises(UnicodeDecodeError): + list(read_chunks(CountingFile(path), chunk_indexes, ChunkReads())) + + def test_buffered_scan_reads_in_large_blocks(tmp_path): """A linear scan through ``buffered`` yields the same records with one read per buffer's worth of the file, and closing the buffer leaves the file open.""" diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index c90fd47d3dbf..bcebdefb8a8a 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -8,6 +8,7 @@ import importlib.util import json import os +import time import pyarrow as pa import pytest @@ -42,14 +43,18 @@ infer_schema, list_manifests, mixed_encoding_messages, + pack, read_all, record_parsed_chunks, round_robin_messages, scanner_for, selected_messages, summary_of, + write_corrupt_chunk, + write_corrupt_chunk_header, write_cut_short, write_mcap, + write_mcap_without_summary_schemas, write_mixed_channels, ) @@ -128,6 +133,288 @@ def test_reading_a_file_cut_short_fails_naming_it(tmp_path, chunked_file): read_all(datasource, manifest) +CORRUPT_CHUNK = 4 # the chunk, and so the message, a bad file loses + + +def write_task_files(tmp_path, bad, corrupt=write_corrupt_chunk): + """Write ``a.mcap``, ``b.mcap`` and ``c.mcap`` (nine messages, one per chunk) + and corrupt chunk ``CORRUPT_CHUNK`` of the files named in ``bad`` with + ``corrupt``; return the three paths.""" + paths = [os.path.join(tmp_path, f"{name}.mcap") for name in "abc"] + for path in paths: + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + if os.path.basename(path) in bad: + corrupt_in_place(path, CORRUPT_CHUNK, corrupt) + return paths + + +def corrupt_in_place(path, index, corrupt=write_corrupt_chunk): + """Corrupt chunk ``index`` of the MCAP file ``path`` with ``corrupt``.""" + corrupt(path + ".tmp", path, index) + os.replace(path + ".tmp", path) + + +def read_one_task(paths, max_errored_blocks=1, **options): + """Read ``paths`` as one read task under ``max_errored_blocks``, with the + datasource ``options``; return the tables it yields and its error.""" + from ray.data.context import DataContext + + DataContext.get_current().max_errored_blocks = max_errored_blocks + datasource = MCAPDatasourceV2(paths, include_paths=True, **options) + # preserve_order keeps the files in the order given; the listing threads + # otherwise finish in any order. + task = FileManifest.concat(list_manifests(datasource, preserve_order=True)) + tables = [] + with pytest.raises(Exception) as error: + for table in scanner_for(datasource).create_reader().read(task): + tables.append(table) + return tables, error.value + + +def rows_of(tables): + """The ``(path, seq)`` of each row of ``tables``, in order.""" + return [ + (row["path"], row["data"]["seq"]) + for table in tables + for row in table.select(["path", "data"]).to_pylist() + ] + + +def file_rows(path, skipped=()): + """The ``(path, seq)`` rows of a file of ``round_robin_messages``, in order, + without the messages of the ``skipped`` chunks (one message per chunk).""" + return [(path, seq) for seq in range(CHUNKED_FILE_MESSAGES) if seq not in skipped] + + +def chunk_offset(path, index): + """The byte offset of chunk ``index`` of the MCAP file ``path``.""" + return summary_of(path).chunk_indexes[index].chunk_start_offset + + +@pytest.fixture +def restore_max_errored_blocks(): + """Put ``DataContext.max_errored_blocks`` back after the test.""" + from ray.data.context import DataContext + + saved = DataContext.get_current().max_errored_blocks + yield + DataContext.get_current().max_errored_blocks = saved + + +CORRUPTIONS = {"payload": write_corrupt_chunk, "header": write_corrupt_chunk_header} + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("corrupt", CORRUPTIONS.values(), ids=CORRUPTIONS.keys()) +def test_a_corrupt_chunk_costs_only_its_own_rows( + tmp_path, corrupt, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a task whose middle file has a corrupt chunk + yields every other row of the three files, then raises naming that file and + chunk. That holds for a payload that does not decompress and a record that + does not parse, in log-time and in file order.""" + a, b, c = write_task_files(tmp_path, bad={"b.mcap"}, corrupt=corrupt) + + tables, error = read_one_task([a, b, c], log_time_order=log_time_order) + + skipped = {CORRUPT_CHUNK} + assert rows_of(tables) == file_rows(a) + file_rows(b, skipped) + file_rows(c) + assert isinstance(error, ValueError) + assert str(error).startswith( + f"Skipped 1 corrupt chunk at byte offset {chunk_offset(b, CORRUPT_CHUNK)} " + f"of MCAP file {b!r}: " + ) + + +def test_one_error_names_every_corrupt_chunk_of_a_file( + tmp_path, restore_max_errored_blocks +): + """A file with two corrupt chunks, one of each kind, loses only their two + rows, and one error names both chunks.""" + a, b, c = write_task_files(tmp_path, bad=set()) + corrupt_in_place(b, 2, write_corrupt_chunk) + corrupt_in_place(b, 6, write_corrupt_chunk_header) + + tables, error = read_one_task([a, b, c]) + + assert rows_of(tables) == file_rows(a) + file_rows(b, {2, 6}) + file_rows(c) + assert str(error).startswith( + f"Skipped 2 corrupt chunks at byte offsets {chunk_offset(b, 2)}, " + f"{chunk_offset(b, 6)} of MCAP file {b!r}: " + ) + + +def test_a_task_names_every_file_it_could_not_read_in_full( + tmp_path, restore_max_errored_blocks +): + """Two files with a corrupt chunk fail their task once, with one error that + names both, after every other row of the three files is read.""" + a, b, c = write_task_files(tmp_path, bad={"a.mcap", "c.mcap"}) + + tables, error = read_one_task([a, b, c], max_errored_blocks=-1) + + skipped = {CORRUPT_CHUNK} + expected = file_rows(a, skipped) + file_rows(b) + file_rows(c, skipped) + assert rows_of(tables) == expected + assert isinstance(error, ValueError) + assert "2 MCAP files of this read task could not be read in full" in str(error) + assert a in str(error) and c in str(error) + + +def test_a_task_fails_at_its_first_corrupt_chunk_when_errors_are_not_skipped( + tmp_path, restore_max_errored_blocks +): + """With ``max_errored_blocks=0`` the read fails anyway, so the task raises at + the corrupt chunk without reading the file after it.""" + a, b, c = write_task_files(tmp_path, bad={"b.mcap"}) + + tables, error = read_one_task([a, b, c], max_errored_blocks=0) + + # b's rows before the corrupt chunk were still being built into a table. + assert rows_of(tables) == file_rows(a) + assert "b.mcap" in str(error) + + +def test_an_error_reading_the_file_is_not_skipped_as_a_corrupt_chunk( + tmp_path, monkeypatch +): + """An ``OSError`` while a chunk is decoded, here reading back an earlier + chunk for a schema, raises even when corrupt chunks are skipped, so the + file's read is retried rather than the chunk dropped.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + + path = os.path.join(tmp_path, "schemas_in_chunks.mcap") + write_mcap_without_summary_schemas(path) + summary = summary_of(path) + # Chunks 2 to 5 declare no schema, so reading them reads chunk 0 back. + owned = {c.chunk_start_offset for c in summary.chunk_indexes[2:]} + + def read_chunk(f, chunk_index): + raise OSError("connection reset") + + monkeypatch.setattr(mcap_chunks, "read_chunk", read_chunk) + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + reported = [] + + with open(path, "rb") as f, pytest.raises(OSError, match="connection reset"): + for _ in reader.iter_chunks( + f, path, summary, owned, on_corrupt=lambda c, e: reported.append(c) + ): + pass + + assert reported == [] + + +def test_a_corrupt_chunk_read_back_for_a_schema_costs_only_itself(tmp_path): + """A task reading back for a schema skips an earlier corrupt chunk and finds + the schema in the next one, so its own chunk is not reported as corrupt.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean, b_declared_in_chunk_1=True) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + summary = summary_of(path) + # The task owns chunks 3 to 5. Chunk 3 (/b) reads back through chunk 0, + # which is corrupt, to chunk 1, which declares other_schema. + owned = {c.chunk_start_offset for c in summary.chunk_indexes[3:]} + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + reported = [] + + with open(path, "rb") as f: + selected = list( + reader.iter_chunks( + f, path, summary, owned, on_corrupt=lambda c, e: reported.append(c) + ) + ) + + assert [channel.topic for _, channel, _, _ in selected] == ["/b", "/a", "/b"] + assert reported == [] + + +def test_a_retried_read_reports_only_its_own_corrupt_chunks( + chunked_file, monkeypatch, restore_max_errored_blocks +): + """A file read that is retried starts with no corrupt chunks recorded, so a + chunk the failed attempt recorded but the retry reads cleanly raises no + error.""" + from ray.data._internal.datasource_v2.formats.mcap import mcap_chunks + from ray.data.context import DataContext + + context = DataContext.get_current() + context.max_errored_blocks = 1 + monkeypatch.setattr(context, "retried_io_errors", ["connection reset"]) + # Skip the retry's backoff wait. + monkeypatch.setattr(time, "sleep", lambda seconds: None) + real_read_chunks = mcap_chunks.read_chunks + attempts = [] + + def read_chunks(f, chunk_indexes, reads, on_corrupt=None): + # The first attempt records its first chunk as corrupt, then fails. + attempts.append(1) + if len(attempts) == 1: + assert on_corrupt is not None + on_corrupt(list(chunk_indexes)[0], ValueError("stale")) + raise OSError("connection reset") + yield from real_read_chunks(f, chunk_indexes, reads, on_corrupt) + + monkeypatch.setattr(mcap_chunks, "read_chunks", read_chunks) + datasource = MCAPDatasourceV2([chunked_file]) + + table = read_all(datasource, *list_manifests(datasource)) + + assert len(attempts) == 2 + assert table.num_rows == CHUNKED_FILE_MESSAGES + + +# Small files that share one read task, and which of them have a corrupt chunk: +# the first and the last are among the five. +TASK_FILES = 8 +BAD_FILES = {1: [3], 5: [0, 2, 3, 5, 7]} + + +@pytest.mark.parametrize("num_bad", BAD_FILES.keys()) +def test_a_corrupt_chunk_in_a_shared_read_costs_only_its_own_rows( + tmp_path, num_bad, restore_max_errored_blocks +): + """A corrupt chunk fetched in one ranged read with its neighbours costs only + its own rows: the bad files and the task's other files keep every other + row, and the error names each bad file, with one and with five bad files of + eight.""" + from ray.data.context import DataContext + + paths = [os.path.join(tmp_path, f"f{i}.mcap") for i in range(TASK_FILES)] + for path in paths: + write_mcap(path, round_robin_messages(CHUNKED_FILE_MESSAGES)) + bad = [paths[i] for i in BAD_FILES[num_bad]] + for path in bad: + corrupt_in_place(path, CORRUPT_CHUNK) + # One allowed error is enough: the task raises once for all its bad files. + DataContext.get_current().max_errored_blocks = 1 + datasource = MCAPDatasourceV2(paths, include_paths=True) + # The eight small files share one read task, which carries their summaries. + (task,) = pack(datasource, preserve_order=True) + seek_log = SeekLog() + scanner = dataclasses.replace(scanner_for(datasource), filesystem=seek_log) + + tables = [] + with pytest.raises(Exception) as error: + for table in scanner.create_reader().read(task): + tables.append(table) + + # Each file, bad ones too, is one read from its first chunk: the corrupt + # chunk came in the same read as its neighbours, and no summary was read. + for path in paths: + assert seek_log.seeks[path] == [chunk_offset(path, 0)] + in_task_order = [str(p) for p in dict.fromkeys(task.paths.tolist())] + good = [path for path in in_task_order if path not in bad] + expected = [ + row + for path in in_task_order + for row in file_rows(path, () if path in good else {CORRUPT_CHUNK}) + ] + assert rows_of(tables) == expected + assert all(path in str(error.value) for path in bad) + assert not any(path in str(error.value) for path in good) + + def test_non_json_channel_under_a_decoded_plan_fails(tmp_path): """A read planned as decoded JSON fails on a non-JSON file the sample missed.""" json_path = os.path.join(tmp_path, "json.mcap") From 3e0b9027c6db11d294709adf8447d9477fa82366 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Thu, 29 Oct 2026 08:36:19 -0700 Subject: [PATCH 03/13] [Data] Emit a partial block when an actor task won't be rerun Problem: - An actor task counted as rerun on any error its retry_exceptions names, even when its actor's max_task_retries is 0. Ray then does not rerun it, so a skipped error dropped the task's buffered rows. Change: - An actor task counts as rerun only while max_task_retries allows it. The task option overrides the actor's; Ray defaults it to 0, as it stays with max_restarts=0. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../operators/actor_pool_map_operator.py | 21 +++++++++++- .../tests/test_actor_pool_map_operator.py | 33 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 6b922ecbb210..8278e5ff2305 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -118,6 +118,23 @@ def get_map_worker_cls_name(op_name: str) -> str: return f"MapWorker({op_name})" +def _actor_task_retried_exceptions( + task_args: Dict[str, Any], actor_args: Dict[str, Any] +) -> Union[bool, List[type]]: + """The exceptions Ray reruns an actor task on. + + Ray reruns an actor task only while ``max_task_retries`` allows it. The task + option overrides the actor's, and Ray defaults it to 0, as it stays when + ``max_restarts=0``. + """ + max_task_retries = task_args.get( + "max_task_retries", actor_args.get("max_task_retries", 0) + ) + if max_task_retries == 0: + return False + return task_args.get("retry_exceptions", False) + + class ActorPoolMapOperator(MapOperator, ReportsExtraResourceUsage): """A MapOperator implementation that executes tasks on an actor pool. @@ -510,7 +527,9 @@ def _try_schedule_tasks_internal(self) -> int: task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, - ray_retried_exceptions=actor_task_args.get("retry_exceptions", False), + ray_retried_exceptions=_actor_task_retried_exceptions( + actor_task_args, self._ray_remote_args + ), ) extra_labels = actor_task_args.pop("_labels", None) or {} gen = actor.submit.options( diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index e7f520fa1490..1176d02361c1 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -38,6 +38,8 @@ from ray.data._internal.execution.interfaces.ref_bundle import BlockEntry, RefBundle from ray.data._internal.execution.interfaces.task_context import TaskContext from ray.data._internal.execution.operators.actor_pool_map_operator import ( + ActorPoolMapOperator, + _actor_task_retried_exceptions, _ActorPool, _MapWorker, ) @@ -1975,6 +1977,37 @@ def test_merge_ray_remote_args_op_wins_on_collision(restore_data_context): assert merged["label_selector"] == {"subcluster": "val", "node": "X"} +@pytest.mark.parametrize( + "actor_args, expected", + [ + # By default the operator gives actors unlimited task retries. + ({}, [ValueError]), + # With ``max_restarts=0`` it leaves ``max_task_retries`` at Ray's 0. + ({"max_restarts": 0}, False), + ({"max_task_retries": 0}, False), + ], +) +def test_actor_task_retried_exceptions_follow_max_task_retries( + restore_data_context, actor_args, expected +): + """An actor task counts as rerun on ``actor_task_retry_on_errors`` only when + its actor's ``max_task_retries`` lets Ray rerun it, so a task that won't + rerun emits its buffered rows before an error the job skips.""" + context = DataContext.get_current() + context.actor_task_retry_on_errors = [ValueError] + actor_args = ActorPoolMapOperator._apply_default_remote_args(actor_args, context) + task_args = ActorPoolMapOperator._apply_default_actor_task_remote_args( + None, context + ) + + retried = _actor_task_retried_exceptions(task_args, actor_args) + + if expected is False: + assert retried is False + else: + assert ValueError in retried + + if __name__ == "__main__": import sys From fbff3518f7f790b08c7089e2d2470823e4c549b2 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Thu, 29 Oct 2026 08:53:09 -0700 Subject: [PATCH 04/13] [Data] Emit a partial block on a system error Ray won't rerun Problem: - A map task counted as rerun on every RaySystemError, even with max_retries=0 or max_task_retries=0. Ray then does not rerun it, so a skipped error dropped the task's buffered rows. Change: - RaySystemError counts as rerun only through the task's retry_exceptions, which Ray Data always extends with it and which are empty when retries are off. - The task-pool retry check adds RaySystemError as cached_remote_fn does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../execution/operators/map_transformer.py | 13 ++--- .../operators/task_pool_map_operator.py | 14 ++++- python/ray/data/tests/test_map_transformer.py | 55 +++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/python/ray/data/_internal/execution/operators/map_transformer.py b/python/ray/data/_internal/execution/operators/map_transformer.py index bf5ec6f6d55c..d729c4cbbf60 100644 --- a/python/ray/data/_internal/execution/operators/map_transformer.py +++ b/python/ray/data/_internal/execution/operators/map_transformer.py @@ -903,18 +903,13 @@ def _emit_rows_before_error(error: Exception) -> bool: and skips as many as were already emitted, so a last block cut short at the error would hide rows of the rerun's block in its place. ``_map_task`` reruns on an error that ``retried_map_errors`` matches while it has attempts - left. Ray reruns a task on a ``RaySystemError`` and on an error its - ``retry_exceptions`` names, which for an actor task come from - ``actor_task_retry_on_errors``. + left. Ray reruns a task, while its retries allow, on an error its + ``retry_exceptions`` names. Ray Data adds ``RaySystemError`` to them, and + for an actor task they come from ``actor_task_retry_on_errors``. """ - import ray.exceptions from ray.data.context import DataContext - may_rerun = ( - _map_task_reruns_on(error) - or isinstance(error, ray.exceptions.RaySystemError) - or _ray_reruns_on(error) - ) + may_rerun = _map_task_reruns_on(error) or _ray_reruns_on(error) return DataContext.get_current().max_errored_blocks != 0 and not may_rerun diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index e33b13fd0923..764fd8f9d9d2 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -24,7 +24,10 @@ _map_task, ) from ray.data._internal.execution.operators.map_transformer import MapTransformer -from ray.data._internal.remote_fn import cached_remote_fn +from ray.data._internal.remote_fn import ( + _add_system_error_to_retry_exceptions, + cached_remote_fn, +) from ray.data._internal.utils.cached_ray_internals import ( get_local_ongoing_lineage_reconstruction_tasks, ) @@ -338,8 +341,13 @@ def _ray_retried_exceptions(ray_remote_args: Dict[str, Any]) -> Union[bool, List """The exceptions Ray reruns a task with ``ray_remote_args`` on. ``cached_remote_fn`` defaults ``max_retries`` to -1 (unlimited); with 0, Ray - reruns on nothing. + reruns on nothing. Otherwise it also reruns on ``RaySystemError``, which + ``cached_remote_fn`` adds to ``retry_exceptions``. """ if ray_remote_args.get("max_retries", -1) == 0: return False - return ray_remote_args.get("retry_exceptions", False) + retried = ray_remote_args.get("retry_exceptions", False) + # A copy, since adding ``RaySystemError`` appends to a list in place. + args = {"retry_exceptions": list(retried) if isinstance(retried, list) else retried} + _add_system_error_to_retry_exceptions(args) + return args["retry_exceptions"] diff --git a/python/ray/data/tests/test_map_transformer.py b/python/ray/data/tests/test_map_transformer.py index 649cc1873e96..a232a3449757 100644 --- a/python/ray/data/tests/test_map_transformer.py +++ b/python/ray/data/tests/test_map_transformer.py @@ -447,6 +447,61 @@ def test_an_error_ray_does_not_rerun_the_task_on_emits_the_buffered_rows_first( next(out) +def _failing_system_error_transformer() -> MapTransformer: + """A transformer that yields two one-row tables into one buffered block, + then raises a ``RaySystemError``.""" + from ray.exceptions import RaySystemError + + def fn(blocks, ctx): + yield pa.table({"x": [0]}) + yield pa.table({"x": [1]}) + raise RaySystemError("lost an object") + + return MapTransformer( + [ + BlockMapTransformFn( + fn, + output_block_size_option=OutputBlockSizeOption.of( + target_max_block_size=1024 * 1024 + ), + ) + ] + ) + + +@pytest.mark.parametrize( + "remote_args, emits_rows", + [ + # Ray Data's default: unlimited retries, ``RaySystemError`` among them. + ({}, False), + # With ``max_retries=0`` Ray reruns on nothing. + ({"max_retries": 0}, True), + ], +) +def test_a_system_error_emits_the_buffered_rows_only_when_ray_wont_rerun( + skip_errors, remote_args, emits_rows +): + """Ray reruns a task on a ``RaySystemError`` only while its retries allow, so + with ``max_retries=0`` the buffered rows come out before the error.""" + from ray.data._internal.execution.operators.task_pool_map_operator import ( + _ray_retried_exceptions, + ) + from ray.exceptions import RaySystemError + + out = _outputs( + _failing_system_error_transformer(), + ray_retried_exceptions=_ray_retried_exceptions(remote_args), + ) + + if emits_rows: + assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [ + 0, + 1, + ] + with pytest.raises(RaySystemError, match="lost an object"): + next(out) + + def test_the_last_map_attempt_emits_the_buffered_rows_first(skip_errors): """On its last attempt ``_map_task`` won't rerun the transform even for an error a pattern matches, so the buffered rows come out first.""" From b546f17e3305ed3b7823e9d6e4f3658373f1c4d9 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 6 Nov 2026 23:42:57 -0800 Subject: [PATCH 05/13] [Data] Keep a fused batcher's rows and rerun by default on a skipped error Problem: - A fused downstream batcher dropped the rows it held when an upstream stage raised an error the job skips. Map then MapBatches(batch_size=4) kept rows 0 to 3 of 0 to 6. - A TaskContext built without retry settings meant "never rerun", so a shuffle reduce task emitted partial output it did not opt in to. - The retry settings were sentinels on TaskContext, and the pattern check was a copy of iterate_with_retry's. - The max_errored_blocks docs still said failed blocks are dropped. Change: - A batch transform ends its input at an error the task skips, hands on a short last batch, then raises the error. - A frozen RerunPolicy replaces the TaskContext retry fields. Its default reruns on any error; map operators and _map_task fill it in. - iterate_with_retry and RerunPolicy share is_retryable_error. - _emit_rows_before_error is renamed _should_emit_buffered_rows. - The context docstring and the docs page say which rows are kept. - Dataset-level tests cover a skipped map_batches error and a fused batcher. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- doc/source/data/execution-configurations.md | 2 +- .../execution/interfaces/task_context.py | 97 +++++++++++++-- .../operators/actor_pool_map_operator.py | 13 ++- .../execution/operators/map_operator.py | 5 +- .../execution/operators/map_transformer.py | 110 +++++++++++------- .../operators/task_pool_map_operator.py | 13 ++- python/ray/data/_internal/util.py | 15 ++- python/ray/data/context.py | 9 +- .../tests/test_actor_pool_map_operator.py | 17 ++- python/ray/data/tests/test_map_transformer.py | 59 +++++++--- .../test_streaming_executor_errored_blocks.py | 55 +++++++++ 11 files changed, 290 insertions(+), 105 deletions(-) diff --git a/doc/source/data/execution-configurations.md b/doc/source/data/execution-configurations.md index 914ea80b2de6..6ce104615269 100644 --- a/doc/source/data/execution-configurations.md +++ b/doc/source/data/execution-configurations.md @@ -60,7 +60,7 @@ ctx.verbose_stats_logs = True Many {class}`~ray.data.DataContext` options are for advanced use cases or debugging, and you usually don't need to modify them. The following are some of the most important options: -* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. Ray Data drops the data in the failed blocks. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors. +* `max_errored_blocks`: The maximum number of blocks that can have errors. A negative value means no limit. Set this option to tolerate application-level exceptions in block processing tasks. UDFs can raise these exceptions, for example on corrupted data samples, and IO errors can also cause them. A failed map task keeps the rows it produced before the error, and Ray Data drops the rest of the task's input. While Ray Data or Ray can still rerun the task, through `retried_map_errors` or the task's `max_retries` and `retry_exceptions`, a failed attempt emits no rows, so the rerun produces each row once. Use this option to keep a long-running job from failing because of a small number of bad blocks. By default, Ray Data tolerates no blocks with errors. * `retried_io_errors`: A list of error message patterns that trigger a retry when reading or writing files. Ray Data matches each pattern first as literal text and then as a regular expression. Use it to handle transient errors from remote storage systems. By default, Ray Data retries on common transient AWS S3 errors. This option replaces the deprecated `write_file_retry_on_errors`. * `verbose_stats_logs`: Whether stats logs are verbose. Verbose logs include fields such as `extra_metrics` in the stats output, which are otherwise excluded. Off by default. * `log_internal_stack_trace`: Whether to write the full internal stack frames from Ray Data and Ray Core to the Ray Data log file when logging a user-code error. Ray Data always omits these internal frames from `stdout`, and by default also omits them from the log file. Set this to `True` to include them in the log file. Off by default. diff --git a/python/ray/data/_internal/execution/interfaces/task_context.py b/python/ray/data/_internal/execution/interfaces/task_context.py index 810165ffd354..db23642eb275 100644 --- a/python/ray/data/_internal/execution/interfaces/task_context.py +++ b/python/ray/data/_internal/execution/interfaces/task_context.py @@ -1,7 +1,18 @@ import contextlib +import dataclasses import threading from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + Optional, + Sequence, + Tuple, + Type, + Union, +) if TYPE_CHECKING: from ray.data._internal.execution.operators.map_transformer import MapTransformer @@ -11,6 +22,78 @@ _thread_local = threading.local() +@dataclass(frozen=True) +class RerunPolicy: + """The errors on which a map task runs again after it raises. + + A rerun produces the task's blocks from the start, so a task emits the rows + it buffered before an error the job skips only when no rerun follows. The + default reruns on any error: a task emits partial output only when its + operator states how Ray retries it. + """ + + # Whether Ray reruns the task on any error, and otherwise the exception + # types it reruns on, matched against the error and its causes. + ray_reruns_any_error: bool = True + ray_retried_types: Tuple[Type[BaseException], ...] = () + + # How many more attempts ``_map_task`` makes, and the ``retried_map_errors`` + # patterns it makes them on. ``None`` matches any error, as for + # ``iterate_with_retry``. + map_attempts_left: int = 0 + map_retried_patterns: Optional[Tuple[str, ...]] = () + + @classmethod + def from_retry_exceptions( + cls, retry_exceptions: Union[bool, Sequence[Type[BaseException]]] + ) -> "RerunPolicy": + """The policy of a task Ray reruns on ``retry_exceptions``. + + ``retry_exceptions`` takes the form of the remote arg: ``True`` for any + error, ``False`` for none, or a list of exception types. + """ + if isinstance(retry_exceptions, bool): + return cls(ray_reruns_any_error=retry_exceptions) + return cls( + ray_reruns_any_error=False, ray_retried_types=tuple(retry_exceptions) + ) + + def with_map_retries( + self, attempts_left: int, patterns: Optional[Sequence[str]] + ) -> "RerunPolicy": + """This policy for an attempt of ``_map_task`` with ``attempts_left``.""" + return dataclasses.replace( + self, + map_attempts_left=attempts_left, + map_retried_patterns=None if patterns is None else tuple(patterns), + ) + + def may_rerun(self, error: Exception) -> bool: + """Whether Ray or ``_map_task`` runs the task again after ``error``.""" + return self._ray_reruns_on(error) or self._map_task_reruns_on(error) + + def _ray_reruns_on(self, error: Exception) -> bool: + if self.ray_reruns_any_error: + return True + # A transform may wrap the error, so its causes count too. + cause: Optional[BaseException] = error + while cause is not None: + if isinstance(cause, self.ray_retried_types): + return True + cause = cause.__cause__ + return False + + def _map_task_reruns_on(self, error: Exception) -> bool: + # Matched as ``_map_task`` matches it: the message with its causes. + from ray._common.retry import format_exception + from ray.data._internal.util import is_retryable_error + + if self.map_attempts_left <= 0: + return False + error_str = format_exception(error, include_cause=True) + return is_retryable_error(error_str, self.map_retried_patterns) + + @dataclass class TaskContext: """This describes the information of a task running block transform.""" @@ -47,15 +130,9 @@ class TaskContext: # Override of the target max-block-size for the task target_max_block_size_override: Optional[int] = None - # The exceptions Ray reruns this task on: its ``retry_exceptions`` remote - # arg, or ``False`` when ``max_retries`` is 0. ``True`` means any exception. - ray_retried_exceptions: Union[bool, List[type]] = False - - # Set by ``_map_task`` for each attempt: how many more times it reruns the - # transform, and the ``retried_map_errors`` patterns it reruns on (``None`` - # means any error). - map_retries_left: int = 0 - map_retried_errors: Optional[List[str]] = field(default_factory=list) + # The errors on which this task runs again. A map operator sets Ray's part + # and ``_map_task`` its own part for each attempt. + rerun_policy: RerunPolicy = field(default_factory=RerunPolicy) # Additional keyword arguments passed to the task. kwargs: Dict[str, Any] = field(default_factory=dict) diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 8278e5ff2305..ef1a51da718c 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -57,6 +57,7 @@ TaskContext, actor_resource_dict, ) +from ray.data._internal.execution.interfaces.task_context import RerunPolicy from ray.data._internal.execution.node_trackers.actor_location import ( ActorLocationTracker, get_or_create_actor_location_tracker, @@ -118,10 +119,10 @@ def get_map_worker_cls_name(op_name: str) -> str: return f"MapWorker({op_name})" -def _actor_task_retried_exceptions( +def _actor_task_rerun_policy( task_args: Dict[str, Any], actor_args: Dict[str, Any] -) -> Union[bool, List[type]]: - """The exceptions Ray reruns an actor task on. +) -> RerunPolicy: + """The errors on which Ray reruns an actor task. Ray reruns an actor task only while ``max_task_retries`` allows it. The task option overrides the actor's, and Ray defaults it to 0, as it stays when @@ -131,8 +132,8 @@ def _actor_task_retried_exceptions( "max_task_retries", actor_args.get("max_task_retries", 0) ) if max_task_retries == 0: - return False - return task_args.get("retry_exceptions", False) + return RerunPolicy.from_retry_exceptions(False) + return RerunPolicy.from_retry_exceptions(task_args.get("retry_exceptions", False)) class ActorPoolMapOperator(MapOperator, ReportsExtraResourceUsage): @@ -527,7 +528,7 @@ def _try_schedule_tasks_internal(self) -> int: task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, - ray_retried_exceptions=_actor_task_retried_exceptions( + rerun_policy=_actor_task_rerun_policy( actor_task_args, self._ray_remote_args ), ) diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index c528a2bce89d..44481cd0210f 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -1001,8 +1001,9 @@ def transform_iter_factory(): # Tell the transform whether this attempt can still be rerun, so it # knows whether to emit its buffered rows before an error. attempts_started += 1 - ctx.map_retries_left = max_attempts - attempts_started - ctx.map_retried_errors = to_match + ctx.rerun_policy = ctx.rerun_policy.with_map_retries( + max_attempts - attempts_started, to_match + ) # Clear any per-task custom stats before each attempt (the reporter # is reused across retries of this task), so a prior attempt's stats # can't leak into this one. A producing transform repopulates it diff --git a/python/ray/data/_internal/execution/operators/map_transformer.py b/python/ray/data/_internal/execution/operators/map_transformer.py index d729c4cbbf60..0bcbe0077f8f 100644 --- a/python/ray/data/_internal/execution/operators/map_transformer.py +++ b/python/ray/data/_internal/execution/operators/map_transformer.py @@ -19,7 +19,10 @@ from ray._common.utils import env_integer from ray.data._internal.block_batching.block_batching import batch_blocks -from ray.data._internal.execution.interfaces.task_context import TaskContext +from ray.data._internal.execution.interfaces.task_context import ( + RerunPolicy, + TaskContext, +) from ray.data._internal.output_buffer import BlockOutputBuffer, OutputBlockSizeOption from ray.data.block import ( BatchFormat, @@ -763,6 +766,13 @@ def __init__( self._target_batch_size_bytes = target_batch_size_bytes def _pre_process(self, blocks: Iterable[Block]) -> Iterable[MapTransformFnData]: + # An upstream stage of a fused task may raise an error the task skips. + # The batcher then sees the end of its input, so the rows it holds come + # out as a short batch before the error is raised. + input_blocks = _InputEndingAtSkippedError(blocks) + return _ErrorRaisedAtEnd(self._batches(input_blocks), input_blocks) + + def _batches(self, blocks: Iterable[Block]) -> Iterable[MapTransformFnData]: # TODO make batch-udf zero-copy by default if self._batch_size == "auto": batch_size, blocks = _compute_auto_batch_size( @@ -839,7 +849,7 @@ class _BlockShapingIterator(Iterator[Block]): When the results raise an error the job skips, the rows already buffered come out as a last block before the error, so the skipped task keeps the - output it produced (see ``_emit_rows_before_error``). + output it produced (see ``_should_emit_buffered_rows``). """ def __init__( @@ -888,66 +898,76 @@ def __next__(self) -> Block: except Exception as e: if not self._buffer.has_buffered_rows(): raise - if not _emit_rows_before_error(e): + if not _should_emit_buffered_rows(e): raise self._error = e self._buffer.finalize() self._finalized = True -def _emit_rows_before_error(error: Exception) -> bool: +def _should_emit_buffered_rows(error: Exception) -> bool: """Whether a map task that raised ``error`` first emits its buffered rows. Only when the job skips the error (``DataContext.max_errored_blocks`` is not - 0) and the task cannot run again. A rerun produces its blocks from the start - and skips as many as were already emitted, so a last block cut short at the - error would hide rows of the rerun's block in its place. ``_map_task`` - reruns on an error that ``retried_map_errors`` matches while it has attempts - left. Ray reruns a task, while its retries allow, on an error its - ``retry_exceptions`` names. Ray Data adds ``RaySystemError`` to them, and - for an actor task they come from ``actor_task_retry_on_errors``. + 0) and the task does not run again. A rerun produces its blocks from the + start and skips as many as were already emitted, so a last block cut short + at the error would hide rows of the rerun's block in its place. The task's + ``RerunPolicy`` says when it runs again. """ from ray.data.context import DataContext - may_rerun = _map_task_reruns_on(error) or _ray_reruns_on(error) - return DataContext.get_current().max_errored_blocks != 0 and not may_rerun + if DataContext.get_current().max_errored_blocks == 0: + return False + task_context = TaskContext.get_current() + policy = task_context.rerun_policy if task_context else RerunPolicy() + return not policy.may_rerun(error) -def _ray_reruns_on(error: Exception) -> bool: - """Whether Ray reruns the running task on ``error``. +class _InputEndingAtSkippedError(Iterator[Any]): + """Input that ends where an upstream stage raises an error the task skips. - Checks ``error`` and its causes against the task's ``retry_exceptions``, so - an error a transform wraps still counts. - """ - task_context = TaskContext.get_current() - retried = task_context.ray_retried_exceptions if task_context else False - if isinstance(retried, bool): - return retried - cause: Optional[BaseException] = error - while cause is not None: - if isinstance(cause, tuple(retried)): - return True - cause = cause.__cause__ - return False - - -def _map_task_reruns_on(error: Exception) -> bool: - """Whether ``_map_task`` reruns the running attempt on ``error``. - - It does while attempts are left and the error's message, with its causes, - matches a ``retried_map_errors`` pattern, as ``iterate_with_retry`` checks - it. ``None`` patterns match any error. + The error is held, so a stage that buffers this input, such as a batcher, + hands on the rows it holds as at the end of its input. + ``_ErrorRaisedAtEnd`` then raises the held error. """ - from ray._common.retry import format_exception, matches_error - task_context = TaskContext.get_current() - if task_context is None or task_context.map_retries_left <= 0: - return False - patterns = task_context.map_retried_errors - if patterns is None: - return True - text = format_exception(error, include_cause=True) - return any(matches_error(pattern, text) for pattern in patterns) + def __init__(self, items: Iterable[Any]): + self._items = iter(items) + self.error: Optional[Exception] = None + + def __iter__(self) -> "_InputEndingAtSkippedError": + return self + + def __next__(self) -> Any: + try: + return next(self._items) + except StopIteration: + raise + except Exception as e: + if not _should_emit_buffered_rows(e): + raise + self.error = e + raise StopIteration + + +class _ErrorRaisedAtEnd(Iterator[Any]): + """Passes ``items`` through, then raises the error ``source`` held, if any.""" + + def __init__(self, items: Iterable[Any], source: _InputEndingAtSkippedError): + self._items = iter(items) + self._source = source + + def __iter__(self) -> "_ErrorRaisedAtEnd": + return self + + def __next__(self) -> Any: + try: + return next(self._items) + except StopIteration: + if self._source.error is None: + raise + error, self._source.error = self._source.error, None + raise error class _RowBasedIterator(Iterator[Row]): diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index 764fd8f9d9d2..e7bf7c44e33d 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -1,6 +1,6 @@ import copy import warnings -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple if TYPE_CHECKING: import pyarrow as pa @@ -19,6 +19,7 @@ TaskContext, task_resource_dict, ) +from ray.data._internal.execution.interfaces.task_context import RerunPolicy from ray.data._internal.execution.operators.map_operator import ( MapOperator, _map_task, @@ -192,7 +193,7 @@ def _try_schedule_task(self, bundle: RefBundle, strict: bool): task_idx=self._next_data_task_idx, op_name=self.name, target_max_block_size_override=self.target_max_block_size_override, - ray_retried_exceptions=_ray_retried_exceptions(dynamic_ray_remote_args), + rerun_policy=_task_rerun_policy(dynamic_ray_remote_args), ) dynamic_ray_remote_args["name"] = self.name logical_usage = ExecutionResources.from_resource_dict(dynamic_ray_remote_args) @@ -337,17 +338,17 @@ def all_inputs_done(self): ) -def _ray_retried_exceptions(ray_remote_args: Dict[str, Any]) -> Union[bool, List[type]]: - """The exceptions Ray reruns a task with ``ray_remote_args`` on. +def _task_rerun_policy(ray_remote_args: Dict[str, Any]) -> RerunPolicy: + """The errors on which Ray reruns a task with ``ray_remote_args``. ``cached_remote_fn`` defaults ``max_retries`` to -1 (unlimited); with 0, Ray reruns on nothing. Otherwise it also reruns on ``RaySystemError``, which ``cached_remote_fn`` adds to ``retry_exceptions``. """ if ray_remote_args.get("max_retries", -1) == 0: - return False + return RerunPolicy.from_retry_exceptions(False) retried = ray_remote_args.get("retry_exceptions", False) # A copy, since adding ``RaySystemError`` appends to a list in place. args = {"retry_exceptions": list(retried) if isinstance(retried, list) else retried} _add_system_error_to_retry_exceptions(args) - return args["retry_exceptions"] + return RerunPolicy.from_retry_exceptions(args["retry_exceptions"]) diff --git a/python/ray/data/_internal/util.py b/python/ray/data/_internal/util.py index 1aadb7e9ec8a..e4072a2cc81d 100644 --- a/python/ray/data/_internal/util.py +++ b/python/ray/data/_internal/util.py @@ -23,6 +23,7 @@ Iterator, List, Optional, + Sequence, Tuple, TypeVar, Union, @@ -1657,6 +1658,15 @@ def _annotate_exception_with_retry_context( return exc +def is_retryable_error(error_str: str, match: Optional[Sequence[str]]) -> bool: + """Whether ``iterate_with_retry`` retries an error formatted as ``error_str``. + + ``match`` holds patterns, each checked as a substring, then as a regex. + ``None`` retries any error. + """ + return match is None or any(matches_error(pattern, error_str) for pattern in match) + + def iterate_with_retry( iterable_factory: Callable[[], Iterable], description: str, @@ -1707,10 +1717,7 @@ def iterate_with_retry( return except Exception as e: error_str = format_exception(e, include_cause=unwrap_cause) - is_retryable = match is None or any( - matches_error(pattern, error_str) for pattern in match - ) - if is_retryable and attempt + 1 < max_attempts: + if is_retryable_error(error_str, match) and attempt + 1 < max_attempts: # Retry with binary expoential backoff with random jitter. backoff = min((2 ** (attempt + 1)), max_backoff_s) * random.random() total_backoff_s += backoff diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 20f7c3f3a744..f5338ec0f88d 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -860,9 +860,12 @@ class DataContext: max_errored_blocks: Max number of blocks that are allowed to have errors, unlimited if negative. This option allows application-level exceptions in block processing tasks. These exceptions may be caused by UDFs (e.g., due to - corrupted data samples) or IO errors. Data in the failed blocks are dropped. - This option can be useful to prevent a long-running job from failing due to - a small number of bad blocks. + corrupted data samples) or IO errors. A failed map task keeps the rows it + produced before the error and drops the rest of its input. While the + task can still be rerun (``retried_map_errors``, or Ray's ``max_retries`` + with ``retry_exceptions``), a failed attempt emits no rows, so the rerun + produces each row once. This option can be useful to prevent a + long-running job from failing due to a small number of bad blocks. log_internal_stack_trace: Whether to write the full Ray Data/Ray Core internal code stack frames to the Ray Data log file when logging a user-code error. These internal frames are always omitted from diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index 1176d02361c1..03085ba79fb9 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -39,7 +39,7 @@ from ray.data._internal.execution.interfaces.task_context import TaskContext from ray.data._internal.execution.operators.actor_pool_map_operator import ( ActorPoolMapOperator, - _actor_task_retried_exceptions, + _actor_task_rerun_policy, _ActorPool, _MapWorker, ) @@ -1978,17 +1978,17 @@ def test_merge_ray_remote_args_op_wins_on_collision(restore_data_context): @pytest.mark.parametrize( - "actor_args, expected", + "actor_args, reruns", [ # By default the operator gives actors unlimited task retries. - ({}, [ValueError]), + ({}, True), # With ``max_restarts=0`` it leaves ``max_task_retries`` at Ray's 0. ({"max_restarts": 0}, False), ({"max_task_retries": 0}, False), ], ) -def test_actor_task_retried_exceptions_follow_max_task_retries( - restore_data_context, actor_args, expected +def test_actor_task_rerun_policy_follows_max_task_retries( + restore_data_context, actor_args, reruns ): """An actor task counts as rerun on ``actor_task_retry_on_errors`` only when its actor's ``max_task_retries`` lets Ray rerun it, so a task that won't @@ -2000,12 +2000,9 @@ def test_actor_task_retried_exceptions_follow_max_task_retries( None, context ) - retried = _actor_task_retried_exceptions(task_args, actor_args) + policy = _actor_task_rerun_policy(task_args, actor_args) - if expected is False: - assert retried is False - else: - assert ValueError in retried + assert policy.may_rerun(ValueError("bad input")) is reruns if __name__ == "__main__": diff --git a/python/ray/data/tests/test_map_transformer.py b/python/ray/data/tests/test_map_transformer.py index a232a3449757..885be3418300 100644 --- a/python/ray/data/tests/test_map_transformer.py +++ b/python/ray/data/tests/test_map_transformer.py @@ -7,7 +7,10 @@ import pyarrow as pa import pytest -from ray.data._internal.execution.interfaces.task_context import TaskContext +from ray.data._internal.execution.interfaces.task_context import ( + RerunPolicy, + TaskContext, +) from ray.data._internal.execution.operators.map_transformer import ( BatchMapTransformFn, BlockMapTransformFn, @@ -317,15 +320,18 @@ def fn(blocks, ctx): ) -def _outputs(transformer, **retry_settings): +# A task that Ray does not rerun, for the tests of ``_map_task``'s own retries. +NO_RAY_RERUN = RerunPolicy(ray_reruns_any_error=False) + + +def _outputs(transformer, rerun_policy=NO_RAY_RERUN): """The transformer's output blocks for one one-row input block, in a task - whose ``TaskContext`` carries ``retry_settings`` (how ``_map_task`` and Ray - rerun it). + that runs again as ``rerun_policy`` says. A generator, so the task context is current while the blocks are drawn, as ``_map_task`` sets it. """ - ctx = TaskContext(task_idx=0, op_name="test", **retry_settings) + ctx = TaskContext(task_idx=0, op_name="test", rerun_policy=rerun_policy) with TaskContext.current(ctx): yield from transformer.apply_transform( iter([pa.table({"x": [0]})]), ctx, clock=TransformClock() @@ -353,6 +359,21 @@ def test_a_skipped_error_emits_the_buffered_rows_first(skip_errors): next(out) +def test_a_task_without_retry_settings_raises_without_a_partial_block(skip_errors): + """A task whose ``TaskContext`` does not state its retry settings may be + rerun, so its buffered rows are not emitted even when the job skips errors.""" + ctx = TaskContext(task_idx=0, op_name="test") + transformer = _failing_block_transformer(tables_before_error=2) + + with TaskContext.current(ctx): + out = transformer.apply_transform( + iter([pa.table({"x": [0]})]), ctx, clock=TransformClock() + ) + + with pytest.raises(ValueError, match="bad input"): + next(out) + + def test_an_error_that_fails_the_job_emits_no_partial_block(): """With ``max_errored_blocks=0``, the default, the job fails anyway, so the transform raises without emitting its buffered rows.""" @@ -378,8 +399,7 @@ def test_a_transform_that_may_rerun_raises_without_a_partial_block(skip_errors): # ``None`` patterns retry any error, as ``retried_map_errors=True`` sets them. out = _outputs( _failing_block_transformer(tables_before_error=2), - map_retries_left=1, - map_retried_errors=None, + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=None), ) with pytest.raises(ValueError, match="bad input"): @@ -394,8 +414,7 @@ def test_an_error_no_retry_pattern_matches_emits_the_buffered_rows_first( out first.""" out = _outputs( _failing_block_transformer(tables_before_error=2), - map_retries_left=1, - map_retried_errors=["connection reset"], + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["connection reset"]), ) assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] @@ -410,8 +429,7 @@ def test_an_error_a_retry_pattern_matches_raises_without_a_partial_block( transform, so its buffered rows are not emitted.""" out = _outputs( _failing_block_transformer(tables_before_error=2), - map_retries_left=1, - map_retried_errors=["bad input"], + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["bad input"]), ) with pytest.raises(ValueError, match="bad input"): @@ -425,7 +443,7 @@ def test_an_error_ray_reruns_the_task_on_raises_without_a_partial_block( buffered rows are not emitted, as for ``retried_map_errors``.""" out = _outputs( _failing_block_transformer(tables_before_error=2), - ray_retried_exceptions=[ValueError], + RerunPolicy.from_retry_exceptions([ValueError]), ) with pytest.raises(ValueError, match="bad input"): @@ -439,7 +457,7 @@ def test_an_error_ray_does_not_rerun_the_task_on_emits_the_buffered_rows_first( so its buffered rows come out first.""" out = _outputs( _failing_block_transformer(tables_before_error=2), - ray_retried_exceptions=[KeyError], + RerunPolicy.from_retry_exceptions([KeyError]), ) assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] @@ -484,13 +502,13 @@ def test_a_system_error_emits_the_buffered_rows_only_when_ray_wont_rerun( """Ray reruns a task on a ``RaySystemError`` only while its retries allow, so with ``max_retries=0`` the buffered rows come out before the error.""" from ray.data._internal.execution.operators.task_pool_map_operator import ( - _ray_retried_exceptions, + _task_rerun_policy, ) from ray.exceptions import RaySystemError out = _outputs( _failing_system_error_transformer(), - ray_retried_exceptions=_ray_retried_exceptions(remote_args), + _task_rerun_policy(remote_args), ) if emits_rows: @@ -507,8 +525,7 @@ def test_the_last_map_attempt_emits_the_buffered_rows_first(skip_errors): error a pattern matches, so the buffered rows come out first.""" out = _outputs( _failing_block_transformer(tables_before_error=2), - map_retries_left=0, - map_retried_errors=["bad input"], + NO_RAY_RERUN.with_map_retries(attempts_left=0, patterns=["bad input"]), ) assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] @@ -522,8 +539,14 @@ def test_actor_retry_settings_do_not_stop_a_task_pool_map_from_emitting( """``actor_task_retry_on_errors`` applies to actor tasks only, through their ``retry_exceptions``. A task that Ray won't rerun still emits its buffered rows when the setting is on.""" + from ray.data._internal.execution.operators.task_pool_map_operator import ( + _task_rerun_policy, + ) + monkeypatch.setattr(skip_errors, "actor_task_retry_on_errors", True) - out = _outputs(_failing_block_transformer(tables_before_error=2)) + out = _outputs( + _failing_block_transformer(tables_before_error=2), _task_rerun_policy({}) + ) assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] with pytest.raises(ValueError, match="bad input"): diff --git a/python/ray/data/tests/test_streaming_executor_errored_blocks.py b/python/ray/data/tests/test_streaming_executor_errored_blocks.py index ed6560ea177f..ee3741744357 100644 --- a/python/ray/data/tests/test_streaming_executor_errored_blocks.py +++ b/python/ray/data/tests/test_streaming_executor_errored_blocks.py @@ -99,6 +99,61 @@ def flaky(batch): assert sorted(row["id"] for row in ds.take_all()) == list(range(10)) +NUM_ROWS = 10 # rows of the one-block datasets below, one read task +FAILING_ROW = 7 # the row whose UDF call raises + + +def fail_at_failing_row(batch): + """A batch UDF that raises on the batch holding ``FAILING_ROW``.""" + if FAILING_ROW in batch["id"]: + raise ValueError(f"bad row {FAILING_ROW}") + return batch + + +def fail_row_at_failing_row(row): + """A row UDF that raises on ``FAILING_ROW``.""" + if row["id"] == FAILING_ROW: + raise ValueError(f"bad row {FAILING_ROW}") + return row + + +@pytest.mark.parametrize("max_errored_blocks", [1, -1]) +def test_a_skipped_error_keeps_the_rows_a_task_produced_before_it( + restore_data_context, max_errored_blocks +): + """A one-task ``map_batches`` that fails at the batch holding row 7 keeps the + rows of the batches before it when the job skips the error, with one error + allowed and with any number.""" + ctx = ray.data.DataContext.get_current() + ctx.max_errored_blocks = max_errored_blocks + + ds = ray.data.range(NUM_ROWS, override_num_blocks=1).map_batches( + fail_at_failing_row, batch_size=2 + ) + rows = sorted(row["id"] for row in ds.take_all()) + + # Batches of two: [6, 7] fails, so the batches before it, rows 0 to 5, stay. + assert rows == list(range(6)) + + +def test_a_skipped_error_keeps_the_rows_a_fused_batcher_holds(restore_data_context): + """In one fused task, ``map`` fails at row 7 while the downstream + ``map_batches`` batcher holds rows 4 to 6 of an unfinished batch of four. + The batcher hands them on as a short batch, so every row before the error + is kept.""" + ctx = ray.data.DataContext.get_current() + ctx.max_errored_blocks = 1 + + ds = ( + ray.data.range(NUM_ROWS, override_num_blocks=1) + .map(fail_row_at_failing_row) + .map_batches(lambda batch: batch, batch_size=4) + ) + rows = sorted(row["id"] for row in ds.take_all()) + + assert rows == list(range(FAILING_ROW)) + + if __name__ == "__main__": import sys From 18be598a1fd75ec43f238dc83c231c07459aab60 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 6 Nov 2026 23:46:00 -0800 Subject: [PATCH 06/13] [Data] read_mcap: skip only corrupt chunks and raise other errors at once Problem: - A corrupt chunk read back for a declaration was always skipped. When it alone declared a schema, rows came out with no schema, even with max_errored_blocks=0. - Any error while a chunk was read, such as the reader's own missing channel error, a MemoryError or a bug, was reported as a skipped corrupt chunk. - A task joined every failed file into one ValueError caused by the first. A later OSError was hidden, so the task was skipped instead of rerun. Change: - is_corrupt_data_error says which errors mean the bytes do not decode. Only those are skipped, and only around decoding a chunk. - A chunk read back for a declaration is skipped only when corrupt chunks are skipped. A schema still missing after the look-back raises, naming the corrupt chunk. - A file's corrupt data is deferred to the task's one error. Any other error raises at once. _read_owned_file holds one file's read. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_chunks.py | 83 ++++++++++------ .../datasource_v2/formats/mcap/mcap_io.py | 34 ++++++- .../datasource_v2/formats/mcap/mcap_reader.py | 79 +++++++++------ .../tests/unit/datasource_v2/mcap_testing.py | 32 +++++++ .../unit/datasource_v2/test_mcap_reader.py | 96 ++++++++++++++++++- 5 files changed, 258 insertions(+), 66 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 043a788d6fe4..160124924f3a 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -9,7 +9,7 @@ """ import heapq -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, Any, @@ -26,6 +26,7 @@ from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( ChunkReads, OnCorrupt, + is_corrupt_data_error, read_chunk, read_chunks, ) @@ -36,7 +37,7 @@ ) if TYPE_CHECKING: - from mcap.records import Channel, Chunk, ChunkIndex, Message, Schema + from mcap.records import Channel, Chunk, ChunkIndex, McapRecord, Message, Schema from mcap.summary import Summary # One selected message as the reader sees it: its schema (``None`` for a @@ -63,32 +64,46 @@ class _Declared: a channel declared only in an earlier chunk serves the later ones. ``scanned`` holds the offsets of the chunks already read back for records. ``file_summary`` returns the file's whole summary, whose chunk indexes say - where to read back. + where to read back. With ``skip_corrupt``, a chunk read back that does not + decode is skipped and its offset kept in ``undecodable``. """ channels: Dict[int, "Channel"] schemas: Dict[int, "Schema"] scanned: Set[int] file_summary: Callable[[], "Summary"] + skip_corrupt: bool = False + undecodable: List[int] = field(default_factory=list) + + def undecodable_note(self) -> str: + """A sentence naming the chunks read back that did not decode, if any.""" + if not self.undecodable: + return "" + offsets = ", ".join(str(offset) for offset in self.undecodable) + if len(self.undecodable) == 1: + return f" The chunk at byte offset {offsets}, read back for it, is corrupt." + return f" The chunks at byte offsets {offsets}, read back for it, are corrupt." def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None: """Add the channel and schema records of one chunk to ``declared``. A record already in ``declared`` is not replaced. The chunk is marked as - scanned. A chunk that does not decode adds nothing, so the walk goes on to - the next earlier chunk; the task that owns it reports it. An error reading - ``f`` still raises. + scanned. With ``declared.skip_corrupt``, a chunk that does not decode adds + nothing and is noted in ``declared.undecodable``, so the walk goes on to the + next earlier chunk. It is not reported as skipped: the task that owns it + does that. Any other error, and every error without ``skip_corrupt``, raises. """ from mcap.records import Channel, Schema from mcap.stream_reader import breakup_chunk declared.scanned.add(chunk_index.chunk_start_offset) try: - records = list(breakup_chunk(read_chunk(f, chunk_index))) - except OSError: - raise - except Exception: # noqa: BLE001 - a corrupt chunk declares nothing here + records = breakup_chunk(read_chunk(f, chunk_index)) + except Exception as e: + if not declared.skip_corrupt or not is_corrupt_data_error(e): + raise + declared.undecodable.append(chunk_index.chunk_start_offset) return for record in records: if isinstance(record, Channel): @@ -148,6 +163,7 @@ def iter_chunks( dict(summary.schemas), set(), file_summary or (lambda: summary), + skip_corrupt=on_corrupt is not None, ) if not self._log_time_order: chunks = read_chunks(f, chunk_indexes, self._chunk_reads, on_corrupt) @@ -253,26 +269,28 @@ def _decoded_messages( on_corrupt: Optional[OnCorrupt], with_index: bool = False, ) -> Iterator[Any]: - """``_chunk_messages``, or the messages before a decoding error. + """Decode ``chunk`` and yield ``_chunk_messages`` of its records. ``chunk`` is ``None`` when its record did not parse, which - ``read_chunks`` has already handed to ``on_corrupt``. A chunk that fails - while it is decompressed or parsed stops there and goes to - ``on_corrupt``. Without ``on_corrupt``, or for an error reading ``f``, - the error raises. + ``read_chunks`` has already handed to ``on_corrupt``. A chunk that does + not decompress or parse (``is_corrupt_data_error``) goes to + ``on_corrupt`` and yields nothing. Without ``on_corrupt`` it raises, as + does any error after the chunk is decoded. """ + from mcap.stream_reader import breakup_chunk + if chunk is None: return try: - yield from self._chunk_messages( - f, path, summary, chunk_index, chunk, selected, declared, with_index - ) - except OSError: - raise - except Exception as e: # noqa: BLE001 - a corrupt chunk costs only itself - if on_corrupt is None: + records = breakup_chunk(chunk) + except Exception as e: + if on_corrupt is None or not is_corrupt_data_error(e): raise on_corrupt(chunk_index, e) + return + yield from self._chunk_messages( + f, path, summary, chunk_index, records, selected, declared, with_index + ) def _chunk_messages( self, @@ -280,12 +298,12 @@ def _chunk_messages( path: str, summary: "Summary", chunk_index: "ChunkIndex", - chunk: "Chunk", + records: List["McapRecord"], selected: Set[int], declared: "_Declared", with_index: bool = False, ) -> Iterator[Any]: - """Decompress one chunk and yield its selected messages in file order. + """Yield the selected messages of one decoded chunk, in file order. Channel and schema records inside the chunk take precedence over the summary's. ``row_id`` counts every message record of the chunk, so it @@ -293,10 +311,9 @@ def _chunk_messages( yielded as ``(index in chunk, message)``. """ from mcap.records import Channel, Message, Schema - from mcap.stream_reader import breakup_chunk index = -1 - for record in breakup_chunk(chunk): + for record in records: if isinstance(record, Message): index += 1 channel, schema = self._channel_and_schema( @@ -336,16 +353,24 @@ def _channel_and_schema( raise ValueError( f"MCAP file {path!r} has a message on channel " f"{message.channel_id}, which neither the summary nor any " - "chunk up to this one declares." + f"chunk up to this one declares.{declared.undecodable_note()}" ) - if channel.schema_id and channel.schema_id not in declared.schemas: + if not channel.schema_id: + return channel, None + if channel.schema_id not in declared.schemas: # The schema record may be only in an earlier chunk # (``repeat_schemas=False``). ``message_types`` and the schema # columns need it. self._find_in_earlier_chunks( f, chunk_index, declared, schema_id=channel.schema_id ) - schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None + schema = declared.schemas.get(channel.schema_id) + if schema is None: + raise ValueError( + f"MCAP file {path!r} has channel {channel.topic!r} with schema " + f"{channel.schema_id}, which neither the summary nor any chunk " + f"up to this one declares.{declared.undecodable_note()}" + ) return channel, schema def _channel_is_selected( diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py index cd225f9ca981..37f48fba22e9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_io.py @@ -120,6 +120,33 @@ def plan_chunk_reads( OnCorrupt = Callable[["ChunkIndex", Exception], None] +def is_corrupt_data_error(error: BaseException) -> bool: + """Whether ``error`` says MCAP bytes do not decode, rather than that a read + failed or the reader has a bug. + + Parsing a malformed record raises an mcap ``McapError``, ``struct.error`` or + a ``ValueError`` such as ``UnicodeDecodeError``. Decompressing raises + ``zstandard.ZstdError``, or for lz4 a ``RuntimeError`` whose message starts + with ``LZ4F_``. An ``OSError`` and a missing codec + (``UnsupportedCompressionError``) are not corruption. + """ + import struct + + from mcap.exceptions import McapError, UnsupportedCompressionError + + if isinstance(error, (OSError, UnsupportedCompressionError)): + return False + if isinstance(error, (McapError, struct.error, ValueError)): + return True + if type(error) is RuntimeError: + return str(error).startswith("LZ4F_") + try: + import zstandard + except ImportError: + return False + return isinstance(error, zstandard.ZstdError) + + def read_chunks( f: Any, chunk_indexes: Iterable["ChunkIndex"], @@ -267,11 +294,12 @@ def _parse_chunk_or_report( on_corrupt: Optional[OnCorrupt], ) -> Optional["Chunk"]: """``_parse_chunk``, or ``None`` after handing a corrupt record to - ``on_corrupt``. Without ``on_corrupt`` the error raises.""" + ``on_corrupt``. Without ``on_corrupt``, or for an error that is not + corruption (``is_corrupt_data_error``), the error raises.""" try: return _parse_chunk(stream, position) - except Exception as e: # noqa: BLE001 - any parse failure is a corrupt record - if on_corrupt is None: + except Exception as e: + if on_corrupt is None or not is_corrupt_data_error(e): raise on_corrupt(chunk_index, e) return None diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 6984dfc0632b..9ca565cd2cbd 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -45,6 +45,7 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ( buffered, + is_corrupt_data_error, parse_summary_records, read_summary_from, ) @@ -143,48 +144,65 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: When ``DataContext.max_errored_blocks`` lets errors be skipped, a corrupt chunk costs only itself: the reader skips it and reads the - file's other chunks. A file that fails as a whole does not stop the - task's other files. Once they are read, one error names every file and - chunk that failed, so the task counts once against the budget. - Otherwise the first failure fails the task at once. + file's other chunks. A file whose data does not decode as a whole does + not stop the task's other files. Once they are read, one error names + every file and chunk that is corrupt, so the task counts once against + the budget. Any other error, such as a failed read, raises at once, so + the task's retry settings see it. Without skipping, the first failure + fails the task at once. """ from ray.data.context import DataContext if len(input_split) == 0: return - context = DataContext.get_current() - retried_io_errors = context.retried_io_errors - read_past_failures = context.max_errored_blocks != 0 + skip_corrupt = DataContext.get_current().max_errored_blocks != 0 remaining = self._limit failures: List[Tuple[str, Exception]] = [] for owned in _owned_files(input_split): if remaining is not None and remaining <= 0: break - # The chunks skipped as corrupt, by offset. Each attempt starts it - # afresh, so it holds the last attempt's. - corrupt: Optional[Dict[int, Exception]] = {} if read_past_failures else None - tables = iterate_with_retry( - partial(self._read_file, owned, remaining, corrupt), - f"read MCAP file {owned.path}", - match=retried_io_errors, - ) - try: - for table in tables: - if remaining is not None: - remaining -= table.num_rows - yield table - except Exception as e: # noqa: BLE001 - raised once the others are read - if not read_past_failures: - raise - failures.append((owned.path, e)) - else: - if corrupt: - failures.append( - (owned.path, _corrupt_chunks_error(owned.path, corrupt)) - ) + for table in self._read_owned_file( + owned, remaining, skip_corrupt, failures + ): + if remaining is not None: + remaining -= table.num_rows + yield table if failures: raise _read_error(failures) + def _read_owned_file( + self, + owned: "_OwnedFile", + limit: Optional[int], + skip_corrupt: bool, + failures: List[Tuple[str, Exception]], + ) -> Iterator[pa.Table]: + """Yield the tables of one file, retrying as ``retried_io_errors`` allows. + + With ``skip_corrupt``, the file's corrupt data is appended to + ``failures`` rather than raised: its skipped chunks, or its own error + when it does not decode as a whole (``is_corrupt_data_error``). + """ + from ray.data.context import DataContext + + # The chunks skipped as corrupt, by offset. Each attempt starts it + # afresh, so it holds the last attempt's. + corrupt: Optional[Dict[int, Exception]] = {} if skip_corrupt else None + tables = iterate_with_retry( + partial(self._read_file, owned, limit, corrupt), + f"read MCAP file {owned.path}", + match=DataContext.get_current().retried_io_errors, + ) + try: + yield from tables + except Exception as e: + if not skip_corrupt or not is_corrupt_data_error(e): + raise + failures.append((owned.path, e)) + return + if corrupt: + failures.append((owned.path, _corrupt_chunks_error(owned.path, corrupt))) + # -- one file ---------------------------------------------------------- def _read_file( @@ -396,10 +414,11 @@ def _corrupt_chunks_error(path: str, corrupt: Dict[int, Exception]) -> ValueErro def _read_error(failures: List[Tuple[str, Exception]]) -> Exception: - """The error for the files a task could not read in full. + """The error for the files whose data a task found corrupt. A single failure is its own error, whose message already names the file. Several become one ``ValueError`` naming each file, caused by the first. + Every failure is corruption, so no rerun would read any of them. """ if len(failures) == 1: return failures[0][1] diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index a3bdd6950288..d659c49791e1 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -162,6 +162,38 @@ def declare(topic, name): writer.finish() +UNDECLARED_CHANNEL = 99 # the channel id ``write_undeclared_channel`` never declares + + +def write_undeclared_channel(path): + """Write one chunk that holds a message on ``/a``, then a message on channel + ``UNDECLARED_CHANNEL``, which no record of the file declares. + + The chunk decodes, but the reader cannot place its second message. + """ + from mcap.writer import CompressionType, IndexType, Writer + + with open(path, "wb") as stream: + writer = Writer( + stream, compression=CompressionType.ZSTD, index_types=IndexType.ALL + ) + writer.start(profile="", library="ray-test") + schema_id = writer.register_schema( + name="test_schema", encoding="jsonschema", data=b'{"type": "object"}' + ) + channel_id = writer.register_channel( + schema_id=schema_id, topic="/a", message_encoding="json" + ) + for i, channel in enumerate([channel_id, UNDECLARED_CHANNEL]): + writer.add_message( + channel_id=channel, + log_time=BASE_TIME + i * STEP, + publish_time=BASE_TIME + i * STEP, + data=json.dumps({"seq": i}).encode(), + ) + writer.finish() + + MIXED_TOPICS = ("/a", "/b", "/raw", "/ü") # channels of ``write_mixed_channels`` SHARED_SCHEMA = b"float64 x\n" * 30 # the definition /a and /b both use diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index bcebdefb8a8a..d68cfdcf4957 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -37,6 +37,7 @@ MIXED_TOPICS, SHARED_SCHEMA, STEP, + UNDECLARED_CHANNEL, SeekLog, as_read_task, chunks_read, @@ -56,6 +57,7 @@ write_mcap, write_mcap_without_summary_schemas, write_mixed_channels, + write_undeclared_channel, ) pytestmark = pytest.mark.skipif( @@ -163,7 +165,7 @@ def read_one_task(paths, max_errored_blocks=1, **options): datasource = MCAPDatasourceV2(paths, include_paths=True, **options) # preserve_order keeps the files in the order given; the listing threads # otherwise finish in any order. - task = FileManifest.concat(list_manifests(datasource, preserve_order=True)) + task = as_read_task(*list_manifests(datasource, preserve_order=True)) tables = [] with pytest.raises(Exception) as error: for table in scanner_for(datasource).create_reader().read(task): @@ -312,9 +314,9 @@ def test_a_corrupt_chunk_read_back_for_a_schema_costs_only_itself(tmp_path): write_mcap_without_summary_schemas(clean, b_declared_in_chunk_1=True) path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) summary = summary_of(path) - # The task owns chunks 3 to 5. Chunk 3 (/b) reads back through chunk 0, + # The task owns the /b chunks 3 and 5. Chunk 3 reads back through chunk 0, # which is corrupt, to chunk 1, which declares other_schema. - owned = {c.chunk_start_offset for c in summary.chunk_indexes[3:]} + owned = {summary.chunk_indexes[i].chunk_start_offset for i in (3, 5)} reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) reported = [] @@ -325,10 +327,96 @@ def test_a_corrupt_chunk_read_back_for_a_schema_costs_only_itself(tmp_path): ) ) - assert [channel.topic for _, channel, _, _ in selected] == ["/b", "/a", "/b"] + assert [channel.topic for _, channel, _, _ in selected] == ["/b", "/b"] + assert all(schema and schema.name == "other_schema" for schema, *_ in selected) assert reported == [] +def read_all_chunks(path, owned_from, on_corrupt): + """The selected messages of chunks ``owned_from`` onward of the MCAP file + ``path``, read with ``on_corrupt`` as one task reads them.""" + summary = summary_of(path) + owned = {c.chunk_start_offset for c in summary.chunk_indexes[owned_from:]} + reader = SelectedMessageReader(MCAPSelection(), log_time_order=False) + with open(path, "rb") as f: + return list(reader.iter_chunks(f, path, summary, owned, on_corrupt=on_corrupt)) + + +def test_a_schema_only_a_corrupt_chunk_declares_fails_an_unskipped_read(tmp_path): + """Without ``on_corrupt``, a task that reads back for a schema only a corrupt + chunk declares raises that chunk's decoding error rather than yield rows + without their schema.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + + # Chunk 0 alone declares both schemas; the task owns chunks 2 to 5. + with pytest.raises(Exception) as error: + read_all_chunks(path, owned_from=2, on_corrupt=None) + + assert type(error.value).__name__ == "ZstdError" + + +def test_a_schema_only_a_corrupt_chunk_declares_fails_a_skipping_read(tmp_path): + """With ``on_corrupt``, a task whose schema only a corrupt earlier chunk + declares raises an error naming that chunk. Neither chunk is reported as + corrupt by this task: the owner of the corrupt one reports it.""" + clean = os.path.join(tmp_path, "clean.mcap") + write_mcap_without_summary_schemas(clean) + path = write_corrupt_chunk(os.path.join(tmp_path, "bad.mcap"), clean, 0) + reported = [] + + with pytest.raises(ValueError) as error: + read_all_chunks(path, owned_from=2, on_corrupt=lambda c, e: reported.append(c)) + + assert "neither the summary nor any chunk up to this one declares" in str( + error.value + ) + assert ( + f"chunk at byte offset {chunk_offset(path, 0)}, read back for it, is corrupt" + in str(error.value) + ) + assert reported == [] + + +def test_a_message_on_an_undeclared_channel_is_not_skipped_as_corrupt(tmp_path): + """A chunk that decodes but holds a message on a channel nothing declares + raises that error even when corrupt chunks are skipped, rather than being + reported as a corrupt chunk.""" + path = os.path.join(tmp_path, "undeclared.mcap") + write_undeclared_channel(path) + reported = [] + + with pytest.raises(ValueError, match=f"on channel {UNDECLARED_CHANNEL}, which"): + read_all_chunks(path, owned_from=0, on_corrupt=lambda c, e: reported.append(c)) + + assert reported == [] + + +def test_an_error_reading_a_file_raises_before_skipped_corrupt_chunks( + tmp_path, monkeypatch, restore_max_errored_blocks +): + """When errors are skipped, a task whose first file has a corrupt chunk and + whose second file fails to read raises the read's ``OSError`` at once, so + the task's retry settings see it, rather than one error for both files.""" + from ray.data._internal.datasource_v2.formats.mcap.mcap_reader import MCAPReader + + a, b, c = write_task_files(tmp_path, bad={"a.mcap"}) + real_read_file = MCAPReader._read_file + + def read_file(self, owned, *args): + if owned.path == b: + raise OSError("disk unplugged") + return real_read_file(self, owned, *args) + + monkeypatch.setattr(MCAPReader, "_read_file", read_file) + + tables, error = read_one_task([a, b, c]) + + assert isinstance(error, OSError) + assert rows_of(tables) == file_rows(a, skipped={CORRUPT_CHUNK}) + + def test_a_retried_read_reports_only_its_own_corrupt_chunks( chunked_file, monkeypatch, restore_max_errored_blocks ): From ab54d41be168b46f9d5511dc1f151ce8e6f5b272 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 8 Nov 2026 13:22:53 -0800 Subject: [PATCH 07/13] [Data] read_mcap: fold the partial-block rerun tests into one; trim duplicates Problem: - Ten unit tests each ran one rerun policy and checked whether the buffered rows came out first. - The errored-rows test ran max_errored_blocks 1 and -1, which take the same path. - Two read_mcap tests also ran with a fused map, which the fused batcher test already covers. Change: - Make the ten tests one test over a small case dataclass of the rerun policy and whether the rows come out, with readable ids. - Let the failing transformer helper raise a given error type, and drop the separate system-error transformer. - Keep only max_errored_blocks=-1, which also catches a check that skips only on a positive limit. - Drop the fused_map parameter from the two read_mcap tests. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/tests/datasource/test_mcap.py | 23 +- python/ray/data/tests/test_map_transformer.py | 301 +++++++----------- .../test_streaming_executor_errored_blocks.py | 12 +- 3 files changed, 132 insertions(+), 204 deletions(-) diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 39e3b83c436f..9c12c3e73f20 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -793,20 +793,15 @@ def write_folder_with_a_corrupt_chunk(tmp_path) -> str: return folder -@pytest.mark.parametrize("fused_map", [False, True]) def test_read_mcap_max_errored_blocks_keeps_files_sharing_a_task_with_a_corrupt_chunk( - ray_start_regular_shared, tmp_path, datasource_v2, fused_map + ray_start_regular_shared, tmp_path, datasource_v2 ): """With ``max_errored_blocks=1``, a recording whose only chunk is corrupt is - skipped and every other file is read, including files in the same read task, - also when a map is fused into the read.""" + skipped and every other file is read, including files in the same read task.""" folder = write_folder_with_a_corrupt_chunk(tmp_path) ray.data.DataContext.get_current().max_errored_blocks = 1 - ds = ray.data.read_mcap(folder, include_paths=True) - if fused_map: - ds = ds.map_batches(lambda batch: batch) - rows = ds.take_all() + rows = ray.data.read_mcap(folder, include_paths=True).take_all() # V2 packs the three small files into one read task; legacy splits them # over two tasks on this one-CPU cluster, a.mcap with b.mcap. @@ -814,13 +809,12 @@ def test_read_mcap_max_errored_blocks_keeps_files_sharing_a_task_with_a_corrupt_ assert files == ["a.mcap"] * 3 + ["c.mcap"] * 3 -@pytest.mark.parametrize("fused_map", [False, True]) def test_read_mcap_max_errored_blocks_loses_only_the_rows_of_corrupt_chunks( - ray_start_regular_shared, tmp_path, datasource_v2, fused_map + ray_start_regular_shared, tmp_path, datasource_v2 ): """With ``max_errored_blocks=1``, V2 loses only the rows of a file's two - corrupt chunks, also when a map is fused into the read. Their read task - raises once for both, so one allowed error is enough.""" + corrupt chunks. Their read task raises once for both, so one allowed error is + enough.""" if not datasource_v2: pytest.skip("the legacy reader stops at a file's first corrupt chunk") from ray.data.tests.unit.datasource_v2.mcap_testing import ( @@ -838,11 +832,8 @@ def test_read_mcap_max_errored_blocks_loses_only_the_rows_of_corrupt_chunks( once = write_corrupt_chunk(os.path.join(tmp_path, "once.mcap"), good, 2) write_corrupt_chunk(os.path.join(folder, "b.mcap"), once, 6) ray.data.DataContext.get_current().max_errored_blocks = 1 - ds = ray.data.read_mcap(folder, include_paths=True) - if fused_map: - ds = ds.map_batches(lambda batch: batch) - rows = ds.take_all() + rows = ray.data.read_mcap(folder, include_paths=True).take_all() seqs = sorted((os.path.basename(row["path"]), row["data"]["seq"]) for row in rows) expected = [("a.mcap", i) for i in range(CHUNKED_FILE_MESSAGES)] diff --git a/python/ray/data/tests/test_map_transformer.py b/python/ray/data/tests/test_map_transformer.py index 885be3418300..34acb1f7c67e 100644 --- a/python/ray/data/tests/test_map_transformer.py +++ b/python/ray/data/tests/test_map_transformer.py @@ -2,6 +2,8 @@ import time import weakref from collections import deque +from dataclasses import dataclass +from typing import Type import pandas as pd import pyarrow as pa @@ -17,11 +19,16 @@ MapTransformer, TransformClock, ) +from ray.data._internal.execution.operators.task_pool_map_operator import ( + _task_rerun_policy, +) from ray.data._internal.output_buffer import OutputBlockSizeOption from ray.data._internal.planner.plan_udf_map_op import ( _generate_transform_fn_for_map_batches, ) from ray.data.block import BlockAccessor, DataBatch +from ray.data.context import DataContext +from ray.exceptions import RaySystemError def _create_chained_transformer(udf, n): @@ -298,15 +305,15 @@ def make_input_blocks(): ) -def _failing_block_transformer(tables_before_error): +def _failing_block_transformer(tables_before_error, error_type=ValueError): """A one-stage transformer whose function yields ``tables_before_error`` - one-row tables, then raises ``ValueError("bad input")``. Its output blocks + one-row tables, then raises ``error_type("bad input")``. Its output blocks are 1 MiB, so the tables share one buffered block.""" def fn(blocks, ctx): for i in range(tables_before_error): yield pa.table({"x": [i]}) - raise ValueError("bad input") + raise error_type("bad input") return MapTransformer( [ @@ -342,20 +349,122 @@ def _outputs(transformer, rerun_policy=NO_RAY_RERUN): def skip_errors(monkeypatch): """Let the job skip an errored block (``max_errored_blocks=1``); returns the patched ``DataContext``.""" - from ray.data.context import DataContext - context = DataContext.get_current() monkeypatch.setattr(context, "max_errored_blocks", 1) return context -def test_a_skipped_error_emits_the_buffered_rows_first(skip_errors): - """When the job skips errors, rows a transform produced before raising come - out as one last block, then the error is raised, so the task keeps them.""" - out = _outputs(_failing_block_transformer(tables_before_error=2)) +@dataclass(frozen=True) +class BufferedRowsCase: + """A map task that raises after buffering rows 0 and 1, and whether it emits + them as a last block before the error.""" - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] - with pytest.raises(ValueError, match="bad input"): + rerun_policy: RerunPolicy + emits_rows: bool + error_type: Type[Exception] = ValueError + # The job's ``DataContext`` settings; 1 lets the job skip the error. + max_errored_blocks: int = 1 + actor_task_retry_on_errors: bool = False + + +@pytest.mark.parametrize( + "case", + [ + pytest.param( + BufferedRowsCase(NO_RAY_RERUN, emits_rows=True), id="skipped-error" + ), + pytest.param( + BufferedRowsCase(NO_RAY_RERUN, emits_rows=False, max_errored_blocks=0), + id="error-fails-the-job", + ), + # ``None`` patterns retry any error, as ``retried_map_errors=True`` sets them. + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=None), + emits_rows=False, + ), + id="map-task-retries-any-error", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries( + attempts_left=1, patterns=["connection reset"] + ), + emits_rows=True, + ), + id="no-map-retry-pattern-matches", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["bad input"]), + emits_rows=False, + ), + id="a-map-retry-pattern-matches", + ), + pytest.param( + BufferedRowsCase( + NO_RAY_RERUN.with_map_retries(attempts_left=0, patterns=["bad input"]), + emits_rows=True, + ), + id="last-map-attempt", + ), + pytest.param( + BufferedRowsCase( + RerunPolicy.from_retry_exceptions([ValueError]), emits_rows=False + ), + id="ray-retries-the-error", + ), + pytest.param( + BufferedRowsCase( + RerunPolicy.from_retry_exceptions([KeyError]), emits_rows=True + ), + id="ray-retries-other-errors", + ), + # Ray Data's default for a task: unlimited retries, ``RaySystemError`` + # among them. With ``max_retries=0`` Ray reruns on nothing. + pytest.param( + BufferedRowsCase( + _task_rerun_policy({}), emits_rows=False, error_type=RaySystemError + ), + id="system-error-ray-retries", + ), + pytest.param( + BufferedRowsCase( + _task_rerun_policy({"max_retries": 0}), + emits_rows=True, + error_type=RaySystemError, + ), + id="system-error-no-ray-retries", + ), + # ``actor_task_retry_on_errors`` reaches actor tasks only, through their + # ``retry_exceptions``. + pytest.param( + BufferedRowsCase( + _task_rerun_policy({}), emits_rows=True, actor_task_retry_on_errors=True + ), + id="task-with-actor-retry-setting", + ), + ], +) +def test_a_failing_task_emits_its_buffered_rows_only_without_a_rerun(case, monkeypatch): + """A map task that raises after buffering rows emits them as one last block + before the error only when the job skips the error and neither Ray nor + ``_map_task`` runs the task again.""" + context = DataContext.get_current() + monkeypatch.setattr(context, "max_errored_blocks", case.max_errored_blocks) + monkeypatch.setattr( + context, "actor_task_retry_on_errors", case.actor_task_retry_on_errors + ) + transformer = _failing_block_transformer( + tables_before_error=2, error_type=case.error_type + ) + + out = _outputs(transformer, case.rerun_policy) + + if case.emits_rows: + rows = BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() + assert rows == [0, 1] + with pytest.raises(case.error_type, match="bad input"): next(out) @@ -374,15 +483,6 @@ def test_a_task_without_retry_settings_raises_without_a_partial_block(skip_error next(out) -def test_an_error_that_fails_the_job_emits_no_partial_block(): - """With ``max_errored_blocks=0``, the default, the job fails anyway, so the - transform raises without emitting its buffered rows.""" - out = _outputs(_failing_block_transformer(tables_before_error=2)) - - with pytest.raises(ValueError, match="bad input"): - next(out) - - def test_a_transform_failing_before_any_output_emits_no_block(skip_errors): """A transform that raises before producing a row raises at once, without an empty block.""" @@ -392,167 +492,6 @@ def test_a_transform_failing_before_any_output_emits_no_block(skip_errors): next(out) -def test_a_transform_that_may_rerun_raises_without_a_partial_block(skip_errors): - """When ``_map_task`` may rerun the transform (``retried_map_errors``), the - buffered rows are not emitted: the rerun skips as many blocks as were - emitted, so a partial block would hide rows of the rerun's first block.""" - # ``None`` patterns retry any error, as ``retried_map_errors=True`` sets them. - out = _outputs( - _failing_block_transformer(tables_before_error=2), - NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=None), - ) - - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def test_an_error_no_retry_pattern_matches_emits_the_buffered_rows_first( - skip_errors, -): - """``_map_task`` retries only errors that match ``retried_map_errors``, so an - error that matches none won't rerun the transform, and its buffered rows come - out first.""" - out = _outputs( - _failing_block_transformer(tables_before_error=2), - NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["connection reset"]), - ) - - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def test_an_error_a_retry_pattern_matches_raises_without_a_partial_block( - skip_errors, -): - """An error that matches a ``retried_map_errors`` pattern may rerun the - transform, so its buffered rows are not emitted.""" - out = _outputs( - _failing_block_transformer(tables_before_error=2), - NO_RAY_RERUN.with_map_retries(attempts_left=1, patterns=["bad input"]), - ) - - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def test_an_error_ray_reruns_the_task_on_raises_without_a_partial_block( - skip_errors, -): - """Ray reruns a task on an error its ``retry_exceptions`` names, so the - buffered rows are not emitted, as for ``retried_map_errors``.""" - out = _outputs( - _failing_block_transformer(tables_before_error=2), - RerunPolicy.from_retry_exceptions([ValueError]), - ) - - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def test_an_error_ray_does_not_rerun_the_task_on_emits_the_buffered_rows_first( - skip_errors, -): - """An error that ``retry_exceptions`` does not name won't rerun the task, - so its buffered rows come out first.""" - out = _outputs( - _failing_block_transformer(tables_before_error=2), - RerunPolicy.from_retry_exceptions([KeyError]), - ) - - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def _failing_system_error_transformer() -> MapTransformer: - """A transformer that yields two one-row tables into one buffered block, - then raises a ``RaySystemError``.""" - from ray.exceptions import RaySystemError - - def fn(blocks, ctx): - yield pa.table({"x": [0]}) - yield pa.table({"x": [1]}) - raise RaySystemError("lost an object") - - return MapTransformer( - [ - BlockMapTransformFn( - fn, - output_block_size_option=OutputBlockSizeOption.of( - target_max_block_size=1024 * 1024 - ), - ) - ] - ) - - -@pytest.mark.parametrize( - "remote_args, emits_rows", - [ - # Ray Data's default: unlimited retries, ``RaySystemError`` among them. - ({}, False), - # With ``max_retries=0`` Ray reruns on nothing. - ({"max_retries": 0}, True), - ], -) -def test_a_system_error_emits_the_buffered_rows_only_when_ray_wont_rerun( - skip_errors, remote_args, emits_rows -): - """Ray reruns a task on a ``RaySystemError`` only while its retries allow, so - with ``max_retries=0`` the buffered rows come out before the error.""" - from ray.data._internal.execution.operators.task_pool_map_operator import ( - _task_rerun_policy, - ) - from ray.exceptions import RaySystemError - - out = _outputs( - _failing_system_error_transformer(), - _task_rerun_policy(remote_args), - ) - - if emits_rows: - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [ - 0, - 1, - ] - with pytest.raises(RaySystemError, match="lost an object"): - next(out) - - -def test_the_last_map_attempt_emits_the_buffered_rows_first(skip_errors): - """On its last attempt ``_map_task`` won't rerun the transform even for an - error a pattern matches, so the buffered rows come out first.""" - out = _outputs( - _failing_block_transformer(tables_before_error=2), - NO_RAY_RERUN.with_map_retries(attempts_left=0, patterns=["bad input"]), - ) - - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] - with pytest.raises(ValueError, match="bad input"): - next(out) - - -def test_actor_retry_settings_do_not_stop_a_task_pool_map_from_emitting( - skip_errors, monkeypatch -): - """``actor_task_retry_on_errors`` applies to actor tasks only, through their - ``retry_exceptions``. A task that Ray won't rerun still emits its buffered - rows when the setting is on.""" - from ray.data._internal.execution.operators.task_pool_map_operator import ( - _task_rerun_policy, - ) - - monkeypatch.setattr(skip_errors, "actor_task_retry_on_errors", True) - out = _outputs( - _failing_block_transformer(tables_before_error=2), _task_rerun_policy({}) - ) - - assert BlockAccessor.for_block(next(out)).to_arrow()["x"].to_pylist() == [0, 1] - with pytest.raises(ValueError, match="bad input"): - next(out) - - if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_streaming_executor_errored_blocks.py b/python/ray/data/tests/test_streaming_executor_errored_blocks.py index ee3741744357..cbf1a0729234 100644 --- a/python/ray/data/tests/test_streaming_executor_errored_blocks.py +++ b/python/ray/data/tests/test_streaming_executor_errored_blocks.py @@ -117,15 +117,13 @@ def fail_row_at_failing_row(row): return row -@pytest.mark.parametrize("max_errored_blocks", [1, -1]) -def test_a_skipped_error_keeps_the_rows_a_task_produced_before_it( - restore_data_context, max_errored_blocks -): +def test_a_skipped_error_keeps_the_rows_a_task_produced_before_it(restore_data_context): """A one-task ``map_batches`` that fails at the batch holding row 7 keeps the - rows of the batches before it when the job skips the error, with one error - allowed and with any number.""" + rows of the batches before it when the job skips the error.""" ctx = ray.data.DataContext.get_current() - ctx.max_errored_blocks = max_errored_blocks + # Any limit other than 0 skips the error. -1 (no limit) also catches a check + # that treats only a positive limit as skipping. + ctx.max_errored_blocks = -1 ds = ray.data.range(NUM_ROWS, override_num_blocks=1).map_batches( fail_at_failing_row, batch_size=2 From 9e15f8cfcd01d9a14f7e23c6e8fac384aab3a858 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Sun, 8 Nov 2026 17:00:37 -0800 Subject: [PATCH 08/13] [Data] read_mcap: skip one message that is not JSON, not its file Problem: - With max_errored_blocks != 0, one message whose payload is not valid JSON dropped the rest of its file, rows already built included. - The error did not say that the rest of the file was dropped. Change: - decode_payload raises InvalidJsonPayload, a ValueError, for a payload that is not valid JSON. A wrong message encoding still raises a ValueError that fails the file. - When errors can be skipped, the reader skips that one message and keeps every other row of the file, as it does for a corrupt chunk. - One error per file names the skipped chunks by byte offset, counts the skipped messages and names the first by row id, so the task counts once. The record keeps only the first message. - The table builder decodes a payload before it adds anything, so a skipped message leaves it consistent. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../formats/mcap/mcap_message_rows.py | 22 ++- .../datasource_v2/formats/mcap/mcap_reader.py | 157 ++++++++++++++---- python/ray/data/tests/datasource/test_mcap.py | 36 ++++ .../tests/unit/datasource_v2/mcap_testing.py | 18 +- .../unit/datasource_v2/test_mcap_reader.py | 123 ++++++++++++++ 5 files changed, 315 insertions(+), 41 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py index fd3994aa7a83..85c4f22979fc 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_message_rows.py @@ -128,14 +128,18 @@ def message_schema( return pa.schema(fields) +class InvalidJsonPayload(ValueError): + """A message on a JSON-encoded channel whose payload is not valid JSON.""" + + def decode_payload(channel: "Channel", data: bytes, where: str) -> Any: """Decode a JSON payload into Python values. The reader calls this only when ``data`` was planned as decoded JSON, and planning calls it to type that column. Arrow has no column type for a mix of - bytes and decoded values, so a channel of another encoding, or a payload - that is not valid JSON, raises a ``ValueError`` that names ``where``: a - ``row_id`` or a file. + bytes and decoded values, so a channel of another encoding raises a + ``ValueError``, and a payload that is not valid JSON an + ``InvalidJsonPayload``. Both name ``where``: a ``row_id`` or a file. """ if channel.message_encoding != "json": raise ValueError( @@ -147,7 +151,7 @@ def decode_payload(channel: "Channel", data: bytes, where: str) -> Any: try: return json.loads(data.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError) as e: - raise ValueError( + raise InvalidJsonPayload( f"{where}: message on JSON-encoded topic {channel.topic!r} is not valid " f"JSON: {e}" ) from e @@ -383,7 +387,15 @@ def reset(self) -> None: ) def add(self, selected: _Selected) -> None: + """Add one message as a row. + + The payload is decoded before anything is added, so a payload that + raises ``InvalidJsonPayload`` leaves the builder as it was. + """ schema, channel, message, row_id = selected + value = None + if self._json_values is not None: + value = decode_payload(channel, message.data, row_id) slot = self._slot(channel, schema) self._slot_ids.append(slot) self.num_rows += 1 @@ -394,7 +406,7 @@ def add(self, selected: _Selected) -> None: self._payloads.append(message.data) self.estimated_bytes += len(message.data) elif self._json_values is not None: - self._json_values.append(decode_payload(channel, message.data, row_id)) + self._json_values.append(value) payload_bytes = len(message.data) self.estimated_bytes += _DECODED_JSON_BYTES_PER_PAYLOAD_BYTE * payload_bytes if self._row_ids is not None: diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 9ca565cd2cbd..d789cf8cb001 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -21,7 +21,7 @@ import itertools import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from functools import partial from typing import ( TYPE_CHECKING, @@ -50,6 +50,7 @@ read_summary_from, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( + InvalidJsonPayload, _MessageTableBuilder, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection @@ -144,12 +145,13 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: When ``DataContext.max_errored_blocks`` lets errors be skipped, a corrupt chunk costs only itself: the reader skips it and reads the - file's other chunks. A file whose data does not decode as a whole does - not stop the task's other files. Once they are read, one error names - every file and chunk that is corrupt, so the task counts once against - the budget. Any other error, such as a failed read, raises at once, so - the task's retry settings see it. Without skipping, the first failure - fails the task at once. + file's other chunks. A message whose payload is not valid JSON likewise + costs only its own row. A file whose data does not decode as a whole + does not stop the task's other files. Once they are read, one error + names every file, chunk and message skipped, so the task counts once + against the budget. Any other error, such as a failed read, raises at + once, so the task's retry settings see it. Without skipping, the first + failure fails the task at once. """ from ray.data.context import DataContext @@ -180,16 +182,17 @@ def _read_owned_file( """Yield the tables of one file, retrying as ``retried_io_errors`` allows. With ``skip_corrupt``, the file's corrupt data is appended to - ``failures`` rather than raised: its skipped chunks, or its own error - when it does not decode as a whole (``is_corrupt_data_error``). + ``failures`` rather than raised: its skipped chunks and messages, or + its own error when it does not decode as a whole + (``is_corrupt_data_error``). """ from ray.data.context import DataContext - # The chunks skipped as corrupt, by offset. Each attempt starts it - # afresh, so it holds the last attempt's. - corrupt: Optional[Dict[int, Exception]] = {} if skip_corrupt else None + # What the read skips. Each attempt starts it afresh, so it holds the + # last attempt's. + skipped = _SkippedData() if skip_corrupt else None tables = iterate_with_retry( - partial(self._read_file, owned, limit, corrupt), + partial(self._read_file, owned, limit, skipped), f"read MCAP file {owned.path}", match=DataContext.get_current().retried_io_errors, ) @@ -200,8 +203,8 @@ def _read_owned_file( raise failures.append((owned.path, e)) return - if corrupt: - failures.append((owned.path, _corrupt_chunks_error(owned.path, corrupt))) + if skipped is not None and (skipped.chunks or skipped.num_messages): + failures.append((owned.path, _skipped_data_error(owned.path, skipped))) # -- one file ---------------------------------------------------------- @@ -209,22 +212,23 @@ def _read_file( self, owned: "_OwnedFile", limit: Optional[int], - corrupt: Optional[Dict[int, Exception]] = None, + skipped: Optional["_SkippedData"] = None, ) -> Iterator[pa.Table]: """Yield the tables of one file, limited to the chunks it owns. ``owned.offsets`` is ``None`` for a whole-file row, which means every chunk. A file without a chunk index is scanned linearly, through a read buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most - ``limit`` rows are yielded. With ``corrupt``, a chunk that cannot be - decoded is recorded there by offset and skipped; a scanned file has no - chunk boundaries to skip to, so it still fails whole. A retry reads the - file from its start, so ``corrupt`` is cleared first. + ``limit`` messages are read, a skipped one included. With ``skipped``, a + chunk that cannot be decoded and a message that is not valid JSON are + recorded there and skipped; a scanned file has no chunk boundaries to + skip to, so a corrupt chunk still fails it whole. A retry reads the + file from its start, so ``skipped`` is cleared first. """ from ray.data.context import DataContext - if corrupt is not None: - corrupt.clear() + if skipped is not None: + skipped.clear() path, offsets = owned.path, owned.offsets filesystem = self._filesystem or LocalFileSystem() @@ -252,20 +256,27 @@ def _read_file( offsets, file_summary, on_corrupt=None - if corrupt is None - else partial(_record_corrupt_chunk, corrupt), + if skipped is None + else partial(_record_corrupt_chunk, skipped.chunks), ) - yield from self._tables(itertools.islice(messages, limit), path, offsets) + yield from self._tables( + itertools.islice(messages, limit), path, offsets, skipped + ) # -- rows to tables ---------------------------------------------------- def _tables( - self, messages: Iterator[_Selected], path: str, offsets: Optional[Set[int]] + self, + messages: Iterator[_Selected], + path: str, + offsets: Optional[Set[int]], + skipped: Optional["_SkippedData"] = None, ) -> Iterator[pa.Table]: """Build tables of about ``target_block_size`` bytes from the messages. Messages are pulled one at a time, so a chunk is read only once its - first message is the next one needed. + first message is the next one needed. With ``skipped``, a message that + is not valid JSON is recorded there instead of becoming a row. """ # Every table reports one read unit: the file for a whole-file read, else # the row's first chunk. Listing excludes read units chunk by chunk @@ -286,7 +297,8 @@ def _tables( ) rows_before = 0 for selected in messages: - builder.add(selected) + if not _add_message(builder, selected, skipped): + continue if ( self._target_block_size is not None and builder.estimated_bytes >= self._target_block_size @@ -392,6 +404,62 @@ def _partition_value_array(self, name: str, value: Any, num_rows: int) -> pa.Arr return array +@dataclass(frozen=True) +class _SkippedMessage: + """A message whose payload is not valid JSON, and the error it raised.""" + + row_id: str + error: Exception + + +@dataclass +class _SkippedData: + """What a read of one file skipped. + + Every corrupt chunk is kept, with its first error. Messages that are not + valid JSON are counted and only the first is kept, so a file of such + messages costs constant memory. + """ + + # Corrupt chunks, by byte offset. + chunks: Dict[int, Exception] = field(default_factory=dict) + num_messages: int = 0 + first_message: Optional[_SkippedMessage] = None + + def record_message(self, row_id: str, error: Exception) -> None: + """Count a skipped message, keeping it if it is the first.""" + if self.first_message is None: + self.first_message = _SkippedMessage(row_id, error) + self.num_messages += 1 + + def clear(self) -> None: + """Forget what was skipped, for a read that starts the file over.""" + self.chunks.clear() + self.num_messages = 0 + self.first_message = None + + +def _add_message( + builder: _MessageTableBuilder, + selected: _Selected, + skipped: Optional[_SkippedData], +) -> bool: + """Add one message to ``builder`` and return whether it became a row. + + With ``skipped``, a message whose payload is not valid JSON is recorded + there rather than raised. + """ + try: + builder.add(selected) + except InvalidJsonPayload as e: + if skipped is None: + raise + _, _, _, row_id = selected + skipped.record_message(row_id, e) + return False + return True + + def _record_corrupt_chunk( corrupt: Dict[int, Exception], chunk_index: "ChunkIndex", error: Exception ) -> None: @@ -399,14 +467,33 @@ def _record_corrupt_chunk( corrupt.setdefault(chunk_index.chunk_start_offset, error) -def _corrupt_chunks_error(path: str, corrupt: Dict[int, Exception]) -> ValueError: - """The error for the chunks of ``path`` that a read skipped as corrupt.""" - offsets = sorted(corrupt) - first = corrupt[offsets[0]] - where = "chunk at byte offset" if len(offsets) == 1 else "chunks at byte offsets" +def _skipped_data_error(path: str, skipped: _SkippedData) -> ValueError: + """The error for the chunks and messages of ``path`` that a read skipped. + + It names every chunk and the first message, and quotes the first error: + the first chunk's, or the first message's when no chunk was skipped. + """ + offsets = sorted(skipped.chunks) + parts = [] + if offsets: + where = ( + "chunk at byte offset" if len(offsets) == 1 else "chunks at byte offsets" + ) + listed = ", ".join(str(offset) for offset in offsets) + parts.append(f"{len(offsets)} corrupt {where} {listed}") + message = skipped.first_message + if message is not None: + count = skipped.num_messages + what = "message that is" if count == 1 else "messages that are" + which = "row id" if count == 1 else "first: row id" + parts.append(f"{count} {what} not valid JSON ({which} {message.row_id})") + if offsets: + first = skipped.chunks[offsets[0]] + else: + assert message is not None + first = message.error error = ValueError( - f"Skipped {len(offsets)} corrupt {where} " - f"{', '.join(str(offset) for offset in offsets)} of MCAP file {path!r}: " + f"Skipped {' and '.join(parts)} of MCAP file {path!r}: " f"{type(first).__name__}: {_first_line(first)}" ) error.__cause__ = first diff --git a/python/ray/data/tests/datasource/test_mcap.py b/python/ray/data/tests/datasource/test_mcap.py index 9c12c3e73f20..3e64d1e8b7aa 100644 --- a/python/ray/data/tests/datasource/test_mcap.py +++ b/python/ray/data/tests/datasource/test_mcap.py @@ -841,6 +841,42 @@ def test_read_mcap_max_errored_blocks_loses_only_the_rows_of_corrupt_chunks( assert seqs == expected +def test_read_mcap_max_errored_blocks_loses_only_a_message_that_is_not_json( + ray_start_regular_shared, tmp_path, datasource_v2 +): + """With ``max_errored_blocks=1``, V2 loses only the row of a message whose + payload is not valid JSON and keeps the rest of its file. With + ``max_errored_blocks=0``, the read fails naming the message's ``row_id``.""" + if not datasource_v2: + pytest.skip("the legacy reader keeps a payload that is not JSON as bytes") + from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import ( + message_row_id, + ) + from ray.data.tests.unit.datasource_v2.mcap_testing import ( + CHUNKED_FILE_MESSAGES, + summary_of, + write_invalid_json, + ) + + bad = 4 + # One message per chunk, so ``bad`` is the only message of chunk ``bad``. + path = write_invalid_json(os.path.join(tmp_path, "bad.mcap"), bad={bad}) + bad_offset = summary_of(path).chunk_indexes[bad].chunk_start_offset + ctx = ray.data.DataContext.get_current() + + ctx.max_errored_blocks = 1 + rows = ray.data.read_mcap(path).take_all() + + seqs = sorted(row["data"]["seq"] for row in rows) + assert seqs == [i for i in range(CHUNKED_FILE_MESSAGES) if i != bad] + + ctx.max_errored_blocks = 0 + with pytest.raises(Exception) as error: + ray.data.read_mcap(path).take_all() + + assert message_row_id(path, bad_offset, 0) in str(error.value) + + def test_read_mcap_lists_files_in_parallel( ray_start_regular_shared, tmp_path, datasource_v2 ): diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index d659c49791e1..316e05858073 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -9,7 +9,7 @@ import json import shutil from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional import pyarrow as pa from pyarrow.fs import LocalFileSystem @@ -359,6 +359,22 @@ def write_corrupt_chunk(path: str, source: str, index: int) -> str: return path +NOT_JSON = b"not json" # a payload on a JSON-encoded topic that does not decode + + +def write_invalid_json(path: str, bad: Iterable[int], **options: Any) -> str: + """Write ``CHUNKED_FILE_MESSAGES`` ``round_robin_messages`` to ``path``, with + the payloads of the messages whose ``seq`` is in ``bad`` set to ``NOT_JSON``. + + ``options`` go to ``write_mcap``; by default each message is its own chunk. + """ + messages = round_robin_messages(CHUNKED_FILE_MESSAGES) + for seq in bad: + messages[seq]["data"] = NOT_JSON + write_mcap(path, messages, **options) + return path + + # Where a chunk record's compression name starts: after the opcode (1 byte), # the record length (8), the message start and end times (8 each), the # uncompressed size (8), its CRC (4) and the name's length (4). diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index d68cfdcf4957..9ff26795133d 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -8,6 +8,7 @@ import importlib.util import json import os +import re import time import pyarrow as pa @@ -22,6 +23,7 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ChunkReads from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( + InvalidJsonPayload, _MessageTableBuilder, message_schema, ) @@ -30,6 +32,7 @@ MCAPSelection, ) from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner +from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import message_row_id from ray.data._internal.datasource_v2.interfaces.file_manifest import FileManifest from ray.data.tests.unit.datasource_v2.mcap_testing import ( BASE_TIME, @@ -54,6 +57,7 @@ write_corrupt_chunk, write_corrupt_chunk_header, write_cut_short, + write_invalid_json, write_mcap, write_mcap_without_summary_schemas, write_mixed_channels, @@ -246,6 +250,102 @@ def test_one_error_names_every_corrupt_chunk_of_a_file( ) +BAD_JSON = 4 # the message whose payload is not valid JSON + + +def message_row_id_of(path, seq, one_chunk): + """The ``row_id`` of message ``seq`` of a file of ``write_invalid_json``, + written with one chunk per message or with every message in one chunk.""" + if one_chunk: + return message_row_id(path, chunk_offset(path, 0), seq) + return message_row_id(path, chunk_offset(path, seq), 0) + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("one_chunk", [False, True], ids=["own_chunk", "mid_chunk"]) +def test_a_message_that_is_not_json_costs_only_its_own_row( + tmp_path, one_chunk, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a message that is not valid JSON in the middle + file costs only its own row, whether it fills a chunk or sits between good + messages of one chunk. The kept rows' columns line up, and the task then + raises naming the message.""" + a, b, c = write_task_files(tmp_path, bad=set()) + # A chunk size past the file's size puts every message in one chunk. + write_invalid_json(b, bad={BAD_JSON}, chunk_size=1 << 20 if one_chunk else 1) + + tables, error = read_one_task( + [a, b, c], log_time_order=log_time_order, include_row_id=True + ) + + assert rows_of(tables) == file_rows(a) + file_rows(b, {BAD_JSON}) + file_rows(c) + rows = [row for table in tables for row in table.to_pylist() if row["path"] == b] + seqs = [row["data"]["seq"] for row in rows] + assert [row["log_time"] for row in rows] == [BASE_TIME + s * STEP for s in seqs] + expected_ids = [message_row_id_of(b, s, one_chunk) for s in seqs] + assert [row[ROW_ID_COLUMN] for row in rows] == expected_ids + row_id = message_row_id_of(b, BAD_JSON, one_chunk) + assert str(error).startswith( + f"Skipped 1 message that is not valid JSON (row id {row_id}) of MCAP file " + f"{b!r}: InvalidJsonPayload: {row_id}: message on JSON-encoded topic" + ) + + +def test_one_error_names_the_corrupt_chunks_and_the_messages_that_are_not_json( + tmp_path, restore_max_errored_blocks +): + """A file with a corrupt chunk and two messages that are not valid JSON loses + only those three rows, and one error names the chunk, counts the messages + and names the first of them, with the chunk's error.""" + a, b, c = write_task_files(tmp_path, bad=set()) + write_invalid_json(b, bad={5, 6}) + corrupt_in_place(b, 2) + + tables, error = read_one_task([a, b, c]) + + assert rows_of(tables) == file_rows(a) + file_rows(b, {2, 5, 6}) + file_rows(c) + first_row_id = message_row_id_of(b, 5, False) + assert str(error).startswith( + f"Skipped 1 corrupt chunk at byte offset {chunk_offset(b, 2)} and " + f"2 messages that are not valid JSON (first: row id {first_row_id}) of " + f"MCAP file {b!r}: ZstdError: " + ) + + +def test_a_file_of_messages_that_are_not_json_is_counted_not_listed( + tmp_path, restore_max_errored_blocks +): + """A file whose every message is not valid JSON loses all its rows, and its + error counts the messages and quotes only the first one's error.""" + a, b, c = write_task_files(tmp_path, bad=set()) + write_invalid_json(b, bad=range(CHUNKED_FILE_MESSAGES)) + + tables, error = read_one_task([a, b, c]) + + assert rows_of(tables) == file_rows(a) + file_rows(c) + first_row_id = message_row_id_of(b, 0, False) + assert str(error).startswith( + f"Skipped {CHUNKED_FILE_MESSAGES} messages that are not valid JSON " + f"(first: row id {first_row_id}) of MCAP file {b!r}: " + f"InvalidJsonPayload: {first_row_id}: " + ) + + +def test_a_message_that_is_not_json_fails_its_task_when_errors_are_not_skipped( + tmp_path, restore_max_errored_blocks +): + """With ``max_errored_blocks=0``, a message that is not valid JSON fails the + task at once, with an error that names the message.""" + a, b, c = write_task_files(tmp_path, bad=set()) + write_invalid_json(b, bad={BAD_JSON}) + + tables, error = read_one_task([a, b, c], max_errored_blocks=0) + + assert rows_of(tables) == file_rows(a) + assert isinstance(error, InvalidJsonPayload) + assert str(error).startswith(f"{message_row_id_of(b, BAD_JSON, False)}: ") + + def test_a_task_names_every_file_it_could_not_read_in_full( tmp_path, restore_max_errored_blocks ): @@ -1102,6 +1202,29 @@ def test_build_empties_the_builder(chunked_file): assert second.column(ROW_ID_COLUMN).to_pylist() == expected_ids +def test_a_payload_that_is_not_json_leaves_the_builder_unchanged(tmp_path): + """Adding a message whose payload is not valid JSON raises + ``InvalidJsonPayload`` before the builder changes, so the rows added around + it build into a consistent table.""" + path = write_invalid_json(os.path.join(tmp_path, "bad.mcap"), bad={1}) + builder = _MessageTableBuilder( + columns=None, include_metadata=True, include_row_id=True, decode_json=True + ) + good, bad, *_ = selected_messages(path) + builder.add(good) + before = (builder.num_rows, builder.estimated_bytes) + + with pytest.raises( + InvalidJsonPayload, match=re.escape(bad[3]) + ": .* not valid JSON" + ): + builder.add(bad) + + assert (builder.num_rows, builder.estimated_bytes) == before + table = builder.build() + assert table.column(ROW_ID_COLUMN).to_pylist() == [good[3]] + assert table.column("log_time").to_pylist() == [BASE_TIME] + + def test_payload_buffer_starts_a_chunk_before_its_offsets_overflow(monkeypatch): """A ``binary`` buffer is sealed into its own chunk before its int32 offsets would overflow, and the chunks keep the values in order.""" From 38d54b5b7d75bfbac8f6094b224b4c56891ab9f1 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Mon, 9 Nov 2026 09:02:55 -0800 Subject: [PATCH 09/13] [Data] read_mcap: keep the limit and the rows built when skipping Problem: - With max_errored_blocks set, a message that is not valid JSON used up a place in the row limit, so a limited read could return fewer rows than asked although later messages were valid. - A file that failed mid-way, for example on a topic of another encoding, dropped the rows already built into its current table. What the file kept depended on the block size. Change: - The limit counts rows built. A skipped message frees its place, and no message is pulled once the limit is met. - An ordered scan of a file without a chunk index holds every message when it can skip, so a skipped one cannot shorten the result. - When errors are skipped, a file error yields the rows built before it, then fails the rest of the file as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_reader.py | 78 ++++++++++++------- .../unit/datasource_v2/test_mcap_reader.py | 65 +++++++++++++++- 2 files changed, 112 insertions(+), 31 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index d789cf8cb001..6546446bd77c 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -19,7 +19,6 @@ - ``mcap_message_rows`` builds message rows. """ -import itertools import logging from dataclasses import dataclass, field from functools import partial @@ -146,12 +145,12 @@ def read(self, input_split: FileManifest) -> Iterator[pa.Table]: When ``DataContext.max_errored_blocks`` lets errors be skipped, a corrupt chunk costs only itself: the reader skips it and reads the file's other chunks. A message whose payload is not valid JSON likewise - costs only its own row. A file whose data does not decode as a whole - does not stop the task's other files. Once they are read, one error - names every file, chunk and message skipped, so the task counts once - against the budget. Any other error, such as a failed read, raises at - once, so the task's retry settings see it. Without skipping, the first - failure fails the task at once. + costs only its own row. A file whose other data does not decode keeps + the rows before the failure and does not stop the task's other files. + Once they are read, one error names every file, chunk and message + skipped, so the task counts once against the budget. Any other error, + such as a failed read, raises at once, so the task's retry settings see + it. Without skipping, the first failure fails the task at once. """ from ray.data.context import DataContext @@ -219,11 +218,12 @@ def _read_file( ``owned.offsets`` is ``None`` for a whole-file row, which means every chunk. A file without a chunk index is scanned linearly, through a read buffer of ``DataContext.streaming_read_buffer_size`` bytes. At most - ``limit`` messages are read, a skipped one included. With ``skipped``, a - chunk that cannot be decoded and a message that is not valid JSON are - recorded there and skipped; a scanned file has no chunk boundaries to - skip to, so a corrupt chunk still fails it whole. A retry reads the - file from its start, so ``skipped`` is cleared first. + ``limit`` rows are built. With ``skipped``, a chunk that cannot be + decoded and a message that is not valid JSON are recorded there and + skipped, and a skipped message does not count against ``limit``. A + scanned file has no chunk boundaries to skip to, so a corrupt chunk + still fails it whole. A retry reads the file from its start, so + ``skipped`` is cleared first. """ from ray.data.context import DataContext @@ -245,8 +245,10 @@ def _read_file( file_summary = None if offsets is None else _FileSummary(f, path) if summary is None or not summary.chunk_indexes: buffer_size = DataContext.get_current().streaming_read_buffer_size + # A skipped message leaves its place in the limit to the next + # one, so a read that can skip holds every selected message. messages = self._message_reader.iter_unindexed( - buffered(f, buffer_size), path, limit + buffered(f, buffer_size), path, limit if skipped is None else None ) else: messages = self._message_reader.iter_chunks( @@ -259,9 +261,7 @@ def _read_file( if skipped is None else partial(_record_corrupt_chunk, skipped.chunks), ) - yield from self._tables( - itertools.islice(messages, limit), path, offsets, skipped - ) + yield from self._tables(messages, path, offsets, limit, skipped) # -- rows to tables ---------------------------------------------------- @@ -270,13 +270,18 @@ def _tables( messages: Iterator[_Selected], path: str, offsets: Optional[Set[int]], + limit: Optional[int], skipped: Optional["_SkippedData"] = None, ) -> Iterator[pa.Table]: """Build tables of about ``target_block_size`` bytes from the messages. Messages are pulled one at a time, so a chunk is read only once its - first message is the next one needed. With ``skipped``, a message that - is not valid JSON is recorded there instead of becoming a row. + first message is the next one needed, and no message is pulled once + ``limit`` rows are built. With ``skipped``, a message that is not valid + JSON is recorded there instead of becoming a row. An error that fails + the rest of the file (``is_corrupt_data_error``) first yields the rows + built before it, so what the file keeps does not depend on the block + size. """ # Every table reports one read unit: the file for a whole-file read, else # the row's first chunk. Listing excludes read units chunk by chunk @@ -296,21 +301,36 @@ def _tables( extra_row_bytes=self._appended_row_bytes(path, unit, wanted), ) rows_before = 0 - for selected in messages: - if not _add_message(builder, selected, skipped): - continue - if ( - self._target_block_size is not None - and builder.estimated_bytes >= self._target_block_size - ): - num_rows = builder.num_rows - # ``build`` empties the builder, so the yielded table is the - # only copy of its rows. + try: + for selected in messages: + if not _add_message(builder, selected, skipped): + continue + if limit is not None and rows_before + builder.num_rows >= limit: + break + if self._block_is_full(builder): + num_rows = builder.num_rows + # ``build`` empties the builder, so the yielded table is the + # only copy of its rows. + yield self._finish(builder.build(), path, unit, rows_before) + rows_before += num_rows + except Exception as e: + if skipped is None or not is_corrupt_data_error(e): + raise + # The file's bytes fail the same way on every read, so a retried + # read yields these same tables and the retry skips them all. + if builder.num_rows > 0: yield self._finish(builder.build(), path, unit, rows_before) - rows_before += num_rows + raise if builder.num_rows > 0: yield self._finish(builder.build(), path, unit, rows_before) + def _block_is_full(self, builder: _MessageTableBuilder) -> bool: + """Whether ``builder`` holds ``target_block_size`` bytes or more.""" + return ( + self._target_block_size is not None + and builder.estimated_bytes >= self._target_block_size + ) + def _appended_row_bytes( self, path: str, unit: ReadUnit, wanted: Optional[Set[str]] ) -> int: diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 9ff26795133d..851216572ae3 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -163,16 +163,22 @@ def corrupt_in_place(path, index, corrupt=write_corrupt_chunk): def read_one_task(paths, max_errored_blocks=1, **options): """Read ``paths`` as one read task under ``max_errored_blocks``, with the datasource ``options``; return the tables it yields and its error.""" + datasource = MCAPDatasourceV2(paths, include_paths=True, **options) + return read_task_with(scanner_for(datasource), datasource, max_errored_blocks) + + +def read_task_with(scanner, datasource, max_errored_blocks=1): + """Read the files of ``datasource`` as one read task with ``scanner``, under + ``max_errored_blocks``; return the tables it yields and its error.""" from ray.data.context import DataContext DataContext.get_current().max_errored_blocks = max_errored_blocks - datasource = MCAPDatasourceV2(paths, include_paths=True, **options) # preserve_order keeps the files in the order given; the listing threads # otherwise finish in any order. task = as_read_task(*list_manifests(datasource, preserve_order=True)) tables = [] with pytest.raises(Exception) as error: - for table in scanner_for(datasource).create_reader().read(task): + for table in scanner.create_reader().read(task): tables.append(table) return tables, error.value @@ -346,6 +352,61 @@ def test_a_message_that_is_not_json_fails_its_task_when_errors_are_not_skipped( assert str(error).startswith(f"{message_row_id_of(b, BAD_JSON, False)}: ") +LIMIT = 4 # rows a limited read asks for + + +@pytest.mark.parametrize("log_time_order", [True, False]) +@pytest.mark.parametrize("index", [True, False], ids=["indexed", "unindexed"]) +def test_a_skipped_message_does_not_count_against_the_limit( + tmp_path, index, log_time_order, restore_max_errored_blocks +): + """When errors can be skipped, a limited read of a file whose first two + messages are not valid JSON still yields ``LIMIT`` rows, the next ones in + the file, with and without a chunk index, in log-time and in file order.""" + path = write_invalid_json(os.path.join(tmp_path, "b.mcap"), bad={0, 1}, index=index) + datasource = MCAPDatasourceV2( + [path], include_paths=True, log_time_order=log_time_order + ) + + tables, error = read_task_with( + scanner_for(datasource).push_limit(LIMIT), datasource + ) + + assert rows_of(tables) == [(path, seq) for seq in range(2, 2 + LIMIT)] + assert str(error).startswith("Skipped 2 messages that are not valid JSON") + + +LATE_CDR = 5 # the message on a CDR-encoded topic, after five JSON messages + + +@pytest.mark.parametrize( + "target_block_size", [None, 1], ids=["one_block", "block_per_row"] +) +def test_a_file_that_fails_mid_way_keeps_the_rows_before_the_failure( + tmp_path, target_block_size, restore_max_errored_blocks +): + """When errors can be skipped, a file that message ``LATE_CDR`` fails keeps + the rows before it, whether they share one block or fill one each. The + task then raises the file's error.""" + planned_on = os.path.join(tmp_path, "json.mcap") + write_mcap(planned_on, round_robin_messages(3)) + path = os.path.join(tmp_path, "late_cdr.mcap") + messages = round_robin_messages(LATE_CDR + 1) + messages[LATE_CDR].update(topic="/cdr", data=b"\x00\x01") + write_mcap(path, messages, encodings={"/cdr": "cdr"}) + # Planned on a JSON-only file, so the read decodes JSON and the CDR + # message fails it. + planned = MCAPDatasourceV2([planned_on], include_paths=True) + scanner = dataclasses.replace( + scanner_for(planned), target_block_size=target_block_size + ) + + tables, error = read_task_with(scanner, MCAPDatasourceV2([path])) + + assert rows_of(tables) == [(path, seq) for seq in range(LATE_CDR)] + assert "topic '/cdr' is 'cdr'-encoded" in str(error) + + def test_a_task_names_every_file_it_could_not_read_in_full( tmp_path, restore_max_errored_blocks ): From 8f45b938f5343dc0b48b7ad813718ea1b797cc5b Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Mon, 9 Nov 2026 12:09:29 -0800 Subject: [PATCH 10/13] [Data] read_mcap: skip an unselected channel before its schema Problem: - A missing schema raised for every message, before the selection check ran. - So a topic the read filters out could fail the rest of its file when its schema lived only in a skipped corrupt chunk. The later selected rows were lost and a max_errored_blocks slot was spent. Change: - Split the channel and schema lookups. A message whose channel the listing or the topic filter rejects, or whose log time is out of range, is skipped before its schema is looked up. - A message that may be read still fails on a missing schema and names the corrupt chunk read back for it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_chunks.py | 49 +++++++++++++++---- .../tests/unit/datasource_v2/mcap_testing.py | 9 ++-- .../unit/datasource_v2/test_mcap_reader.py | 27 ++++++++++ 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py index 160124924f3a..fc7b7380c656 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_chunks.py @@ -316,13 +316,16 @@ def _chunk_messages( for record in records: if isinstance(record, Message): index += 1 - channel, schema = self._channel_and_schema( - f, path, chunk_index, declared, record - ) - if not self._channel_is_selected(channel, schema, summary, selected): + channel = self._declared_channel(f, path, chunk_index, declared, record) + # Only a message that may be read needs its schema, so one the + # topic or time filters drop never fails on a missing schema. + if self._rejected_without_schema(channel, summary, selected): continue if not self._selection.in_time_range(record.log_time): continue + schema = self._declared_schema(f, path, chunk_index, declared, channel) + if not self._channel_is_selected(channel, schema, summary, selected): + continue row_id = message_row_id(path, chunk_index.chunk_start_offset, index) item = (schema, channel, record, row_id) yield (index, item) if with_index else item @@ -331,17 +334,17 @@ def _chunk_messages( elif isinstance(record, Schema): declared.schemas[record.id] = record - def _channel_and_schema( + def _declared_channel( self, f: Any, path: str, chunk_index: "ChunkIndex", declared: _Declared, message: "Message", - ) -> Tuple["Channel", Optional["Schema"]]: - """Look up the channel and schema of ``message``. + ) -> "Channel": + """Look up the channel of ``message``. - A record not met yet is read back from the earlier chunks of the file. + A channel not met yet is read back from the earlier chunks of the file. """ channel = declared.channels.get(message.channel_id) if channel is None: @@ -355,8 +358,22 @@ def _channel_and_schema( f"{message.channel_id}, which neither the summary nor any " f"chunk up to this one declares.{declared.undecodable_note()}" ) + return channel + + def _declared_schema( + self, + f: Any, + path: str, + chunk_index: "ChunkIndex", + declared: _Declared, + channel: "Channel", + ) -> Optional["Schema"]: + """Look up the schema of ``channel``, or ``None`` if it has none. + + A schema not met yet is read back from the earlier chunks of the file. + """ if not channel.schema_id: - return channel, None + return None if channel.schema_id not in declared.schemas: # The schema record may be only in an earlier chunk # (``repeat_schemas=False``). ``message_types`` and the schema @@ -371,7 +388,19 @@ def _channel_and_schema( f"{channel.schema_id}, which neither the summary nor any chunk " f"up to this one declares.{declared.undecodable_note()}" ) - return channel, schema + return schema + + def _rejected_without_schema( + self, channel: "Channel", summary: "Summary", selected: Set[int] + ) -> bool: + """Whether messages on ``channel`` fail the selection whatever its schema. + + That is the case for a channel the summary lists but ``selected`` + leaves out, and for a channel whose topic is not selected. + """ + if channel.id in summary.channels and channel.id not in selected: + return True + return not self._selection.accepts_channel(channel, None) def _channel_is_selected( self, diff --git a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py index 316e05858073..a7edbdb28b47 100644 --- a/python/ray/data/tests/unit/datasource_v2/mcap_testing.py +++ b/python/ray/data/tests/unit/datasource_v2/mcap_testing.py @@ -111,7 +111,9 @@ def mixed_encoding_messages(n=6): return messages -def write_mcap_without_summary_schemas(path, b_declared_in_chunk_1=False): +def write_mcap_without_summary_schemas( + path, b_declared_in_chunk_1=False, message_indexes=True +): """Write two JSON channels with two schemas, one message per chunk. The schema records are written only inside the chunks @@ -122,7 +124,8 @@ def write_mcap_without_summary_schemas(path, b_declared_in_chunk_1=False): topic /a /b /a /b /a /b Both schemas are declared in chunk 0, or with ``b_declared_in_chunk_1``, - ``other_schema`` only in chunk 1. + ``other_schema`` only in chunk 1. Without ``message_indexes``, the chunk + indexes do not say which channels a chunk holds, so a read opens every chunk. """ from mcap.writer import CompressionType, IndexType, Writer @@ -131,7 +134,7 @@ def write_mcap_without_summary_schemas(path, b_declared_in_chunk_1=False): stream, chunk_size=1, compression=CompressionType.ZSTD, - index_types=IndexType.ALL, + index_types=IndexType.ALL if message_indexes else IndexType.CHUNK, repeat_schemas=False, ) writer.start(profile="", library="ray-test") diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 851216572ae3..66fce07d5daf 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -540,6 +540,33 @@ def test_a_schema_only_a_corrupt_chunk_declares_fails_a_skipping_read(tmp_path): assert reported == [] +UNDECODABLE_SCHEMA_CHUNK = 1 # the only chunk that declares ``/b``'s schema + + +def test_an_unselected_topic_whose_schema_is_corrupt_costs_no_rows( + tmp_path, restore_max_errored_blocks +): + """When errors are skipped, a read of ``/a`` whose file declares the + unselected ``/b``'s schema only in a corrupt chunk yields every ``/a`` row. + It reports that chunk as skipped, not the missing schema.""" + clean = os.path.join(tmp_path, "clean.mcap") + # Without message indexes, the read opens the /b chunks 3 and 5 too. + write_mcap_without_summary_schemas( + clean, b_declared_in_chunk_1=True, message_indexes=False + ) + path = write_corrupt_chunk( + os.path.join(tmp_path, "bad.mcap"), clean, UNDECODABLE_SCHEMA_CHUNK + ) + + tables, error = read_one_task([path], topics=["/a"]) + + assert rows_of(tables) == [(path, seq) for seq in (0, 2, 4)] + assert str(error).startswith( + "Skipped 1 corrupt chunk at byte offset " + f"{chunk_offset(path, UNDECODABLE_SCHEMA_CHUNK)} of MCAP file {path!r}: " + ) + + def test_a_message_on_an_undeclared_channel_is_not_skipped_as_corrupt(tmp_path): """A chunk that decodes but holds a message on a channel nothing declares raises that error even when corrupt chunks are skipped, rather than being From ff354e82e184edfcb46bf3a768fb0f7bb2b5b955 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Tue, 10 Nov 2026 19:20:06 -0800 Subject: [PATCH 11/13] [Data] read_mcap: keep one error for a file's corrupt chunks Problem: - A read that skips corrupt chunks keeps each chunk's error until the file is done. An error's traceback holds its chunk's bytes, so every corrupt chunk adds its size to the read's memory. Change: - Keep the byte offsets of a file's corrupt chunks and only the error at the lowest offset. The error the read raises is unchanged. - A test reads a file whose every chunk is corrupt and bounds its peak. - Cut four reader tests whose cases other tests already cover. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- .../datasource_v2/formats/mcap/mcap_reader.py | 46 +++++-- .../unit/datasource_v2/test_mcap_reader.py | 122 ++++++++---------- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py index 6546446bd77c..6c76ed0825d9 100644 --- a/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py +++ b/python/ray/data/_internal/datasource_v2/formats/mcap/mcap_reader.py @@ -25,7 +25,6 @@ from typing import ( TYPE_CHECKING, Any, - Dict, Iterator, List, Optional, @@ -202,7 +201,7 @@ def _read_owned_file( raise failures.append((owned.path, e)) return - if skipped is not None and (skipped.chunks or skipped.num_messages): + if skipped is not None and (skipped.chunks.offsets or skipped.num_messages): failures.append((owned.path, _skipped_data_error(owned.path, skipped))) # -- one file ---------------------------------------------------------- @@ -432,17 +431,37 @@ class _SkippedMessage: error: Exception +@dataclass +class _CorruptOffsets: + """The byte offsets of what a read skipped as corrupt, and one error. + + Only the error at the lowest offset is kept. An error's traceback holds the + bytes that failed to parse, so keeping every error would hold every + corrupt chunk until the file is done. + """ + + offsets: Set[int] = field(default_factory=set) + lowest: Optional[int] = None + error: Optional[Exception] = None + + def add(self, offset: int, error: Exception) -> None: + """Record ``offset``, keeping ``error`` if ``offset`` is the lowest.""" + if self.lowest is None or offset < self.lowest: + self.lowest = offset + self.error = error + self.offsets.add(offset) + + @dataclass class _SkippedData: """What a read of one file skipped. - Every corrupt chunk is kept, with its first error. Messages that are not - valid JSON are counted and only the first is kept, so a file of such - messages costs constant memory. + Every corrupt chunk is named by its offset, and one error is kept for them + all. Messages that are not valid JSON are counted and only the first is + kept. """ - # Corrupt chunks, by byte offset. - chunks: Dict[int, Exception] = field(default_factory=dict) + chunks: _CorruptOffsets = field(default_factory=_CorruptOffsets) num_messages: int = 0 first_message: Optional[_SkippedMessage] = None @@ -454,7 +473,7 @@ def record_message(self, row_id: str, error: Exception) -> None: def clear(self) -> None: """Forget what was skipped, for a read that starts the file over.""" - self.chunks.clear() + self.chunks = _CorruptOffsets() self.num_messages = 0 self.first_message = None @@ -481,10 +500,10 @@ def _add_message( def _record_corrupt_chunk( - corrupt: Dict[int, Exception], chunk_index: "ChunkIndex", error: Exception + corrupt: _CorruptOffsets, chunk_index: "ChunkIndex", error: Exception ) -> None: - """Keep the first error of each chunk a read skips as corrupt.""" - corrupt.setdefault(chunk_index.chunk_start_offset, error) + """Record a chunk a read skips as corrupt.""" + corrupt.add(chunk_index.chunk_start_offset, error) def _skipped_data_error(path: str, skipped: _SkippedData) -> ValueError: @@ -493,7 +512,7 @@ def _skipped_data_error(path: str, skipped: _SkippedData) -> ValueError: It names every chunk and the first message, and quotes the first error: the first chunk's, or the first message's when no chunk was skipped. """ - offsets = sorted(skipped.chunks) + offsets = sorted(skipped.chunks.offsets) parts = [] if offsets: where = ( @@ -508,7 +527,8 @@ def _skipped_data_error(path: str, skipped: _SkippedData) -> ValueError: which = "row id" if count == 1 else "first: row id" parts.append(f"{count} {what} not valid JSON ({which} {message.row_id})") if offsets: - first = skipped.chunks[offsets[0]] + first = skipped.chunks.error + assert first is not None else: assert message is not None first = message.error diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 66fce07d5daf..b6aaed637063 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -8,13 +8,18 @@ import importlib.util import json import os -import re +import random import time +import tracemalloc +from functools import partial import pyarrow as pa import pytest -from ray.data._internal.datasource_v2.formats.mcap import mcap_message_rows +from ray.data._internal.datasource_v2.formats.mcap import ( + mcap_message_rows, + mcap_reader, +) from ray.data._internal.datasource_v2.formats.mcap.mcap_chunks import ( SelectedMessageReader, ) @@ -23,7 +28,6 @@ ) from ray.data._internal.datasource_v2.formats.mcap.mcap_io import ChunkReads from ray.data._internal.datasource_v2.formats.mcap.mcap_message_rows import ( - InvalidJsonPayload, _MessageTableBuilder, message_schema, ) @@ -238,22 +242,55 @@ def test_a_corrupt_chunk_costs_only_its_own_rows( ) -def test_one_error_names_every_corrupt_chunk_of_a_file( - tmp_path, restore_max_errored_blocks +CORRUPT_CHUNKS = 16 # chunks of a file whose every chunk is corrupt +CHUNK_BYTES = 1 << 20 # each chunk's payload, random so it does not compress + + +def test_a_file_of_corrupt_chunks_keeps_one_error( + tmp_path, monkeypatch, restore_max_errored_blocks ): - """A file with two corrupt chunks, one of each kind, loses only their two - rows, and one error names both chunks.""" - a, b, c = write_task_files(tmp_path, bad=set()) - corrupt_in_place(b, 2, write_corrupt_chunk) - corrupt_in_place(b, 6, write_corrupt_chunk_header) + """A file whose every chunk is corrupt loses all its rows, and one error + names every chunk. The read keeps one chunk's error rather than one per + chunk, so its peak does not grow with the number of corrupt chunks.""" + path = os.path.join(tmp_path, "b.mcap") + rng = random.Random(0) + write_mcap( + path, + [ + { + "topic": "/a", + "log_time": BASE_TIME + i, + "data": rng.randbytes(CHUNK_BYTES), + } + for i in range(CORRUPT_CHUNKS) + ], + message_encoding="cdr", + ) + offsets = [chunk_offset(path, i) for i in range(CORRUPT_CHUNKS)] + for index in range(CORRUPT_CHUNKS): + corrupt_in_place(path, index) + # One chunk per read, so the read's own buffers hold one chunk at a time. + monkeypatch.setattr( + mcap_reader, + "SelectedMessageReader", + partial(SelectedMessageReader, chunk_reads=ONE_CHUNK_PER_READ), + ) - tables, error = read_one_task([a, b, c]) + tracemalloc.start() + try: + tables, error = read_one_task([path]) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() - assert rows_of(tables) == file_rows(a) + file_rows(b, {2, 6}) + file_rows(c) + assert rows_of(tables) == [] assert str(error).startswith( - f"Skipped 2 corrupt chunks at byte offsets {chunk_offset(b, 2)}, " - f"{chunk_offset(b, 6)} of MCAP file {b!r}: " + f"Skipped {CORRUPT_CHUNKS} corrupt chunks at byte offsets " + f"{', '.join(map(str, offsets))} of MCAP file {path!r}: ZstdError: " ) + # The read's buffers hold a few chunks at once. An error's traceback holds + # its chunk's bytes, so keeping every error would hold all of them. + assert peak < CORRUPT_CHUNKS // 2 * CHUNK_BYTES BAD_JSON = 4 # the message whose payload is not valid JSON @@ -318,40 +355,6 @@ def test_one_error_names_the_corrupt_chunks_and_the_messages_that_are_not_json( ) -def test_a_file_of_messages_that_are_not_json_is_counted_not_listed( - tmp_path, restore_max_errored_blocks -): - """A file whose every message is not valid JSON loses all its rows, and its - error counts the messages and quotes only the first one's error.""" - a, b, c = write_task_files(tmp_path, bad=set()) - write_invalid_json(b, bad=range(CHUNKED_FILE_MESSAGES)) - - tables, error = read_one_task([a, b, c]) - - assert rows_of(tables) == file_rows(a) + file_rows(c) - first_row_id = message_row_id_of(b, 0, False) - assert str(error).startswith( - f"Skipped {CHUNKED_FILE_MESSAGES} messages that are not valid JSON " - f"(first: row id {first_row_id}) of MCAP file {b!r}: " - f"InvalidJsonPayload: {first_row_id}: " - ) - - -def test_a_message_that_is_not_json_fails_its_task_when_errors_are_not_skipped( - tmp_path, restore_max_errored_blocks -): - """With ``max_errored_blocks=0``, a message that is not valid JSON fails the - task at once, with an error that names the message.""" - a, b, c = write_task_files(tmp_path, bad=set()) - write_invalid_json(b, bad={BAD_JSON}) - - tables, error = read_one_task([a, b, c], max_errored_blocks=0) - - assert rows_of(tables) == file_rows(a) - assert isinstance(error, InvalidJsonPayload) - assert str(error).startswith(f"{message_row_id_of(b, BAD_JSON, False)}: ") - - LIMIT = 4 # rows a limited read asks for @@ -1290,29 +1293,6 @@ def test_build_empties_the_builder(chunked_file): assert second.column(ROW_ID_COLUMN).to_pylist() == expected_ids -def test_a_payload_that_is_not_json_leaves_the_builder_unchanged(tmp_path): - """Adding a message whose payload is not valid JSON raises - ``InvalidJsonPayload`` before the builder changes, so the rows added around - it build into a consistent table.""" - path = write_invalid_json(os.path.join(tmp_path, "bad.mcap"), bad={1}) - builder = _MessageTableBuilder( - columns=None, include_metadata=True, include_row_id=True, decode_json=True - ) - good, bad, *_ = selected_messages(path) - builder.add(good) - before = (builder.num_rows, builder.estimated_bytes) - - with pytest.raises( - InvalidJsonPayload, match=re.escape(bad[3]) + ": .* not valid JSON" - ): - builder.add(bad) - - assert (builder.num_rows, builder.estimated_bytes) == before - table = builder.build() - assert table.column(ROW_ID_COLUMN).to_pylist() == [good[3]] - assert table.column("log_time").to_pylist() == [BASE_TIME] - - def test_payload_buffer_starts_a_chunk_before_its_offsets_overflow(monkeypatch): """A ``binary`` buffer is sealed into its own chunk before its int32 offsets would overflow, and the chunks keep the values in order.""" From f9130901786e756da61195446f97827e31dc8030 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 13 Nov 2026 09:06:42 -0800 Subject: [PATCH 12/13] [Data] read_mcap: find the file name in the error's notes Problem: - Master's retry wrapper now names the file in a note on the error instead of rewriting its message. The test that a task fails at its first corrupt chunk looked for the file name in str(error), so it fails on master. Change: - Look for the file name in the error's __notes__. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index b6aaed637063..84748a4fc5b6 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -438,7 +438,8 @@ def test_a_task_fails_at_its_first_corrupt_chunk_when_errors_are_not_skipped( # b's rows before the corrupt chunk were still being built into a table. assert rows_of(tables) == file_rows(a) - assert "b.mcap" in str(error) + # The retry wrapper names the file in a note, leaving the message as raised. + assert "b.mcap" in "\n".join(getattr(error, "__notes__", [])) def test_an_error_reading_the_file_is_not_skipped_as_a_corrupt_chunk( From 8132257457e4061036e7978e3049e491ef72e619 Mon Sep 17 00:00:00 2001 From: Marwan Sarieddine Date: Fri, 13 Nov 2026 09:43:39 -0800 Subject: [PATCH 13/13] [Data] read_mcap: check the corrupt chunk's error, not its notes Problem: - The retry wrapper adds the file name as an exception note, and Python 3.10 has no notes, so CI's error carries no file name at all. Change: - Check that the task raised the corrupt chunk's ZstdError. The rows check already shows the task stopped in b. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Marwan Sarieddine --- python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py index 84748a4fc5b6..17bfa93f7ba4 100644 --- a/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py +++ b/python/ray/data/tests/unit/datasource_v2/test_mcap_reader.py @@ -438,8 +438,7 @@ def test_a_task_fails_at_its_first_corrupt_chunk_when_errors_are_not_skipped( # b's rows before the corrupt chunk were still being built into a table. assert rows_of(tables) == file_rows(a) - # The retry wrapper names the file in a note, leaving the message as raised. - assert "b.mcap" in "\n".join(getattr(error, "__notes__", [])) + assert type(error).__name__ == "ZstdError" def test_an_error_reading_the_file_is_not_skipped_as_a_corrupt_chunk(