diff --git a/python/ray/data/_internal/planner/checkpoint/plan_write_op.py b/python/ray/data/_internal/planner/checkpoint/plan_write_op.py index 76641ea330ab..6ffb2e1290d5 100644 --- a/python/ray/data/_internal/planner/checkpoint/plan_write_op.py +++ b/python/ray/data/_internal/planner/checkpoint/plan_write_op.py @@ -27,6 +27,10 @@ from ray.data.datasource.file_datasink import _FileDatasink from ray.data.datasource.filename_provider import _split_base_and_ext +# Key for storing the per-task ID-only blocks picked up by +# `_generate_capture_id_columns_transform` for the post-write checkpoint. +CHECKPOINT_ID_BLOCKS_KWARG_NAME = "_checkpoint_id_blocks" + def _validate_id_column_exists(id_column: str, block: Block) -> None: """Validate that the ID column exists in the block. @@ -142,14 +146,19 @@ def plan_write_op_with_checkpoint_writer( f"the checkpoint is saved, duplicate data may be written on retry. " f"This will be addressed in a future version." ) + # Pre-write transform: pick up the ID columns while the blocks stream + # to the write, since the write does not re-emit them afterwards. + capture_id_columns_fn = _generate_capture_id_columns_transform(data_context) write_checkpoint_fn = _generate_non_atomic_write_checkpoint_transform( - data_context, checkpoint_writer + checkpoint_writer ) post_transformations = [ write_checkpoint_fn, collect_stats_fn, ] - pre_transformations = [] + pre_transformations = [ + capture_id_columns_fn, + ] physical_op = _plan_write_op_internal( op, @@ -264,6 +273,46 @@ def prepare_checkpoint( ) +def _generate_capture_id_columns_transform( + data_context: DataContext, +) -> BlockMapTransformFn: + """Generate transform for collecting ID columns while the blocks stream. + + Non-file datasinks write their checkpoint AFTER the data (see + `_generate_non_atomic_write_checkpoint_transform`), but the write consumes + the blocks without re-emitting them: keeping every block around for a + second reader is what inflated write-task memory. So pick up each block's + ID column on its way to the write and keep only that -- one column instead + of the whole block. + + Steps: + 1. Validates the ID column exists on every non-empty block + 2. Stores per-block ID-only blocks in ctx.kwargs for the post-write + checkpoint transform + 3. Passes the original blocks on to the write + """ + + def capture_id_columns( + blocks: Iterable[Block], ctx: TaskContext + ) -> Iterable[Block]: + id_column = data_context.checkpoint_config.id_column + id_blocks: List[Block] = [] + for block in blocks: + ba = BlockAccessor.for_block(block) + if ba.num_rows() > 0: + # Validate ID column exists + _validate_id_column_exists(id_column, block) + id_blocks.append(ba.select(columns=[id_column])) + yield block + + ctx.kwargs[CHECKPOINT_ID_BLOCKS_KWARG_NAME] = id_blocks + + return BlockMapTransformFn( + capture_id_columns, + disable_block_shaping=True, + ) + + def _generate_commit_checkpoint_transform( checkpoint_writer: CheckpointWriter, ) -> BlockMapTransformFn: @@ -285,8 +334,11 @@ def commit_checkpoints( ) -> Iterable[Block]: # Each stage runs on the first pull from it, so nothing upstream has # run yet. `prepare_checkpoint` is what leaves the pending checkpoints - # on `ctx`. Drain first, or there is nothing here to commit. - blocks = list(blocks) + # on `ctx`. Drain first, or there is nothing here to commit. The write + # emits no blocks of its own -- its stats are on `ctx` -- so there is + # nothing here to hold on to. + for _ in blocks: + pass # Get pending checkpoints written in pre-write phase pending_checkpoints: List[PendingCheckpoint] = ctx.kwargs.get( @@ -297,7 +349,7 @@ def commit_checkpoints( for pending in pending_checkpoints: checkpoint_writer.commit_checkpoint(pending) - return blocks + return iter(()) return BlockMapTransformFn( commit_checkpoints, @@ -306,7 +358,6 @@ def commit_checkpoints( def _generate_non_atomic_write_checkpoint_transform( - data_context: DataContext, checkpoint_writer: CheckpointWriter, ) -> BlockMapTransformFn: """Generate transform for writing checkpoints AFTER data write (non-file datasinks). @@ -324,26 +375,33 @@ def _generate_non_atomic_write_checkpoint_transform( For idempotent operations (upserts with unique keys), this is safe. For non-idempotent operations (inserts), duplicates may result. + The blocks are gone by the time this runs -- the write consumes them + without re-emitting them -- so the ID columns were picked up on their way + past by `_generate_capture_id_columns_transform`. + TODO: For datasinks that support deletions (e.g., SQL DELETE by ID), we could store written IDs in pending checkpoints and delete them on recovery, avoiding duplicates even for non-idempotent operations. """ def write_checkpoint(blocks: Iterable[Block], ctx: TaskContext) -> Iterable[Block]: - # Combine all blocks - block_list, combined_block = _combine_blocks(blocks) + # Drain before reading `ctx` below: consuming the input is what runs + # the write stage. + for _ in blocks: + pass + + # Combine the ID-only blocks captured before the write + _, combined_block = _combine_blocks( + ctx.kwargs.pop(CHECKPOINT_ID_BLOCKS_KWARG_NAME, []) + ) ba = BlockAccessor.for_block(combined_block) if ba.num_rows() > 0: - # Validate ID column exists - id_column = data_context.checkpoint_config.id_column - _validate_id_column_exists(id_column, combined_block) - # Write checkpoint directly (no 2-phase commit) # No data_file_path since non-file datasinks don't have file paths checkpoint_writer.write_block_checkpoint(ba) - return iter(block_list) + return iter(()) return BlockMapTransformFn( write_checkpoint, diff --git a/python/ray/data/_internal/planner/plan_write_op.py b/python/ray/data/_internal/planner/plan_write_op.py index 7d9d3132c8f5..1c210bcefb46 100644 --- a/python/ray/data/_internal/planner/plan_write_op.py +++ b/python/ray/data/_internal/planner/plan_write_op.py @@ -1,6 +1,5 @@ -import itertools import uuid -from typing import TYPE_CHECKING, Callable, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, List, Optional, Union from ray.data._internal.execution.bundle_queue import EstimateBytes, RebundleQueue from ray.data._internal.execution.interfaces import PhysicalOperator @@ -21,6 +20,53 @@ WRITE_UUID_KWARG_NAME = "write_uuid" # Key for storing pending checkpoint paths for commit phase PENDING_CHECKPOINTS_KWARG_NAME = "_pending_checkpoints" +# Key for storing the write stats accumulator on `TaskContext.kwargs` +WRITE_STATS_KWARG_NAME = "_write_stats" + + +class _WriteStats: + """Row and byte totals for one write task's input. + + This is all the output stats block needs from the input, so it is + accumulated while blocks are on their way to the sink rather than by + holding the blocks and measuring them afterwards. + """ + + __slots__ = ("num_rows", "size_bytes") + + def __init__(self) -> None: + self.num_rows = 0 + self.size_bytes = 0 + + def add(self, block: Block) -> None: + accessor = BlockAccessor.for_block(block) + self.num_rows += accessor.num_rows() + self.size_bytes += accessor.size_bytes() + + +class _StatsCollectingBlocks(Iterator[Block]): + """Block pass-through that accumulates :class:`_WriteStats` as blocks flow. + + ``itertools.tee`` used to give the write and the stats collection their own + view of the same blocks, but the two views are read one after the other: + the sink drains its copy first, so tee buffers the whole input for the + later reader. The stats are two integers, so count blocks on the way past + and hold none of them. + """ + + __slots__ = ("_blocks", "_stats") + + def __init__(self, blocks: Iterable[Block], stats: _WriteStats) -> None: + self._blocks = iter(blocks) + self._stats = stats + + def __iter__(self) -> "_StatsCollectingBlocks": + return self + + def __next__(self) -> Block: + block = next(self._blocks) + self._stats.add(block) + return block def generate_write_fn( @@ -29,17 +75,28 @@ def generate_write_fn( def fn(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]: """Writes the blocks to the given datasink or legacy datasource. - Outputs the original blocks to be written.""" - # Create a copy of the iterator, so we can return the original blocks. - it1, it2 = itertools.tee(blocks, 2) + Outputs no blocks: the write stats accumulate on `ctx.kwargs`, and + `generate_collect_write_stats_fn` turns them into the operator's + single output block. Returning the written blocks would mean holding + every one of them until the stats stage read them.""" + stats = _WriteStats() + ctx.kwargs[WRITE_STATS_KWARG_NAME] = stats + blocks = _StatsCollectingBlocks(blocks, stats) + if isinstance(datasink_or_legacy_datasource, Datasink): ctx.kwargs["_datasink_write_return"] = datasink_or_legacy_datasource.write( - it1, ctx + blocks, ctx ) else: - datasink_or_legacy_datasource.write(it1, ctx, **write_args) + datasink_or_legacy_datasource.write(blocks, ctx, **write_args) - return it2 + # The stats cover the whole input, not just what the sink pulled: a + # sink may stop early, and the blocks it did not read still arrived at + # this task. Drain the rest through the counter, dropping each block. + for _ in blocks: + pass + + return iter(()) return fn @@ -52,10 +109,15 @@ def generate_collect_write_stats_fn() -> BlockMapTransformFn: def fn(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]: """Handles stats collection for block writes.""" # Drain before reading `ctx` below: consuming the input is what runs - # the write stage, and the write is what sets `_datasink_write_return`. - block_accessors = [BlockAccessor.for_block(block) for block in blocks] - total_num_rows = sum(ba.num_rows() for ba in block_accessors) - total_size_bytes = sum(ba.size_bytes() for ba in block_accessors) + # the write stage, and the write is what sets `_datasink_write_return` + # and the stats. Count the blocks as they drain, so callers that pass + # the blocks straight in (rather than driving this from + # `generate_write_fn`) still get their stats without holding them. + drained = _WriteStats() + for block in blocks: + drained.add(block) + + stats = ctx.kwargs.pop(WRITE_STATS_KWARG_NAME, drained) # NOTE: Write tasks can return anything, so we need to wrap it in a valid block # type. @@ -63,9 +125,9 @@ def fn(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]: block = pd.DataFrame( { - "num_rows": [total_num_rows], - "size_bytes": [total_size_bytes], - "write_return": [ctx.kwargs.get("_datasink_write_return", None)], + "num_rows": [stats.num_rows], + "size_bytes": [stats.size_bytes], + "write_return": [ctx.kwargs.pop("_datasink_write_return", None)], } ) return iter([block]) diff --git a/python/ray/data/tests/unit/test_plan_write_op.py b/python/ray/data/tests/unit/test_plan_write_op.py new file mode 100644 index 000000000000..606635798a07 --- /dev/null +++ b/python/ray/data/tests/unit/test_plan_write_op.py @@ -0,0 +1,248 @@ +"""Unit tests for the write-op transforms in ``plan_write_op.py``. + +These drive the transform fns directly instead of through Ray tasks: what +matters here is what each fn does to the blocks it is handed and to +``TaskContext.kwargs``. +""" + +import gc +import weakref +from types import SimpleNamespace +from typing import List, Optional + +import pandas as pd +import pytest + +from ray.data._internal.execution.interfaces.task_context import TaskContext +from ray.data._internal.execution.operators.map_transformer import ( + BlockMapTransformFn, + MapTransformer, + TransformClock, +) +from ray.data._internal.planner.checkpoint.plan_write_op import ( + _generate_capture_id_columns_transform, + _generate_non_atomic_write_checkpoint_transform, +) +from ray.data._internal.planner.plan_write_op import ( + generate_collect_write_stats_fn, + generate_write_fn, +) +from ray.data.block import BlockAccessor +from ray.data.datasource.datasink import Datasink + + +def _block(num_rows: int) -> pd.DataFrame: + return pd.DataFrame({"x": list(range(num_rows))}) + + +class _RecordingSink(Datasink): + """Datasink that records what it saw, optionally stopping early.""" + + def __init__(self, max_blocks: Optional[int] = None, write_return=None): + self.num_blocks = 0 + self.num_rows = 0 + self.max_blocks = max_blocks + self.write_return = write_return + + def write(self, blocks, ctx): + for block in blocks: + self.num_blocks += 1 + self.num_rows += len(block) + if self.max_blocks is not None and self.num_blocks >= self.max_blocks: + break + return self.write_return + + +class _RecordingLegacyDatasource: + """Legacy write target: anything that is not a Datasink.""" + + def __init__(self): + self.num_rows = 0 + self.write_args = None + + def write(self, blocks, ctx, **write_args): + for block in blocks: + self.num_rows += len(block) + self.write_args = write_args + + +class _RecordingCheckpointWriter: + """Stand-in for CheckpointWriter that records what would be checkpointed.""" + + def __init__(self): + self.checkpointed_ids: Optional[List] = None + + def write_block_checkpoint(self, block: BlockAccessor): + self.checkpointed_ids = block.to_arrow().column(0).to_pylist() + + +def _stats_row(ctx: TaskContext, blocks=()) -> dict: + """Run the stats transform the way the operator chains it: on empty input.""" + stats = next(generate_collect_write_stats_fn()(iter(blocks), ctx)) + return dict(stats.iloc[0]) + + +def test_write_fn_collects_stats_and_emits_no_blocks(): + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink(write_return="done") + blocks = [_block(2), _block(3)] + + out = generate_write_fn(sink)(iter(blocks), ctx) + + # The written blocks are not re-emitted: holding them for the stats stage + # is what used to keep the whole input in memory. + assert list(out) == [] + assert sink.num_blocks == 2 + assert sink.num_rows == 5 + + stats = _stats_row(ctx) + assert stats["num_rows"] == 5 + assert stats["size_bytes"] == sum( + BlockAccessor.for_block(b).size_bytes() for b in blocks + ) + assert stats["write_return"] == "done" + + +def test_write_stats_cover_blocks_the_sink_did_not_read(): + # A sink may stop early. The stats still describe the whole task input, + # as they did when the tee-based implementation drained its second copy. + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink(max_blocks=1) + blocks = [_block(2), _block(3), _block(4)] + + generate_write_fn(sink)(iter(blocks), ctx) + + assert sink.num_rows == 2 + assert _stats_row(ctx)["num_rows"] == 9 + + +def test_write_fn_holds_no_blocks_for_the_stats(): + # Regression test for the `itertools.tee` implementation, which kept every + # written block buffered in the returned iterator for the stats stage: a + # write task then held its whole input in memory until the stats ran. + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink() + refs = [] + + def blocks(): + for _ in range(3): + block = _block(2) + refs.append(weakref.ref(block)) + yield block + + out = generate_write_fn(sink)(blocks(), ctx) + + # The stats stage has not run yet and `out` is still alive. Nothing may be + # holding the written blocks at this point. + gc.collect() + assert all(ref() is None for ref in refs), "written blocks are still held" + assert list(out) == [] + + +def test_legacy_datasource_write_collects_stats(): + ctx = TaskContext(task_idx=0, op_name="test") + source = _RecordingLegacyDatasource() + blocks = [_block(2), _block(3)] + + out = generate_write_fn(source, mode="x")(iter(blocks), ctx) + + assert list(out) == [] + assert source.num_rows == 5 + assert source.write_args == {"mode": "x"} + assert _stats_row(ctx)["num_rows"] == 5 + + +def test_write_fn_with_empty_input(): + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink() + + assert list(generate_write_fn(sink)(iter(()), ctx)) == [] + + stats = _stats_row(ctx) + assert stats["num_rows"] == 0 + assert stats["write_return"] is None + + +def test_collect_write_stats_without_write_fn_counts_blocks(): + # `generate_collect_write_stats_fn` is also driven directly, with no + # `generate_write_fn` upstream (see test_bigquery). It must still derive + # the stats from the blocks it is given. + ctx = TaskContext(task_idx=0, op_name="test") + blocks = [_block(2), _block(3)] + + stats = _stats_row(ctx, blocks) + + assert stats["num_rows"] == 5 + assert stats["size_bytes"] == sum( + BlockAccessor.for_block(b).size_bytes() for b in blocks + ) + assert stats["write_return"] is None + + +def test_write_stats_read_after_write_stage_runs(): + # The write stage is lazy: it only runs when the stats stage first pulls + # on its output. So the stats must not be read off `ctx` before that pull, + # or they are read before the write has counted anything. Drive the real + # transform chain from its output end, the way a map task does. + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink(write_return="done") + transformer = MapTransformer( + [ + BlockMapTransformFn(generate_write_fn(sink), disable_block_shaping=True), + generate_collect_write_stats_fn(), + ] + ) + + out = transformer.apply_transform( + iter([_block(2), _block(3)]), ctx, clock=TransformClock() + ) + stats = next(iter(out)) + + assert sink.num_rows == 5 + assert dict(stats.iloc[0])["num_rows"] == 5 + assert dict(stats.iloc[0])["write_return"] == "done" + + +def test_non_atomic_checkpoint_captures_ids_during_write(): + # Non-file sinks write their checkpoint AFTER the data, but the write + # consumes the blocks without re-emitting them, so the ID columns are + # picked up on their way past and only they are kept. + data_context = SimpleNamespace(checkpoint_config=SimpleNamespace(id_column="id")) + capture_fn = _generate_capture_id_columns_transform(data_context) + checkpoint_writer = _RecordingCheckpointWriter() + write_checkpoint_fn = _generate_non_atomic_write_checkpoint_transform( + checkpoint_writer + ) + + ctx = TaskContext(task_idx=0, op_name="test") + sink = _RecordingSink(write_return="done") + write_fn = generate_write_fn(sink) + blocks = [ + pd.DataFrame({"id": [1, 2], "x": ["a", "b"]}), + pd.DataFrame({"id": [3], "x": ["c"]}), + ] + + # Chain them the way the operator does: each stage pulls the one before. + captured = capture_fn(iter(blocks), ctx) + written = write_fn(captured, ctx) + checkpointed = write_checkpoint_fn(written, ctx) + + stats = next(generate_collect_write_stats_fn()(checkpointed, ctx)) + assert dict(stats.iloc[0])["num_rows"] == 3 + assert sink.num_rows == 3 + assert checkpoint_writer.checkpointed_ids == [1, 2, 3] + + +def test_non_atomic_checkpoint_validates_id_column_before_write(): + data_context = SimpleNamespace(checkpoint_config=SimpleNamespace(id_column="id")) + capture_fn = _generate_capture_id_columns_transform(data_context) + ctx = TaskContext(task_idx=0, op_name="test") + + with pytest.raises(ValueError, match="is absent"): + list(capture_fn(iter([pd.DataFrame({"x": [1, 2]})]), ctx)) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__]))