Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 71 additions & 13 deletions python/ray/data/_internal/planner/checkpoint/plan_write_op.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
from ray.data.datasource.file_datasink import _FileDatasink
from ray.data.datasource.filename_provider import _split_base_and_ext

# Key for storing the per-task ID-only blocks picked up by
# `_generate_capture_id_columns_transform` for the post-write checkpoint.
CHECKPOINT_ID_BLOCKS_KWARG_NAME = "_checkpoint_id_blocks"


def _validate_id_column_exists(id_column: str, block: Block) -> None:
"""Validate that the ID column exists in the block.
Expand Down Expand Up @@ -142,14 +146,19 @@ def plan_write_op_with_checkpoint_writer(
f"the checkpoint is saved, duplicate data may be written on retry. "
f"This will be addressed in a future version."
)
# Pre-write transform: pick up the ID columns while the blocks stream
# to the write, since the write does not re-emit them afterwards.
capture_id_columns_fn = _generate_capture_id_columns_transform(data_context)
write_checkpoint_fn = _generate_non_atomic_write_checkpoint_transform(
data_context, checkpoint_writer
checkpoint_writer
)
post_transformations = [
write_checkpoint_fn,
collect_stats_fn,
]
pre_transformations = []
pre_transformations = [
capture_id_columns_fn,
]

physical_op = _plan_write_op_internal(
op,
Expand Down Expand Up @@ -264,6 +273,46 @@ def prepare_checkpoint(
)


def _generate_capture_id_columns_transform(
data_context: DataContext,
) -> BlockMapTransformFn:
"""Generate transform for collecting ID columns while the blocks stream.

Non-file datasinks write their checkpoint AFTER the data (see
`_generate_non_atomic_write_checkpoint_transform`), but the write consumes
the blocks without re-emitting them: keeping every block around for a
second reader is what inflated write-task memory. So pick up each block's
ID column on its way to the write and keep only that -- one column instead
of the whole block.

Steps:
1. Validates the ID column exists on every non-empty block
2. Stores per-block ID-only blocks in ctx.kwargs for the post-write
checkpoint transform
3. Passes the original blocks on to the write
"""

def capture_id_columns(
blocks: Iterable[Block], ctx: TaskContext
) -> Iterable[Block]:
id_column = data_context.checkpoint_config.id_column
id_blocks: List[Block] = []
for block in blocks:
ba = BlockAccessor.for_block(block)
if ba.num_rows() > 0:
# Validate ID column exists
_validate_id_column_exists(id_column, block)
id_blocks.append(ba.select(columns=[id_column]))
yield block

ctx.kwargs[CHECKPOINT_ID_BLOCKS_KWARG_NAME] = id_blocks

return BlockMapTransformFn(
capture_id_columns,
disable_block_shaping=True,
)


def _generate_commit_checkpoint_transform(
checkpoint_writer: CheckpointWriter,
) -> BlockMapTransformFn:
Expand All @@ -285,8 +334,11 @@ def commit_checkpoints(
) -> Iterable[Block]:
# Each stage runs on the first pull from it, so nothing upstream has
# run yet. `prepare_checkpoint` is what leaves the pending checkpoints
# on `ctx`. Drain first, or there is nothing here to commit.
blocks = list(blocks)
# on `ctx`. Drain first, or there is nothing here to commit. The write
# emits no blocks of its own -- its stats are on `ctx` -- so there is
# nothing here to hold on to.
for _ in blocks:
pass

# Get pending checkpoints written in pre-write phase
pending_checkpoints: List[PendingCheckpoint] = ctx.kwargs.get(
Expand All @@ -297,7 +349,7 @@ def commit_checkpoints(
for pending in pending_checkpoints:
checkpoint_writer.commit_checkpoint(pending)

return blocks
return iter(())

return BlockMapTransformFn(
commit_checkpoints,
Expand All @@ -306,7 +358,6 @@ def commit_checkpoints(


def _generate_non_atomic_write_checkpoint_transform(
data_context: DataContext,
checkpoint_writer: CheckpointWriter,
) -> BlockMapTransformFn:
"""Generate transform for writing checkpoints AFTER data write (non-file datasinks).
Expand All @@ -324,26 +375,33 @@ def _generate_non_atomic_write_checkpoint_transform(
For idempotent operations (upserts with unique keys), this is safe. For
non-idempotent operations (inserts), duplicates may result.

The blocks are gone by the time this runs -- the write consumes them
without re-emitting them -- so the ID columns were picked up on their way
past by `_generate_capture_id_columns_transform`.

TODO: For datasinks that support deletions (e.g., SQL DELETE by ID), we
could store written IDs in pending checkpoints and delete them on recovery,
avoiding duplicates even for non-idempotent operations.
"""

def write_checkpoint(blocks: Iterable[Block], ctx: TaskContext) -> Iterable[Block]:
# Combine all blocks
block_list, combined_block = _combine_blocks(blocks)
# Drain before reading `ctx` below: consuming the input is what runs
# the write stage.
for _ in blocks:
pass

# Combine the ID-only blocks captured before the write
_, combined_block = _combine_blocks(
ctx.kwargs.pop(CHECKPOINT_ID_BLOCKS_KWARG_NAME, [])
)
Comment thread
FreemanDane marked this conversation as resolved.
Resolved
ba = BlockAccessor.for_block(combined_block)

if ba.num_rows() > 0:
# Validate ID column exists
id_column = data_context.checkpoint_config.id_column
_validate_id_column_exists(id_column, combined_block)

# Write checkpoint directly (no 2-phase commit)
# No data_file_path since non-file datasinks don't have file paths
checkpoint_writer.write_block_checkpoint(ba)

return iter(block_list)
return iter(())

return BlockMapTransformFn(
write_checkpoint,
Expand Down
92 changes: 77 additions & 15 deletions python/ray/data/_internal/planner/plan_write_op.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import itertools
import uuid
from typing import TYPE_CHECKING, Callable, Iterator, List, Optional, Union
from typing import TYPE_CHECKING, Callable, Iterable, Iterator, List, Optional, Union

from ray.data._internal.execution.bundle_queue import EstimateBytes, RebundleQueue
from ray.data._internal.execution.interfaces import PhysicalOperator
Expand All @@ -21,6 +20,53 @@
WRITE_UUID_KWARG_NAME = "write_uuid"
# Key for storing pending checkpoint paths for commit phase
PENDING_CHECKPOINTS_KWARG_NAME = "_pending_checkpoints"
# Key for storing the write stats accumulator on `TaskContext.kwargs`
WRITE_STATS_KWARG_NAME = "_write_stats"


class _WriteStats:
"""Row and byte totals for one write task's input.

This is all the output stats block needs from the input, so it is
accumulated while blocks are on their way to the sink rather than by
holding the blocks and measuring them afterwards.
"""

__slots__ = ("num_rows", "size_bytes")

def __init__(self) -> None:
self.num_rows = 0
self.size_bytes = 0

def add(self, block: Block) -> None:
accessor = BlockAccessor.for_block(block)
self.num_rows += accessor.num_rows()
self.size_bytes += accessor.size_bytes()


class _StatsCollectingBlocks(Iterator[Block]):
"""Block pass-through that accumulates :class:`_WriteStats` as blocks flow.

``itertools.tee`` used to give the write and the stats collection their own
view of the same blocks, but the two views are read one after the other:
the sink drains its copy first, so tee buffers the whole input for the
later reader. The stats are two integers, so count blocks on the way past
and hold none of them.
"""

__slots__ = ("_blocks", "_stats")

def __init__(self, blocks: Iterable[Block], stats: _WriteStats) -> None:
self._blocks = iter(blocks)
self._stats = stats

def __iter__(self) -> "_StatsCollectingBlocks":
return self

def __next__(self) -> Block:
block = next(self._blocks)
self._stats.add(block)
return block


def generate_write_fn(
Expand All @@ -29,17 +75,28 @@ def generate_write_fn(
def fn(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]:
"""Writes the blocks to the given datasink or legacy datasource.

Outputs the original blocks to be written."""
# Create a copy of the iterator, so we can return the original blocks.
it1, it2 = itertools.tee(blocks, 2)
Outputs no blocks: the write stats accumulate on `ctx.kwargs`, and
`generate_collect_write_stats_fn` turns them into the operator's
single output block. Returning the written blocks would mean holding
every one of them until the stats stage read them."""
stats = _WriteStats()
ctx.kwargs[WRITE_STATS_KWARG_NAME] = stats
blocks = _StatsCollectingBlocks(blocks, stats)

if isinstance(datasink_or_legacy_datasource, Datasink):
ctx.kwargs["_datasink_write_return"] = datasink_or_legacy_datasource.write(
it1, ctx
blocks, ctx
)
else:
datasink_or_legacy_datasource.write(it1, ctx, **write_args)
datasink_or_legacy_datasource.write(blocks, ctx, **write_args)

return it2
# The stats cover the whole input, not just what the sink pulled: a
# sink may stop early, and the blocks it did not read still arrived at
# this task. Drain the rest through the counter, dropping each block.
for _ in blocks:
pass

return iter(())

return fn

Expand All @@ -52,20 +109,25 @@ def generate_collect_write_stats_fn() -> BlockMapTransformFn:
def fn(blocks: Iterator[Block], ctx: TaskContext) -> Iterator[Block]:
"""Handles stats collection for block writes."""
# Drain before reading `ctx` below: consuming the input is what runs
# the write stage, and the write is what sets `_datasink_write_return`.
block_accessors = [BlockAccessor.for_block(block) for block in blocks]
total_num_rows = sum(ba.num_rows() for ba in block_accessors)
total_size_bytes = sum(ba.size_bytes() for ba in block_accessors)
# the write stage, and the write is what sets `_datasink_write_return`
# and the stats. Count the blocks as they drain, so callers that pass
# the blocks straight in (rather than driving this from
# `generate_write_fn`) still get their stats without holding them.
drained = _WriteStats()
for block in blocks:
drained.add(block)

stats = ctx.kwargs.pop(WRITE_STATS_KWARG_NAME, drained)

# NOTE: Write tasks can return anything, so we need to wrap it in a valid block
# type.
import pandas as pd

block = pd.DataFrame(
{
"num_rows": [total_num_rows],
"size_bytes": [total_size_bytes],
"write_return": [ctx.kwargs.get("_datasink_write_return", None)],
"num_rows": [stats.num_rows],
"size_bytes": [stats.size_bytes],
"write_return": [ctx.kwargs.pop("_datasink_write_return", None)],
}
)
return iter([block])
Expand Down
Loading
Loading