Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 89 additions & 24 deletions cpp/src/arrow/dataset/dataset_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,14 @@ struct DatasetWriterState {
// the dataset writer will pause and it will also close the largest open file.
Throttle open_files_throttle;
// Control for how many rows the dataset writer will allow to be staged. A row is
// staged if it is waiting for more rows to reach minimum_batch_size. If this is
// exceeded then the largest staged batch is unstaged (no backpressure is applied)
// staged if it is waiting for more rows to reach min_rows_per_group. If this is
// exceeded then the file with the most staged rows has its staged rows written out
// (no backpressure is applied)
std::atomic<uint64_t> staged_rows_count;
// If too many rows get staged we will end up with poor performance and, if more rows
// are staged than max_rows_queued we will end up with deadlock. To avoid this, once
// we have too many staged rows we just ignore min_rows_per_group
// we have too many staged rows we write out the largest group of staged rows even
// though it is smaller than min_rows_per_group
const uint64_t max_rows_staged;
// Mutex to guard access to the file visitors in the writer options
std::mutex visitors_mutex;
Expand Down Expand Up @@ -201,35 +203,42 @@ class DatasetWriterFileQueue
"DatasetWriter::WriteBatch"sv);
}

Result<int64_t> PopAndDeliverStagedBatch() {
Status PopAndDeliverStagedBatch() {
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> next_batch, PopStagedBatch());
int64_t rows_popped = next_batch->num_rows();
rows_currently_staged_ -= next_batch->num_rows();
uint64_t rows_popped = static_cast<uint64_t>(next_batch->num_rows());
rows_currently_staged_ -= rows_popped;
writer_state_->staged_rows_count -= rows_popped;
ScheduleBatch(std::move(next_batch));
return rows_popped;
return Status::OK();
}

// Stage batches, popping and delivering batches if enough data has arrived
Status Push(std::shared_ptr<RecordBatch> batch) {
uint64_t delta_staged = batch->num_rows();
rows_currently_staged_ += delta_staged;
uint64_t rows_pushed = static_cast<uint64_t>(batch->num_rows());
rows_currently_staged_ += rows_pushed;
writer_state_->staged_rows_count += rows_pushed;
staged_batches_.push_back(std::move(batch));
while (!staged_batches_.empty() &&
(writer_state_->StagingFull() ||
rows_currently_staged_ >= options_.min_rows_per_group)) {
ARROW_ASSIGN_OR_RAISE(int64_t rows_popped, PopAndDeliverStagedBatch());
delta_staged -= rows_popped;
rows_currently_staged_ >= options_.min_rows_per_group) {
RETURN_NOT_OK(PopAndDeliverStagedBatch());
}
return Status::OK();
}

uint64_t rows_staged() const { return rows_currently_staged_; }

// Deliver all staged batches even though there are fewer than min_rows_per_group
Status FlushStaged() {
while (!staged_batches_.empty()) {
RETURN_NOT_OK(PopAndDeliverStagedBatch());
}
// Note, delta_staged may be negative if we were able to deliver some data
writer_state_->staged_rows_count += delta_staged;
return Status::OK();
}

Status Finish() {
writer_state_->staged_rows_count -= rows_currently_staged_;
while (!staged_batches_.empty()) {
RETURN_NOT_OK(PopAndDeliverStagedBatch().status().OrElse(
[&](auto&&) { file_tasks_.reset(); }));
RETURN_NOT_OK(
PopAndDeliverStagedBatch().OrElse([&](auto&&) { file_tasks_.reset(); }));
}
// At this point all write tasks have been added. Because the scheduler
// is a 1-task FIFO we know this task will run at the very end and can
Expand Down Expand Up @@ -377,6 +386,14 @@ class DatasetWriterDirectoryQueue

uint64_t rows_written() const { return rows_written_; }

uint64_t rows_staged() const {
return latest_open_file_ ? latest_open_file_->rows_staged() : 0;
}

Status FlushStaged() {
return latest_open_file_ ? latest_open_file_->FlushStaged() : Status::OK();
}

void PrepareDirectory() {
if (directory_.empty() || !write_options_.create_dir) {
return;
Expand Down Expand Up @@ -517,11 +534,28 @@ Status EnsureDestinationValid(const FileSystemDatasetWriteOptions& options) {
return Status::OK();
}

// Rule of thumb for the max rows to stage. It will grow with max_rows_queued until
// max_rows_queued starts to get too large and then it caps out at 8 million rows.
// Rule of thumb for the max rows to stage. It grows with max_rows_queued so that
// raising max_rows_queued lets more files reach min_rows_per_group at the same time.
// Feel free to replace with something more meaningful, this is just a random heuristic.
uint64_t CalculateMaxRowsStaged(uint64_t max_rows_queued) {
return std::min(static_cast<uint64_t>(1 << 23), max_rows_queued / 4);
//
// A single file can only reach min_rows_per_group if that many rows can be staged so
// the limit is never smaller than min_rows_per_group.
uint64_t CalculateMaxRowsStaged(uint64_t max_rows_queued, uint64_t min_rows_per_group) {
return std::max(max_rows_queued / 4, min_rows_per_group);
}

// Staged rows count as queued rows. If the staged rows alone could reach
// max_rows_queued then the writer would pause forever so, if needed, max_rows_queued
// is raised to leave room for both the staged rows and the rows being written.
uint64_t CalculateMaxRowsQueued(uint64_t max_rows_queued, uint64_t max_rows_staged) {
if (max_rows_queued == 0) {
// Unthrottled
return max_rows_queued;
}
if (max_rows_staged > std::numeric_limits<uint64_t>::max() / 2) {
return std::numeric_limits<uint64_t>::max();
}
return std::max(max_rows_queued, 2 * max_rows_staged);
}

} // namespace
Expand All @@ -542,8 +576,11 @@ class DatasetWriter::DatasetWriterImpl {
})),
write_options_(std::move(write_options)),
writer_state_(std::make_shared<DatasetWriterState>(
max_rows_queued, write_options_.max_open_files,
CalculateMaxRowsStaged(max_rows_queued))),
CalculateMaxRowsQueued(
max_rows_queued, CalculateMaxRowsStaged(
max_rows_queued, write_options_.min_rows_per_group)),
write_options_.max_open_files,
CalculateMaxRowsStaged(max_rows_queued, write_options_.min_rows_per_group))),
pause_callback_(std::move(pause_callback)),
resume_callback_(std::move(resume_callback)) {}

Expand Down Expand Up @@ -651,6 +688,33 @@ class DatasetWriter::DatasetWriterImpl {
return largest->FinishCurrentFile();
}

// If too many rows are staged then write out the staged rows of the file that has
// the most of them, repeating until we are back under the limit. This gives the
// largest row groups we can still make.
Status EvictStagedRowsIfNeeded() {
while (writer_state_->StagingFull()) {
const std::string* largest_key = nullptr;
DatasetWriterDirectoryQueue* largest = nullptr;
uint64_t largest_rows_staged = 0;
for (const auto& dir_queue : directory_queues_) {
uint64_t rows_staged = dir_queue.second->rows_staged();
// Break ties on the key so the choice does not depend on iteration order
if (rows_staged > largest_rows_staged ||
(rows_staged > 0 && rows_staged == largest_rows_staged &&
dir_queue.first < *largest_key)) {
largest_key = &dir_queue.first;
largest = dir_queue.second.get();
largest_rows_staged = rows_staged;
}
}
if (largest == nullptr) {
break;
}
RETURN_NOT_OK(largest->FlushStaged());
}
return Status::OK();
}

Future<> DoWriteRecordBatch(std::shared_ptr<RecordBatch> batch,
const std::string& directory, const std::string& prefix) {
ARROW_ASSIGN_OR_RAISE(
Expand Down Expand Up @@ -705,6 +769,7 @@ class DatasetWriter::DatasetWriterImpl {
writer_state_->rows_in_flight_throttle.Release(next_chunk->num_rows());
return s;
}
RETURN_NOT_OK(EvictStagedRowsIfNeeded());
batch = std::move(remainder);
if (batch) {
RETURN_NOT_OK(dir_queue->FinishCurrentFile());
Expand Down
49 changes: 49 additions & 0 deletions cpp/src/arrow/dataset/dataset_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,55 @@ TEST_F(DatasetWriterTestFixture, MinRowGroupBackpressure) {
AssertCreatedData(expected_files);
}

TEST_F(DatasetWriterTestFixture, MinRowGroupLargerThanStagingLimit) {
// GH-55642: a single file should reach min_rows_per_group even when that is more
// rows than the dataset writer would otherwise be willing to stage
write_options_.min_rows_per_group = 60;
auto dataset_writer = MakeDatasetWriter(/*max_rows=*/100);
for (int i = 0; i < 12; i++) {
dataset_writer->WriteRecordBatch(MakeBatch(10), "");
}
EndWriterChecked(dataset_writer.get());
AssertCreatedData({{"testdir/chunk-0.arrow", 0, 120, 2}});
}

TEST_F(DatasetWriterTestFixture, MinRowGroupEvictsLargestStaged) {
// GH-55642: when too many rows are staged the file with the most staged rows should
// be written out, not the file that happened to receive the latest batch
write_options_.min_rows_per_group = 50;
auto dataset_writer = MakeDatasetWriter(/*max_rows=*/100);
dataset_writer->WriteRecordBatch(MakeBatch(30), "a");
for (int i = 0; i < 4; i++) {
dataset_writer->WriteRecordBatch(MakeBatch(5), "b");
}
EndWriterChecked(dataset_writer.get());
AssertCreatedData(
{{"testdir/a/chunk-0.arrow", 0, 30, 1}, {"testdir/b/chunk-0.arrow", 30, 20, 1}});
}

TEST_F(DatasetWriterTestFixture, MinRowGroupManyFilesWithRoomToStage) {
// GH-55642: every file should reach min_rows_per_group when max_rows_queued leaves
// enough room to stage min_rows_per_group rows for each of them
write_options_.min_rows_per_group = 50;
write_options_.max_rows_per_group = 50;
// 4 files * 50 rows = 200 rows staged at most, the staging limit is 1000 / 4
auto dataset_writer = MakeDatasetWriter(/*max_rows=*/1000);
for (int i = 0; i < 30; i++) {
for (int file = 0; file < 4; file++) {
dataset_writer->WriteRecordBatch(MakeBatch(0, 5), std::to_string(file));
}
}
EndWriterChecked(dataset_writer.get());
for (int file = 0; file < 4; file++) {
auto written = FindFile("testdir/" + std::to_string(file) + "/chunk-0.arrow");
ASSERT_TRUE(written.has_value());
int num_batches = 0;
auto batch = ReadAsBatch(written->data, &num_batches);
ASSERT_EQ(150, batch->num_rows());
ASSERT_EQ(3, num_batches);
}
}

TEST_F(DatasetWriterTestFixture, ConcurrentWritesSameFile) {
#ifndef ARROW_ENABLE_THREADING
GTEST_SKIP() << "Concurrent writes tests need threads";
Expand Down
15 changes: 8 additions & 7 deletions cpp/src/arrow/dataset/file_base.cc
Original file line number Diff line number Diff line change
Expand Up @@ -417,12 +417,12 @@ class DatasetWritingSinkNodeConsumer : public acero::SinkNodeConsumer {
} else {
schema_ = schema;
}
ARROW_ASSIGN_OR_RAISE(
dataset_writer_,
internal::DatasetWriter::Make(
write_options_, plan->query_context()->async_scheduler(),
[backpressure_control] { backpressure_control->Pause(); },
[backpressure_control] { backpressure_control->Resume(); }, [] {}));
ARROW_ASSIGN_OR_RAISE(dataset_writer_,
internal::DatasetWriter::Make(
write_options_, plan->query_context()->async_scheduler(),
[backpressure_control] { backpressure_control->Pause(); },
[backpressure_control] { backpressure_control->Resume(); },
[] {}, write_options_.max_rows_queued));
return Status::OK();
}

Expand Down Expand Up @@ -578,7 +578,8 @@ class TeeNode : public acero::MapNode,
dataset_writer_,
internal::DatasetWriter::Make(
write_options_, plan_->query_context()->async_scheduler(),
[this] { Pause(); }, [this] { Resume(); }, [this] { MapNode::Finish(); }));
[this] { Pause(); }, [this] { Resume(); }, [this] { MapNode::Finish(); },
write_options_.max_rows_queued));
return MapNode::StartProducing();
}

Expand Down
14 changes: 14 additions & 0 deletions cpp/src/arrow/dataset/file_base.h
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,14 @@ struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions {
/// and only write the row groups to the disk when sufficient rows have accumulated.
/// The final row group size may be less than this value and other options such as
/// `max_open_files` or `max_rows_per_file` lead to smaller row group sizes.
///
/// Rows waiting for a row group to fill are held in memory. The dataset writer
/// limits how many rows it holds this way, across all files being written, to the
/// larger of `max_rows_queued` / 4 and this value. When the limit is reached the file
/// holding the most waiting rows has them written out as a smaller row group. A
/// single file therefore always reaches this value, but a partitioned write may not
/// unless `max_rows_queued` is at least 4 times this value multiplied by the number
/// of partitions being written at the same time.
uint64_t min_rows_per_group = 0;

/// If greater than 0 then the dataset writer may split up large incoming batches into
Expand All @@ -442,6 +450,12 @@ struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions {
/// group size is just barely larger than this value).
uint64_t max_rows_per_group = 1 << 20;

/// The maximum number of rows the dataset writer will hold in memory, either waiting
/// to form a row group or waiting to be written to disk, before it applies
/// backpressure. If 0 then there is no limit. This is raised automatically, if
/// needed, to twice `min_rows_per_group`.
uint64_t max_rows_queued = 8 * 1024 * 1024;

/// Controls what happens if an output directory already exists.
ExistingDataBehavior existing_data_behavior = ExistingDataBehavior::kError;

Expand Down
10 changes: 10 additions & 0 deletions docs/source/python/dataset.rst
Original file line number Diff line number Diff line change
Expand Up @@ -758,6 +758,16 @@ defined with the ``min_rows_per_group`` parameter of :meth:`write_dataset`.
less than this value if other options such as ``max_open_files`` or
``max_rows_per_file`` force smaller row group sizes.

Rows waiting for a row group to fill are held in memory. The writer limits
how many rows it holds this way, across all files being written, to the
larger of ``max_rows_queued / 4`` and ``min_rows_per_group``. When the
limit is reached, the file holding the most waiting rows has them written
out as a smaller row group. A single file therefore always reaches
``min_rows_per_group``, but a partitioned write may not unless the
``max_rows_queued`` parameter of :meth:`write_dataset` is at least 4 times
``min_rows_per_group`` multiplied by the number of partitions being written
at the same time. Raising ``max_rows_queued`` uses more memory.

The maximum number of rows allowed per group is defined with the
``max_rows_per_group`` parameter of :meth:`write_dataset`.

Expand Down
5 changes: 4 additions & 1 deletion python/pyarrow/_dataset.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -4144,7 +4144,8 @@ def _filesystemdataset_write(
int max_rows_per_file,
int min_rows_per_group,
int max_rows_per_group,
bool create_dir
bool create_dir,
max_rows_queued=None
):
"""
CFileSystemDataset.Write wrapper
Expand All @@ -4164,6 +4165,8 @@ def _filesystemdataset_write(
c_options.max_rows_per_file = max_rows_per_file
c_options.max_rows_per_group = max_rows_per_group
c_options.min_rows_per_group = min_rows_per_group
if max_rows_queued is not None:
c_options.max_rows_queued = max_rows_queued
c_options.basename_template = tobytes(basename_template)
if existing_data_behavior == 'error':
c_options.existing_data_behavior = ExistingDataBehavior_ERROR
Expand Down
21 changes: 19 additions & 2 deletions python/pyarrow/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,8 @@ def write_dataset(data, base_dir, *, basename_template=None, format=None,
preserve_order=False, max_partitions=None, max_open_files=None,
max_rows_per_file=None, min_rows_per_group=None,
max_rows_per_group=None, file_visitor=None,
existing_data_behavior='error', create_dir=True):
existing_data_behavior='error', create_dir=True,
max_rows_queued=None):
"""
Write a dataset to a given format and partitioning.

Expand Down Expand Up @@ -913,6 +914,15 @@ def write_dataset(data, base_dir, *, basename_template=None, format=None,
Minimum number of rows per group. When the value is greater than 0,
the dataset writer will batch incoming data and only write the row
groups to the disk when sufficient rows have accumulated.
Rows waiting for a row group to fill are held in memory, and the
writer limits how many rows it holds this way, across all files
being written, to the larger of ``max_rows_queued / 4`` and this
value. When the limit is reached, the file holding the most waiting
rows has them written out as a smaller row group. A single file
therefore always reaches this value, but a partitioned write may
not unless ``max_rows_queued`` is at least 4 times this value
multiplied by the number of partitions being written at the same
time.
max_rows_per_group : int, default 1024 * 1024
Maximum number of rows per group. If the value is greater than 0,
then the dataset writer may split up large incoming batches into
Expand Down Expand Up @@ -957,6 +967,13 @@ def file_visitor(written_file):
create_dir : bool, default True
If False, directories will not be created. This can be useful for
filesystems that do not require directories.
max_rows_queued : int, default 8 * 1024 * 1024
The maximum number of rows the dataset writer will hold in memory,
either waiting to form a row group or waiting to be written to
disk, before it applies backpressure. If 0 there is no limit.
Raise this to let a partitioned write reach min_rows_per_group in
more partitions at the same time, at the cost of memory. It is
raised automatically, if needed, to twice min_rows_per_group.
"""
from pyarrow.fs import _resolve_filesystem_and_path

Expand Down Expand Up @@ -1036,5 +1053,5 @@ def file_visitor(written_file):
scanner, base_dir, basename_template, filesystem, partitioning,
preserve_order, file_options, max_partitions, file_visitor,
existing_data_behavior, max_open_files, max_rows_per_file,
min_rows_per_group, max_rows_per_group, create_dir
min_rows_per_group, max_rows_per_group, create_dir, max_rows_queued
)
1 change: 1 addition & 0 deletions python/pyarrow/includes/libarrow_dataset.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
uint64_t max_rows_per_file
uint64_t min_rows_per_group
uint64_t max_rows_per_group
uint64_t max_rows_queued

cdef cppclass CFileSystemDataset \
"arrow::dataset::FileSystemDataset"(CDataset):
Expand Down
Loading
Loading