From 827ccfaf33ddf6dcaf4d43d85f18061d8d3f6634 Mon Sep 17 00:00:00 2001 From: tolleybot Date: Tue, 17 Nov 2026 09:15:34 -0800 Subject: [PATCH 1/4] GH-55642: [C++][Python][Docs] Document that min_rows_per_group is best effort The dataset writer stops staging rows once 2,097,152 rows are held back across all files being written, after which row groups smaller than min_rows_per_group are written. Describe this in the Python docstring, the C++ option documentation and the Python user guide. --- cpp/src/arrow/dataset/file_base.h | 7 +++++++ docs/source/python/dataset.rst | 8 ++++++++ python/pyarrow/dataset.py | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h index c628d76f36f4..64a2901c8354 100644 --- a/cpp/src/arrow/dataset/file_base.h +++ b/cpp/src/arrow/dataset/file_base.h @@ -434,6 +434,13 @@ struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions { /// and only write the row groups to the disk when sufficient rows have accumulated. /// The final row group size may be less than this value and other options such as /// `max_open_files` or `max_rows_per_file` lead to smaller row group sizes. + /// + /// This is best effort. The writer holds back at most 2,097,152 rows in total, + /// across all files being written, while waiting for row groups to fill. Once that + /// many rows are held back, smaller row groups are written. Values above this limit + /// are therefore not met, and a partitioned write can produce row groups below this + /// value when min_rows_per_group multiplied by the number of partitions being written + /// at the same time exceeds the limit. uint64_t min_rows_per_group = 0; /// If greater than 0 then the dataset writer may split up large incoming batches into diff --git a/docs/source/python/dataset.rst b/docs/source/python/dataset.rst index 0c51458c8e64..f491e2637c57 100644 --- a/docs/source/python/dataset.rst +++ b/docs/source/python/dataset.rst @@ -758,6 +758,14 @@ defined with the ``min_rows_per_group`` parameter of :meth:`write_dataset`. less than this value if other options such as ``max_open_files`` or ``max_rows_per_file`` force smaller row group sizes. + ``min_rows_per_group`` is best effort. The writer holds back at most + 2,097,152 rows in total, across all files being written, while waiting for + row groups to fill. Once that many rows are held back, smaller row groups + are written. Values above this limit are therefore not met, and a + partitioned write can produce row groups below ``min_rows_per_group`` when + ``min_rows_per_group`` multiplied by the number of partitions being written + at the same time exceeds the limit. + The maximum number of rows allowed per group is defined with the ``max_rows_per_group`` parameter of :meth:`write_dataset`. diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index f696d5f03ba5..a865b71a35e3 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -913,6 +913,13 @@ def write_dataset(data, base_dir, *, basename_template=None, format=None, Minimum number of rows per group. When the value is greater than 0, the dataset writer will batch incoming data and only write the row groups to the disk when sufficient rows have accumulated. + This is best effort. The writer holds back at most 2,097,152 rows + in total, across all files being written, while waiting for row + groups to fill. Once that many rows are held back, smaller row + groups are written. Values above this limit are therefore not + met, and a partitioned write can produce row groups below this + value when min_rows_per_group multiplied by the number of + partitions being written at the same time exceeds the limit. max_rows_per_group : int, default 1024 * 1024 Maximum number of rows per group. If the value is greater than 0, then the dataset writer may split up large incoming batches into From 5ca71b05da9eb85b7c10901cdfb6a6edcb23df75 Mon Sep 17 00:00:00 2001 From: tolleybot Date: Tue, 17 Nov 2026 09:51:24 -0800 Subject: [PATCH 2/4] GH-55642: [C++][Dataset] Honor min_rows_per_group for a single file and evict the largest staged file When more rows are staged than the dataset writer allows, write out the staged rows of the file that holds the most of them instead of the file that received the latest batch, which could produce very small row groups. Never let the staging limit be smaller than min_rows_per_group, raising the limit on queued rows when needed, so that a single file always reaches min_rows_per_group. --- cpp/src/arrow/dataset/dataset_writer.cc | 111 +++++++++++++++---- cpp/src/arrow/dataset/dataset_writer_test.cc | 26 +++++ 2 files changed, 115 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/dataset/dataset_writer.cc b/cpp/src/arrow/dataset/dataset_writer.cc index f5104efb70b1..ff38032a8f9d 100644 --- a/cpp/src/arrow/dataset/dataset_writer.cc +++ b/cpp/src/arrow/dataset/dataset_writer.cc @@ -111,12 +111,14 @@ struct DatasetWriterState { // the dataset writer will pause and it will also close the largest open file. Throttle open_files_throttle; // Control for how many rows the dataset writer will allow to be staged. A row is - // staged if it is waiting for more rows to reach minimum_batch_size. If this is - // exceeded then the largest staged batch is unstaged (no backpressure is applied) + // staged if it is waiting for more rows to reach min_rows_per_group. If this is + // exceeded then the file with the most staged rows has its staged rows written out + // (no backpressure is applied) std::atomic staged_rows_count; // If too many rows get staged we will end up with poor performance and, if more rows // are staged than max_rows_queued we will end up with deadlock. To avoid this, once - // we have too many staged rows we just ignore min_rows_per_group + // we have too many staged rows we write out the largest group of staged rows even + // though it is smaller than min_rows_per_group const uint64_t max_rows_staged; // Mutex to guard access to the file visitors in the writer options std::mutex visitors_mutex; @@ -201,35 +203,42 @@ class DatasetWriterFileQueue "DatasetWriter::WriteBatch"sv); } - Result PopAndDeliverStagedBatch() { + Status PopAndDeliverStagedBatch() { ARROW_ASSIGN_OR_RAISE(std::shared_ptr next_batch, PopStagedBatch()); - int64_t rows_popped = next_batch->num_rows(); - rows_currently_staged_ -= next_batch->num_rows(); + uint64_t rows_popped = static_cast(next_batch->num_rows()); + rows_currently_staged_ -= rows_popped; + writer_state_->staged_rows_count -= rows_popped; ScheduleBatch(std::move(next_batch)); - return rows_popped; + return Status::OK(); } // Stage batches, popping and delivering batches if enough data has arrived Status Push(std::shared_ptr batch) { - uint64_t delta_staged = batch->num_rows(); - rows_currently_staged_ += delta_staged; + uint64_t rows_pushed = static_cast(batch->num_rows()); + rows_currently_staged_ += rows_pushed; + writer_state_->staged_rows_count += rows_pushed; staged_batches_.push_back(std::move(batch)); while (!staged_batches_.empty() && - (writer_state_->StagingFull() || - rows_currently_staged_ >= options_.min_rows_per_group)) { - ARROW_ASSIGN_OR_RAISE(int64_t rows_popped, PopAndDeliverStagedBatch()); - delta_staged -= rows_popped; + rows_currently_staged_ >= options_.min_rows_per_group) { + RETURN_NOT_OK(PopAndDeliverStagedBatch()); + } + return Status::OK(); + } + + uint64_t rows_staged() const { return rows_currently_staged_; } + + // Deliver all staged batches even though there are fewer than min_rows_per_group + Status FlushStaged() { + while (!staged_batches_.empty()) { + RETURN_NOT_OK(PopAndDeliverStagedBatch()); } - // Note, delta_staged may be negative if we were able to deliver some data - writer_state_->staged_rows_count += delta_staged; return Status::OK(); } Status Finish() { - writer_state_->staged_rows_count -= rows_currently_staged_; while (!staged_batches_.empty()) { - RETURN_NOT_OK(PopAndDeliverStagedBatch().status().OrElse( - [&](auto&&) { file_tasks_.reset(); })); + RETURN_NOT_OK( + PopAndDeliverStagedBatch().OrElse([&](auto&&) { file_tasks_.reset(); })); } // At this point all write tasks have been added. Because the scheduler // is a 1-task FIFO we know this task will run at the very end and can @@ -377,6 +386,14 @@ class DatasetWriterDirectoryQueue uint64_t rows_written() const { return rows_written_; } + uint64_t rows_staged() const { + return latest_open_file_ ? latest_open_file_->rows_staged() : 0; + } + + Status FlushStaged() { + return latest_open_file_ ? latest_open_file_->FlushStaged() : Status::OK(); + } + void PrepareDirectory() { if (directory_.empty() || !write_options_.create_dir) { return; @@ -520,8 +537,26 @@ Status EnsureDestinationValid(const FileSystemDatasetWriteOptions& options) { // Rule of thumb for the max rows to stage. It will grow with max_rows_queued until // max_rows_queued starts to get too large and then it caps out at 8 million rows. // Feel free to replace with something more meaningful, this is just a random heuristic. -uint64_t CalculateMaxRowsStaged(uint64_t max_rows_queued) { - return std::min(static_cast(1 << 23), max_rows_queued / 4); +// +// A single file can only reach min_rows_per_group if that many rows can be staged so +// the limit is never smaller than min_rows_per_group. +uint64_t CalculateMaxRowsStaged(uint64_t max_rows_queued, uint64_t min_rows_per_group) { + return std::max(std::min(static_cast(1 << 23), max_rows_queued / 4), + min_rows_per_group); +} + +// Staged rows count as queued rows. If the staged rows alone could reach +// max_rows_queued then the writer would pause forever so, if needed, max_rows_queued +// is raised to leave room for both the staged rows and the rows being written. +uint64_t CalculateMaxRowsQueued(uint64_t max_rows_queued, uint64_t max_rows_staged) { + if (max_rows_queued == 0) { + // Unthrottled + return max_rows_queued; + } + if (max_rows_staged > std::numeric_limits::max() / 2) { + return std::numeric_limits::max(); + } + return std::max(max_rows_queued, 2 * max_rows_staged); } } // namespace @@ -542,8 +577,12 @@ class DatasetWriter::DatasetWriterImpl { })), write_options_(std::move(write_options)), writer_state_(std::make_shared( - max_rows_queued, write_options_.max_open_files, - CalculateMaxRowsStaged(max_rows_queued))), + CalculateMaxRowsQueued( + max_rows_queued, + CalculateMaxRowsStaged(max_rows_queued, + write_options_.min_rows_per_group)), + write_options_.max_open_files, + CalculateMaxRowsStaged(max_rows_queued, write_options_.min_rows_per_group))), pause_callback_(std::move(pause_callback)), resume_callback_(std::move(resume_callback)) {} @@ -651,6 +690,33 @@ class DatasetWriter::DatasetWriterImpl { return largest->FinishCurrentFile(); } + // If too many rows are staged then write out the staged rows of the file that has + // the most of them, repeating until we are back under the limit. This gives the + // largest row groups we can still make. + Status EvictStagedRowsIfNeeded() { + while (writer_state_->StagingFull()) { + const std::string* largest_key = nullptr; + DatasetWriterDirectoryQueue* largest = nullptr; + uint64_t largest_rows_staged = 0; + for (const auto& dir_queue : directory_queues_) { + uint64_t rows_staged = dir_queue.second->rows_staged(); + // Break ties on the key so the choice does not depend on iteration order + if (rows_staged > largest_rows_staged || + (rows_staged > 0 && rows_staged == largest_rows_staged && + dir_queue.first < *largest_key)) { + largest_key = &dir_queue.first; + largest = dir_queue.second.get(); + largest_rows_staged = rows_staged; + } + } + if (largest == nullptr) { + break; + } + RETURN_NOT_OK(largest->FlushStaged()); + } + return Status::OK(); + } + Future<> DoWriteRecordBatch(std::shared_ptr batch, const std::string& directory, const std::string& prefix) { ARROW_ASSIGN_OR_RAISE( @@ -705,6 +771,7 @@ class DatasetWriter::DatasetWriterImpl { writer_state_->rows_in_flight_throttle.Release(next_chunk->num_rows()); return s; } + RETURN_NOT_OK(EvictStagedRowsIfNeeded()); batch = std::move(remainder); if (batch) { RETURN_NOT_OK(dir_queue->FinishCurrentFile()); diff --git a/cpp/src/arrow/dataset/dataset_writer_test.cc b/cpp/src/arrow/dataset/dataset_writer_test.cc index 45a36deccea6..ad8df5a426eb 100644 --- a/cpp/src/arrow/dataset/dataset_writer_test.cc +++ b/cpp/src/arrow/dataset/dataset_writer_test.cc @@ -466,6 +466,32 @@ TEST_F(DatasetWriterTestFixture, MinRowGroupBackpressure) { AssertCreatedData(expected_files); } +TEST_F(DatasetWriterTestFixture, MinRowGroupLargerThanStagingLimit) { + // GH-55642: a single file should reach min_rows_per_group even when that is more + // rows than the dataset writer would otherwise be willing to stage + write_options_.min_rows_per_group = 60; + auto dataset_writer = MakeDatasetWriter(/*max_rows=*/100); + for (int i = 0; i < 12; i++) { + dataset_writer->WriteRecordBatch(MakeBatch(10), ""); + } + EndWriterChecked(dataset_writer.get()); + AssertCreatedData({{"testdir/chunk-0.arrow", 0, 120, 2}}); +} + +TEST_F(DatasetWriterTestFixture, MinRowGroupEvictsLargestStaged) { + // GH-55642: when too many rows are staged the file with the most staged rows should + // be written out, not the file that happened to receive the latest batch + write_options_.min_rows_per_group = 50; + auto dataset_writer = MakeDatasetWriter(/*max_rows=*/100); + dataset_writer->WriteRecordBatch(MakeBatch(30), "a"); + for (int i = 0; i < 4; i++) { + dataset_writer->WriteRecordBatch(MakeBatch(5), "b"); + } + EndWriterChecked(dataset_writer.get()); + AssertCreatedData( + {{"testdir/a/chunk-0.arrow", 0, 30, 1}, {"testdir/b/chunk-0.arrow", 30, 20, 1}}); +} + TEST_F(DatasetWriterTestFixture, ConcurrentWritesSameFile) { #ifndef ARROW_ENABLE_THREADING GTEST_SKIP() << "Concurrent writes tests need threads"; From 6d20e217e704d6a342c6759e765f6e571835b418 Mon Sep 17 00:00:00 2001 From: tolleybot Date: Tue, 17 Nov 2026 10:07:47 -0800 Subject: [PATCH 3/4] GH-55642: [C++][Python][Dataset] Add max_rows_queued to the dataset write options Expose the limit on rows held in memory by the dataset writer as FileSystemDatasetWriteOptions::max_rows_queued and as the max_rows_queued argument of pyarrow.dataset.write_dataset. The limit on staged rows now grows with it without a fixed ceiling, so a partitioned write can reach min_rows_per_group in every partition when given enough room. --- cpp/src/arrow/dataset/dataset_writer.cc | 7 ++-- cpp/src/arrow/dataset/dataset_writer_test.cc | 23 +++++++++++ cpp/src/arrow/dataset/file_base.cc | 6 ++- cpp/src/arrow/dataset/file_base.h | 14 +++++++ python/pyarrow/_dataset.pyx | 5 ++- python/pyarrow/dataset.py | 21 +++++++++- python/pyarrow/includes/libarrow_dataset.pxd | 1 + python/pyarrow/tests/test_dataset.py | 42 ++++++++++++++++++++ 8 files changed, 110 insertions(+), 9 deletions(-) diff --git a/cpp/src/arrow/dataset/dataset_writer.cc b/cpp/src/arrow/dataset/dataset_writer.cc index ff38032a8f9d..fe9987f3e790 100644 --- a/cpp/src/arrow/dataset/dataset_writer.cc +++ b/cpp/src/arrow/dataset/dataset_writer.cc @@ -534,15 +534,14 @@ Status EnsureDestinationValid(const FileSystemDatasetWriteOptions& options) { return Status::OK(); } -// Rule of thumb for the max rows to stage. It will grow with max_rows_queued until -// max_rows_queued starts to get too large and then it caps out at 8 million rows. +// Rule of thumb for the max rows to stage. It grows with max_rows_queued so that +// raising max_rows_queued lets more files reach min_rows_per_group at the same time. // Feel free to replace with something more meaningful, this is just a random heuristic. // // A single file can only reach min_rows_per_group if that many rows can be staged so // the limit is never smaller than min_rows_per_group. uint64_t CalculateMaxRowsStaged(uint64_t max_rows_queued, uint64_t min_rows_per_group) { - return std::max(std::min(static_cast(1 << 23), max_rows_queued / 4), - min_rows_per_group); + return std::max(max_rows_queued / 4, min_rows_per_group); } // Staged rows count as queued rows. If the staged rows alone could reach diff --git a/cpp/src/arrow/dataset/dataset_writer_test.cc b/cpp/src/arrow/dataset/dataset_writer_test.cc index ad8df5a426eb..96069cc504f7 100644 --- a/cpp/src/arrow/dataset/dataset_writer_test.cc +++ b/cpp/src/arrow/dataset/dataset_writer_test.cc @@ -492,6 +492,29 @@ TEST_F(DatasetWriterTestFixture, MinRowGroupEvictsLargestStaged) { {{"testdir/a/chunk-0.arrow", 0, 30, 1}, {"testdir/b/chunk-0.arrow", 30, 20, 1}}); } +TEST_F(DatasetWriterTestFixture, MinRowGroupManyFilesWithRoomToStage) { + // GH-55642: every file should reach min_rows_per_group when max_rows_queued leaves + // enough room to stage min_rows_per_group rows for each of them + write_options_.min_rows_per_group = 50; + write_options_.max_rows_per_group = 50; + // 4 files * 50 rows = 200 rows staged at most, the staging limit is 1000 / 4 + auto dataset_writer = MakeDatasetWriter(/*max_rows=*/1000); + for (int i = 0; i < 30; i++) { + for (int file = 0; file < 4; file++) { + dataset_writer->WriteRecordBatch(MakeBatch(0, 5), std::to_string(file)); + } + } + EndWriterChecked(dataset_writer.get()); + for (int file = 0; file < 4; file++) { + auto written = FindFile("testdir/" + std::to_string(file) + "/chunk-0.arrow"); + ASSERT_TRUE(written.has_value()); + int num_batches = 0; + auto batch = ReadAsBatch(written->data, &num_batches); + ASSERT_EQ(150, batch->num_rows()); + ASSERT_EQ(3, num_batches); + } +} + TEST_F(DatasetWriterTestFixture, ConcurrentWritesSameFile) { #ifndef ARROW_ENABLE_THREADING GTEST_SKIP() << "Concurrent writes tests need threads"; diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc index 8ad2254aa64b..3c1e887367be 100644 --- a/cpp/src/arrow/dataset/file_base.cc +++ b/cpp/src/arrow/dataset/file_base.cc @@ -422,7 +422,8 @@ class DatasetWritingSinkNodeConsumer : public acero::SinkNodeConsumer { internal::DatasetWriter::Make( write_options_, plan->query_context()->async_scheduler(), [backpressure_control] { backpressure_control->Pause(); }, - [backpressure_control] { backpressure_control->Resume(); }, [] {})); + [backpressure_control] { backpressure_control->Resume(); }, [] {}, + write_options_.max_rows_queued)); return Status::OK(); } @@ -578,7 +579,8 @@ class TeeNode : public acero::MapNode, dataset_writer_, internal::DatasetWriter::Make( write_options_, plan_->query_context()->async_scheduler(), - [this] { Pause(); }, [this] { Resume(); }, [this] { MapNode::Finish(); })); + [this] { Pause(); }, [this] { Resume(); }, [this] { MapNode::Finish(); }, + write_options_.max_rows_queued)); return MapNode::StartProducing(); } diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h index c628d76f36f4..1e23c8a3db6a 100644 --- a/cpp/src/arrow/dataset/file_base.h +++ b/cpp/src/arrow/dataset/file_base.h @@ -434,6 +434,14 @@ struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions { /// and only write the row groups to the disk when sufficient rows have accumulated. /// The final row group size may be less than this value and other options such as /// `max_open_files` or `max_rows_per_file` lead to smaller row group sizes. + /// + /// Rows waiting for a row group to fill are held in memory. The dataset writer + /// limits how many rows it holds this way, across all files being written, to the + /// larger of `max_rows_queued` / 4 and this value. When the limit is reached the file + /// holding the most waiting rows has them written out as a smaller row group. A + /// single file therefore always reaches this value, but a partitioned write may not + /// unless `max_rows_queued` is at least 4 times this value multiplied by the number + /// of partitions being written at the same time. uint64_t min_rows_per_group = 0; /// If greater than 0 then the dataset writer may split up large incoming batches into @@ -442,6 +450,12 @@ struct ARROW_DS_EXPORT FileSystemDatasetWriteOptions { /// group size is just barely larger than this value). uint64_t max_rows_per_group = 1 << 20; + /// The maximum number of rows the dataset writer will hold in memory, either waiting + /// to form a row group or waiting to be written to disk, before it applies + /// backpressure. If 0 then there is no limit. This is raised automatically, if + /// needed, to twice `min_rows_per_group`. + uint64_t max_rows_queued = 8 * 1024 * 1024; + /// Controls what happens if an output directory already exists. ExistingDataBehavior existing_data_behavior = ExistingDataBehavior::kError; diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 4820027ee821..9f0a79e56903 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -4144,7 +4144,8 @@ def _filesystemdataset_write( int max_rows_per_file, int min_rows_per_group, int max_rows_per_group, - bool create_dir + bool create_dir, + max_rows_queued=None ): """ CFileSystemDataset.Write wrapper @@ -4164,6 +4165,8 @@ def _filesystemdataset_write( c_options.max_rows_per_file = max_rows_per_file c_options.max_rows_per_group = max_rows_per_group c_options.min_rows_per_group = min_rows_per_group + if max_rows_queued is not None: + c_options.max_rows_queued = max_rows_queued c_options.basename_template = tobytes(basename_template) if existing_data_behavior == 'error': c_options.existing_data_behavior = ExistingDataBehavior_ERROR diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index f696d5f03ba5..e198790b1fc0 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -849,7 +849,8 @@ def write_dataset(data, base_dir, *, basename_template=None, format=None, preserve_order=False, max_partitions=None, max_open_files=None, max_rows_per_file=None, min_rows_per_group=None, max_rows_per_group=None, file_visitor=None, - existing_data_behavior='error', create_dir=True): + existing_data_behavior='error', create_dir=True, + max_rows_queued=None): """ Write a dataset to a given format and partitioning. @@ -913,6 +914,15 @@ def write_dataset(data, base_dir, *, basename_template=None, format=None, Minimum number of rows per group. When the value is greater than 0, the dataset writer will batch incoming data and only write the row groups to the disk when sufficient rows have accumulated. + Rows waiting for a row group to fill are held in memory, and the + writer limits how many rows it holds this way, across all files + being written, to the larger of ``max_rows_queued / 4`` and this + value. When the limit is reached, the file holding the most waiting + rows has them written out as a smaller row group. A single file + therefore always reaches this value, but a partitioned write may + not unless ``max_rows_queued`` is at least 4 times this value + multiplied by the number of partitions being written at the same + time. max_rows_per_group : int, default 1024 * 1024 Maximum number of rows per group. If the value is greater than 0, then the dataset writer may split up large incoming batches into @@ -957,6 +967,13 @@ def file_visitor(written_file): create_dir : bool, default True If False, directories will not be created. This can be useful for filesystems that do not require directories. + max_rows_queued : int, default 8 * 1024 * 1024 + The maximum number of rows the dataset writer will hold in memory, + either waiting to form a row group or waiting to be written to + disk, before it applies backpressure. If 0 there is no limit. + Raise this to let a partitioned write reach min_rows_per_group in + more partitions at the same time, at the cost of memory. It is + raised automatically, if needed, to twice min_rows_per_group. """ from pyarrow.fs import _resolve_filesystem_and_path @@ -1036,5 +1053,5 @@ def file_visitor(written_file): scanner, base_dir, basename_template, filesystem, partitioning, preserve_order, file_options, max_partitions, file_visitor, existing_data_behavior, max_open_files, max_rows_per_file, - min_rows_per_group, max_rows_per_group, create_dir + min_rows_per_group, max_rows_per_group, create_dir, max_rows_queued ) diff --git a/python/pyarrow/includes/libarrow_dataset.pxd b/python/pyarrow/includes/libarrow_dataset.pxd index 12cf82733935..eecd29419c10 100644 --- a/python/pyarrow/includes/libarrow_dataset.pxd +++ b/python/pyarrow/includes/libarrow_dataset.pxd @@ -234,6 +234,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil: uint64_t max_rows_per_file uint64_t min_rows_per_group uint64_t max_rows_per_group + uint64_t max_rows_queued cdef cppclass CFileSystemDataset \ "arrow::dataset::FileSystemDataset"(CDataset): diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index fd5a418aaf2d..dd9d8af791ce 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4589,6 +4589,48 @@ def test_write_dataset_min_rows_per_group(tempdir): @pytest.mark.parquet +def _row_group_sizes_per_file(base_dir): + return { + os.path.relpath(path, base_dir): [ + md.row_group(i).num_rows for i in range(md.num_row_groups)] + for path in sorted(ds.dataset(base_dir, format="parquet").files) + for md in [pq.ParquetFile(path).metadata] + } + + +@pytest.mark.parquet +def test_write_dataset_min_rows_per_group_single_file(tempdir): + # GH-55642: a single file reaches min_rows_per_group even when that is + # more rows than the writer would otherwise hold back + table = pa.table({"x": pa.array(range(100_000), type=pa.int32())}) + ds.write_dataset(table.to_batches(max_chunksize=1_000), tempdir / "ds", + schema=table.schema, format="parquet", + min_rows_per_group=30_000, max_rows_per_group=30_000, + max_rows_queued=40_000) + assert _row_group_sizes_per_file(tempdir / "ds") == { + "part-0.parquet": [30_000, 30_000, 30_000, 10_000]} + + +@pytest.mark.parquet +def test_write_dataset_min_rows_per_group_partitioned(tempdir): + # GH-55642: every partition reaches min_rows_per_group when + # max_rows_queued leaves room to hold that many rows for each of them + num_rows, num_parts, rows_per_group = 120_000, 4, 10_000 + table = pa.table({ + "x": pa.array(range(num_rows), type=pa.int32()), + "p": pa.array([i % num_parts for i in range(num_rows)], type=pa.int32()), + }) + ds.write_dataset(table.to_batches(max_chunksize=1_000), tempdir / "ds", + schema=table.schema, format="parquet", + partitioning=["p"], partitioning_flavor="hive", + min_rows_per_group=rows_per_group, + max_rows_per_group=rows_per_group, + max_rows_queued=4 * num_parts * rows_per_group, + use_threads=False) + assert _row_group_sizes_per_file(tempdir / "ds") == { + f"p={p}/part-0.parquet": [rows_per_group] * 3 for p in range(num_parts)} + + def test_write_dataset_max_rows_per_group(tempdir): directory = tempdir / 'ds' max_rows_per_group = 18 From 750fc44be94b284fa8f8e705018377cea5045efe Mon Sep 17 00:00:00 2001 From: tolleybot Date: Tue, 17 Nov 2026 10:13:16 -0800 Subject: [PATCH 4/4] GH-55642: [C++] Apply clang-format --- cpp/src/arrow/dataset/dataset_writer.cc | 5 ++--- cpp/src/arrow/dataset/file_base.cc | 13 ++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/dataset/dataset_writer.cc b/cpp/src/arrow/dataset/dataset_writer.cc index fe9987f3e790..cd69629ab4d6 100644 --- a/cpp/src/arrow/dataset/dataset_writer.cc +++ b/cpp/src/arrow/dataset/dataset_writer.cc @@ -577,9 +577,8 @@ class DatasetWriter::DatasetWriterImpl { write_options_(std::move(write_options)), writer_state_(std::make_shared( CalculateMaxRowsQueued( - max_rows_queued, - CalculateMaxRowsStaged(max_rows_queued, - write_options_.min_rows_per_group)), + max_rows_queued, CalculateMaxRowsStaged( + max_rows_queued, write_options_.min_rows_per_group)), write_options_.max_open_files, CalculateMaxRowsStaged(max_rows_queued, write_options_.min_rows_per_group))), pause_callback_(std::move(pause_callback)), diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc index 3c1e887367be..c4cf8c39888b 100644 --- a/cpp/src/arrow/dataset/file_base.cc +++ b/cpp/src/arrow/dataset/file_base.cc @@ -417,13 +417,12 @@ class DatasetWritingSinkNodeConsumer : public acero::SinkNodeConsumer { } else { schema_ = schema; } - ARROW_ASSIGN_OR_RAISE( - dataset_writer_, - internal::DatasetWriter::Make( - write_options_, plan->query_context()->async_scheduler(), - [backpressure_control] { backpressure_control->Pause(); }, - [backpressure_control] { backpressure_control->Resume(); }, [] {}, - write_options_.max_rows_queued)); + ARROW_ASSIGN_OR_RAISE(dataset_writer_, + internal::DatasetWriter::Make( + write_options_, plan->query_context()->async_scheduler(), + [backpressure_control] { backpressure_control->Pause(); }, + [backpressure_control] { backpressure_control->Resume(); }, + [] {}, write_options_.max_rows_queued)); return Status::OK(); }