From 25cc647bc63abd966396712c66824eaf95eaef8e Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Mon, 16 Nov 2026 20:38:19 +0100 Subject: [PATCH 1/3] GH-55638: [C++][Parquet] Keep content-defined chunk boundaries of list columns at array chunk starts The nested branch of the chunker skipped the empty chunk at offset 0 of a WriteArrow() call, which dropped the boundary instead of closing the previous page. --- cpp/src/parquet/chunker_internal.cc | 12 +++++------- cpp/src/parquet/chunker_internal_test.cc | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cpp/src/parquet/chunker_internal.cc b/cpp/src/parquet/chunker_internal.cc index aa0a2217b831..02a7a3afbfe8 100644 --- a/cpp/src/parquet/chunker_internal.cc +++ b/cpp/src/parquet/chunker_internal.cc @@ -326,13 +326,11 @@ class ContentDefinedChunker::Impl { } if (rep_level == 0 && gearhash.NeedNewChunk()) { - // if we are at a record boundary and need a new chunk, we create a new chunk - auto levels_to_write = offset - prev_offset; - if (levels_to_write > 0) { - chunks.push_back({prev_offset, prev_value_offset, levels_to_write}); - prev_offset = offset; - prev_value_offset = value_offset; - } + // if we are at a record boundary and need a new chunk, we create a new chunk, + // at offset 0 it is empty and closes the page of the previous call + chunks.push_back({prev_offset, prev_value_offset, offset - prev_offset}); + prev_offset = offset; + prev_value_offset = value_offset; } if (def_level >= level_info_.repeated_ancestor_def_level) { // we only increment the value offset if we have a leaf value diff --git a/cpp/src/parquet/chunker_internal_test.cc b/cpp/src/parquet/chunker_internal_test.cc index 7b4c9bb7cce1..a87815f17c9b 100644 --- a/cpp/src/parquet/chunker_internal_test.cc +++ b/cpp/src/parquet/chunker_internal_test.cc @@ -396,6 +396,7 @@ ParquetInfo GetColumnParquetInfo(const std::shared_ptr& data, int column while (auto page = page_reader->NextPage()) { if (page->type() == PageType::DATA_PAGE || page->type() == PageType::DATA_PAGE_V2) { auto data_page = static_cast(page.get()); + EXPECT_GT(data_page->num_values(), 0) << "empty data page"; column_info.page_sizes.push_back(data_page->uncompressed_size()); column_info.page_lengths.push_back(data_page->num_values()); } else if (page->type() == PageType::DICTIONARY_PAGE) { @@ -1468,6 +1469,25 @@ TEST_P(TestCDCSingleRowGroup, ArrayOffsets) { } } +TEST_P(TestCDCSingleRowGroup, IndependentOfArrayChunks) { + // Each array chunk is written by a separate WriteArrow() call + const auto& param = GetParam(); + ASSERT_OK_AND_ASSIGN(auto chunked, ConcatenateTables({part1_, part2_, part3_})); + ASSERT_OK_AND_ASSIGN(auto combined, chunked->CombineChunks()); + ASSERT_EQ(chunked->column(0)->num_chunks(), 3); + ASSERT_EQ(combined->column(0)->num_chunks(), 1); + ASSERT_OK_AND_ASSIGN( + auto chunked_parquet, + WriteTableToBuffer(chunked, kMinChunkSize, kMaxChunkSize, kRowGroupLength, false, + param.data_page_version)); + ASSERT_OK_AND_ASSIGN( + auto combined_parquet, + WriteTableToBuffer(combined, kMinChunkSize, kMaxChunkSize, kRowGroupLength, false, + param.data_page_version)); + ASSERT_EQ(GetPageEnds(chunked_parquet, /*column_index=*/0), + GetPageEnds(combined_parquet, /*column_index=*/0)); +} + #if defined(ADDRESS_SANITIZER) || defined(ARROW_VALGRIND) // Instantiate the test suite with a reduced set of types to avoid slow tests INSTANTIATE_TEST_SUITE_P( From 0430b9cacf2783d51270d859d1b82c26e62e3ff6 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Tue, 17 Nov 2026 09:39:42 +0100 Subject: [PATCH 2/3] Document that the first chunk may be empty --- cpp/src/parquet/chunker_internal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/chunker_internal.cc b/cpp/src/parquet/chunker_internal.cc index 02a7a3afbfe8..eda59a84698b 100644 --- a/cpp/src/parquet/chunker_internal.cc +++ b/cpp/src/parquet/chunker_internal.cc @@ -229,7 +229,8 @@ class ContentDefinedChunker::Impl { // chunks must be non-empty and monotonic increasing ARROW_DCHECK(!chunks.empty()); - // the first chunk must start at the first level + // the first chunk must start at the first level, it may be empty to close the + // page left open by the previous call auto first_chunk = chunks.front(); ARROW_DCHECK_EQ(first_chunk.level_offset, 0); ARROW_DCHECK_EQ(first_chunk.value_offset, 0); From ae71e1651760aebedbec78eb97b766ec5481cffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 17 Nov 2026 09:56:36 +0100 Subject: [PATCH 3/3] Update cpp/src/parquet/chunker_internal_test.cc Co-authored-by: Antoine Pitrou --- cpp/src/parquet/chunker_internal_test.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/chunker_internal_test.cc b/cpp/src/parquet/chunker_internal_test.cc index a87815f17c9b..a351e05a35ff 100644 --- a/cpp/src/parquet/chunker_internal_test.cc +++ b/cpp/src/parquet/chunker_internal_test.cc @@ -1470,7 +1470,9 @@ TEST_P(TestCDCSingleRowGroup, ArrayOffsets) { } TEST_P(TestCDCSingleRowGroup, IndependentOfArrayChunks) { - // Each array chunk is written by a separate WriteArrow() call + // Content-defined chunking should produce the same pages + // for logically equal inputs, regardless of the inputs' own + // in-memory chunking. const auto& param = GetParam(); ASSERT_OK_AND_ASSIGN(auto chunked, ConcatenateTables({part1_, part2_, part3_})); ASSERT_OK_AND_ASSIGN(auto combined, chunked->CombineChunks());