Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions cpp/src/parquet/chunker_internal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ class ContentDefinedChunker::Impl {
// chunks must be non-empty and monotonic increasing
ARROW_DCHECK(!chunks.empty());

// the first chunk must start at the first level
// the first chunk must start at the first level, it may be empty to close the
// page left open by the previous call
auto first_chunk = chunks.front();
ARROW_DCHECK_EQ(first_chunk.level_offset, 0);
ARROW_DCHECK_EQ(first_chunk.value_offset, 0);
Expand Down Expand Up @@ -326,13 +327,11 @@ class ContentDefinedChunker::Impl {
}

if (rep_level == 0 && gearhash.NeedNewChunk()) {
// if we are at a record boundary and need a new chunk, we create a new chunk
auto levels_to_write = offset - prev_offset;
if (levels_to_write > 0) {
chunks.push_back({prev_offset, prev_value_offset, levels_to_write});
prev_offset = offset;
prev_value_offset = value_offset;
}
// if we are at a record boundary and need a new chunk, we create a new chunk,
// at offset 0 it is empty and closes the page of the previous call
chunks.push_back({prev_offset, prev_value_offset, offset - prev_offset});
prev_offset = offset;
prev_value_offset = value_offset;
}
if (def_level >= level_info_.repeated_ancestor_def_level) {
// we only increment the value offset if we have a leaf value
Expand Down
22 changes: 22 additions & 0 deletions cpp/src/parquet/chunker_internal_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,7 @@ ParquetInfo GetColumnParquetInfo(const std::shared_ptr<Buffer>& data, int column
while (auto page = page_reader->NextPage()) {
if (page->type() == PageType::DATA_PAGE || page->type() == PageType::DATA_PAGE_V2) {
auto data_page = static_cast<DataPage*>(page.get());
EXPECT_GT(data_page->num_values(), 0) << "empty data page";
Comment thread
kszucs marked this conversation as resolved.
Resolved
column_info.page_sizes.push_back(data_page->uncompressed_size());
column_info.page_lengths.push_back(data_page->num_values());
} else if (page->type() == PageType::DICTIONARY_PAGE) {
Expand Down Expand Up @@ -1468,6 +1469,27 @@ TEST_P(TestCDCSingleRowGroup, ArrayOffsets) {
}
}

TEST_P(TestCDCSingleRowGroup, IndependentOfArrayChunks) {
// Content-defined chunking should produce the same pages
// for logically equal inputs, regardless of the inputs' own
// in-memory chunking.
const auto& param = GetParam();
ASSERT_OK_AND_ASSIGN(auto chunked, ConcatenateTables({part1_, part2_, part3_}));
ASSERT_OK_AND_ASSIGN(auto combined, chunked->CombineChunks());
ASSERT_EQ(chunked->column(0)->num_chunks(), 3);
ASSERT_EQ(combined->column(0)->num_chunks(), 1);
ASSERT_OK_AND_ASSIGN(
auto chunked_parquet,
WriteTableToBuffer(chunked, kMinChunkSize, kMaxChunkSize, kRowGroupLength, false,
param.data_page_version));
ASSERT_OK_AND_ASSIGN(
auto combined_parquet,
WriteTableToBuffer(combined, kMinChunkSize, kMaxChunkSize, kRowGroupLength, false,
param.data_page_version));
ASSERT_EQ(GetPageEnds(chunked_parquet, /*column_index=*/0),
GetPageEnds(combined_parquet, /*column_index=*/0));
}

#if defined(ADDRESS_SANITIZER) || defined(ARROW_VALGRIND)
// Instantiate the test suite with a reduced set of types to avoid slow tests
INSTANTIATE_TEST_SUITE_P(
Expand Down
Loading