Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 125 additions & 2 deletions cpp/src/parquet/arrow/arrow_reader_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4691,8 +4691,7 @@ TEST(TestArrowReaderAdHoc, OldDataPageV2) {
GTEST_SKIP() << "ARROW_TEST_DATA not set.";
}
std::stringstream ss;
ss << c_root << "/"
<< "parquet/ARROW-17100.parquet";
ss << c_root << "/" << "parquet/ARROW-17100.parquet";
std::string path = ss.str();
TryReadDataFile(path);
}
Expand Down Expand Up @@ -5225,6 +5224,130 @@ TEST_P(TestArrowWriteDictionary, StatisticsUnifiedDictionary) {
ASSERT_EQ(rg1_page_reader->NextPage(), nullptr);
}

TEST_P(TestArrowWriteDictionary, StatisticsWithNullsMatchingDictLength) {
// Check that dict-encoded inputs produce correct statistics (GH-55272)
auto dict_type = ::arrow::dictionary(::arrow::int32(), ::arrow::utf8());
auto schema = ::arrow::schema({::arrow::field("values", dict_type)});

struct DictionaryCase {
std::string dictionary_json;
std::string indices_json;
int expected_null_count;
std::string expected_min;
std::string expected_max;
};
const auto dict_cases = std::vector<DictionaryCase>{
{R"(["a", "b"])", R"([0, 0])", 0, "a", "a"},
{R"(["a", "b"])", R"([0, null, 0])", 1, "a", "a"},
{R"(["a", "b"])", R"([1, 0])", 0, "a", "b"},
{R"(["a", "b"])", R"([1, null, 0])", 1, "a", "b"},
};

for (const auto& dict_case : dict_cases) {
auto dictionary = ArrayFromJSON(::arrow::utf8(), dict_case.dictionary_json);
auto indices = ArrayFromJSON(::arrow::int32(), dict_case.indices_json);
ASSERT_OK_AND_ASSIGN(auto dict_array, ::arrow::DictionaryArray::FromArrays(
dict_type, indices, dictionary));
auto table = ::arrow::Table::Make(schema, {dict_array});

std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
auto out_stream = std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);

std::shared_ptr<WriterProperties> writer_properties =
WriterProperties::Builder()
.data_page_version(this->GetParquetDataPageVersion())
->build();
std::unique_ptr<FileWriter> writer;
ASSERT_OK_AND_ASSIGN(
writer, FileWriter::Open(*schema, ::arrow::default_memory_pool(), out_stream,
writer_properties, default_arrow_writer_properties()));
ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
ASSERT_OK(writer->Close());
ASSERT_OK(out_stream->Close());

auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
std::unique_ptr<ParquetFileReader> parquet_reader =
ParquetFileReader::Open(std::move(buffer_reader));

std::shared_ptr<FileMetaData> metadata = parquet_reader->metadata();
ASSERT_EQ(metadata->num_row_groups(), 1);
auto stats = metadata->RowGroup(0)->ColumnChunk(0)->statistics();
ASSERT_NE(stats, nullptr);
EXPECT_EQ(stats->num_values(), 2);
EXPECT_EQ(stats->null_count(), dict_case.expected_null_count);
EXPECT_EQ(stats->EncodeMin(), dict_case.expected_min);
EXPECT_EQ(stats->EncodeMax(), dict_case.expected_max);
}
}

TEST_P(TestArrowWriteDictionary, NestedDictionaryParentNullStatistics) {
// Ensure statistics accurately reflect parent nulls for nested dictionary columns.
auto dict_type = ::arrow::dictionary(::arrow::int32(), ::arrow::utf8());
auto inner_field = ::arrow::field("inner", dict_type, /*nullable=*/false);
auto struct_type = ::arrow::struct_({inner_field});
auto schema = ::arrow::schema({::arrow::field("root", struct_type, /*nullable=*/true)});

auto dictionary = ArrayFromJSON(::arrow::utf8(), R"(["a", "b"])");
auto indices = ArrayFromJSON(::arrow::int32(), R"([0, 0, 1])");
ASSERT_OK_AND_ASSIGN(auto inner_dict, ::arrow::DictionaryArray::FromArrays(
dict_type, indices, dictionary));

// Parent struct validity bitmap: [valid, null, valid]
::arrow::TypedBufferBuilder<bool> bitmap_builder;
ASSERT_OK(bitmap_builder.Append(true));
ASSERT_OK(bitmap_builder.Append(false));
ASSERT_OK(bitmap_builder.Append(true));
ASSERT_OK_AND_ASSIGN(auto null_bitmap, bitmap_builder.Finish());

ASSERT_OK_AND_ASSIGN(auto struct_array, ::arrow::StructArray::Make(
{inner_dict}, {inner_field}, null_bitmap));
auto table = ::arrow::Table::Make(schema, {struct_array});

std::shared_ptr<::arrow::ResizableBuffer> serialized_data = AllocateBuffer();
auto out_stream = std::make_shared<::arrow::io::BufferOutputStream>(serialized_data);

std::shared_ptr<WriterProperties> writer_properties =
WriterProperties::Builder()
.data_page_version(this->GetParquetDataPageVersion())
->build();
std::unique_ptr<FileWriter> writer;
ASSERT_OK_AND_ASSIGN(
writer, FileWriter::Open(*schema, ::arrow::default_memory_pool(), out_stream,
writer_properties, default_arrow_writer_properties()));
ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
ASSERT_OK(writer->Close());
ASSERT_OK(out_stream->Close());

auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(serialized_data);
std::unique_ptr<ParquetFileReader> parquet_reader =
ParquetFileReader::Open(std::move(buffer_reader));

std::shared_ptr<FileMetaData> metadata = parquet_reader->metadata();
ASSERT_EQ(metadata->num_row_groups(), 1);
auto stats = metadata->RowGroup(0)->ColumnChunk(0)->statistics();
ASSERT_NE(stats, nullptr);
// There are 2 valid inner values and 1 null (from the parent struct)
EXPECT_EQ(stats->num_values(), 2);
EXPECT_EQ(stats->null_count(), 1);
EXPECT_EQ(stats->EncodeMin(), "a");
EXPECT_EQ(stats->EncodeMax(), "b");

// In V2, verify that the data page header num_nulls matches page statistics
if (this->GetParquetDataPageVersion() == ParquetDataPageVersion::V2) {
auto rg_reader = parquet_reader->RowGroup(0)->GetColumnPageReader(0);
auto dict_page = rg_reader->NextPage();
ASSERT_NE(dict_page, nullptr);
ASSERT_EQ(dict_page->type(), PageType::DICTIONARY_PAGE);
auto page = rg_reader->NextPage();
ASSERT_NE(page, nullptr);
ASSERT_EQ(page->type(), PageType::DATA_PAGE_V2);
auto data_page_v2 = std::static_pointer_cast<DataPageV2>(page);
EXPECT_EQ(data_page_v2->num_nulls(), 1);
EXPECT_EQ(data_page_v2->num_rows(), 3);
EXPECT_EQ(data_page_v2->num_values(), 3);
}
}

// ----------------------------------------------------------------------
// Tests for directly reading DictionaryArray

Expand Down
15 changes: 10 additions & 5 deletions cpp/src/parquet/column_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2023,8 +2023,13 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
PARQUET_ASSIGN_OR_THROW(::arrow::Datum referenced_indices,
::arrow::compute::Unique(*chunk_indices, &exec_ctx));

// On first run, we might be able to re-use the existing dictionary
if (referenced_indices.length() == dictionary->length()) {
// Use the original dictionary if we can detect that all indices are referenced,
// otherwise call Take to find referenced dictionary values.
const bool all_indices_referenced =
(referenced_indices.null_count() == 0)
? referenced_indices.length() == dictionary->length()
: referenced_indices.length() == dictionary->length() + 1;
if (all_indices_referenced) {
referenced_dictionary = dictionary;
} else {
PARQUET_ASSIGN_OR_THROW(
Expand Down Expand Up @@ -2064,12 +2069,12 @@ Status TypedColumnWriterImpl<ParquetType>::WriteArrowDictionary(
AddIfNotNull(rep_levels, offset));
std::shared_ptr<Array> writeable_indices =
indices->Slice(value_offset, batch_num_spaced_values);
if (page_statistics_ || bloom_filter_writer_) {
update_stats(/*num_chunk_levels=*/batch_size, writeable_indices);
}
PARQUET_ASSIGN_OR_THROW(
writeable_indices,
MaybeReplaceValidity(writeable_indices, null_count, ctx->memory_pool));
if (page_statistics_ || bloom_filter_writer_) {
update_stats(/*num_chunk_levels=*/batch_size, writeable_indices);
}
Comment thread
Awesome06 marked this conversation as resolved.
Resolved
dict_encoder->PutIndices(*writeable_indices);
// Update unencoded byte array data size to size statistics
UpdateUnencodedDataBytes();
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/parquet/column_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1134,7 +1134,7 @@ TEST(TestColumnWriter, LARGE_MEMORY_TEST(WriteLargeDictEncodedPage)) {
if (page_count == 0) {
ASSERT_EQ(page->type(), PageType::DICTIONARY_PAGE);
} else {
ASSERT_EQ(page->type(), PageType::DATA_PAGE);
ASSERT_EQ(page->type(), PageType::DATA_PAGE_V2);
}
page_count++;
}
Expand Down
9 changes: 8 additions & 1 deletion cpp/src/parquet/file_serialize_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,15 @@ class TestSerialize : public PrimitiveTypedTest<TestType> {
const int64_t total_compressed_size = rg_metadata->total_compressed_size();
if (expected_codec_type == Compression::UNCOMPRESSED) {
ASSERT_EQ(total_byte_size, total_compressed_size);
} else {
} else if (writer_properties->data_page_version() == ParquetDataPageVersion::V1) {
ASSERT_NE(total_byte_size, total_compressed_size);
} else {
// Under Parquet V2 data pages, small or incompressible data pages remain
// uncompressed due to selective compression. Unconditionally compressed
// dictionary pages can compress to smaller, larger (due to codec overhead),
// or identical size.
ASSERT_GT(total_byte_size, 0);
ASSERT_GT(total_compressed_size, 0);
}

int64_t total_column_byte_size = 0;
Expand Down
4 changes: 2 additions & 2 deletions cpp/src/parquet/properties.h
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ class PARQUET_EXPORT WriterProperties {
pagesize_(kDefaultDataPageSize),
max_rows_per_page_(kDefaultMaxRowsPerPage),
version_(ParquetVersion::PARQUET_2_6),
data_page_version_(ParquetDataPageVersion::V1),
data_page_version_(ParquetDataPageVersion::V2),
created_by_(DEFAULT_CREATED_BY),
store_decimal_as_integer_(false),
page_checksum_enabled_(false),
Expand Down Expand Up @@ -525,7 +525,7 @@ class PARQUET_EXPORT WriterProperties {
}

/// Specify the data page version.
/// Default V1.
/// Default V2.
Builder* data_page_version(ParquetDataPageVersion data_page_version) {
data_page_version_ = data_page_version;
return this;
Expand Down
9 changes: 5 additions & 4 deletions cpp/src/parquet/properties_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ TEST(TestWriterProperties, Basics) {
ASSERT_EQ(kDefaultDataPageSize, props->data_pagesize());
ASSERT_EQ(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT, props->dictionary_pagesize_limit());
ASSERT_EQ(ParquetVersion::PARQUET_2_6, props->version());
ASSERT_EQ(ParquetDataPageVersion::V1, props->data_page_version());
ASSERT_EQ(ParquetDataPageVersion::V2, props->data_page_version());
ASSERT_FALSE(props->page_checksum_enabled());
}

Expand Down Expand Up @@ -112,14 +112,15 @@ TEST(TestWriterProperties, MinCompressionSavingsRequiresDataPageV2) {
for (double savings : {2 * kDefaultMinCompressionSavings, 0.1, 1.0}) {
WriterProperties::Builder builder;
builder.min_compression_savings(savings);
EXPECT_NO_THROW(builder.build());

builder.data_page_version(ParquetDataPageVersion::V1);
EXPECT_THROW(builder.build(), ParquetException);

builder.data_page_version(ParquetDataPageVersion::V2);
EXPECT_NO_THROW(builder.build());

builder.data_page_version(ParquetDataPageVersion::V1);
EXPECT_THROW(builder.build(), ParquetException);

builder.min_compression_savings(0.0);
EXPECT_NO_THROW(builder.build());
builder.min_compression_savings(kDefaultMinCompressionSavings);
Expand Down Expand Up @@ -418,7 +419,7 @@ std::vector<WriterPropertiesTestCase> writer_properties_test_cases() {
builder.max_row_group_length(DEFAULT_MAX_ROW_GROUP_LENGTH - 1);
builder.data_pagesize(kDefaultDataPageSize - 1);
builder.max_rows_per_page(kDefaultMaxRowsPerPage - 1);
builder.data_page_version(ParquetDataPageVersion::V2);
builder.data_page_version(ParquetDataPageVersion::V1);
builder.version(ParquetVersion::type::PARQUET_2_4);
builder.enable_store_decimal_as_integer();
builder.disable_write_page_index();
Expand Down
4 changes: 2 additions & 2 deletions cpp/src/parquet/type_fwd.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ namespace parquet {
/// versions).
///
/// However, some features - such as compression algorithms, encryption,
/// or the improved "v2" data page format - must be enabled separately in
/// ArrowWriterProperties.
/// or opting out of the "v2" data page format - are configured separately in
/// WriterProperties.
struct ParquetVersion {
enum type : int {
/// Enable only pre-2.2 Parquet format features when writing
Expand Down
11 changes: 6 additions & 5 deletions docs/source/cpp/parquet.rst
Original file line number Diff line number Diff line change
Expand Up @@ -267,16 +267,17 @@ group that takes precedent over the ``chunk_size`` passed in the write methods.

You can set the version of Parquet to write with ``version``, which determines
which logical types are available. In addition, you can set the data page version
with ``data_page_version``. It's V1 by default; setting to V2 will allow more
optimal compression (skipping compressing pages where there isn't a space
benefit), but not all readers support this data page version.
with ``data_page_version``. It defaults to V2, which allows more optimal
compression (skipping compressing pages where there isn't a space benefit) and
page-level null count metadata. If compatibility with older readers is needed,
it can be set to V1.

For V2 pages, use ``min_compression_savings`` to set how much space compression must
save. Use a value from 0.0 to 1.0. For example, ``min_compression_savings(0.1)`` requires
at least 10% savings. If the space savings for a page do not meet the threshold,
it is not compressed. The default is ``0.01``, so compression must save at least 1%.
Setting a value above this default with V1 or using a number outside the allowed
range throws ``ParquetException``.
Setting a value above this default with V1 data pages or using a number outside
the allowed range throws ``ParquetException``.

Compression is off by default, but to get the most out of Parquet, you should
also choose a compression codec. You can choose one for the whole file or
Expand Down
6 changes: 3 additions & 3 deletions docs/source/python/parquet/parquet.rst
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ such as the row groups and column chunk metadata and statistics:
<pyarrow._parquet.RowGroupMetaData object at ...>
num_columns: 4
num_rows: 3
total_byte_size: 290
total_byte_size: 299
sorting_columns: ()
>>> metadata.row_group(0).column(0)
<pyarrow._parquet.ColumnChunkMetaData object at ...>
Expand Down Expand Up @@ -333,8 +333,8 @@ such as the row groups and column chunk metadata and statistics:
has_dictionary_page: True
dictionary_page_offset: 4
data_page_offset: 36
total_compressed_size: 106
total_uncompressed_size: 102
total_compressed_size: 105
total_uncompressed_size: 103
bloom_filter_offset: None
bloom_filter_length: None

Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/_dataset_parquet.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -707,7 +707,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions):
compression_level=None,
use_byte_stream_split=False,
column_encoding=None,
data_page_version="1.0",
data_page_version="2.0",
min_compression_savings=0.01,
use_deprecated_int96_timestamps=False,
coerce_timestamps=None,
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/_parquet.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -2090,7 +2090,7 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
savings = min_compression_savings
if not (0.0 <= savings <= 1.0):
raise ValueError("min_compression_savings must be in range [0, 1]")
if savings > 0.01 and data_page_version in (None, "1.0"):
if savings > 0.01 and data_page_version == "1.0":
raise ValueError("min_compression_savings requires V2 data pages")
props.min_compression_savings(savings)

Expand Down
16 changes: 8 additions & 8 deletions python/pyarrow/parquet/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -770,8 +770,8 @@ def _sanitize_table(table, new_schema, flavor):
maximizes file compatibility.
UINT32 and some logical types are only available with version '2.4'.
Nanosecond timestamps are only available with version '2.6'.
Other features such as compression algorithms or the new serialized
data page format must be enabled separately (see 'compression' and
Other features such as compression algorithms or the serialized
data page format are configured separately (see 'compression' and
'data_page_version').
use_dictionary : bool or list, default True
Specify if we should use dictionary encoding in general or only for
Expand Down Expand Up @@ -840,9 +840,9 @@ def _sanitize_table(table, new_schema, flavor):
Certain encodings are only compatible with certain data types.
Please refer to the encodings section of `Reading and writing Parquet
files <https://arrow.apache.org/docs/cpp/parquet.html#encodings>`_.
data_page_version : {"1.0", "2.0"}, default "1.0"
data_page_version : {"1.0", "2.0"}, default "2.0"
The serialized Parquet data page format version to write, defaults to
1.0. This does not impact the file schema logical types and Arrow to
2.0. This does not impact the file schema logical types and Arrow to
Parquet type casting behavior; for that use the "version" option.
use_compliant_nested_type : bool, default True
Whether to write compliant Parquet nested type (lists) as defined
Expand Down Expand Up @@ -999,8 +999,8 @@ def _sanitize_table(table, new_schema, flavor):
Use a value from 0.0 to 1.0. For example, 0.1 requires at least 10% savings.
Savings are measured on encoded values, excluding repetition and definition
levels. The default is 0.01, so compression must save at least 1%.
Values above this default require ``data_page_version="2.0"``. Using
a larger value with V1 or a number outside the allowed range raises ``ValueError``.
Using a larger value with V1 data pages or a number outside
the allowed range raises ``ValueError``.
"""

_parquet_writer_example_doc = """\
Expand Down Expand Up @@ -1087,7 +1087,7 @@ def __init__(self, where, schema, filesystem=None,
use_byte_stream_split=False,
column_encoding=None,
writer_engine_version=None,
data_page_version='1.0',
data_page_version='2.0',
use_compliant_nested_type=True,
encryption_properties=None,
write_batch_size=None,
Expand Down Expand Up @@ -2050,7 +2050,7 @@ def write_table(table, where, row_group_size=None, version='2.6',
compression_level=None,
use_byte_stream_split=False,
column_encoding=None,
data_page_version='1.0',
data_page_version='2.0',
use_compliant_nested_type=True,
encryption_properties=None,
write_batch_size=None,
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/tests/parquet/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def test_parquet_metadata_api():
assert col_meta.is_stats_set is True
assert isinstance(col_meta.statistics, pq.Statistics)
assert col_meta.compression == 'SNAPPY'
assert set(col_meta.encodings) == {'PLAIN', 'RLE'}
assert set(col_meta.encodings) == {'RLE'}
assert col_meta.has_dictionary_page is False
assert col_meta.dictionary_page_offset is None
assert col_meta.data_page_offset > 0
Expand Down
Loading
Loading