From 22ce40cea1b871e7b56e006f37a200a8dd2d3522 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:15:59 +0000 Subject: [PATCH 1/5] Fix reading dictionary-encoded BYTE_ARRAY decimals in Parquet reader --- cpp/src/io/parquet/page_decode.cuh | 8 +- .../parquet/reader_impl_preprocess_utils.cuh | 8 +- cpp/tests/io/parquet_reader_test.cpp | 78 +++++++++++++++++++ 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/cpp/src/io/parquet/page_decode.cuh b/cpp/src/io/parquet/page_decode.cuh index 499cb9dfbcc8..54db9c1d873c 100644 --- a/cpp/src/io/parquet/page_decode.cuh +++ b/cpp/src/io/parquet/page_decode.cuh @@ -1352,12 +1352,8 @@ inline __device__ bool setup_local_page_info(auto* const s, case Encoding::PLAIN_DICTIONARY: case Encoding::RLE_DICTIONARY: { // RLE-packed dictionary indices, first byte indicates index length in bits - auto const is_decimal = s->setup.col.logical_type.has_value() and - s->setup.col.logical_type->type == LogicalType::DECIMAL; - if ((s->setup.col.physical_type == Type::BYTE_ARRAY or - s->setup.col.physical_type == Type::FIXED_LEN_BYTE_ARRAY) and - not is_decimal and s->setup.col.str_dict_index != nullptr) { - // String dictionary: use index + if (s->setup.col.str_dict_index != nullptr) { + // String dictionary (incl. BYTE_ARRAY decimals): use index s->stream.dict_base = reinterpret_cast(s->setup.col.str_dict_index); s->stream.dict_size = s->setup.col.dict_page->num_input_values * sizeof(string_index_pair); diff --git a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh index a1610b633613..46a55e21e97a 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh +++ b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh @@ -146,15 +146,15 @@ void decode_sparse_page_headers(pass_intermediate_data& pass, cudf::memory_resources mr); /** - * @brief Check if the column chunk has a string (byte array or FLBA) type + * @brief Check if the column chunk needs a string dictionary index: any byte array (including + * decimals, which are variable width) or a non-decimal FLBA */ __device__ constexpr inline bool is_string_chunk(ColumnChunkDesc const& chunk) { auto const is_decimal = chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL; - auto const is_binary = - chunk.physical_type == Type::BYTE_ARRAY or chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY; - return is_binary and not is_decimal; + return chunk.physical_type == Type::BYTE_ARRAY or + (chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY and not is_decimal); } /** diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index f425b0e9a8fe..db7fd73d2e88 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -1790,6 +1791,83 @@ TEST_F(ParquetReaderTest, DecimalRead) } } +template +struct ParquetReaderByteArrayDecimalTest : public ParquetReaderTest {}; + +TYPED_TEST_SUITE(ParquetReaderByteArrayDecimalTest, cudf::test::FixedPointTypes); + +TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictAndPlain) +{ + using T = TypeParam; + using rep = typename T::rep; + auto constexpr num_rows = 1000; + auto constexpr scale = 2; + auto constexpr precision = cuda::std::is_same_v ? 9 + : cuda::std::is_same_v ? 18 + : 38; + + // Few distinct values so the dictionary is used, each in the fewest big-endian bytes that hold it + auto const values = cudf::detail::make_counting_transform_iterator( + 0, [](int64_t i) { return (i % 10 - 5) * int64_t{70'001}; }); + auto const byte_array_decimals = cudf::detail::make_counting_transform_iterator(0, [&](int i) { + auto const value = values[i]; + auto const magnitude = static_cast(value < 0 ? ~value : value); + auto const num_bytes = static_cast(std::bit_width(magnitude) / 8 + 1); + std::string bytes(num_bytes, '\0'); + for (auto b = 0; b < num_bytes; ++b) { + bytes[num_bytes - 1 - b] = static_cast(value >> (8 * b)); + } + return bytes; + }); + auto const col = + cudf::test::strings_column_wrapper(byte_array_decimals, byte_array_decimals + num_rows); + auto const table = cudf::table_view{{col}}; + auto const expected = cudf::test::fixed_point_column_wrapper( + values, values + num_rows, numeric::scale_type{-scale}); + + for (auto const policy : + {cudf::io::dictionary_policy::ALWAYS, cudf::io::dictionary_policy::NEVER}) { + // No writer produces BYTE_ARRAY decimals, so write a binary column and annotate it as decimal + std::vector buffer; + auto metadata = cudf::io::table_input_metadata{table}; + metadata.column_metadata[0].set_name("dec").set_output_as_binary(true); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, table) + .metadata(std::move(metadata)) + .dictionary_policy(policy) + .build()); + + auto footer = + cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ + cudf::host_span{buffer.data(), buffer.size()}})) + .front(); + auto& schema = footer.schema[1]; + schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; + schema.decimal_scale = scale; + schema.decimal_precision = precision; + schema.logical_type = cudf::io::parquet::LogicalType{ + cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; + + // Replace the footer, which trails the data pages, with the annotated one + uint32_t old_footer_len{}; + std::memcpy(&old_footer_len, buffer.data() + buffer.size() - 8, sizeof(old_footer_len)); + buffer.resize(buffer.size() - 8 - old_footer_len); + auto const new_footer = parquet_delta_test::serialize_footer(footer); + auto const new_footer_len = static_cast(new_footer.size()); + buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); + buffer.insert(buffer.end(), + reinterpret_cast(&new_footer_len), + reinterpret_cast(&new_footer_len) + sizeof(new_footer_len)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span{buffer.data(), buffer.size()}}) + .build()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), expected); + } +} + // Reading Parquet DELTA-encoded pages whose mini-blocks hold more than 64 values. No stock // writer emits over 64 values/mini-block (cudf and parquet-mr write 32, pyarrow and arrow-rs at // most 64), so the files are built in-memory by the parquet_delta_test_utils.hpp helpers. From 2c5eac48bc4677869cf83f3509d5d7dbbf1cee1f Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:07:29 +0000 Subject: [PATCH 2/5] Trim BYTE_ARRAY decimal dictionary test --- cpp/tests/io/parquet_reader_test.cpp | 105 ++++++++++++--------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index db7fd73d2e88..e79145adc6a5 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -1796,76 +1795,68 @@ struct ParquetReaderByteArrayDecimalTest : public ParquetReaderTest {}; TYPED_TEST_SUITE(ParquetReaderByteArrayDecimalTest, cudf::test::FixedPointTypes); -TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictAndPlain) +TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) { using T = TypeParam; - using rep = typename T::rep; auto constexpr num_rows = 1000; auto constexpr scale = 2; auto constexpr precision = cuda::std::is_same_v ? 9 : cuda::std::is_same_v ? 18 : 38; - // Few distinct values so the dictionary is used, each in the fewest big-endian bytes that hold it + // Few distinct values so the dictionary is used, each as 8 big-endian bytes auto const values = cudf::detail::make_counting_transform_iterator( 0, [](int64_t i) { return (i % 10 - 5) * int64_t{70'001}; }); - auto const byte_array_decimals = cudf::detail::make_counting_transform_iterator(0, [&](int i) { - auto const value = values[i]; - auto const magnitude = static_cast(value < 0 ? ~value : value); - auto const num_bytes = static_cast(std::bit_width(magnitude) / 8 + 1); - std::string bytes(num_bytes, '\0'); - for (auto b = 0; b < num_bytes; ++b) { - bytes[num_bytes - 1 - b] = static_cast(value >> (8 * b)); + auto const bytes = cudf::detail::make_counting_transform_iterator(0, [&](int i) { + std::string out(8, '\0'); + for (auto b = 0; b < 8; ++b) { + out[7 - b] = static_cast(values[i] >> (8 * b)); } - return bytes; + return out; }); - auto const col = - cudf::test::strings_column_wrapper(byte_array_decimals, byte_array_decimals + num_rows); - auto const table = cudf::table_view{{col}}; - auto const expected = cudf::test::fixed_point_column_wrapper( - values, values + num_rows, numeric::scale_type{-scale}); - - for (auto const policy : - {cudf::io::dictionary_policy::ALWAYS, cudf::io::dictionary_policy::NEVER}) { - // No writer produces BYTE_ARRAY decimals, so write a binary column and annotate it as decimal - std::vector buffer; - auto metadata = cudf::io::table_input_metadata{table}; - metadata.column_metadata[0].set_name("dec").set_output_as_binary(true); - cudf::io::write_parquet( - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, table) - .metadata(std::move(metadata)) - .dictionary_policy(policy) - .build()); + auto const col = cudf::test::strings_column_wrapper(bytes, bytes + num_rows); + auto const table = cudf::table_view{{col}}; - auto footer = - cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ - cudf::host_span{buffer.data(), buffer.size()}})) - .front(); - auto& schema = footer.schema[1]; - schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; - schema.decimal_scale = scale; - schema.decimal_precision = precision; - schema.logical_type = cudf::io::parquet::LogicalType{ - cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; - - // Replace the footer, which trails the data pages, with the annotated one - uint32_t old_footer_len{}; - std::memcpy(&old_footer_len, buffer.data() + buffer.size() - 8, sizeof(old_footer_len)); - buffer.resize(buffer.size() - 8 - old_footer_len); - auto const new_footer = parquet_delta_test::serialize_footer(footer); - auto const new_footer_len = static_cast(new_footer.size()); - buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); - buffer.insert(buffer.end(), - reinterpret_cast(&new_footer_len), - reinterpret_cast(&new_footer_len) + sizeof(new_footer_len)); - buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + // No writer produces BYTE_ARRAY decimals, so write a binary column and annotate it as decimal + std::vector buffer; + auto metadata = cudf::io::table_input_metadata{table}; + metadata.column_metadata[0].set_output_as_binary(true); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, table) + .metadata(std::move(metadata)) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .build()); - auto const result = cudf::io::read_parquet( - cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{buffer.data(), buffer.size()}}) - .build()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), expected); - } + auto footer = + cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ + cudf::host_span{buffer.data(), buffer.size()}})) + .front(); + auto& schema = footer.schema[1]; + schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; + schema.decimal_scale = scale; + schema.decimal_precision = precision; + schema.logical_type = cudf::io::parquet::LogicalType{ + cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; + + // Swap the trailing footer for the annotated one, keeping the data pages + uint32_t footer_len{}; + std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len)); + auto const new_footer = parquet_delta_test::serialize_footer(footer); + buffer.resize(buffer.size() - 8 - footer_len); + buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); + footer_len = new_footer.size(); + buffer.insert(buffer.end(), + reinterpret_cast(&footer_len), + reinterpret_cast(&footer_len + 1)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span{buffer.data(), buffer.size()}}) + .build()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), + cudf::test::fixed_point_column_wrapper( + values, values + num_rows, numeric::scale_type{-scale})); } // Reading Parquet DELTA-encoded pages whose mini-blocks hold more than 64 values. No stock From ed6e90534de29f084d40ccb78e282bfb9e9c32b6 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:26:59 -0800 Subject: [PATCH 3/5] Assert the test column is dictionary-encoded --- cpp/tests/io/parquet_reader_test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index e79145adc6a5..edd96f58a84e 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -1831,6 +1831,10 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ cudf::host_span{buffer.data(), buffer.size()}})) .front(); + auto const& encodings = footer.row_groups[0].columns[0].meta_data.encodings; + ASSERT_NE(std::ranges::find(encodings, cudf::io::parquet::Encoding::RLE_DICTIONARY), + encodings.end()); + auto& schema = footer.schema[1]; schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; schema.decimal_scale = scale; From 937a2dcb55128034e0a72080705c17b08e28f13d Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:34:47 +0000 Subject: [PATCH 4/5] Rename is_string_chunk to needs_str_dict_index --- cpp/src/io/parquet/reader_impl_preprocess_utils.cuh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh index 46a55e21e97a..6263a1a69b1d 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh +++ b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh @@ -149,7 +149,7 @@ void decode_sparse_page_headers(pass_intermediate_data& pass, * @brief Check if the column chunk needs a string dictionary index: any byte array (including * decimals, which are variable width) or a non-decimal FLBA */ -__device__ constexpr inline bool is_string_chunk(ColumnChunkDesc const& chunk) +__device__ constexpr inline bool needs_str_dict_index(ColumnChunkDesc const& chunk) { auto const is_decimal = chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL; @@ -168,7 +168,7 @@ struct set_str_dict_index_count { { auto const& chunk = chunks[page.chunk_idx]; if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0 and chunk.num_dict_pages > 0 and - is_string_chunk(chunk)) { + needs_str_dict_index(chunk)) { // there is only ever one dictionary page per chunk, so this is safe to do in parallel. str_dict_index_count[page.chunk_idx] = page.num_input_values; } @@ -191,7 +191,7 @@ struct set_str_dict_index_ptr { if (sparse_page_io and (chunk.dict_page == nullptr or chunk.dict_page->page_data == nullptr)) { return; } - if (chunk.num_dict_pages > 0 and is_string_chunk(chunk)) { + if (chunk.num_dict_pages > 0 and needs_str_dict_index(chunk)) { chunk.str_dict_index = base + str_dict_index_offsets[i]; } } From 51e80d9983f7c48109685318c472bd35e0c989fd Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:20:39 +0000 Subject: [PATCH 5/5] Accept PLAIN_DICTIONARY or RLE_DICTIONARY in test assert --- cpp/tests/io/parquet_reader_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index edd96f58a84e..13f6723f773c 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -1832,8 +1832,10 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) cudf::host_span{buffer.data(), buffer.size()}})) .front(); auto const& encodings = footer.row_groups[0].columns[0].meta_data.encodings; - ASSERT_NE(std::ranges::find(encodings, cudf::io::parquet::Encoding::RLE_DICTIONARY), - encodings.end()); + ASSERT_TRUE(std::ranges::any_of(encodings, [](auto enc) { + return enc == cudf::io::parquet::Encoding::PLAIN_DICTIONARY or + enc == cudf::io::parquet::Encoding::RLE_DICTIONARY; + })); auto& schema = footer.schema[1]; schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;