Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions cpp/src/io/parquet/page_decode.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1352,12 +1352,8 @@ inline __device__ bool setup_local_page_info(auto* const s,
case Encoding::PLAIN_DICTIONARY:
case Encoding::RLE_DICTIONARY: {
// RLE-packed dictionary indices, first byte indicates index length in bits
auto const is_decimal = s->setup.col.logical_type.has_value() and
s->setup.col.logical_type->type == LogicalType::DECIMAL;
if ((s->setup.col.physical_type == Type::BYTE_ARRAY or
s->setup.col.physical_type == Type::FIXED_LEN_BYTE_ARRAY) and
not is_decimal and s->setup.col.str_dict_index != nullptr) {
// String dictionary: use index
if (s->setup.col.str_dict_index != nullptr) {
// String dictionary (incl. BYTE_ARRAY decimals): use index
s->stream.dict_base = reinterpret_cast<uint8_t const*>(s->setup.col.str_dict_index);
s->stream.dict_size =
s->setup.col.dict_page->num_input_values * sizeof(string_index_pair);
Expand Down
14 changes: 7 additions & 7 deletions cpp/src/io/parquet/reader_impl_preprocess_utils.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -146,15 +146,15 @@ void decode_sparse_page_headers(pass_intermediate_data& pass,
cudf::memory_resources mr);

/**
* @brief Check if the column chunk has a string (byte array or FLBA) type
* @brief Check if the column chunk needs a string dictionary index: any byte array (including
* decimals, which are variable width) or a non-decimal FLBA
*/
__device__ constexpr inline bool is_string_chunk(ColumnChunkDesc const& chunk)
__device__ constexpr inline bool needs_str_dict_index(ColumnChunkDesc const& chunk)
{
auto const is_decimal =
chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL;
auto const is_binary =
chunk.physical_type == Type::BYTE_ARRAY or chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY;
return is_binary and not is_decimal;
return chunk.physical_type == Type::BYTE_ARRAY or
(chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY and not is_decimal);
}

/**
Expand All @@ -168,7 +168,7 @@ struct set_str_dict_index_count {
{
auto const& chunk = chunks[page.chunk_idx];
if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0 and chunk.num_dict_pages > 0 and
is_string_chunk(chunk)) {
needs_str_dict_index(chunk)) {
// there is only ever one dictionary page per chunk, so this is safe to do in parallel.
str_dict_index_count[page.chunk_idx] = page.num_input_values;
}
Expand All @@ -191,7 +191,7 @@ struct set_str_dict_index_ptr {
if (sparse_page_io and (chunk.dict_page == nullptr or chunk.dict_page->page_data == nullptr)) {
return;
}
if (chunk.num_dict_pages > 0 and is_string_chunk(chunk)) {
if (chunk.num_dict_pages > 0 and needs_str_dict_index(chunk)) {
chunk.str_dict_index = base + str_dict_index_offsets[i];
}
}
Expand Down
75 changes: 75 additions & 0 deletions cpp/tests/io/parquet_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1790,6 +1790,81 @@ TEST_F(ParquetReaderTest, DecimalRead)
}
}

template <typename T>
struct ParquetReaderByteArrayDecimalTest : public ParquetReaderTest {};

TYPED_TEST_SUITE(ParquetReaderByteArrayDecimalTest, cudf::test::FixedPointTypes);

TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded)
{
using T = TypeParam;
auto constexpr num_rows = 1000;
auto constexpr scale = 2;
auto constexpr precision = cuda::std::is_same_v<T, numeric::decimal32> ? 9
: cuda::std::is_same_v<T, numeric::decimal64> ? 18
: 38;

// Few distinct values so the dictionary is used, each as 8 big-endian bytes
auto const values = cudf::detail::make_counting_transform_iterator(
0, [](int64_t i) { return (i % 10 - 5) * int64_t{70'001}; });
auto const bytes = cudf::detail::make_counting_transform_iterator(0, [&](int i) {
std::string out(8, '\0');
for (auto b = 0; b < 8; ++b) {
out[7 - b] = static_cast<char>(values[i] >> (8 * b));
}
return out;
});
auto const col = cudf::test::strings_column_wrapper(bytes, bytes + num_rows);
auto const table = cudf::table_view{{col}};

// No writer produces BYTE_ARRAY decimals, so write a binary column and annotate it as decimal
std::vector<char> buffer;
auto metadata = cudf::io::table_input_metadata{table};
metadata.column_metadata[0].set_output_as_binary(true);
cudf::io::write_parquet(
cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, table)
.metadata(std::move(metadata))
.dictionary_policy(cudf::io::dictionary_policy::ALWAYS)
.build());

auto footer =
cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{
cudf::host_span<char const>{buffer.data(), buffer.size()}}))
.front();
auto const& encodings = footer.row_groups[0].columns[0].meta_data.encodings;
ASSERT_TRUE(std::ranges::any_of(encodings, [](auto enc) {
return enc == cudf::io::parquet::Encoding::PLAIN_DICTIONARY or
enc == cudf::io::parquet::Encoding::RLE_DICTIONARY;
}));

auto& schema = footer.schema[1];
Comment thread
mhaseeb123 marked this conversation as resolved.
Resolved
schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;
schema.decimal_scale = scale;
schema.decimal_precision = precision;
schema.logical_type = cudf::io::parquet::LogicalType{
cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}};

// Swap the trailing footer for the annotated one, keeping the data pages
uint32_t footer_len{};
std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len));
auto const new_footer = parquet_delta_test::serialize_footer(footer);
buffer.resize(buffer.size() - 8 - footer_len);
buffer.insert(buffer.end(), new_footer.begin(), new_footer.end());
footer_len = new_footer.size();
buffer.insert(buffer.end(),
reinterpret_cast<char const*>(&footer_len),
reinterpret_cast<char const*>(&footer_len + 1));
buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
Comment on lines +1830 to +1857

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once #25339 merges, this can use the shared edit_footer helper from parquet_common.hpp:

Suggested change
auto footer =
cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{
cudf::host_span<char const>{buffer.data(), buffer.size()}}))
.front();
auto& schema = footer.schema[1];
schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;
schema.decimal_scale = scale;
schema.decimal_precision = precision;
schema.logical_type = cudf::io::parquet::LogicalType{
cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}};
// Swap the trailing footer for the annotated one, keeping the data pages
uint32_t footer_len{};
std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len));
auto const new_footer = parquet_delta_test::serialize_footer(footer);
buffer.resize(buffer.size() - 8 - footer_len);
buffer.insert(buffer.end(), new_footer.begin(), new_footer.end());
footer_len = new_footer.size();
buffer.insert(buffer.end(),
reinterpret_cast<char const*>(&footer_len),
reinterpret_cast<char const*>(&footer_len + 1));
buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
edit_footer(buffer, [&](cudf::io::parquet::FileMetaData& footer) {
auto& schema = footer.schema[1];
schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;
schema.decimal_scale = scale;
schema.decimal_precision = precision;
schema.logical_type = cudf::io::parquet::LogicalType{
cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}};
});


auto const result = cudf::io::read_parquet(
cudf::io::parquet_reader_options::builder(
cudf::io::source_info{cudf::host_span<char const>{buffer.data(), buffer.size()}})
.build());
CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0),
cudf::test::fixed_point_column_wrapper<typename T::rep>(
values, values + num_rows, numeric::scale_type{-scale}));
}

// Reading Parquet DELTA-encoded pages whose mini-blocks hold more than 64 values. No stock
// writer emits over 64 values/mini-block (cudf and parquet-mr write 32, pyarrow and arrow-rs at
// most 64), so the files are built in-memory by the parquet_delta_test_utils.hpp helpers.
Expand Down
Loading