diff --git a/cpp/src/io/parquet/page_decode.cuh b/cpp/src/io/parquet/page_decode.cuh index 499cb9dfbcc8..54db9c1d873c 100644 --- a/cpp/src/io/parquet/page_decode.cuh +++ b/cpp/src/io/parquet/page_decode.cuh @@ -1352,12 +1352,8 @@ inline __device__ bool setup_local_page_info(auto* const s, case Encoding::PLAIN_DICTIONARY: case Encoding::RLE_DICTIONARY: { // RLE-packed dictionary indices, first byte indicates index length in bits - auto const is_decimal = s->setup.col.logical_type.has_value() and - s->setup.col.logical_type->type == LogicalType::DECIMAL; - if ((s->setup.col.physical_type == Type::BYTE_ARRAY or - s->setup.col.physical_type == Type::FIXED_LEN_BYTE_ARRAY) and - not is_decimal and s->setup.col.str_dict_index != nullptr) { - // String dictionary: use index + if (s->setup.col.str_dict_index != nullptr) { + // String dictionary (incl. BYTE_ARRAY decimals): use index s->stream.dict_base = reinterpret_cast(s->setup.col.str_dict_index); s->stream.dict_size = s->setup.col.dict_page->num_input_values * sizeof(string_index_pair); diff --git a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh index a1610b633613..6263a1a69b1d 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh +++ b/cpp/src/io/parquet/reader_impl_preprocess_utils.cuh @@ -146,15 +146,15 @@ void decode_sparse_page_headers(pass_intermediate_data& pass, cudf::memory_resources mr); /** - * @brief Check if the column chunk has a string (byte array or FLBA) type + * @brief Check if the column chunk needs a string dictionary index: any byte array (including + * decimals, which are variable width) or a non-decimal FLBA */ -__device__ constexpr inline bool is_string_chunk(ColumnChunkDesc const& chunk) +__device__ constexpr inline bool needs_str_dict_index(ColumnChunkDesc const& chunk) { auto const is_decimal = chunk.logical_type.has_value() and chunk.logical_type->type == LogicalType::DECIMAL; - auto const is_binary = - chunk.physical_type == Type::BYTE_ARRAY or chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY; - return is_binary and not is_decimal; + return chunk.physical_type == Type::BYTE_ARRAY or + (chunk.physical_type == Type::FIXED_LEN_BYTE_ARRAY and not is_decimal); } /** @@ -168,7 +168,7 @@ struct set_str_dict_index_count { { auto const& chunk = chunks[page.chunk_idx]; if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0 and chunk.num_dict_pages > 0 and - is_string_chunk(chunk)) { + needs_str_dict_index(chunk)) { // there is only ever one dictionary page per chunk, so this is safe to do in parallel. str_dict_index_count[page.chunk_idx] = page.num_input_values; } @@ -191,7 +191,7 @@ struct set_str_dict_index_ptr { if (sparse_page_io and (chunk.dict_page == nullptr or chunk.dict_page->page_data == nullptr)) { return; } - if (chunk.num_dict_pages > 0 and is_string_chunk(chunk)) { + if (chunk.num_dict_pages > 0 and needs_str_dict_index(chunk)) { chunk.str_dict_index = base + str_dict_index_offsets[i]; } } diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index f425b0e9a8fe..13f6723f773c 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -1790,6 +1790,81 @@ TEST_F(ParquetReaderTest, DecimalRead) } } +template +struct ParquetReaderByteArrayDecimalTest : public ParquetReaderTest {}; + +TYPED_TEST_SUITE(ParquetReaderByteArrayDecimalTest, cudf::test::FixedPointTypes); + +TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) +{ + using T = TypeParam; + auto constexpr num_rows = 1000; + auto constexpr scale = 2; + auto constexpr precision = cuda::std::is_same_v ? 9 + : cuda::std::is_same_v ? 18 + : 38; + + // Few distinct values so the dictionary is used, each as 8 big-endian bytes + auto const values = cudf::detail::make_counting_transform_iterator( + 0, [](int64_t i) { return (i % 10 - 5) * int64_t{70'001}; }); + auto const bytes = cudf::detail::make_counting_transform_iterator(0, [&](int i) { + std::string out(8, '\0'); + for (auto b = 0; b < 8; ++b) { + out[7 - b] = static_cast(values[i] >> (8 * b)); + } + return out; + }); + auto const col = cudf::test::strings_column_wrapper(bytes, bytes + num_rows); + auto const table = cudf::table_view{{col}}; + + // No writer produces BYTE_ARRAY decimals, so write a binary column and annotate it as decimal + std::vector buffer; + auto metadata = cudf::io::table_input_metadata{table}; + metadata.column_metadata[0].set_output_as_binary(true); + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, table) + .metadata(std::move(metadata)) + .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) + .build()); + + auto footer = + cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ + cudf::host_span{buffer.data(), buffer.size()}})) + .front(); + auto const& encodings = footer.row_groups[0].columns[0].meta_data.encodings; + ASSERT_TRUE(std::ranges::any_of(encodings, [](auto enc) { + return enc == cudf::io::parquet::Encoding::PLAIN_DICTIONARY or + enc == cudf::io::parquet::Encoding::RLE_DICTIONARY; + })); + + auto& schema = footer.schema[1]; + schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; + schema.decimal_scale = scale; + schema.decimal_precision = precision; + schema.logical_type = cudf::io::parquet::LogicalType{ + cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; + + // Swap the trailing footer for the annotated one, keeping the data pages + uint32_t footer_len{}; + std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len)); + auto const new_footer = parquet_delta_test::serialize_footer(footer); + buffer.resize(buffer.size() - 8 - footer_len); + buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); + footer_len = new_footer.size(); + buffer.insert(buffer.end(), + reinterpret_cast(&footer_len), + reinterpret_cast(&footer_len + 1)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + + auto const result = cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span{buffer.data(), buffer.size()}}) + .build()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), + cudf::test::fixed_point_column_wrapper( + values, values + num_rows, numeric::scale_type{-scale})); +} + // Reading Parquet DELTA-encoded pages whose mini-blocks hold more than 64 values. No stock // writer emits over 64 values/mini-block (cudf and parquet-mr write 32, pyarrow and arrow-rs at // most 64), so the files are built in-memory by the parquet_delta_test_utils.hpp helpers.