diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index e7fa94be07f0..f4220f6539f7 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -488,13 +488,17 @@ void reader_impl::create_global_chunk_info() auto col = _input_columns[i]; // look up metadata auto& col_meta = _metadata->get_column_metadata(rg.index, rg.source_index, col.schema_idx); - auto& schema = _metadata->get_schema( - _metadata->map_schema_index(col.schema_idx, rg.source_index), rg.source_index); + auto const mapped_schema_idx = _metadata->map_schema_index(col.schema_idx, rg.source_index); + auto& schema = _metadata->get_schema(mapped_schema_idx, rg.source_index); + // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories + auto const strings_to_categorical = + _strings_to_categorical and + not _metadata->is_binary_leaf(mapped_schema_idx, rg.source_index); auto [clock_rate, logical_type] = conversion_info(to_type_id(schema, _metadata->arrow_type(col.schema_idx), - _strings_to_categorical, + strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width), _options.timestamp_type.id(), @@ -532,7 +536,7 @@ void reader_impl::create_global_chunk_info() col.schema_idx, chunk_info, list_bytes_per_row_est, - schema.type == Type::BYTE_ARRAY and _strings_to_categorical, + schema.type == Type::BYTE_ARRAY and strings_to_categorical, rg.source_index); } // Adjust for skip_rows when updating the remaining rows after the first group diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index f40335f8c50d..de252bf4454a 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1075,16 +1075,19 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, // Validate that all sources have the same schema unless we are reading select columns // from mismatched sources, in which case, we will only check the projected columns later. if (not has_cols_from_mismatched_srcs) { - // `SchemaElement::operator==` ignores logical types, so compare FILE groups separately + // `SchemaElement::operator==` ignores logical types, so compare FILE groups and binary + // leaves of logical groups (FILE, VARIANT) separately auto const is_file_group = [](std::vector const& tree, size_t idx) { return is_valid_file_group( tree[idx], tree[idx].children_idx | std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; })); }; - auto const same_file_groups = [&](std::vector const& other) { + auto const same_logical_groups = [&](metadata const& pfm) { + auto const pfm_idx = static_cast(&pfm - per_file_metadata.data()); return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) { - return is_file_group(schema, idx) == is_file_group(other, idx); + return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and + is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx); }); }; @@ -1094,7 +1097,7 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(), "All non-empty sources must have the same number of columns"); } - CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema), + CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm), "All sources must have the same schema"); } @@ -1682,6 +1685,25 @@ std::unique_ptr aggregate_reader_metadata::read_column_chunk_bounds( return std::make_unique
(std::move(columns)); } +bool aggregate_reader_metadata::is_binary_leaf(int schema_idx, int pfm_idx) const +{ + auto const& schema_elem = get_schema(schema_idx, pfm_idx); + if (schema_elem.type != Type::BYTE_ARRAY or schema_elem.num_children != 0) { return false; } + auto const& parent = get_schema(schema_elem.parent_idx, pfm_idx); + if (not parent.logical_type.has_value()) { return false; } + switch (parent.logical_type->type) { + case LogicalType::VARIANT: return schema_elem.name == "metadata" or schema_elem.name == "value"; + case LogicalType::FILE: + return schema_elem.name == "inline" and + is_valid_file_group( + parent, + parent.children_idx | std::views::transform([&](int idx) -> SchemaElement const& { + return get_schema(idx, pfm_idx); + })); + default: return false; + } +} + bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const { // Check if schema_idx or src_idx is invalid @@ -2213,12 +2235,15 @@ aggregate_reader_metadata::select_columns( auto const& parent_schema = get_schema(schema_elem.parent_idx); auto const one_level_list = schema_elem.is_one_level_list(parent_schema); + // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories + auto const binary_leaf = is_binary_leaf(schema_idx); + auto const leaf_to_categorical = strings_to_categorical and not binary_leaf; // if we're at the root, this is a new output column auto const col_type = one_level_list ? type_id::LIST : to_type_id(schema_elem, arrow_type(schema_idx), - strings_to_categorical, + leaf_to_categorical, timestamp_type_id, decimal_type_id); auto const dtype = to_data_type(col_type, schema_elem); @@ -2260,15 +2285,6 @@ aggregate_reader_metadata::select_columns( } } - // Flag the `inline` child of a valid FILE group so that `make_column` materializes it as - // `list` instead of strings. - if (output_col.is_file) { - auto const inline_col = std::ranges::find(output_col.children, - std::string_view{"inline"}, - &cudf::io::detail::inline_column_buffer::name); - if (inline_col != output_col.children.end()) { inline_col->string_as_binary = true; } - } - // if I have no children, we're at a leaf and I'm an input column (that is, one with actual // data stored) so add me to the list. if (schema_elem.num_children == 0) { @@ -2280,7 +2296,7 @@ aggregate_reader_metadata::select_columns( // determine the element data type auto const element_type = to_type_id(schema_elem, arrow_type(schema_idx), - strings_to_categorical, + leaf_to_categorical, timestamp_type_id, decimal_type_id); auto const element_dtype = to_data_type(element_type, schema_elem); @@ -2305,13 +2321,10 @@ aggregate_reader_metadata::select_columns( // pop off the extra nesting element. if (one_level_list) { nesting.pop_back(); } - // Flag the `metadata` / `value` BYTE_ARRAY children of a VARIANT group so that - // `make_column` materializes them as `list` instead of strings. - if (schema_elem.type == Type::BYTE_ARRAY && parent_schema.logical_type.has_value() && - parent_schema.logical_type->type == LogicalType::VARIANT && - (schema_elem.name == "metadata" || schema_elem.name == "value")) { - output_col.string_as_binary = true; - } + // Flag binary leaves of logical groups (the `inline` child of a valid FILE group and the + // `metadata` / `value` children of a VARIANT group) so that `make_column` materializes + // them as `list` instead of strings. + if (binary_leaf) { output_col.string_as_binary = true; } path_is_valid = true; // If we're able to reach leaf then path is valid } @@ -2383,6 +2396,15 @@ aggregate_reader_metadata::select_columns( src_schema_elem.name), std::invalid_argument); + // Binary leaves of logical groups (FILE, VARIANT) are read as binary, so they must be + // binary leaves in all sources + CUDF_EXPECTS( + is_binary_leaf(src_schema_idx) == is_binary_leaf(dst_schema_idx, src_idx), + std::format("Encountered mismatching binary leaf of a logical group across the Parquet " + "sources for column '{}'", + src_schema_elem.name), + std::invalid_argument); + // Get the schema_idx_map for this data source (pfm) auto& schema_idx_map = schema_idx_maps[src_idx - 1]; // Map the schema index from 0th tree (src) to the one in the current (dst) tree. diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index d0020a055929..9626e96e3771 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -711,6 +711,19 @@ class aggregate_reader_metadata { return nullable_across_sources.contains(schema_idx); } + /** + * @brief Checks if a schema element is a binary BYTE_ARRAY leaf of a logical group, i.e. the + * `inline` child of a valid FILE group or the `metadata` / `value` child of a VARIANT group + * + * Such leaves are always read as `list` and are never converted to categories. + * + * @param schema_idx The index of the SchemaElement in the pfm_idx'th file + * @param pfm_idx The index of the per_file_metadata, default = 0 if not specified + * + * @return True if the schema element is a binary leaf of a logical group + */ + [[nodiscard]] bool is_binary_leaf(int schema_idx, int pfm_idx = 0) const; + /** * @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file * diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index 9f0cf17ce823..3c5622878a6f 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -258,17 +258,19 @@ void reader_impl::allocate_nesting_info() } // Get the schema from the current input source. - auto& actual_cur_schema = _metadata->get_schema( - _metadata->map_schema_index(schema_idx, src_file_idx), src_file_idx); + auto const actual_cur_schema_idx = _metadata->map_schema_index(schema_idx, src_file_idx); + auto& actual_cur_schema = _metadata->get_schema(actual_cur_schema_idx, src_file_idx); // values indexed by output column index nesting_info[cur_depth].max_def_level = actual_cur_schema.max_definition_level; pni[cur_depth].size = 0; - pni[cur_depth].type = to_type_id(actual_cur_schema, - _metadata->arrow_type(schema_idx), - _strings_to_categorical, - _options.timestamp_type.id(), - _options.decimal_width); + pni[cur_depth].type = + to_type_id(actual_cur_schema, + _metadata->arrow_type(schema_idx), + _strings_to_categorical and + not _metadata->is_binary_leaf(actual_cur_schema_idx, src_file_idx), + _options.timestamp_type.id(), + _options.decimal_width); pni[cur_depth].nullable = cur_schema.repetition_type == FieldRepetitionType::OPTIONAL; // the implicit list of a one-level list is a required list with its parent's def level @@ -736,11 +738,12 @@ void reader_impl::preprocess_file(read_mode mode) printf("# Input columns: %'lu\n", _input_columns.size()); for (size_t idx = 0; idx < _input_columns.size(); idx++) { auto const& schema = _metadata->get_schema(_input_columns[idx].schema_idx); - auto const type_id = to_type_id(schema, - _metadata->arrow_type(_input_columns[idx].schema_idx), - _strings_to_categorical, - _options.timestamp_type.id(), - _options.decimal_width); + auto const type_id = to_type_id( + schema, + _metadata->arrow_type(_input_columns[idx].schema_idx), + _strings_to_categorical and not _metadata->is_binary_leaf(_input_columns[idx].schema_idx), + _options.timestamp_type.id(), + _options.decimal_width); printf("\tC(%'lu, %s): %s\n", idx, _input_columns[idx].name.c_str(), diff --git a/cpp/tests/io/parquet_common.cpp b/cpp/tests/io/parquet_common.cpp index 14bb9b2b869e..4139d0d0e9d3 100644 --- a/cpp/tests/io/parquet_common.cpp +++ b/cpp/tests/io/parquet_common.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include +#include #include #include #include @@ -255,6 +257,32 @@ void read_footer(std::unique_ptr const& source, file_meta_data); } +void edit_footer(std::vector& buffer, + std::function const& edit) +{ + namespace pq_experimental = cudf::io::parquet::experimental; + + // parquet files end with 4-byte footer_length and 4-byte magic == "PAR1" + auto constexpr ender_size = 2 * sizeof(uint32_t); + CUDF_EXPECTS(buffer.size() >= ender_size, "Buffer is too small to be a Parquet file"); + uint32_t footer_size{}; + std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size)); + CUDF_EXPECTS(buffer.size() >= ender_size + footer_size, "Invalid Parquet footer length"); + auto const footer_begin = buffer.size() - ender_size - footer_size; + + auto file_meta_data = pq_experimental::read_parquet_footer_bytes( + {reinterpret_cast(buffer.data()) + footer_begin, footer_size}); + edit(file_meta_data); + auto const footer = pq_experimental::write_parquet_footer_bytes(file_meta_data); + + auto const new_footer_size = static_cast(footer.size()); + auto const size_bytes = reinterpret_cast(&new_footer_size); + buffer.resize(footer_begin); + buffer.insert(buffer.end(), footer.begin(), footer.end()); + buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); +} + // returns the number of bits used for dictionary encoding data at the given page location. // this assumes the data is uncompressed. // throws cudf::logic_error if the page_loc data is invalid. diff --git a/cpp/tests/io/parquet_common.hpp b/cpp/tests/io/parquet_common.hpp index 58472745ff6f..4f24b80720e8 100644 --- a/cpp/tests/io/parquet_common.hpp +++ b/cpp/tests/io/parquet_common.hpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -134,6 +135,18 @@ std::unique_ptr make_parquet_list_list_col( void read_footer(std::unique_ptr const& source, cudf::io::parquet::FileMetaData* file_meta_data); +/** + * @brief Rewrites the footer of an in-memory Parquet file + * + * Decodes the trailing footer of `buffer`, applies `edit` to it, and replaces it with the + * re-encoded footer, keeping the data pages as is. + * + * @param buffer Parquet file bytes to update in place + * @param edit Callable that modifies the decoded footer + */ +void edit_footer(std::vector& buffer, + std::function const& edit); + /** * @brief Returns the number of bits used for dictionary encoding the uncompressed page at * `page_loc` diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index fde7dc2d83d0..e63c0e8e7050 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -1866,7 +1865,7 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) .build()); - auto footer = + auto const footer = cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ cudf::host_span{buffer.data(), buffer.size()}})) .front(); @@ -1876,24 +1875,14 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) enc == cudf::io::parquet::Encoding::RLE_DICTIONARY; })); - auto& schema = footer.schema[1]; - schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; - schema.decimal_scale = scale; - schema.decimal_precision = precision; - schema.logical_type = cudf::io::parquet::LogicalType{ - cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; - - // Swap the trailing footer for the annotated one, keeping the data pages - uint32_t footer_len{}; - std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len)); - auto const new_footer = parquet_delta_test::serialize_footer(footer); - buffer.resize(buffer.size() - 8 - footer_len); - buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); - footer_len = new_footer.size(); - buffer.insert(buffer.end(), - reinterpret_cast(&footer_len), - reinterpret_cast(&footer_len + 1)); - buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + edit_footer(buffer, [&](cudf::io::parquet::FileMetaData& fmd) { + auto& schema = fmd.schema[1]; + schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; + schema.decimal_scale = scale; + schema.decimal_precision = precision; + schema.logical_type = cudf::io::parquet::LogicalType{ + cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; + }); auto const result = cudf::io::read_parquet( cudf::io::parquet_reader_options::builder( @@ -2825,33 +2814,20 @@ TEST_F(ParquetReaderTest, FileLogicalType) file_metadata.child(0).set_name("offset").set_nullability(true); file_metadata.child(1).set_name("inline").set_nullability(true).set_output_as_binary(true); - std::vector buffer; + std::vector file_group; cudf::io::write_parquet( - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input) + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&file_group}, input) .metadata(std::move(input_metadata)) .build()); + edit_footer(file_group, [](cudf::io::parquet::FileMetaData& fmd) { + EXPECT_EQ(fmd.schema[1].name, "file"); + EXPECT_EQ(fmd.schema[2].name, "offset"); + EXPECT_EQ(fmd.schema[3].name, "inline"); + fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; + // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE + fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; + }); - auto const source = cudf::io::datasource::create(cudf::host_span{ - reinterpret_cast(buffer.data()), buffer.size()}); - auto fmd = cudf::io::read_parquet_footers({&source, 1}).front(); - ASSERT_EQ(fmd.schema[1].name, "file"); - ASSERT_EQ(fmd.schema[2].name, "offset"); - ASSERT_EQ(fmd.schema[3].name, "inline"); - fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; - // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE - fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; - - // Appends `fmd` as the new footer, leaving the original one unreferenced - auto const rewrite_file_footer = [&] { - auto const footer = cudf::io::parquet::experimental::write_parquet_footer_bytes(fmd); - auto const footer_size = static_cast(footer.size()); - auto const size_bytes = reinterpret_cast(&footer_size); - auto file = buffer; - file.insert(file.end(), footer.begin(), footer.end()); - file.insert(file.end(), size_bytes, size_bytes + sizeof(footer_size)); - file.insert(file.end(), {'P', 'A', 'R', '1'}); - return file; - }; auto const read_metadata = [](std::span const> files, bool allow_mismatched = true) { std::vector> sources; @@ -2867,15 +2843,13 @@ TEST_F(ParquetReaderTest, FileLogicalType) .metadata; }; - auto const file_group = rewrite_file_footer(); - auto const metadata = read_metadata(std::array{file_group}); + auto const metadata = read_metadata(std::array{file_group}); EXPECT_EQ(metadata.schema_info[0].is_file, true); EXPECT_EQ(metadata.schema_info[0].children[1].is_binary, true); // A FILE group and a plain struct across sources throw, with or without mismatched schemas - fmd.schema[1].logical_type.reset(); - auto const plain_struct = rewrite_file_footer(); - fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; + auto plain_struct = file_group; + edit_footer(plain_struct, [](auto& fmd) { fmd.schema[1].logical_type.reset(); }); for (auto const allow_mismatched : {true, false}) { EXPECT_THROW(read_metadata(std::array{file_group, plain_struct}, allow_mismatched), std::logic_error); @@ -2884,8 +2858,67 @@ TEST_F(ParquetReaderTest, FileLogicalType) } // An annotated `inline` breaks the FILE spec, so the group is read as a plain struct - fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; - EXPECT_FALSE(read_metadata(std::array{rewrite_file_footer()}).schema_info[0].is_file.has_value()); + auto annotated_inline = file_group; + edit_footer(annotated_inline, + [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; }); + EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value()); +} + +TEST_F(ParquetReaderTest, VariantGroupsAcrossSources) +{ + // VARIANT binary leaves read from multiple sources keep their bytes when strings are converted + // to categories. A VARIANT group and a plain struct with the same binary children across + // sources throw, with or without mismatched schemas, as only the VARIANT binary leaves skip + // category conversion + + cudf::test::strings_column_wrapper metadata_col{"m"}; + cudf::test::strings_column_wrapper value_col{"v"}; + auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, value_col}}; + + auto const input = table_view{{variant_col}}; + cudf::io::table_input_metadata input_metadata(input); + auto& variant_metadata = input_metadata.column_metadata[0].set_name("variant"); + variant_metadata.child(0).set_name("metadata").set_output_as_binary(true); + variant_metadata.child(1).set_name("value").set_output_as_binary(true); + + std::vector plain_struct; + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&plain_struct}, input) + .metadata(std::move(input_metadata)) + .build()); + + // cudf cannot write VARIANT groups yet, so retag the struct group as VARIANT in the footer + auto variant_group = plain_struct; + edit_footer(variant_group, [](cudf::io::parquet::FileMetaData& fmd) { + EXPECT_EQ(fmd.schema[1].name, "variant"); + fmd.schema[1].logical_type = + cudf::io::parquet::LogicalType{cudf::io::parquet::LogicalType::VARIANT}; + }); + + auto const read = [](std::span const> files, bool allow_mismatched) { + std::vector> sources; + for (auto const& file : files) { + sources.emplace_back(reinterpret_cast(file.data()), file.size()); + } + return cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span>{sources}}) + .allow_mismatched_pq_schemas(allow_mismatched) + .column_names({"variant"}) + .convert_strings_to_categories(true)); + }; + + for (auto const allow_mismatched : {true, false}) { + auto const result = read(std::array{variant_group, variant_group}, allow_mismatched); + auto const variant = result.tbl->view().column(0); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(0), + cudf::test::lists_column_wrapper{{'m'}, {'m'}}); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(1), + cudf::test::lists_column_wrapper{{'v'}, {'v'}}); + + EXPECT_THROW(read(std::array{variant_group, plain_struct}, allow_mismatched), std::logic_error); + EXPECT_THROW(read(std::array{plain_struct, variant_group}, allow_mismatched), std::logic_error); + } } TEST_F(ParquetReaderTest, NestingOptimizationTest) diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py index 2212cdbbc2f9..38dc10976bd3 100644 --- a/python/cudf/cudf/tests/input_output/test_parquet.py +++ b/python/cudf/cudf/tests/input_output/test_parquet.py @@ -774,6 +774,64 @@ def file_type(*names): assert_eq(cudf.DataFrame.from_arrow(expect), got) +@pytest.mark.parametrize( + "filename, columns", + [ + ("file_logical_type.parquet", None), + ("variant_multirow.parquet", None), + ("variant_nested_unshredded.parquet", None), + ("variant_nested_shredded.parquet", None), + # Skip the shredded `v.typed_value.{n,k}.value` leaves, which are still + # hashed as their parents aren't annotated VARIANT (see #25425) + ( + "duckdb_variant_sample.parquet", + [ + "id", + "v.metadata", + "v.value", + "v.typed_value.n.typed_value", + "v.typed_value.k.typed_value", + ], + ), + ], +) +def test_parquet_reader_logical_group_binary_not_categorized( + datadir, filename, columns +): + # Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`) + # groups stay `list` when strings are converted to categories, while + # all other strings are read as INT32 hashes + import pylibcudf as plc + + def read_types(convert_strings_to_categories): + source = plc.io.SourceInfo([str(datadir / filename)]) + options = ( + plc.io.parquet.ParquetReaderOptions.builder(source) + .convert_strings_to_categories(convert_strings_to_categories) + .build() + ) + if columns is not None: + options.set_column_names(columns) + return plc.io.parquet.read_parquet(options).tbl.to_arrow().schema.types + + def categorized(typ): + if pa.types.is_string(typ) or pa.types.is_large_string(typ): + return pa.int32() + if pa.types.is_struct(typ): + return pa.struct([f.with_type(categorized(f.type)) for f in typ]) + if pa.types.is_list(typ): + return pa.list_( + typ.value_field.with_type(categorized(typ.value_type)) + ) + if pa.types.is_large_list(typ): + return pa.large_list( + typ.value_field.with_type(categorized(typ.value_type)) + ) + return typ + + assert read_types(True) == [categorized(t) for t in read_types(False)] + + @pytest.mark.parametrize("columns", [["a"], ["b", "a"], None]) def test_parquet_reader_decimal128(datadir, columns): fname = datadir / "nested_decimal128_file.parquet"