From c51c3c4ce663201c773693da439250555416f073 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:47:41 +0000 Subject: [PATCH 01/11] Never categorize binary children of FILE and VARIANT groups in the Parquet reader --- .../hybrid_scan/hybrid_scan_preprocess.cu | 11 ++- cpp/src/io/parquet/reader_impl_chunking.cu | 12 ++- cpp/src/io/parquet/reader_impl_helpers.cpp | 46 +++++---- cpp/src/io/parquet/reader_impl_helpers.hpp | 13 +++ cpp/src/io/parquet/reader_impl_preprocess.cu | 19 ++-- cpp/tests/io/parquet_reader_test.cpp | 93 +++++++++++++++++++ 6 files changed, 162 insertions(+), 32 deletions(-) diff --git a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu index c1818b34c8bd..8908a9036611 100644 --- a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu +++ b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu @@ -292,8 +292,13 @@ hybrid_scan_reader_impl::prepare_dictionaries( // look up metadata auto& col_meta = _extended_metadata->get_column_metadata(rg.index, rg.source_index, col_schema_idx); - auto& schema = _extended_metadata->get_schema( - _extended_metadata->map_schema_index(col_schema_idx, rg.source_index), rg.source_index); + auto const mapped_schema_idx = + _extended_metadata->map_schema_index(col_schema_idx, rg.source_index); + auto& schema = _extended_metadata->get_schema(mapped_schema_idx, rg.source_index); + // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories + auto const strings_to_categorical = + options.is_enabled_convert_strings_to_categories() and + not _extended_metadata->is_binary_leaf(mapped_schema_idx, rg.source_index); // dictionary data buffer for this column chunk auto& dict_page_data = dictionary_page_data[chunk_idx]; @@ -304,7 +309,7 @@ hybrid_scan_reader_impl::prepare_dictionaries( auto const [clock_rate, logical_type] = parquet::detail::conversion_info( parquet::detail::to_type_id(schema, - options.is_enabled_convert_strings_to_categories(), + strings_to_categorical, options.get_timestamp_type().id(), options.get_decimal_width()), options.get_timestamp_type().id(), diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index beb7605b55fa..1df89803be80 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -483,12 +483,16 @@ void reader_impl::create_global_chunk_info() auto col = _input_columns[i]; // look up metadata auto& col_meta = _metadata->get_column_metadata(rg.index, rg.source_index, col.schema_idx); - auto& schema = _metadata->get_schema( - _metadata->map_schema_index(col.schema_idx, rg.source_index), rg.source_index); + auto const mapped_schema_idx = _metadata->map_schema_index(col.schema_idx, rg.source_index); + auto& schema = _metadata->get_schema(mapped_schema_idx, rg.source_index); + // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories + auto const strings_to_categorical = + _strings_to_categorical and + not _metadata->is_binary_leaf(mapped_schema_idx, rg.source_index); auto [clock_rate, logical_type] = conversion_info( to_type_id( - schema, _strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width), + schema, strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width), _options.timestamp_type.id(), schema.type, schema.logical_type); @@ -524,7 +528,7 @@ void reader_impl::create_global_chunk_info() col.schema_idx, chunk_info, list_bytes_per_row_est, - schema.type == Type::BYTE_ARRAY and _strings_to_categorical, + schema.type == Type::BYTE_ARRAY and strings_to_categorical, rg.source_index); } // Adjust for skip_rows when updating the remaining rows after the first group diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 3cfcb1f15c9e..97ba236a9f62 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1617,6 +1617,25 @@ std::unique_ptr aggregate_reader_metadata::read_column_chunk_bounds( return std::make_unique
(std::move(columns)); } +bool aggregate_reader_metadata::is_binary_leaf(int schema_idx, int pfm_idx) const +{ + auto const& schema_elem = get_schema(schema_idx, pfm_idx); + if (schema_elem.type != Type::BYTE_ARRAY or schema_elem.num_children != 0) { return false; } + auto const& parent = get_schema(schema_elem.parent_idx, pfm_idx); + if (not parent.logical_type.has_value()) { return false; } + switch (parent.logical_type->type) { + case LogicalType::VARIANT: return schema_elem.name == "metadata" or schema_elem.name == "value"; + case LogicalType::FILE: + return schema_elem.name == "inline" and + is_valid_file_group( + parent, + parent.children_idx | std::views::transform([&](int idx) -> SchemaElement const& { + return get_schema(idx, pfm_idx); + })); + default: return false; + } +} + bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const { // Check if schema_idx or src_idx is invalid @@ -2148,12 +2167,15 @@ aggregate_reader_metadata::select_columns( auto const& parent_schema = get_schema(schema_elem.parent_idx); auto const one_level_list = schema_elem.is_one_level_list(parent_schema); + // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories + auto const binary_leaf = is_binary_leaf(schema_idx); + auto const leaf_to_categorical = strings_to_categorical and not binary_leaf; // if we're at the root, this is a new output column auto const col_type = one_level_list ? type_id::LIST - : to_type_id(schema_elem, strings_to_categorical, timestamp_type_id, decimal_type_id); + : to_type_id(schema_elem, leaf_to_categorical, timestamp_type_id, decimal_type_id); auto const dtype = to_data_type(col_type, schema_elem); cudf::io::detail::inline_column_buffer output_col( @@ -2193,15 +2215,6 @@ aggregate_reader_metadata::select_columns( } } - // Flag the `inline` child of a valid FILE group so that `make_column` materializes it as - // `list` instead of strings. - if (output_col.is_file) { - auto const inline_col = std::ranges::find(output_col.children, - std::string_view{"inline"}, - &cudf::io::detail::inline_column_buffer::name); - if (inline_col != output_col.children.end()) { inline_col->string_as_binary = true; } - } - // if I have no children, we're at a leaf and I'm an input column (that is, one with actual // data stored) so add me to the list. if (schema_elem.num_children == 0) { @@ -2212,7 +2225,7 @@ aggregate_reader_metadata::select_columns( if (one_level_list) { // determine the element data type auto const element_type = - to_type_id(schema_elem, strings_to_categorical, timestamp_type_id, decimal_type_id); + to_type_id(schema_elem, leaf_to_categorical, timestamp_type_id, decimal_type_id); auto const element_dtype = to_data_type(element_type, schema_elem); cudf::io::detail::inline_column_buffer element_col( @@ -2235,13 +2248,10 @@ aggregate_reader_metadata::select_columns( // pop off the extra nesting element. if (one_level_list) { nesting.pop_back(); } - // Flag the `metadata` / `value` BYTE_ARRAY children of a VARIANT group so that - // `make_column` materializes them as `list` instead of strings. - if (schema_elem.type == Type::BYTE_ARRAY && parent_schema.logical_type.has_value() && - parent_schema.logical_type->type == LogicalType::VARIANT && - (schema_elem.name == "metadata" || schema_elem.name == "value")) { - output_col.string_as_binary = true; - } + // Flag binary leaves of logical groups (the `inline` child of a valid FILE group and the + // `metadata` / `value` children of a VARIANT group) so that `make_column` materializes + // them as `list` instead of strings. + if (binary_leaf) { output_col.string_as_binary = true; } path_is_valid = true; // If we're able to reach leaf then path is valid } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index a1796605e195..1a00d662c005 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -717,6 +717,19 @@ class aggregate_reader_metadata { return nullable_across_sources.contains(schema_idx); } + /** + * @brief Checks if a schema element is a binary BYTE_ARRAY leaf of a logical group, i.e. the + * `inline` child of a valid FILE group or the `metadata` / `value` child of a VARIANT group + * + * Such leaves are always read as `list` and are never converted to categories. + * + * @param schema_idx The index of the SchemaElement in the pfm_idx'th file + * @param pfm_idx The index of the per_file_metadata, default = 0 if not specified + * + * @return True if the schema element is a binary leaf of a logical group + */ + [[nodiscard]] bool is_binary_leaf(int schema_idx, int pfm_idx = 0) const; + /** * @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file * diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index a0463b00694c..666fdf611898 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -279,16 +279,18 @@ void reader_impl::allocate_nesting_info() } // Get the schema from the current input source. - auto& actual_cur_schema = _metadata->get_schema( - _metadata->map_schema_index(schema_idx, src_file_idx), src_file_idx); + auto const actual_cur_schema_idx = _metadata->map_schema_index(schema_idx, src_file_idx); + auto& actual_cur_schema = _metadata->get_schema(actual_cur_schema_idx, src_file_idx); // values indexed by output column index nesting_info[cur_depth].max_def_level = actual_cur_schema.max_definition_level; pni[cur_depth].size = 0; - pni[cur_depth].type = to_type_id(actual_cur_schema, - _strings_to_categorical, - _options.timestamp_type.id(), - _options.decimal_width); + pni[cur_depth].type = + to_type_id(actual_cur_schema, + _strings_to_categorical and + not _metadata->is_binary_leaf(actual_cur_schema_idx, src_file_idx), + _options.timestamp_type.id(), + _options.decimal_width); pni[cur_depth].nullable = cur_schema.repetition_type == FieldRepetitionType::OPTIONAL; // the implicit list of a one-level list is a required list with its parent's def level @@ -770,7 +772,10 @@ void reader_impl::preprocess_file(read_mode mode) for (size_t idx = 0; idx < _input_columns.size(); idx++) { auto const& schema = _metadata->get_schema(_input_columns[idx].schema_idx); auto const type_id = to_type_id( - schema, _strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width); + schema, + _strings_to_categorical and not _metadata->is_binary_leaf(_input_columns[idx].schema_idx), + _options.timestamp_type.id(), + _options.decimal_width); printf("\tC(%'lu, %s): %s\n", idx, _input_columns[idx].name.c_str(), diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 9cd3267b67be..566b9b49e644 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2851,6 +2851,99 @@ TEST_F(ParquetReaderTest, FileLogicalType) EXPECT_FALSE(read_metadata(std::array{rewrite_file_footer()}).schema_info[0].is_file.has_value()); } +// Binary leaves of FILE and VARIANT groups are read as `list` even when strings are +// converted to categories. cudf cannot write these groups yet, so retag a written struct group. +TEST_F(ParquetReaderTest, LogicalGroupBinaryLeavesNotCategorized) +{ + using cudf::io::parquet::LogicalType; + namespace pq_experimental = cudf::io::parquet::experimental; + + // Writes `str` (strings) and `group` (struct of `names`) and retags `group` as `logical_type` + auto const write_with_group_type = [](cudf::column_view group_col, + std::vector const& names, + cudf::column_view str_col, + LogicalType::Type logical_type) { + auto const input = table_view{{str_col, group_col}}; + cudf::io::table_input_metadata input_metadata(input); + input_metadata.column_metadata[0].set_name("str"); + auto& group_metadata = input_metadata.column_metadata[1].set_name("group"); + for (std::size_t i = 0; i < names.size(); ++i) { + group_metadata.child(i).set_name(names[i]).set_nullability(true); + if (group_col.child(i).type().id() == cudf::type_id::STRING and names[i] != "uri") { + group_metadata.child(i).set_output_as_binary(true); + } + } + + std::vector buffer; + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input) + .metadata(std::move(input_metadata)) + .build()); + + auto constexpr ender_size = 2 * sizeof(uint32_t); + uint32_t footer_size{}; + std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size)); + auto const footer_begin = buffer.size() - ender_size - footer_size; + auto fmd = pq_experimental::read_parquet_footer_bytes( + {reinterpret_cast(buffer.data()) + footer_begin, footer_size}); + EXPECT_EQ(fmd.schema[2].name, "group"); + fmd.schema[2].logical_type = LogicalType{logical_type}; + + auto const footer = pq_experimental::write_parquet_footer_bytes(fmd); + auto const new_footer_size = static_cast(footer.size()); + auto const size_bytes = reinterpret_cast(&new_footer_size); + buffer.resize(footer_begin); + buffer.insert(buffer.end(), footer.begin(), footer.end()); + buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + return buffer; + }; + + auto const read_categorized = [](std::vector const& file) { + auto const source = cudf::io::source_info{cudf::host_span{ + reinterpret_cast(file.data()), file.size()}}; + return cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(source).convert_strings_to_categories(true)); + }; + + cudf::test::strings_column_wrapper str_col{"x", "yy", "x", "zzz"}; + cudf::test::strings_column_wrapper bin_col{"a", "bc", "hello", "d"}; + cudf::test::lists_column_wrapper bin_bytes{ + {'a'}, {'b', 'c'}, {'h', 'e', 'l', 'l', 'o'}, {'d'}}; + + // FILE: `inline` is binary, while `uri` and the top-level strings are categorized + { + cudf::test::strings_column_wrapper uri_col{"s3://a", "s3://b", "", "s3://d"}; + auto file_col = cudf::test::structs_column_wrapper{{uri_col, bin_col}}; + auto const result = read_categorized( + write_with_group_type(file_col, {"uri", "inline"}, str_col, LogicalType::FILE)); + + auto const tbl = result.tbl->view(); + EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32); + EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT); + EXPECT_EQ(tbl.column(1).child(0).type().id(), cudf::type_id::INT32); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1)); + EXPECT_EQ(result.metadata.schema_info[1].is_file, true); + EXPECT_EQ(result.metadata.schema_info[1].children[1].is_binary, true); + } + + // VARIANT: `metadata` and `value` are binary, while the top-level strings are categorized + { + cudf::test::strings_column_wrapper metadata_col{"m0", "m1", "m2", "m3"}; + cudf::test::lists_column_wrapper metadata_bytes{ + {'m', '0'}, {'m', '1'}, {'m', '2'}, {'m', '3'}}; + auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, bin_col}}; + auto const result = read_categorized( + write_with_group_type(variant_col, {"metadata", "value"}, str_col, LogicalType::VARIANT)); + + auto const tbl = result.tbl->view(); + EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32); + EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(metadata_bytes, tbl.column(1).child(0)); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1)); + } +} + TEST_F(ParquetReaderTest, NestingOptimizationTest) { // test nesting levels > cudf::io::parquet::detail::max_cacheable_nesting_decode_info deep. From 562ef5739e5ef6aaaf31fd3b8f45b1696d19f413 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:47:29 +0000 Subject: [PATCH 02/11] Add edit_footer test helper and move the categorization test to Python --- cpp/tests/io/parquet_common.cpp | 28 ++++ cpp/tests/io/parquet_common.hpp | 13 ++ cpp/tests/io/parquet_reader_test.cpp | 137 +++--------------- .../cudf/tests/input_output/test_parquet.py | 41 ++++++ 4 files changed, 100 insertions(+), 119 deletions(-) diff --git a/cpp/tests/io/parquet_common.cpp b/cpp/tests/io/parquet_common.cpp index 14bb9b2b869e..4139d0d0e9d3 100644 --- a/cpp/tests/io/parquet_common.cpp +++ b/cpp/tests/io/parquet_common.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include +#include #include #include #include @@ -255,6 +257,32 @@ void read_footer(std::unique_ptr const& source, file_meta_data); } +void edit_footer(std::vector& buffer, + std::function const& edit) +{ + namespace pq_experimental = cudf::io::parquet::experimental; + + // parquet files end with 4-byte footer_length and 4-byte magic == "PAR1" + auto constexpr ender_size = 2 * sizeof(uint32_t); + CUDF_EXPECTS(buffer.size() >= ender_size, "Buffer is too small to be a Parquet file"); + uint32_t footer_size{}; + std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size)); + CUDF_EXPECTS(buffer.size() >= ender_size + footer_size, "Invalid Parquet footer length"); + auto const footer_begin = buffer.size() - ender_size - footer_size; + + auto file_meta_data = pq_experimental::read_parquet_footer_bytes( + {reinterpret_cast(buffer.data()) + footer_begin, footer_size}); + edit(file_meta_data); + auto const footer = pq_experimental::write_parquet_footer_bytes(file_meta_data); + + auto const new_footer_size = static_cast(footer.size()); + auto const size_bytes = reinterpret_cast(&new_footer_size); + buffer.resize(footer_begin); + buffer.insert(buffer.end(), footer.begin(), footer.end()); + buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size)); + buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); +} + // returns the number of bits used for dictionary encoding data at the given page location. // this assumes the data is uncompressed. // throws cudf::logic_error if the page_loc data is invalid. diff --git a/cpp/tests/io/parquet_common.hpp b/cpp/tests/io/parquet_common.hpp index 58472745ff6f..4f24b80720e8 100644 --- a/cpp/tests/io/parquet_common.hpp +++ b/cpp/tests/io/parquet_common.hpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -134,6 +135,18 @@ std::unique_ptr make_parquet_list_list_col( void read_footer(std::unique_ptr const& source, cudf::io::parquet::FileMetaData* file_meta_data); +/** + * @brief Rewrites the footer of an in-memory Parquet file + * + * Decodes the trailing footer of `buffer`, applies `edit` to it, and replaces it with the + * re-encoded footer, keeping the data pages as is. + * + * @param buffer Parquet file bytes to update in place + * @param edit Callable that modifies the decoded footer + */ +void edit_footer(std::vector& buffer, + std::function const& edit); + /** * @brief Returns the number of bits used for dictionary encoding the uncompressed page at * `page_loc` diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 566b9b49e644..6564d6d54b18 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -2794,25 +2793,18 @@ TEST_F(ParquetReaderTest, FileLogicalType) .metadata(std::move(input_metadata)) .build()); - auto const source = cudf::io::datasource::create(cudf::host_span{ - reinterpret_cast(buffer.data()), buffer.size()}); - auto fmd = cudf::io::read_parquet_footers({&source, 1}).front(); - ASSERT_EQ(fmd.schema[1].name, "file"); - ASSERT_EQ(fmd.schema[2].name, "offset"); - ASSERT_EQ(fmd.schema[3].name, "inline"); - fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; - // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE - fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; - - // Appends `fmd` as the new footer, leaving the original one unreferenced - auto const rewrite_file_footer = [&] { - auto const footer = cudf::io::parquet::experimental::write_parquet_footer_bytes(fmd); - auto const footer_size = static_cast(footer.size()); - auto const size_bytes = reinterpret_cast(&footer_size); - auto file = buffer; - file.insert(file.end(), footer.begin(), footer.end()); - file.insert(file.end(), size_bytes, size_bytes + sizeof(footer_size)); - file.insert(file.end(), {'P', 'A', 'R', '1'}); + // Returns a copy of `buffer` whose footer retags `file` as FILE and then applies `edit` + auto const rewrite_file_footer = [&](auto const& edit) { + auto file = buffer; + edit_footer(file, [&](cudf::io::parquet::FileMetaData& fmd) { + EXPECT_EQ(fmd.schema[1].name, "file"); + EXPECT_EQ(fmd.schema[2].name, "offset"); + EXPECT_EQ(fmd.schema[3].name, "inline"); + fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; + // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE + fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; + edit(fmd); + }); return file; }; auto const read_metadata = [](std::span const> files, @@ -2830,15 +2822,14 @@ TEST_F(ParquetReaderTest, FileLogicalType) .metadata; }; - auto const file_group = rewrite_file_footer(); + auto const file_group = rewrite_file_footer([](auto&) {}); auto const metadata = read_metadata(std::array{file_group}); EXPECT_EQ(metadata.schema_info[0].is_file, true); EXPECT_EQ(metadata.schema_info[0].children[1].is_binary, true); // A FILE group and a plain struct across sources throw, with or without mismatched schemas - fmd.schema[1].logical_type.reset(); - auto const plain_struct = rewrite_file_footer(); - fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; + auto const plain_struct = + rewrite_file_footer([](auto& fmd) { fmd.schema[1].logical_type.reset(); }); for (auto const allow_mismatched : {true, false}) { EXPECT_THROW(read_metadata(std::array{file_group, plain_struct}, allow_mismatched), std::logic_error); @@ -2847,101 +2838,9 @@ TEST_F(ParquetReaderTest, FileLogicalType) } // An annotated `inline` breaks the FILE spec, so the group is read as a plain struct - fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; - EXPECT_FALSE(read_metadata(std::array{rewrite_file_footer()}).schema_info[0].is_file.has_value()); -} - -// Binary leaves of FILE and VARIANT groups are read as `list` even when strings are -// converted to categories. cudf cannot write these groups yet, so retag a written struct group. -TEST_F(ParquetReaderTest, LogicalGroupBinaryLeavesNotCategorized) -{ - using cudf::io::parquet::LogicalType; - namespace pq_experimental = cudf::io::parquet::experimental; - - // Writes `str` (strings) and `group` (struct of `names`) and retags `group` as `logical_type` - auto const write_with_group_type = [](cudf::column_view group_col, - std::vector const& names, - cudf::column_view str_col, - LogicalType::Type logical_type) { - auto const input = table_view{{str_col, group_col}}; - cudf::io::table_input_metadata input_metadata(input); - input_metadata.column_metadata[0].set_name("str"); - auto& group_metadata = input_metadata.column_metadata[1].set_name("group"); - for (std::size_t i = 0; i < names.size(); ++i) { - group_metadata.child(i).set_name(names[i]).set_nullability(true); - if (group_col.child(i).type().id() == cudf::type_id::STRING and names[i] != "uri") { - group_metadata.child(i).set_output_as_binary(true); - } - } - - std::vector buffer; - cudf::io::write_parquet( - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input) - .metadata(std::move(input_metadata)) - .build()); - - auto constexpr ender_size = 2 * sizeof(uint32_t); - uint32_t footer_size{}; - std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size)); - auto const footer_begin = buffer.size() - ender_size - footer_size; - auto fmd = pq_experimental::read_parquet_footer_bytes( - {reinterpret_cast(buffer.data()) + footer_begin, footer_size}); - EXPECT_EQ(fmd.schema[2].name, "group"); - fmd.schema[2].logical_type = LogicalType{logical_type}; - - auto const footer = pq_experimental::write_parquet_footer_bytes(fmd); - auto const new_footer_size = static_cast(footer.size()); - auto const size_bytes = reinterpret_cast(&new_footer_size); - buffer.resize(footer_begin); - buffer.insert(buffer.end(), footer.begin(), footer.end()); - buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size)); - buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); - return buffer; - }; - - auto const read_categorized = [](std::vector const& file) { - auto const source = cudf::io::source_info{cudf::host_span{ - reinterpret_cast(file.data()), file.size()}}; - return cudf::io::read_parquet( - cudf::io::parquet_reader_options::builder(source).convert_strings_to_categories(true)); - }; - - cudf::test::strings_column_wrapper str_col{"x", "yy", "x", "zzz"}; - cudf::test::strings_column_wrapper bin_col{"a", "bc", "hello", "d"}; - cudf::test::lists_column_wrapper bin_bytes{ - {'a'}, {'b', 'c'}, {'h', 'e', 'l', 'l', 'o'}, {'d'}}; - - // FILE: `inline` is binary, while `uri` and the top-level strings are categorized - { - cudf::test::strings_column_wrapper uri_col{"s3://a", "s3://b", "", "s3://d"}; - auto file_col = cudf::test::structs_column_wrapper{{uri_col, bin_col}}; - auto const result = read_categorized( - write_with_group_type(file_col, {"uri", "inline"}, str_col, LogicalType::FILE)); - - auto const tbl = result.tbl->view(); - EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32); - EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT); - EXPECT_EQ(tbl.column(1).child(0).type().id(), cudf::type_id::INT32); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1)); - EXPECT_EQ(result.metadata.schema_info[1].is_file, true); - EXPECT_EQ(result.metadata.schema_info[1].children[1].is_binary, true); - } - - // VARIANT: `metadata` and `value` are binary, while the top-level strings are categorized - { - cudf::test::strings_column_wrapper metadata_col{"m0", "m1", "m2", "m3"}; - cudf::test::lists_column_wrapper metadata_bytes{ - {'m', '0'}, {'m', '1'}, {'m', '2'}, {'m', '3'}}; - auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, bin_col}}; - auto const result = read_categorized( - write_with_group_type(variant_col, {"metadata", "value"}, str_col, LogicalType::VARIANT)); - - auto const tbl = result.tbl->view(); - EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32); - EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(metadata_bytes, tbl.column(1).child(0)); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1)); - } + auto const annotated_inline = rewrite_file_footer( + [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; }); + EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value()); } TEST_F(ParquetReaderTest, NestingOptimizationTest) diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py index 2212cdbbc2f9..f9fef5f84431 100644 --- a/python/cudf/cudf/tests/input_output/test_parquet.py +++ b/python/cudf/cudf/tests/input_output/test_parquet.py @@ -774,6 +774,47 @@ def file_type(*names): assert_eq(cudf.DataFrame.from_arrow(expect), got) +@pytest.mark.parametrize( + "filename", + [ + "file_logical_type.parquet", + "variant_multirow.parquet", + "duckdb_variant_sample.parquet", + ], +) +def test_parquet_reader_logical_group_binary_not_categorized(datadir, filename): + # Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`) + # groups stay `list` when strings are converted to categories, while + # all other strings are read as INT32 hashes + import pylibcudf as plc + + def read_types(convert_strings_to_categories): + source = plc.io.SourceInfo([str(datadir / filename)]) + options = ( + plc.io.parquet.ParquetReaderOptions.builder(source) + .convert_strings_to_categories(convert_strings_to_categories) + .build() + ) + return plc.io.parquet.read_parquet(options).tbl.to_arrow().schema.types + + def categorized(typ): + if pa.types.is_string(typ) or pa.types.is_large_string(typ): + return pa.int32() + if pa.types.is_struct(typ): + return pa.struct([f.with_type(categorized(f.type)) for f in typ]) + if pa.types.is_list(typ): + return pa.list_( + typ.value_field.with_type(categorized(typ.value_type)) + ) + if pa.types.is_large_list(typ): + return pa.large_list( + typ.value_field.with_type(categorized(typ.value_type)) + ) + return typ + + assert read_types(True) == [categorized(t) for t in read_types(False)] + + @pytest.mark.parametrize("columns", [["a"], ["b", "a"], None]) def test_parquet_reader_decimal128(datadir, columns): fname = datadir / "nested_decimal128_file.parquet" From d8ffcf4295f6e70347bd20fa74f69d84a8d00f28 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:51:04 +0000 Subject: [PATCH 03/11] Fix ruff format --- python/cudf/cudf/tests/input_output/test_parquet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py index f9fef5f84431..b925db4f8bcc 100644 --- a/python/cudf/cudf/tests/input_output/test_parquet.py +++ b/python/cudf/cudf/tests/input_output/test_parquet.py @@ -782,7 +782,9 @@ def file_type(*names): "duckdb_variant_sample.parquet", ], ) -def test_parquet_reader_logical_group_binary_not_categorized(datadir, filename): +def test_parquet_reader_logical_group_binary_not_categorized( + datadir, filename +): # Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`) # groups stay `list` when strings are converted to categories, while # all other strings are read as INT32 hashes From 780c4400ce7a0494b9654e6d38b8dfb53c4bdb99 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:53:18 +0000 Subject: [PATCH 04/11] Reject mismatched logical group binary leaves across Parquet sources --- cpp/src/io/parquet/reader_impl_helpers.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 97ba236a9f62..7ac8a9bbfd9c 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -2323,6 +2323,15 @@ aggregate_reader_metadata::select_columns( src_schema_elem.name), std::invalid_argument); + // Binary leaves of logical groups (FILE, VARIANT) are read as binary, so they must be + // binary leaves in all sources + CUDF_EXPECTS( + is_binary_leaf(src_schema_idx) == is_binary_leaf(dst_schema_idx, src_idx), + std::format("Encountered mismatching binary leaf of a logical group across the Parquet " + "sources for column '{}'", + src_schema_elem.name), + std::invalid_argument); + // Get the schema_idx_map for this data source (pfm) auto& schema_idx_map = schema_idx_maps[src_idx - 1]; // Map the schema index from 0th tree (src) to the one in the current (dst) tree. From b554f28170689edbdffd1924780eb8c07a66204a Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:48:16 +0000 Subject: [PATCH 05/11] Reject mismatched logical group binary leaves in default multi-source reads --- cpp/src/io/parquet/reader_impl_helpers.cpp | 14 ++++--- cpp/tests/io/parquet_reader_test.cpp | 48 ++++++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 7ac8a9bbfd9c..162284c44390 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1043,26 +1043,30 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, // Validate that all sources have the same schema unless we are reading select columns // from mismatched sources, in which case, we will only check the projected columns later. if (not has_cols_from_mismatched_srcs) { - // `SchemaElement::operator==` ignores logical types, so compare FILE groups separately + // `SchemaElement::operator==` ignores logical types, so compare FILE groups and binary + // leaves of logical groups (FILE, VARIANT) separately auto const is_file_group = [](std::vector const& tree, size_t idx) { return is_valid_file_group( tree[idx], tree[idx].children_idx | std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; })); }; - auto const same_file_groups = [&](std::vector const& other) { + auto const same_logical_groups = [&](size_t pfm_idx) { + auto const& other = per_file_metadata[pfm_idx].schema; return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) { - return is_file_group(schema, idx) == is_file_group(other, idx); + return is_file_group(schema, idx) == is_file_group(other, idx) and + is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx); }); }; // Verify that the input files have matching numbers of columns and schema. - for (auto const& pfm : per_file_metadata) { + for (size_t pfm_idx = 0; pfm_idx < per_file_metadata.size(); ++pfm_idx) { + auto const& pfm = per_file_metadata[pfm_idx]; if (pfm.row_groups.size() > 0) { CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(), "All non-empty sources must have the same number of columns"); } - CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema), + CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm_idx), "All sources must have the same schema"); } diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 6564d6d54b18..1ad97b121729 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2843,6 +2843,54 @@ TEST_F(ParquetReaderTest, FileLogicalType) EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value()); } +TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources) +{ + // A VARIANT group and a plain struct with the same binary children across sources throw, with + // or without mismatched schemas, as only the VARIANT binary leaves skip category conversion + + cudf::test::strings_column_wrapper metadata_col{"m"}; + cudf::test::strings_column_wrapper value_col{"v"}; + auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, value_col}}; + + auto const input = table_view{{variant_col}}; + cudf::io::table_input_metadata input_metadata(input); + auto& variant_metadata = input_metadata.column_metadata[0].set_name("variant"); + variant_metadata.child(0).set_name("metadata").set_output_as_binary(true); + variant_metadata.child(1).set_name("value").set_output_as_binary(true); + + std::vector plain_struct; + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&plain_struct}, input) + .metadata(std::move(input_metadata)) + .build()); + + // cudf cannot write VARIANT groups yet, so retag the struct group as VARIANT in the footer + auto variant_group = plain_struct; + edit_footer(variant_group, [](cudf::io::parquet::FileMetaData& fmd) { + EXPECT_EQ(fmd.schema[1].name, "variant"); + fmd.schema[1].logical_type = + cudf::io::parquet::LogicalType{cudf::io::parquet::LogicalType::VARIANT}; + }); + + auto const read = [](std::span const> files, bool allow_mismatched) { + std::vector> sources; + for (auto const& file : files) { + sources.emplace_back(reinterpret_cast(file.data()), file.size()); + } + return cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span>{sources}}) + .allow_mismatched_pq_schemas(allow_mismatched) + .column_names({"variant"}) + .convert_strings_to_categories(true)); + }; + + for (auto const allow_mismatched : {true, false}) { + EXPECT_THROW(read(std::array{variant_group, plain_struct}, allow_mismatched), std::logic_error); + EXPECT_THROW(read(std::array{plain_struct, variant_group}, allow_mismatched), std::logic_error); + } +} + TEST_F(ParquetReaderTest, NestingOptimizationTest) { // test nesting levels > cudf::io::parquet::detail::max_cacheable_nesting_decode_info deep. From 8072627d1340f0e6d1de8d4bb0d9b412a521ec09 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:52:05 +0000 Subject: [PATCH 06/11] Use range-for in the multi-source logical group check --- cpp/src/io/parquet/reader_impl_helpers.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 162284c44390..d98b9363481d 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1051,22 +1051,21 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, tree[idx].children_idx | std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; })); }; - auto const same_logical_groups = [&](size_t pfm_idx) { - auto const& other = per_file_metadata[pfm_idx].schema; + auto const same_logical_groups = [&](metadata const& pfm) { + auto const pfm_idx = static_cast(std::distance(per_file_metadata.data(), &pfm)); return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) { - return is_file_group(schema, idx) == is_file_group(other, idx) and + return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx); }); }; // Verify that the input files have matching numbers of columns and schema. - for (size_t pfm_idx = 0; pfm_idx < per_file_metadata.size(); ++pfm_idx) { - auto const& pfm = per_file_metadata[pfm_idx]; + for (auto const& pfm : per_file_metadata) { if (pfm.row_groups.size() > 0) { CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(), "All non-empty sources must have the same number of columns"); } - CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm_idx), + CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm), "All sources must have the same schema"); } From 7cfa64ab04f4bc44dd907fa87c907899ac25f4a0 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:16:30 +0000 Subject: [PATCH 07/11] Use pointer subtraction for the source index --- cpp/src/io/parquet/reader_impl_helpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index d98b9363481d..84d035c33f41 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1052,7 +1052,7 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; })); }; auto const same_logical_groups = [&](metadata const& pfm) { - auto const pfm_idx = static_cast(std::distance(per_file_metadata.data(), &pfm)); + auto const pfm_idx = static_cast(&pfm - per_file_metadata.data()); return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) { return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx); From eb8e2963333aedbb153ff80fa1da6f2c1c69bc68 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:07:06 +0000 Subject: [PATCH 08/11] Use edit_footer in the BYTE_ARRAY decimal dictionary test --- cpp/tests/io/parquet_reader_test.cpp | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 1ad97b121729..231e9c4cf943 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -1828,7 +1828,7 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) .dictionary_policy(cudf::io::dictionary_policy::ALWAYS) .build()); - auto footer = + auto const footer = cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{ cudf::host_span{buffer.data(), buffer.size()}})) .front(); @@ -1838,24 +1838,14 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded) enc == cudf::io::parquet::Encoding::RLE_DICTIONARY; })); - auto& schema = footer.schema[1]; - schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; - schema.decimal_scale = scale; - schema.decimal_precision = precision; - schema.logical_type = cudf::io::parquet::LogicalType{ - cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; - - // Swap the trailing footer for the annotated one, keeping the data pages - uint32_t footer_len{}; - std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len)); - auto const new_footer = parquet_delta_test::serialize_footer(footer); - buffer.resize(buffer.size() - 8 - footer_len); - buffer.insert(buffer.end(), new_footer.begin(), new_footer.end()); - footer_len = new_footer.size(); - buffer.insert(buffer.end(), - reinterpret_cast(&footer_len), - reinterpret_cast(&footer_len + 1)); - buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); + edit_footer(buffer, [&](cudf::io::parquet::FileMetaData& fmd) { + auto& schema = fmd.schema[1]; + schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL; + schema.decimal_scale = scale; + schema.decimal_precision = precision; + schema.logical_type = cudf::io::parquet::LogicalType{ + cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}}; + }); auto const result = cudf::io::read_parquet( cudf::io::parquet_reader_options::builder( From aa1eee915368d73672bcc0f33ccaf6bf6340f065 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:17:38 +0000 Subject: [PATCH 09/11] Flatten FILE footer edits in the FileLogicalType test --- cpp/tests/io/parquet_reader_test.cpp | 38 ++++++++++++---------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 231e9c4cf943..268728637c47 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2777,26 +2777,20 @@ TEST_F(ParquetReaderTest, FileLogicalType) file_metadata.child(0).set_name("offset").set_nullability(true); file_metadata.child(1).set_name("inline").set_nullability(true).set_output_as_binary(true); - std::vector buffer; + std::vector file_group; cudf::io::write_parquet( - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input) + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&file_group}, input) .metadata(std::move(input_metadata)) .build()); + edit_footer(file_group, [](cudf::io::parquet::FileMetaData& fmd) { + EXPECT_EQ(fmd.schema[1].name, "file"); + EXPECT_EQ(fmd.schema[2].name, "offset"); + EXPECT_EQ(fmd.schema[3].name, "inline"); + fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; + // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE + fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; + }); - // Returns a copy of `buffer` whose footer retags `file` as FILE and then applies `edit` - auto const rewrite_file_footer = [&](auto const& edit) { - auto file = buffer; - edit_footer(file, [&](cudf::io::parquet::FileMetaData& fmd) { - EXPECT_EQ(fmd.schema[1].name, "file"); - EXPECT_EQ(fmd.schema[2].name, "offset"); - EXPECT_EQ(fmd.schema[3].name, "inline"); - fmd.schema[1].logical_type = LogicalType{LogicalType::FILE}; - // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE - fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}}; - edit(fmd); - }); - return file; - }; auto const read_metadata = [](std::span const> files, bool allow_mismatched = true) { std::vector> sources; @@ -2812,14 +2806,13 @@ TEST_F(ParquetReaderTest, FileLogicalType) .metadata; }; - auto const file_group = rewrite_file_footer([](auto&) {}); - auto const metadata = read_metadata(std::array{file_group}); + auto const metadata = read_metadata(std::array{file_group}); EXPECT_EQ(metadata.schema_info[0].is_file, true); EXPECT_EQ(metadata.schema_info[0].children[1].is_binary, true); // A FILE group and a plain struct across sources throw, with or without mismatched schemas - auto const plain_struct = - rewrite_file_footer([](auto& fmd) { fmd.schema[1].logical_type.reset(); }); + auto plain_struct = file_group; + edit_footer(plain_struct, [](auto& fmd) { fmd.schema[1].logical_type.reset(); }); for (auto const allow_mismatched : {true, false}) { EXPECT_THROW(read_metadata(std::array{file_group, plain_struct}, allow_mismatched), std::logic_error); @@ -2828,8 +2821,9 @@ TEST_F(ParquetReaderTest, FileLogicalType) } // An annotated `inline` breaks the FILE spec, so the group is read as a plain struct - auto const annotated_inline = rewrite_file_footer( - [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; }); + auto annotated_inline = file_group; + edit_footer(annotated_inline, + [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; }); EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value()); } From f8fe97761d66af86537b27329ec57f77ee4b3083 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:44:12 +0000 Subject: [PATCH 10/11] Test multi-source VARIANT bytes and nested VARIANT files with categories --- cpp/tests/io/parquet_reader_test.cpp | 15 ++++++++--- .../cudf/tests/input_output/test_parquet.py | 25 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 268728637c47..0cc9cdc3bb97 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2827,10 +2827,12 @@ TEST_F(ParquetReaderTest, FileLogicalType) EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value()); } -TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources) +TEST_F(ParquetReaderTest, VariantGroupsAcrossSources) { - // A VARIANT group and a plain struct with the same binary children across sources throw, with - // or without mismatched schemas, as only the VARIANT binary leaves skip category conversion + // VARIANT binary leaves read from multiple sources keep their bytes when strings are converted + // to categories. A VARIANT group and a plain struct with the same binary children across + // sources throw, with or without mismatched schemas, as only the VARIANT binary leaves skip + // category conversion cudf::test::strings_column_wrapper metadata_col{"m"}; cudf::test::strings_column_wrapper value_col{"v"}; @@ -2870,6 +2872,13 @@ TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources) }; for (auto const allow_mismatched : {true, false}) { + auto const result = read(std::array{variant_group, variant_group}, allow_mismatched); + auto const variant = result.tbl->view().column(0); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(0), + cudf::test::lists_column_wrapper{{'m'}, {'m'}}); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(1), + cudf::test::lists_column_wrapper{{'v'}, {'v'}}); + EXPECT_THROW(read(std::array{variant_group, plain_struct}, allow_mismatched), std::logic_error); EXPECT_THROW(read(std::array{plain_struct, variant_group}, allow_mismatched), std::logic_error); } diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py index b925db4f8bcc..38dc10976bd3 100644 --- a/python/cudf/cudf/tests/input_output/test_parquet.py +++ b/python/cudf/cudf/tests/input_output/test_parquet.py @@ -775,15 +775,28 @@ def file_type(*names): @pytest.mark.parametrize( - "filename", + "filename, columns", [ - "file_logical_type.parquet", - "variant_multirow.parquet", - "duckdb_variant_sample.parquet", + ("file_logical_type.parquet", None), + ("variant_multirow.parquet", None), + ("variant_nested_unshredded.parquet", None), + ("variant_nested_shredded.parquet", None), + # Skip the shredded `v.typed_value.{n,k}.value` leaves, which are still + # hashed as their parents aren't annotated VARIANT (see #25425) + ( + "duckdb_variant_sample.parquet", + [ + "id", + "v.metadata", + "v.value", + "v.typed_value.n.typed_value", + "v.typed_value.k.typed_value", + ], + ), ], ) def test_parquet_reader_logical_group_binary_not_categorized( - datadir, filename + datadir, filename, columns ): # Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`) # groups stay `list` when strings are converted to categories, while @@ -797,6 +810,8 @@ def read_types(convert_strings_to_categories): .convert_strings_to_categories(convert_strings_to_categories) .build() ) + if columns is not None: + options.set_column_names(columns) return plc.io.parquet.read_parquet(options).tbl.to_arrow().schema.types def categorized(typ): From ac3b79408a003c4e666a663aa32f64ad5d164af0 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:49:35 +0000 Subject: [PATCH 11/11] Drop the no-op binary leaf check in hybrid scan dictionaries --- .../io/parquet/hybrid_scan/hybrid_scan_preprocess.cu | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu index 8908a9036611..c1818b34c8bd 100644 --- a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu +++ b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu @@ -292,13 +292,8 @@ hybrid_scan_reader_impl::prepare_dictionaries( // look up metadata auto& col_meta = _extended_metadata->get_column_metadata(rg.index, rg.source_index, col_schema_idx); - auto const mapped_schema_idx = - _extended_metadata->map_schema_index(col_schema_idx, rg.source_index); - auto& schema = _extended_metadata->get_schema(mapped_schema_idx, rg.source_index); - // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories - auto const strings_to_categorical = - options.is_enabled_convert_strings_to_categories() and - not _extended_metadata->is_binary_leaf(mapped_schema_idx, rg.source_index); + auto& schema = _extended_metadata->get_schema( + _extended_metadata->map_schema_index(col_schema_idx, rg.source_index), rg.source_index); // dictionary data buffer for this column chunk auto& dict_page_data = dictionary_page_data[chunk_idx]; @@ -309,7 +304,7 @@ hybrid_scan_reader_impl::prepare_dictionaries( auto const [clock_rate, logical_type] = parquet::detail::conversion_info( parquet::detail::to_type_id(schema, - strings_to_categorical, + options.is_enabled_convert_strings_to_categories(), options.get_timestamp_type().id(), options.get_decimal_width()), options.get_timestamp_type().id(),