From c51c3c4ce663201c773693da439250555416f073 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Mon, 16 Nov 2026 18:47:41 +0000
Subject: [PATCH 01/11] Never categorize binary children of FILE and VARIANT
groups in the Parquet reader
---
.../hybrid_scan/hybrid_scan_preprocess.cu | 11 ++-
cpp/src/io/parquet/reader_impl_chunking.cu | 12 ++-
cpp/src/io/parquet/reader_impl_helpers.cpp | 46 +++++----
cpp/src/io/parquet/reader_impl_helpers.hpp | 13 +++
cpp/src/io/parquet/reader_impl_preprocess.cu | 19 ++--
cpp/tests/io/parquet_reader_test.cpp | 93 +++++++++++++++++++
6 files changed, 162 insertions(+), 32 deletions(-)
diff --git a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
index c1818b34c8bd..8908a9036611 100644
--- a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
+++ b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
@@ -292,8 +292,13 @@ hybrid_scan_reader_impl::prepare_dictionaries(
// look up metadata
auto& col_meta =
_extended_metadata->get_column_metadata(rg.index, rg.source_index, col_schema_idx);
- auto& schema = _extended_metadata->get_schema(
- _extended_metadata->map_schema_index(col_schema_idx, rg.source_index), rg.source_index);
+ auto const mapped_schema_idx =
+ _extended_metadata->map_schema_index(col_schema_idx, rg.source_index);
+ auto& schema = _extended_metadata->get_schema(mapped_schema_idx, rg.source_index);
+ // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
+ auto const strings_to_categorical =
+ options.is_enabled_convert_strings_to_categories() and
+ not _extended_metadata->is_binary_leaf(mapped_schema_idx, rg.source_index);
// dictionary data buffer for this column chunk
auto& dict_page_data = dictionary_page_data[chunk_idx];
@@ -304,7 +309,7 @@ hybrid_scan_reader_impl::prepare_dictionaries(
auto const [clock_rate, logical_type] = parquet::detail::conversion_info(
parquet::detail::to_type_id(schema,
- options.is_enabled_convert_strings_to_categories(),
+ strings_to_categorical,
options.get_timestamp_type().id(),
options.get_decimal_width()),
options.get_timestamp_type().id(),
diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu
index beb7605b55fa..1df89803be80 100644
--- a/cpp/src/io/parquet/reader_impl_chunking.cu
+++ b/cpp/src/io/parquet/reader_impl_chunking.cu
@@ -483,12 +483,16 @@ void reader_impl::create_global_chunk_info()
auto col = _input_columns[i];
// look up metadata
auto& col_meta = _metadata->get_column_metadata(rg.index, rg.source_index, col.schema_idx);
- auto& schema = _metadata->get_schema(
- _metadata->map_schema_index(col.schema_idx, rg.source_index), rg.source_index);
+ auto const mapped_schema_idx = _metadata->map_schema_index(col.schema_idx, rg.source_index);
+ auto& schema = _metadata->get_schema(mapped_schema_idx, rg.source_index);
+ // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
+ auto const strings_to_categorical =
+ _strings_to_categorical and
+ not _metadata->is_binary_leaf(mapped_schema_idx, rg.source_index);
auto [clock_rate, logical_type] = conversion_info(
to_type_id(
- schema, _strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width),
+ schema, strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width),
_options.timestamp_type.id(),
schema.type,
schema.logical_type);
@@ -524,7 +528,7 @@ void reader_impl::create_global_chunk_info()
col.schema_idx,
chunk_info,
list_bytes_per_row_est,
- schema.type == Type::BYTE_ARRAY and _strings_to_categorical,
+ schema.type == Type::BYTE_ARRAY and strings_to_categorical,
rg.source_index);
}
// Adjust for skip_rows when updating the remaining rows after the first group
diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp
index 3cfcb1f15c9e..97ba236a9f62 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.cpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.cpp
@@ -1617,6 +1617,25 @@ std::unique_ptr
aggregate_reader_metadata::read_column_chunk_bounds(
return std::make_unique(std::move(columns));
}
+bool aggregate_reader_metadata::is_binary_leaf(int schema_idx, int pfm_idx) const
+{
+ auto const& schema_elem = get_schema(schema_idx, pfm_idx);
+ if (schema_elem.type != Type::BYTE_ARRAY or schema_elem.num_children != 0) { return false; }
+ auto const& parent = get_schema(schema_elem.parent_idx, pfm_idx);
+ if (not parent.logical_type.has_value()) { return false; }
+ switch (parent.logical_type->type) {
+ case LogicalType::VARIANT: return schema_elem.name == "metadata" or schema_elem.name == "value";
+ case LogicalType::FILE:
+ return schema_elem.name == "inline" and
+ is_valid_file_group(
+ parent,
+ parent.children_idx | std::views::transform([&](int idx) -> SchemaElement const& {
+ return get_schema(idx, pfm_idx);
+ }));
+ default: return false;
+ }
+}
+
bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const
{
// Check if schema_idx or src_idx is invalid
@@ -2148,12 +2167,15 @@ aggregate_reader_metadata::select_columns(
auto const& parent_schema = get_schema(schema_elem.parent_idx);
auto const one_level_list = schema_elem.is_one_level_list(parent_schema);
+ // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
+ auto const binary_leaf = is_binary_leaf(schema_idx);
+ auto const leaf_to_categorical = strings_to_categorical and not binary_leaf;
// if we're at the root, this is a new output column
auto const col_type =
one_level_list
? type_id::LIST
- : to_type_id(schema_elem, strings_to_categorical, timestamp_type_id, decimal_type_id);
+ : to_type_id(schema_elem, leaf_to_categorical, timestamp_type_id, decimal_type_id);
auto const dtype = to_data_type(col_type, schema_elem);
cudf::io::detail::inline_column_buffer output_col(
@@ -2193,15 +2215,6 @@ aggregate_reader_metadata::select_columns(
}
}
- // Flag the `inline` child of a valid FILE group so that `make_column` materializes it as
- // `list` instead of strings.
- if (output_col.is_file) {
- auto const inline_col = std::ranges::find(output_col.children,
- std::string_view{"inline"},
- &cudf::io::detail::inline_column_buffer::name);
- if (inline_col != output_col.children.end()) { inline_col->string_as_binary = true; }
- }
-
// if I have no children, we're at a leaf and I'm an input column (that is, one with actual
// data stored) so add me to the list.
if (schema_elem.num_children == 0) {
@@ -2212,7 +2225,7 @@ aggregate_reader_metadata::select_columns(
if (one_level_list) {
// determine the element data type
auto const element_type =
- to_type_id(schema_elem, strings_to_categorical, timestamp_type_id, decimal_type_id);
+ to_type_id(schema_elem, leaf_to_categorical, timestamp_type_id, decimal_type_id);
auto const element_dtype = to_data_type(element_type, schema_elem);
cudf::io::detail::inline_column_buffer element_col(
@@ -2235,13 +2248,10 @@ aggregate_reader_metadata::select_columns(
// pop off the extra nesting element.
if (one_level_list) { nesting.pop_back(); }
- // Flag the `metadata` / `value` BYTE_ARRAY children of a VARIANT group so that
- // `make_column` materializes them as `list` instead of strings.
- if (schema_elem.type == Type::BYTE_ARRAY && parent_schema.logical_type.has_value() &&
- parent_schema.logical_type->type == LogicalType::VARIANT &&
- (schema_elem.name == "metadata" || schema_elem.name == "value")) {
- output_col.string_as_binary = true;
- }
+ // Flag binary leaves of logical groups (the `inline` child of a valid FILE group and the
+ // `metadata` / `value` children of a VARIANT group) so that `make_column` materializes
+ // them as `list` instead of strings.
+ if (binary_leaf) { output_col.string_as_binary = true; }
path_is_valid = true; // If we're able to reach leaf then path is valid
}
diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp
index a1796605e195..1a00d662c005 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.hpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.hpp
@@ -717,6 +717,19 @@ class aggregate_reader_metadata {
return nullable_across_sources.contains(schema_idx);
}
+ /**
+ * @brief Checks if a schema element is a binary BYTE_ARRAY leaf of a logical group, i.e. the
+ * `inline` child of a valid FILE group or the `metadata` / `value` child of a VARIANT group
+ *
+ * Such leaves are always read as `list` and are never converted to categories.
+ *
+ * @param schema_idx The index of the SchemaElement in the pfm_idx'th file
+ * @param pfm_idx The index of the per_file_metadata, default = 0 if not specified
+ *
+ * @return True if the schema element is a binary leaf of a logical group
+ */
+ [[nodiscard]] bool is_binary_leaf(int schema_idx, int pfm_idx = 0) const;
+
/**
* @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file
*
diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu
index a0463b00694c..666fdf611898 100644
--- a/cpp/src/io/parquet/reader_impl_preprocess.cu
+++ b/cpp/src/io/parquet/reader_impl_preprocess.cu
@@ -279,16 +279,18 @@ void reader_impl::allocate_nesting_info()
}
// Get the schema from the current input source.
- auto& actual_cur_schema = _metadata->get_schema(
- _metadata->map_schema_index(schema_idx, src_file_idx), src_file_idx);
+ auto const actual_cur_schema_idx = _metadata->map_schema_index(schema_idx, src_file_idx);
+ auto& actual_cur_schema = _metadata->get_schema(actual_cur_schema_idx, src_file_idx);
// values indexed by output column index
nesting_info[cur_depth].max_def_level = actual_cur_schema.max_definition_level;
pni[cur_depth].size = 0;
- pni[cur_depth].type = to_type_id(actual_cur_schema,
- _strings_to_categorical,
- _options.timestamp_type.id(),
- _options.decimal_width);
+ pni[cur_depth].type =
+ to_type_id(actual_cur_schema,
+ _strings_to_categorical and
+ not _metadata->is_binary_leaf(actual_cur_schema_idx, src_file_idx),
+ _options.timestamp_type.id(),
+ _options.decimal_width);
pni[cur_depth].nullable = cur_schema.repetition_type == FieldRepetitionType::OPTIONAL;
// the implicit list of a one-level list is a required list with its parent's def level
@@ -770,7 +772,10 @@ void reader_impl::preprocess_file(read_mode mode)
for (size_t idx = 0; idx < _input_columns.size(); idx++) {
auto const& schema = _metadata->get_schema(_input_columns[idx].schema_idx);
auto const type_id = to_type_id(
- schema, _strings_to_categorical, _options.timestamp_type.id(), _options.decimal_width);
+ schema,
+ _strings_to_categorical and not _metadata->is_binary_leaf(_input_columns[idx].schema_idx),
+ _options.timestamp_type.id(),
+ _options.decimal_width);
printf("\tC(%'lu, %s): %s\n",
idx,
_input_columns[idx].name.c_str(),
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 9cd3267b67be..566b9b49e644 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -2851,6 +2851,99 @@ TEST_F(ParquetReaderTest, FileLogicalType)
EXPECT_FALSE(read_metadata(std::array{rewrite_file_footer()}).schema_info[0].is_file.has_value());
}
+// Binary leaves of FILE and VARIANT groups are read as `list` even when strings are
+// converted to categories. cudf cannot write these groups yet, so retag a written struct group.
+TEST_F(ParquetReaderTest, LogicalGroupBinaryLeavesNotCategorized)
+{
+ using cudf::io::parquet::LogicalType;
+ namespace pq_experimental = cudf::io::parquet::experimental;
+
+ // Writes `str` (strings) and `group` (struct of `names`) and retags `group` as `logical_type`
+ auto const write_with_group_type = [](cudf::column_view group_col,
+ std::vector const& names,
+ cudf::column_view str_col,
+ LogicalType::Type logical_type) {
+ auto const input = table_view{{str_col, group_col}};
+ cudf::io::table_input_metadata input_metadata(input);
+ input_metadata.column_metadata[0].set_name("str");
+ auto& group_metadata = input_metadata.column_metadata[1].set_name("group");
+ for (std::size_t i = 0; i < names.size(); ++i) {
+ group_metadata.child(i).set_name(names[i]).set_nullability(true);
+ if (group_col.child(i).type().id() == cudf::type_id::STRING and names[i] != "uri") {
+ group_metadata.child(i).set_output_as_binary(true);
+ }
+ }
+
+ std::vector buffer;
+ cudf::io::write_parquet(
+ cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input)
+ .metadata(std::move(input_metadata))
+ .build());
+
+ auto constexpr ender_size = 2 * sizeof(uint32_t);
+ uint32_t footer_size{};
+ std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size));
+ auto const footer_begin = buffer.size() - ender_size - footer_size;
+ auto fmd = pq_experimental::read_parquet_footer_bytes(
+ {reinterpret_cast(buffer.data()) + footer_begin, footer_size});
+ EXPECT_EQ(fmd.schema[2].name, "group");
+ fmd.schema[2].logical_type = LogicalType{logical_type};
+
+ auto const footer = pq_experimental::write_parquet_footer_bytes(fmd);
+ auto const new_footer_size = static_cast(footer.size());
+ auto const size_bytes = reinterpret_cast(&new_footer_size);
+ buffer.resize(footer_begin);
+ buffer.insert(buffer.end(), footer.begin(), footer.end());
+ buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size));
+ buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
+ return buffer;
+ };
+
+ auto const read_categorized = [](std::vector const& file) {
+ auto const source = cudf::io::source_info{cudf::host_span{
+ reinterpret_cast(file.data()), file.size()}};
+ return cudf::io::read_parquet(
+ cudf::io::parquet_reader_options::builder(source).convert_strings_to_categories(true));
+ };
+
+ cudf::test::strings_column_wrapper str_col{"x", "yy", "x", "zzz"};
+ cudf::test::strings_column_wrapper bin_col{"a", "bc", "hello", "d"};
+ cudf::test::lists_column_wrapper bin_bytes{
+ {'a'}, {'b', 'c'}, {'h', 'e', 'l', 'l', 'o'}, {'d'}};
+
+ // FILE: `inline` is binary, while `uri` and the top-level strings are categorized
+ {
+ cudf::test::strings_column_wrapper uri_col{"s3://a", "s3://b", "", "s3://d"};
+ auto file_col = cudf::test::structs_column_wrapper{{uri_col, bin_col}};
+ auto const result = read_categorized(
+ write_with_group_type(file_col, {"uri", "inline"}, str_col, LogicalType::FILE));
+
+ auto const tbl = result.tbl->view();
+ EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32);
+ EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT);
+ EXPECT_EQ(tbl.column(1).child(0).type().id(), cudf::type_id::INT32);
+ CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1));
+ EXPECT_EQ(result.metadata.schema_info[1].is_file, true);
+ EXPECT_EQ(result.metadata.schema_info[1].children[1].is_binary, true);
+ }
+
+ // VARIANT: `metadata` and `value` are binary, while the top-level strings are categorized
+ {
+ cudf::test::strings_column_wrapper metadata_col{"m0", "m1", "m2", "m3"};
+ cudf::test::lists_column_wrapper metadata_bytes{
+ {'m', '0'}, {'m', '1'}, {'m', '2'}, {'m', '3'}};
+ auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, bin_col}};
+ auto const result = read_categorized(
+ write_with_group_type(variant_col, {"metadata", "value"}, str_col, LogicalType::VARIANT));
+
+ auto const tbl = result.tbl->view();
+ EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32);
+ EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT);
+ CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(metadata_bytes, tbl.column(1).child(0));
+ CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1));
+ }
+}
+
TEST_F(ParquetReaderTest, NestingOptimizationTest)
{
// test nesting levels > cudf::io::parquet::detail::max_cacheable_nesting_decode_info deep.
From 562ef5739e5ef6aaaf31fd3b8f45b1696d19f413 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 01:47:29 +0000
Subject: [PATCH 02/11] Add edit_footer test helper and move the categorization
test to Python
---
cpp/tests/io/parquet_common.cpp | 28 ++++
cpp/tests/io/parquet_common.hpp | 13 ++
cpp/tests/io/parquet_reader_test.cpp | 137 +++---------------
.../cudf/tests/input_output/test_parquet.py | 41 ++++++
4 files changed, 100 insertions(+), 119 deletions(-)
diff --git a/cpp/tests/io/parquet_common.cpp b/cpp/tests/io/parquet_common.cpp
index 14bb9b2b869e..4139d0d0e9d3 100644
--- a/cpp/tests/io/parquet_common.cpp
+++ b/cpp/tests/io/parquet_common.cpp
@@ -9,6 +9,7 @@
#include
#include
+#include
#include
#include
#include
@@ -20,6 +21,7 @@
#include
#include
+#include
#include
#include
#include
@@ -255,6 +257,32 @@ void read_footer(std::unique_ptr const& source,
file_meta_data);
}
+void edit_footer(std::vector& buffer,
+ std::function const& edit)
+{
+ namespace pq_experimental = cudf::io::parquet::experimental;
+
+ // parquet files end with 4-byte footer_length and 4-byte magic == "PAR1"
+ auto constexpr ender_size = 2 * sizeof(uint32_t);
+ CUDF_EXPECTS(buffer.size() >= ender_size, "Buffer is too small to be a Parquet file");
+ uint32_t footer_size{};
+ std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size));
+ CUDF_EXPECTS(buffer.size() >= ender_size + footer_size, "Invalid Parquet footer length");
+ auto const footer_begin = buffer.size() - ender_size - footer_size;
+
+ auto file_meta_data = pq_experimental::read_parquet_footer_bytes(
+ {reinterpret_cast(buffer.data()) + footer_begin, footer_size});
+ edit(file_meta_data);
+ auto const footer = pq_experimental::write_parquet_footer_bytes(file_meta_data);
+
+ auto const new_footer_size = static_cast(footer.size());
+ auto const size_bytes = reinterpret_cast(&new_footer_size);
+ buffer.resize(footer_begin);
+ buffer.insert(buffer.end(), footer.begin(), footer.end());
+ buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size));
+ buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
+}
+
// returns the number of bits used for dictionary encoding data at the given page location.
// this assumes the data is uncompressed.
// throws cudf::logic_error if the page_loc data is invalid.
diff --git a/cpp/tests/io/parquet_common.hpp b/cpp/tests/io/parquet_common.hpp
index 58472745ff6f..4f24b80720e8 100644
--- a/cpp/tests/io/parquet_common.hpp
+++ b/cpp/tests/io/parquet_common.hpp
@@ -19,6 +19,7 @@
#include
+#include
#include
#include
#include
@@ -134,6 +135,18 @@ std::unique_ptr make_parquet_list_list_col(
void read_footer(std::unique_ptr const& source,
cudf::io::parquet::FileMetaData* file_meta_data);
+/**
+ * @brief Rewrites the footer of an in-memory Parquet file
+ *
+ * Decodes the trailing footer of `buffer`, applies `edit` to it, and replaces it with the
+ * re-encoded footer, keeping the data pages as is.
+ *
+ * @param buffer Parquet file bytes to update in place
+ * @param edit Callable that modifies the decoded footer
+ */
+void edit_footer(std::vector& buffer,
+ std::function const& edit);
+
/**
* @brief Returns the number of bits used for dictionary encoding the uncompressed page at
* `page_loc`
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 566b9b49e644..6564d6d54b18 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -20,7 +20,6 @@
#include
#include
#include
-#include
#include
#include
#include
@@ -2794,25 +2793,18 @@ TEST_F(ParquetReaderTest, FileLogicalType)
.metadata(std::move(input_metadata))
.build());
- auto const source = cudf::io::datasource::create(cudf::host_span{
- reinterpret_cast(buffer.data()), buffer.size()});
- auto fmd = cudf::io::read_parquet_footers({&source, 1}).front();
- ASSERT_EQ(fmd.schema[1].name, "file");
- ASSERT_EQ(fmd.schema[2].name, "offset");
- ASSERT_EQ(fmd.schema[3].name, "inline");
- fmd.schema[1].logical_type = LogicalType{LogicalType::FILE};
- // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE
- fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}};
-
- // Appends `fmd` as the new footer, leaving the original one unreferenced
- auto const rewrite_file_footer = [&] {
- auto const footer = cudf::io::parquet::experimental::write_parquet_footer_bytes(fmd);
- auto const footer_size = static_cast(footer.size());
- auto const size_bytes = reinterpret_cast(&footer_size);
- auto file = buffer;
- file.insert(file.end(), footer.begin(), footer.end());
- file.insert(file.end(), size_bytes, size_bytes + sizeof(footer_size));
- file.insert(file.end(), {'P', 'A', 'R', '1'});
+ // Returns a copy of `buffer` whose footer retags `file` as FILE and then applies `edit`
+ auto const rewrite_file_footer = [&](auto const& edit) {
+ auto file = buffer;
+ edit_footer(file, [&](cudf::io::parquet::FileMetaData& fmd) {
+ EXPECT_EQ(fmd.schema[1].name, "file");
+ EXPECT_EQ(fmd.schema[2].name, "offset");
+ EXPECT_EQ(fmd.schema[3].name, "inline");
+ fmd.schema[1].logical_type = LogicalType{LogicalType::FILE};
+ // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE
+ fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}};
+ edit(fmd);
+ });
return file;
};
auto const read_metadata = [](std::span const> files,
@@ -2830,15 +2822,14 @@ TEST_F(ParquetReaderTest, FileLogicalType)
.metadata;
};
- auto const file_group = rewrite_file_footer();
+ auto const file_group = rewrite_file_footer([](auto&) {});
auto const metadata = read_metadata(std::array{file_group});
EXPECT_EQ(metadata.schema_info[0].is_file, true);
EXPECT_EQ(metadata.schema_info[0].children[1].is_binary, true);
// A FILE group and a plain struct across sources throw, with or without mismatched schemas
- fmd.schema[1].logical_type.reset();
- auto const plain_struct = rewrite_file_footer();
- fmd.schema[1].logical_type = LogicalType{LogicalType::FILE};
+ auto const plain_struct =
+ rewrite_file_footer([](auto& fmd) { fmd.schema[1].logical_type.reset(); });
for (auto const allow_mismatched : {true, false}) {
EXPECT_THROW(read_metadata(std::array{file_group, plain_struct}, allow_mismatched),
std::logic_error);
@@ -2847,101 +2838,9 @@ TEST_F(ParquetReaderTest, FileLogicalType)
}
// An annotated `inline` breaks the FILE spec, so the group is read as a plain struct
- fmd.schema[3].logical_type = LogicalType{LogicalType::STRING};
- EXPECT_FALSE(read_metadata(std::array{rewrite_file_footer()}).schema_info[0].is_file.has_value());
-}
-
-// Binary leaves of FILE and VARIANT groups are read as `list` even when strings are
-// converted to categories. cudf cannot write these groups yet, so retag a written struct group.
-TEST_F(ParquetReaderTest, LogicalGroupBinaryLeavesNotCategorized)
-{
- using cudf::io::parquet::LogicalType;
- namespace pq_experimental = cudf::io::parquet::experimental;
-
- // Writes `str` (strings) and `group` (struct of `names`) and retags `group` as `logical_type`
- auto const write_with_group_type = [](cudf::column_view group_col,
- std::vector const& names,
- cudf::column_view str_col,
- LogicalType::Type logical_type) {
- auto const input = table_view{{str_col, group_col}};
- cudf::io::table_input_metadata input_metadata(input);
- input_metadata.column_metadata[0].set_name("str");
- auto& group_metadata = input_metadata.column_metadata[1].set_name("group");
- for (std::size_t i = 0; i < names.size(); ++i) {
- group_metadata.child(i).set_name(names[i]).set_nullability(true);
- if (group_col.child(i).type().id() == cudf::type_id::STRING and names[i] != "uri") {
- group_metadata.child(i).set_output_as_binary(true);
- }
- }
-
- std::vector buffer;
- cudf::io::write_parquet(
- cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input)
- .metadata(std::move(input_metadata))
- .build());
-
- auto constexpr ender_size = 2 * sizeof(uint32_t);
- uint32_t footer_size{};
- std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size));
- auto const footer_begin = buffer.size() - ender_size - footer_size;
- auto fmd = pq_experimental::read_parquet_footer_bytes(
- {reinterpret_cast(buffer.data()) + footer_begin, footer_size});
- EXPECT_EQ(fmd.schema[2].name, "group");
- fmd.schema[2].logical_type = LogicalType{logical_type};
-
- auto const footer = pq_experimental::write_parquet_footer_bytes(fmd);
- auto const new_footer_size = static_cast(footer.size());
- auto const size_bytes = reinterpret_cast(&new_footer_size);
- buffer.resize(footer_begin);
- buffer.insert(buffer.end(), footer.begin(), footer.end());
- buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size));
- buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
- return buffer;
- };
-
- auto const read_categorized = [](std::vector const& file) {
- auto const source = cudf::io::source_info{cudf::host_span{
- reinterpret_cast(file.data()), file.size()}};
- return cudf::io::read_parquet(
- cudf::io::parquet_reader_options::builder(source).convert_strings_to_categories(true));
- };
-
- cudf::test::strings_column_wrapper str_col{"x", "yy", "x", "zzz"};
- cudf::test::strings_column_wrapper bin_col{"a", "bc", "hello", "d"};
- cudf::test::lists_column_wrapper bin_bytes{
- {'a'}, {'b', 'c'}, {'h', 'e', 'l', 'l', 'o'}, {'d'}};
-
- // FILE: `inline` is binary, while `uri` and the top-level strings are categorized
- {
- cudf::test::strings_column_wrapper uri_col{"s3://a", "s3://b", "", "s3://d"};
- auto file_col = cudf::test::structs_column_wrapper{{uri_col, bin_col}};
- auto const result = read_categorized(
- write_with_group_type(file_col, {"uri", "inline"}, str_col, LogicalType::FILE));
-
- auto const tbl = result.tbl->view();
- EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32);
- EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT);
- EXPECT_EQ(tbl.column(1).child(0).type().id(), cudf::type_id::INT32);
- CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1));
- EXPECT_EQ(result.metadata.schema_info[1].is_file, true);
- EXPECT_EQ(result.metadata.schema_info[1].children[1].is_binary, true);
- }
-
- // VARIANT: `metadata` and `value` are binary, while the top-level strings are categorized
- {
- cudf::test::strings_column_wrapper metadata_col{"m0", "m1", "m2", "m3"};
- cudf::test::lists_column_wrapper metadata_bytes{
- {'m', '0'}, {'m', '1'}, {'m', '2'}, {'m', '3'}};
- auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, bin_col}};
- auto const result = read_categorized(
- write_with_group_type(variant_col, {"metadata", "value"}, str_col, LogicalType::VARIANT));
-
- auto const tbl = result.tbl->view();
- EXPECT_EQ(tbl.column(0).type().id(), cudf::type_id::INT32);
- EXPECT_EQ(tbl.column(1).type().id(), cudf::type_id::STRUCT);
- CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(metadata_bytes, tbl.column(1).child(0));
- CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(bin_bytes, tbl.column(1).child(1));
- }
+ auto const annotated_inline = rewrite_file_footer(
+ [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; });
+ EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value());
}
TEST_F(ParquetReaderTest, NestingOptimizationTest)
diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py
index 2212cdbbc2f9..f9fef5f84431 100644
--- a/python/cudf/cudf/tests/input_output/test_parquet.py
+++ b/python/cudf/cudf/tests/input_output/test_parquet.py
@@ -774,6 +774,47 @@ def file_type(*names):
assert_eq(cudf.DataFrame.from_arrow(expect), got)
+@pytest.mark.parametrize(
+ "filename",
+ [
+ "file_logical_type.parquet",
+ "variant_multirow.parquet",
+ "duckdb_variant_sample.parquet",
+ ],
+)
+def test_parquet_reader_logical_group_binary_not_categorized(datadir, filename):
+ # Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`)
+ # groups stay `list` when strings are converted to categories, while
+ # all other strings are read as INT32 hashes
+ import pylibcudf as plc
+
+ def read_types(convert_strings_to_categories):
+ source = plc.io.SourceInfo([str(datadir / filename)])
+ options = (
+ plc.io.parquet.ParquetReaderOptions.builder(source)
+ .convert_strings_to_categories(convert_strings_to_categories)
+ .build()
+ )
+ return plc.io.parquet.read_parquet(options).tbl.to_arrow().schema.types
+
+ def categorized(typ):
+ if pa.types.is_string(typ) or pa.types.is_large_string(typ):
+ return pa.int32()
+ if pa.types.is_struct(typ):
+ return pa.struct([f.with_type(categorized(f.type)) for f in typ])
+ if pa.types.is_list(typ):
+ return pa.list_(
+ typ.value_field.with_type(categorized(typ.value_type))
+ )
+ if pa.types.is_large_list(typ):
+ return pa.large_list(
+ typ.value_field.with_type(categorized(typ.value_type))
+ )
+ return typ
+
+ assert read_types(True) == [categorized(t) for t in read_types(False)]
+
+
@pytest.mark.parametrize("columns", [["a"], ["b", "a"], None])
def test_parquet_reader_decimal128(datadir, columns):
fname = datadir / "nested_decimal128_file.parquet"
From d8ffcf4295f6e70347bd20fa74f69d84a8d00f28 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 01:51:04 +0000
Subject: [PATCH 03/11] Fix ruff format
---
python/cudf/cudf/tests/input_output/test_parquet.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py
index f9fef5f84431..b925db4f8bcc 100644
--- a/python/cudf/cudf/tests/input_output/test_parquet.py
+++ b/python/cudf/cudf/tests/input_output/test_parquet.py
@@ -782,7 +782,9 @@ def file_type(*names):
"duckdb_variant_sample.parquet",
],
)
-def test_parquet_reader_logical_group_binary_not_categorized(datadir, filename):
+def test_parquet_reader_logical_group_binary_not_categorized(
+ datadir, filename
+):
# Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`)
# groups stay `list` when strings are converted to categories, while
# all other strings are read as INT32 hashes
From 780c4400ce7a0494b9654e6d38b8dfb53c4bdb99 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 19:53:18 +0000
Subject: [PATCH 04/11] Reject mismatched logical group binary leaves across
Parquet sources
---
cpp/src/io/parquet/reader_impl_helpers.cpp | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp
index 97ba236a9f62..7ac8a9bbfd9c 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.cpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.cpp
@@ -2323,6 +2323,15 @@ aggregate_reader_metadata::select_columns(
src_schema_elem.name),
std::invalid_argument);
+ // Binary leaves of logical groups (FILE, VARIANT) are read as binary, so they must be
+ // binary leaves in all sources
+ CUDF_EXPECTS(
+ is_binary_leaf(src_schema_idx) == is_binary_leaf(dst_schema_idx, src_idx),
+ std::format("Encountered mismatching binary leaf of a logical group across the Parquet "
+ "sources for column '{}'",
+ src_schema_elem.name),
+ std::invalid_argument);
+
// Get the schema_idx_map for this data source (pfm)
auto& schema_idx_map = schema_idx_maps[src_idx - 1];
// Map the schema index from 0th tree (src) to the one in the current (dst) tree.
From b554f28170689edbdffd1924780eb8c07a66204a Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 21:48:16 +0000
Subject: [PATCH 05/11] Reject mismatched logical group binary leaves in
default multi-source reads
---
cpp/src/io/parquet/reader_impl_helpers.cpp | 14 ++++---
cpp/tests/io/parquet_reader_test.cpp | 48 ++++++++++++++++++++++
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp
index 7ac8a9bbfd9c..162284c44390 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.cpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.cpp
@@ -1043,26 +1043,30 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema,
// Validate that all sources have the same schema unless we are reading select columns
// from mismatched sources, in which case, we will only check the projected columns later.
if (not has_cols_from_mismatched_srcs) {
- // `SchemaElement::operator==` ignores logical types, so compare FILE groups separately
+ // `SchemaElement::operator==` ignores logical types, so compare FILE groups and binary
+ // leaves of logical groups (FILE, VARIANT) separately
auto const is_file_group = [](std::vector const& tree, size_t idx) {
return is_valid_file_group(
tree[idx],
tree[idx].children_idx |
std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; }));
};
- auto const same_file_groups = [&](std::vector const& other) {
+ auto const same_logical_groups = [&](size_t pfm_idx) {
+ auto const& other = per_file_metadata[pfm_idx].schema;
return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) {
- return is_file_group(schema, idx) == is_file_group(other, idx);
+ return is_file_group(schema, idx) == is_file_group(other, idx) and
+ is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx);
});
};
// Verify that the input files have matching numbers of columns and schema.
- for (auto const& pfm : per_file_metadata) {
+ for (size_t pfm_idx = 0; pfm_idx < per_file_metadata.size(); ++pfm_idx) {
+ auto const& pfm = per_file_metadata[pfm_idx];
if (pfm.row_groups.size() > 0) {
CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(),
"All non-empty sources must have the same number of columns");
}
- CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema),
+ CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm_idx),
"All sources must have the same schema");
}
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 6564d6d54b18..1ad97b121729 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -2843,6 +2843,54 @@ TEST_F(ParquetReaderTest, FileLogicalType)
EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value());
}
+TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources)
+{
+ // A VARIANT group and a plain struct with the same binary children across sources throw, with
+ // or without mismatched schemas, as only the VARIANT binary leaves skip category conversion
+
+ cudf::test::strings_column_wrapper metadata_col{"m"};
+ cudf::test::strings_column_wrapper value_col{"v"};
+ auto variant_col = cudf::test::structs_column_wrapper{{metadata_col, value_col}};
+
+ auto const input = table_view{{variant_col}};
+ cudf::io::table_input_metadata input_metadata(input);
+ auto& variant_metadata = input_metadata.column_metadata[0].set_name("variant");
+ variant_metadata.child(0).set_name("metadata").set_output_as_binary(true);
+ variant_metadata.child(1).set_name("value").set_output_as_binary(true);
+
+ std::vector plain_struct;
+ cudf::io::write_parquet(
+ cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&plain_struct}, input)
+ .metadata(std::move(input_metadata))
+ .build());
+
+ // cudf cannot write VARIANT groups yet, so retag the struct group as VARIANT in the footer
+ auto variant_group = plain_struct;
+ edit_footer(variant_group, [](cudf::io::parquet::FileMetaData& fmd) {
+ EXPECT_EQ(fmd.schema[1].name, "variant");
+ fmd.schema[1].logical_type =
+ cudf::io::parquet::LogicalType{cudf::io::parquet::LogicalType::VARIANT};
+ });
+
+ auto const read = [](std::span const> files, bool allow_mismatched) {
+ std::vector> sources;
+ for (auto const& file : files) {
+ sources.emplace_back(reinterpret_cast(file.data()), file.size());
+ }
+ return cudf::io::read_parquet(
+ cudf::io::parquet_reader_options::builder(
+ cudf::io::source_info{cudf::host_span>{sources}})
+ .allow_mismatched_pq_schemas(allow_mismatched)
+ .column_names({"variant"})
+ .convert_strings_to_categories(true));
+ };
+
+ for (auto const allow_mismatched : {true, false}) {
+ EXPECT_THROW(read(std::array{variant_group, plain_struct}, allow_mismatched), std::logic_error);
+ EXPECT_THROW(read(std::array{plain_struct, variant_group}, allow_mismatched), std::logic_error);
+ }
+}
+
TEST_F(ParquetReaderTest, NestingOptimizationTest)
{
// test nesting levels > cudf::io::parquet::detail::max_cacheable_nesting_decode_info deep.
From 8072627d1340f0e6d1de8d4bb0d9b412a521ec09 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 21:52:05 +0000
Subject: [PATCH 06/11] Use range-for in the multi-source logical group check
---
cpp/src/io/parquet/reader_impl_helpers.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp
index 162284c44390..d98b9363481d 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.cpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.cpp
@@ -1051,22 +1051,21 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema,
tree[idx].children_idx |
std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; }));
};
- auto const same_logical_groups = [&](size_t pfm_idx) {
- auto const& other = per_file_metadata[pfm_idx].schema;
+ auto const same_logical_groups = [&](metadata const& pfm) {
+ auto const pfm_idx = static_cast(std::distance(per_file_metadata.data(), &pfm));
return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) {
- return is_file_group(schema, idx) == is_file_group(other, idx) and
+ return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and
is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx);
});
};
// Verify that the input files have matching numbers of columns and schema.
- for (size_t pfm_idx = 0; pfm_idx < per_file_metadata.size(); ++pfm_idx) {
- auto const& pfm = per_file_metadata[pfm_idx];
+ for (auto const& pfm : per_file_metadata) {
if (pfm.row_groups.size() > 0) {
CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(),
"All non-empty sources must have the same number of columns");
}
- CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm_idx),
+ CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm),
"All sources must have the same schema");
}
From 7cfa64ab04f4bc44dd907fa87c907899ac25f4a0 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Tue, 17 Nov 2026 22:16:30 +0000
Subject: [PATCH 07/11] Use pointer subtraction for the source index
---
cpp/src/io/parquet/reader_impl_helpers.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp
index d98b9363481d..84d035c33f41 100644
--- a/cpp/src/io/parquet/reader_impl_helpers.cpp
+++ b/cpp/src/io/parquet/reader_impl_helpers.cpp
@@ -1052,7 +1052,7 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema,
std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; }));
};
auto const same_logical_groups = [&](metadata const& pfm) {
- auto const pfm_idx = static_cast(std::distance(per_file_metadata.data(), &pfm));
+ auto const pfm_idx = static_cast(&pfm - per_file_metadata.data());
return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) {
return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and
is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx);
From eb8e2963333aedbb153ff80fa1da6f2c1c69bc68 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Wed, 18 Nov 2026 01:07:06 +0000
Subject: [PATCH 08/11] Use edit_footer in the BYTE_ARRAY decimal dictionary
test
---
cpp/tests/io/parquet_reader_test.cpp | 28 +++++++++-------------------
1 file changed, 9 insertions(+), 19 deletions(-)
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 1ad97b121729..231e9c4cf943 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -1828,7 +1828,7 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded)
.dictionary_policy(cudf::io::dictionary_policy::ALWAYS)
.build());
- auto footer =
+ auto const footer =
cudf::io::read_parquet_footers(cudf::io::make_datasources(cudf::io::source_info{
cudf::host_span{buffer.data(), buffer.size()}}))
.front();
@@ -1838,24 +1838,14 @@ TYPED_TEST(ParquetReaderByteArrayDecimalTest, DictionaryEncoded)
enc == cudf::io::parquet::Encoding::RLE_DICTIONARY;
}));
- auto& schema = footer.schema[1];
- schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;
- schema.decimal_scale = scale;
- schema.decimal_precision = precision;
- schema.logical_type = cudf::io::parquet::LogicalType{
- cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}};
-
- // Swap the trailing footer for the annotated one, keeping the data pages
- uint32_t footer_len{};
- std::memcpy(&footer_len, buffer.data() + buffer.size() - 8, sizeof(footer_len));
- auto const new_footer = parquet_delta_test::serialize_footer(footer);
- buffer.resize(buffer.size() - 8 - footer_len);
- buffer.insert(buffer.end(), new_footer.begin(), new_footer.end());
- footer_len = new_footer.size();
- buffer.insert(buffer.end(),
- reinterpret_cast(&footer_len),
- reinterpret_cast(&footer_len + 1));
- buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
+ edit_footer(buffer, [&](cudf::io::parquet::FileMetaData& fmd) {
+ auto& schema = fmd.schema[1];
+ schema.converted_type = cudf::io::parquet::ConvertedType::DECIMAL;
+ schema.decimal_scale = scale;
+ schema.decimal_precision = precision;
+ schema.logical_type = cudf::io::parquet::LogicalType{
+ cudf::io::parquet::DecimalType{.scale = scale, .precision = precision}};
+ });
auto const result = cudf::io::read_parquet(
cudf::io::parquet_reader_options::builder(
From aa1eee915368d73672bcc0f33ccaf6bf6340f065 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Wed, 18 Nov 2026 04:17:38 +0000
Subject: [PATCH 09/11] Flatten FILE footer edits in the FileLogicalType test
---
cpp/tests/io/parquet_reader_test.cpp | 38 ++++++++++++----------------
1 file changed, 16 insertions(+), 22 deletions(-)
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 231e9c4cf943..268728637c47 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -2777,26 +2777,20 @@ TEST_F(ParquetReaderTest, FileLogicalType)
file_metadata.child(0).set_name("offset").set_nullability(true);
file_metadata.child(1).set_name("inline").set_nullability(true).set_output_as_binary(true);
- std::vector buffer;
+ std::vector file_group;
cudf::io::write_parquet(
- cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&buffer}, input)
+ cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&file_group}, input)
.metadata(std::move(input_metadata))
.build());
+ edit_footer(file_group, [](cudf::io::parquet::FileMetaData& fmd) {
+ EXPECT_EQ(fmd.schema[1].name, "file");
+ EXPECT_EQ(fmd.schema[2].name, "offset");
+ EXPECT_EQ(fmd.schema[3].name, "inline");
+ fmd.schema[1].logical_type = LogicalType{LogicalType::FILE};
+ // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE
+ fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}};
+ });
- // Returns a copy of `buffer` whose footer retags `file` as FILE and then applies `edit`
- auto const rewrite_file_footer = [&](auto const& edit) {
- auto file = buffer;
- edit_footer(file, [&](cudf::io::parquet::FileMetaData& fmd) {
- EXPECT_EQ(fmd.schema[1].name, "file");
- EXPECT_EQ(fmd.schema[2].name, "offset");
- EXPECT_EQ(fmd.schema[3].name, "inline");
- fmd.schema[1].logical_type = LogicalType{LogicalType::FILE};
- // A signed 64-bit INTEGER annotation on `offset` is allowed and keeps the group FILE
- fmd.schema[2].logical_type = LogicalType{cudf::io::parquet::IntType{64, true}};
- edit(fmd);
- });
- return file;
- };
auto const read_metadata = [](std::span const> files,
bool allow_mismatched = true) {
std::vector> sources;
@@ -2812,14 +2806,13 @@ TEST_F(ParquetReaderTest, FileLogicalType)
.metadata;
};
- auto const file_group = rewrite_file_footer([](auto&) {});
- auto const metadata = read_metadata(std::array{file_group});
+ auto const metadata = read_metadata(std::array{file_group});
EXPECT_EQ(metadata.schema_info[0].is_file, true);
EXPECT_EQ(metadata.schema_info[0].children[1].is_binary, true);
// A FILE group and a plain struct across sources throw, with or without mismatched schemas
- auto const plain_struct =
- rewrite_file_footer([](auto& fmd) { fmd.schema[1].logical_type.reset(); });
+ auto plain_struct = file_group;
+ edit_footer(plain_struct, [](auto& fmd) { fmd.schema[1].logical_type.reset(); });
for (auto const allow_mismatched : {true, false}) {
EXPECT_THROW(read_metadata(std::array{file_group, plain_struct}, allow_mismatched),
std::logic_error);
@@ -2828,8 +2821,9 @@ TEST_F(ParquetReaderTest, FileLogicalType)
}
// An annotated `inline` breaks the FILE spec, so the group is read as a plain struct
- auto const annotated_inline = rewrite_file_footer(
- [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; });
+ auto annotated_inline = file_group;
+ edit_footer(annotated_inline,
+ [](auto& fmd) { fmd.schema[3].logical_type = LogicalType{LogicalType::STRING}; });
EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value());
}
From f8fe97761d66af86537b27329ec57f77ee4b3083 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Thu, 19 Nov 2026 20:44:12 +0000
Subject: [PATCH 10/11] Test multi-source VARIANT bytes and nested VARIANT
files with categories
---
cpp/tests/io/parquet_reader_test.cpp | 15 ++++++++---
.../cudf/tests/input_output/test_parquet.py | 25 +++++++++++++++----
2 files changed, 32 insertions(+), 8 deletions(-)
diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp
index 268728637c47..0cc9cdc3bb97 100644
--- a/cpp/tests/io/parquet_reader_test.cpp
+++ b/cpp/tests/io/parquet_reader_test.cpp
@@ -2827,10 +2827,12 @@ TEST_F(ParquetReaderTest, FileLogicalType)
EXPECT_FALSE(read_metadata(std::array{annotated_inline}).schema_info[0].is_file.has_value());
}
-TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources)
+TEST_F(ParquetReaderTest, VariantGroupsAcrossSources)
{
- // A VARIANT group and a plain struct with the same binary children across sources throw, with
- // or without mismatched schemas, as only the VARIANT binary leaves skip category conversion
+ // VARIANT binary leaves read from multiple sources keep their bytes when strings are converted
+ // to categories. A VARIANT group and a plain struct with the same binary children across
+ // sources throw, with or without mismatched schemas, as only the VARIANT binary leaves skip
+ // category conversion
cudf::test::strings_column_wrapper metadata_col{"m"};
cudf::test::strings_column_wrapper value_col{"v"};
@@ -2870,6 +2872,13 @@ TEST_F(ParquetReaderTest, VariantAndPlainStructAcrossSources)
};
for (auto const allow_mismatched : {true, false}) {
+ auto const result = read(std::array{variant_group, variant_group}, allow_mismatched);
+ auto const variant = result.tbl->view().column(0);
+ CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(0),
+ cudf::test::lists_column_wrapper{{'m'}, {'m'}});
+ CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(variant.child(1),
+ cudf::test::lists_column_wrapper{{'v'}, {'v'}});
+
EXPECT_THROW(read(std::array{variant_group, plain_struct}, allow_mismatched), std::logic_error);
EXPECT_THROW(read(std::array{plain_struct, variant_group}, allow_mismatched), std::logic_error);
}
diff --git a/python/cudf/cudf/tests/input_output/test_parquet.py b/python/cudf/cudf/tests/input_output/test_parquet.py
index b925db4f8bcc..38dc10976bd3 100644
--- a/python/cudf/cudf/tests/input_output/test_parquet.py
+++ b/python/cudf/cudf/tests/input_output/test_parquet.py
@@ -775,15 +775,28 @@ def file_type(*names):
@pytest.mark.parametrize(
- "filename",
+ "filename, columns",
[
- "file_logical_type.parquet",
- "variant_multirow.parquet",
- "duckdb_variant_sample.parquet",
+ ("file_logical_type.parquet", None),
+ ("variant_multirow.parquet", None),
+ ("variant_nested_unshredded.parquet", None),
+ ("variant_nested_shredded.parquet", None),
+ # Skip the shredded `v.typed_value.{n,k}.value` leaves, which are still
+ # hashed as their parents aren't annotated VARIANT (see #25425)
+ (
+ "duckdb_variant_sample.parquet",
+ [
+ "id",
+ "v.metadata",
+ "v.value",
+ "v.typed_value.n.typed_value",
+ "v.typed_value.k.typed_value",
+ ],
+ ),
],
)
def test_parquet_reader_logical_group_binary_not_categorized(
- datadir, filename
+ datadir, filename, columns
):
# Binary children of FILE (`inline`) and VARIANT (`metadata`, `value`)
# groups stay `list` when strings are converted to categories, while
@@ -797,6 +810,8 @@ def read_types(convert_strings_to_categories):
.convert_strings_to_categories(convert_strings_to_categories)
.build()
)
+ if columns is not None:
+ options.set_column_names(columns)
return plc.io.parquet.read_parquet(options).tbl.to_arrow().schema.types
def categorized(typ):
From ac3b79408a003c4e666a663aa32f64ad5d164af0 Mon Sep 17 00:00:00 2001
From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com>
Date: Thu, 19 Nov 2026 20:49:35 +0000
Subject: [PATCH 11/11] Drop the no-op binary leaf check in hybrid scan
dictionaries
---
.../io/parquet/hybrid_scan/hybrid_scan_preprocess.cu | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
index 8908a9036611..c1818b34c8bd 100644
--- a/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
+++ b/cpp/src/io/parquet/hybrid_scan/hybrid_scan_preprocess.cu
@@ -292,13 +292,8 @@ hybrid_scan_reader_impl::prepare_dictionaries(
// look up metadata
auto& col_meta =
_extended_metadata->get_column_metadata(rg.index, rg.source_index, col_schema_idx);
- auto const mapped_schema_idx =
- _extended_metadata->map_schema_index(col_schema_idx, rg.source_index);
- auto& schema = _extended_metadata->get_schema(mapped_schema_idx, rg.source_index);
- // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
- auto const strings_to_categorical =
- options.is_enabled_convert_strings_to_categories() and
- not _extended_metadata->is_binary_leaf(mapped_schema_idx, rg.source_index);
+ auto& schema = _extended_metadata->get_schema(
+ _extended_metadata->map_schema_index(col_schema_idx, rg.source_index), rg.source_index);
// dictionary data buffer for this column chunk
auto& dict_page_data = dictionary_page_data[chunk_idx];
@@ -309,7 +304,7 @@ hybrid_scan_reader_impl::prepare_dictionaries(
auto const [clock_rate, logical_type] = parquet::detail::conversion_info(
parquet::detail::to_type_id(schema,
- strings_to_categorical,
+ options.is_enabled_convert_strings_to_categories(),
options.get_timestamp_type().id(),
options.get_decimal_width()),
options.get_timestamp_type().id(),