Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions cpp/src/io/parquet/reader_impl_chunking.cu
Original file line number Diff line number Diff line change
Expand Up @@ -488,13 +488,17 @@ void reader_impl::create_global_chunk_info()
auto col = _input_columns[i];
// look up metadata
auto& col_meta = _metadata->get_column_metadata(rg.index, rg.source_index, col.schema_idx);
auto& schema = _metadata->get_schema(
_metadata->map_schema_index(col.schema_idx, rg.source_index), rg.source_index);
auto const mapped_schema_idx = _metadata->map_schema_index(col.schema_idx, rg.source_index);
auto& schema = _metadata->get_schema(mapped_schema_idx, rg.source_index);
// Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
auto const strings_to_categorical =
_strings_to_categorical and
not _metadata->is_binary_leaf(mapped_schema_idx, rg.source_index);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

auto [clock_rate, logical_type] =
conversion_info(to_type_id(schema,
_metadata->arrow_type(col.schema_idx),
_strings_to_categorical,
strings_to_categorical,
_options.timestamp_type.id(),
_options.decimal_width),
_options.timestamp_type.id(),
Expand Down Expand Up @@ -532,7 +536,7 @@ void reader_impl::create_global_chunk_info()
col.schema_idx,
chunk_info,
list_bytes_per_row_est,
schema.type == Type::BYTE_ARRAY and _strings_to_categorical,
schema.type == Type::BYTE_ARRAY and strings_to_categorical,
rg.source_index);
}
// Adjust for skip_rows when updating the remaining rows after the first group
Expand Down
66 changes: 44 additions & 22 deletions cpp/src/io/parquet/reader_impl_helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1075,16 +1075,19 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema,
// Validate that all sources have the same schema unless we are reading select columns
// from mismatched sources, in which case, we will only check the projected columns later.
if (not has_cols_from_mismatched_srcs) {
// `SchemaElement::operator==` ignores logical types, so compare FILE groups separately
// `SchemaElement::operator==` ignores logical types, so compare FILE groups and binary
// leaves of logical groups (FILE, VARIANT) separately
auto const is_file_group = [](std::vector<SchemaElement> const& tree, size_t idx) {
return is_valid_file_group(
tree[idx],
tree[idx].children_idx |
std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; }));
};
auto const same_file_groups = [&](std::vector<SchemaElement> const& other) {
auto const same_logical_groups = [&](metadata const& pfm) {
auto const pfm_idx = static_cast<int>(&pfm - per_file_metadata.data());
return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) {
return is_file_group(schema, idx) == is_file_group(other, idx);
return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and
is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx);
});
};

Expand All @@ -1094,7 +1097,7 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema,
CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(),
"All non-empty sources must have the same number of columns");
}
CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema),
CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed fixed on 1b675ce. These checks also reject a VARIANT/plain-struct pair with categories off, which was read before. Could we note that in the description?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a note to the description.

"All sources must have the same schema");
}

Expand Down Expand Up @@ -1682,6 +1685,25 @@ std::unique_ptr<table> aggregate_reader_metadata::read_column_chunk_bounds(
return std::make_unique<table>(std::move(columns));
}

bool aggregate_reader_metadata::is_binary_leaf(int schema_idx, int pfm_idx) const
{
auto const& schema_elem = get_schema(schema_idx, pfm_idx);
if (schema_elem.type != Type::BYTE_ARRAY or schema_elem.num_children != 0) { return false; }
auto const& parent = get_schema(schema_elem.parent_idx, pfm_idx);
if (not parent.logical_type.has_value()) { return false; }
switch (parent.logical_type->type) {
case LogicalType::VARIANT: return schema_elem.name == "metadata" or schema_elem.name == "value";
Comment thread
vuule marked this conversation as resolved.
Resolved
case LogicalType::FILE:
return schema_elem.name == "inline" and
is_valid_file_group(
parent,
parent.children_idx | std::views::transform([&](int idx) -> SchemaElement const& {
return get_schema(idx, pfm_idx);
}));
default: return false;
}
}

bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const
{
// Check if schema_idx or src_idx is invalid
Expand Down Expand Up @@ -2213,12 +2235,15 @@ aggregate_reader_metadata::select_columns(

auto const& parent_schema = get_schema(schema_elem.parent_idx);
auto const one_level_list = schema_elem.is_one_level_list(parent_schema);
// Binary leaves of logical groups (FILE, VARIANT) are never converted to categories
auto const binary_leaf = is_binary_leaf(schema_idx);
auto const leaf_to_categorical = strings_to_categorical and not binary_leaf;

// if we're at the root, this is a new output column
auto const col_type = one_level_list ? type_id::LIST
: to_type_id(schema_elem,
arrow_type(schema_idx),
strings_to_categorical,
leaf_to_categorical,
timestamp_type_id,
decimal_type_id);
auto const dtype = to_data_type(col_type, schema_elem);
Expand Down Expand Up @@ -2260,15 +2285,6 @@ aggregate_reader_metadata::select_columns(
}
}

// Flag the `inline` child of a valid FILE group so that `make_column` materializes it as
// `list<uint8>` instead of strings.
if (output_col.is_file) {
auto const inline_col = std::ranges::find(output_col.children,
std::string_view{"inline"},
&cudf::io::detail::inline_column_buffer::name);
if (inline_col != output_col.children.end()) { inline_col->string_as_binary = true; }
}

// if I have no children, we're at a leaf and I'm an input column (that is, one with actual
// data stored) so add me to the list.
if (schema_elem.num_children == 0) {
Expand All @@ -2280,7 +2296,7 @@ aggregate_reader_metadata::select_columns(
// determine the element data type
auto const element_type = to_type_id(schema_elem,
arrow_type(schema_idx),
strings_to_categorical,
leaf_to_categorical,
timestamp_type_id,
decimal_type_id);
auto const element_dtype = to_data_type(element_type, schema_elem);
Expand All @@ -2305,13 +2321,10 @@ aggregate_reader_metadata::select_columns(
// pop off the extra nesting element.
if (one_level_list) { nesting.pop_back(); }

// Flag the `metadata` / `value` BYTE_ARRAY children of a VARIANT group so that
// `make_column` materializes them as `list<uint8>` instead of strings.
if (schema_elem.type == Type::BYTE_ARRAY && parent_schema.logical_type.has_value() &&
parent_schema.logical_type->type == LogicalType::VARIANT &&
(schema_elem.name == "metadata" || schema_elem.name == "value")) {
output_col.string_as_binary = true;
}
// Flag binary leaves of logical groups (the `inline` child of a valid FILE group and the
// `metadata` / `value` children of a VARIANT group) so that `make_column` materializes
// them as `list<uint8>` instead of strings.
if (binary_leaf) { output_col.string_as_binary = true; }

path_is_valid = true; // If we're able to reach leaf then path is valid
}
Expand Down Expand Up @@ -2383,6 +2396,15 @@ aggregate_reader_metadata::select_columns(
src_schema_elem.name),
std::invalid_argument);

// Binary leaves of logical groups (FILE, VARIANT) are read as binary, so they must be
// binary leaves in all sources
CUDF_EXPECTS(
is_binary_leaf(src_schema_idx) == is_binary_leaf(dst_schema_idx, src_idx),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Disclaimer: this is a critical issue claimed by AI, so I'd like to check with you whether it holds.

Could we double check the multi-source case outside this path? This check only runs for a projected read with allow_mismatched_pq_schemas. The default path compares FILE groups only, so a VARIANT group in one source and the same plain struct in another is still accepted, and each source then decides categorization on its own.

I tried it locally on c39c366 with a struct{metadata, value} of binary leaves (4 rows), one copy retagged as VARIANT, and convert_strings_to_categories(true):

  • {variant, plain}: list<uint8> children with 8 bytes instead of 16, the second source's rows are empty
  • {plain, variant}: INT32 children whose rows 4-7 are 0, 2, 4, 6

Does this make sense to you, or am I missing something that rules this input out?

CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema),

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think its a valid find. We also need this same check in initialize_internals where we ensure same schema when allow_mismatched_pq_schemas is false

@mhaseeb123 mhaseeb123 Nov 17, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in 1b675ce and added VariantAndPlainStructAcrossSources test

std::format("Encountered mismatching binary leaf of a logical group across the Parquet "
"sources for column '{}'",
src_schema_elem.name),
std::invalid_argument);

// Get the schema_idx_map for this data source (pfm)
auto& schema_idx_map = schema_idx_maps[src_idx - 1];
// Map the schema index from 0th tree (src) to the one in the current (dst) tree.
Expand Down
13 changes: 13 additions & 0 deletions cpp/src/io/parquet/reader_impl_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -711,6 +711,19 @@ class aggregate_reader_metadata {
return nullable_across_sources.contains(schema_idx);
}

/**
* @brief Checks if a schema element is a binary BYTE_ARRAY leaf of a logical group, i.e. the
* `inline` child of a valid FILE group or the `metadata` / `value` child of a VARIANT group
*
* Such leaves are always read as `list<uint8>` and are never converted to categories.
*
* @param schema_idx The index of the SchemaElement in the pfm_idx'th file
* @param pfm_idx The index of the per_file_metadata, default = 0 if not specified
*
* @return True if the schema element is a binary leaf of a logical group
*/
[[nodiscard]] bool is_binary_leaf(int schema_idx, int pfm_idx = 0) const;

/**
* @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file
*
Expand Down
27 changes: 15 additions & 12 deletions cpp/src/io/parquet/reader_impl_preprocess.cu
Original file line number Diff line number Diff line change
Expand Up @@ -258,17 +258,19 @@ void reader_impl::allocate_nesting_info()
}

// Get the schema from the current input source.
auto& actual_cur_schema = _metadata->get_schema(
_metadata->map_schema_index(schema_idx, src_file_idx), src_file_idx);
auto const actual_cur_schema_idx = _metadata->map_schema_index(schema_idx, src_file_idx);
auto& actual_cur_schema = _metadata->get_schema(actual_cur_schema_idx, src_file_idx);

// values indexed by output column index
nesting_info[cur_depth].max_def_level = actual_cur_schema.max_definition_level;
pni[cur_depth].size = 0;
pni[cur_depth].type = to_type_id(actual_cur_schema,
_metadata->arrow_type(schema_idx),
_strings_to_categorical,
_options.timestamp_type.id(),
_options.decimal_width);
pni[cur_depth].type =
to_type_id(actual_cur_schema,
_metadata->arrow_type(schema_idx),
_strings_to_categorical and
not _metadata->is_binary_leaf(actual_cur_schema_idx, src_file_idx),
_options.timestamp_type.id(),
_options.decimal_width);
pni[cur_depth].nullable = cur_schema.repetition_type == FieldRepetitionType::OPTIONAL;

// the implicit list of a one-level list is a required list with its parent's def level
Expand Down Expand Up @@ -736,11 +738,12 @@ void reader_impl::preprocess_file(read_mode mode)
printf("# Input columns: %'lu\n", _input_columns.size());
for (size_t idx = 0; idx < _input_columns.size(); idx++) {
auto const& schema = _metadata->get_schema(_input_columns[idx].schema_idx);
auto const type_id = to_type_id(schema,
_metadata->arrow_type(_input_columns[idx].schema_idx),
_strings_to_categorical,
_options.timestamp_type.id(),
_options.decimal_width);
auto const type_id = to_type_id(
schema,
_metadata->arrow_type(_input_columns[idx].schema_idx),
_strings_to_categorical and not _metadata->is_binary_leaf(_input_columns[idx].schema_idx),
_options.timestamp_type.id(),
_options.decimal_width);
printf("\tC(%'lu, %s): %s\n",
idx,
_input_columns[idx].name.c_str(),
Expand Down
28 changes: 28 additions & 0 deletions cpp/tests/io/parquet_common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include <cudf/concatenate.hpp>
#include <cudf/copying.hpp>
#include <cudf/io/experimental/parquet_footer.hpp>
#include <cudf/io/parquet.hpp>
#include <cudf/io/parquet_io_utils.hpp>
#include <cudf/io/parquet_metadata.hpp>
Expand All @@ -20,6 +21,7 @@
#include <src/io/parquet/compact_protocol_reader.hpp>
#include <src/io/parquet/timestamp_utils.cuh>

#include <cstdint>
#include <cstring>
#include <format>
#include <numeric>
Expand Down Expand Up @@ -255,6 +257,32 @@ void read_footer(std::unique_ptr<cudf::io::datasource> const& source,
file_meta_data);
}

void edit_footer(std::vector<char>& buffer,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This common util will also be used in PRs #25338 and #25349. Added a comment there as well

std::function<void(cudf::io::parquet::FileMetaData&)> const& edit)
{
namespace pq_experimental = cudf::io::parquet::experimental;

// parquet files end with 4-byte footer_length and 4-byte magic == "PAR1"
auto constexpr ender_size = 2 * sizeof(uint32_t);
CUDF_EXPECTS(buffer.size() >= ender_size, "Buffer is too small to be a Parquet file");
uint32_t footer_size{};
std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size));
CUDF_EXPECTS(buffer.size() >= ender_size + footer_size, "Invalid Parquet footer length");
auto const footer_begin = buffer.size() - ender_size - footer_size;

auto file_meta_data = pq_experimental::read_parquet_footer_bytes(
{reinterpret_cast<uint8_t const*>(buffer.data()) + footer_begin, footer_size});
edit(file_meta_data);
auto const footer = pq_experimental::write_parquet_footer_bytes(file_meta_data);

auto const new_footer_size = static_cast<uint32_t>(footer.size());
auto const size_bytes = reinterpret_cast<char const*>(&new_footer_size);
buffer.resize(footer_begin);
buffer.insert(buffer.end(), footer.begin(), footer.end());
buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size));
buffer.insert(buffer.end(), {'P', 'A', 'R', '1'});
}

// returns the number of bits used for dictionary encoding data at the given page location.
// this assumes the data is uncompressed.
// throws cudf::logic_error if the page_loc data is invalid.
Expand Down
13 changes: 13 additions & 0 deletions cpp/tests/io/parquet_common.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <cuda/std/utility>

#include <functional>
#include <random>
#include <string>
#include <string_view>
Expand Down Expand Up @@ -134,6 +135,18 @@ std::unique_ptr<cudf::column> make_parquet_list_list_col(
void read_footer(std::unique_ptr<cudf::io::datasource> const& source,
cudf::io::parquet::FileMetaData* file_meta_data);

/**
* @brief Rewrites the footer of an in-memory Parquet file
*
* Decodes the trailing footer of `buffer`, applies `edit` to it, and replaces it with the
* re-encoded footer, keeping the data pages as is.
*
* @param buffer Parquet file bytes to update in place
* @param edit Callable that modifies the decoded footer
*/
void edit_footer(std::vector<char>& buffer,
std::function<void(cudf::io::parquet::FileMetaData&)> const& edit);

/**
* @brief Returns the number of bits used for dictionary encoding the uncompressed page at
* `page_loc`
Expand Down
Loading
Loading