Repository navigation
Skip strings-to-categories for FILE and VARIANT binaries #25339
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
c51c3c4
562ef57
d8ffcf4
780c440
b554f28
8072627
7cfa64a
eb8e296
aa1eee9
f8fe977
ac3b794
2369420
98021cd
25c139f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||
|---|---|---|---|---|
|
|
@@ -1075,16 +1075,19 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, | |||
| // Validate that all sources have the same schema unless we are reading select columns | ||||
| // from mismatched sources, in which case, we will only check the projected columns later. | ||||
| if (not has_cols_from_mismatched_srcs) { | ||||
| // `SchemaElement::operator==` ignores logical types, so compare FILE groups separately | ||||
| // `SchemaElement::operator==` ignores logical types, so compare FILE groups and binary | ||||
| // leaves of logical groups (FILE, VARIANT) separately | ||||
| auto const is_file_group = [](std::vector<SchemaElement> const& tree, size_t idx) { | ||||
| return is_valid_file_group( | ||||
| tree[idx], | ||||
| tree[idx].children_idx | | ||||
| std::views::transform([&](int child) -> SchemaElement const& { return tree[child]; })); | ||||
| }; | ||||
| auto const same_file_groups = [&](std::vector<SchemaElement> const& other) { | ||||
| auto const same_logical_groups = [&](metadata const& pfm) { | ||||
| auto const pfm_idx = static_cast<int>(&pfm - per_file_metadata.data()); | ||||
| return std::ranges::all_of(std::views::iota(size_t{0}, schema.size()), [&](size_t idx) { | ||||
| return is_file_group(schema, idx) == is_file_group(other, idx); | ||||
| return is_file_group(schema, idx) == is_file_group(pfm.schema, idx) and | ||||
| is_binary_leaf(idx) == is_binary_leaf(idx, pfm_idx); | ||||
| }); | ||||
| }; | ||||
|
|
||||
|
|
@@ -1094,7 +1097,7 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, | |||
| CUDF_EXPECTS(pfm.row_groups.empty() or num_cols == pfm.row_groups.front().columns.size(), | ||||
| "All non-empty sources must have the same number of columns"); | ||||
| } | ||||
| CUDF_EXPECTS(schema == pfm.schema and same_file_groups(pfm.schema), | ||||
| CUDF_EXPECTS(schema == pfm.schema and same_logical_groups(pfm), | ||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Confirmed fixed on 1b675ce. These checks also reject a VARIANT/plain-struct pair with categories off, which was read before. Could we note that in the description?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added a note to the description. |
||||
| "All sources must have the same schema"); | ||||
| } | ||||
|
|
||||
|
|
@@ -1682,6 +1685,25 @@ std::unique_ptr<table> aggregate_reader_metadata::read_column_chunk_bounds( | |||
| return std::make_unique<table>(std::move(columns)); | ||||
| } | ||||
|
|
||||
| bool aggregate_reader_metadata::is_binary_leaf(int schema_idx, int pfm_idx) const | ||||
| { | ||||
| auto const& schema_elem = get_schema(schema_idx, pfm_idx); | ||||
| if (schema_elem.type != Type::BYTE_ARRAY or schema_elem.num_children != 0) { return false; } | ||||
| auto const& parent = get_schema(schema_elem.parent_idx, pfm_idx); | ||||
| if (not parent.logical_type.has_value()) { return false; } | ||||
| switch (parent.logical_type->type) { | ||||
| case LogicalType::VARIANT: return schema_elem.name == "metadata" or schema_elem.name == "value"; | ||||
|
vuule marked this conversation as resolved.
Resolved
|
||||
| case LogicalType::FILE: | ||||
| return schema_elem.name == "inline" and | ||||
| is_valid_file_group( | ||||
| parent, | ||||
| parent.children_idx | std::views::transform([&](int idx) -> SchemaElement const& { | ||||
| return get_schema(idx, pfm_idx); | ||||
| })); | ||||
| default: return false; | ||||
| } | ||||
| } | ||||
|
|
||||
| bool aggregate_reader_metadata::is_schema_index_mapped(int schema_idx, int src_idx) const | ||||
| { | ||||
| // Check if schema_idx or src_idx is invalid | ||||
|
|
@@ -2213,12 +2235,15 @@ aggregate_reader_metadata::select_columns( | |||
|
|
||||
| auto const& parent_schema = get_schema(schema_elem.parent_idx); | ||||
| auto const one_level_list = schema_elem.is_one_level_list(parent_schema); | ||||
| // Binary leaves of logical groups (FILE, VARIANT) are never converted to categories | ||||
| auto const binary_leaf = is_binary_leaf(schema_idx); | ||||
| auto const leaf_to_categorical = strings_to_categorical and not binary_leaf; | ||||
|
|
||||
| // if we're at the root, this is a new output column | ||||
| auto const col_type = one_level_list ? type_id::LIST | ||||
| : to_type_id(schema_elem, | ||||
| arrow_type(schema_idx), | ||||
| strings_to_categorical, | ||||
| leaf_to_categorical, | ||||
| timestamp_type_id, | ||||
| decimal_type_id); | ||||
| auto const dtype = to_data_type(col_type, schema_elem); | ||||
|
|
@@ -2260,15 +2285,6 @@ aggregate_reader_metadata::select_columns( | |||
| } | ||||
| } | ||||
|
|
||||
| // Flag the `inline` child of a valid FILE group so that `make_column` materializes it as | ||||
| // `list<uint8>` instead of strings. | ||||
| if (output_col.is_file) { | ||||
| auto const inline_col = std::ranges::find(output_col.children, | ||||
| std::string_view{"inline"}, | ||||
| &cudf::io::detail::inline_column_buffer::name); | ||||
| if (inline_col != output_col.children.end()) { inline_col->string_as_binary = true; } | ||||
| } | ||||
|
|
||||
| // if I have no children, we're at a leaf and I'm an input column (that is, one with actual | ||||
| // data stored) so add me to the list. | ||||
| if (schema_elem.num_children == 0) { | ||||
|
|
@@ -2280,7 +2296,7 @@ aggregate_reader_metadata::select_columns( | |||
| // determine the element data type | ||||
| auto const element_type = to_type_id(schema_elem, | ||||
| arrow_type(schema_idx), | ||||
| strings_to_categorical, | ||||
| leaf_to_categorical, | ||||
| timestamp_type_id, | ||||
| decimal_type_id); | ||||
| auto const element_dtype = to_data_type(element_type, schema_elem); | ||||
|
|
@@ -2305,13 +2321,10 @@ aggregate_reader_metadata::select_columns( | |||
| // pop off the extra nesting element. | ||||
| if (one_level_list) { nesting.pop_back(); } | ||||
|
|
||||
| // Flag the `metadata` / `value` BYTE_ARRAY children of a VARIANT group so that | ||||
| // `make_column` materializes them as `list<uint8>` instead of strings. | ||||
| if (schema_elem.type == Type::BYTE_ARRAY && parent_schema.logical_type.has_value() && | ||||
| parent_schema.logical_type->type == LogicalType::VARIANT && | ||||
| (schema_elem.name == "metadata" || schema_elem.name == "value")) { | ||||
| output_col.string_as_binary = true; | ||||
| } | ||||
| // Flag binary leaves of logical groups (the `inline` child of a valid FILE group and the | ||||
| // `metadata` / `value` children of a VARIANT group) so that `make_column` materializes | ||||
| // them as `list<uint8>` instead of strings. | ||||
| if (binary_leaf) { output_col.string_as_binary = true; } | ||||
|
|
||||
| path_is_valid = true; // If we're able to reach leaf then path is valid | ||||
| } | ||||
|
|
@@ -2383,6 +2396,15 @@ aggregate_reader_metadata::select_columns( | |||
| src_schema_elem.name), | ||||
| std::invalid_argument); | ||||
|
|
||||
| // Binary leaves of logical groups (FILE, VARIANT) are read as binary, so they must be | ||||
| // binary leaves in all sources | ||||
| CUDF_EXPECTS( | ||||
| is_binary_leaf(src_schema_idx) == is_binary_leaf(dst_schema_idx, src_idx), | ||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Disclaimer: this is a critical issue claimed by AI, so I'd like to check with you whether it holds. Could we double check the multi-source case outside this path? This check only runs for a projected read with I tried it locally on c39c366 with a
Does this make sense to you, or am I missing something that rules this input out? cudf/cpp/src/io/parquet/reader_impl_helpers.cpp Line 1065 in c39c366
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think its a valid find. We also need this same check in
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done in 1b675ce and added |
||||
| std::format("Encountered mismatching binary leaf of a logical group across the Parquet " | ||||
| "sources for column '{}'", | ||||
| src_schema_elem.name), | ||||
| std::invalid_argument); | ||||
|
|
||||
| // Get the schema_idx_map for this data source (pfm) | ||||
| auto& schema_idx_map = schema_idx_maps[src_idx - 1]; | ||||
| // Map the schema index from 0th tree (src) to the one in the current (dst) tree. | ||||
|
|
||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ | |
|
|
||
| #include <cudf/concatenate.hpp> | ||
| #include <cudf/copying.hpp> | ||
| #include <cudf/io/experimental/parquet_footer.hpp> | ||
| #include <cudf/io/parquet.hpp> | ||
| #include <cudf/io/parquet_io_utils.hpp> | ||
| #include <cudf/io/parquet_metadata.hpp> | ||
|
|
@@ -20,6 +21,7 @@ | |
| #include <src/io/parquet/compact_protocol_reader.hpp> | ||
| #include <src/io/parquet/timestamp_utils.cuh> | ||
|
|
||
| #include <cstdint> | ||
| #include <cstring> | ||
| #include <format> | ||
| #include <numeric> | ||
|
|
@@ -255,6 +257,32 @@ void read_footer(std::unique_ptr<cudf::io::datasource> const& source, | |
| file_meta_data); | ||
| } | ||
|
|
||
| void edit_footer(std::vector<char>& buffer, | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| std::function<void(cudf::io::parquet::FileMetaData&)> const& edit) | ||
| { | ||
| namespace pq_experimental = cudf::io::parquet::experimental; | ||
|
|
||
| // parquet files end with 4-byte footer_length and 4-byte magic == "PAR1" | ||
| auto constexpr ender_size = 2 * sizeof(uint32_t); | ||
| CUDF_EXPECTS(buffer.size() >= ender_size, "Buffer is too small to be a Parquet file"); | ||
| uint32_t footer_size{}; | ||
| std::memcpy(&footer_size, buffer.data() + buffer.size() - ender_size, sizeof(footer_size)); | ||
| CUDF_EXPECTS(buffer.size() >= ender_size + footer_size, "Invalid Parquet footer length"); | ||
| auto const footer_begin = buffer.size() - ender_size - footer_size; | ||
|
|
||
| auto file_meta_data = pq_experimental::read_parquet_footer_bytes( | ||
| {reinterpret_cast<uint8_t const*>(buffer.data()) + footer_begin, footer_size}); | ||
| edit(file_meta_data); | ||
| auto const footer = pq_experimental::write_parquet_footer_bytes(file_meta_data); | ||
|
|
||
| auto const new_footer_size = static_cast<uint32_t>(footer.size()); | ||
| auto const size_bytes = reinterpret_cast<char const*>(&new_footer_size); | ||
| buffer.resize(footer_begin); | ||
| buffer.insert(buffer.end(), footer.begin(), footer.end()); | ||
| buffer.insert(buffer.end(), size_bytes, size_bytes + sizeof(new_footer_size)); | ||
| buffer.insert(buffer.end(), {'P', 'A', 'R', '1'}); | ||
| } | ||
|
|
||
| // returns the number of bits used for dictionary encoding data at the given page location. | ||
| // this assumes the data is uncompressed. | ||
| // throws cudf::logic_error if the page_loc data is invalid. | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.