diff --git a/cpp/src/io/parquet/hybrid_scan/page_index_filter.cu b/cpp/src/io/parquet/hybrid_scan/page_index_filter.cu index 4326e03952fd..935c2fc21e84 100644 --- a/cpp/src/io/parquet/hybrid_scan/page_index_filter.cu +++ b/cpp/src/io/parquet/hybrid_scan/page_index_filter.cu @@ -152,8 +152,14 @@ struct page_stats_caster : public stats_caster_base { } // Translate binary data to Type then to - min.set_index(column_page_idx, min_value, colchunk.meta_data.type, ts_scale); - max.set_index(column_page_idx, max_value, colchunk.meta_data.type, ts_scale); + if (per_file_metadata[src_idx].stats_ignored(mapped_schema_idx)) { + // The logical type was dropped as unsupported, so min/max can't be interpreted + min.set_index(column_page_idx, std::nullopt, colchunk.meta_data.type); + max.set_index(column_page_idx, std::nullopt, colchunk.meta_data.type); + } else { + min.set_index(column_page_idx, min_value, colchunk.meta_data.type, ts_scale); + max.set_index(column_page_idx, max_value, colchunk.meta_data.type, ts_scale); + } has_nan.set_value(column_page_idx, parquet::detail::has_nans( column_index.nan_counts.has_value() diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 2ac4a5b4949d..b5e6503ffb3e 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -424,6 +424,55 @@ type_id to_type_id(SchemaElement const& schema, return type_id::EMPTY; } +namespace { + +/** + * @brief Checks whether a leaf column's logical type annotation is recognized and valid for its + * physical type, per the Parquet LogicalTypes spec + * + * Unrecognized logical types (parsed as `LogicalType::UNDEFINED`) and unrecognized logical/physical + * combinations (e.g. DECIMAL on BOOLEAN) are reported as unsupported. + */ +[[nodiscard]] bool is_supported_logical_physical_combination(SchemaElement const& elem) +{ + auto const& lt = elem.logical_type; + if (not lt.has_value()) { return true; } + + auto const physical = elem.type; + auto const is_int = physical == Type::INT32 or physical == Type::INT64; + switch (lt->type) { + case LogicalType::STRING: + case LogicalType::ENUM: + case LogicalType::JSON: + case LogicalType::BSON: + case LogicalType::GEOMETRY: + case LogicalType::GEOGRAPHY: return physical == Type::BYTE_ARRAY; + case LogicalType::UUID: return physical == Type::FIXED_LEN_BYTE_ARRAY; + case LogicalType::FLOAT16: return physical == Type::FIXED_LEN_BYTE_ARRAY; + case LogicalType::DATE: return physical == Type::INT32; + case LogicalType::TIME: + if (lt->is_time_millis()) { return physical == Type::INT32; } + return (lt->is_time_micros() or lt->is_time_nanos()) and physical == Type::INT64; + case LogicalType::TIMESTAMP: + return physical == Type::INT64 and + (lt->is_timestamp_millis() or lt->is_timestamp_micros() or lt->is_timestamp_nanos()); + case LogicalType::INTEGER: { + auto const bw = lt->bit_width(); + if (bw == 8 or bw == 16 or bw == 32) { return physical == Type::INT32; } + return bw == 64 and physical == Type::INT64; + } + case LogicalType::DECIMAL: + return is_int or physical == Type::FIXED_LEN_BYTE_ARRAY or physical == Type::BYTE_ARRAY; + // Null type may annotate any physical type + case LogicalType::UNKNOWN: return true; + // MAP, LIST, VARIANT and FILE annotate groups, so they are not valid on leaf columns. UNDEFINED + // is an unrecognized type + default: return false; + } +} + +} // namespace + void metadata::sanitize_schema() { // Parquet isn't very strict about incoming metadata. Lots of things can and should be inferred. @@ -501,6 +550,18 @@ void metadata::sanitize_schema() schema[schema_idx].logical_type = converted_to_logical_type(schema[schema_idx]); } + // Unrecognized logical types and logical/physical combinations: ignore the annotation and the + // column statistics, and read the column using its physical type only. + // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unsupported-logical-types + if (auto& elem = schema[schema_idx]; elem.type != Type::UNDEFINED and + elem.logical_type.has_value() and + not is_supported_logical_physical_combination(elem)) { + elem.logical_type = cuda::std::nullopt; + elem.converted_type = std::nullopt; + elem.logical_type_payload = {}; + stats_ignored_schema_idx_.insert(static_cast(schema_idx)); + } + // Recursive `process` may append to `schema`; index into it each iteration // instead of holding a reference. auto const num_children = schema[schema_idx].num_children; diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index a1796605e195..e793a6d03fc9 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -214,11 +214,25 @@ struct metadata : public FileMetaData { */ [[nodiscard]] bool is_page_index_setup() const { return is_page_index_setup_; } + /** + * @brief Returns whether min/max statistics of a leaf schema element must be ignored + * + * True for columns whose logical type annotation was dropped because it, or its combination + * with the physical type, is unrecognized. Their statistics can't be interpreted reliably. + * + * @param schema_idx Index of the schema element + */ + [[nodiscard]] bool stats_ignored(int schema_idx) const + { + return stats_ignored_schema_idx_.contains(schema_idx); + } + protected: void sanitize_schema(); private: bool is_page_index_setup_ = false; + std::unordered_set stats_ignored_schema_idx_; }; /** diff --git a/cpp/src/io/parquet/row_group_stats_helpers.hpp b/cpp/src/io/parquet/row_group_stats_helpers.hpp index 3ccdde7e48c7..2317b14981d3 100644 --- a/cpp/src/io/parquet/row_group_stats_helpers.hpp +++ b/cpp/src/io/parquet/row_group_stats_helpers.hpp @@ -127,8 +127,14 @@ struct row_group_stats_caster : public stats_caster_base { if (not max_value->has_value()) { max_value = &stats.max; } } // translate binary data to Type then to - min.set_index(stats_idx, *min_value, colchunk.meta_data.type, ts_scale); - max.set_index(stats_idx, *max_value, colchunk.meta_data.type, ts_scale); + if (source_metadata.stats_ignored(mapped_schema_idx)) { + // The logical type was dropped as unsupported, so min/max can't be interpreted + min.set_index(stats_idx, std::nullopt, {}); + max.set_index(stats_idx, std::nullopt, {}); + } else { + min.set_index(stats_idx, *min_value, colchunk.meta_data.type, ts_scale); + max.set_index(stats_idx, *max_value, colchunk.meta_data.type, ts_scale); + } // Check the nullability and NaN presence of this column chunk if (with_null_and_nan_stats) { has_nan->set_value(stats_idx, has_nans(stats.nan_count)); diff --git a/cpp/tests/io/experimental/parquet_footer_facade_test.cpp b/cpp/tests/io/experimental/parquet_footer_facade_test.cpp index 4534d23e595f..06aae40b0b11 100644 --- a/cpp/tests/io/experimental/parquet_footer_facade_test.cpp +++ b/cpp/tests/io/experimental/parquet_footer_facade_test.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include using namespace cudf::io::parquet; @@ -1195,4 +1197,44 @@ TEST_F(ParquetFooterFacadeTest, LogicalTypeFieldWithWrongWireTypeIsSkipped) EXPECT_THROW((void)experimental::read_parquet_footer_bytes(footer), cudf::logic_error); } +// Unrecognized logical/physical combinations drop the logical annotation (the column is read as +// plain physical data); valid combinations keep it. Only a footer is needed, no data pages. +TEST_F(ParquetFooterFacadeTest, UnsupportedLogicalPhysicalCombinationDropsAnnotation) +{ + auto const make_leaf = [](std::string name, Type type, LogicalType logical_type) { + SchemaElement leaf; + leaf.name = std::move(name); + leaf.type = type; + leaf.logical_type = logical_type; + return leaf; + }; + + FileMetaData meta; + meta.version = 2; + meta.num_rows = 0; + SchemaElement root; + root.name = "schema"; + root.num_children = 6; + meta.schema = {root, + make_leaf("decimal_on_bool", Type::BOOLEAN, LogicalType{DecimalType{2, 9}}), + make_leaf("int32_on_int64", Type::INT64, LogicalType{IntType{32, true}}), + make_leaf("decimal_on_int64", Type::INT64, LogicalType{DecimalType{2, 18}}), + make_leaf("date_on_int32", Type::INT32, LogicalType{LogicalType::DATE}), + make_leaf("string_on_binary", Type::BYTE_ARRAY, LogicalType{LogicalType::STRING}), + make_leaf("file_on_binary", Type::BYTE_ARRAY, LogicalType{LogicalType::FILE})}; + + auto const bytes = experimental::write_parquet_footer_bytes(meta); + cudf::io::parquet::experimental::hybrid_scan_reader reader(bytes, + cudf::io::parquet_reader_options{}); + auto const schema = reader.parquet_metadata().schema; + ASSERT_EQ(schema.size(), 7); + + EXPECT_FALSE(schema[1].logical_type.has_value()); // DECIMAL on BOOLEAN + EXPECT_FALSE(schema[2].logical_type.has_value()); // INTEGER(32) on INT64 + EXPECT_TRUE(schema[3].logical_type.has_value()); // DECIMAL on INT64 + EXPECT_TRUE(schema[4].logical_type.has_value()); // DATE on INT32 + EXPECT_TRUE(schema[5].logical_type.has_value()); // STRING on BYTE_ARRAY + EXPECT_FALSE(schema[6].logical_type.has_value()); // FILE annotates groups, not leaves +} + CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/io/parquet_misc_test.cpp b/cpp/tests/io/parquet_misc_test.cpp index dbfc86a41988..cafdd19a3a47 100644 --- a/cpp/tests/io/parquet_misc_test.cpp +++ b/cpp/tests/io/parquet_misc_test.cpp @@ -8,12 +8,22 @@ #include #include +#include #include +#include #include +#include + #include #include +#include +#include +#include +#include +#include +#include #include //////////////////////////////// @@ -234,3 +244,87 @@ TYPED_TEST(ParquetWriterComparableTypeTest, ThreeColumnSorted) EXPECT_EQ(ci.boundary_order, expected_orders[i]); } } + +///////////////////////////////////////////// +// unsupported logical/physical type handling + +namespace { + +// Writes an INT64 column [0, 100) with statistics, then rewrites the footer with the given logical +// type on the column and bogus [0, 0] min/max statistics. Returns the path of the patched file. +std::string write_patched_int64_file(std::string const& name, + std::optional logical_type) +{ + namespace pq = cudf::io::parquet; + std::vector values(100); + std::iota(values.begin(), values.end(), int64_t{0}); + cudf::test::fixed_width_column_wrapper col(values.begin(), values.end()); + auto const filepath = temp_env->get_temp_filepath(name + "_orig.parquet"); + cudf::io::write_parquet(cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, + cudf::table_view{{col}}) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN)); + + pq::FileMetaData fmd; + read_footer(cudf::io::datasource::create(filepath), &fmd); + + // Make the statistics claim every value is 0, so a reader that trusts them prunes `a > 50` + std::vector zero(sizeof(int64_t), 0); + auto& stats = fmd.row_groups[0].columns[0].meta_data.statistics; + stats.min = stats.max = stats.min_value = stats.max_value = zero; + if (logical_type.has_value()) { fmd.schema[1].logical_type = *logical_type; } + + std::vector footer; + pq::detail::CompactProtocolWriter{&footer}.write(fmd); + + std::ifstream in(filepath, std::ios::binary); + std::vector bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + uint32_t old_len; + std::memcpy(&old_len, bytes.data() + bytes.size() - 8, sizeof(old_len)); + bytes.resize(bytes.size() - 8 - old_len); + bytes.insert(bytes.end(), footer.begin(), footer.end()); + auto const new_len = static_cast(footer.size()); + bytes.insert(bytes.end(), + reinterpret_cast(&new_len), + reinterpret_cast(&new_len) + sizeof(new_len)); + bytes.insert(bytes.end(), {'P', 'A', 'R', '1'}); + + auto const patched = temp_env->get_temp_filepath(name + ".parquet"); + std::ofstream(patched, std::ios::binary).write(bytes.data(), bytes.size()); + return patched; +} + +cudf::io::table_with_metadata read_with_filter(std::string const& path) +{ + auto literal = cudf::numeric_scalar(50); + auto literal_ref = cudf::ast::literal(literal); + auto col_ref = cudf::ast::column_reference(0); + auto filter = cudf::ast::operation(cudf::ast::ast_operator::GREATER, col_ref, literal_ref); + return cudf::io::read_parquet( + cudf::io::parquet_reader_options::builder(cudf::io::source_info{path}).filter(filter)); +} + +} // namespace + +// Unsupported logical/physical combinations drop the annotation and ignore the statistics, so the +// column is read as plain INT64 and the bogus statistics don't prune. The control (no annotation) +// shows the statistics are otherwise honored. +TEST_F(ParquetReaderTest, UnsupportedLogicalPhysicalCombinationIgnoresAnnotationAndStats) +{ + using cudf::io::parquet::IntType; + using cudf::io::parquet::LogicalType; + struct test_case { + std::string name; + std::optional logical_type; + cudf::size_type expected_rows; + }; + std::vector const cases{{"Control", std::nullopt, 0}, + {"IntegerOnInt64", LogicalType{IntType{32, true}}, 49}, + {"DateOnInt64", LogicalType{LogicalType::DATE}, 49}}; + + for (auto const& tc : cases) { + auto const path = write_patched_int64_file(tc.name, tc.logical_type); + auto const filtered = read_with_filter(path); + EXPECT_EQ(filtered.tbl->view().column(0).type().id(), cudf::type_id::INT64) << tc.name; + EXPECT_EQ(filtered.tbl->num_rows(), tc.expected_rows) << tc.name; + } +}