Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions cpp/src/io/parquet/hybrid_scan/page_index_filter.cu
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,14 @@ struct page_stats_caster : public stats_caster_base {
}

// Translate binary data to Type then to <T>
min.set_index(column_page_idx, min_value, colchunk.meta_data.type, ts_scale);
max.set_index(column_page_idx, max_value, colchunk.meta_data.type, ts_scale);
if (per_file_metadata[src_idx].stats_ignored(mapped_schema_idx)) {
// The logical type was dropped as unsupported, so min/max can't be interpreted
min.set_index(column_page_idx, std::nullopt, colchunk.meta_data.type);
max.set_index(column_page_idx, std::nullopt, colchunk.meta_data.type);
} else {
min.set_index(column_page_idx, min_value, colchunk.meta_data.type, ts_scale);
max.set_index(column_page_idx, max_value, colchunk.meta_data.type, ts_scale);
}
has_nan.set_value(column_page_idx,
parquet::detail::has_nans(
column_index.nan_counts.has_value()
Expand Down
61 changes: 61 additions & 0 deletions cpp/src/io/parquet/reader_impl_helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,55 @@ type_id to_type_id(SchemaElement const& schema,
return type_id::EMPTY;
}

namespace {

/**
* @brief Checks whether a leaf column's logical type annotation is recognized and valid for its
* physical type, per the Parquet LogicalTypes spec
*
* Unrecognized logical types (parsed as `LogicalType::UNDEFINED`) and unrecognized logical/physical
* combinations (e.g. DECIMAL on BOOLEAN) are reported as unsupported.
*/
[[nodiscard]] bool is_supported_logical_physical_combination(SchemaElement const& elem)
{
auto const& lt = elem.logical_type;
if (not lt.has_value()) { return true; }

auto const physical = elem.type;
auto const is_int = physical == Type::INT32 or physical == Type::INT64;
switch (lt->type) {
case LogicalType::STRING:
case LogicalType::ENUM:
case LogicalType::JSON:
case LogicalType::BSON:
case LogicalType::GEOMETRY:
case LogicalType::GEOGRAPHY: return physical == Type::BYTE_ARRAY;
case LogicalType::UUID: return physical == Type::FIXED_LEN_BYTE_ARRAY;
case LogicalType::FLOAT16: return physical == Type::FIXED_LEN_BYTE_ARRAY;
case LogicalType::DATE: return physical == Type::INT32;
case LogicalType::TIME:
if (lt->is_time_millis()) { return physical == Type::INT32; }
return (lt->is_time_micros() or lt->is_time_nanos()) and physical == Type::INT64;
case LogicalType::TIMESTAMP:
return physical == Type::INT64 and
(lt->is_timestamp_millis() or lt->is_timestamp_micros() or lt->is_timestamp_nanos());
case LogicalType::INTEGER: {
auto const bw = lt->bit_width();
if (bw == 8 or bw == 16 or bw == 32) { return physical == Type::INT32; }
return bw == 64 and physical == Type::INT64;
}
case LogicalType::DECIMAL:
return is_int or physical == Type::FIXED_LEN_BYTE_ARRAY or physical == Type::BYTE_ARRAY;
// Null type may annotate any physical type
case LogicalType::UNKNOWN: return true;
// MAP, LIST, VARIANT and FILE annotate groups, so they are not valid on leaf columns. UNDEFINED
// is an unrecognized type
default: return false;
}
}

} // namespace

void metadata::sanitize_schema()
{
// Parquet isn't very strict about incoming metadata. Lots of things can and should be inferred.
Expand Down Expand Up @@ -501,6 +550,18 @@ void metadata::sanitize_schema()
schema[schema_idx].logical_type = converted_to_logical_type(schema[schema_idx]);
}

// Unrecognized logical types and logical/physical combinations: ignore the annotation and the
// column statistics, and read the column using its physical type only.
// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unsupported-logical-types
if (auto& elem = schema[schema_idx]; elem.type != Type::UNDEFINED and
elem.logical_type.has_value() and
not is_supported_logical_physical_combination(elem)) {
elem.logical_type = cuda::std::nullopt;
elem.converted_type = std::nullopt;
elem.logical_type_payload = {};
stats_ignored_schema_idx_.insert(static_cast<int>(schema_idx));
}

// Recursive `process` may append to `schema`; index into it each iteration
// instead of holding a reference.
auto const num_children = schema[schema_idx].num_children;
Expand Down
14 changes: 14 additions & 0 deletions cpp/src/io/parquet/reader_impl_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -214,11 +214,25 @@ struct metadata : public FileMetaData {
*/
[[nodiscard]] bool is_page_index_setup() const { return is_page_index_setup_; }

/**
* @brief Returns whether min/max statistics of a leaf schema element must be ignored
*
* True for columns whose logical type annotation was dropped because it, or its combination
* with the physical type, is unrecognized. Their statistics can't be interpreted reliably.
*
* @param schema_idx Index of the schema element
*/
[[nodiscard]] bool stats_ignored(int schema_idx) const
{
return stats_ignored_schema_idx_.contains(schema_idx);
}

protected:
void sanitize_schema();

private:
bool is_page_index_setup_ = false;
std::unordered_set<int> stats_ignored_schema_idx_;
};

/**
Expand Down
10 changes: 8 additions & 2 deletions cpp/src/io/parquet/row_group_stats_helpers.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,14 @@ struct row_group_stats_caster : public stats_caster_base {
if (not max_value->has_value()) { max_value = &stats.max; }
}
// translate binary data to Type then to <T>
min.set_index(stats_idx, *min_value, colchunk.meta_data.type, ts_scale);
max.set_index(stats_idx, *max_value, colchunk.meta_data.type, ts_scale);
if (source_metadata.stats_ignored(mapped_schema_idx)) {
// The logical type was dropped as unsupported, so min/max can't be interpreted
min.set_index(stats_idx, std::nullopt, {});
max.set_index(stats_idx, std::nullopt, {});
} else {
min.set_index(stats_idx, *min_value, colchunk.meta_data.type, ts_scale);
max.set_index(stats_idx, *max_value, colchunk.meta_data.type, ts_scale);
}
// Check the nullability and NaN presence of this column chunk
if (with_null_and_nan_stats) {
has_nan->set_value(stats_idx, has_nans(stats.nan_count));
Expand Down
42 changes: 42 additions & 0 deletions cpp/tests/io/experimental/parquet_footer_facade_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <cudf_test/testing_main.hpp>

#include <cudf/io/datasource.hpp>
#include <cudf/io/experimental/hybrid_scan.hpp>
#include <cudf/io/experimental/parquet_footer.hpp>
#include <cudf/io/parquet.hpp>
#include <cudf/io/parquet_metadata.hpp>
Expand All @@ -24,6 +25,7 @@
#include <format>
#include <optional>
#include <string>
#include <utility>
#include <vector>

using namespace cudf::io::parquet;
Expand Down Expand Up @@ -1195,4 +1197,44 @@ TEST_F(ParquetFooterFacadeTest, LogicalTypeFieldWithWrongWireTypeIsSkipped)
EXPECT_THROW((void)experimental::read_parquet_footer_bytes(footer), cudf::logic_error);
}

// Unrecognized logical/physical combinations drop the logical annotation (the column is read as
// plain physical data); valid combinations keep it. Only a footer is needed, no data pages.
TEST_F(ParquetFooterFacadeTest, UnsupportedLogicalPhysicalCombinationDropsAnnotation)
{
auto const make_leaf = [](std::string name, Type type, LogicalType logical_type) {
SchemaElement leaf;
leaf.name = std::move(name);
leaf.type = type;
leaf.logical_type = logical_type;
return leaf;
};

FileMetaData meta;
meta.version = 2;
meta.num_rows = 0;
SchemaElement root;
root.name = "schema";
root.num_children = 6;
meta.schema = {root,
make_leaf("decimal_on_bool", Type::BOOLEAN, LogicalType{DecimalType{2, 9}}),
make_leaf("int32_on_int64", Type::INT64, LogicalType{IntType{32, true}}),
make_leaf("decimal_on_int64", Type::INT64, LogicalType{DecimalType{2, 18}}),
make_leaf("date_on_int32", Type::INT32, LogicalType{LogicalType::DATE}),
make_leaf("string_on_binary", Type::BYTE_ARRAY, LogicalType{LogicalType::STRING}),
make_leaf("file_on_binary", Type::BYTE_ARRAY, LogicalType{LogicalType::FILE})};

auto const bytes = experimental::write_parquet_footer_bytes(meta);
cudf::io::parquet::experimental::hybrid_scan_reader reader(bytes,
cudf::io::parquet_reader_options{});
auto const schema = reader.parquet_metadata().schema;
ASSERT_EQ(schema.size(), 7);

EXPECT_FALSE(schema[1].logical_type.has_value()); // DECIMAL on BOOLEAN
EXPECT_FALSE(schema[2].logical_type.has_value()); // INTEGER(32) on INT64
EXPECT_TRUE(schema[3].logical_type.has_value()); // DECIMAL on INT64
EXPECT_TRUE(schema[4].logical_type.has_value()); // DATE on INT32
EXPECT_TRUE(schema[5].logical_type.has_value()); // STRING on BYTE_ARRAY
EXPECT_FALSE(schema[6].logical_type.has_value()); // FILE annotates groups, not leaves
}

CUDF_TEST_PROGRAM_MAIN()
94 changes: 94 additions & 0 deletions cpp/tests/io/parquet_misc_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,22 @@
#include <cudf_test/base_fixture.hpp>
#include <cudf_test/table_utilities.hpp>

#include <cudf/ast/expressions.hpp>
#include <cudf/io/parquet.hpp>
#include <cudf/scalar/scalar.hpp>

#include <cuda/iterator>

#include <src/io/parquet/compact_protocol_writer.hpp>

#include <algorithm>
#include <array>
#include <cstring>
#include <fstream>
#include <iterator>
#include <numeric>
#include <optional>
#include <string>
#include <vector>

////////////////////////////////
Expand Down Expand Up @@ -234,3 +244,87 @@ TYPED_TEST(ParquetWriterComparableTypeTest, ThreeColumnSorted)
EXPECT_EQ(ci.boundary_order, expected_orders[i]);
}
}

/////////////////////////////////////////////
// unsupported logical/physical type handling

namespace {

// Writes an INT64 column [0, 100) with statistics, then rewrites the footer with the given logical
// type on the column and bogus [0, 0] min/max statistics. Returns the path of the patched file.
std::string write_patched_int64_file(std::string const& name,
std::optional<cudf::io::parquet::LogicalType> logical_type)
{
namespace pq = cudf::io::parquet;
std::vector<int64_t> values(100);
std::iota(values.begin(), values.end(), int64_t{0});
cudf::test::fixed_width_column_wrapper<int64_t> col(values.begin(), values.end());
auto const filepath = temp_env->get_temp_filepath(name + "_orig.parquet");
cudf::io::write_parquet(cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath},
cudf::table_view{{col}})
.stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN));

pq::FileMetaData fmd;
read_footer(cudf::io::datasource::create(filepath), &fmd);

// Make the statistics claim every value is 0, so a reader that trusts them prunes `a > 50`
std::vector<uint8_t> zero(sizeof(int64_t), 0);
auto& stats = fmd.row_groups[0].columns[0].meta_data.statistics;
stats.min = stats.max = stats.min_value = stats.max_value = zero;
if (logical_type.has_value()) { fmd.schema[1].logical_type = *logical_type; }

std::vector<uint8_t> footer;
pq::detail::CompactProtocolWriter{&footer}.write(fmd);

std::ifstream in(filepath, std::ios::binary);
std::vector<char> bytes((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
uint32_t old_len;
std::memcpy(&old_len, bytes.data() + bytes.size() - 8, sizeof(old_len));
bytes.resize(bytes.size() - 8 - old_len);
bytes.insert(bytes.end(), footer.begin(), footer.end());
auto const new_len = static_cast<uint32_t>(footer.size());
bytes.insert(bytes.end(),
reinterpret_cast<char const*>(&new_len),
reinterpret_cast<char const*>(&new_len) + sizeof(new_len));
bytes.insert(bytes.end(), {'P', 'A', 'R', '1'});

auto const patched = temp_env->get_temp_filepath(name + ".parquet");
std::ofstream(patched, std::ios::binary).write(bytes.data(), bytes.size());
Comment on lines +267 to +292

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once #25339 merges, this can use the shared edit_footer helper from parquet_common.hpp:

Suggested change
pq::FileMetaData fmd;
read_footer(cudf::io::datasource::create(filepath), &fmd);
// Make the statistics claim every value is 0, so a reader that trusts them prunes `a > 50`
std::vector<uint8_t> zero(sizeof(int64_t), 0);
auto& stats = fmd.row_groups[0].columns[0].meta_data.statistics;
stats.min = stats.max = stats.min_value = stats.max_value = zero;
if (logical_type.has_value()) { fmd.schema[1].logical_type = *logical_type; }
std::vector<uint8_t> footer;
pq::detail::CompactProtocolWriter{&footer}.write(fmd);
std::ifstream in(filepath, std::ios::binary);
std::vector<char> bytes((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
uint32_t old_len;
std::memcpy(&old_len, bytes.data() + bytes.size() - 8, sizeof(old_len));
bytes.resize(bytes.size() - 8 - old_len);
bytes.insert(bytes.end(), footer.begin(), footer.end());
auto const new_len = static_cast<uint32_t>(footer.size());
bytes.insert(bytes.end(),
reinterpret_cast<char const*>(&new_len),
reinterpret_cast<char const*>(&new_len) + sizeof(new_len));
bytes.insert(bytes.end(), {'P', 'A', 'R', '1'});
auto const patched = temp_env->get_temp_filepath(name + ".parquet");
std::ofstream(patched, std::ios::binary).write(bytes.data(), bytes.size());
std::ifstream in(filepath, std::ios::binary);
std::vector<char> bytes((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
edit_footer(bytes, [&](pq::FileMetaData& fmd) {
// Make the statistics claim every value is 0, so a reader that trusts them prunes `a > 50`
std::vector<uint8_t> zero(sizeof(int64_t), 0);
auto& stats = fmd.row_groups[0].columns[0].meta_data.statistics;
stats.min = stats.max = stats.min_value = stats.max_value = zero;
if (logical_type.has_value()) { fmd.schema[1].logical_type = *logical_type; }
});
auto const patched = temp_env->get_temp_filepath(name + ".parquet");
std::ofstream(patched, std::ios::binary).write(bytes.data(), bytes.size());

return patched;
}

cudf::io::table_with_metadata read_with_filter(std::string const& path)
{
auto literal = cudf::numeric_scalar<int64_t>(50);
auto literal_ref = cudf::ast::literal(literal);
auto col_ref = cudf::ast::column_reference(0);
auto filter = cudf::ast::operation(cudf::ast::ast_operator::GREATER, col_ref, literal_ref);
return cudf::io::read_parquet(
cudf::io::parquet_reader_options::builder(cudf::io::source_info{path}).filter(filter));
}

} // namespace

// Unsupported logical/physical combinations drop the annotation and ignore the statistics, so the
// column is read as plain INT64 and the bogus statistics don't prune. The control (no annotation)
// shows the statistics are otherwise honored.
TEST_F(ParquetReaderTest, UnsupportedLogicalPhysicalCombinationIgnoresAnnotationAndStats)
{
using cudf::io::parquet::IntType;
using cudf::io::parquet::LogicalType;
struct test_case {
std::string name;
std::optional<LogicalType> logical_type;
cudf::size_type expected_rows;
};
std::vector<test_case> const cases{{"Control", std::nullopt, 0},
{"IntegerOnInt64", LogicalType{IntType{32, true}}, 49},
{"DateOnInt64", LogicalType{LogicalType::DATE}, 49}};

for (auto const& tc : cases) {
auto const path = write_patched_int64_file(tc.name, tc.logical_type);
auto const filtered = read_with_filter(path);
EXPECT_EQ(filtered.tbl->view().column(0).type().id(), cudf::type_id::INT64) << tc.name;
EXPECT_EQ(filtered.tbl->num_rows(), tc.expected_rows) << tc.name;
}
}
Loading