From f4d3c308a99be5e2dd887bb8b77653da252fa8da Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 1 Nov 2026 17:46:26 -0600 Subject: [PATCH 1/6] [C++][Parquet] Add fixed-size-list reader baseline benchmark --- .../parquet/arrow/reader_writer_benchmark.cc | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/cpp/src/parquet/arrow/reader_writer_benchmark.cc b/cpp/src/parquet/arrow/reader_writer_benchmark.cc index a5e84243b7ff..b9fc30440fa2 100644 --- a/cpp/src/parquet/arrow/reader_writer_benchmark.cc +++ b/cpp/src/parquet/arrow/reader_writer_benchmark.cc @@ -36,6 +36,7 @@ #include "arrow/array/data.h" #include "arrow/compute/cast.h" #include "arrow/io/memory.h" +#include "arrow/memory_pool.h" #include "arrow/table.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/random.h" @@ -722,6 +723,111 @@ static void BM_ReadListColumn(::benchmark::State& state) { BENCHMARK(BM_ReadListColumn)->Apply(NestedReadArguments); +// Compare the same float values as a flat column and as fixed-size lists. +// Fixture generation, writing, and the checked warm-up read are outside timing. +static void BM_ReadFloatLayout(::benchmark::State& state) { + const bool use_lists = state.range(0) == 1; + const int32_t list_size = static_cast(state.range(1)); + const auto page_version = + state.range(2) == 1 ? ParquetDataPageVersion::V1 : ParquetDataPageVersion::V2; + const bool use_zstd = state.range(3) != 0; + const int64_t page_size = state.range(4); + const int64_t num_values = (BENCHMARK_SIZE / list_size) * list_size; + + ::arrow::random::RandomArrayGenerator rng(500); + auto values = rng.Float32(num_values, /*min=*/-1.0f, /*max=*/1.0f, + /*null_probability=*/0.0); + std::shared_ptr column = values; + if (use_lists) { + auto item = ::arrow::field("item", ::arrow::float32(), /*nullable=*/false); + auto type = ::arrow::fixed_size_list(item, list_size); + PARQUET_ASSIGN_OR_THROW(column, + ::arrow::FixedSizeListArray::FromArrays(values, type)); + } + auto schema = + ::arrow::schema({::arrow::field("column", column->type(), /*nullable=*/false)}); + auto expected = Table::Make(schema, {column}); + EXIT_NOT_OK(expected->ValidateFull()); + + auto properties = + WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->compression(use_zstd ? Compression::ZSTD : Compression::UNCOMPRESSED) + ->data_page_version(page_version) + ->data_pagesize(page_size) + ->build(); + // Preserve the Arrow type so the nested case exercises FixedSizeListReader. + auto arrow_properties = ArrowWriterProperties::Builder().store_schema()->build(); + auto output = CreateOutputStream(); + EXIT_NOT_OK(WriteTable(*expected, ::arrow::default_memory_pool(), output, + expected->num_rows(), properties, arrow_properties)); + PARQUET_ASSIGN_OR_THROW(auto buffer, output->Finish()); + + ArrowReaderProperties reader_properties; + reader_properties.set_use_threads(false); + // ReadTable reads the whole row group; both layouts contain the same number + // of child values. The streaming record-batch size does not apply here. + auto read = [&](::arrow::MemoryPool* pool) { + auto parquet_reader = ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), ReaderProperties(pool)); + PARQUET_ASSIGN_OR_THROW(auto reader, FileReader::Make(pool, std::move(parquet_reader), + reader_properties)); + PARQUET_ASSIGN_OR_THROW(auto table, reader->ReadTable()); + return table; + }; + + // Measure Arrow-managed allocations on one untimed read. This includes the + // decoded output, but excludes the input file and non-pool allocations. + ::arrow::ProxyMemoryPool read_pool(::arrow::default_memory_pool()); + { + auto actual = read(&read_pool); + EXIT_NOT_OK(actual->ValidateFull()); + if (!expected->Equals(*actual)) { + state.SkipWithError( + "Float layout did not round-trip with identical type and values"); + return; + } + } + state.counters["peak_read_bytes"] = static_cast(read_pool.max_memory()); + state.counters["read_allocations"] = static_cast(read_pool.num_allocations()); + state.counters["file_bytes"] = static_cast(buffer->size()); + state.counters["float_values"] = static_cast(num_values); + state.SetLabel(use_lists ? "fixed_size_list" : "flat"); + + // Each iteration opens a fresh reader over the same in-memory file and reads + // the whole column. Reader setup and result destruction are timed as well. + for (auto _ : state) { + auto actual = read(::arrow::default_memory_pool()); + ::benchmark::DoNotOptimize(actual.get()); + } + state.SetItemsProcessed(state.iterations() * num_values); + state.SetBytesProcessed(state.iterations() * num_values * sizeof(float)); +} + +static void FloatLayoutReadArguments(::benchmark::internal::Benchmark* b) { + for (int64_t list_size : {1, 3, 8, 9, 15, 16, 768}) { + for (int64_t page_version : {1, 2}) { + for (int64_t zstd : {0, 1}) { + for (int64_t layout : {0, 1}) { + b->Args({layout, list_size, page_version, zstd, 1024 * 1024}); + } + } + // Also measure many small pages for coordinates and embeddings. + if (list_size == 3 || list_size == 768) { + for (int64_t layout : {0, 1}) { + b->Args({layout, list_size, page_version, 0, 8 * 1024}); + } + } + } + } +} + +BENCHMARK(BM_ReadFloatLayout) + ->Apply(FloatLayoutReadArguments) + ->ArgNames({"layout", "list_size", "page_version", "zstd", "page_bytes"}) + ->UseRealTime(); + static void BM_ReadListOfStructColumn(::benchmark::State& state) { constexpr int64_t kNumValues = BENCHMARK_SIZE / 10; const double null_probability = static_cast(state.range(0)) / 100.0; From 6578d602fbfb07a727e19e128b27f50611ea934d Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Thu, 12 Nov 2026 11:22:13 -0600 Subject: [PATCH 2/6] [C++][Parquet] Fast path for reading fixed-size lists --- .../parquet/arrow/arrow_reader_writer_test.cc | 86 ++++++ cpp/src/parquet/arrow/reader.cc | 35 ++- cpp/src/parquet/column_reader.cc | 155 +++++++++- cpp/src/parquet/column_reader.h | 5 + cpp/src/parquet/column_reader_test.cc | 281 ++++++++++++++++++ cpp/src/parquet/fixed_size_list_internal.h | 168 +++++++++++ cpp/src/parquet/level_decoder_internal.h | 16 +- 7 files changed, 735 insertions(+), 11 deletions(-) create mode 100644 cpp/src/parquet/fixed_size_list_internal.h diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 421769521d1e..8e46ae835fa5 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -3572,6 +3572,92 @@ TEST(ArrowReadWrite, EmptyListView) { ASSERT_EQ(0, list_view.value_sizes()->size()); } +TEST(ArrowReadWrite, FixedSizeListFastPathPagesAndBatches) { + for (int width : {1, 3, 8, 9, 15, 16, 768}) { + for (auto version : {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) { + for (bool dictionary : {false, true}) { + for (bool nullable : {false, true}) { + SCOPED_TRACE(::testing::Message() << width << "/" << int(version) << "/" + << dictionary << "/" << nullable); + ::arrow::Int32Builder values; + for (int i = 0; i < width * 97; ++i) ASSERT_OK(values.Append(i)); + ASSERT_OK_AND_ASSIGN(auto child, values.Finish()); + auto type = ::arrow::fixed_size_list( + ::arrow::field("item", ::arrow::int32(), nullable), width); + ASSERT_OK_AND_ASSIGN(auto array, + ::arrow::FixedSizeListArray::FromArrays(child, type)); + auto table = Table::Make( + ::arrow::schema({::arrow::field("root", type, nullable)}), {array}); + auto wp = WriterProperties::Builder() + .data_page_version(version) + ->data_pagesize(256) + ->write_batch_size(7) + ->compression(Compression::ZSTD) + ->build(); + if (!dictionary) + wp = WriterProperties::Builder(*wp).disable_dictionary()->build(); + auto ap = ArrowWriterProperties::Builder().store_schema()->build(); + auto sink = CreateOutputStream(); + ASSERT_OK(WriteTable(*table, default_memory_pool(), sink, 31, wp, ap)); + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + for (int batch_size : {1, 7, 32, 1024}) { + ArrowReaderProperties rp; + rp.set_batch_size(batch_size); + FileReaderBuilder builder; + ASSERT_OK(builder.Open(std::make_shared(buffer))); + ASSERT_OK_AND_ASSIGN(auto reader, builder.properties(rp)->Build()); + ASSERT_OK_AND_ASSIGN(auto batches, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto actual, + Table::FromRecordBatchReader(batches.get())); + ASSERT_OK(actual->ValidateFull()); + ASSERT_TRUE(table->Equals(*actual)); + } + } + } + } + } +} + +TEST(ArrowReadWrite, FixedSizeListFastPathMixedNullPages) { + auto type = ::arrow::fixed_size_list(::arrow::int32(), 3); + std::string json = "["; + for (int row = 0; row < 200; ++row) { + if (row) json += ","; + if (row == 60) + json += "null"; + else if (row == 120) + json += "[1,null,3]"; + else + json += "[1,2,3]"; + } + json += "]"; + auto array = ::arrow::ArrayFromJSON(type, json); + auto table = Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + for (auto version : {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) { + auto wp = WriterProperties::Builder() + .data_page_version(version) + ->disable_dictionary() + ->data_pagesize(128) + ->write_batch_size(7) + ->build(); + auto ap = ArrowWriterProperties::Builder().store_schema()->build(); + auto sink = CreateOutputStream(); + ASSERT_OK(WriteTable(*table, default_memory_pool(), sink, 200, wp, ap)); + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + for (int batch_size : {1, 13, 200}) { + ArrowReaderProperties rp; + rp.set_batch_size(batch_size); + FileReaderBuilder builder; + ASSERT_OK(builder.Open(std::make_shared(buffer))); + ASSERT_OK_AND_ASSIGN(auto reader, builder.properties(rp)->Build()); + ASSERT_OK_AND_ASSIGN(auto batches, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto actual, Table::FromRecordBatchReader(batches.get())); + ASSERT_OK(actual->ValidateFull()); + ASSERT_TRUE(table->Equals(*actual)); + } + } +} + struct FixedSizeListTestCase { std::shared_ptr type; std::string json; diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc index 872d1b44928d..4d0a7b29181e 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -127,6 +127,8 @@ class ColumnReaderImpl : public ColumnReader { virtual ::arrow::Status BuildArray(int64_t length_upper_bound, std::shared_ptr<::arrow::ChunkedArray>* out) = 0; virtual bool IsOrHasRepeatedChild() const = 0; + virtual bool SetFixedSizeList(int32_t /*width*/) { return false; } + virtual bool dense_fixed_size_list() const { return false; } }; namespace { @@ -498,6 +500,12 @@ class LeafReader : public ColumnReaderImpl { } bool IsOrHasRepeatedChild() const final { return false; } + bool SetFixedSizeList(int32_t width) override { + return record_reader_->SetFixedSizeList(width); + } + bool dense_fixed_size_list() const override { + return record_reader_->dense_fixed_size_list(); + } Status LoadBatch(int64_t records_to_read) final { BEGIN_PARQUET_CATCH_EXCEPTIONS @@ -686,6 +694,7 @@ class ListReader : public ColumnReaderImpl { const std::shared_ptr field() override { return field_; } protected: + ColumnReaderImpl* item_reader() const { return item_reader_.get(); } std::shared_ptr ctx_; private: @@ -730,7 +739,31 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader { ::parquet::internal::LevelInfo level_info, std::unique_ptr child_reader) : ListReader(std::move(ctx), std::move(field), level_info, - std::move(child_reader)) {} + std::move(child_reader)) { + const auto& type = + checked_cast(*this->field()->type()); + this->item_reader()->SetFixedSizeList(type.list_size()); + } + + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override { + if (!item_reader()->dense_fixed_size_list()) { + return ListReader::BuildArray(length_upper_bound, out); + } + const auto& type = checked_cast(*field()->type()); + std::shared_ptr child; + RETURN_NOT_OK(item_reader()->BuildArray(length_upper_bound, &child)); + ARROW_ASSIGN_OR_RAISE(auto child_data, ChunksToSingle(*child)); + const int64_t rows = child_data->length / type.list_size(); + if (child_data->length % type.list_size() != 0) { + return Status::Invalid("Incomplete fixed-size list at end of column"); + } + auto data = ::arrow::ArrayData::Make(field()->type(), rows, {nullptr}, + {std::move(child_data)}, 0); + *out = std::make_shared(::arrow::MakeArray(std::move(data))); + return Status::OK(); + } + ::arrow::Result> AssembleArray( std::shared_ptr data) final { DCHECK_EQ(data->buffers.size(), 2); diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 6aca8a23ce16..921174c15792 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -47,6 +47,7 @@ #include "parquet/encryption/encryption_internal.h" #include "parquet/encryption/internal_file_decryptor.h" #include "parquet/exception.h" +#include "parquet/fixed_size_list_internal.h" #include "parquet/level_conversion.h" #include "parquet/level_decoder_internal.h" #include "parquet/properties.h" @@ -918,6 +919,20 @@ class LevelSinkBuffer : public DataSinkBuffer { return LevelSinkBuffer(AllocateBuffer(pool)); } + void AppendFixed(int64_t count, int16_t max_level, int32_t width, bool repetitions) { + Base::ReadFromCallback( + [&]() { + auto* out = Base::write_start(); + if (repetitions) { + for (int64_t i = 0; i < count; ++i) out[i] = i % width != 0; + } else { + std::fill_n(out, count, max_level); + } + return count; + }, + count); + } + /// Decode levels into the sink buffer. void Decode(auto& decoder, int32_t batch_size) { const auto decoded = Base::ReadFromCallback( @@ -964,6 +979,15 @@ class ValiditySinkBuffer : private DataSinkBuffer return ValiditySinkBuffer(AllocateBuffer(pool)); } + void AppendPresent(int32_t count) { + Base::ReadFromCallback( + [&]() { + ::arrow::bit_util::SetBitsTo(data(), values_count(), count, true); + return count; + }, + count); + } + struct ReadResult { int64_t values_read = 0; int64_t null_count = 0; @@ -1172,6 +1196,9 @@ class ColumnChunkReader { return rep_levels_decoder_.Decode(levels, batch_size); } + int64_t page_number() const { return page_number_; } + int32_t page_value_count() const { return num_buffered_values_; } + // Available values in the current data page, value includes repeated values and nulls. int32_t available_values_current_page() const { const int32_t out = num_buffered_values_ - num_decoded_values_; @@ -1223,6 +1250,7 @@ class ColumnChunkReader { // than levels, and this tells you how many encoded levels there are in that // case. int32_t num_buffered_values_ = 0; + int64_t page_number_ = 0; // The number of values from the current data page that have been decoded // into memory or skipped over. int32_t num_decoded_values_ = 0; @@ -1365,6 +1393,7 @@ void ColumnChunkReader::InitializeDataPage(std::shared_ptr page) { // Some compiler do not support `static_assert(false, ...)` in a discarded branch static_assert(!std::is_same_v, "Unknown data page type"); } + ++page_number_; num_buffered_values_ = page->num_values(); num_decoded_values_ = 0; InitializeDataDecoder(*page, byte_size); @@ -1745,13 +1774,21 @@ class TypedRecordReader : public ColumnChunkReader(this)->MaterializeFixedLevels(); + return def_levels_.data(); + } - int16_t* rep_levels() const final { return rep_levels_.data(); } + int16_t* rep_levels() const final { + const_cast(this)->MaterializeFixedLevels(); + return rep_levels_.data(); + } - int64_t levels_position() const final { return levels_position_; } + int64_t levels_position() const final { return levels_position_ + fixed_levels_; } - int64_t levels_written() const final { return def_levels_.values_count(); } + int64_t levels_written() const final { + return def_levels_.values_count() + fixed_levels_; + } int64_t null_count() const final { return null_count_; } @@ -1774,6 +1811,19 @@ class TypedRecordReader : public ColumnChunkReader && + !std::is_same_v && + !std::is_same_v && !std::is_same_v) + return false; + if (width <= 0 || this->max_rep_level() != 1 || read_dictionary()) return false; + fixed_width_ = width; + return true; + } + bool dense_fixed_size_list() const override { + return fixed_width_ > 0 && !generic_batch_ && fixed_phase_ == 0; + } + // Skip records that we have in our buffer. This function is only for // non-repeated fields. int64_t SkipRecordsInBufferNonRepeated(int64_t num_records); @@ -1800,8 +1850,8 @@ class TypedRecordReader : public ColumnChunkReader ReleaseValues() override { @@ -1864,6 +1914,52 @@ class TypedRecordReader : public ColumnChunkReader const ValueSink& { return value_sink_; } private: + // Fast batches retain only a count of predictable levels. Materialize them + // lazily if a caller needs levels or a later page needs generic reconstruction. + void MaterializeFixedLevels() { + if (fixed_levels_ == 0) return; + def_levels_.AppendFixed(fixed_levels_, this->max_def_level(), fixed_width_, false); + rep_levels_.AppendFixed(fixed_levels_, this->max_def_level(), fixed_width_, true); + levels_position_ += fixed_levels_; + fixed_levels_ = 0; + } + void UseGeneric() { + MaterializeFixedLevels(); + generic_batch_ = true; + fixed_page_ = -1; + fixed_phase_ = 0; + // Fast reads leave the level decoders untouched. Synchronize once before + // the generic path starts decoding the unconsumed suffix of this page. + const int32_t consumed = + this->page_value_count() - this->available_values_current_page(); + const int32_t behind = + this->def_levels_decoder_.remaining() - this->available_values_current_page(); + if (consumed > 0 && behind > 0) { + CheckNumberDecoded(this->def_levels_decoder_.Skip(behind), behind); + CheckNumberDecoded(this->rep_levels_decoder_.Skip(behind), behind); + } + } + bool MatchFixedPage() { + if (fixed_page_ == this->page_number()) return true; + if (this->available_values_current_page() != this->page_value_count()) return false; + const auto& def = this->def_levels_decoder_; + const auto& rep = this->rep_levels_decoder_; + using Matcher = internal::FixedSizeListLevelMatcher; + if (!def.is_rle() || !rep.is_rle() || + !Matcher::AllPresent(def.encoded_rle(), this->max_def_level(), + this->page_value_count()) || + !Matcher::Repetitions(rep.encoded_rle(), this->page_value_count(), fixed_width_, + fixed_phase_)) + return false; + fixed_page_ = this->page_number(); + return true; + } + + int32_t fixed_width_ = 0; + int32_t fixed_phase_ = 0; + int64_t fixed_levels_ = 0; + int64_t fixed_page_ = -1; + bool generic_batch_ = false; ValueSink value_sink_; /// \brief Each bit corresponds to one element in 'values_' and specifies if it /// is null or not null. @@ -1921,6 +2017,42 @@ int64_t TypedRecordReader::ReadRecords(int64_t num_records) { break; } + if (fixed_width_ > 0 && !generic_batch_ && !has_buffered_levels()) { + if (MatchFixedPage()) { + // A completed row at a page end is only counted after the next page + // proves its first marker is zero (or EOF completes the last row). + if (!at_record_start_ && fixed_phase_ == 0) { + ++records_read; + at_record_start_ = true; + if (records_read == num_records) break; + } + const int64_t remaining_records = num_records - records_read; + int64_t target; + if (MultiplyWithOverflow(remaining_records, int64_t{fixed_width_}, &target)) { + throw ParquetException("Fixed-size list read size overflow"); + } + const int32_t take = + narrow_min(target - fixed_phase_, this->available_values_current_page()); + if (auto* dict = this->PopNewDictionary()) value_sink_.OnNewDictionary(*dict); + value_sink_.ReadValuesDense(*this->current_decoder_.get(), take); + valid_bits_.AppendPresent(take); + fixed_levels_ += take; + const int64_t progressed = static_cast(fixed_phase_) + take; + records_read += progressed / fixed_width_; + fixed_phase_ = static_cast(progressed % fixed_width_); + this->MarkValuesAsConsumed(take); + at_record_start_ = fixed_phase_ == 0; + if (at_record_start_ && this->available_values_current_page() == 0) { + --records_read; + at_record_start_ = false; + } + continue; + } + UseGeneric(); + } else if (fixed_width_ > 0 && !generic_batch_) { + UseGeneric(); + } + /// We perform multiple batch reads until we either exhaust the row group /// or observe the desired number of records const int32_t batch_size = @@ -2068,6 +2200,7 @@ void TypedRecordReader::SkipValuesInPage(int64_t num_values) { template int64_t TypedRecordReader::SkipRecords(int64_t num_records) { if (num_records == 0) return 0; + if (fixed_width_ > 0) UseGeneric(); // Top level required field. Number of records equals number of levels, // and there is no read-ahead for levels. @@ -2153,13 +2286,17 @@ template void TypedRecordReader::Reserve(int64_t extra_values) { value_sink_.ReserveValues(extra_values); valid_bits_.ReserveValues(extra_values); // potentially no-op if void - def_levels_.ReserveValues(extra_values); // potentially no-op if void - rep_levels_.ReserveValues(extra_values); // potentially no-op if void + if (fixed_width_ == 0 || generic_batch_) { + def_levels_.ReserveValues(extra_values); // potentially no-op if void + rep_levels_.ReserveValues(extra_values); // potentially no-op if void + } } template void TypedRecordReader::Reset() { null_count_ = 0; + fixed_levels_ = 0; + generic_batch_ = has_buffered_levels() || !at_record_start_; value_sink_.ResetValues(); valid_bits_.ResetValues(); // potentially no-op if void // Must keep eagerly decoded levels @@ -2171,6 +2308,8 @@ void TypedRecordReader::Reset() { template void TypedRecordReader::SetPageReader(std::unique_ptr reader) { at_record_start_ = true; + fixed_phase_ = 0; + fixed_page_ = -1; Base::SetPageReader(std::move(reader)); } diff --git a/cpp/src/parquet/column_reader.h b/cpp/src/parquet/column_reader.h index b51cb255f7fa..f5623915005a 100644 --- a/cpp/src/parquet/column_reader.h +++ b/cpp/src/parquet/column_reader.h @@ -336,6 +336,11 @@ class PARQUET_EXPORT RecordReader { /// \return number of records read virtual int64_t ReadRecords(int64_t num_records) = 0; + // Internal Arrow list-reader contract. Ordinary record-reader callers retain + // fully materialized levels unless they explicitly configure a known width. + virtual bool SetFixedSizeList(int32_t /*width*/) { return false; } + virtual bool dense_fixed_size_list() const { return false; } + /// \brief Attempt to skip indicated number of records from column chunk. /// Note that for repeated fields, a record may have more than one value /// and all of them are skipped. diff --git a/cpp/src/parquet/column_reader_test.cc b/cpp/src/parquet/column_reader_test.cc index 1b636d178305..ac8a1a985477 100644 --- a/cpp/src/parquet/column_reader_test.cc +++ b/cpp/src/parquet/column_reader_test.cc @@ -37,6 +37,7 @@ #include "arrow/util/macros.h" #include "parquet/column_page.h" #include "parquet/column_reader.h" +#include "parquet/fixed_size_list_internal.h" #include "parquet/schema.h" #include "parquet/test_util.h" #include "parquet/types.h" @@ -2003,5 +2004,285 @@ INSTANTIATE_TEST_SUITE_P( : "NoLevelsOptimization"); }); +namespace { + +void AppendLevelHeader(uint32_t value, std::vector* bytes) { + do { + uint8_t byte = value & 127; + value >>= 7; + bytes->push_back(byte | (value ? 128 : 0)); + } while (value); +} + +std::vector PackedFixedLevels(int32_t width, int32_t phase, int32_t count) { + std::vector bytes; + AppendLevelHeader(((count + 7) / 8) * 2 + 1, &bytes); + for (int32_t i = 0; i < count; i += 8) { + uint8_t byte = 0; + for (int bit = 0; bit < 8; ++bit) { + if ((static_cast(phase) + i + bit) % width != 0) byte |= 1U << bit; + } + bytes.push_back(byte); + } + return bytes; +} + +} // namespace + +TEST(FixedSizeListLevels, PackedPhasesAndNearMisses) { + using Matcher = internal::FixedSizeListLevelMatcher; + for (int32_t width : {1, 3, 8, 9, 15, 16, 768}) { + for (int32_t phase : {0, width / 2, width - 1}) { + for (int32_t count : {1, 7, 8, 9, 63, 64, 65, 4097}) { + SCOPED_TRACE(::testing::Message() << width << "/" << phase << "/" << count); + auto bytes = PackedFixedLevels(width, phase, count); + ASSERT_TRUE(Matcher::Repetitions(bytes, count, width, phase)); + // A final logical bit mismatch must be rejected; final padding is ignored. + bytes.back() ^= 1U << ((count - 1) % 8); + ASSERT_FALSE(Matcher::Repetitions(bytes, count, width, phase)); + bytes.pop_back(); + ASSERT_FALSE(Matcher::Repetitions(bytes, count, width, phase)); + } + } + } +} + +TEST(FixedSizeListLevels, RepeatedAndMixedRuns) { + using Matcher = internal::FixedSizeListLevelMatcher; + for (int32_t width : {1, 3, 8, 9, 15, 16, 768}) { + std::vector bytes; + const int32_t packed = width * 8; + bytes = PackedFixedLevels(width, 0, packed); + for (int row = 0; row < 20; ++row) { + AppendLevelHeader(2, &bytes); + bytes.push_back(0); + if (width > 1) { + AppendLevelHeader(2 * (width - 1), &bytes); + bytes.push_back(1); + } + } + ASSERT_TRUE(Matcher::Repetitions(bytes, packed + 20 * width, width, 0)); + ASSERT_FALSE(Matcher::Repetitions(bytes, packed + 20 * width - 1, width, 0)); + bytes.back() = 2; + ASSERT_FALSE(Matcher::Repetitions(bytes, packed + 20 * width, width, 0)); + } + for (const auto& bad : std::vector>{ + {}, {0}, {1}, {128}, {255, 255, 255, 255, 255}, {2}, {3}, {2, 2}}) { + ASSERT_FALSE(Matcher::Repetitions(bad, 1, 3, 0)); + } +} + +// Extracted from parquet-java 1.16.0 files: required LIST, +// 32 rows, widths 1/3/8/9/15/16/768, dictionary disabled, 4096-byte page target, +// PARQUET_1_0 and PARQUET_2_0 writers. Duplicate streams are stored only once. +// Keep the encoded bytes: regenerating these with the C++ encoder would lose +// the independent writer's choices of packed/RLE run boundaries. +TEST(FixedSizeListLevels, ParquetJavaStreams) { + struct Case { + int32_t width; + int32_t count; + std::vector definitions; + std::vector repetitions; + }; + const std::vector cases = { + {1, 32, {64, 1}, {64, 0}}, + {15, 480, {192, 7, 1}, {5, 254, 127, 28, 1, 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, + 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, + 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, + 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, + 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, 5, 254, 127, 28, 1, + 5, 254, 127, 28, 1}}, + {16, + 512, + {128, 8, 1}, + {3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, 3, 254, 16, 1, + 3, 254, 16, 1, 3, 254, 16, 1}}, + {3, + 96, + {192, 1, 1}, + {25, 182, 109, 219, 182, 109, 219, 182, 109, 219, 182, 109, 219}}, + {768, 7680, {128, 120, 1}, {3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1}}, + {768, 1536, {128, 24, 1}, {3, 254, 240, 11, 1, 3, 254, 240, 11, 1}}, + {768, + 24576, + {128, 128, 3, 1}, + {3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, + 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1, 3, 254, 240, 11, 1}}, + {8, 256, {128, 4, 1}, {65, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, + 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254}}, + {9, 288, {192, 4, 1}, {17, 254, 253, 251, 247, 239, 223, 191, 127, 16, 1, + 17, 254, 253, 251, 247, 239, 223, 191, 127, 16, 1, + 17, 254, 253, 251, 247, 239, 223, 191, 127, 16, 1, + 17, 254, 253, 251, 247, 239, 223, 191, 127, 16, 1}}, + }; + for (const auto& c : cases) { + SCOPED_TRACE(::testing::Message() << c.width << "/" << c.count); + using Matcher = internal::FixedSizeListLevelMatcher; + ASSERT_TRUE(Matcher::AllPresent(c.definitions, 1, c.count)); + ASSERT_TRUE(Matcher::Repetitions(c.repetitions, c.count, c.width, 0)); + LevelDecoder decoder; + decoder.SetDataV2(static_cast(c.repetitions.size()), 1, c.count, + c.repetitions.data()); + std::vector levels(c.count); + ASSERT_EQ(decoder.Decode(c.count, levels.data()), c.count); + for (int i = 0; i < c.count; ++i) ASSERT_EQ(levels[i], i % c.width != 0); + + // Exercise the real record reader with these independent writer streams, + // not only the recognition helper. Values are PLAIN little-endian int32s. + std::vector bytes(c.repetitions); + bytes.insert(bytes.end(), c.definitions.begin(), c.definitions.end()); + for (int32_t i = 0; i < c.count; ++i) { + for (int b = 0; b < 4; ++b) bytes.push_back((i >> (b * 8)) & 255); + } + auto buffer = ::arrow::Buffer::FromVector(std::move(bytes)); + auto page = std::make_shared( + buffer, c.count, 0, c.count / c.width, Encoding::PLAIN, + static_cast(c.definitions.size()), + static_cast(c.repetitions.size()), buffer->size()); + auto column = GroupNode::Make( + "p", Repetition::REQUIRED, + {GroupNode::Make( + "list", Repetition::REPEATED, + {PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32)})}); + SchemaDescriptor schema; + schema.Init(GroupNode::Make("root", Repetition::REQUIRED, {column})); + const auto* descr = schema.Column(0); + auto reader = internal::RecordReader::Make( + {.descr = descr, .leaf_info = ComputeLevelInfo(descr)}); + ASSERT_TRUE(reader->SetFixedSizeList(c.width)); + std::vector> pages{page}; + reader->SetPageReader(std::make_unique(pages)); + int32_t value_offset = 0; + while (value_offset < c.count) { + const int64_t rows = reader->ReadRecords(3); + ASSERT_GT(rows, 0); + ASSERT_TRUE(reader->dense_fixed_size_list()); + ASSERT_EQ(reader->values_written(), rows * c.width); + const auto* values = reinterpret_cast(reader->values()); + for (int64_t i = 0; i < reader->values_written(); ++i) { + ASSERT_EQ(values[i], value_offset++); + } + reader->Reset(); + } + } +} + +TEST(FixedSizeListLevels, DefinitionGate) { + using Matcher = internal::FixedSizeListLevelMatcher; + for (int16_t level : {1, 2, 3, 256, 32767}) { + std::vector bytes; + AppendLevelHeader(2 * 12345, &bytes); + bytes.push_back(level & 255); + if (level > 255) bytes.push_back(level >> 8); + ASSERT_TRUE(Matcher::AllPresent(bytes, level, 12345)); + ASSERT_FALSE(Matcher::AllPresent(bytes, level, 12344)); + bytes.back() ^= 1; + ASSERT_FALSE(Matcher::AllPresent(bytes, level, 12345)); + } +} + +TEST(FixedSizeListRecordReader, DifferentialPagesBatchesAndSkips) { + for (int32_t width : {1, 3, 8, 9, 15, 16, 768}) { + for (bool irregular : {false, true}) { + auto column = GroupNode::Make( + "p", Repetition::OPTIONAL, + {GroupNode::Make( + "list", Repetition::REPEATED, + {PrimitiveNode::Make("element", Repetition::OPTIONAL, Type::INT32)})}); + SchemaDescriptor schema; + schema.Init(GroupNode::Make("root", Repetition::REQUIRED, {column})); + const auto* descr = schema.Column(0); + std::vector defs, reps; + for (int row = 0; row < 100; ++row) { + const int length = irregular && row == 30 ? width + 1 : width; + for (int i = 0; i < length; ++i) { + reps.push_back(i == 0 ? 0 : 1); + defs.push_back(irregular && row == 60 ? 2 : 3); + } + } + std::vector> pages; + // Split both inside records and exactly at record boundaries. + size_t start = 0; + for (int page = 0; start < defs.size(); ++page) { + const size_t length = std::min( + defs.size() - start, page % 2 == 0 ? 12 * width : 7 * width + 1); + std::vector d(defs.begin() + start, defs.begin() + start + length); + std::vector r(reps.begin() + start, reps.begin() + start + length); + std::vector values; + for (size_t i = 0; i < length; ++i) { + if (d[i] == 3) values.push_back(static_cast(start + i)); + } + pages.push_back( + test::MakeDataPage(descr, values, static_cast(length), + Encoding::PLAIN, nullptr, 0, d, 3, r, 1)); + start += length; + } + for (bool skip : {false, true}) { + auto generic = internal::RecordReader::Make( + {.descr = descr, .leaf_info = ComputeLevelInfo(descr)}); + auto fast = internal::RecordReader::Make( + {.descr = descr, .leaf_info = ComputeLevelInfo(descr)}); + ASSERT_TRUE(fast->SetFixedSizeList(width)); + generic->SetPageReader(std::make_unique(pages)); + fast->SetPageReader(std::make_unique(pages)); + for (int step = 0; step < 35; ++step) { + SCOPED_TRACE(::testing::Message() + << width << "/" << irregular << "/" << skip << "/" << step); + if (skip && step % 3 == 1) { + ASSERT_EQ(generic->SkipRecords(2), fast->SkipRecords(2)); + } else { + ASSERT_EQ(generic->ReadRecords(4), fast->ReadRecords(4)); + if (step == 0) ASSERT_TRUE(fast->dense_fixed_size_list()); + ASSERT_EQ(generic->values_written(), fast->values_written()); + ASSERT_EQ(generic->null_count(), fast->null_count()); + const auto* expected = reinterpret_cast(generic->values()); + const auto* actual = reinterpret_cast(fast->values()); + // Nullable leaf slots have unspecified values; compare present slots. + auto expected_valid = generic->ReleaseIsValid(); + auto actual_valid = fast->ReleaseIsValid(); + for (int64_t i = 0; i < generic->values_written(); ++i) { + const bool present = + !expected_valid || ::arrow::bit_util::GetBit(expected_valid->data(), i); + ASSERT_EQ(present, !actual_valid || + ::arrow::bit_util::GetBit(actual_valid->data(), i)); + if (present) { + ASSERT_EQ(expected[i], actual[i]); + } + } + ASSERT_EQ(generic->levels_position(), fast->levels_position()); + const int64_t count = generic->levels_position(); + ASSERT_EQ( + std::vector(generic->def_levels(), + generic->def_levels() + count), + std::vector(fast->def_levels(), fast->def_levels() + count)); + ASSERT_EQ( + std::vector(generic->rep_levels(), + generic->rep_levels() + count), + std::vector(fast->rep_levels(), fast->rep_levels() + count)); + } + generic->Reset(); + fast->Reset(); + } + } + } + } +} + } // namespace test } // namespace parquet diff --git a/cpp/src/parquet/fixed_size_list_internal.h b/cpp/src/parquet/fixed_size_list_internal.h new file mode 100644 index 000000000000..60f0f4af5e48 --- /dev/null +++ b/cpp/src/parquet/fixed_size_list_internal.h @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace parquet::internal { + +// Non-consuming recognition of the hybrid RLE/bit-packed streams used for a +// single repeated leaf. Unrecognized layouts are handled by the normal decoder. +class FixedSizeListLevelMatcher { + public: + static bool AllPresent(std::span bytes, int16_t max_level, + int32_t count) { + if (max_level == 0) return bytes.empty(); + uint32_t header; + if (!ReadHeader(&bytes, &header) || (header & 1) || + (header >> 1) != static_cast(count)) + return false; + const int value_bytes = max_level <= 255 ? 1 : 2; + if (bytes.size() != static_cast(value_bytes)) return false; + const int value = bytes[0] | (value_bytes == 2 ? bytes[1] << 8 : 0); + return value == max_level; + } + + static bool Repetitions(std::span bytes, int32_t count, int32_t width, + int32_t phase) { + if (count <= 0 || width <= 0 || phase < 0 || phase >= width) return false; + // Cache every small-width phase once. The hot loop compares eight encoded + // bytes at a time rather than regenerating the expected bits for each word. + static const auto patterns = [] { + std::array, 16>, 17> out{}; + for (int w = 1; w <= 16; ++w) { + for (int p = 0; p < w; ++p) { + for (int b = 0; b < 8; ++b) { + out[w][p][b] = ExpectedByte(w, (p + b * 8) % w); + } + } + } + return out; + }(); + const auto original = bytes; + const int32_t initial_phase = phase; + bool tried_stride = false; + int64_t remaining = count; + while (remaining > 0) { + uint32_t header; + if (!ReadHeader(&bytes, &header) || header < 2) return false; + if (!(header & 1)) { + const int64_t run = header >> 1; + if (bytes.empty() || run > remaining) return false; + const uint8_t value = bytes.front(); + bytes = bytes.subspan(1); + if (value == 0) { + if (phase != 0 || (width != 1 && run != 1)) return false; + } else if (value == 1) { + if (phase == 0 || run > width - phase) return false; + } else { + return false; + } + phase = static_cast((phase + run) % width); + remaining -= run; + } else { + // One packed group contains eight one-bit repetition levels. Only the + // final group may contain padding beyond the page's logical count. + const uint64_t groups = header >> 1; + if (groups > bytes.size() || groups * 8 > static_cast(remaining) + 7) + return false; + int64_t bits = std::min(remaining, groups * 8); + const int64_t logical_bits = bits; + size_t offset = 0; + // The packed pattern repeats after width/gcd(width,8) bytes. Small + // widths therefore use word-sized comparisons without expanding levels. + while (bits >= 64) { + if (width <= 16) { + if (std::memcmp(bytes.data() + offset, patterns[width][phase].data(), 8) != + 0) { + return false; + } + phase = (phase + 64) % width; + offset += 8; + bits -= 64; + continue; + } + uint8_t expected[8]; + int32_t next = phase; + for (auto& byte : expected) { + byte = ExpectedByte(width, next); + next = static_cast((static_cast(next) + 8) % width); + } + if (std::memcmp(bytes.data() + offset, expected, 8) != 0) return false; + phase = next; + offset += 8; + bits -= 64; + } + while (bits > 0) { + const int take = static_cast(std::min(8, bits)); + const uint8_t mask = static_cast((1U << take) - 1); + if ((bytes[offset] & mask) != (ExpectedByte(width, phase) & mask)) { + return false; + } + phase = static_cast((static_cast(phase) + take) % width); + ++offset; + bits -= take; + } + bytes = bytes.subspan(static_cast(groups)); + remaining -= logical_bits; + } + // Once a validated encoded prefix returns to its incoming list phase, + // exact copies describe the same whole number of lists. Compare those + // strides directly; a different legal run split resumes the run walker. + if (width > 8 && !tried_stride && phase == initial_phase) { + tried_stride = true; + const int64_t period = count - remaining; + const size_t stride = original.size() - bytes.size(); + while (remaining >= period && bytes.size() >= stride && + std::memcmp(bytes.data(), original.data(), stride) == 0) { + bytes = bytes.subspan(stride); + remaining -= period; + } + } + } + return bytes.empty(); + } + + private: + static uint8_t ExpectedByte(int32_t width, int32_t phase) { + if (width == 1) return 0; + uint8_t byte = 0xff; + int64_t zero = phase == 0 ? 0 : width - phase; + for (; zero < 8; zero += width) byte &= static_cast(~(1U << zero)); + return byte; + } + + static bool ReadHeader(std::span* bytes, uint32_t* header) { + *header = 0; + for (int shift = 0; shift < 35; shift += 7) { + if (bytes->empty()) return false; + const uint8_t byte = bytes->front(); + *bytes = bytes->subspan(1); + if (shift == 28 && byte > 15) return false; + *header |= static_cast(byte & 127) << shift; + if (!(byte & 128)) return true; + } + return false; + } +}; + +} // namespace parquet::internal diff --git a/cpp/src/parquet/level_decoder_internal.h b/cpp/src/parquet/level_decoder_internal.h index e956abc93eb1..bd3690bd26c7 100644 --- a/cpp/src/parquet/level_decoder_internal.h +++ b/cpp/src/parquet/level_decoder_internal.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -159,7 +160,12 @@ class PageLevelDecoder { /// Return the number of values left to be decoded. int32_t remaining() const { return num_values_remaining_; } + std::span encoded_rle() const { return encoded_rle_; } + bool is_rle() const { return is_rle_; } + private: + std::span encoded_rle_; + bool is_rle_ = false; std::variant decoder_; /// Number of values remaining. The underlying decoder zero pads bit packed values /// up to a multiple of 8 so it cannot know the exact number of remaining values. @@ -249,6 +255,8 @@ int32_t PageLevelDecoder::SetDataV1(Encoding::type encoding, if (ARROW_PREDICT_FALSE(data_size < 0 || data_size > max_data_size - 4)) { throw ParquetException("Received invalid number of bytes (corrupt data page?)"); } + encoded_rle_ = {data + 4, static_cast(data_size)}; + is_rle_ = true; decoder_ = RleDec(/* data= */ data + 4, /* data_size= */ data_size, params); return 4 + data_size; } @@ -262,6 +270,8 @@ int32_t PageLevelDecoder::SetDataV1(Encoding::type encoding, if (ARROW_PREDICT_FALSE(data_size < 0 || data_size > max_data_size)) { throw ParquetException("Received invalid number of bytes (corrupt data page?)"); } + encoded_rle_ = {}; + is_rle_ = false; decoder_ = BitDec(/* data= */ data, /* data_size= */ data_size, params); return data_size; } @@ -279,6 +289,8 @@ void PageLevelDecoder::SetDataV2(const uint8_t* data, int32_t da } num_values_remaining_ = params.value_count; max_level_ = params.max_level; + encoded_rle_ = {data, static_cast(data_size)}; + is_rle_ = true; decoder_ = RleDec(data, data_size, params); } @@ -302,8 +314,8 @@ int32_t PageLevelDecoder::Skip(int32_t batch_size) { } template -auto PageLevelDecoder::CountUpTo(int16_t value, - int32_t batch_size) -> CountUpToResult { +auto PageLevelDecoder::CountUpTo(int16_t value, int32_t batch_size) + -> CountUpToResult { const int32_t num_values = std::min(num_values_remaining_, batch_size); const auto result = std::visit([&](auto& dec) { return dec.CountUpTo(value, num_values); }, decoder_); From b915f6de82d0ec317d6bf7e2fb9830664aca949c Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Tue, 17 Nov 2026 16:09:29 -0600 Subject: [PATCH 3/6] Fix CountUpTo formatting --- cpp/src/parquet/level_decoder_internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/level_decoder_internal.h b/cpp/src/parquet/level_decoder_internal.h index bd3690bd26c7..17904d69fd61 100644 --- a/cpp/src/parquet/level_decoder_internal.h +++ b/cpp/src/parquet/level_decoder_internal.h @@ -314,8 +314,8 @@ int32_t PageLevelDecoder::Skip(int32_t batch_size) { } template -auto PageLevelDecoder::CountUpTo(int16_t value, int32_t batch_size) - -> CountUpToResult { +auto PageLevelDecoder::CountUpTo(int16_t value, + int32_t batch_size) -> CountUpToResult { const int32_t num_values = std::min(num_values_remaining_, batch_size); const auto result = std::visit([&](auto& dec) { return dec.CountUpTo(value, num_values); }, decoder_); From ae9c87ff8ae86d3c2f6082952ae94989152e44b3 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Tue, 17 Nov 2026 16:25:07 -0600 Subject: [PATCH 4/6] Explain fixed-size list reader behavior --- cpp/src/parquet/arrow/reader.cc | 1 + cpp/src/parquet/column_reader.cc | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc index cfc77e6276bf..9a624c1b0898 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -762,6 +762,7 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader { if (child_data->length % type.list_size() != 0) { return Status::Invalid("Incomplete fixed-size list at end of column"); } + // Every list and value in this batch is present so no null bitmap is needed auto data = ::arrow::ArrayData::Make(field()->type(), rows, {nullptr}, {std::move(child_data)}, 0); *out = std::make_shared(::arrow::MakeArray(std::move(data))); diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 9767fd4fc9b4..258974e0c5c6 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -1933,8 +1933,7 @@ class TypedRecordReader : public ColumnChunkReader const ValueSink& { return value_sink_; } private: - // Fast batches retain only a count of predictable levels. Materialize them - // lazily if a caller needs levels or a later page needs generic reconstruction. + // Rebuild the skipped markers only when the existing reader or a caller needs them void MaterializeFixedLevels() { if (fixed_levels_ == 0) return; def_levels_.AppendFixed(fixed_levels_, this->max_def_level(), fixed_width_, false); @@ -2053,6 +2052,7 @@ int64_t TypedRecordReader::ReadRecords(int64_t num_records) { const int32_t take = narrow_min(target - fixed_phase_, this->available_values_current_page()); if (auto* dict = this->PopNewDictionary()) value_sink_.OnNewDictionary(*dict); + // The dictionary decoder gives us the actual values for the lists value_sink_.ReadValuesDense(*this->current_decoder_.get(), take); valid_bits_.AppendPresent(take); fixed_levels_ += take; From 40ea43d3d4d3769b85d4a3c19bcea9ac54f9dbfa Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Tue, 17 Nov 2026 16:51:53 -0600 Subject: [PATCH 5/6] Test fast reads on Arrow-written lists --- cpp/src/parquet/arrow/arrow_reader_writer_test.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index ba68268ca2e8..d33d540fcd5a 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -3643,6 +3643,20 @@ TEST(ArrowReadWrite, FixedSizeListFastPathPagesAndBatches) { auto sink = CreateOutputStream(); ASSERT_OK(WriteTable(*table, default_memory_pool(), sink, 31, wp, ap)); ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + if (!nullable && !dictionary && width == 3) { + auto file = ParquetFileReader::Open(std::make_shared(buffer)); + const auto* column = file->metadata()->schema()->Column(0); + auto reader = ::parquet::internal::RecordReader::Make( + column, ::parquet::internal::LevelInfo::ComputeLevelInfo(column)); + ASSERT_TRUE(reader->SetFixedSizeList(width)); + reader->SetPageReader(file->RowGroup(0)->GetColumnPageReader(0)); + + ASSERT_EQ(reader->ReadRecords(7), 7); + ASSERT_TRUE(reader->dense_fixed_size_list()); + ASSERT_EQ(reader->values_written(), 7 * width); + } + for (int batch_size : {1, 7, 32, 1024}) { ArrowReaderProperties rp; rp.set_batch_size(batch_size); From ca51df148b3141ed5fc6a81b79319aa25bbf5196 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Tue, 17 Nov 2026 18:50:56 -0600 Subject: [PATCH 6/6] Add braces around fixed-size list test assertion --- cpp/src/parquet/column_reader_test.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/column_reader_test.cc b/cpp/src/parquet/column_reader_test.cc index ac8a1a985477..2b87e8cbf5f7 100644 --- a/cpp/src/parquet/column_reader_test.cc +++ b/cpp/src/parquet/column_reader_test.cc @@ -2248,7 +2248,9 @@ TEST(FixedSizeListRecordReader, DifferentialPagesBatchesAndSkips) { ASSERT_EQ(generic->SkipRecords(2), fast->SkipRecords(2)); } else { ASSERT_EQ(generic->ReadRecords(4), fast->ReadRecords(4)); - if (step == 0) ASSERT_TRUE(fast->dense_fixed_size_list()); + if (step == 0) { + ASSERT_TRUE(fast->dense_fixed_size_list()); + } ASSERT_EQ(generic->values_written(), fast->values_written()); ASSERT_EQ(generic->null_count(), fast->null_count()); const auto* expected = reinterpret_cast(generic->values());