Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions cpp/src/parquet/arrow/arrow_reader_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3615,6 +3615,106 @@ TEST(ArrowReadWrite, EmptyListView) {
ASSERT_EQ(0, list_view.value_sizes()->size());
}

TEST(ArrowReadWrite, FixedSizeListFastPathPagesAndBatches) {
for (int width : {1, 3, 8, 9, 15, 16, 768}) {
for (auto version : {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) {
for (bool dictionary : {false, true}) {
for (bool nullable : {false, true}) {
SCOPED_TRACE(::testing::Message() << width << "/" << int(version) << "/"
<< dictionary << "/" << nullable);
::arrow::Int32Builder values;
for (int i = 0; i < width * 97; ++i) ASSERT_OK(values.Append(i));
ASSERT_OK_AND_ASSIGN(auto child, values.Finish());
auto type = ::arrow::fixed_size_list(
::arrow::field("item", ::arrow::int32(), nullable), width);
ASSERT_OK_AND_ASSIGN(auto array,
::arrow::FixedSizeListArray::FromArrays(child, type));
auto table = Table::Make(
::arrow::schema({::arrow::field("root", type, nullable)}), {array});
auto wp = WriterProperties::Builder()
.data_page_version(version)
->data_pagesize(256)
->write_batch_size(7)
->compression(Compression::ZSTD)
->build();
if (!dictionary)
wp = WriterProperties::Builder(*wp).disable_dictionary()->build();
auto ap = ArrowWriterProperties::Builder().store_schema()->build();
auto sink = CreateOutputStream();
ASSERT_OK(WriteTable(*table, default_memory_pool(), sink, 31, wp, ap));
ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish());

if (!nullable && !dictionary && width == 3) {
auto file = ParquetFileReader::Open(std::make_shared<BufferReader>(buffer));
const auto* column = file->metadata()->schema()->Column(0);
auto reader = ::parquet::internal::RecordReader::Make(
column, ::parquet::internal::LevelInfo::ComputeLevelInfo(column));
ASSERT_TRUE(reader->SetFixedSizeList(width));
reader->SetPageReader(file->RowGroup(0)->GetColumnPageReader(0));

ASSERT_EQ(reader->ReadRecords(7), 7);
ASSERT_TRUE(reader->dense_fixed_size_list());
ASSERT_EQ(reader->values_written(), 7 * width);
}

for (int batch_size : {1, 7, 32, 1024}) {
ArrowReaderProperties rp;
rp.set_batch_size(batch_size);
FileReaderBuilder builder;
ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
ASSERT_OK_AND_ASSIGN(auto reader, builder.properties(rp)->Build());
ASSERT_OK_AND_ASSIGN(auto batches, reader->GetRecordBatchReader());
ASSERT_OK_AND_ASSIGN(auto actual,
Table::FromRecordBatchReader(batches.get()));
ASSERT_OK(actual->ValidateFull());
ASSERT_TRUE(table->Equals(*actual));
}
}
}
}
}
}

TEST(ArrowReadWrite, FixedSizeListFastPathMixedNullPages) {
auto type = ::arrow::fixed_size_list(::arrow::int32(), 3);
std::string json = "[";
for (int row = 0; row < 200; ++row) {
if (row) json += ",";
if (row == 60)
json += "null";
else if (row == 120)
json += "[1,null,3]";
else
json += "[1,2,3]";
}
json += "]";
auto array = ::arrow::ArrayFromJSON(type, json);
auto table = Table::Make(::arrow::schema({::arrow::field("root", type)}), {array});
for (auto version : {ParquetDataPageVersion::V1, ParquetDataPageVersion::V2}) {
auto wp = WriterProperties::Builder()
.data_page_version(version)
->disable_dictionary()
->data_pagesize(128)
->write_batch_size(7)
->build();
auto ap = ArrowWriterProperties::Builder().store_schema()->build();
auto sink = CreateOutputStream();
ASSERT_OK(WriteTable(*table, default_memory_pool(), sink, 200, wp, ap));
ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish());
for (int batch_size : {1, 13, 200}) {
ArrowReaderProperties rp;
rp.set_batch_size(batch_size);
FileReaderBuilder builder;
ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
ASSERT_OK_AND_ASSIGN(auto reader, builder.properties(rp)->Build());
ASSERT_OK_AND_ASSIGN(auto batches, reader->GetRecordBatchReader());
ASSERT_OK_AND_ASSIGN(auto actual, Table::FromRecordBatchReader(batches.get()));
ASSERT_OK(actual->ValidateFull());
ASSERT_TRUE(table->Equals(*actual));
}
}
}

struct FixedSizeListTestCase {
std::shared_ptr<DataType> type;
std::string json;
Expand Down
36 changes: 35 additions & 1 deletion cpp/src/parquet/arrow/reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ class ColumnReaderImpl : public ColumnReader {
virtual ::arrow::Status BuildArray(int64_t length_upper_bound,
std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
virtual bool IsOrHasRepeatedChild() const = 0;
virtual bool SetFixedSizeList(int32_t /*width*/) { return false; }
virtual bool dense_fixed_size_list() const { return false; }
};

namespace {
Expand Down Expand Up @@ -501,6 +503,12 @@ class LeafReader : public ColumnReaderImpl {
}

bool IsOrHasRepeatedChild() const final { return false; }
bool SetFixedSizeList(int32_t width) override {
return record_reader_->SetFixedSizeList(width);
}
bool dense_fixed_size_list() const override {
return record_reader_->dense_fixed_size_list();
}

Status LoadBatch(int64_t records_to_read) final {
BEGIN_PARQUET_CATCH_EXCEPTIONS
Expand Down Expand Up @@ -690,6 +698,7 @@ class ListReader : public ColumnReaderImpl {
const std::shared_ptr<Field> field() override { return field_; }

protected:
ColumnReaderImpl* item_reader() const { return item_reader_.get(); }
std::shared_ptr<ReaderContext> ctx_;

private:
Expand Down Expand Up @@ -734,7 +743,32 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader<int32_t> {
::parquet::internal::LevelInfo level_info,
std::unique_ptr<ColumnReaderImpl> child_reader)
: ListReader(std::move(ctx), std::move(field), level_info,
std::move(child_reader)) {}
std::move(child_reader)) {
const auto& type =
checked_cast<const ::arrow::FixedSizeListType&>(*this->field()->type());
this->item_reader()->SetFixedSizeList(type.list_size());
}

Status BuildArray(int64_t length_upper_bound,
std::shared_ptr<ChunkedArray>* out) override {
if (!item_reader()->dense_fixed_size_list()) {
return ListReader<int32_t>::BuildArray(length_upper_bound, out);
}
const auto& type = checked_cast<const ::arrow::FixedSizeListType&>(*field()->type());
std::shared_ptr<ChunkedArray> child;
RETURN_NOT_OK(item_reader()->BuildArray(length_upper_bound, &child));
ARROW_ASSIGN_OR_RAISE(auto child_data, ChunksToSingle(*child, ctx_->pool));
const int64_t rows = child_data->length / type.list_size();
if (child_data->length % type.list_size() != 0) {
return Status::Invalid("Incomplete fixed-size list at end of column");
}
// Every list and value in this batch is present so no null bitmap is needed
auto data = ::arrow::ArrayData::Make(field()->type(), rows, {nullptr},
{std::move(child_data)}, 0);
*out = std::make_shared<ChunkedArray>(::arrow::MakeArray(std::move(data)));
return Status::OK();
}

::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(
std::shared_ptr<ArrayData> data) final {
DCHECK_EQ(data->buffers.size(), 2);
Expand Down
106 changes: 106 additions & 0 deletions cpp/src/parquet/arrow/reader_writer_benchmark.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include "arrow/array/data.h"
#include "arrow/compute/cast.h"
#include "arrow/io/memory.h"
#include "arrow/memory_pool.h"
#include "arrow/table.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/random.h"
Expand Down Expand Up @@ -722,6 +723,111 @@ static void BM_ReadListColumn(::benchmark::State& state) {

BENCHMARK(BM_ReadListColumn)->Apply(NestedReadArguments);

// Compare the same float values as a flat column and as fixed-size lists.
// Fixture generation, writing, and the checked warm-up read are outside timing.
static void BM_ReadFloatLayout(::benchmark::State& state) {
const bool use_lists = state.range(0) == 1;
const int32_t list_size = static_cast<int32_t>(state.range(1));
const auto page_version =
state.range(2) == 1 ? ParquetDataPageVersion::V1 : ParquetDataPageVersion::V2;
const bool use_zstd = state.range(3) != 0;
const int64_t page_size = state.range(4);
const int64_t num_values = (BENCHMARK_SIZE / list_size) * list_size;

::arrow::random::RandomArrayGenerator rng(500);
auto values = rng.Float32(num_values, /*min=*/-1.0f, /*max=*/1.0f,
/*null_probability=*/0.0);
std::shared_ptr<Array> column = values;
if (use_lists) {
auto item = ::arrow::field("item", ::arrow::float32(), /*nullable=*/false);
auto type = ::arrow::fixed_size_list(item, list_size);
PARQUET_ASSIGN_OR_THROW(column,
::arrow::FixedSizeListArray::FromArrays(values, type));
}
auto schema =
::arrow::schema({::arrow::field("column", column->type(), /*nullable=*/false)});
auto expected = Table::Make(schema, {column});
EXIT_NOT_OK(expected->ValidateFull());

auto properties =
WriterProperties::Builder()
.disable_dictionary()
->encoding(Encoding::PLAIN)
->compression(use_zstd ? Compression::ZSTD : Compression::UNCOMPRESSED)
->data_page_version(page_version)
->data_pagesize(page_size)
->build();
// Preserve the Arrow type so the nested case exercises FixedSizeListReader.
auto arrow_properties = ArrowWriterProperties::Builder().store_schema()->build();
auto output = CreateOutputStream();
EXIT_NOT_OK(WriteTable(*expected, ::arrow::default_memory_pool(), output,
expected->num_rows(), properties, arrow_properties));
PARQUET_ASSIGN_OR_THROW(auto buffer, output->Finish());

ArrowReaderProperties reader_properties;
reader_properties.set_use_threads(false);
// ReadTable reads the whole row group; both layouts contain the same number
// of child values. The streaming record-batch size does not apply here.
auto read = [&](::arrow::MemoryPool* pool) {
auto parquet_reader = ParquetFileReader::Open(
std::make_shared<::arrow::io::BufferReader>(buffer), ReaderProperties(pool));
PARQUET_ASSIGN_OR_THROW(auto reader, FileReader::Make(pool, std::move(parquet_reader),
reader_properties));
PARQUET_ASSIGN_OR_THROW(auto table, reader->ReadTable());
return table;
};

// Measure Arrow-managed allocations on one untimed read. This includes the
// decoded output, but excludes the input file and non-pool allocations.
::arrow::ProxyMemoryPool read_pool(::arrow::default_memory_pool());
{
auto actual = read(&read_pool);
EXIT_NOT_OK(actual->ValidateFull());
if (!expected->Equals(*actual)) {
state.SkipWithError(
"Float layout did not round-trip with identical type and values");
return;
}
}
state.counters["peak_read_bytes"] = static_cast<double>(read_pool.max_memory());
state.counters["read_allocations"] = static_cast<double>(read_pool.num_allocations());
state.counters["file_bytes"] = static_cast<double>(buffer->size());
state.counters["float_values"] = static_cast<double>(num_values);
state.SetLabel(use_lists ? "fixed_size_list" : "flat");

// Each iteration opens a fresh reader over the same in-memory file and reads
// the whole column. Reader setup and result destruction are timed as well.
for (auto _ : state) {
auto actual = read(::arrow::default_memory_pool());
::benchmark::DoNotOptimize(actual.get());
}
state.SetItemsProcessed(state.iterations() * num_values);
state.SetBytesProcessed(state.iterations() * num_values * sizeof(float));
}

static void FloatLayoutReadArguments(::benchmark::internal::Benchmark* b) {
for (int64_t list_size : {1, 3, 8, 9, 15, 16, 768}) {
for (int64_t page_version : {1, 2}) {
for (int64_t zstd : {0, 1}) {
for (int64_t layout : {0, 1}) {
b->Args({layout, list_size, page_version, zstd, 1024 * 1024});
}
}
// Also measure many small pages for coordinates and embeddings.
if (list_size == 3 || list_size == 768) {
for (int64_t layout : {0, 1}) {
b->Args({layout, list_size, page_version, 0, 8 * 1024});
}
}
}
}
}

BENCHMARK(BM_ReadFloatLayout)
->Apply(FloatLayoutReadArguments)
->ArgNames({"layout", "list_size", "page_version", "zstd", "page_bytes"})
->UseRealTime();

static void BM_ReadListOfStructColumn(::benchmark::State& state) {
constexpr int64_t kNumValues = BENCHMARK_SIZE / 10;
const double null_probability = static_cast<double>(state.range(0)) / 100.0;
Expand Down
Loading
Loading