Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions cpp/src/parquet/column_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -762,6 +762,7 @@ class ColumnWriterImpl {
total_bytes_written_(0),
total_compressed_bytes_(0),
closed_(false),
failed_(false),
fallback_(false),
definition_levels_sink_(allocator_),
repetition_levels_sink_(allocator_),
Expand Down Expand Up @@ -894,6 +895,9 @@ class ColumnWriterImpl {
// Flag to check if the Writer has been closed
bool closed_;

// Failed writes may leave the encoder in an invalid state. Do not finalize it.
bool failed_;

// Flag to infer if dictionary encoding has fallen back to PLAIN
bool fallback_;

Expand Down Expand Up @@ -1113,6 +1117,9 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size,
int64_t ColumnWriterImpl::Close() {
if (!closed_) {
closed_ = true;
if (failed_) {
throw ParquetException("Cannot close a column writer after a failed write");
}
if (has_dictionary_ && !fallback_) {
WriteDictionaryPage();
}
Expand Down Expand Up @@ -1338,7 +1345,14 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
"Content-defined chunking is not supported in WriteBatch() or "
"WriteBatchSpaced(), use WriteArrow() instead.");
}
return WriteBatchInternal(num_values, def_levels, rep_levels, values);
if (closed_ || failed_) {
throw ParquetException("Cannot write to a closed or failed column writer");
}
failed_ = true;
const int64_t values_written =
WriteBatchInternal(num_values, def_levels, rep_levels, values);
failed_ = false;
return values_written;
}

int64_t WriteBatchInternal(int64_t num_values, const int16_t* def_levels,
Expand Down Expand Up @@ -1381,8 +1395,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
"Content-defined chunking is not supported in WriteBatch() or "
"WriteBatchSpaced(), use WriteArrow() instead.");
}
return WriteBatchSpacedInternal(num_values, def_levels, rep_levels, valid_bits,
valid_bits_offset, values);
if (closed_ || failed_) {
throw ParquetException("Cannot write to a closed or failed column writer");
}
failed_ = true;
WriteBatchSpacedInternal(num_values, def_levels, rep_levels, valid_bits,
valid_bits_offset, values);
failed_ = false;
}

void WriteBatchSpacedInternal(int64_t num_values, const int16_t* def_levels,
Expand Down Expand Up @@ -1427,6 +1446,20 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels,
int64_t num_levels, const ::arrow::Array& leaf_array,
ArrowWriteContext* ctx, bool leaf_field_nullable) override {
if (closed_ || failed_) {
return Status::Invalid("Cannot write to a closed or failed column writer");
}
// Leave failed_ set for both exceptions and non-OK Status returns.
failed_ = true;
Status status = WriteArrowInternal(def_levels, rep_levels, num_levels, leaf_array,
ctx, leaf_field_nullable);
failed_ = !status.ok();
return status;
}

Status WriteArrowInternal(const int16_t* def_levels, const int16_t* rep_levels,
int64_t num_levels, const ::arrow::Array& leaf_array,
ArrowWriteContext* ctx, bool leaf_field_nullable) {
BEGIN_PARQUET_CATCH_EXCEPTIONS
// Leaf nulls are canonical when there is only a single null element after a list
// and it is at the leaf.
Expand Down
129 changes: 129 additions & 0 deletions cpp/src/parquet/column_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@
#include <cmath>
#include <memory>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>

#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include "arrow/array/array_primitive.h"
#include "arrow/io/buffered.h"
#include "arrow/io/file.h"
#include "arrow/memory_pool.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/bitmap_builders.h"
Expand Down Expand Up @@ -1760,6 +1763,132 @@ TEST(TestColumnWriter, WriteDataPagesChangeOnRecordBoundariesWithSmallBatches) {
}
}

class DictionaryGrowthFailingPool : public ::arrow::ProxyMemoryPool {
public:
DictionaryGrowthFailingPool() : ProxyMemoryPool(::arrow::system_memory_pool()) {}

void FailAllocationsAtOrAbove(int64_t size) { failure_threshold_ = size; }

::arrow::Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override {
if (failure_threshold_ > 0 && size >= failure_threshold_) {
return ::arrow::Status::OutOfMemory("injected dictionary allocation failure");
}
return ProxyMemoryPool::Allocate(size, alignment, out);
}

::arrow::Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment,
uint8_t** ptr) override {
if (failure_threshold_ > 0 && new_size >= failure_threshold_) {
return ::arrow::Status::OutOfMemory("injected dictionary allocation failure");
}
return ProxyMemoryPool::Reallocate(old_size, new_size, alignment, ptr);
}

private:
int64_t failure_threshold_ = 0;
};

enum class FailedWriteMethod { Batch, Spaced, Arrow };

class FailedColumnWriterTest
: public ::testing::TestWithParam<std::tuple<FailedWriteMethod, bool, bool>> {};

TEST_P(FailedColumnWriterTest, DictionaryGrowthOutOfMemory) {
const auto [method, close_explicitly, recover_memory] = GetParam();
DictionaryGrowthFailingPool pool;
auto sink = CreateOutputStream();
auto schema = std::static_pointer_cast<GroupNode>(GroupNode::Make(
"schema", Repetition::REQUIRED, {schema::Double("value", Repetition::REQUIRED)}));
auto properties = WriterProperties::Builder()
.memory_pool(&pool)
->enable_dictionary()
->disable_statistics()
->compression(Compression::UNCOMPRESSED)
->build();
{
auto writer = ParquetFileWriter::Open(sink, schema, properties);
auto* column =
static_cast<DoubleWriter*>(writer->AppendBufferedRowGroup()->column(0));
auto arrow_properties = default_arrow_writer_properties();
ArrowWriteContext context(&pool, arrow_properties.get());
auto write = [&](const double value) -> ::arrow::Status {
switch (method) {
case FailedWriteMethod::Batch:
column->WriteBatch(1, nullptr, nullptr, &value);
break;
case FailedWriteMethod::Spaced: {
const uint8_t valid_bits = 1;
column->WriteBatchSpaced(1, nullptr, nullptr, &valid_bits, 0, &value);
break;
}
case FailedWriteMethod::Arrow: {
::arrow::DoubleArray array(1, ::arrow::Buffer::Wrap(&value, 1));
return column->WriteArrow(nullptr, nullptr, 1, array, &context, false);
}
}
return ::arrow::Status::OK();
};
// The dictionary starts with 1024 hash slots and grows on the 512th value.
for (int i = 0; i < 511; ++i) {
ASSERT_OK(write(static_cast<double>(i)));
}
// Fail hash table growth, while allowing the smaller indices buffer.
pool.FailAllocationsAtOrAbove(64 * 1024);
if (method == FailedWriteMethod::Arrow) {
ASSERT_FALSE(write(511.0).ok());
} else {
ASSERT_THROW(static_cast<void>(write(511.0)), ParquetException);
}

pool.FailAllocationsAtOrAbove(recover_memory ? 0 : 1);
ASSERT_OK_AND_ASSIGN(const int64_t position, sink->Tell());
const int64_t allocations = pool.num_allocations();
// Once failed, a writer must reject further writes even if memory recovers.
if (method == FailedWriteMethod::Arrow) {
ASSERT_RAISES(Invalid, write(0.0));
} else {
ASSERT_THROW(static_cast<void>(write(0.0)), ParquetException);
}
if (close_explicitly) {
ASSERT_THROW(writer->Close(), ParquetException);
ASSERT_NO_THROW(writer->Close());
}
ASSERT_NO_THROW(writer.reset());
ASSERT_OK_AND_ASSIGN(const int64_t final_position, sink->Tell());
ASSERT_EQ(final_position, position);
ASSERT_EQ(pool.num_allocations(), allocations);
}
ASSERT_EQ(pool.bytes_allocated(), 0);
}

INSTANTIATE_TEST_SUITE_P(FailureCleanup, FailedColumnWriterTest,
::testing::Combine(::testing::Values(FailedWriteMethod::Batch,
FailedWriteMethod::Spaced,
FailedWriteMethod::Arrow),
::testing::Bool(), ::testing::Bool()));

TEST(TestColumnWriter, CloseAfterWriteArrowReturnsError) {
auto sink = CreateOutputStream();
auto schema = std::static_pointer_cast<GroupNode>(GroupNode::Make(
"schema", Repetition::REQUIRED, {schema::Double("value", Repetition::REQUIRED)}));
auto writer = ParquetFileWriter::Open(sink, schema);
auto* column = writer->AppendBufferedRowGroup()->column(0);
auto arrow_properties = default_arrow_writer_properties();
ArrowWriteContext context(::arrow::default_memory_pool(), arrow_properties.get());
const double value = 0;
const uint8_t valid_bits = 0;
::arrow::DoubleArray array(1, ::arrow::Buffer::Wrap(&value, 1),
::arrow::Buffer::Wrap(&valid_bits, 1), 1);
// A returned Status, without a thrown exception, must also mark the writer failed.
ASSERT_RAISES(Invalid, column->WriteArrow(nullptr, nullptr, 1, array, &context, false));
ASSERT_OK_AND_ASSIGN(const int64_t position, sink->Tell());
ASSERT_THROW(writer->Close(), ParquetException);
ASSERT_NO_THROW(writer->Close());
ASSERT_NO_THROW(writer.reset());
ASSERT_OK_AND_ASSIGN(const int64_t final_position, sink->Tell());
ASSERT_EQ(final_position, position);
}

class ColumnWriterTestSizeEstimated : public ::testing::Test {
public:
void SetUp() {
Expand Down
Loading