From ffea4e452faac67f232d1014cf45365cf8ef21bc Mon Sep 17 00:00:00 2001 From: Tuco Date: Fri, 13 Nov 2026 18:27:43 +0800 Subject: [PATCH] GH-55628: [C++][Parquet] Stop finalizing column writers after failed writes Co-Authored-By: Claude Code --- cpp/src/parquet/column_writer.cc | 39 +++++++- cpp/src/parquet/column_writer_test.cc | 129 ++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 3 deletions(-) diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index c899f5a9847a..95ca5652519e 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -762,6 +762,7 @@ class ColumnWriterImpl { total_bytes_written_(0), total_compressed_bytes_(0), closed_(false), + failed_(false), fallback_(false), definition_levels_sink_(allocator_), repetition_levels_sink_(allocator_), @@ -894,6 +895,9 @@ class ColumnWriterImpl { // Flag to check if the Writer has been closed bool closed_; + // Failed writes may leave the encoder in an invalid state. Do not finalize it. + bool failed_; + // Flag to infer if dictionary encoding has fallen back to PLAIN bool fallback_; @@ -1113,6 +1117,9 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size, int64_t ColumnWriterImpl::Close() { if (!closed_) { closed_ = true; + if (failed_) { + throw ParquetException("Cannot close a column writer after a failed write"); + } if (has_dictionary_ && !fallback_) { WriteDictionaryPage(); } @@ -1338,7 +1345,14 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, "Content-defined chunking is not supported in WriteBatch() or " "WriteBatchSpaced(), use WriteArrow() instead."); } - return WriteBatchInternal(num_values, def_levels, rep_levels, values); + if (closed_ || failed_) { + throw ParquetException("Cannot write to a closed or failed column writer"); + } + failed_ = true; + const int64_t values_written = + WriteBatchInternal(num_values, def_levels, rep_levels, values); + failed_ = false; + return values_written; } int64_t WriteBatchInternal(int64_t num_values, const int16_t* def_levels, @@ -1381,8 +1395,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, "Content-defined chunking is not supported in WriteBatch() or " "WriteBatchSpaced(), use WriteArrow() instead."); } - return WriteBatchSpacedInternal(num_values, def_levels, rep_levels, valid_bits, - valid_bits_offset, values); + if (closed_ || failed_) { + throw ParquetException("Cannot write to a closed or failed column writer"); + } + failed_ = true; + WriteBatchSpacedInternal(num_values, def_levels, rep_levels, valid_bits, + valid_bits_offset, values); + failed_ = false; } void WriteBatchSpacedInternal(int64_t num_values, const int16_t* def_levels, @@ -1427,6 +1446,20 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, const ::arrow::Array& leaf_array, ArrowWriteContext* ctx, bool leaf_field_nullable) override { + if (closed_ || failed_) { + return Status::Invalid("Cannot write to a closed or failed column writer"); + } + // Leave failed_ set for both exceptions and non-OK Status returns. + failed_ = true; + Status status = WriteArrowInternal(def_levels, rep_levels, num_levels, leaf_array, + ctx, leaf_field_nullable); + failed_ = !status.ok(); + return status; + } + + Status WriteArrowInternal(const int16_t* def_levels, const int16_t* rep_levels, + int64_t num_levels, const ::arrow::Array& leaf_array, + ArrowWriteContext* ctx, bool leaf_field_nullable) { BEGIN_PARQUET_CATCH_EXCEPTIONS // Leaf nulls are canonical when there is only a single null element after a list // and it is at the leaf. diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index b1f8b4e3be38..ec8bb67477ea 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -18,14 +18,17 @@ #include #include #include +#include #include #include #include #include +#include "arrow/array/array_primitive.h" #include "arrow/io/buffered.h" #include "arrow/io/file.h" +#include "arrow/memory_pool.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_builders.h" @@ -1760,6 +1763,132 @@ TEST(TestColumnWriter, WriteDataPagesChangeOnRecordBoundariesWithSmallBatches) { } } +class DictionaryGrowthFailingPool : public ::arrow::ProxyMemoryPool { + public: + DictionaryGrowthFailingPool() : ProxyMemoryPool(::arrow::system_memory_pool()) {} + + void FailAllocationsAtOrAbove(int64_t size) { failure_threshold_ = size; } + + ::arrow::Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override { + if (failure_threshold_ > 0 && size >= failure_threshold_) { + return ::arrow::Status::OutOfMemory("injected dictionary allocation failure"); + } + return ProxyMemoryPool::Allocate(size, alignment, out); + } + + ::arrow::Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, + uint8_t** ptr) override { + if (failure_threshold_ > 0 && new_size >= failure_threshold_) { + return ::arrow::Status::OutOfMemory("injected dictionary allocation failure"); + } + return ProxyMemoryPool::Reallocate(old_size, new_size, alignment, ptr); + } + + private: + int64_t failure_threshold_ = 0; +}; + +enum class FailedWriteMethod { Batch, Spaced, Arrow }; + +class FailedColumnWriterTest + : public ::testing::TestWithParam> {}; + +TEST_P(FailedColumnWriterTest, DictionaryGrowthOutOfMemory) { + const auto [method, close_explicitly, recover_memory] = GetParam(); + DictionaryGrowthFailingPool pool; + auto sink = CreateOutputStream(); + auto schema = std::static_pointer_cast(GroupNode::Make( + "schema", Repetition::REQUIRED, {schema::Double("value", Repetition::REQUIRED)})); + auto properties = WriterProperties::Builder() + .memory_pool(&pool) + ->enable_dictionary() + ->disable_statistics() + ->compression(Compression::UNCOMPRESSED) + ->build(); + { + auto writer = ParquetFileWriter::Open(sink, schema, properties); + auto* column = + static_cast(writer->AppendBufferedRowGroup()->column(0)); + auto arrow_properties = default_arrow_writer_properties(); + ArrowWriteContext context(&pool, arrow_properties.get()); + auto write = [&](const double value) -> ::arrow::Status { + switch (method) { + case FailedWriteMethod::Batch: + column->WriteBatch(1, nullptr, nullptr, &value); + break; + case FailedWriteMethod::Spaced: { + const uint8_t valid_bits = 1; + column->WriteBatchSpaced(1, nullptr, nullptr, &valid_bits, 0, &value); + break; + } + case FailedWriteMethod::Arrow: { + ::arrow::DoubleArray array(1, ::arrow::Buffer::Wrap(&value, 1)); + return column->WriteArrow(nullptr, nullptr, 1, array, &context, false); + } + } + return ::arrow::Status::OK(); + }; + // The dictionary starts with 1024 hash slots and grows on the 512th value. + for (int i = 0; i < 511; ++i) { + ASSERT_OK(write(static_cast(i))); + } + // Fail hash table growth, while allowing the smaller indices buffer. + pool.FailAllocationsAtOrAbove(64 * 1024); + if (method == FailedWriteMethod::Arrow) { + ASSERT_FALSE(write(511.0).ok()); + } else { + ASSERT_THROW(static_cast(write(511.0)), ParquetException); + } + + pool.FailAllocationsAtOrAbove(recover_memory ? 0 : 1); + ASSERT_OK_AND_ASSIGN(const int64_t position, sink->Tell()); + const int64_t allocations = pool.num_allocations(); + // Once failed, a writer must reject further writes even if memory recovers. + if (method == FailedWriteMethod::Arrow) { + ASSERT_RAISES(Invalid, write(0.0)); + } else { + ASSERT_THROW(static_cast(write(0.0)), ParquetException); + } + if (close_explicitly) { + ASSERT_THROW(writer->Close(), ParquetException); + ASSERT_NO_THROW(writer->Close()); + } + ASSERT_NO_THROW(writer.reset()); + ASSERT_OK_AND_ASSIGN(const int64_t final_position, sink->Tell()); + ASSERT_EQ(final_position, position); + ASSERT_EQ(pool.num_allocations(), allocations); + } + ASSERT_EQ(pool.bytes_allocated(), 0); +} + +INSTANTIATE_TEST_SUITE_P(FailureCleanup, FailedColumnWriterTest, + ::testing::Combine(::testing::Values(FailedWriteMethod::Batch, + FailedWriteMethod::Spaced, + FailedWriteMethod::Arrow), + ::testing::Bool(), ::testing::Bool())); + +TEST(TestColumnWriter, CloseAfterWriteArrowReturnsError) { + auto sink = CreateOutputStream(); + auto schema = std::static_pointer_cast(GroupNode::Make( + "schema", Repetition::REQUIRED, {schema::Double("value", Repetition::REQUIRED)})); + auto writer = ParquetFileWriter::Open(sink, schema); + auto* column = writer->AppendBufferedRowGroup()->column(0); + auto arrow_properties = default_arrow_writer_properties(); + ArrowWriteContext context(::arrow::default_memory_pool(), arrow_properties.get()); + const double value = 0; + const uint8_t valid_bits = 0; + ::arrow::DoubleArray array(1, ::arrow::Buffer::Wrap(&value, 1), + ::arrow::Buffer::Wrap(&valid_bits, 1), 1); + // A returned Status, without a thrown exception, must also mark the writer failed. + ASSERT_RAISES(Invalid, column->WriteArrow(nullptr, nullptr, 1, array, &context, false)); + ASSERT_OK_AND_ASSIGN(const int64_t position, sink->Tell()); + ASSERT_THROW(writer->Close(), ParquetException); + ASSERT_NO_THROW(writer->Close()); + ASSERT_NO_THROW(writer.reset()); + ASSERT_OK_AND_ASSIGN(const int64_t final_position, sink->Tell()); + ASSERT_EQ(final_position, position); +} + class ColumnWriterTestSizeEstimated : public ::testing::Test { public: void SetUp() {