From 5ea85cf665075ea830ede52dd41b846390810195 Mon Sep 17 00:00:00 2001 From: Awesome06 Date: Wed, 18 Nov 2026 16:18:06 +0530 Subject: [PATCH 1/5] [C++][IPC] Validate buffer bounds and replace raw pointer arithmetic in MessageDecoder * Replace raw pointer arithmetic and unchecked memcpy offsets in MessageDecoderImpl with std::span and AddWithOverflow * Enforce validation against negative data size in MessageDecoder::Consume * Add opt-in max_metadata_length, max_body_length, and max_decompression_ratio ceilings to IpcReadOptions (defaulting to unlimited: -1) * Use CHECK_FLATBUFFERS_NOT_NULL guards in Tensor metadata decoders * Add deterministic unit tests covering negative sizes, opt-in limits, and decompression bounds in read_write_test --- cpp/src/arrow/ipc/message.cc | 96 +++++++++++++++--- cpp/src/arrow/ipc/message.h | 23 +++++ cpp/src/arrow/ipc/metadata_internal.cc | 8 ++ cpp/src/arrow/ipc/options.h | 9 ++ cpp/src/arrow/ipc/read_write_test.cc | 129 +++++++++++++++++++++++++ cpp/src/arrow/ipc/reader.cc | 38 +++++++- cpp/src/arrow/ipc/reader_internal.h | 12 +++ 7 files changed, 298 insertions(+), 17 deletions(-) diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 34ec6993aa4b..4edbea5adf8f 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include "arrow/status.h" #include "arrow/util/endian.h" #include "arrow/util/future.h" +#include "arrow/util/int_util_overflow.h" #include "arrow/util/logging_internal.h" #include "arrow/util/ubsan.h" @@ -674,8 +676,21 @@ class MessageDecoder::MessageDecoderImpl { explicit MessageDecoderImpl(std::shared_ptr listener, State initial_state, int64_t initial_next_required_size, MemoryPool* pool, bool skip_body) + : MessageDecoderImpl( + std::move(listener), initial_state, initial_next_required_size, + [pool] { + auto opts = IpcReadOptions::Defaults(); + opts.memory_pool = pool; + return opts; + }(), + skip_body) {} + + explicit MessageDecoderImpl(std::shared_ptr listener, + State initial_state, int64_t initial_next_required_size, + IpcReadOptions options, bool skip_body) : listener_(std::move(listener)), - pool_(pool), + options_(std::move(options)), + pool_(options_.memory_pool), memory_manager_(CPUDevice::memory_manager(pool_)), state_(initial_state), next_required_size_(initial_next_required_size), @@ -685,15 +700,20 @@ class MessageDecoder::MessageDecoderImpl { skip_body_(skip_body) {} Status ConsumeData(const uint8_t* data, int64_t size) { + if (size < 0) { + return Status::Invalid("Invalid data size: negative byte count"); + } if (buffered_size_ == 0) { - while (size > 0 && size >= next_required_size_) { + std::span data_span(data, static_cast(size)); + while (data_span.size() >= static_cast(next_required_size_)) { auto used_size = next_required_size_; switch (state_) { case State::INITIAL: - RETURN_NOT_OK(ConsumeInitialData(data, next_required_size_)); + RETURN_NOT_OK(ConsumeInitialData(data_span.data(), next_required_size_)); break; case State::METADATA_LENGTH: - RETURN_NOT_OK(ConsumeMetadataLengthData(data, next_required_size_)); + RETURN_NOT_OK( + ConsumeMetadataLengthData(data_span.data(), next_required_size_)); break; case State::METADATA: { // We need to copy metadata because it's used in @@ -704,7 +724,8 @@ class MessageDecoder::MessageDecoderImpl { // here. ARROW_ASSIGN_OR_RAISE(std::shared_ptr buffer, AllocateBuffer(next_required_size_, pool_)); - memcpy(buffer->mutable_data(), data, next_required_size_); + memcpy(buffer->mutable_data(), data_span.data(), + static_cast(next_required_size_)); RETURN_NOT_OK(ConsumeMetadataBuffer(buffer)); } break; case State::BODY: { @@ -714,15 +735,16 @@ class MessageDecoder::MessageDecoderImpl { // the passed message is needed to be valid after the // call, it's a listener_'s responsibility. The listener_ // may copy the data for it. - auto buffer = std::make_shared(data, next_required_size_); + auto buffer = std::make_shared(data_span.data(), next_required_size_); RETURN_NOT_OK(ConsumeBodyBuffer(buffer)); } break; case State::EOS: return Status::OK(); } - data += used_size; - size -= used_size; + data_span = data_span.subspan(static_cast(used_size)); } + data = data_span.data(); + size = static_cast(data_span.size()); } if (size == 0) { @@ -847,6 +869,12 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(listener_->OnEOS()); return Status::OK(); } else if (continuation > 0) { + if (options_.max_metadata_length >= 0 && + continuation > options_.max_metadata_length) { + return Status::Invalid("IPC metadata length ", continuation, + " exceeds maximum allowed metadata length ", + options_.max_metadata_length); + } state_ = State::METADATA; // ARROW-6314: Backwards compatibility for reading old IPC // messages produced prior to version 0.15.0 @@ -881,6 +909,12 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(listener_->OnEOS()); return Status::OK(); } else if (metadata_length > 0) { + if (options_.max_metadata_length >= 0 && + metadata_length > options_.max_metadata_length) { + return Status::Invalid("IPC metadata length ", metadata_length, + " exceeds maximum allowed metadata length ", + options_.max_metadata_length); + } state_ = State::METADATA; next_required_size_ = metadata_length; RETURN_NOT_OK(listener_->OnMetadata()); @@ -929,6 +963,11 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(MaybeAlignMetadata(&metadata_)); int64_t body_length = -1; RETURN_NOT_OK(CheckMetadataAndGetBodyLength(*metadata_, &body_length)); + if (options_.max_body_length >= 0 && body_length > options_.max_body_length) { + return Status::Invalid("IPC body length ", body_length, + " exceeds maximum allowed body length ", + options_.max_body_length); + } state_ = State::BODY; next_required_size_ = skip_body_ ? 0 : body_length; @@ -987,7 +1026,10 @@ class MessageDecoder::MessageDecoderImpl { } Status ConsumeDataChunks(int64_t nbytes, void* out) { - size_t offset = 0; + DCHECK_GE(nbytes, 0); + DCHECK_GE(buffered_size_, nbytes); + std::span out_span(static_cast(out), static_cast(nbytes)); + int64_t offset = 0; size_t n_used_chunks = 0; auto required_size = nbytes; std::shared_ptr last_chunk; @@ -995,15 +1037,23 @@ class MessageDecoder::MessageDecoderImpl { if (!chunk->is_cpu()) { ARROW_ASSIGN_OR_RAISE(chunk, Buffer::ViewOrCopy(chunk, memory_manager_)); } - auto data = chunk->data(); - auto data_size = chunk->size(); - auto copy_size = std::min(required_size, data_size); - memcpy(static_cast(out) + offset, data, copy_size); + std::span chunk_span(chunk->data(), + static_cast(chunk->size())); + auto copy_size = std::min(required_size, chunk->size()); + std::memcpy( + out_span.subspan(static_cast(offset), static_cast(copy_size)) + .data(), + chunk_span.data(), static_cast(copy_size)); n_used_chunks++; - offset += copy_size; + int64_t next_offset = 0; + if (ARROW_PREDICT_FALSE( + ::arrow::internal::AddWithOverflow(offset, copy_size, &next_offset))) { + return Status::Invalid("Chunk offset calculation overflowed"); + } + offset = next_offset; required_size -= copy_size; if (required_size == 0) { - if (data_size != copy_size) { + if (chunk->size() != copy_size) { last_chunk = SliceBuffer(chunk, copy_size); } break; @@ -1018,6 +1068,7 @@ class MessageDecoder::MessageDecoderImpl { } std::shared_ptr listener_; + IpcReadOptions options_; MemoryPool* pool_; std::shared_ptr memory_manager_; State state_; @@ -1042,6 +1093,21 @@ MessageDecoder::MessageDecoder(std::shared_ptr listener, initial_next_required_size, pool, skip_body)); } +MessageDecoder::MessageDecoder(std::shared_ptr listener, + IpcReadOptions options, bool skip_body) { + impl_.reset(new MessageDecoderImpl(std::move(listener), State::INITIAL, + kMessageDecoderNextRequiredSizeInitial, + std::move(options), skip_body)); +} + +MessageDecoder::MessageDecoder(std::shared_ptr listener, + State initial_state, int64_t initial_next_required_size, + IpcReadOptions options, bool skip_body) { + impl_.reset(new MessageDecoderImpl(std::move(listener), initial_state, + initial_next_required_size, std::move(options), + skip_body)); +} + MessageDecoder::~MessageDecoder() {} Status MessageDecoder::Consume(const uint8_t* data, int64_t size) { diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index 90346bd1972f..f9c5adb3822b 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -26,6 +26,7 @@ #include #include "arrow/io/type_fwd.h" +#include "arrow/ipc/options.h" #include "arrow/ipc/type_fwd.h" #include "arrow/result.h" #include "arrow/status.h" @@ -289,6 +290,28 @@ class ARROW_EXPORT MessageDecoder { int64_t initial_next_required_size, MemoryPool* pool = default_memory_pool(), bool skip_body = false); + /// \brief Construct a message decoder with options. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] options options for reading IPC messages + /// \param[in] skip_body if true the body will be skipped even if the message has a body + explicit MessageDecoder(std::shared_ptr listener, + IpcReadOptions options, bool skip_body = false); + + /// \brief Construct a message decoder with the specified state and options. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] initial_state an initial state of the decoder + /// \param[in] initial_next_required_size the number of bytes needed + /// to run the next action + /// \param[in] options options for reading IPC messages + /// \param[in] skip_body if true the body will be skipped even if the message has a body + MessageDecoder(std::shared_ptr listener, State initial_state, + int64_t initial_next_required_size, IpcReadOptions options, + bool skip_body = false); + virtual ~MessageDecoder(); /// \brief Feed data to the decoder as a raw data. diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc index 02134aa805ab..e3c5157ab3c8 100644 --- a/cpp/src/arrow/ipc/metadata_internal.cc +++ b/cpp/src/arrow/ipc/metadata_internal.cc @@ -1290,6 +1290,7 @@ Result> GetKeyValueMetadata( metadata->reserve(fb_metadata->size()); for (const auto pair : *fb_metadata) { + CHECK_FLATBUFFERS_NOT_NULL(pair, "custom_metadata.pair"); CHECK_FLATBUFFERS_NOT_NULL(pair->key(), "custom_metadata.key"); CHECK_FLATBUFFERS_NOT_NULL(pair->value(), "custom_metadata.value"); metadata->Append(pair->key()->str(), pair->value()->str()); @@ -1474,11 +1475,16 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr* type if (tensor == nullptr) { return Status::IOError("Header-type of flatbuffer-encoded Message is not Tensor."); } + CHECK_FLATBUFFERS_NOT_NULL(tensor->shape(), "Tensor.shape"); flatbuffers::uoffset_t ndim = tensor->shape()->size(); for (flatbuffers::uoffset_t i = 0; i < ndim; ++i) { auto dim = tensor->shape()->Get(i); + CHECK_FLATBUFFERS_NOT_NULL(dim, "Tensor.shape[i]"); + if (dim->size() < 0) { + return Status::Invalid("Invalid tensor dimension size: ", dim->size()); + } shape->push_back(dim->size()); dim_names->push_back(StringFromFlatbuffers(dim->name())); @@ -1551,11 +1557,13 @@ Status GetSparseTensorMetadata(const Buffer& metadata, std::shared_ptr return Status::IOError( "Header-type of flatbuffer-encoded Message is not SparseTensor."); } + CHECK_FLATBUFFERS_NOT_NULL(sparse_tensor->shape(), "SparseTensor.shape"); int ndim = static_cast(sparse_tensor->shape()->size()); if (shape || dim_names) { for (int i = 0; i < ndim; ++i) { auto dim = sparse_tensor->shape()->Get(i); + CHECK_FLATBUFFERS_NOT_NULL(dim, "SparseTensor.shape[i]"); if (shape) { if (dim->size() < 0) { diff --git a/cpp/src/arrow/ipc/options.h b/cpp/src/arrow/ipc/options.h index 254a4693866e..229ad2472dc8 100644 --- a/cpp/src/arrow/ipc/options.h +++ b/cpp/src/arrow/ipc/options.h @@ -189,6 +189,15 @@ struct ARROW_EXPORT IpcReadOptions { /// The lazy property will always be reset to true to deliver the expected behavior io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults(); + /// \brief Maximum allowed metadata length in bytes (-1 means unlimited). + int64_t max_metadata_length = -1; + + /// \brief Maximum allowed body length in bytes (-1 means unlimited). + int64_t max_body_length = -1; + + /// \brief Maximum allowed decompression expansion ratio (-1 means unlimited). + int32_t max_decompression_ratio = -1; + static IpcReadOptions Defaults(); }; diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 811f620f40a5..0b959329d2ff 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3401,6 +3401,135 @@ TEST(PreBuffering, MixedAccess) { ASSERT_EQ(2, stats.num_record_batches); } +TEST(TestMessageDecoderBounds, NegativeConsumeSize) { + std::unique_ptr message; + auto listener = std::make_shared(&message); + MessageDecoder decoder(listener); + const uint8_t data[4] = {0}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("Invalid data size: negative byte count"), + decoder.Consume(data, -1)); +} + +TEST(TestMessageDecoderBounds, NegativeContinuationToken) { + std::unique_ptr message; + auto listener = std::make_shared(&message); + MessageDecoder decoder(listener); + int32_t negative_token = -5; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("negative continuation token"), + decoder.Consume(reinterpret_cast(&negative_token), + sizeof(int32_t))); +} + +TEST(TestMessageDecoderBounds, NegativeMetadataLength) { + std::unique_ptr message; + auto listener = std::make_shared(&message); + MessageDecoder decoder(listener); + int32_t buf[2] = {internal::kIpcContinuationToken, -10}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("negative metadata length"), + decoder.Consume(reinterpret_cast(buf), sizeof(buf))); +} + +TEST(TestMessageDecoderBounds, MaxMetadataLengthExceeded) { + std::unique_ptr message; + auto listener = std::make_shared(&message); + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 50; + MessageDecoder decoder(listener, options); + int32_t buf[2] = {internal::kIpcContinuationToken, 100}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + decoder.Consume(reinterpret_cast(buf), sizeof(buf))); +} + +TEST(TestStreamDecoderLimits, MaxBodyLengthExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + StreamWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + auto listener = std::make_shared(); + StreamDecoder decoder(listener, options); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + decoder.Consume(writer_helper.buffer_)); + + auto default_listener = std::make_shared(); + StreamDecoder default_decoder(default_listener); + ASSERT_OK(default_decoder.Consume(writer_helper.buffer_)); + ASSERT_EQ(default_listener->num_record_batches(), 1); +} + +TEST(TestDecompressBufferLimits, NegativeUncompressedSize) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + int64_t uncompressed_size = -5; + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + buf_data.append(16, 'x'); + auto buf = Buffer::FromString(std::move(buf_data)); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("negative capacity: -5"), + internal::DecompressBuffer(buf, IpcReadOptions::Defaults(), codec.get())); +} + +TEST(TestDecompressBufferLimits, ZeroCompressedPayloadWithNonZeroUncompressed) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + int64_t uncompressed_size = 100; + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + auto buf = Buffer::FromString(std::move(buf_data)); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr( + "Compressed buffer has 0 payload bytes but non-zero uncompressed size 100"), + internal::DecompressBuffer(buf, IpcReadOptions::Defaults(), codec.get())); +} + +TEST(TestDecompressBufferLimits, MaxDecompressionRatioExceeded) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + int64_t uncompressed_size = 1000; + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + buf_data.append(10, 'x'); + auto buf = Buffer::FromString(std::move(buf_data)); + + auto options = IpcReadOptions::Defaults(); + options.max_decompression_ratio = 5; + + EXPECT_RAISES_WITH_MESSAGE_THAT( + CapacityError, + ::testing::HasSubstr("exceeds maximum allowed decompression ratio (5)"), + internal::DecompressBuffer(buf, options, codec.get())); +} + +TEST(TestMetadataBounds, EmptyOrInvalidTensorMetadata) { + auto empty_buf = Buffer::FromString(""); + std::shared_ptr type; + std::vector shape; + std::vector strides; + std::vector dim_names; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("Invalid flatbuffers message"), + internal::GetTensorMetadata(*empty_buf, &type, &shape, &strides, &dim_names)); +} + } // namespace test } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 8a2c47298854..ce9852ab3166 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -555,6 +556,10 @@ class ArrayLoader { ArrayData* out_ = nullptr; }; +} // namespace + +namespace internal { + Result> DecompressBuffer(const std::shared_ptr& buf, const IpcReadOptions& options, util::Codec* codec) { @@ -576,12 +581,35 @@ Result> DecompressBuffer(const std::shared_ptr& return SliceBuffer(buf, sizeof(int64_t), compressed_size); } + if (compressed_size == 0 && uncompressed_size > 0) { + return Status::Invalid( + "Compressed buffer has 0 payload bytes but non-zero uncompressed size ", + uncompressed_size); + } + + if (options.max_decompression_ratio > 0) { + int64_t max_allowed = 0; + if (ARROW_PREDICT_FALSE(::arrow::internal::MultiplyWithOverflow( + compressed_size, static_cast(options.max_decompression_ratio), + &max_allowed))) { + max_allowed = std::numeric_limits::max(); + } + if (ARROW_PREDICT_FALSE(uncompressed_size > max_allowed)) { + return Status::CapacityError("Decompressed size (", uncompressed_size, + ") exceeds maximum allowed decompression ratio (", + options.max_decompression_ratio, ")"); + } + } + + std::span compressed_payload(data + sizeof(int64_t), + static_cast(compressed_size)); + ARROW_ASSIGN_OR_RAISE(auto uncompressed, AllocateBuffer(uncompressed_size, options.memory_pool)); ARROW_ASSIGN_OR_RAISE( int64_t actual_decompressed, - codec->Decompress(compressed_size, data + sizeof(int64_t), uncompressed_size, + codec->Decompress(compressed_size, compressed_payload.data(), uncompressed_size, uncompressed->mutable_data())); if (actual_decompressed != uncompressed_size) { return Status::Invalid("Failed to fully decompress buffer, expected ", @@ -592,6 +620,12 @@ Result> DecompressBuffer(const std::shared_ptr& return uncompressed; } +} // namespace internal + +namespace { + +using internal::DecompressBuffer; + Status DecompressBuffers(Compression::type compression, const IpcReadOptions& options, ArrayDataVector* fields) { struct BufferAccumulator { @@ -2167,7 +2201,7 @@ class StreamDecoder::StreamDecoderImpl : public StreamDecoderInternal { explicit StreamDecoderImpl(std::shared_ptr listener, IpcReadOptions options) : StreamDecoderInternal(std::move(listener), options), message_decoder_(std::shared_ptr(this, [](void*) {}), - options.memory_pool) {} + options) {} Status Consume(const uint8_t* data, int64_t size) { return message_decoder_.Consume(data, size); diff --git a/cpp/src/arrow/ipc/reader_internal.h b/cpp/src/arrow/ipc/reader_internal.h index d61ac96f0f14..984be965a958 100644 --- a/cpp/src/arrow/ipc/reader_internal.h +++ b/cpp/src/arrow/ipc/reader_internal.h @@ -29,9 +29,21 @@ namespace io { struct ReadRange; } +namespace util { +class Codec; +} + namespace ipc { +struct IpcReadOptions; + namespace internal { + +ARROW_EXPORT +Result> DecompressBuffer(const std::shared_ptr& buf, + const IpcReadOptions& options, + util::Codec* codec); + /// \class IoRecordedRandomAccessFile /// \brief An RandomAccessFile that doesn't perform real IO, but only save all the IO /// operations it receives, including read operation's , for replaying From 912977184d7bb10d8cdd623c782b6b0bfaf5c35e Mon Sep 17 00:00:00 2001 From: Awesome06 Date: Wed, 18 Nov 2026 21:49:18 +0530 Subject: [PATCH 2/5] GH-55669: [C++][IPC] Fix use-after-free on EOS in MessageDecoder and enforce reader limits * Avoid use-after-free in MessageDecoderImpl when listener OnEOS resets the decoder * Early exit when data span is fully consumed before trailing member dereferences * Wire IpcReadOptions through ReadMessage, MessageReader, and RecordBatchFileReader * Add block bounds validation and test coverage for IPC reader limits --- cpp/src/arrow/ipc/message.cc | 196 +++++++++++++++++++++++---- cpp/src/arrow/ipc/message.h | 25 ++++ cpp/src/arrow/ipc/read_write_test.cc | 95 +++++++++++++ cpp/src/arrow/ipc/reader.cc | 47 +++++-- 4 files changed, 327 insertions(+), 36 deletions(-) diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 4edbea5adf8f..923f07f174cf 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -330,6 +330,10 @@ struct ReadMessageState { : listener(std::make_shared(&result)), decoder(std::make_shared(listener)) {} + explicit ReadMessageState(const IpcReadOptions& options) + : listener(std::make_shared(&result)), + decoder(std::make_shared(listener, options)) {} + // ReadMessageState points into itself, so it shouldn't be moved ReadMessageState(ReadMessageState&&) = delete; ReadMessageState& operator=(ReadMessageState&&) = delete; @@ -420,11 +424,25 @@ static Result> ReadMessageContinued( } // namespace Result> ReadMessage(std::shared_ptr metadata, - std::shared_ptr body) { + std::shared_ptr body, + const IpcReadOptions& options) { std::unique_ptr result; auto listener = std::make_shared(&result); // If the user does not pass in a body buffer then we assume they are skipping it - MessageDecoder decoder(listener, default_memory_pool(), body == nullptr); + MessageDecoder decoder(listener, options, body == nullptr); + + if (options.max_metadata_length >= 0 && + metadata->size() > options.max_metadata_length) { + return Status::Invalid("IPC metadata length ", metadata->size(), + " exceeds maximum allowed metadata length ", + options.max_metadata_length); + } + if (body != nullptr && options.max_body_length >= 0 && + body->size() > options.max_body_length) { + return Status::Invalid("IPC body length ", body->size(), + " exceeds maximum allowed body length ", + options.max_body_length); + } if (metadata->size() < decoder.next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -467,15 +485,31 @@ Result> ReadMessage(std::shared_ptr metadata, } } +Result> ReadMessage(std::shared_ptr metadata, + std::shared_ptr body) { + return ReadMessage(std::move(metadata), std::move(body), IpcReadOptions::Defaults()); +} + // Common helper for the two ReadMessage overloads that take a file + offset. // When body_length is provided, metadata and body are read in a single IO. // When body_length is absent, metadata is read first, then the body is read // separately. static Result> ReadMessageInternal( int64_t offset, int32_t metadata_length, std::optional body_length, - io::RandomAccessFile* file, MemoryPool* pool, + io::RandomAccessFile* file, const IpcReadOptions& options, const FieldsLoaderFunction& fields_loader) { - ReadMessageState state; + if (options.max_metadata_length >= 0 && metadata_length > options.max_metadata_length) { + return Status::Invalid("IPC metadata length ", metadata_length, + " exceeds maximum allowed metadata length ", + options.max_metadata_length); + } + if (body_length.has_value() && options.max_body_length >= 0 && + *body_length > options.max_body_length) { + return Status::Invalid("IPC body length ", *body_length, + " exceeds maximum allowed body length ", + options.max_body_length); + } + ReadMessageState state(options); if (metadata_length < state.decoder->next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -491,15 +525,17 @@ static Result> ReadMessageInternal( file->ReadAt(offset, read_size, /*allow_short_read=*/false)); - return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, pool, - fields_loader, &state); + return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, + options.memory_pool, fields_loader, &state); } Result> ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = file->io_context().pool(); return ReadMessageInternal(offset, metadata_length, /*body_length=*/std::nullopt, file, - file->io_context().pool(), fields_loader); + options, fields_loader); } Result> ReadMessage(const int64_t offset, @@ -507,17 +543,40 @@ Result> ReadMessage(const int64_t offset, const int64_t body_length, io::RandomAccessFile* file, MemoryPool* pool, const FieldsLoaderFunction& fields_loader) { - return ReadMessageInternal(offset, metadata_length, body_length, file, pool, + auto options = IpcReadOptions::Defaults(); + options.memory_pool = pool; + return ReadMessageInternal(offset, metadata_length, body_length, file, options, + fields_loader); +} + +Result> ReadMessage(const int64_t offset, + const int32_t metadata_length, + const int64_t body_length, + io::RandomAccessFile* file, + const IpcReadOptions& options, + const FieldsLoaderFunction& fields_loader) { + return ReadMessageInternal(offset, metadata_length, body_length, file, options, fields_loader); } Future> ReadMessageAsync(int64_t offset, int32_t metadata_length, int64_t body_length, io::RandomAccessFile* file, + const IpcReadOptions& options, const io::IOContext& context) { + if (options.max_metadata_length >= 0 && metadata_length > options.max_metadata_length) { + return Status::Invalid("IPC metadata length ", metadata_length, + " exceeds maximum allowed metadata length ", + options.max_metadata_length); + } + if (options.max_body_length >= 0 && body_length > options.max_body_length) { + return Status::Invalid("IPC body length ", body_length, + " exceeds maximum allowed body length ", + options.max_body_length); + } // Make a std::shared_ptr so as to have a stable ReadMessageState pointer, // since state->listener will point back to state->result. - auto state = std::make_shared(); + auto state = std::make_shared(options); if (metadata_length < state->decoder->next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -530,11 +589,20 @@ Future> ReadMessageAsync(int64_t offset, int32_t metada // Pass a nullptr file to ensure that no further IO occurs // (we have fetched all the required bytes). return ReadMessageContinued(offset, metadata_length, body_length, metadata, - /*file=*/nullptr, context.pool(), + /*file=*/nullptr, options.memory_pool, /*fields_loader=*/{}, state.get()); }); } +Future> ReadMessageAsync(int64_t offset, int32_t metadata_length, + int64_t body_length, + io::RandomAccessFile* file, + const io::IOContext& context) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = context.pool(); + return ReadMessageAsync(offset, metadata_length, body_length, file, options, context); +} + Status AlignStream(io::InputStream* stream, int32_t alignment) { ARROW_ASSIGN_OR_RAISE(int64_t position, stream->Tell()); return stream->Advance(PaddedLength(position, alignment) - position); @@ -614,10 +682,11 @@ Status DecodeMessage(MessageDecoder* decoder, io::InputStream* file) { } } -Result> ReadMessage(io::InputStream* file, MemoryPool* pool) { +Result> ReadMessage(io::InputStream* file, + const IpcReadOptions& options) { std::unique_ptr message; auto listener = std::make_shared(&message); - MessageDecoder decoder(listener, pool); + MessageDecoder decoder(listener, options); ARROW_RETURN_NOT_OK(DecodeMessage(&decoder, file)); if (!message) { return nullptr; @@ -626,6 +695,12 @@ Result> ReadMessage(io::InputStream* file, MemoryPool* } } +Result> ReadMessage(io::InputStream* file, MemoryPool* pool) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = pool; + return ReadMessage(file, options); +} + Status WriteMessage(const Buffer& message, const IpcWriteOptions& options, io::OutputStream* file, int32_t* message_length) { const int32_t prefix_size = options.write_legacy_ipc_format ? 4 : 8; @@ -705,16 +780,29 @@ class MessageDecoder::MessageDecoderImpl { } if (buffered_size_ == 0) { std::span data_span(data, static_cast(size)); - while (data_span.size() >= static_cast(next_required_size_)) { + while (!data_span.empty() && + data_span.size() >= static_cast(next_required_size_)) { auto used_size = next_required_size_; switch (state_) { - case State::INITIAL: + case State::INITIAL: { + bool is_eos = (bit_util::FromLittleEndian( + util::SafeLoadAs(data_span.data())) == 0); RETURN_NOT_OK(ConsumeInitialData(data_span.data(), next_required_size_)); + if (is_eos) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: + } + case State::METADATA_LENGTH: { + bool is_eos = (bit_util::FromLittleEndian( + util::SafeLoadAs(data_span.data())) == 0); RETURN_NOT_OK( ConsumeMetadataLengthData(data_span.data(), next_required_size_)); + if (is_eos) { + return Status::OK(); + } break; + } case State::METADATA: { // We need to copy metadata because it's used in // ConsumeBody(). ConsumeBody() may be called from another @@ -741,7 +829,13 @@ class MessageDecoder::MessageDecoderImpl { case State::EOS: return Status::OK(); } + if (data_span.size() == static_cast(used_size)) { + return Status::OK(); + } data_span = data_span.subspan(static_cast(used_size)); + if (state_ == State::EOS) { + return Status::OK(); + } } data = data_span.data(); size = static_cast(data_span.size()); @@ -751,6 +845,10 @@ class MessageDecoder::MessageDecoderImpl { return Status::OK(); } + if (state_ == State::EOS) { + return Status::OK(); + } + // We need to copy unused data because the given data for the // current ConsumeData() call may be invalid in the next // ConsumeData() call. @@ -763,15 +861,27 @@ class MessageDecoder::MessageDecoderImpl { Status ConsumeBuffer(std::shared_ptr buffer) { if (buffered_size_ == 0) { - while (buffer->size() >= next_required_size_) { + while (buffer->size() > 0 && buffer->size() >= next_required_size_) { auto used_size = next_required_size_; switch (state_) { - case State::INITIAL: + case State::INITIAL: { + ARROW_ASSIGN_OR_RAISE(auto token, ConsumeDataBufferInt32(buffer)); + bool is_eos = (bit_util::FromLittleEndian(token) == 0); RETURN_NOT_OK(ConsumeInitialBuffer(buffer)); + if (is_eos) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: + } + case State::METADATA_LENGTH: { + ARROW_ASSIGN_OR_RAISE(auto length, ConsumeDataBufferInt32(buffer)); + bool is_eos = (bit_util::FromLittleEndian(length) == 0); RETURN_NOT_OK(ConsumeMetadataLengthBuffer(buffer)); + if (is_eos) { + return Status::OK(); + } break; + } case State::METADATA: if (buffer->size() == next_required_size_) { return ConsumeMetadataBuffer(buffer); @@ -795,6 +905,9 @@ class MessageDecoder::MessageDecoderImpl { return Status::OK(); } buffer = SliceBuffer(buffer, used_size); + if (state_ == State::EOS) { + return Status::OK(); + } } } @@ -802,6 +915,10 @@ class MessageDecoder::MessageDecoderImpl { return Status::OK(); } + if (state_ == State::EOS) { + return Status::OK(); + } + buffered_size_ += buffer->size(); chunks_.push_back(std::move(buffer)); return ConsumeChunks(); @@ -821,12 +938,26 @@ class MessageDecoder::MessageDecoderImpl { } switch (state_) { - case State::INITIAL: - RETURN_NOT_OK(ConsumeInitialChunks()); + case State::INITIAL: { + int32_t continuation = 0; + RETURN_NOT_OK(ConsumeDataChunks(sizeof(int32_t), &continuation)); + auto token = bit_util::FromLittleEndian(continuation); + RETURN_NOT_OK(ConsumeInitial(token)); + if (token == 0) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: - RETURN_NOT_OK(ConsumeMetadataLengthChunks()); + } + case State::METADATA_LENGTH: { + int32_t metadata_length = 0; + RETURN_NOT_OK(ConsumeDataChunks(sizeof(int32_t), &metadata_length)); + auto length = bit_util::FromLittleEndian(metadata_length); + RETURN_NOT_OK(ConsumeMetadataLength(length)); + if (length == 0) { + return Status::OK(); + } break; + } case State::METADATA: RETURN_NOT_OK(ConsumeMetadataChunks()); break; @@ -1130,14 +1261,17 @@ MessageDecoder::State MessageDecoder::state() const { return impl_->state(); } /// \brief Implementation of MessageReader that reads from InputStream class InputStreamMessageReader : public MessageReader, public MessageDecoderListener { public: - explicit InputStreamMessageReader(io::InputStream* stream) + explicit InputStreamMessageReader(io::InputStream* stream, + IpcReadOptions options = IpcReadOptions::Defaults()) : stream_(stream), owned_stream_(), message_(), - decoder_(std::shared_ptr(this, [](void*) {})) {} + decoder_(std::shared_ptr(this, [](void*) {}), + std::move(options)) {} - explicit InputStreamMessageReader(const std::shared_ptr& owned_stream) - : InputStreamMessageReader(owned_stream.get()) { + explicit InputStreamMessageReader(const std::shared_ptr& owned_stream, + IpcReadOptions options = IpcReadOptions::Defaults()) + : InputStreamMessageReader(owned_stream.get(), std::move(options)) { owned_stream_ = owned_stream; } @@ -1178,10 +1312,20 @@ std::unique_ptr MessageReader::Open(io::InputStream* stream) { return std::make_unique(stream); } +std::unique_ptr MessageReader::Open(io::InputStream* stream, + const IpcReadOptions& options) { + return std::make_unique(stream, options); +} + std::unique_ptr MessageReader::Open( const std::shared_ptr& owned_stream) { return std::make_unique(owned_stream); } +std::unique_ptr MessageReader::Open( + const std::shared_ptr& owned_stream, const IpcReadOptions& options) { + return std::make_unique(owned_stream, options); +} + } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index f9c5adb3822b..cfba9dc89ae9 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -462,10 +462,15 @@ class ARROW_EXPORT MessageReader { /// \brief Create MessageReader that reads from InputStream static std::unique_ptr Open(io::InputStream* stream); + static std::unique_ptr Open(io::InputStream* stream, + const IpcReadOptions& options); /// \brief Create MessageReader that reads from owned InputStream static std::unique_ptr Open( const std::shared_ptr& owned_stream); + static std::unique_ptr Open( + const std::shared_ptr& owned_stream, + const IpcReadOptions& options); /// \brief Read next Message from the interface /// @@ -522,6 +527,12 @@ Result> ReadMessage( io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(), const FieldsLoaderFunction& fields_loader = {}); +ARROW_EXPORT +Result> ReadMessage( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, const IpcReadOptions& options, + const FieldsLoaderFunction& fields_loader = {}); + /// \brief Read encapsulated IPC message from cached buffers /// /// The buffers should contain an entire message. Partial reads are not handled. @@ -541,11 +552,21 @@ Result> ReadMessage( ARROW_EXPORT Result> ReadMessage( std::shared_ptr metadata, std::shared_ptr body); +ARROW_EXPORT Result> ReadMessage( + std::shared_ptr metadata, std::shared_ptr body, + const IpcReadOptions& options); + ARROW_EXPORT Future> ReadMessageAsync( const int64_t offset, const int32_t metadata_length, const int64_t body_length, io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context()); +ARROW_EXPORT +Future> ReadMessageAsync( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, const IpcReadOptions& options, + const io::IOContext& context = io::default_io_context()); + /// \brief Advance stream to an 8-byte offset if its position is not a multiple /// of 8 already /// \param[in] stream an input stream @@ -581,6 +602,10 @@ ARROW_EXPORT Result> ReadMessage(io::InputStream* stream, MemoryPool* pool = default_memory_pool()); +ARROW_EXPORT +Result> ReadMessage(io::InputStream* stream, + const IpcReadOptions& options); + /// \brief Feed data from InputStream to MessageDecoder to decode an /// encapsulated IPC message (metadata and body) /// diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 0b959329d2ff..297bce7ed293 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3466,6 +3466,61 @@ TEST(TestStreamDecoderLimits, MaxBodyLengthExceeded) { ASSERT_EQ(default_listener->num_record_batches(), 1); } +TEST(TestStreamReaderLimits, MaxBodyLengthExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + StreamWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + io::BufferReader stream_reader_buf(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchStreamReader::Open(&stream_reader_buf, options)); + std::shared_ptr out_batch; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadNext(&out_batch)); +} + +TEST(TestFileReaderLimits, MaxMetadataLengthExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + FileWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 10; + auto buf_reader = std::make_shared(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(buf_reader.get(), options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + reader->ReadRecordBatch(0)); +} + +TEST(TestFileReaderLimits, MaxBodyLengthExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + FileWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + auto buf_reader = std::make_shared(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(buf_reader.get(), options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadRecordBatch(0)); +} + TEST(TestDecompressBufferLimits, NegativeUncompressedSize) { if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { GTEST_SKIP() << "LZ4 compression not available"; @@ -3519,6 +3574,46 @@ TEST(TestDecompressBufferLimits, MaxDecompressionRatioExceeded) { internal::DecompressBuffer(buf, options, codec.get())); } +TEST(TestDecompressBufferLimits, ZeroDecompressionRatioExceeded) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + int64_t uncompressed_size = 1000; + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + buf_data.append(10, 'x'); + auto buf = Buffer::FromString(std::move(buf_data)); + + auto options = IpcReadOptions::Defaults(); + options.max_decompression_ratio = 0; + + EXPECT_RAISES_WITH_MESSAGE_THAT( + CapacityError, + ::testing::HasSubstr("exceeds maximum allowed decompression ratio (0)"), + internal::DecompressBuffer(buf, options, codec.get())); +} + +TEST(TestDecompressBufferLimits, NegativeDecompressionRatioInvalid) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + int64_t uncompressed_size = 1000; + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + buf_data.append(10, 'x'); + auto buf = Buffer::FromString(std::move(buf_data)); + + auto options = IpcReadOptions::Defaults(); + options.max_decompression_ratio = -2; + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_decompression_ratio must be -1 or non-negative, got -2"), + internal::DecompressBuffer(buf, options, codec.get())); +} + TEST(TestMetadataBounds, EmptyOrInvalidTensorMetadata) { auto empty_buf = Buffer::FromString(""); std::shared_ptr type; diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index ce9852ab3166..d149c5f6e843 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -587,7 +587,7 @@ Result> DecompressBuffer(const std::shared_ptr& uncompressed_size); } - if (options.max_decompression_ratio > 0) { + if (options.max_decompression_ratio >= 0) { int64_t max_allowed = 0; if (ARROW_PREDICT_FALSE(::arrow::internal::MultiplyWithOverflow( compressed_size, static_cast(options.max_decompression_ratio), @@ -599,6 +599,9 @@ Result> DecompressBuffer(const std::shared_ptr& ") exceeds maximum allowed decompression ratio (", options.max_decompression_ratio, ")"); } + } else if (ARROW_PREDICT_FALSE(options.max_decompression_ratio < -1)) { + return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ", + options.max_decompression_ratio); } std::span compressed_payload(data + sizeof(int64_t), @@ -829,7 +832,7 @@ Status GetCompressionExperimental(const flatbuf::Message* message, Status ReadContiguousPayload(io::InputStream* file, const IpcReadOptions& options, std::unique_ptr* message) { - ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options.memory_pool)); + ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options)); if (*message == nullptr) { return Status::Invalid("Unable to read metadata at offset"); } @@ -1275,12 +1278,12 @@ Result> RecordBatchStreamReader::Open( Result> RecordBatchStreamReader::Open( io::InputStream* stream, const IpcReadOptions& options) { - return Open(MessageReader::Open(stream), options); + return Open(MessageReader::Open(stream, options), options); } Result> RecordBatchStreamReader::Open( const std::shared_ptr& stream, const IpcReadOptions& options) { - return Open(MessageReader::Open(stream), options); + return Open(MessageReader::Open(stream, options), options); } // ---------------------------------------------------------------------- @@ -1314,6 +1317,21 @@ Status CheckAligned(const FileBlock& block) { return Status::OK(); } +Status CheckBlockLimits(const FileBlock& block, const IpcReadOptions& options) { + if (options.max_metadata_length >= 0 && + block.metadata_length > options.max_metadata_length) { + return Status::Invalid("IPC metadata length ", block.metadata_length, + " exceeds maximum allowed metadata length ", + options.max_metadata_length); + } + if (options.max_body_length >= 0 && block.body_length > options.max_body_length) { + return Status::Invalid("IPC body length ", block.body_length, + " exceeds maximum allowed body length ", + options.max_body_length); + } + return Status::OK(); +} + template Result CheckBodyLength(MessagePtr message, const FileBlock& block) { if (message->body_length() != block.body_length) { @@ -1331,9 +1349,10 @@ Result> ReadMessageFromBlock( const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options)); ARROW_ASSIGN_OR_RAISE( auto message, ReadMessage(block.offset, block.metadata_length, block.body_length, - file, options.memory_pool, fields_loader)); + file, options, fields_loader)); return CheckBodyLength(std::move(message), block); } @@ -1341,8 +1360,9 @@ Future> ReadMessageFromBlockAsync( const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, const io::IOContext& io_context) { RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options)); return ReadMessageAsync(block.offset, block.metadata_length, block.body_length, file, - io_context.with_pool(options.memory_pool)) + options, io_context) .Then([block](std::shared_ptr message) { return CheckBodyLength(std::move(message), block); }); @@ -1636,10 +1656,16 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { all_metadata_ready.Then([this, index]() -> Result> { stats_.num_messages.fetch_add(1, std::memory_order_relaxed); ARROW_ASSIGN_OR_RAISE(FileBlock block, GetRecordBatchBlock(index)); + if (options_.max_metadata_length >= 0 && + block.metadata_length > options_.max_metadata_length) { + return Status::Invalid("IPC metadata length ", block.metadata_length, + " exceeds maximum allowed metadata length ", + options_.max_metadata_length); + } ARROW_ASSIGN_OR_RAISE( std::shared_ptr metadata, metadata_cache_->Read({block.offset, block.metadata_length})); - return ReadMessage(std::move(metadata), nullptr); + return ReadMessage(std::move(metadata), nullptr, options_); }); cached_metadata_.emplace(index, metadata_loaded); } @@ -2080,15 +2106,16 @@ WholeIpcFileRecordBatchGenerator::operator()() { Future> WholeIpcFileRecordBatchGenerator::ReadBlock( const FileBlock& block) { + RETURN_NOT_OK(CheckBlockLimits(block, state_->options_)); if (cached_source_) { auto cached_source = cached_source_; io::ReadRange range{block.offset, block.metadata_length + block.body_length}; - auto pool = state_->options_.memory_pool; + auto options = state_->options_; return cached_source->WaitFor({range}).Then( - [cached_source, pool, range]() -> Result> { + [cached_source, options, range]() -> Result> { ARROW_ASSIGN_OR_RAISE(auto buffer, cached_source->Read(range)); io::BufferReader stream(std::move(buffer)); - return ReadMessage(&stream, pool); + return ReadMessage(&stream, options); }); } else { return ReadMessageFromBlockAsync(block, state_->options_, state_->file_, io_context_); From 138d08726e7d20e0523a9610fc01dc182573c027 Mon Sep 17 00:00:00 2001 From: Awesome06 Date: Thu, 19 Nov 2026 00:15:14 +0530 Subject: [PATCH 3/5] GH-55669: [C++][IPC] Reject invalid negative read limits and enforce file footer bound * Add IpcReadOptions::Validate() to reject negative values below -1 * Wire options validation to stream and file reader entry points * Enforce max_metadata_length against footer_length in ReadFooterAsync * Add tests for footer bounds, negative options validation, and decoder rejection --- cpp/src/arrow/ipc/message.cc | 5 ++ cpp/src/arrow/ipc/options.cc | 20 +++++++ cpp/src/arrow/ipc/options.h | 5 ++ cpp/src/arrow/ipc/read_write_test.cc | 82 ++++++++++++++++++++++++++-- cpp/src/arrow/ipc/reader.cc | 17 +++++- 5 files changed, 122 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 923f07f174cf..2411316da37d 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -426,6 +426,7 @@ static Result> ReadMessageContinued( Result> ReadMessage(std::shared_ptr metadata, std::shared_ptr body, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); std::unique_ptr result; auto listener = std::make_shared(&result); // If the user does not pass in a body buffer then we assume they are skipping it @@ -498,6 +499,7 @@ static Result> ReadMessageInternal( int64_t offset, int32_t metadata_length, std::optional body_length, io::RandomAccessFile* file, const IpcReadOptions& options, const FieldsLoaderFunction& fields_loader) { + RETURN_NOT_OK(options.Validate()); if (options.max_metadata_length >= 0 && metadata_length > options.max_metadata_length) { return Status::Invalid("IPC metadata length ", metadata_length, " exceeds maximum allowed metadata length ", @@ -564,6 +566,7 @@ Future> ReadMessageAsync(int64_t offset, int32_t metada io::RandomAccessFile* file, const IpcReadOptions& options, const io::IOContext& context) { + RETURN_NOT_OK(options.Validate()); if (options.max_metadata_length >= 0 && metadata_length > options.max_metadata_length) { return Status::Invalid("IPC metadata length ", metadata_length, " exceeds maximum allowed metadata length ", @@ -775,6 +778,7 @@ class MessageDecoder::MessageDecoderImpl { skip_body_(skip_body) {} Status ConsumeData(const uint8_t* data, int64_t size) { + RETURN_NOT_OK(options_.Validate()); if (size < 0) { return Status::Invalid("Invalid data size: negative byte count"); } @@ -860,6 +864,7 @@ class MessageDecoder::MessageDecoderImpl { } Status ConsumeBuffer(std::shared_ptr buffer) { + RETURN_NOT_OK(options_.Validate()); if (buffered_size_ == 0) { while (buffer->size() > 0 && buffer->size() >= next_required_size_) { auto used_size = next_required_size_; diff --git a/cpp/src/arrow/ipc/options.cc b/cpp/src/arrow/ipc/options.cc index e5b14a47face..d316e5abc01b 100644 --- a/cpp/src/arrow/ipc/options.cc +++ b/cpp/src/arrow/ipc/options.cc @@ -26,6 +26,26 @@ IpcWriteOptions IpcWriteOptions::Defaults() { return IpcWriteOptions(); } IpcReadOptions IpcReadOptions::Defaults() { return IpcReadOptions(); } +Status IpcReadOptions::Validate() const { + if (max_recursion_depth < 0) { + return Status::Invalid("max_recursion_depth must be non-negative, got ", + max_recursion_depth); + } + if (max_metadata_length < -1) { + return Status::Invalid("max_metadata_length must be -1 or non-negative, got ", + max_metadata_length); + } + if (max_body_length < -1) { + return Status::Invalid("max_body_length must be -1 or non-negative, got ", + max_body_length); + } + if (max_decompression_ratio < -1) { + return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ", + max_decompression_ratio); + } + return Status::OK(); +} + namespace internal { Status CheckCompressionSupported(Compression::type codec) { diff --git a/cpp/src/arrow/ipc/options.h b/cpp/src/arrow/ipc/options.h index 229ad2472dc8..1e92f6eb2f32 100644 --- a/cpp/src/arrow/ipc/options.h +++ b/cpp/src/arrow/ipc/options.h @@ -190,6 +190,8 @@ struct ARROW_EXPORT IpcReadOptions { io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults(); /// \brief Maximum allowed metadata length in bytes (-1 means unlimited). + /// + /// Enforced on message metadata in IPC streams and file footer metadata in IPC files. int64_t max_metadata_length = -1; /// \brief Maximum allowed body length in bytes (-1 means unlimited). @@ -198,6 +200,9 @@ struct ARROW_EXPORT IpcReadOptions { /// \brief Maximum allowed decompression expansion ratio (-1 means unlimited). int32_t max_decompression_ratio = -1; + /// \brief Validate read options. + Status Validate() const; + static IpcReadOptions Defaults(); }; diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 297bce7ed293..a8f691bca13b 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3485,7 +3485,7 @@ TEST(TestStreamReaderLimits, MaxBodyLengthExceeded) { reader->ReadNext(&out_batch)); } -TEST(TestFileReaderLimits, MaxMetadataLengthExceeded) { +TEST(TestFileReaderLimits, MaxFooterMetadataLengthExceeded) { std::shared_ptr batch; ASSERT_OK(MakeIntRecordBatch(&batch)); FileWriterHelper writer_helper; @@ -3496,11 +3496,9 @@ TEST(TestFileReaderLimits, MaxMetadataLengthExceeded) { auto options = IpcReadOptions::Defaults(); options.max_metadata_length = 10; auto buf_reader = std::make_shared(writer_helper.buffer_); - ASSERT_OK_AND_ASSIGN(auto reader, - RecordBatchFileReader::Open(buf_reader.get(), options)); EXPECT_RAISES_WITH_MESSAGE_THAT( Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), - reader->ReadRecordBatch(0)); + RecordBatchFileReader::Open(buf_reader.get(), options)); } TEST(TestFileReaderLimits, MaxBodyLengthExceeded) { @@ -3521,6 +3519,82 @@ TEST(TestFileReaderLimits, MaxBodyLengthExceeded) { reader->ReadRecordBatch(0)); } +TEST(TestIpcReadOptions, ValidateNegativeLimits) { + auto options = IpcReadOptions::Defaults(); + ASSERT_OK(options.Validate()); + + options.max_metadata_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_metadata_length must be -1 or non-negative, got -2"), + options.Validate()); + + options = IpcReadOptions::Defaults(); + options.max_body_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_body_length must be -1 or non-negative, got -2"), + options.Validate()); + + options = IpcReadOptions::Defaults(); + options.max_decompression_ratio = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_decompression_ratio must be -1 or non-negative, got -2"), + options.Validate()); + + options = IpcReadOptions::Defaults(); + options.max_recursion_depth = -1; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_recursion_depth must be non-negative, got -1"), + options.Validate()); +} + +TEST(TestStreamReaderLimits, NegativeLimitsInvalid) { + io::BufferReader empty_reader(""); + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_metadata_length must be -1 or non-negative, got -2"), + RecordBatchStreamReader::Open(&empty_reader, options)); + + options = IpcReadOptions::Defaults(); + options.max_body_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_body_length must be -1 or non-negative, got -2"), + RecordBatchStreamReader::Open(&empty_reader, options)); +} + +TEST(TestFileReaderLimits, NegativeLimitsInvalid) { + io::BufferReader empty_reader(""); + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_metadata_length must be -1 or non-negative, got -2"), + RecordBatchFileReader::Open(&empty_reader, options)); + + options = IpcReadOptions::Defaults(); + options.max_body_length = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_body_length must be -1 or non-negative, got -2"), + RecordBatchFileReader::Open(&empty_reader, options)); +} + +TEST(TestMessageDecoderBounds, NegativeLimitsInvalid) { + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = -2; + auto empty_buf = Buffer::FromString(""); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("max_metadata_length must be -1 or non-negative, got -2"), + ReadMessage(empty_buf, nullptr, options)); +} + TEST(TestDecompressBufferLimits, NegativeUncompressedSize) { if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { GTEST_SKIP() << "LZ4 compression not available"; diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index d149c5f6e843..eba438799b10 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -563,6 +563,7 @@ namespace internal { Result> DecompressBuffer(const std::shared_ptr& buf, const IpcReadOptions& options, util::Codec* codec) { + RETURN_NOT_OK(options.Validate()); if (buf == nullptr || buf->size() == 0) { return buf; } @@ -599,9 +600,6 @@ Result> DecompressBuffer(const std::shared_ptr& ") exceeds maximum allowed decompression ratio (", options.max_decompression_ratio, ")"); } - } else if (ARROW_PREDICT_FALSE(options.max_decompression_ratio < -1)) { - return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ", - options.max_decompression_ratio); } std::span compressed_payload(data + sizeof(int64_t), @@ -1269,6 +1267,7 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader, Result> RecordBatchStreamReader::Open( std::unique_ptr message_reader, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // Private ctor auto result = std::make_shared(std::move(message_reader), options); @@ -1278,11 +1277,13 @@ Result> RecordBatchStreamReader::Open( Result> RecordBatchStreamReader::Open( io::InputStream* stream, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); return Open(MessageReader::Open(stream, options), options); } Result> RecordBatchStreamReader::Open( const std::shared_ptr& stream, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); return Open(MessageReader::Open(stream, options), options); } @@ -1318,6 +1319,7 @@ Status CheckAligned(const FileBlock& block) { } Status CheckBlockLimits(const FileBlock& block, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); if (options.max_metadata_length >= 0 && block.metadata_length > options.max_metadata_length) { return Status::Invalid("IPC metadata length ", block.metadata_length, @@ -1555,6 +1557,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { Status Open(io::RandomAccessFile* file, int64_t footer_offset, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // The metadata_cache_ may have already been constructed with an owned file in the // owning overload of Open if (!metadata_cache_) { @@ -1584,6 +1587,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { Future<> OpenAsync(io::RandomAccessFile* file, int64_t footer_offset, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // The metadata_cache_ may have already been constructed with an owned file in the // owning overload of OpenAsync if (!metadata_cache_) { @@ -1984,6 +1988,13 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { return Status::Invalid("File is smaller than indicated metadata size"); } + if (self->options_.max_metadata_length >= 0 && + footer_length > self->options_.max_metadata_length) { + return Status::Invalid("IPC file footer metadata length ", footer_length, + " exceeds maximum allowed metadata length ", + self->options_.max_metadata_length); + } + // Now read the footer auto read_footer = self->file_->ReadAsync( self->footer_offset_ - footer_length - kTotalMagicSize, footer_length, From f47e80d5a8d6d325a52f15d3333ec3b075a78625 Mon Sep 17 00:00:00 2001 From: Awesome06 Date: Thu, 19 Nov 2026 00:40:54 +0530 Subject: [PATCH 4/5] GH-55669: [C++][IPC] Enforce metadata and body limits on custom MessageReader streams * Validate metadata and body lengths in StreamDecoderInternal::OnMessageDecoded * Protect custom MessageReader pathways against unbounded allocations * Add unit test coverage with MockCustomMessageReader --- cpp/src/arrow/ipc/read_write_test.cc | 69 ++++++++++++++++++++++++++++ cpp/src/arrow/ipc/reader.cc | 14 ++++++ 2 files changed, 83 insertions(+) diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index a8f691bca13b..a094197f0ebe 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3485,6 +3485,75 @@ TEST(TestStreamReaderLimits, MaxBodyLengthExceeded) { reader->ReadNext(&out_batch)); } +namespace { + +class MockCustomMessageReader : public MessageReader { + public: + explicit MockCustomMessageReader(std::vector> messages) + : messages_(std::move(messages)) {} + + Result> ReadNextMessage() override { + if (index_ < messages_.size()) { + return std::move(messages_[index_++]); + } + return nullptr; + } + + private: + std::vector> messages_; + size_t index_ = 0; +}; + +} // namespace + +TEST(TestStreamReaderLimits, CustomMessageReaderLimitsExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + StreamWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + // Test schema metadata length limit on custom reader + { + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 5; + std::vector> test_messages; + io::BufferReader r(writer_helper.buffer_); + auto mr = MessageReader::Open(&r); + while (true) { + ASSERT_OK_AND_ASSIGN(auto msg, mr->ReadNextMessage()); + if (!msg) break; + test_messages.push_back(std::move(msg)); + } + auto custom_mr = std::make_unique(std::move(test_messages)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + RecordBatchStreamReader::Open(std::move(custom_mr), options)); + } + + // Test body length limit on custom reader + { + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + std::vector> test_messages; + io::BufferReader r(writer_helper.buffer_); + auto mr = MessageReader::Open(&r); + while (true) { + ASSERT_OK_AND_ASSIGN(auto msg, mr->ReadNextMessage()); + if (!msg) break; + test_messages.push_back(std::move(msg)); + } + auto custom_mr = std::make_unique(std::move(test_messages)); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchStreamReader::Open(std::move(custom_mr), options)); + std::shared_ptr out_batch; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadNext(&out_batch)); + } +} + TEST(TestFileReaderLimits, MaxFooterMetadataLengthExceeded) { std::shared_ptr batch; ASSERT_OK(MakeIntRecordBatch(&batch)); diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index eba438799b10..07799abc2e27 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -1073,6 +1073,20 @@ class StreamDecoderInternal : public MessageDecoderListener { swap_endian_(false) {} Status OnMessageDecoded(std::unique_ptr message) override { + if (ARROW_PREDICT_TRUE(message != nullptr)) { + if (options_.max_metadata_length >= 0 && + message->metadata()->size() > options_.max_metadata_length) { + return Status::Invalid("IPC metadata length ", message->metadata()->size(), + " exceeds maximum allowed metadata length ", + options_.max_metadata_length); + } + if (options_.max_body_length >= 0 && + message->body_length() > options_.max_body_length) { + return Status::Invalid("IPC body length ", message->body_length(), + " exceeds maximum allowed body length ", + options_.max_body_length); + } + } ++stats_.num_messages; switch (state_) { case State::SCHEMA: From 65c48c8197512c2de89d5194a45cc138373907da Mon Sep 17 00:00:00 2001 From: Awesome06 Date: Thu, 19 Nov 2026 01:03:34 +0530 Subject: [PATCH 5/5] GH-55669: [C++][IPC] Validate block limits before pre-buffering file ranges --- cpp/src/arrow/ipc/read_write_test.cc | 59 +++++++++++++++++++++++++++- cpp/src/arrow/ipc/reader.cc | 34 +++++++++++----- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index a094197f0ebe..c67612661f58 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3588,6 +3588,61 @@ TEST(TestFileReaderLimits, MaxBodyLengthExceeded) { reader->ReadRecordBatch(0)); } +TEST(TestFileReaderLimits, PreBufferMetadataExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + auto metadata = key_value_metadata({"large_field"}, {std::string(2000, 'x')}); + FileWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch, metadata)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 500; + auto buf_reader = std::make_shared(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(buf_reader.get(), options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + reader->PreBufferMetadata({0})); +} + +TEST(TestFileReaderLimits, PreBufferBodyLengthExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + FileWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + auto buf_reader = std::make_shared(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(buf_reader.get(), options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->PreBufferMetadata({0})); +} + +TEST(TestFileReaderLimits, GeneratorCoalesceBlockLimitsExceeded) { + std::shared_ptr batch; + ASSERT_OK(MakeIntRecordBatch(&batch)); + FileWriterHelper writer_helper; + ASSERT_OK(writer_helper.Init(batch->schema(), IpcWriteOptions::Defaults())); + ASSERT_OK(writer_helper.WriteBatch(batch)); + ASSERT_OK(writer_helper.Finish()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + auto buf_reader = std::make_shared(writer_helper.buffer_); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(buf_reader, options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->GetRecordBatchGenerator(/*coalesce=*/true)); +} + TEST(TestIpcReadOptions, ValidateNegativeLimits) { auto options = IpcReadOptions::Defaults(); ASSERT_OK(options.Validate()); @@ -3621,7 +3676,7 @@ TEST(TestIpcReadOptions, ValidateNegativeLimits) { } TEST(TestStreamReaderLimits, NegativeLimitsInvalid) { - io::BufferReader empty_reader(""); + io::BufferReader empty_reader(Buffer::FromString("")); auto options = IpcReadOptions::Defaults(); options.max_metadata_length = -2; EXPECT_RAISES_WITH_MESSAGE_THAT( @@ -3638,7 +3693,7 @@ TEST(TestStreamReaderLimits, NegativeLimitsInvalid) { } TEST(TestFileReaderLimits, NegativeLimitsInvalid) { - io::BufferReader empty_reader(""); + io::BufferReader empty_reader(Buffer::FromString("")); auto options = IpcReadOptions::Defaults(); options.max_metadata_length = -2; EXPECT_RAISES_WITH_MESSAGE_THAT( diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 07799abc2e27..507fc55420a4 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -1655,6 +1655,16 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { std::shared_ptr cached_source; if (coalesce && !file_->supports_zero_copy()) { if (!owned_file_) return Status::Invalid("Cannot coalesce without an owned file"); + for (int i = 0; i < state->num_dictionaries(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto block, state->GetDictionaryBlock(i)); + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); + } + for (int i = 0; i < state->num_record_batches(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto block, state->GetRecordBatchBlock(i)); + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); + } // Since the user is asking for all fields then we can cache the entire // file (up to the footer) cached_source = std::make_shared(file_, io_context, @@ -1674,12 +1684,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { all_metadata_ready.Then([this, index]() -> Result> { stats_.num_messages.fetch_add(1, std::memory_order_relaxed); ARROW_ASSIGN_OR_RAISE(FileBlock block, GetRecordBatchBlock(index)); - if (options_.max_metadata_length >= 0 && - block.metadata_length > options_.max_metadata_length) { - return Status::Invalid("IPC metadata length ", block.metadata_length, - " exceeds maximum allowed metadata length ", - options_.max_metadata_length); - } + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); ARROW_ASSIGN_OR_RAISE( std::shared_ptr metadata, metadata_cache_->Read({block.offset, block.metadata_length})); @@ -1786,6 +1792,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { // Adds all dictionaries to the range cache for (int i = 0; i < num_dictionaries(); ++i) { ARROW_ASSIGN_OR_RAISE(FileBlock block, GetDictionaryBlock(i)); + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); ranges->push_back({block.offset, block.metadata_length + block.body_length}); } return Status::OK(); @@ -1795,6 +1803,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { std::vector* ranges) { for (int index : indices) { ARROW_ASSIGN_OR_RAISE(FileBlock block, GetRecordBatchBlock(index)); + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); ranges->push_back({block.offset, block.metadata_length}); } return Status::OK(); @@ -1951,7 +1961,11 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { .Then([this, index](const std::shared_ptr& message_obj) -> Future> { ARROW_ASSIGN_OR_RAISE(auto block, GetRecordBatchBlock(index)); - ARROW_ASSIGN_OR_RAISE(auto message, GetFlatbufMessage(message_obj)); + RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlockLimits(block, options_)); + ARROW_ASSIGN_OR_RAISE(auto checked_message, + CheckBodyLength(message_obj, block)); + ARROW_ASSIGN_OR_RAISE(auto message, GetFlatbufMessage(checked_message)); ARROW_ASSIGN_OR_RAISE(auto batch, GetBatchFromMessage(message)); ARROW_ASSIGN_OR_RAISE(auto context, GetIpcReadContext(message, batch)); @@ -2131,16 +2145,18 @@ WholeIpcFileRecordBatchGenerator::operator()() { Future> WholeIpcFileRecordBatchGenerator::ReadBlock( const FileBlock& block) { + RETURN_NOT_OK(CheckAligned(block)); RETURN_NOT_OK(CheckBlockLimits(block, state_->options_)); if (cached_source_) { auto cached_source = cached_source_; io::ReadRange range{block.offset, block.metadata_length + block.body_length}; auto options = state_->options_; return cached_source->WaitFor({range}).Then( - [cached_source, options, range]() -> Result> { + [cached_source, options, range, block]() -> Result> { ARROW_ASSIGN_OR_RAISE(auto buffer, cached_source->Read(range)); io::BufferReader stream(std::move(buffer)); - return ReadMessage(&stream, options); + ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(&stream, options)); + return CheckBodyLength(std::move(message), block); }); } else { return ReadMessageFromBlockAsync(block, state_->options_, state_->file_, io_context_);