diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h index 8cb982470715..25350fe94c72 100644 --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h @@ -89,6 +89,11 @@ struct ARROW_EXPORT IOContext { StopToken stop_token() const { return stop_token_; } + /// Return a copy of this IOContext with the given memory pool + IOContext with_pool(MemoryPool* pool) const { + return IOContext(pool, executor_, stop_token_, external_id_); + } + private: MemoryPool* pool_; ::arrow::internal::Executor* executor_; diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 662b6bc3731d..34ec6993aa4b 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -308,8 +308,9 @@ Status ReadFieldsSubset(int64_t offset, io::RandomAccessFile* file, RETURN_NOT_OK(fields_loader(batch, &io_recorded_random_access_file)); const auto& read_ranges = io_recorded_random_access_file.GetReadRanges(); for (const auto& range : read_ranges) { - auto read_result = file->ReadAt(offset + metadata->size() + range.offset, - range.length, body->mutable_data() + range.offset); + auto read_result = + file->ReadAt(offset + metadata->size() + range.offset, range.length, + /*allow_short_read=*/false, body->mutable_data() + range.offset); if (!read_result.ok()) { return Status::IOError("Failed to read message body, error ", read_result.status().ToString()); @@ -332,17 +333,26 @@ struct ReadMessageState { ReadMessageState& operator=(ReadMessageState&&) = delete; }; +Status CheckBodyLength(int64_t offset, int64_t body_length, io::RandomAccessFile* file) { + ARROW_ASSIGN_OR_RAISE(auto file_size, file->GetSize()); + if (body_length > file_size - offset) { + return Status::IOError( + "Invalid IPC file: advertised body length would read beyond end of file"); + } + return Status::OK(); +} + // A common continuation callback for ReadMessage and ReadMessageAsync overloads static Result> ReadMessageContinued( int64_t offset, int32_t metadata_length, std::optional body_length, - std::shared_ptr metadata, io::RandomAccessFile* file, + std::shared_ptr metadata, io::RandomAccessFile* file, MemoryPool* pool, const FieldsLoaderFunction& fields_loader, ReadMessageState* state) { MessageDecoder* decoder = state->decoder.get(); - if (body_length.has_value()) { - // If body length was known, the given buffer should contain exactly the metadata - // followed by the body. - DCHECK_EQ(metadata->size(), metadata_length + *body_length); - } + // The given buffer should contain exactly the metadata, optionally followed by the + // body. + DCHECK(metadata->size() == metadata_length || + metadata->size() == metadata_length + body_length.value_or(0)); + ARROW_RETURN_NOT_OK(decoder->Consume(SliceBuffer(metadata, 0, metadata_length))); if (decoder->buffered_size() > 0) { return Status::Invalid("Message metadata too long by ", decoder->buffered_size(), @@ -360,17 +370,7 @@ static Result> ReadMessageContinued( ", metadata length: ", metadata_length); case MessageDecoder::State::BODY: { std::shared_ptr body; - if (fields_loader) { - // Selective field loading: allocate a body buffer and read only the - // requested field ranges into it. - DCHECK_NE(file, nullptr); - ARROW_ASSIGN_OR_RAISE( - body, AllocateBuffer(decoder->next_required_size(), default_memory_pool())); - RETURN_NOT_OK(ReadFieldsSubset(offset, file, fields_loader, - SliceBuffer(metadata, 0, metadata_length), - decoder->next_required_size(), body)); - } else if (body_length.has_value()) { - // Body was already read as part of the combined IO; just slice it out. + if (body_length.has_value()) { if (*body_length != decoder->next_required_size()) { // The streaming decoder got out of sync with the actual advertised // metadata and body size, which signals an invalid IPC file. @@ -378,14 +378,31 @@ static Result> ReadMessageContinued( *body_length, ", but message decoder expects to read ", decoder->next_required_size(), " bytes instead"); } - body = SliceBuffer(metadata, metadata_length, - std::min(*body_length, metadata->size() - metadata_length)); } else { - // Body length was unknown; do a separate IO to read the body. + // Read body length from message decoder + body_length = decoder->next_required_size(); + } + // Check body length against file size if possible. This reduces the blast + // radius of excessive memory allocation if IPC metadata is corrupted. + if (file != nullptr) { + RETURN_NOT_OK(CheckBodyLength(offset + metadata_length, *body_length, file)); + } + if (fields_loader) { + // Selective field loading: allocate a body buffer and read only the + // requested field ranges into it. DCHECK_NE(file, nullptr); - ARROW_ASSIGN_OR_RAISE( - body, file->ReadAt(offset + metadata_length, decoder->next_required_size(), - /*allow_short_read=*/false)); + ARROW_ASSIGN_OR_RAISE(body, AllocateBuffer(*body_length, pool)); + RETURN_NOT_OK(ReadFieldsSubset(offset, file, fields_loader, + SliceBuffer(metadata, 0, metadata_length), + *body_length, body)); + } else if (metadata->size() - metadata_length >= body_length) { + // Body was already read as part of the combined IO; just slice it out. + body = SliceBuffer(metadata, metadata_length, *body_length); + } else { + // Body was not read yet; do a separate IO to read the body. + DCHECK_NE(file, nullptr); + ARROW_ASSIGN_OR_RAISE(body, file->ReadAt(offset + metadata_length, *body_length, + /*allow_short_read=*/false)); } RETURN_NOT_OK(decoder->Consume(body)); @@ -454,20 +471,25 @@ Result> ReadMessage(std::shared_ptr metadata, // separately. static Result> ReadMessageInternal( int64_t offset, int32_t metadata_length, std::optional body_length, - io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { + io::RandomAccessFile* file, MemoryPool* pool, + const FieldsLoaderFunction& fields_loader) { ReadMessageState state; if (metadata_length < state.decoder->next_required_size()) { return Status::Invalid("metadata_length should be at least ", state.decoder->next_required_size()); } - // When body_length is known, read metadata + body in one IO call. + RETURN_NOT_OK(CheckBodyLength(offset + metadata_length, body_length.value_or(0), file)); + // When body_length is known and all fields will be loaded, + // read metadata + body in one IO call. // Otherwise, read only metadata first. + const auto read_size = + metadata_length + ((body_length.has_value() && !fields_loader) ? *body_length : 0); ARROW_ASSIGN_OR_RAISE(std::shared_ptr metadata, - file->ReadAt(offset, metadata_length + body_length.value_or(0), + file->ReadAt(offset, read_size, /*allow_short_read=*/false)); - return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, + return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, pool, fields_loader, &state); } @@ -475,15 +497,16 @@ Result> ReadMessage(int64_t offset, int32_t metadata_le io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { return ReadMessageInternal(offset, metadata_length, /*body_length=*/std::nullopt, file, - fields_loader); + file->io_context().pool(), fields_loader); } Result> ReadMessage(const int64_t offset, const int32_t metadata_length, const int64_t body_length, - io::RandomAccessFile* file) { - return ReadMessageInternal(offset, metadata_length, body_length, file, - /*fields_loader=*/{}); + io::RandomAccessFile* file, MemoryPool* pool, + const FieldsLoaderFunction& fields_loader) { + return ReadMessageInternal(offset, metadata_length, body_length, file, pool, + fields_loader); } Future> ReadMessageAsync(int64_t offset, int32_t metadata_length, @@ -505,7 +528,7 @@ Future> ReadMessageAsync(int64_t offset, int32_t metada // Pass a nullptr file to ensure that no further IO occurs // (we have fetched all the required bytes). return ReadMessageContinued(offset, metadata_length, body_length, metadata, - /*file=*/nullptr, + /*file=*/nullptr, context.pool(), /*fields_loader=*/{}, state.get()); }); } diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index c6701f200f45..90346bd1972f 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -471,7 +471,7 @@ using FieldsLoaderFunction = std::function> ReadMessage( const int64_t offset, const int32_t metadata_length, io::RandomAccessFile* file, @@ -490,13 +490,14 @@ Result> ReadMessage( /// \param[in] metadata_length the total number of bytes to read from file /// \param[in] body_length the number of bytes for the message body /// \param[in] file the seekable file interface to read from +/// \param[in] pool the memory pool to allocate memory from +/// \param[in] fields_loader the function for loading subset of fields from the given file /// \return the message read - ARROW_EXPORT -Result> ReadMessage(const int64_t offset, - const int32_t metadata_length, - const int64_t body_length, - io::RandomAccessFile* file); +Result> ReadMessage( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(), + const FieldsLoaderFunction& fields_loader = {}); /// \brief Read encapsulated IPC message from cached buffers /// diff --git a/cpp/src/arrow/ipc/message_internal_test.cc b/cpp/src/arrow/ipc/message_internal_test.cc index 90b091c6b671..d87cc1807d47 100644 --- a/cpp/src/arrow/ipc/message_internal_test.cc +++ b/cpp/src/arrow/ipc/message_internal_test.cc @@ -391,7 +391,7 @@ TEST_F(MessageDecodingTest, ReadMessage3) { io::BufferReader reader(stream_buf); return ReadMessage(/*offset=*/padding_size, static_cast(sample_message.metadata_bytes->size()), - &reader, /*fields_loader=*/{}); + sample_message.body_bytes->size(), &reader); }; padding_size = 0; for (const auto& params : message_params()) { @@ -427,7 +427,8 @@ TEST_F(MessageDecodingTest, ReadMessage4) { io::BufferReader reader(stream_buf); return ReadMessage(/*offset=*/padding_size, static_cast(sample_message.metadata_bytes->size()), - &reader, fields_loader); + sample_message.body_bytes->size(), &reader, default_memory_pool(), + fields_loader); }; padding_size = 0; for (const auto& params : message_params()) { diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 139e07be0545..811f620f40a5 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -552,8 +552,10 @@ class TestIpcRoundTrip : public ::testing::TestWithParam, ASSERT_OK(WriteRecordBatch(*batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, options_)); + ARROW_SUPPRESS_DEPRECATION_WARNING ASSERT_OK_AND_ASSIGN(std::unique_ptr message1, ReadMessage(0, metadata_length, mmap_.get())); + ARROW_UNSUPPRESS_DEPRECATION_WARNING ASSERT_EQ(expected_version, message1->metadata_version()); ASSERT_OK_AND_ASSIGN(auto message2, diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 96e281c32385..8a2c47298854 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -793,8 +793,9 @@ Status GetCompressionExperimental(const flatbuf::Message* message, return Status::OK(); } -Status ReadContiguousPayload(io::InputStream* file, std::unique_ptr* message) { - ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file)); +Status ReadContiguousPayload(io::InputStream* file, const IpcReadOptions& options, + std::unique_ptr* message) { + ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options.memory_pool)); if (*message == nullptr) { return Status::Invalid("Unable to read metadata at offset"); } @@ -995,7 +996,7 @@ Result> ReadRecordBatch( const std::shared_ptr& schema, const DictionaryMemo* dictionary_memo, const IpcReadOptions& options, io::InputStream* file) { std::unique_ptr message; - RETURN_NOT_OK(ReadContiguousPayload(file, &message)); + RETURN_NOT_OK(ReadContiguousPayload(file, options, &message)); CHECK_HAS_BODY(*message); ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body())); return ReadRecordBatch(*message->metadata(), schema, dictionary_memo, options, @@ -1293,25 +1294,21 @@ Result CheckBodyLength(MessagePtr message, const FileBlock& block) { } Result> ReadMessageFromBlock( - const FileBlock& block, io::RandomAccessFile* file, + const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { RETURN_NOT_OK(CheckAligned(block)); - if (fields_loader) { - ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(block.offset, block.metadata_length, - file, fields_loader)); - return CheckBodyLength(std::move(message), block); - } else { - ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(block.offset, block.metadata_length, - block.body_length, file)); - return CheckBodyLength(std::move(message), block); - } + ARROW_ASSIGN_OR_RAISE( + auto message, ReadMessage(block.offset, block.metadata_length, block.body_length, + file, options.memory_pool, fields_loader)); + return CheckBodyLength(std::move(message), block); } Future> ReadMessageFromBlockAsync( - const FileBlock& block, io::RandomAccessFile* file, const io::IOContext& io_context) { + const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, + const io::IOContext& io_context) { RETURN_NOT_OK(CheckAligned(block)); return ReadMessageAsync(block.offset, block.metadata_length, block.body_length, file, - io_context) + io_context.with_pool(options.memory_pool)) .Then([block](std::shared_ptr message) { return CheckBodyLength(std::move(message), block); }); @@ -1666,8 +1663,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { Result> ReadMessageFromBlock( const FileBlock& block, const FieldsLoaderFunction& fields_loader = {}) { - ARROW_ASSIGN_OR_RAISE(auto message, - arrow::ipc::ReadMessageFromBlock(block, file_, fields_loader)); + ARROW_ASSIGN_OR_RAISE(auto message, arrow::ipc::ReadMessageFromBlock( + block, options_, file_, fields_loader)); stats_.num_messages.fetch_add(1, std::memory_order_relaxed); return message; } @@ -2060,7 +2057,7 @@ Future> WholeIpcFileRecordBatchGenerator::ReadBlock( return ReadMessage(&stream, pool); }); } else { - return ReadMessageFromBlockAsync(block, state_->file_, io_context_); + return ReadMessageFromBlockAsync(block, state_->options_, state_->file_, io_context_); } } @@ -2292,7 +2289,7 @@ Result> ReadSchema(const Message& message, Result> ReadTensor(io::InputStream* file) { std::unique_ptr message; - RETURN_NOT_OK(ReadContiguousPayload(file, &message)); + RETURN_NOT_OK(ReadContiguousPayload(file, IpcReadOptions::Defaults(), &message)); ARROW_SUPPRESS_DEPRECATION_WARNING return ReadTensor(*message); ARROW_UNSUPPRESS_DEPRECATION_WARNING @@ -2746,7 +2743,7 @@ Result> ReadSparseTensor(const Message& message) { Result> ReadSparseTensor(io::InputStream* file) { std::unique_ptr message; - RETURN_NOT_OK(ReadContiguousPayload(file, &message)); + RETURN_NOT_OK(ReadContiguousPayload(file, IpcReadOptions::Defaults(), &message)); CHECK_MESSAGE_TYPE(MessageType::SPARSE_TENSOR, message->type()); CHECK_HAS_BODY(*message); ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));