Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cpp/src/arrow/io/interfaces.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ struct ARROW_EXPORT IOContext {

StopToken stop_token() const { return stop_token_; }

/// Return a copy of this IOContext with the given memory pool
IOContext with_pool(MemoryPool* pool) const {
return IOContext(pool, executor_, stop_token_, external_id_);
}

private:
MemoryPool* pool_;
::arrow::internal::Executor* executor_;
Expand Down
91 changes: 57 additions & 34 deletions cpp/src/arrow/ipc/message.cc
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,9 @@ Status ReadFieldsSubset(int64_t offset, io::RandomAccessFile* file,
RETURN_NOT_OK(fields_loader(batch, &io_recorded_random_access_file));
const auto& read_ranges = io_recorded_random_access_file.GetReadRanges();
for (const auto& range : read_ranges) {
auto read_result = file->ReadAt(offset + metadata->size() + range.offset,
range.length, body->mutable_data() + range.offset);
auto read_result =
file->ReadAt(offset + metadata->size() + range.offset, range.length,
/*allow_short_read=*/false, body->mutable_data() + range.offset);
if (!read_result.ok()) {
return Status::IOError("Failed to read message body, error ",
read_result.status().ToString());
Expand All @@ -332,17 +333,26 @@ struct ReadMessageState {
ReadMessageState& operator=(ReadMessageState&&) = delete;
};

Status CheckBodyLength(int64_t offset, int64_t body_length, io::RandomAccessFile* file) {
ARROW_ASSIGN_OR_RAISE(auto file_size, file->GetSize());
if (body_length > file_size - offset) {
return Status::IOError(
"Invalid IPC file: advertised body length would read beyond end of file");
}
return Status::OK();
}

// A common continuation callback for ReadMessage and ReadMessageAsync overloads
static Result<std::unique_ptr<Message>> ReadMessageContinued(
int64_t offset, int32_t metadata_length, std::optional<int64_t> body_length,
std::shared_ptr<Buffer> metadata, io::RandomAccessFile* file,
std::shared_ptr<Buffer> metadata, io::RandomAccessFile* file, MemoryPool* pool,
const FieldsLoaderFunction& fields_loader, ReadMessageState* state) {
MessageDecoder* decoder = state->decoder.get();
if (body_length.has_value()) {
// If body length was known, the given buffer should contain exactly the metadata
// followed by the body.
DCHECK_EQ(metadata->size(), metadata_length + *body_length);
}
// The given buffer should contain exactly the metadata, optionally followed by the
// body.
DCHECK(metadata->size() == metadata_length ||
metadata->size() == metadata_length + body_length.value_or(0));

ARROW_RETURN_NOT_OK(decoder->Consume(SliceBuffer(metadata, 0, metadata_length)));
if (decoder->buffered_size() > 0) {
return Status::Invalid("Message metadata too long by ", decoder->buffered_size(),
Expand All @@ -360,32 +370,39 @@ static Result<std::unique_ptr<Message>> ReadMessageContinued(
", metadata length: ", metadata_length);
case MessageDecoder::State::BODY: {
std::shared_ptr<Buffer> body;
if (fields_loader) {
// Selective field loading: allocate a body buffer and read only the
// requested field ranges into it.
DCHECK_NE(file, nullptr);
ARROW_ASSIGN_OR_RAISE(
body, AllocateBuffer(decoder->next_required_size(), default_memory_pool()));
RETURN_NOT_OK(ReadFieldsSubset(offset, file, fields_loader,
SliceBuffer(metadata, 0, metadata_length),
decoder->next_required_size(), body));
} else if (body_length.has_value()) {
// Body was already read as part of the combined IO; just slice it out.
if (body_length.has_value()) {
if (*body_length != decoder->next_required_size()) {
// The streaming decoder got out of sync with the actual advertised
// metadata and body size, which signals an invalid IPC file.
return Status::IOError("Invalid IPC file: advertised body size is ",
*body_length, ", but message decoder expects to read ",
decoder->next_required_size(), " bytes instead");
}
body = SliceBuffer(metadata, metadata_length,
std::min(*body_length, metadata->size() - metadata_length));
} else {
// Body length was unknown; do a separate IO to read the body.
// Read body length from message decoder
body_length = decoder->next_required_size();
}
// Check body length against file size if possible. This reduces the blast
// radius of excessive memory allocation if IPC metadata is corrupted.
if (file != nullptr) {
RETURN_NOT_OK(CheckBodyLength(offset + metadata_length, *body_length, file));
}
if (fields_loader) {
// Selective field loading: allocate a body buffer and read only the
// requested field ranges into it.
DCHECK_NE(file, nullptr);
ARROW_ASSIGN_OR_RAISE(
body, file->ReadAt(offset + metadata_length, decoder->next_required_size(),
/*allow_short_read=*/false));
ARROW_ASSIGN_OR_RAISE(body, AllocateBuffer(*body_length, pool));
RETURN_NOT_OK(ReadFieldsSubset(offset, file, fields_loader,
SliceBuffer(metadata, 0, metadata_length),
*body_length, body));
} else if (metadata->size() - metadata_length >= body_length) {
// Body was already read as part of the combined IO; just slice it out.
body = SliceBuffer(metadata, metadata_length, *body_length);
} else {
// Body was not read yet; do a separate IO to read the body.
DCHECK_NE(file, nullptr);
ARROW_ASSIGN_OR_RAISE(body, file->ReadAt(offset + metadata_length, *body_length,
/*allow_short_read=*/false));
}

RETURN_NOT_OK(decoder->Consume(body));
Expand Down Expand Up @@ -454,36 +471,42 @@ Result<std::unique_ptr<Message>> ReadMessage(std::shared_ptr<Buffer> metadata,
// separately.
static Result<std::unique_ptr<Message>> ReadMessageInternal(
int64_t offset, int32_t metadata_length, std::optional<int64_t> body_length,
io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) {
io::RandomAccessFile* file, MemoryPool* pool,
const FieldsLoaderFunction& fields_loader) {
ReadMessageState state;

if (metadata_length < state.decoder->next_required_size()) {
return Status::Invalid("metadata_length should be at least ",
state.decoder->next_required_size());
}
// When body_length is known, read metadata + body in one IO call.
RETURN_NOT_OK(CheckBodyLength(offset + metadata_length, body_length.value_or(0), file));
// When body_length is known and all fields will be loaded,
// read metadata + body in one IO call.
// Otherwise, read only metadata first.
const auto read_size =
metadata_length + ((body_length.has_value() && !fields_loader) ? *body_length : 0);
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Buffer> metadata,
file->ReadAt(offset, metadata_length + body_length.value_or(0),
file->ReadAt(offset, read_size,
/*allow_short_read=*/false));

return ReadMessageContinued(offset, metadata_length, body_length, metadata, file,
return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, pool,
fields_loader, &state);
}

Result<std::unique_ptr<Message>> ReadMessage(int64_t offset, int32_t metadata_length,
io::RandomAccessFile* file,
const FieldsLoaderFunction& fields_loader) {
return ReadMessageInternal(offset, metadata_length, /*body_length=*/std::nullopt, file,
fields_loader);
file->io_context().pool(), fields_loader);
}

Result<std::unique_ptr<Message>> ReadMessage(const int64_t offset,
const int32_t metadata_length,
const int64_t body_length,
io::RandomAccessFile* file) {
return ReadMessageInternal(offset, metadata_length, body_length, file,
/*fields_loader=*/{});
io::RandomAccessFile* file, MemoryPool* pool,
const FieldsLoaderFunction& fields_loader) {
return ReadMessageInternal(offset, metadata_length, body_length, file, pool,
fields_loader);
}

Future<std::shared_ptr<Message>> ReadMessageAsync(int64_t offset, int32_t metadata_length,
Expand All @@ -505,7 +528,7 @@ Future<std::shared_ptr<Message>> ReadMessageAsync(int64_t offset, int32_t metada
// Pass a nullptr file to ensure that no further IO occurs
// (we have fetched all the required bytes).
return ReadMessageContinued(offset, metadata_length, body_length, metadata,
/*file=*/nullptr,
/*file=*/nullptr, context.pool(),
/*fields_loader=*/{}, state.get());
});
}
Expand Down
13 changes: 7 additions & 6 deletions cpp/src/arrow/ipc/message.h
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ using FieldsLoaderFunction = std::function<Status(const void*, io::RandomAccessF
/// \param[in] file the seekable file interface to read from
/// \param[in] fields_loader the function for loading subset of fields from the given file
/// \return the message read

ARROW_DEPRECATED("Deprecated in 27.0.0. Pass body_length explicitly.")
ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(
const int64_t offset, const int32_t metadata_length, io::RandomAccessFile* file,
Expand All @@ -490,13 +490,14 @@ Result<std::unique_ptr<Message>> ReadMessage(
/// \param[in] metadata_length the total number of bytes to read from file
/// \param[in] body_length the number of bytes for the message body
/// \param[in] file the seekable file interface to read from
/// \param[in] pool the memory pool to allocate memory from
/// \param[in] fields_loader the function for loading subset of fields from the given file
/// \return the message read

ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(const int64_t offset,
const int32_t metadata_length,
const int64_t body_length,
io::RandomAccessFile* file);
Result<std::unique_ptr<Message>> ReadMessage(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(),
const FieldsLoaderFunction& fields_loader = {});

/// \brief Read encapsulated IPC message from cached buffers
///
Expand Down
5 changes: 3 additions & 2 deletions cpp/src/arrow/ipc/message_internal_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ TEST_F(MessageDecodingTest, ReadMessage3) {
io::BufferReader reader(stream_buf);
return ReadMessage(/*offset=*/padding_size,
static_cast<int32_t>(sample_message.metadata_bytes->size()),
&reader, /*fields_loader=*/{});
sample_message.body_bytes->size(), &reader);
};
padding_size = 0;
for (const auto& params : message_params()) {
Expand Down Expand Up @@ -427,7 +427,8 @@ TEST_F(MessageDecodingTest, ReadMessage4) {
io::BufferReader reader(stream_buf);
return ReadMessage(/*offset=*/padding_size,
static_cast<int32_t>(sample_message.metadata_bytes->size()),
&reader, fields_loader);
sample_message.body_bytes->size(), &reader, default_memory_pool(),
fields_loader);
};
padding_size = 0;
for (const auto& params : message_params()) {
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/ipc/read_write_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -552,8 +552,10 @@ class TestIpcRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*>,
ASSERT_OK(WriteRecordBatch(*batch, buffer_offset, mmap_.get(), &metadata_length,
&body_length, options_));

ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK_AND_ASSIGN(std::unique_ptr<Message> message1,
ReadMessage(0, metadata_length, mmap_.get()));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_EQ(expected_version, message1->metadata_version());

ASSERT_OK_AND_ASSIGN(auto message2,
Expand Down
37 changes: 17 additions & 20 deletions cpp/src/arrow/ipc/reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -793,8 +793,9 @@ Status GetCompressionExperimental(const flatbuf::Message* message,
return Status::OK();
}

Status ReadContiguousPayload(io::InputStream* file, std::unique_ptr<Message>* message) {
ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file));
Status ReadContiguousPayload(io::InputStream* file, const IpcReadOptions& options,
std::unique_ptr<Message>* message) {
ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options.memory_pool));
if (*message == nullptr) {
return Status::Invalid("Unable to read metadata at offset");
}
Expand Down Expand Up @@ -995,7 +996,7 @@ Result<std::shared_ptr<RecordBatch>> ReadRecordBatch(
const std::shared_ptr<Schema>& schema, const DictionaryMemo* dictionary_memo,
const IpcReadOptions& options, io::InputStream* file) {
std::unique_ptr<Message> message;
RETURN_NOT_OK(ReadContiguousPayload(file, &message));
RETURN_NOT_OK(ReadContiguousPayload(file, options, &message));
CHECK_HAS_BODY(*message);
ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));
return ReadRecordBatch(*message->metadata(), schema, dictionary_memo, options,
Expand Down Expand Up @@ -1293,25 +1294,21 @@ Result<MessagePtr> CheckBodyLength(MessagePtr message, const FileBlock& block) {
}

Result<std::unique_ptr<Message>> ReadMessageFromBlock(
const FileBlock& block, io::RandomAccessFile* file,
const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file,
const FieldsLoaderFunction& fields_loader) {
RETURN_NOT_OK(CheckAligned(block));
if (fields_loader) {
ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(block.offset, block.metadata_length,
file, fields_loader));
return CheckBodyLength(std::move(message), block);
} else {
ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(block.offset, block.metadata_length,
block.body_length, file));
return CheckBodyLength(std::move(message), block);
}
ARROW_ASSIGN_OR_RAISE(
auto message, ReadMessage(block.offset, block.metadata_length, block.body_length,
file, options.memory_pool, fields_loader));
return CheckBodyLength(std::move(message), block);
}

Future<std::shared_ptr<Message>> ReadMessageFromBlockAsync(
const FileBlock& block, io::RandomAccessFile* file, const io::IOContext& io_context) {
const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file,
const io::IOContext& io_context) {
RETURN_NOT_OK(CheckAligned(block));
return ReadMessageAsync(block.offset, block.metadata_length, block.body_length, file,
io_context)
io_context.with_pool(options.memory_pool))
.Then([block](std::shared_ptr<Message> message) {
return CheckBodyLength(std::move(message), block);
});
Expand Down Expand Up @@ -1666,8 +1663,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {

Result<std::unique_ptr<Message>> ReadMessageFromBlock(
const FileBlock& block, const FieldsLoaderFunction& fields_loader = {}) {
ARROW_ASSIGN_OR_RAISE(auto message,
arrow::ipc::ReadMessageFromBlock(block, file_, fields_loader));
ARROW_ASSIGN_OR_RAISE(auto message, arrow::ipc::ReadMessageFromBlock(
block, options_, file_, fields_loader));
stats_.num_messages.fetch_add(1, std::memory_order_relaxed);
return message;
}
Expand Down Expand Up @@ -2060,7 +2057,7 @@ Future<std::shared_ptr<Message>> WholeIpcFileRecordBatchGenerator::ReadBlock(
return ReadMessage(&stream, pool);
});
} else {
return ReadMessageFromBlockAsync(block, state_->file_, io_context_);
return ReadMessageFromBlockAsync(block, state_->options_, state_->file_, io_context_);
}
}

Expand Down Expand Up @@ -2292,7 +2289,7 @@ Result<std::shared_ptr<Schema>> ReadSchema(const Message& message,

Result<std::shared_ptr<Tensor>> ReadTensor(io::InputStream* file) {
std::unique_ptr<Message> message;
RETURN_NOT_OK(ReadContiguousPayload(file, &message));
RETURN_NOT_OK(ReadContiguousPayload(file, IpcReadOptions::Defaults(), &message));
ARROW_SUPPRESS_DEPRECATION_WARNING
return ReadTensor(*message);
ARROW_UNSUPPRESS_DEPRECATION_WARNING
Expand Down Expand Up @@ -2746,7 +2743,7 @@ Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(const Message& message) {

Result<std::shared_ptr<SparseTensor>> ReadSparseTensor(io::InputStream* file) {
std::unique_ptr<Message> message;
RETURN_NOT_OK(ReadContiguousPayload(file, &message));
RETURN_NOT_OK(ReadContiguousPayload(file, IpcReadOptions::Defaults(), &message));
CHECK_MESSAGE_TYPE(MessageType::SPARSE_TENSOR, message->type());
CHECK_HAS_BODY(*message);
ARROW_ASSIGN_OR_RAISE(auto reader, Buffer::GetReader(message->body()));
Expand Down
Loading