Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
295 changes: 255 additions & 40 deletions cpp/src/arrow/ipc/message.cc

Large diffs are not rendered by default.

48 changes: 48 additions & 0 deletions cpp/src/arrow/ipc/message.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <utility>

#include "arrow/io/type_fwd.h"
#include "arrow/ipc/options.h"
#include "arrow/ipc/type_fwd.h"
#include "arrow/result.h"
#include "arrow/status.h"
Expand Down Expand Up @@ -289,6 +290,28 @@ class ARROW_EXPORT MessageDecoder {
int64_t initial_next_required_size,
MemoryPool* pool = default_memory_pool(), bool skip_body = false);

/// \brief Construct a message decoder with options.
///
/// \param[in] listener a MessageDecoderListener that responds events from
/// the decoder
/// \param[in] options options for reading IPC messages
/// \param[in] skip_body if true the body will be skipped even if the message has a body
explicit MessageDecoder(std::shared_ptr<MessageDecoderListener> listener,
IpcReadOptions options, bool skip_body = false);

/// \brief Construct a message decoder with the specified state and options.
///
/// \param[in] listener a MessageDecoderListener that responds events from
/// the decoder
/// \param[in] initial_state an initial state of the decoder
/// \param[in] initial_next_required_size the number of bytes needed
/// to run the next action
/// \param[in] options options for reading IPC messages
/// \param[in] skip_body if true the body will be skipped even if the message has a body
MessageDecoder(std::shared_ptr<MessageDecoderListener> listener, State initial_state,
int64_t initial_next_required_size, IpcReadOptions options,
bool skip_body = false);

virtual ~MessageDecoder();

/// \brief Feed data to the decoder as a raw data.
Expand Down Expand Up @@ -439,10 +462,15 @@ class ARROW_EXPORT MessageReader {

/// \brief Create MessageReader that reads from InputStream
static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
static std::unique_ptr<MessageReader> Open(io::InputStream* stream,
const IpcReadOptions& options);

/// \brief Create MessageReader that reads from owned InputStream
static std::unique_ptr<MessageReader> Open(
const std::shared_ptr<io::InputStream>& owned_stream);
static std::unique_ptr<MessageReader> Open(
const std::shared_ptr<io::InputStream>& owned_stream,
const IpcReadOptions& options);

/// \brief Read next Message from the interface
///
Expand Down Expand Up @@ -499,6 +527,12 @@ Result<std::unique_ptr<Message>> ReadMessage(
io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(),
const FieldsLoaderFunction& fields_loader = {});

ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const IpcReadOptions& options,
const FieldsLoaderFunction& fields_loader = {});

/// \brief Read encapsulated IPC message from cached buffers
///
/// The buffers should contain an entire message. Partial reads are not handled.
Expand All @@ -518,11 +552,21 @@ Result<std::unique_ptr<Message>> ReadMessage(
ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);

ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body,
const IpcReadOptions& options);

ARROW_EXPORT
Future<std::shared_ptr<Message>> ReadMessageAsync(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context());

ARROW_EXPORT
Future<std::shared_ptr<Message>> ReadMessageAsync(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const IpcReadOptions& options,
const io::IOContext& context = io::default_io_context());

/// \brief Advance stream to an 8-byte offset if its position is not a multiple
/// of 8 already
/// \param[in] stream an input stream
Expand Down Expand Up @@ -558,6 +602,10 @@ ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
MemoryPool* pool = default_memory_pool());

ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
const IpcReadOptions& options);

/// \brief Feed data from InputStream to MessageDecoder to decode an
/// encapsulated IPC message (metadata and body)
///
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/arrow/ipc/metadata_internal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1290,6 +1290,7 @@ Result<std::shared_ptr<KeyValueMetadata>> GetKeyValueMetadata(

metadata->reserve(fb_metadata->size());
for (const auto pair : *fb_metadata) {
CHECK_FLATBUFFERS_NOT_NULL(pair, "custom_metadata.pair");
CHECK_FLATBUFFERS_NOT_NULL(pair->key(), "custom_metadata.key");
CHECK_FLATBUFFERS_NOT_NULL(pair->value(), "custom_metadata.value");
metadata->Append(pair->key()->str(), pair->value()->str());
Expand Down Expand Up @@ -1474,11 +1475,16 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type
if (tensor == nullptr) {
return Status::IOError("Header-type of flatbuffer-encoded Message is not Tensor.");
}
CHECK_FLATBUFFERS_NOT_NULL(tensor->shape(), "Tensor.shape");

flatbuffers::uoffset_t ndim = tensor->shape()->size();

for (flatbuffers::uoffset_t i = 0; i < ndim; ++i) {
auto dim = tensor->shape()->Get(i);
CHECK_FLATBUFFERS_NOT_NULL(dim, "Tensor.shape[i]");
if (dim->size() < 0) {
return Status::Invalid("Invalid tensor dimension size: ", dim->size());
}

shape->push_back(dim->size());
dim_names->push_back(StringFromFlatbuffers(dim->name()));
Expand Down Expand Up @@ -1551,11 +1557,13 @@ Status GetSparseTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>
return Status::IOError(
"Header-type of flatbuffer-encoded Message is not SparseTensor.");
}
CHECK_FLATBUFFERS_NOT_NULL(sparse_tensor->shape(), "SparseTensor.shape");
int ndim = static_cast<int>(sparse_tensor->shape()->size());

if (shape || dim_names) {
for (int i = 0; i < ndim; ++i) {
auto dim = sparse_tensor->shape()->Get(i);
CHECK_FLATBUFFERS_NOT_NULL(dim, "SparseTensor.shape[i]");

if (shape) {
if (dim->size() < 0) {
Expand Down
20 changes: 20 additions & 0 deletions cpp/src/arrow/ipc/options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,26 @@ IpcWriteOptions IpcWriteOptions::Defaults() { return IpcWriteOptions(); }

IpcReadOptions IpcReadOptions::Defaults() { return IpcReadOptions(); }

Status IpcReadOptions::Validate() const {
if (max_recursion_depth < 0) {
return Status::Invalid("max_recursion_depth must be non-negative, got ",
max_recursion_depth);
}
if (max_metadata_length < -1) {
return Status::Invalid("max_metadata_length must be -1 or non-negative, got ",
max_metadata_length);
}
if (max_body_length < -1) {
return Status::Invalid("max_body_length must be -1 or non-negative, got ",
max_body_length);
}
if (max_decompression_ratio < -1) {
return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ",
max_decompression_ratio);
}
return Status::OK();
}

namespace internal {

Status CheckCompressionSupported(Compression::type codec) {
Expand Down
14 changes: 14 additions & 0 deletions cpp/src/arrow/ipc/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,20 @@ struct ARROW_EXPORT IpcReadOptions {
/// The lazy property will always be reset to true to deliver the expected behavior
io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults();

/// \brief Maximum allowed metadata length in bytes (-1 means unlimited).
///
/// Enforced on message metadata in IPC streams and file footer metadata in IPC files.
int64_t max_metadata_length = -1;

/// \brief Maximum allowed body length in bytes (-1 means unlimited).
int64_t max_body_length = -1;

/// \brief Maximum allowed decompression expansion ratio (-1 means unlimited).
int32_t max_decompression_ratio = -1;
Comment thread
Awesome06 marked this conversation as resolved.
Resolved
Comment thread
Awesome06 marked this conversation as resolved.
Resolved

/// \brief Validate read options.
Status Validate() const;

static IpcReadOptions Defaults();
};

Expand Down
Loading
Loading