Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions cpp/src/arrow/ipc/read_write_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include "arrow/ipc/reader_internal.h"
#include "arrow/ipc/test_common.h"
#include "arrow/ipc/writer.h"
#include "arrow/memory_pool.h"
#include "arrow/record_batch.h"
#include "arrow/status.h"
#include "arrow/table.h"
Expand Down Expand Up @@ -370,6 +371,27 @@ TEST_F(TestSchemaMetadata, MetadataVersionForwardCompatibility) {
ASSERT_RAISES(Invalid, ReadSchema(schema_v6_file.get(), &placeholder_memo));
}

TEST_F(TestSchemaMetadata, SerializeSchemaUsesGivenMemoryPool) {
// GH-55623: SerializeSchema() must allocate only from the given pool
auto metadata = key_value_metadata({{"foo", "bar"}});
Schema schema({field("f0", int64()), field("f1", utf8())}, metadata);

// A pool with its own statistics, distinct from default_memory_pool() even when
// both use the same allocator
std::unique_ptr<MemoryPool> pool = MemoryPool::CreateDefault();
MemoryPool* default_pool = default_memory_pool();
const int64_t default_allocated_before = default_pool->total_bytes_allocated();
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Buffer> buffer,
SerializeSchema(schema, pool.get()));
ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before);
ASSERT_GT(pool->total_bytes_allocated(), 0);

io::BufferReader reader(buffer);
DictionaryMemo in_memo;
ASSERT_OK_AND_ASSIGN(auto actual_schema, ReadSchema(&reader, &in_memo));
AssertSchemaEqual(schema, *actual_schema, /*check_metadata=*/true);
}

const std::vector<test::MakeRecordBatch*> kBatchCases = {
&MakeIntRecordBatch,
&MakeListRecordBatch,
Expand Down
65 changes: 65 additions & 0 deletions cpp/src/arrow/ipc/tensor_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "arrow/ipc/reader.h"
#include "arrow/ipc/test_common.h"
#include "arrow/ipc/writer.h"
#include "arrow/memory_pool.h"
#include "arrow/sparse_tensor.h"
#include "arrow/status.h"
#include "arrow/tensor.h"
Expand Down Expand Up @@ -657,6 +658,70 @@ TEST(TestSparseTensor, RejectNegativeShapeAndNonZeroLength) {
ARROW_UNSUPPRESS_DEPRECATION_WARNING
}

TEST(TestTensor, GetTensorMessageUsesGivenMemoryPool) {
// GH-55623: GetTensorMessage() must allocate only from the given pool
std::vector<int64_t> values;
randint(24, 0, 100, &values);
auto data = Buffer::Wrap(values);
// A non-contiguous tensor is also copied to a contiguous one from the pool
Tensor contiguous(int64(), data, {4, 6}, {48, 8}, {"foo", "bar"});
Tensor non_contiguous(int64(), data, {4, 3}, {48, 16});

for (const Tensor* tensor : {&contiguous, &non_contiguous}) {
// A pool with its own statistics, distinct from default_memory_pool() even when
// both use the same allocator
std::unique_ptr<MemoryPool> pool = MemoryPool::CreateDefault();
MemoryPool* default_pool = default_memory_pool();
const int64_t default_allocated_before = default_pool->total_bytes_allocated();
ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK_AND_ASSIGN(auto message, GetTensorMessage(*tensor, pool.get()));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before);
ASSERT_GT(pool->total_bytes_allocated(), 0);

ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK_AND_ASSIGN(auto result, ReadTensor(*message));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_TRUE(tensor->Equals(*result));
}
}

TEST(TestSparseTensor, GetSparseTensorPayloadUsesGivenMemoryPool) {
// GH-55623: GetSparseTensorPayload() and GetSparseTensorMessage() must allocate
// only from the given pool
std::vector<int64_t> values = {1, 0, 2, 0, 0, 3, 0, 4, 5, 0, 6, 0};
auto data = Buffer::Wrap(values);
Tensor dense(int64(), data, {3, 4});
ASSERT_OK_AND_ASSIGN(std::shared_ptr<SparseTensor> sparse,
SparseCOOTensor::Make(dense, int64()));

// A pool with its own statistics, distinct from default_memory_pool() even when
// both use the same allocator
std::unique_ptr<MemoryPool> pool = MemoryPool::CreateDefault();
MemoryPool* default_pool = default_memory_pool();
int64_t default_allocated_before = default_pool->total_bytes_allocated();
IpcPayload payload;
ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK(GetSparseTensorPayload(*sparse, pool.get(), &payload));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before);
ASSERT_GT(pool->total_bytes_allocated(), 0);

ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK_AND_ASSIGN(auto result, internal::ReadSparseTensorPayload(payload));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_TRUE(sparse->Equals(*result));

std::unique_ptr<MemoryPool> message_pool = MemoryPool::CreateDefault();
default_allocated_before = default_pool->total_bytes_allocated();
ARROW_SUPPRESS_DEPRECATION_WARNING
ASSERT_OK_AND_ASSIGN(auto message, GetSparseTensorMessage(*sparse, message_pool.get()));
ARROW_UNSUPPRESS_DEPRECATION_WARNING
ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before);
ASSERT_GT(message_pool->total_bytes_allocated(), 0);
ASSERT_TRUE(message->metadata()->Equals(*payload.metadata));
}

} // namespace test
} // namespace ipc
} // namespace arrow
12 changes: 8 additions & 4 deletions cpp/src/arrow/ipc/writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -980,6 +980,7 @@ Result<std::unique_ptr<Message>> GetTensorMessage(const Tensor& tensor,

IpcWriteOptions options;
options.alignment = kTensorAlignment;
options.memory_pool = pool;
std::shared_ptr<Buffer> metadata;
ARROW_ASSIGN_OR_RAISE(metadata,
internal::WriteTensorMessage(*tensor_to_write, 0, options));
Expand All @@ -990,10 +991,12 @@ namespace internal {

class SparseTensorSerializer {
public:
SparseTensorSerializer(int64_t buffer_start_offset, IpcPayload* out)
SparseTensorSerializer(int64_t buffer_start_offset, MemoryPool* pool, IpcPayload* out)
: out_(out),
buffer_start_offset_(buffer_start_offset),
options_(IpcWriteOptions::Defaults()) {}
options_(IpcWriteOptions::Defaults()) {
options_.memory_pool = pool;
}

~SparseTensorSerializer() = default;

Expand Down Expand Up @@ -1103,7 +1106,7 @@ class SparseTensorSerializer {
Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* dst,
int32_t* metadata_length, int64_t* body_length) {
IpcPayload payload;
internal::SparseTensorSerializer writer(0, &payload);
internal::SparseTensorSerializer writer(0, default_memory_pool(), &payload);
RETURN_NOT_OK(writer.Assemble(sparse_tensor));

*body_length = payload.body_length;
Expand All @@ -1112,7 +1115,7 @@ Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* ds

Status GetSparseTensorPayload(const SparseTensor& sparse_tensor, MemoryPool* pool,
IpcPayload* out) {
internal::SparseTensorSerializer writer(0, out);
internal::SparseTensorSerializer writer(0, pool, out);
return writer.Assemble(sparse_tensor);
}

Expand Down Expand Up @@ -1664,6 +1667,7 @@ Result<std::shared_ptr<Buffer>> SerializeSchema(const Schema& schema, MemoryPool
ARROW_ASSIGN_OR_RAISE(auto stream, io::BufferOutputStream::Create(1024, pool));

auto options = IpcWriteOptions::Defaults();
options.memory_pool = pool;
const bool is_file_format = false; // indifferent as we don't write dictionaries
internal::IpcFormatWriter writer(
std::make_unique<internal::PayloadStreamWriter>(stream.get()), schema, options,
Expand Down
Loading