diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 139e07be0545..0016ed81ffbd 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -40,6 +40,7 @@ #include "arrow/ipc/reader_internal.h" #include "arrow/ipc/test_common.h" #include "arrow/ipc/writer.h" +#include "arrow/memory_pool.h" #include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" @@ -370,6 +371,27 @@ TEST_F(TestSchemaMetadata, MetadataVersionForwardCompatibility) { ASSERT_RAISES(Invalid, ReadSchema(schema_v6_file.get(), &placeholder_memo)); } +TEST_F(TestSchemaMetadata, SerializeSchemaUsesGivenMemoryPool) { + // GH-55623: SerializeSchema() must allocate only from the given pool + auto metadata = key_value_metadata({{"foo", "bar"}}); + Schema schema({field("f0", int64()), field("f1", utf8())}, metadata); + + // A pool with its own statistics, distinct from default_memory_pool() even when + // both use the same allocator + std::unique_ptr pool = MemoryPool::CreateDefault(); + MemoryPool* default_pool = default_memory_pool(); + const int64_t default_allocated_before = default_pool->total_bytes_allocated(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr buffer, + SerializeSchema(schema, pool.get())); + ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before); + ASSERT_GT(pool->total_bytes_allocated(), 0); + + io::BufferReader reader(buffer); + DictionaryMemo in_memo; + ASSERT_OK_AND_ASSIGN(auto actual_schema, ReadSchema(&reader, &in_memo)); + AssertSchemaEqual(schema, *actual_schema, /*check_metadata=*/true); +} + const std::vector kBatchCases = { &MakeIntRecordBatch, &MakeListRecordBatch, diff --git a/cpp/src/arrow/ipc/tensor_test.cc b/cpp/src/arrow/ipc/tensor_test.cc index 78ab3cb0a135..d7833736e281 100644 --- a/cpp/src/arrow/ipc/tensor_test.cc +++ b/cpp/src/arrow/ipc/tensor_test.cc @@ -35,6 +35,7 @@ #include "arrow/ipc/reader.h" #include "arrow/ipc/test_common.h" #include "arrow/ipc/writer.h" +#include "arrow/memory_pool.h" #include "arrow/sparse_tensor.h" #include "arrow/status.h" #include "arrow/tensor.h" @@ -657,6 +658,70 @@ TEST(TestSparseTensor, RejectNegativeShapeAndNonZeroLength) { ARROW_UNSUPPRESS_DEPRECATION_WARNING } +TEST(TestTensor, GetTensorMessageUsesGivenMemoryPool) { + // GH-55623: GetTensorMessage() must allocate only from the given pool + std::vector values; + randint(24, 0, 100, &values); + auto data = Buffer::Wrap(values); + // A non-contiguous tensor is also copied to a contiguous one from the pool + Tensor contiguous(int64(), data, {4, 6}, {48, 8}, {"foo", "bar"}); + Tensor non_contiguous(int64(), data, {4, 3}, {48, 16}); + + for (const Tensor* tensor : {&contiguous, &non_contiguous}) { + // A pool with its own statistics, distinct from default_memory_pool() even when + // both use the same allocator + std::unique_ptr pool = MemoryPool::CreateDefault(); + MemoryPool* default_pool = default_memory_pool(); + const int64_t default_allocated_before = default_pool->total_bytes_allocated(); + ARROW_SUPPRESS_DEPRECATION_WARNING + ASSERT_OK_AND_ASSIGN(auto message, GetTensorMessage(*tensor, pool.get())); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before); + ASSERT_GT(pool->total_bytes_allocated(), 0); + + ARROW_SUPPRESS_DEPRECATION_WARNING + ASSERT_OK_AND_ASSIGN(auto result, ReadTensor(*message)); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + ASSERT_TRUE(tensor->Equals(*result)); + } +} + +TEST(TestSparseTensor, GetSparseTensorPayloadUsesGivenMemoryPool) { + // GH-55623: GetSparseTensorPayload() and GetSparseTensorMessage() must allocate + // only from the given pool + std::vector values = {1, 0, 2, 0, 0, 3, 0, 4, 5, 0, 6, 0}; + auto data = Buffer::Wrap(values); + Tensor dense(int64(), data, {3, 4}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr sparse, + SparseCOOTensor::Make(dense, int64())); + + // A pool with its own statistics, distinct from default_memory_pool() even when + // both use the same allocator + std::unique_ptr pool = MemoryPool::CreateDefault(); + MemoryPool* default_pool = default_memory_pool(); + int64_t default_allocated_before = default_pool->total_bytes_allocated(); + IpcPayload payload; + ARROW_SUPPRESS_DEPRECATION_WARNING + ASSERT_OK(GetSparseTensorPayload(*sparse, pool.get(), &payload)); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before); + ASSERT_GT(pool->total_bytes_allocated(), 0); + + ARROW_SUPPRESS_DEPRECATION_WARNING + ASSERT_OK_AND_ASSIGN(auto result, internal::ReadSparseTensorPayload(payload)); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + ASSERT_TRUE(sparse->Equals(*result)); + + std::unique_ptr message_pool = MemoryPool::CreateDefault(); + default_allocated_before = default_pool->total_bytes_allocated(); + ARROW_SUPPRESS_DEPRECATION_WARNING + ASSERT_OK_AND_ASSIGN(auto message, GetSparseTensorMessage(*sparse, message_pool.get())); + ARROW_UNSUPPRESS_DEPRECATION_WARNING + ASSERT_EQ(default_pool->total_bytes_allocated(), default_allocated_before); + ASSERT_GT(message_pool->total_bytes_allocated(), 0); + ASSERT_TRUE(message->metadata()->Equals(*payload.metadata)); +} + } // namespace test } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index eefd88e9f6b4..d64dc68a0cfd 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -980,6 +980,7 @@ Result> GetTensorMessage(const Tensor& tensor, IpcWriteOptions options; options.alignment = kTensorAlignment; + options.memory_pool = pool; std::shared_ptr metadata; ARROW_ASSIGN_OR_RAISE(metadata, internal::WriteTensorMessage(*tensor_to_write, 0, options)); @@ -990,10 +991,12 @@ namespace internal { class SparseTensorSerializer { public: - SparseTensorSerializer(int64_t buffer_start_offset, IpcPayload* out) + SparseTensorSerializer(int64_t buffer_start_offset, MemoryPool* pool, IpcPayload* out) : out_(out), buffer_start_offset_(buffer_start_offset), - options_(IpcWriteOptions::Defaults()) {} + options_(IpcWriteOptions::Defaults()) { + options_.memory_pool = pool; + } ~SparseTensorSerializer() = default; @@ -1103,7 +1106,7 @@ class SparseTensorSerializer { Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) { IpcPayload payload; - internal::SparseTensorSerializer writer(0, &payload); + internal::SparseTensorSerializer writer(0, default_memory_pool(), &payload); RETURN_NOT_OK(writer.Assemble(sparse_tensor)); *body_length = payload.body_length; @@ -1112,7 +1115,7 @@ Status WriteSparseTensor(const SparseTensor& sparse_tensor, io::OutputStream* ds Status GetSparseTensorPayload(const SparseTensor& sparse_tensor, MemoryPool* pool, IpcPayload* out) { - internal::SparseTensorSerializer writer(0, out); + internal::SparseTensorSerializer writer(0, pool, out); return writer.Assemble(sparse_tensor); } @@ -1664,6 +1667,7 @@ Result> SerializeSchema(const Schema& schema, MemoryPool ARROW_ASSIGN_OR_RAISE(auto stream, io::BufferOutputStream::Create(1024, pool)); auto options = IpcWriteOptions::Defaults(); + options.memory_pool = pool; const bool is_file_format = false; // indifferent as we don't write dictionaries internal::IpcFormatWriter writer( std::make_unique(stream.get()), schema, options,