diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index e02390745..5208431f6 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2467,6 +2467,31 @@ static int ArrowIpcDecoderMakeBuffer(struct ArrowIpcArraySetter* setter, int64_t NANOARROW_RETURN_NOT_OK( setter->factory.make_buffer(&setter->factory, &setter->src, out_view, out, error)); + // Body alignment does not imply buffer alignment: IPC offsets may be unaligned. + // Align each typed buffer before validation or endian swapping can read it. + // Decimal and interval values are represented using scalar words of at most 64 bits. + int64_t alignment = setter->src.element_size_bits / 8; + if (alignment > 8) { + alignment = 8; + } + if (setter->src.data_type == NANOARROW_TYPE_BINARY || + setter->src.data_type == NANOARROW_TYPE_STRING) { + alignment = 1; + } + if (alignment > 1 && out_view->size_bytes > 0 && + ((uintptr_t)out_view->data.data % alignment) != 0) { + struct ArrowBuffer aligned; + ArrowBufferInit(&aligned); + int result = ArrowBufferAppendBufferView(&aligned, *out_view); + if (result != NANOARROW_OK) { + ArrowBufferReset(&aligned); + return result; + } + ArrowBufferReset(out); + ArrowBufferMove(&aligned, out); + out_view->data.data = out->data; + } + if (setter->src.swap_endian) { NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSwapEndian(&setter->src, out_view, out, error)); diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 6dc1a5285..143d88c16 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -185,8 +186,177 @@ ArrowErrorCode ArrowIpcInputStreamInitFile(struct ArrowIpcInputStream* stream, return NANOARROW_OK; } +void ArrowIpcRandomAccessInputMove(struct ArrowIpcRandomAccessInput* src, + struct ArrowIpcRandomAccessInput* dst) { + memcpy(dst, src, sizeof(*dst)); + src->release = NULL; +} + +struct ArrowIpcRandomAccessBufferPrivate { + struct ArrowBuffer buffer; +}; + +static ArrowErrorCode ArrowIpcRandomAccessBufferPeek( + struct ArrowIpcRandomAccessInput* input, int64_t offset, int64_t size_bytes, + struct ArrowBufferView* out, struct ArrowError* error) { + if (offset < 0 || size_bytes < 0) { + ArrowErrorSet(error, "Offset and size must be non-negative"); + return EINVAL; + } + struct ArrowIpcRandomAccessBufferPrivate* private_data = + (struct ArrowIpcRandomAccessBufferPrivate*)input->private_data; + out->data.data = NULL; + out->size_bytes = 0; + if (offset >= private_data->buffer.size_bytes || size_bytes == 0) { + return NANOARROW_OK; + } + out->data.as_uint8 = private_data->buffer.data + offset; + out->size_bytes = private_data->buffer.size_bytes - offset; + if (out->size_bytes > size_bytes) { + out->size_bytes = size_bytes; + } + return NANOARROW_OK; +} + +static ArrowErrorCode ArrowIpcRandomAccessBufferRead( + struct ArrowIpcRandomAccessInput* input, int64_t offset, int64_t size_bytes, + struct ArrowBuffer* out, struct ArrowError* error) { + struct ArrowBufferView view; + NANOARROW_RETURN_NOT_OK( + ArrowIpcRandomAccessBufferPeek(input, offset, size_bytes, &view, error)); + struct ArrowIpcRandomAccessBufferPrivate* private_data = + (struct ArrowIpcRandomAccessBufferPrivate*)input->private_data; + ArrowBufferReset(out); + if (view.size_bytes > 0) { + NANOARROW_RETURN_NOT_OK(ArrowSharedBufferClone(&private_data->buffer, out)); + out->data += offset; + out->size_bytes = view.size_bytes; + out->capacity_bytes = view.size_bytes; + } + return NANOARROW_OK; +} + +static void ArrowIpcRandomAccessBufferRelease(struct ArrowIpcRandomAccessInput* input) { + struct ArrowIpcRandomAccessBufferPrivate* private_data = + (struct ArrowIpcRandomAccessBufferPrivate*)input->private_data; + ArrowBufferReset(&private_data->buffer); + ArrowFree(private_data); + input->release = NULL; +} + +ArrowErrorCode ArrowIpcRandomAccessInputInitBuffer(struct ArrowIpcRandomAccessInput* out, + struct ArrowBuffer* input) { + if (input->size_bytes < 0 || !_ArrowInt64FitsInSizeT(input->size_bytes) || + (input->size_bytes > 0 && input->data == NULL)) { + return EINVAL; + } + struct ArrowIpcRandomAccessBufferPrivate* private_data = + (struct ArrowIpcRandomAccessBufferPrivate*)ArrowMalloc(sizeof(*private_data)); + if (private_data == NULL) { + return ENOMEM; + } + if (ArrowIsSharedBuffer(input)) { + ArrowBufferMove(input, &private_data->buffer); + } else { + int result = ArrowSharedBufferInit(&private_data->buffer, input); + if (result != NANOARROW_OK) { + ArrowFree(private_data); + return result; + } + } + out->peek_at = &ArrowIpcRandomAccessBufferPeek; + out->read_at = &ArrowIpcRandomAccessBufferRead; + out->release = &ArrowIpcRandomAccessBufferRelease; + out->private_data = private_data; + return NANOARROW_OK; +} + +struct ArrowIpcRandomAccessFilePrivate { + FILE* file_ptr; + int close_on_release; +}; + +static ArrowErrorCode ArrowIpcRandomAccessFilePeek( + struct ArrowIpcRandomAccessInput* input, int64_t offset, int64_t size_bytes, + struct ArrowBufferView* out, struct ArrowError* error) { + NANOARROW_UNUSED(input); + NANOARROW_UNUSED(offset); + NANOARROW_UNUSED(size_bytes); + NANOARROW_UNUSED(out); + NANOARROW_UNUSED(error); + return ENOTSUP; +} + +static ArrowErrorCode ArrowIpcRandomAccessFileRead( + struct ArrowIpcRandomAccessInput* input, int64_t offset, int64_t size_bytes, + struct ArrowBuffer* out, struct ArrowError* error) { + if (offset < 0 || size_bytes < 0 || !_ArrowInt64FitsInSizeT(size_bytes)) { + ArrowErrorSet(error, "Offset or size is out of range"); + return EINVAL; + } + struct ArrowIpcRandomAccessFilePrivate* private_data = + (struct ArrowIpcRandomAccessFilePrivate*)input->private_data; + ArrowBufferReset(out); + if (size_bytes == 0) { + return NANOARROW_OK; + } +#if defined(_WIN32) + int seek_result = _fseeki64(private_data->file_ptr, offset, SEEK_SET); +#else + if (offset > LONG_MAX) { + ArrowErrorSet(error, "Offset exceeds the range supported by fseek()"); + return EINVAL; + } + int seek_result = fseek(private_data->file_ptr, (long)offset, SEEK_SET); +#endif + if (seek_result != 0) { + ArrowErrorSet(error, "ArrowIpcRandomAccessInput seek failed"); + return EIO; + } + clearerr(private_data->file_ptr); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBufferReserve(out, size_bytes), error); + out->size_bytes = + (int64_t)fread(out->data, 1, (size_t)size_bytes, private_data->file_ptr); + if (ferror(private_data->file_ptr)) { + ArrowErrorSet(error, "ArrowIpcRandomAccessInput read failed"); + return EIO; + } + return NANOARROW_OK; +} + +static void ArrowIpcRandomAccessFileRelease(struct ArrowIpcRandomAccessInput* input) { + struct ArrowIpcRandomAccessFilePrivate* private_data = + (struct ArrowIpcRandomAccessFilePrivate*)input->private_data; + if (private_data->close_on_release) { + fclose(private_data->file_ptr); + } + ArrowFree(private_data); + input->release = NULL; +} + +ArrowErrorCode ArrowIpcRandomAccessInputInitFile(struct ArrowIpcRandomAccessInput* out, + void* file_ptr, int close_on_release) { + if (file_ptr == NULL) { + return EINVAL; + } + struct ArrowIpcRandomAccessFilePrivate* private_data = + (struct ArrowIpcRandomAccessFilePrivate*)ArrowMalloc(sizeof(*private_data)); + if (private_data == NULL) { + return ENOMEM; + } + private_data->file_ptr = (FILE*)file_ptr; + private_data->close_on_release = close_on_release; + out->peek_at = &ArrowIpcRandomAccessFilePeek; + out->read_at = &ArrowIpcRandomAccessFileRead; + out->release = &ArrowIpcRandomAccessFileRelease; + out->private_data = private_data; + return NANOARROW_OK; +} + struct ArrowIpcArrayStreamReaderPrivate { struct ArrowIpcInputStream input; + struct ArrowIpcRandomAccessInput random_input; + int64_t input_offset; struct ArrowIpcDecoder decoder; int use_shared_buffers; enum ArrowValidationLevel validation_level; @@ -207,6 +377,10 @@ static void ArrowIpcArrayStreamReaderRelease(struct ArrowArrayStream* stream) { private_data->input.release(&private_data->input); } + if (private_data->random_input.release != NULL) { + private_data->random_input.release(&private_data->random_input); + } + ArrowIpcDecoderReset(&private_data->decoder); if (private_data->out_schema.release != NULL) { @@ -224,17 +398,81 @@ static void ArrowIpcArrayStreamReaderRelease(struct ArrowArrayStream* stream) { stream->release = NULL; } +static int ArrowIpcArrayStreamReaderCheckRead( + struct ArrowIpcArrayStreamReaderPrivate* private_data, const void* data, + int64_t size_bytes, int64_t requested) { + if (size_bytes < 0 || size_bytes > requested || (size_bytes > 0 && data == NULL)) { + ArrowErrorSet(&private_data->error, + "Random-access input returned an invalid size or pointer"); + return EINVAL; + } + return NANOARROW_OK; +} + +// Adapt either input interface to the existing sequential header/body parser. +static int ArrowIpcArrayStreamReaderRead( + struct ArrowIpcArrayStreamReaderPrivate* private_data, uint8_t* out, + int64_t size_bytes, int64_t* bytes_read) { + if (private_data->random_input.release == NULL) { + return private_data->input.read(&private_data->input, out, size_bytes, bytes_read, + &private_data->error); + } + if (size_bytes < 0 || !_ArrowInt64FitsInSizeT(size_bytes) || + size_bytes > INT64_MAX - private_data->input_offset) { + ArrowErrorSet(&private_data->error, "Input position is out of range"); + return EINVAL; + } + struct ArrowIpcRandomAccessInput* input = &private_data->random_input; + struct ArrowBufferView view; + if (input->peek_at != NULL) { + int result = input->peek_at(input, private_data->input_offset, size_bytes, &view, + &private_data->error); + if (result == NANOARROW_OK) { + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderCheckRead( + private_data, view.data.data, view.size_bytes, size_bytes)); + if (view.size_bytes > 0) { + memcpy(out, view.data.data, _ArrowInt64ToSizeT(view.size_bytes)); + } + *bytes_read = view.size_bytes; + private_data->input_offset += view.size_bytes; + return NANOARROW_OK; + } else if (result != ENOTSUP) { + return result; + } + } + + struct ArrowBuffer buffer; + ArrowBufferInit(&buffer); + int result = input->read_at(input, private_data->input_offset, size_bytes, &buffer, + &private_data->error); + if (result == NANOARROW_OK) { + result = ArrowIpcArrayStreamReaderCheckRead(private_data, buffer.data, + buffer.size_bytes, size_bytes); + } + if (result == NANOARROW_OK) { + if (buffer.size_bytes > 0) { + memcpy(out, buffer.data, _ArrowInt64ToSizeT(buffer.size_bytes)); + } + *bytes_read = buffer.size_bytes; + private_data->input_offset += buffer.size_bytes; + } + ArrowBufferReset(&buffer); + return result; +} + static int ArrowIpcArrayStreamReaderNextHeader( struct ArrowIpcArrayStreamReaderPrivate* private_data, int schema_expected) { + if (private_data->random_input.release != NULL) { + ArrowBufferReset(&private_data->header); + } private_data->header.size_bytes = 0; int64_t bytes_read = 0; // Read 8 bytes (continuation + header size in bytes) NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowBufferReserve(&private_data->header, 8), &private_data->error); - NANOARROW_RETURN_NOT_OK(private_data->input.read(&private_data->input, - private_data->header.data, 8, - &bytes_read, &private_data->error)); + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderRead( + private_data, private_data->header.data, 8, &bytes_read)); private_data->header.size_bytes += bytes_read; if (bytes_read == 0) { @@ -317,21 +555,52 @@ static int ArrowIpcArrayStreamReaderNextHeader( NANOARROW_DCHECK(new_prefix_size_bytes == 8); } - // Read the header bytes int64_t expected_header_bytes = private_data->decoder.header_size_bytes - 8; - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowBufferReserve(&private_data->header, - expected_header_bytes - extra_bytes_already_read), - &private_data->error); - NANOARROW_RETURN_NOT_OK(private_data->input.read( - &private_data->input, private_data->header.data + private_data->header.size_bytes, - expected_header_bytes - extra_bytes_already_read, &bytes_read, - &private_data->error)); - private_data->header.size_bytes += bytes_read; - - // Verify + decode the header + if (prefix_size_bytes == 8 && private_data->random_input.release != NULL) { + // DecodeHeader retains pointers used when decoding the body. A peek is only + // valid until the next input callback, so retain an owned header instead. + // Buffer inputs return a shared slice here without copying the metadata. + int64_t header_start = private_data->input_offset - 8; + if (private_data->decoder.header_size_bytes > INT64_MAX - header_start) { + ArrowErrorSet(&private_data->error, "Input position is out of range"); + return EINVAL; + } + struct ArrowBuffer header; + ArrowBufferInit(&header); + int result = private_data->random_input.read_at( + &private_data->random_input, header_start, + private_data->decoder.header_size_bytes, &header, &private_data->error); + if (result == NANOARROW_OK) { + result = + ArrowIpcArrayStreamReaderCheckRead(private_data, header.data, header.size_bytes, + private_data->decoder.header_size_bytes); + } + if (result == NANOARROW_OK) { + private_data->input_offset = header_start + header.size_bytes; + if (((uintptr_t)header.data % 8) == 0) { + ArrowBufferReset(&private_data->header); + ArrowBufferMove(&header, &private_data->header); + } else { + private_data->header.size_bytes = 0; + result = ArrowBufferAppend(&private_data->header, header.data, header.size_bytes); + } + } + ArrowBufferReset(&header); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(result, &private_data->error); + } else { + // Legacy headers need an inserted continuation token in an aligned buffer. + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferReserve(&private_data->header, + expected_header_bytes - extra_bytes_already_read), + &private_data->error); + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderRead( + private_data, private_data->header.data + private_data->header.size_bytes, + expected_header_bytes - extra_bytes_already_read, &bytes_read)); + private_data->header.size_bytes += bytes_read; + } input_view.data.data = private_data->header.data; input_view.size_bytes = private_data->header.size_bytes; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderVerifyHeader(&private_data->decoder, input_view, &private_data->error)); @@ -363,14 +632,26 @@ static int ArrowIpcArrayStreamReaderNextBody( int64_t bytes_read; int64_t bytes_to_read = private_data->decoder.body_size_bytes; - // Read the body bytes - private_data->body.size_bytes = 0; - NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowBufferReserve(&private_data->body, bytes_to_read), &private_data->error); - NANOARROW_RETURN_NOT_OK(private_data->input.read(&private_data->input, - private_data->body.data, bytes_to_read, - &bytes_read, &private_data->error)); - private_data->body.size_bytes += bytes_read; + if (private_data->use_shared_buffers && private_data->random_input.release != NULL) { + if (bytes_to_read < 0 || bytes_to_read > INT64_MAX - private_data->input_offset) { + ArrowErrorSet(&private_data->error, "Input position is out of range"); + return EINVAL; + } + NANOARROW_RETURN_NOT_OK(private_data->random_input.read_at( + &private_data->random_input, private_data->input_offset, bytes_to_read, + &private_data->body, &private_data->error)); + bytes_read = private_data->body.size_bytes; + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderCheckRead( + private_data, private_data->body.data, bytes_read, bytes_to_read)); + private_data->input_offset += bytes_read; + } else { + private_data->body.size_bytes = 0; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferReserve(&private_data->body, bytes_to_read), &private_data->error); + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderRead( + private_data, private_data->body.data, bytes_to_read, &bytes_read)); + private_data->body.size_bytes += bytes_read; + } if (bytes_read != bytes_to_read) { ArrowErrorSet(&private_data->error, @@ -383,6 +664,17 @@ static int ArrowIpcArrayStreamReaderNextBody( } } +// Transfer the body of the current message to a shared buffer +static ArrowErrorCode ArrowIpcArrayStreamReaderTakeSharedBody( + struct ArrowIpcArrayStreamReaderPrivate* private_data, struct ArrowBuffer* out) { + if (ArrowIsSharedBuffer(&private_data->body)) { + ArrowBufferMove(&private_data->body, out); + return NANOARROW_OK; + } + + return ArrowSharedBufferInit(out, &private_data->body); +} + static int ArrowIpcArrayStreamReaderReadSchemaIfNeeded( struct ArrowIpcArrayStreamReaderPrivate* private_data) { if (private_data->out_schema.release != NULL) { @@ -457,7 +749,8 @@ static int ArrowIpcArrayStreamReaderProcessRecordBatch( if (private_data->use_shared_buffers) { struct ArrowBuffer shared; NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error); + ArrowIpcArrayStreamReaderTakeSharedBody(private_data, &shared), + &private_data->error); ArrowErrorCode result = ArrowIpcDecoderDecodeArrayFromSharedWithDictionaries( &private_data->decoder, &shared, private_data->field_index, &private_data->dictionaries, out, private_data->validation_level, @@ -487,7 +780,8 @@ static int ArrowIpcArrayStreamReaderProcessDictionary( // Decode the dictionary struct ArrowBuffer shared; NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowSharedBufferInit(&shared, &private_data->body), &private_data->error); + ArrowIpcArrayStreamReaderTakeSharedBody(private_data, &shared), + &private_data->error); int result = ArrowIpcDecoderDecodeDictionaryFromShared( &private_data->decoder, &shared, private_data->validation_level, &private_data->dictionaries, &private_data->error); @@ -603,6 +897,8 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit( ArrowBufferInit(&private_data->body); private_data->out_schema.release = NULL; ArrowIpcInputStreamMove(input_stream, &private_data->input); + private_data->random_input.release = NULL; + private_data->input_offset = 0; private_data->expected_header_prefix_size = kExpectedHeaderPrefixSizeNotSet; private_data->dictionaries.private_data = NULL; @@ -624,3 +920,17 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit( return NANOARROW_OK; } + +ArrowErrorCode ArrowIpcArrayStreamReaderInitRandomAccess( + struct ArrowArrayStream* out, struct ArrowIpcRandomAccessInput* input, + struct ArrowIpcArrayStreamReaderOptions* options) { + if (input->release == NULL || input->read_at == NULL) { + return EINVAL; + } + struct ArrowIpcInputStream unused = {NULL, NULL, NULL}; + NANOARROW_RETURN_NOT_OK(ArrowIpcArrayStreamReaderInit(out, &unused, options)); + struct ArrowIpcArrayStreamReaderPrivate* private_data = + (struct ArrowIpcArrayStreamReaderPrivate*)out->private_data; + ArrowIpcRandomAccessInputMove(input, &private_data->random_input); + return NANOARROW_OK; +} diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc index 13545a691..7ff9a234c 100644 --- a/src/nanoarrow/ipc/reader_test.cc +++ b/src/nanoarrow/ipc/reader_test.cc @@ -21,7 +21,7 @@ #include -#include "nanoarrow/nanoarrow_ipc.h" +#include "nanoarrow/nanoarrow_ipc.hpp" static uint8_t kSimpleSchema[] = { 0xff, 0xff, 0xff, 0xff, 0x10, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -169,21 +169,25 @@ struct FileCloser { FILE* file_{}; }; +static FILE* CreateTemporaryFile() { +#ifdef _MSC_VER + FILE* file = nullptr; + if (tmpfile_s(&file) != 0) { + return nullptr; + } + return file; +#else + return tmpfile(); +#endif +} + TEST(NanoarrowIpcReader, InputStreamFile) { struct ArrowIpcInputStream stream; errno = EINVAL; ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL); uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05}; -#ifdef _MSC_VER - // This macro suppresses warnings when using tmpfile and the MSVC compiler - // warning C4996: 'tmpfile': This function or variable may be unsafe. Consider - // using tmpfile_s instead. To disable deprecation, use _CRT_SECURE_NO_WARNINGS - FILE* file_ptr; - ASSERT_EQ(tmpfile_s(&file_ptr), 0); -#else - FILE* file_ptr = tmpfile(); -#endif + FILE* file_ptr = CreateTemporaryFile(); FileCloser closer{file_ptr}; ASSERT_NE(file_ptr, nullptr); ASSERT_EQ(fwrite(input_data, 1, sizeof(input_data), file_ptr), sizeof(input_data)); @@ -296,6 +300,93 @@ TEST(NanoarrowIpcReader, StreamReaderBasicNoSharedBuffers) { ArrowArrayStreamRelease(&stream); } +TEST(NanoarrowIpcReader, StreamReaderBufferInputSharedBuffers) { + // When the input is a buffer, decoded arrays should reference it in place when + // shared buffers are requested (and the input is aligned), and keep it alive + // after the stream has been released. + struct TestCase { + int use_shared_buffers; + bool input_is_shared; + bool input_is_misaligned; + bool expect_values_in_input; + }; + + alignas(8) + uint8_t misaligned_storage[1 + sizeof(kSimpleSchema) + sizeof(kSimpleRecordBatch)]; + + for (const auto& test_case : + {TestCase{1, false, false, true}, TestCase{1, true, false, true}, + TestCase{0, false, false, false}, TestCase{0, true, false, false}, + TestCase{1, false, true, false}}) { + SCOPED_TRACE( + "use_shared_buffers: " + std::to_string(test_case.use_shared_buffers) + + ", input_is_shared: " + std::to_string(test_case.input_is_shared) + + ", input_is_misaligned: " + std::to_string(test_case.input_is_misaligned)); + + struct ArrowBuffer input_buffer; + ArrowBufferInit(&input_buffer); + if (test_case.input_is_misaligned) { + // Borrow storage that starts one byte past an aligned address + uint8_t* data = misaligned_storage + 1; + memcpy(data, kSimpleSchema, sizeof(kSimpleSchema)); + memcpy(data + sizeof(kSimpleSchema), kSimpleRecordBatch, + sizeof(kSimpleRecordBatch)); + ASSERT_EQ(ArrowBufferSetAllocator( + &input_buffer, + ArrowBufferDeallocator( + [](ArrowBufferAllocator*, uint8_t*, int64_t) {}, nullptr)), + NANOARROW_OK); + input_buffer.data = data; + input_buffer.size_bytes = sizeof(kSimpleSchema) + sizeof(kSimpleRecordBatch); + } else { + ASSERT_EQ(ArrowBufferAppend(&input_buffer, kSimpleSchema, sizeof(kSimpleSchema)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(&input_buffer, kSimpleRecordBatch, + sizeof(kSimpleRecordBatch)), + NANOARROW_OK); + } + + if (test_case.input_is_shared) { + struct ArrowBuffer shared; + ASSERT_EQ(ArrowSharedBufferInit(&shared, &input_buffer), NANOARROW_OK); + ArrowBufferMove(&shared, &input_buffer); + } + + const uint8_t* input_begin = input_buffer.data; + const uint8_t* input_end = input_buffer.data + input_buffer.size_bytes; + + struct ArrowIpcRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(&input, &input_buffer), NANOARROW_OK); + + struct ArrowArrayStream stream; + struct ArrowIpcArrayStreamReaderOptions options; + options.field_index = -1; + options.use_shared_buffers = test_case.use_shared_buffers; + options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL; + ASSERT_EQ(ArrowIpcArrayStreamReaderInitRandomAccess(&stream, &input, &options), + NANOARROW_OK); + + struct ArrowArray array; + ASSERT_EQ(ArrowArrayStreamGetNext(&stream, &array, nullptr), NANOARROW_OK); + ArrowArrayStreamRelease(&stream); + + ASSERT_EQ(array.n_children, 1); + ASSERT_EQ(array.children[0]->length, 3); + auto values = reinterpret_cast(array.children[0]->buffers[1]); + bool values_in_input = values >= input_begin && values < input_end; + EXPECT_EQ(values_in_input, test_case.expect_values_in_input); + EXPECT_EQ(reinterpret_cast(values) % 8, 0); + + int32_t int_values[3]; + memcpy(int_values, values, sizeof(int_values)); + EXPECT_EQ(int_values[0], 1); + EXPECT_EQ(int_values[1], 2); + EXPECT_EQ(int_values[2], 3); + + ArrowArrayRelease(&array); + } +} + TEST(NanoarrowIpcReader, StreamReaderBasicWithEndOfStream) { struct ArrowBuffer input_buffer; ArrowBufferInit(&input_buffer); @@ -712,3 +803,354 @@ TEST(NanoarrowIpcReader, StreamReaderRecordBatchWithoutDictionaryBatch) { ArrowArrayStreamRelease(&stream); } + +TEST(NanoarrowIpcReader, RandomAccessBufferBoundsAndLifetime) { + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ(ArrowBufferAppend(buffer.get(), "abcde", 5), NANOARROW_OK); + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(input.get(), buffer.get()), NANOARROW_OK); + EXPECT_EQ(buffer->data, nullptr); + + ArrowBufferView peek; + ASSERT_EQ(input->peek_at(input.get(), 2, 100, &peek, nullptr), NANOARROW_OK); + ASSERT_EQ(peek.size_bytes, 3); + EXPECT_EQ(memcmp(peek.data.data, "cde", 3), 0); + EXPECT_EQ(input->peek_at(input.get(), -1, 1, &peek, nullptr), EINVAL); + EXPECT_EQ(input->peek_at(input.get(), 0, -1, &peek, nullptr), EINVAL); + ASSERT_EQ(input->peek_at(input.get(), INT64_MAX, INT64_MAX, &peek, nullptr), + NANOARROW_OK); + EXPECT_EQ(peek.size_bytes, 0); + + ASSERT_EQ(input->read_at(input.get(), 3, 100, buffer.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(buffer->size_bytes, 2); + EXPECT_EQ(memcmp(buffer->data, "de", 2), 0); + // Absolute offsets allow going backwards, independent of previous reads. + ASSERT_EQ(input->read_at(input.get(), 0, 2, buffer.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(buffer->size_bytes, 2); + EXPECT_EQ(memcmp(buffer->data, "ab", 2), 0); + ASSERT_EQ(input->read_at(input.get(), 0, 0, buffer.get(), nullptr), NANOARROW_OK); + EXPECT_EQ(buffer->size_bytes, 0); + ASSERT_EQ(input->read_at(input.get(), 1, 3, buffer.get(), nullptr), NANOARROW_OK); + nanoarrow::ipc::UniqueRandomAccessInput moved(std::move(input)); + EXPECT_EQ(input->release, nullptr); // NOLINT(clang-analyzer-cplusplus.Move) + moved.reset(); + EXPECT_EQ(memcmp(buffer->data, "bcd", 3), 0); +} + +TEST(NanoarrowIpcReader, RandomAccessFileBoundsAndSeeking) { + FILE* file = CreateTemporaryFile(); + FileCloser closer{file}; + ASSERT_NE(file, nullptr); + ASSERT_EQ(fwrite("abcde", 1, 5, file), 5); + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitFile(input.get(), nullptr, 0), EINVAL); + ASSERT_EQ(ArrowIpcRandomAccessInputInitFile(input.get(), file, 0), NANOARROW_OK); + ArrowBufferView peek; + EXPECT_EQ(input->peek_at(input.get(), 0, 1, &peek, nullptr), ENOTSUP); + nanoarrow::UniqueBuffer out; + EXPECT_EQ(input->read_at(input.get(), -1, 1, out.get(), nullptr), EINVAL); + EXPECT_EQ(input->read_at(input.get(), 0, -1, out.get(), nullptr), EINVAL); + ASSERT_EQ(input->read_at(input.get(), 3, 10, out.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(out->size_bytes, 2); + EXPECT_EQ(memcmp(out->data, "de", 2), 0); + ASSERT_EQ(input->read_at(input.get(), 5, 10, out.get(), nullptr), NANOARROW_OK); + EXPECT_EQ(out->size_bytes, 0); + ASSERT_EQ(input->read_at(input.get(), 1, 3, out.get(), nullptr), NANOARROW_OK); + ASSERT_EQ(out->size_bytes, 3); + EXPECT_EQ(memcmp(out->data, "bcd", 3), 0); + input.reset(); + EXPECT_EQ(fseek(file, 0, SEEK_SET), 0); + EXPECT_EQ(fclose(file), 0); + closer.file_ = nullptr; + EXPECT_EQ(memcmp(out->data, "bcd", 3), 0); +} + +// A custom input whose peek result is invalidated by its next read. This also +// exercises the reader without relying on the built-in callback addresses. +struct RandomAccessReaderTestInput { + nanoarrow::ipc::UniqueRandomAccessInput underlying; + nanoarrow::UniqueBuffer scratch; + int read_count = 0; + int peek_count = 0; + bool peek_unsupported = false; +}; + +static void InitRandomAccessReaderTestInput(RandomAccessReaderTestInput* state, + ArrowIpcRandomAccessInput* input) { + input->private_data = state; + input->peek_at = [](ArrowIpcRandomAccessInput* self, int64_t offset, int64_t size, + ArrowBufferView* out, ArrowError* error) -> ArrowErrorCode { + auto* state = static_cast(self->private_data); + state->peek_count++; + if (state->peek_unsupported) { + return ENOTSUP; + } + NANOARROW_RETURN_NOT_OK(state->underlying->read_at( + state->underlying.get(), offset, size, state->scratch.get(), error)); + // Make a private copy that the next callback can overwrite. + nanoarrow::UniqueBuffer copy; + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(copy.get(), state->scratch->data, state->scratch->size_bytes)); + state->scratch = std::move(copy); + out->data.data = state->scratch->data; + out->size_bytes = state->scratch->size_bytes; + return NANOARROW_OK; + }; + input->read_at = [](ArrowIpcRandomAccessInput* self, int64_t offset, int64_t size, + ArrowBuffer* out, ArrowError* error) -> ArrowErrorCode { + auto* state = static_cast(self->private_data); + state->read_count++; + if (state->scratch->size_bytes > 0) { + memset(state->scratch->data, 0xff, state->scratch->size_bytes); + } + return state->underlying->read_at(state->underlying.get(), offset, size, out, error); + }; + input->release = [](ArrowIpcRandomAccessInput* self) { + auto* state = static_cast(self->private_data); + state->underlying.reset(); + state->scratch.reset(); + self->release = nullptr; + }; +} + +TEST(NanoarrowIpcReader, RandomAccessReaderCustomAndFileInputs) { + for (int backend : {0, 1, 2, 3}) { + for (int sharing : {0, 1}) { + SCOPED_TRACE("backend=" + std::to_string(backend) + + " sharing=" + std::to_string(sharing)); + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kSimpleSchema, sizeof(kSimpleSchema)), + NANOARROW_OK); + ASSERT_EQ( + ArrowBufferAppend(buffer.get(), kSimpleRecordBatch, sizeof(kSimpleRecordBatch)), + NANOARROW_OK); + ASSERT_EQ( + ArrowBufferAppend(buffer.get(), kSimpleRecordBatch, sizeof(kSimpleRecordBatch)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kEndOfStream, sizeof(kEndOfStream)), + NANOARROW_OK); + RandomAccessReaderTestInput state; + if (backend == 3) { + FILE* file = CreateTemporaryFile(); + FileCloser closer{file}; + ASSERT_NE(file, nullptr); + ASSERT_EQ(fwrite(buffer->data, 1, buffer->size_bytes, file), buffer->size_bytes); + ASSERT_EQ(ArrowIpcRandomAccessInputInitFile(state.underlying.get(), file, 1), + NANOARROW_OK); + closer.file_ = nullptr; + } else { + ASSERT_EQ( + ArrowIpcRandomAccessInputInitBuffer(state.underlying.get(), buffer.get()), + NANOARROW_OK); + } + nanoarrow::ipc::UniqueRandomAccessInput input; + InitRandomAccessReaderTestInput(&state, input.get()); + if (backend == 1) { + input->peek_at = nullptr; + } + if (backend == 2) { + state.peek_unsupported = true; + } + ArrowIpcArrayStreamReaderOptions options = {-1, sharing, + NANOARROW_VALIDATION_LEVEL_FULL}; + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + NANOARROW_OK); + EXPECT_EQ(input->release, nullptr); + nanoarrow::UniqueSchema schema; + ASSERT_EQ(ArrowArrayStreamGetSchema(reader.get(), schema.get(), nullptr), + NANOARROW_OK); + nanoarrow::UniqueArray first, second, eos; + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), first.get(), nullptr), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), second.get(), nullptr), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), eos.get(), nullptr), NANOARROW_OK); + EXPECT_EQ(eos->release, nullptr); + reader.reset(); + EXPECT_GT(state.read_count, 0); + if (backend != 1) { + EXPECT_GT(state.peek_count, 0); + } + for (auto* array : {first.get(), second.get()}) { + ASSERT_EQ(array->length, 3); + const auto* values = static_cast(array->children[0]->buffers[1]); + EXPECT_EQ(values[0], 1); + EXPECT_EQ(values[1], 2); + EXPECT_EQ(values[2], 3); + } + } + } +} + +TEST(NanoarrowIpcReader, RandomAccessReaderUnalignedBufferOffset) { + // The body is aligned, but its int32 buffer starts at offset one. + uint8_t batch[sizeof(kSimpleRecordBatch)]; + memcpy(batch, kSimpleRecordBatch, sizeof(batch)); + ASSERT_EQ(batch[104], 0); + batch[104] = 1; // Second Buffer.offset in the flatbuffer header. + memmove(batch + 145, batch + 144, 12); + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kSimpleSchema, sizeof(kSimpleSchema)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(buffer.get(), batch, sizeof(batch)), NANOARROW_OK); + const uint8_t* original_values = buffer->data + sizeof(kSimpleSchema) + 145; + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(input.get(), buffer.get()), NANOARROW_OK); + ArrowIpcArrayStreamReaderOptions options = {-1, 1, NANOARROW_VALIDATION_LEVEL_FULL}; + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + NANOARROW_OK); + nanoarrow::UniqueArray array; + ArrowError error; + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), array.get(), &error), NANOARROW_OK) + << error.message; + const auto* values = static_cast(array->children[0]->buffers[1]); + EXPECT_NE(static_cast(values), original_values); + EXPECT_EQ(reinterpret_cast(values) % alignof(int32_t), 0); + reader.reset(); + EXPECT_EQ(values[0], 1); + EXPECT_EQ(values[1], 2); + EXPECT_EQ(values[2], 3); +} + +TEST(NanoarrowIpcReader, RandomAccessReaderDictionaryLifetime) { + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kDictionarySchema, sizeof(kDictionarySchema)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kDictionaryBatch, sizeof(kDictionaryBatch)), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kDictionaryRecordBatch, + sizeof(kDictionaryRecordBatch)), + NANOARROW_OK); + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(input.get(), buffer.get()), NANOARROW_OK); + ArrowIpcArrayStreamReaderOptions options = {-1, 1, NANOARROW_VALIDATION_LEVEL_FULL}; + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + NANOARROW_OK); + nanoarrow::UniqueArray array; + ArrowError error; + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), array.get(), &error), NANOARROW_OK) + << error.message; + reader.reset(); + ASSERT_NE(array->children[0]->dictionary, nullptr); + const auto* dictionary = array->children[0]->dictionary; + ASSERT_EQ(dictionary->length, 3); + EXPECT_EQ(memcmp(dictionary->buffers[2], "zeroonetwo", 10), 0); +} + +TEST(NanoarrowIpcReader, RandomAccessReaderTruncatedBodyAndInvalidOptions) { + nanoarrow::UniqueBuffer buffer; + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kSimpleSchema, sizeof(kSimpleSchema)), + NANOARROW_OK); + ASSERT_EQ( + ArrowBufferAppend(buffer.get(), kSimpleRecordBatch, sizeof(kSimpleRecordBatch) - 1), + NANOARROW_OK); + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(input.get(), buffer.get()), NANOARROW_OK); + nanoarrow::UniqueArrayStream reader; + ArrowIpcArrayStreamReaderOptions options = {-1, 1, NANOARROW_VALIDATION_LEVEL_FULL}; + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange): deliberately invalid + options.validation_level = static_cast(999); + EXPECT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + EINVAL); + EXPECT_NE(input->release, nullptr); + options.validation_level = NANOARROW_VALIDATION_LEVEL_FULL; + ASSERT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + NANOARROW_OK); + nanoarrow::UniqueArray array; + ArrowError error; + EXPECT_EQ(ArrowArrayStreamGetNext(reader.get(), array.get(), &error), ESPIPE); + EXPECT_STREQ(error.message, + "Expected to be able to read 16 bytes for message body but got 15"); +} + +TEST(NanoarrowIpcReader, RandomAccessReaderCallbackErrors) { + for (int mode : {0, 1, 2, 3}) { + SCOPED_TRACE(mode); + ArrowIpcRandomAccessInput input = {}; + input.private_data = &mode; + input.release = [](ArrowIpcRandomAccessInput* self) { self->release = nullptr; }; + input.read_at = [](ArrowIpcRandomAccessInput* self, int64_t, int64_t size, + ArrowBuffer* out, ArrowError* error) -> ArrowErrorCode { + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(out, "x", 1)); + if (*static_cast(self->private_data) == 2) { + ArrowErrorSet(error, "read failure"); + return EIO; + } + out->size_bytes = size + 1; + return NANOARROW_OK; + }; + if (mode < 2) { + input.peek_at = [](ArrowIpcRandomAccessInput* self, int64_t, int64_t size, + ArrowBufferView* out, ArrowError* error) -> ArrowErrorCode { + if (*static_cast(self->private_data) == 0) { + ArrowErrorSet(error, "peek failure"); + return EIO; + } + out->data.data = nullptr; + out->size_bytes = size + 1; + return NANOARROW_OK; + }; + } + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ(ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), &input, nullptr), + NANOARROW_OK); + nanoarrow::UniqueSchema schema; + ArrowError error; + EXPECT_EQ(ArrowArrayStreamGetSchema(reader.get(), schema.get(), &error), + mode % 2 == 0 ? EIO : EINVAL); + EXPECT_GT(strlen(error.message), 0); + } + ArrowIpcRandomAccessInput invalid = {}; + nanoarrow::UniqueArrayStream reader; + EXPECT_EQ(ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), &invalid, nullptr), + EINVAL); +} + +TEST(NanoarrowIpcReader, RandomAccessReaderLegacyHeaders) { + nanoarrow::UniqueBuffer buffer; + for (ArrowBufferView message : + {ArrowBufferView{{kSimpleSchema}, sizeof(kSimpleSchema)}, + ArrowBufferView{{kSimpleRecordBatch}, sizeof(kSimpleRecordBatch)}}) { + nanoarrow::UniqueBuffer legacy; + ASSERT_EQ(ArrowBufferAppendBufferView(legacy.get(), message), NANOARROW_OK); + // Locate Message.version through the flatbuffer's root and vtable, then set V4. + auto read16 = [](const uint8_t* p) { return p[0] | (p[1] << 8); }; + auto read32 = [](const uint8_t* p) { + return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24); + }; + int table = 8 + read32(legacy->data + 8); + int vtable = table - read32(legacy->data + table); + int version = table + read16(legacy->data + vtable + 4); + legacy->data[version] = 3; // MetadataVersion.V4 + legacy->data[version + 1] = 0; + // Legacy messages omit the continuation token. + ASSERT_EQ(ArrowBufferAppend(buffer.get(), legacy->data + 4, legacy->size_bytes - 4), + NANOARROW_OK); + } + ASSERT_EQ(ArrowBufferAppend(buffer.get(), kEndOfStream + 4, 4), NANOARROW_OK); + nanoarrow::ipc::UniqueRandomAccessInput input; + ASSERT_EQ(ArrowIpcRandomAccessInputInitBuffer(input.get(), buffer.get()), NANOARROW_OK); + ArrowIpcArrayStreamReaderOptions options = {-1, 1, NANOARROW_VALIDATION_LEVEL_FULL}; + nanoarrow::UniqueArrayStream reader; + ASSERT_EQ( + ArrowIpcArrayStreamReaderInitRandomAccess(reader.get(), input.get(), &options), + NANOARROW_OK); + nanoarrow::UniqueArray array, eos; + ArrowError error; + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), array.get(), &error), NANOARROW_OK) + << error.message; + ASSERT_EQ(array->length, 3); + ASSERT_EQ(ArrowArrayStreamGetNext(reader.get(), eos.get(), &error), NANOARROW_OK) + << error.message; + EXPECT_EQ(eos->release, nullptr); + reader.reset(); + EXPECT_EQ(static_cast(array->children[0]->buffers[1])[2], 3); +} diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index d10f2fc18..f31fa376d 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -98,6 +98,14 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcInputStreamMove) #define ArrowIpcArrayStreamReaderInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcArrayStreamReaderInit) +#define ArrowIpcRandomAccessInputInitBuffer \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcRandomAccessInputInitBuffer) +#define ArrowIpcRandomAccessInputInitFile \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcRandomAccessInputInitFile) +#define ArrowIpcRandomAccessInputMove \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcRandomAccessInputMove) +#define ArrowIpcArrayStreamReaderInitRandomAccess \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcArrayStreamReaderInitRandomAccess) #define ArrowIpcEncoderInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcEncoderInit) #define ArrowIpcEncoderReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcEncoderReset) #define ArrowIpcEncoderFinalizeBuffer \ @@ -926,7 +934,60 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcInputStreamInitBuffer( NANOARROW_DLL ArrowErrorCode ArrowIpcInputStreamInitFile( struct ArrowIpcInputStream* stream, void* file_ptr, int close_on_release); -/// \brief Options for ArrowIpcArrayStreamReaderInit() +/// \brief A user-extensible input with absolute byte offsets +/// +/// Reads do not advance a cursor. The array stream reader manages its own position; +/// callbacks receive non-negative offsets relative to the beginning of the input. +/// Callbacks return up to size_bytes bytes, including a short or empty result at EOF. +/// An implementation need not support concurrent calls. +struct ArrowIpcRandomAccessInput { + /// \brief Borrow bytes without IO, or return ENOTSUP + /// + /// Optional: may be NULL. On success, out is valid until the next callback on + /// this input. The pointer need not be aligned. The reader copies unaligned + /// metadata before decoding it. + ArrowErrorCode (*peek_at)(struct ArrowIpcRandomAccessInput* input, int64_t offset, + int64_t size_bytes, struct ArrowBufferView* out, + struct ArrowError* error); + + /// \brief Read bytes into an owned buffer + /// + /// Required. out is initialized by the caller. The implementation replaces its + /// contents with a buffer that remains valid independently of the input. This + /// may be a shared slice of existing memory or a newly allocated buffer. On + /// error, out must remain safe to ArrowBufferReset(). + ArrowErrorCode (*read_at)(struct ArrowIpcRandomAccessInput* input, int64_t offset, + int64_t size_bytes, struct ArrowBuffer* out, + struct ArrowError* error); + + /// \brief Release the input; implementations must set release to NULL + void (*release)(struct ArrowIpcRandomAccessInput* input); + void* private_data; +}; + +/// \brief Transfer ownership of a random-access input +NANOARROW_DLL void ArrowIpcRandomAccessInputMove(struct ArrowIpcRandomAccessInput* src, + struct ArrowIpcRandomAccessInput* dst); + +/// \brief Create a random-access input that takes ownership of input on success +/// +/// peek_at() borrows memory and read_at() returns shared slices. Decoded arrays +/// may retain the whole input buffer until released. If +/// ArrowSharedBufferIsThreadSafe() is false, shared slices must be released on +/// the same thread as the input. +NANOARROW_DLL ArrowErrorCode ArrowIpcRandomAccessInputInitBuffer( + struct ArrowIpcRandomAccessInput* out, struct ArrowBuffer* input); + +/// \brief Create a random-access input from a seekable C FILE* pointer +/// +/// Offsets are absolute file offsets. read_at() seeks and reads the FILE*, so the +/// caller must not use it concurrently. peek_at() returns ENOTSUP. On success, +/// close_on_release determines whether release() closes the FILE*. Close errors +/// cannot be reported by release(); use close_on_release = 0 to handle them yourself. +NANOARROW_DLL ArrowErrorCode ArrowIpcRandomAccessInputInitFile( + struct ArrowIpcRandomAccessInput* out, void* file_ptr, int close_on_release); + +/// \brief Options for the IPC array stream reader initializers struct ArrowIpcArrayStreamReaderOptions { /// \brief The field index to extract. /// @@ -963,6 +1024,16 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcArrayStreamReaderInit( struct ArrowArrayStream* out, struct ArrowIpcInputStream* input_stream, struct ArrowIpcArrayStreamReaderOptions* options); +/// \brief Initialize the same IPC stream reader from a random-access input +/// +/// Reading starts at absolute offset zero, which must contain a Schema message. +/// This reads the IPC stream format, not the IPC file format. On success, out +/// takes ownership of input. With use_shared_buffers enabled, read_at() buffers +/// are shared among decoded arrays. Unaligned array buffers are copied as needed. +NANOARROW_DLL ArrowErrorCode ArrowIpcArrayStreamReaderInitRandomAccess( + struct ArrowArrayStream* out, struct ArrowIpcRandomAccessInput* input, + struct ArrowIpcArrayStreamReaderOptions* options); + /// \brief Encoder for Arrow IPC messages /// /// This structure is intended to be allocated by the caller, diff --git a/src/nanoarrow/nanoarrow_ipc.hpp b/src/nanoarrow/nanoarrow_ipc.hpp index 9629eba31..2ad47bbc7 100644 --- a/src/nanoarrow/nanoarrow_ipc.hpp +++ b/src/nanoarrow/nanoarrow_ipc.hpp @@ -168,6 +168,24 @@ inline void release_pointer(struct ArrowIpcInputStream* data) { } } +template <> +inline void init_pointer(struct ArrowIpcRandomAccessInput* data) { + data->release = nullptr; +} + +template <> +inline void move_pointer(struct ArrowIpcRandomAccessInput* src, + struct ArrowIpcRandomAccessInput* dst) { + ArrowIpcRandomAccessInputMove(src, dst); +} + +template <> +inline void release_pointer(struct ArrowIpcRandomAccessInput* data) { + if (data->release != nullptr) { + data->release(data); + } +} + template <> inline void init_pointer(struct ArrowIpcOutputStream* data) { data->release = nullptr; @@ -241,6 +259,9 @@ using UniqueCompressor = internal::Unique; /// \brief Class wrapping a unique struct ArrowIpcInputStream using UniqueInputStream = internal::Unique; +/// \brief Class wrapping a unique struct ArrowIpcRandomAccessInput +using UniqueRandomAccessInput = internal::Unique; + /// \brief Class wrapping a unique struct ArrowIpcOutputStream using UniqueOutputStream = internal::Unique;