From f7c8cb28e25cc5190566331868c677278abcfadb Mon Sep 17 00:00:00 2001 From: JonSnow1807 Date: Wed, 11 Nov 2026 23:50:42 -0600 Subject: [PATCH] feat(ipc): Support binary view and string view arrays in the IPC reader and writer The IPC reader rejected the BinaryView and Utf8View types and the writer could not encode them. A binary view or string view field has a validity buffer, a views buffer, and a number of variadic data buffers that can change from one RecordBatch to the next, given by the field's entry in RecordBatch.variadicBufferCounts. The decoder now maps both types, checks that a RecordBatch has one variadicBufferCounts entry per view field and that the counts add up to the number of buffers in the message, and points each array view at its variadic buffers. The buffers of a field that is decoded on its own start after the variadic buffers of any view fields before it. The scratch ArrowArray holds the variadic buffer pointers and sizes and any buffer that has to be allocated (e.g., to decompress it), using ArrowArrayBuffer() for the variadic buffers. The encoder writes the views buffer and the variadic buffers of each view array (but not the C data interface's buffer of variadic buffer sizes) and the variadicBufferCounts of RecordBatch and DictionaryBatch messages that have view arrays. As in Arrow C++, view arrays from a message of the other endianness are not supported. --- src/nanoarrow/ipc/decoder.c | 223 ++++++++++++--- src/nanoarrow/ipc/decoder_test.cc | 453 ++++++++++++++++++++++++++++++ src/nanoarrow/ipc/encoder.c | 60 +++- src/nanoarrow/ipc/files_test.cc | 2 +- src/nanoarrow/ipc/writer_test.cc | 71 ++++- 5 files changed, 756 insertions(+), 53 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 844ddc72c..e256d613c 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -47,8 +47,12 @@ struct ArrowIpcField { // array is scratch space for any intermediary allocations (i.e., it is never moved // to the user). struct ArrowArray* array; - // The cumulative number of buffers preceding this node. + // The cumulative number of buffers preceding this node, not counting the variadic + // buffers of any BinaryView or StringView nodes (which vary between messages). int64_t buffer_offset; + // The cumulative number of BinaryView or StringView nodes preceding this node (i.e., + // the number of RecordBatch.variadicBufferCounts entries that precede its own). + int64_t variadic_buffer_count_offset; // Dictionary identifier (or NANOARROW_IPC_NO_DICTIONARY_ID if this is not a // dictionary-encoded field) int64_t dictionary_id; @@ -75,10 +79,13 @@ struct ArrowIpcDecoderPrivate { // the ArrowArrayView/ArrowArray where the depth-first buffer/field walk should start. struct ArrowIpcField* fields; // The number of buffers that future RecordBatch messages must have to match the schema - // that has been set. + // that has been set (plus the number of variadic buffers in each message). int64_t n_buffers; // The number of union fields in the Schema. int64_t n_union_fields; + // The number of BinaryView and StringView fields in the Schema (i.e., the number of + // variadicBufferCounts entries that future RecordBatch messages must have). + int64_t n_variadic_buffer_counts; // A pointer to the last flatbuffers message. const void* last_message; // The custom_metadata of the last flatbuffers Message (i.e., Message.custom_metadata, @@ -1246,15 +1253,13 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema* schema, ns(Field_table_t) case ns(Type_FixedSizeBinary): return ArrowIpcDecoderSetTypeFixedSizeBinary(schema, type, error); case ns(Type_BinaryView): - ArrowErrorSet(error, "BinaryView not yet supported in IPC reader"); - return ENOTSUP; + return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_BINARY_VIEW, error); case ns(Type_Utf8): return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_STRING, error); case ns(Type_LargeUtf8): return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_LARGE_STRING, error); case ns(Type_Utf8View): - ArrowErrorSet(error, "Utf8View not yet supported in IPC reader"); - return ENOTSUP; + return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_STRING_VIEW, error); case ns(Type_Date): return ArrowIpcDecoderSetTypeDate(schema, type, error); case ns(Type_Time): @@ -1558,6 +1563,32 @@ static int ArrowIpcDecoderDecodeRecordBatchHeader(struct ArrowIpcDecoder* decode n_expected_buffers += private_data->n_union_fields; } + // Each BinaryView or StringView field has a number of variadic buffers that is given + // by its entry in variadicBufferCounts + flatbuffers_int64_vec_t variadic_buffer_counts = + ns(RecordBatch_variadicBufferCounts(batch)); + int64_t n_variadic_buffer_counts = flatbuffers_int64_vec_len(variadic_buffer_counts); + if (n_variadic_buffer_counts != private_data->n_variadic_buffer_counts) { + ArrowErrorSet(error, + "Expected %" PRId64 + " variadic buffer counts in message but found %" PRId64, + private_data->n_variadic_buffer_counts, n_variadic_buffer_counts); + return EINVAL; + } + + for (int64_t i = 0; i < n_variadic_buffer_counts; i++) { + int64_t count = flatbuffers_int64_vec_at(variadic_buffer_counts, (size_t)i); + if (count < 0 || count > n_buffers) { + ArrowErrorSet(error, + "Expected variadic buffer count between 0 and %" PRId64 + " but found %" PRId64, + n_buffers, count); + return EINVAL; + } + + n_expected_buffers += count; + } + if ((n_buffers + 1) != n_expected_buffers) { ArrowErrorSet(error, "Expected %" PRId64 " buffers in message but found %" PRId64, n_expected_buffers - 1, n_buffers); @@ -2015,11 +2046,13 @@ static ArrowErrorCode ArrowIpcDecoderInitFields( struct ArrowIpcField* fields, const struct ArrowSchema* schema, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowArrayView* array_view, struct ArrowArray* array, int64_t* n_fields, - int64_t* n_buffers, int64_t* n_union_fields, struct ArrowError* error) { + int64_t* n_buffers, int64_t* n_union_fields, int64_t* n_variadic_buffer_counts, + struct ArrowError* error) { struct ArrowIpcField* field = fields + (*n_fields); field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; + field->variadic_buffer_count_offset = *n_variadic_buffer_counts; field->dictionary_id = NANOARROW_IPC_NO_DICTIONARY_ID; if (schema->dictionary != NULL) { @@ -2056,13 +2089,16 @@ static ArrowErrorCode ArrowIpcDecoderInitFields( } *n_union_fields += array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION || array_view->storage_type == NANOARROW_TYPE_DENSE_UNION; + *n_variadic_buffer_counts += array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; *n_fields += 1; for (int64_t i = 0; i < array_view->n_children; i++) { NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( fields, schema->children[i], dictionary_encodings, array_view->children[i], - array->children[i], n_fields, n_buffers, n_union_fields, error)); + array->children[i], n_fields, n_buffers, n_union_fields, n_variadic_buffer_counts, + error)); } return NANOARROW_OK; @@ -2079,6 +2115,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( private_data->n_buffers = 0; private_data->n_fields = 0; private_data->n_union_fields = 0; + private_data->n_variadic_buffer_counts = 0; ArrowArrayViewReset(&private_data->array_view); if (private_data->array.release != NULL) { ArrowArrayRelease(&private_data->array); @@ -2116,7 +2153,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( private_data->fields, schema, dictionary_encodings, &private_data->array_view, &private_data->array, &field_i, &private_data->n_buffers, - &private_data->n_union_fields, error)); + &private_data->n_union_fields, &private_data->n_variadic_buffer_counts, error)); return NANOARROW_OK; } @@ -2348,6 +2385,12 @@ static int ArrowIpcDecoderSwapEndian(struct ArrowIpcBufferSource* src, case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_BINARY: return NANOARROW_OK; + case NANOARROW_TYPE_BINARY_VIEW: + case NANOARROW_TYPE_STRING_VIEW: + ArrowErrorSet(error, + "Endian swapping of binary view and string view arrays is not " + "supported"); + return ENOTSUP; default: break; } @@ -2457,6 +2500,8 @@ struct ArrowIpcArraySetter { int64_t field_i; ns(Buffer_vec_t) buffers; int64_t buffer_i; + flatbuffers_int64_vec_t variadic_buffer_counts; + int64_t variadic_buffer_count_i; struct ArrowIpcBufferSource src; struct ArrowIpcBufferFactory factory; enum ArrowIpcMetadataVersion version; @@ -2497,28 +2542,122 @@ static int ArrowIpcDecoderMakeBuffer(struct ArrowIpcArraySetter* setter, int64_t return NANOARROW_OK; } +// Make the next buffer in the message, using out if any allocation has to occur +static int ArrowIpcDecoderMakeNextBuffer(struct ArrowIpcArraySetter* setter, + struct ArrowBufferView* out_view, + struct ArrowBuffer* out, + struct ArrowError* error) { + ns(Buffer_struct_t) buffer = + ns(Buffer_vec_at(setter->buffers, (size_t)setter->buffer_i)); + int64_t buffer_offset = ns(Buffer_offset(buffer)); + int64_t buffer_length = ns(Buffer_length(buffer)); + setter->buffer_i += 1; + + // Attempt to re-use any previous allocation unless this buffer is + // wrapping a custom allocator. + if (out->allocator.private_data != NULL) { + ArrowBufferReset(out); + } else { + out->size_bytes = 0; + } + + return ArrowIpcDecoderMakeBuffer(setter, buffer_offset, buffer_length, out_view, out, + error); +} + +// The variadic buffers of a BinaryView or StringView array follow its validity and views +// buffers in the message. The scratch array provides an ArrowBuffer for each of them that +// is used if any allocation has to occur, and storage for the buffer pointers and sizes +// that the array view refers to (as they would be for an ArrowArray). +static int ArrowIpcDecoderWalkSetVariadicBuffers(struct ArrowIpcArraySetter* setter, + struct ArrowArrayView* array_view, + struct ArrowArray* array, + struct ArrowError* error) { + // The count was checked against the number of buffers in the message when the + // header was decoded + int64_t n_variadic_buffers = flatbuffers_int64_vec_at( + setter->variadic_buffer_counts, (size_t)setter->variadic_buffer_count_i); + setter->variadic_buffer_count_i += 1; + + int32_t n_scratch_buffers = ArrowArrayVariadicBufferCount(array); + if (n_variadic_buffers > n_scratch_buffers) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAddVariadicBuffers(array, + (int32_t)(n_variadic_buffers - n_scratch_buffers)), + error); + } + + struct ArrowBuffer* sizes = ArrowArrayBuffer(array, array->n_buffers - 1); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferResize(sizes, n_variadic_buffers * (int64_t)sizeof(int64_t), 0), error); + + // Variadic buffers contain bytes, which never need endian swapping + setter->src.data_type = NANOARROW_TYPE_BINARY; + setter->src.element_size_bits = 8; + + for (int64_t i = 0; i < n_variadic_buffers; i++) { + int64_t buffer_i = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i; + struct ArrowBufferView view; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderMakeNextBuffer( + setter, &view, ArrowArrayBuffer(array, buffer_i), error)); + array->buffers[buffer_i] = view.data.data; + ((int64_t*)sizes->data)[i] = view.size_bytes; + } + + array_view->n_variadic_buffers = (int32_t)n_variadic_buffers; + array_view->variadic_buffers = array->buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS; + array_view->variadic_buffer_sizes = (int64_t*)sizes->data; + return NANOARROW_OK; +} + +// If the scratch buffer was used, move it to the final array. Otherwise, copy the view. +static int ArrowIpcDecoderGetBuffer(struct ArrowBufferView view, + struct ArrowBuffer* scratch_buffer, + struct ArrowBuffer* buffer_out, + struct ArrowError* error) { + if (scratch_buffer->size_bytes == 0) { + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendBufferView(buffer_out, view)); + } else if (scratch_buffer->data == view.data.as_uint8) { + ArrowBufferMove(scratch_buffer, buffer_out); + } else { + ArrowErrorSet( + error, + "Internal: scratch buffer was used but doesn't point to the same data as view"); + return EINVAL; + } + + return NANOARROW_OK; +} + static int ArrowIpcDecoderWalkGetArray(struct ArrowArrayView* array_view, struct ArrowArray* array, struct ArrowArray* out, struct ArrowError* error) { out->length = array_view->length; out->null_count = array_view->null_count; - for (int64_t i = 0; i < array->n_buffers; i++) { - struct ArrowBufferView view = array_view->buffer_views[i]; - struct ArrowBuffer* scratch_buffer = ArrowArrayBuffer(array, i); - struct ArrowBuffer* buffer_out = ArrowArrayBuffer(out, i); - - // If the scratch buffer was used, move it to the final array. Otherwise, - // copy the view. - if (scratch_buffer->size_bytes == 0) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendBufferView(buffer_out, view)); - } else if (scratch_buffer->data == view.data.as_uint8) { - ArrowBufferMove(scratch_buffer, buffer_out); - } else { - ArrowErrorSet( - error, - "Internal: scratch buffer was used but doesn't point to the same data as view"); - return EINVAL; + int is_view = array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; + + // The variadic buffers of a BinaryView or StringView array are handled below (the + // final buffer with their sizes is populated when the array is finished) + int64_t n_buffers = is_view ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS : array->n_buffers; + for (int64_t i = 0; i < n_buffers; i++) { + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderGetBuffer(array_view->buffer_views[i], + ArrowArrayBuffer(array, i), + ArrowArrayBuffer(out, i), error)); + } + + if (is_view && array_view->n_variadic_buffers > 0) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAddVariadicBuffers(out, array_view->n_variadic_buffers), error); + for (int32_t i = 0; i < array_view->n_variadic_buffers; i++) { + struct ArrowBufferView view; + view.data.data = array_view->variadic_buffers[i]; + view.size_bytes = array_view->variadic_buffer_sizes[i]; + int64_t buffer_i = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i; + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderGetBuffer(view, ArrowArrayBuffer(array, buffer_i), + ArrowArrayBuffer(out, buffer_i), error)); } } @@ -2616,29 +2755,17 @@ static int ArrowIpcDecoderWalkSetArrayView(struct ArrowIpcDecoder* decoder, break; } - ns(Buffer_struct_t) buffer = - ns(Buffer_vec_at(setter->buffers, (size_t)setter->buffer_i)); - int64_t buffer_offset = ns(Buffer_offset(buffer)); - int64_t buffer_length = ns(Buffer_length(buffer)); - setter->buffer_i += 1; - - // Provide a buffer that will be used if any allocation has to occur - struct ArrowBuffer* buffer_dst = ArrowArrayBuffer(array, i); - - // Attempt to re-use any previous allocation unless this buffer is - // wrapping a custom allocator. - if (buffer_dst->allocator.private_data != NULL) { - ArrowBufferReset(buffer_dst); - } else { - buffer_dst->size_bytes = 0; - } - setter->src.data_type = array_view->layout.buffer_data_type[i]; setter->src.element_size_bits = array_view->layout.element_size_bits[i]; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderMakeNextBuffer( + setter, &array_view->buffer_views[i], ArrowArrayBuffer(array, i), error)); + } + + if (array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW) { NANOARROW_RETURN_NOT_OK( - ArrowIpcDecoderMakeBuffer(setter, buffer_offset, buffer_length, - &array_view->buffer_views[i], buffer_dst, error)); + ArrowIpcDecoderWalkSetVariadicBuffers(setter, array_view, array, error)); } for (int64_t i = 0; i < array_view->n_children; i++) { @@ -2714,12 +2841,20 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayViewInternal( setter.field_i = field_i; setter.buffers = ns(RecordBatch_buffers(batch)); setter.buffer_i = root->buffer_offset - 1; + setter.variadic_buffer_counts = ns(RecordBatch_variadicBufferCounts(batch)); + setter.variadic_buffer_count_i = root->variadic_buffer_count_offset; setter.factory = factory; setter.src.codec = decoder->codec; setter.src.swap_endian = ArrowIpcDecoderNeedsSwapEndian(decoder); setter.version = decoder->metadata_version; setter.dictionaries = dictionaries; + // The variadic buffers of any BinaryView or StringView fields before this one also + // precede its buffers + for (int64_t i = 0; i < root->variadic_buffer_count_offset; i++) { + setter.buffer_i += flatbuffers_int64_vec_at(setter.variadic_buffer_counts, (size_t)i); + } + // If we are going to need a decompressor here, ensure the default one is // initialized. if (setter.src.codec != NANOARROW_IPC_COMPRESSION_TYPE_NONE) { diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index a2c68c9c5..5605ab3a5 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #endif @@ -55,6 +56,7 @@ struct ArrowIpcField { struct ArrowArrayView* array_view; struct ArrowArray* array; int64_t buffer_offset; + int64_t variadic_buffer_count_offset; int64_t dictionary_id; }; @@ -1641,6 +1643,16 @@ void AssertArrayViewIdentical(const struct ArrowArrayView* actual, } } + ASSERT_EQ(actual->n_variadic_buffers, expected->n_variadic_buffers); + for (int32_t i = 0; i < expected->n_variadic_buffers; i++) { + ASSERT_EQ(actual->variadic_buffer_sizes[i], expected->variadic_buffer_sizes[i]); + if (expected->variadic_buffer_sizes[i] != 0) { + ASSERT_EQ(memcmp(actual->variadic_buffers[i], expected->variadic_buffers[i], + expected->variadic_buffer_sizes[i]), + 0); + } + } + ASSERT_EQ(actual->n_children, expected->n_children); for (int i = 0; i < actual->n_children; i++) { AssertArrayViewIdentical(actual->children[i], expected->children[i]); @@ -2077,6 +2089,11 @@ INSTANTIATE_TEST_SUITE_P( arrow::run_end_encoded(arrow::int16(), arrow::utf8()), arrow::run_end_encoded(arrow::int32(), arrow::int64()), arrow::run_end_encoded(arrow::int64(), arrow::float64()), + // Binary view and string view + arrow::binary_view(), arrow::utf8_view(), + arrow::struct_({arrow::field("col1", arrow::utf8_view()), + arrow::field("col2", arrow::int32()), + arrow::field("col3", arrow::binary_view())}), // Type with nested metadata arrow::list(arrow::field("some_custom_name", arrow::int32(), @@ -2092,6 +2109,8 @@ INSTANTIATE_TEST_SUITE_P( arrow::dictionary(arrow::uint64(), arrow::utf8()), // Ordered dictionary encoding arrow::dictionary(arrow::int32(), arrow::utf8(), true), + // Dictionary-encoded string view + arrow::dictionary(arrow::int32(), arrow::utf8_view()), // Extension type arrow::extension::uuid(), // Extension type with dictionary as the storage type @@ -2821,3 +2840,437 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeEmptyArrayOffsets) { EXPECT_EQ(first_offset, 0); } } + +// A value for the binary view and string view arrays in the tests below. Most are +// longer than the 12 bytes that a view holds inline, so they are stored in variadic +// buffers, and some are empty or inlined. +static std::string ViewTestValue(int64_t i, int64_t max_size) { + std::string value(static_cast((i * 37) % max_size), ' '); + for (size_t j = 0; j < value.size(); j++) { + value[j] = static_cast('a' + (i + static_cast(j)) % 26); + } + return value; +} + +// A struct, i: int32, sv: string_view> array with +// nulls in the binary view and string view columns. Values of up to 400 bytes give +// those arrays several variadic buffers (nanoarrow starts a new one every 32 KB). +static void MakeViewStructArray(struct ArrowSchema* schema, struct ArrowArray* array, + int64_t length) { + ArrowSchemaInit(schema); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema, 4), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_BINARY_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "bv"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[1], "l"), NANOARROW_OK); + ASSERT_EQ( + ArrowSchemaSetType(schema->children[1]->children[0], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[2], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[2], "i"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[3], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[3], "sv"), NANOARROW_OK); + + ASSERT_EQ(ArrowArrayInitFromSchema(array, schema, nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(array), NANOARROW_OK); + for (int64_t i = 0; i < length; i++) { + if (i % 7 == 3) { + ASSERT_EQ(ArrowArrayAppendNull(array->children[0], 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(array->children[3], 1), NANOARROW_OK); + } else { + std::string bv = ViewTestValue(i, 400); + std::string sv = ViewTestValue(i + 1, 400); + ASSERT_EQ(ArrowArrayAppendString(array->children[0], ArrowCharView(bv.c_str())), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(array->children[3], ArrowCharView(sv.c_str())), + NANOARROW_OK); + } + + for (int64_t j = 0; j < i % 4; j++) { + std::string item = ViewTestValue(i * 3 + j, 400); + ASSERT_EQ(ArrowArrayAppendString(array->children[1]->children[0], + ArrowCharView(item.c_str())), + NANOARROW_OK); + } + ASSERT_EQ(ArrowArrayFinishElement(array->children[1]), NANOARROW_OK); + + ASSERT_EQ(ArrowArrayAppendInt(array->children[2], i), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(array), NANOARROW_OK); + } + + ASSERT_EQ(ArrowArrayFinishBuildingDefault(array, nullptr), NANOARROW_OK); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcViewArrayRoundtrip) { + struct ArrowError error; + std::vector codecs = { + NANOARROW_IPC_COMPRESSION_TYPE_NONE}; + if (ArrowIpcGetLZ4CompressionFunction() != nullptr) { + codecs.push_back(NANOARROW_IPC_COMPRESSION_TYPE_LZ4_FRAME); + } + if (ArrowIpcGetZstdCompressionFunction() != nullptr) { + codecs.push_back(NANOARROW_IPC_COMPRESSION_TYPE_ZSTD); + } + + for (auto codec : codecs) { + // One decoder reads batches whose arrays have more, then fewer variadic buffers than + // the batch before. (Lengths are multiples of 8 so that AssertArrayViewIdentical() + // doesn't compare the unset bits at the end of a validity bitmap.) + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + bool decoder_has_schema = false; + + for (int64_t length : {400, 1000, 0, 400}) { + SCOPED_TRACE(::testing::Message() << "codec " << codec << ", length " << length); + nanoarrow::UniqueSchema schema; + nanoarrow::UniqueArray array; + ASSERT_NO_FATAL_FAILURE(MakeViewStructArray(schema.get(), array.get(), length)); + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error), + NANOARROW_OK) + << error.message; + + struct ArrowArrayView* bv = array_view->children[0]; + struct ArrowArrayView* l_sv = array_view->children[1]->children[0]; + struct ArrowArrayView* sv = array_view->children[3]; + if (length > 0) { + ASSERT_GT(bv->n_variadic_buffers, 1); + ASSERT_GT(l_sv->n_variadic_buffers, 1); + ASSERT_GT(sv->n_variadic_buffers, 1); + } + + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + ASSERT_EQ( + ArrowIpcEncoderSetCompression(encoder.get(), codec, + NANOARROW_IPC_COMPRESSION_LEVEL_DEFAULT, &error), + NANOARROW_OK) + << error.message; + nanoarrow::UniqueBuffer message, body; + ASSERT_EQ(ArrowIpcEncoderEncodeSimpleRecordBatch(encoder.get(), array_view.get(), + body.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, + message.get()), + NANOARROW_OK); + + // The message has a variadicBufferCounts entry for each view array in field order + ns(Message_table_t) fb_message = ns(Message_as_root(message->data + 8)); + ns(RecordBatch_table_t) fb_batch = + (ns(RecordBatch_table_t))ns(Message_header(fb_message)); + flatbuffers_int64_vec_t counts = ns(RecordBatch_variadicBufferCounts(fb_batch)); + ASSERT_EQ(flatbuffers_int64_vec_len(counts), 3); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 0), bv->n_variadic_buffers); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 1), l_sv->n_variadic_buffers); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 2), sv->n_variadic_buffers); + + if (!decoder_has_schema) { + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), + NANOARROW_OK) + << error.message; + decoder_has_schema = true; + } + ASSERT_EQ(ArrowIpcDecoderDecodeHeader( + decoder.get(), {{message->data}, message->size_bytes}, &error), + NANOARROW_OK) + << error.message; + struct ArrowBufferView body_view = {{body->data}, body->size_bytes}; + + // Decode the whole batch and each field on its own (in the depth-first order of + // field indices), whose buffers follow the variadic buffers of any view arrays + // before it + struct ArrowArrayView* decoded; + ASSERT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, -1, &decoded, &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE(AssertArrayViewIdentical(decoded, array_view.get())); + const struct ArrowArrayView* fields[] = {bv, array_view->children[1], l_sv, + array_view->children[2], sv}; + for (int64_t i = 0; i < 5; i++) { + SCOPED_TRACE(::testing::Message() << "field " << i); + ASSERT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, i, &decoded, &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE(AssertArrayViewIdentical(decoded, fields[i])); + } + + // Decode arrays that own a copy of the body or share it. (The view of an empty + // array has no offsets for the list column, so only compare a non-empty one.) + if (length == 0) { + continue; + } + + nanoarrow::UniqueBuffer body_copy; + ASSERT_EQ(ArrowBufferAppend(body_copy.get(), body->data, body->size_bytes), + NANOARROW_OK); + nanoarrow::UniqueBuffer shared_body; + ASSERT_EQ(ArrowIpcSharedBufferInit(shared_body.get(), body_copy.get()), + NANOARROW_OK); + for (bool shared : {false, true}) { + SCOPED_TRACE(shared ? "shared body" : "copied body"); + nanoarrow::UniqueArray decoded_array; + if (shared) { + ASSERT_EQ(ArrowIpcDecoderDecodeArrayFromShared( + decoder.get(), shared_body.get(), -1, decoded_array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } else { + ASSERT_EQ(ArrowIpcDecoderDecodeArray(decoder.get(), body_view, -1, + decoded_array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } + + nanoarrow::UniqueArrayView decoded_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(decoded_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(decoded_view.get(), decoded_array.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE( + AssertArrayViewIdentical(decoded_view.get(), array_view.get())); + } + + // As in Arrow C++, view arrays can't be read from a message of the other + // endianness +#if defined(__BIG_ENDIAN__) + ASSERT_EQ( + ArrowIpcDecoderSetEndianness(decoder.get(), NANOARROW_IPC_ENDIANNESS_LITTLE), + NANOARROW_OK); +#else + ASSERT_EQ(ArrowIpcDecoderSetEndianness(decoder.get(), NANOARROW_IPC_ENDIANNESS_BIG), + NANOARROW_OK); +#endif + ASSERT_EQ(ArrowIpcDecoderDecodeHeader( + decoder.get(), {{message->data}, message->size_bytes}, &error), + NANOARROW_OK) + << error.message; + EXPECT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, 0, &decoded, &error), + ENOTSUP); + EXPECT_STREQ(error.message, + "Endian swapping of binary view and string view arrays is not " + "supported"); + ASSERT_EQ(ArrowIpcDecoderSetEndianness(decoder.get(), + NANOARROW_IPC_ENDIANNESS_UNINITIALIZED), + NANOARROW_OK); + } + } +} + +// Builds an encapsulated RecordBatch message with one field node, n_buffers empty +// buffers, and the given variadicBufferCounts (omitted if counts is nullptr) +static void MakeRecordBatchWithVariadicBufferCounts(int64_t n_buffers, + const std::vector* counts, + struct ArrowBuffer* out) { + flatcc_builder_t builder; + flatcc_builder_init(&builder); + + ASSERT_EQ(ns(Message_start_as_root(&builder)), 0); + ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0); + ASSERT_EQ(ns(Message_header_RecordBatch_start(&builder)), 0); + struct ns(FieldNode) node = {0, 0}; + ASSERT_EQ(ns(RecordBatch_nodes_create(&builder, &node, 1)), 0); + std::vector buffers(n_buffers); + ASSERT_EQ(ns(RecordBatch_buffers_create(&builder, buffers.data(), buffers.size())), 0); + if (counts != nullptr) { + ASSERT_EQ(ns(RecordBatch_variadicBufferCounts_create(&builder, counts->data(), + counts->size())), + 0); + } + ASSERT_EQ(ns(Message_header_RecordBatch_end(&builder)), 0); + ASSERT_NE(ns(Message_end_as_root(&builder)), 0); + + ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out)); + flatcc_builder_clear(&builder); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeVariadicBufferCounts) { + struct ArrowError error; + + // A string view column has a validity buffer, a views buffer, and one buffer for + // each variadic buffer + nanoarrow::UniqueSchema schema; + ArrowSchemaInit(schema.get()); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema.get(), 1), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + + struct Case { + int64_t n_buffers; + bool has_counts; + std::vector counts; + std::string expected_error; + }; + const std::vector cases = { + {2, false, {}, "Expected 1 variadic buffer counts in message but found 0"}, + {2, true, {}, "Expected 1 variadic buffer counts in message but found 0"}, + {2, true, {0, 0}, "Expected 1 variadic buffer counts in message but found 2"}, + {2, true, {-1}, "Expected variadic buffer count between 0 and 2 but found -1"}, + {2, + true, + {INT64_MAX}, + "Expected variadic buffer count between 0 and 2 but found 9223372036854775807"}, + {2, true, {1}, "Expected 3 buffers in message but found 2"}, + {3, true, {0}, "Expected 2 buffers in message but found 3"}, + {2, true, {0}, ""}, + {4, true, {2}, ""}}; + + for (const auto& c : cases) { + SCOPED_TRACE(::testing::Message() << "buffers=" << c.n_buffers << ", counts=" + << ::testing::PrintToString(c.counts)); + nanoarrow::UniqueBuffer message; + ASSERT_NO_FATAL_FAILURE(MakeRecordBatchWithVariadicBufferCounts( + c.n_buffers, c.has_counts ? &c.counts : nullptr, message.get())); + struct ArrowBufferView data = {{message->data}, message->size_bytes}; + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + + if (!c.expected_error.empty()) { + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), EINVAL); + EXPECT_EQ(std::string(error.message), c.expected_error); + continue; + } + + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + struct ArrowArrayView* decoded; + ASSERT_EQ(ArrowIpcDecoderDecodeArrayView(decoder.get(), {{nullptr}, 0}, -1, &decoded, + &error), + NANOARROW_OK) + << error.message; + EXPECT_EQ(decoded->children[0]->n_variadic_buffers, c.counts[0]); + } + + // A schema without view fields does not expect any variadic buffer counts + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + std::vector counts = {0}; + nanoarrow::UniqueBuffer message; + ASSERT_NO_FATAL_FAILURE( + MakeRecordBatchWithVariadicBufferCounts(3, &counts, message.get())); + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), + {{message->data}, message->size_bytes}, &error), + EINVAL); + EXPECT_STREQ(error.message, "Expected 0 variadic buffer counts in message but found 1"); +} + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) +TEST(NanoarrowIpcTest, NanoarrowIpcViewArrayArrowInterop) { + struct ArrowError error; + constexpr int64_t kLength = 200; + + // Values of up to 40 bytes in blocks of 64 bytes give the binary view array a + // variadic buffer for every one or two values that are not inlined. The string view + // array is a slice. + arrow::BinaryViewBuilder bv_builder; + bv_builder.SetBlockSize(64); + arrow::StringViewBuilder sv_builder; + for (int64_t i = 0; i < kLength + 10; i++) { + if (i % 5 == 1) { + ASSERT_TRUE(bv_builder.AppendNull().ok()); + ASSERT_TRUE(sv_builder.AppendNull().ok()); + } else { + ASSERT_TRUE(bv_builder.Append(ViewTestValue(i, 41)).ok()); + ASSERT_TRUE(sv_builder.Append(ViewTestValue(i + 1, 41)).ok()); + } + } + auto bv = bv_builder.Finish().ValueOrDie()->Slice(0, kLength); + auto sv = sv_builder.Finish().ValueOrDie()->Slice(10, kLength); + ASSERT_GT(bv->data()->buffers.size(), static_cast(20)); + + auto arrow_schema = arrow::schema( + {arrow::field("bv", arrow::binary_view()), arrow::field("sv", arrow::utf8_view())}); + auto batch = arrow::RecordBatch::Make(arrow_schema, kLength, {bv, sv}); + nanoarrow::UniqueSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, schema.get()).ok()); + + std::vector codecs = {arrow::Compression::UNCOMPRESSED}; + if (ArrowIpcGetLZ4DecompressionFunction() != nullptr && + arrow::util::Codec::IsAvailable(arrow::Compression::LZ4_FRAME)) { + codecs.push_back(arrow::Compression::LZ4_FRAME); + } + if (ArrowIpcGetZstdDecompressionFunction() != nullptr && + arrow::util::Codec::IsAvailable(arrow::Compression::ZSTD)) { + codecs.push_back(arrow::Compression::ZSTD); + } + + for (auto codec : codecs) { + SCOPED_TRACE(arrow::util::Codec::GetCodecAsString(codec)); + auto options = arrow::ipc::IpcWriteOptions::Defaults(); + if (codec != arrow::Compression::UNCOMPRESSED) { + options.codec = arrow::util::Codec::Create(codec).ValueOrDie(); + } + auto maybe_serialized = arrow::ipc::SerializeRecordBatch(*batch, options); + ASSERT_TRUE(maybe_serialized.ok()) << maybe_serialized.status(); + auto serialized = maybe_serialized.ValueUnsafe(); + + // nanoarrow reads the message that Arrow C++ wrote + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + struct ArrowBufferView data = {{serialized->data()}, serialized->size()}; + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + data.data.as_uint8 += decoder->header_size_bytes; + data.size_bytes -= decoder->header_size_bytes; + nanoarrow::UniqueArray array; + ASSERT_EQ(ArrowIpcDecoderDecodeArray(decoder.get(), data, -1, array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + + // Arrow C++ reads the message that nanoarrow writes from what it read + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error), NANOARROW_OK) + << error.message; + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + nanoarrow::UniqueBuffer message, body; + ASSERT_EQ(ArrowIpcEncoderEncodeSimpleRecordBatch(encoder.get(), array_view.get(), + body.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, message.get()), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(message.get(), body->data, body->size_bytes), + NANOARROW_OK); + + arrow::io::BufferReader message_reader( + arrow::Buffer::Wrap(message->data, message->size_bytes)); + auto maybe_message = arrow::ipc::ReadMessage(&message_reader); + ASSERT_TRUE(maybe_message.ok()) << maybe_message.status(); + arrow::ipc::DictionaryMemo memo; + auto maybe_roundtripped = + arrow::ipc::ReadRecordBatch(*maybe_message.ValueUnsafe(), arrow_schema, &memo, + arrow::ipc::IpcReadOptions::Defaults()); + ASSERT_TRUE(maybe_roundtripped.ok()) << maybe_roundtripped.status(); + ASSERT_TRUE(maybe_roundtripped.ValueUnsafe()->ValidateFull().ok()); + EXPECT_TRUE(maybe_roundtripped.ValueUnsafe()->Equals(*batch)); + + auto maybe_batch = arrow::ImportRecordBatch(array.get(), arrow_schema); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status(); + ASSERT_TRUE(maybe_batch.ValueUnsafe()->ValidateFull().ok()); + EXPECT_TRUE(maybe_batch.ValueUnsafe()->Equals(*batch)); + } +} +#endif diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index 5b78f315a..b54799732 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -58,6 +58,9 @@ struct ArrowIpcEncoderPrivate { int has_compressed_body; // Views of the body buffers of the message being encoded, in body order struct ArrowBuffer buffer_views; + // The number of variadic buffers of each BinaryView or StringView array in the + // message being encoded, in the order of their field nodes + struct ArrowBuffer variadic_buffer_counts; // Compressed copies of those buffers (reused between messages) struct ArrowBuffer* compressed_buffers; int64_t n_compressed_buffers; @@ -85,6 +88,7 @@ ArrowErrorCode ArrowIpcEncoderInit(struct ArrowIpcEncoder* encoder) { private->compressor.release = NULL; private->has_compressed_body = 0; ArrowBufferInit(&private->buffer_views); + ArrowBufferInit(&private->variadic_buffer_counts); private->compressed_buffers = NULL; private->n_compressed_buffers = 0; return NANOARROW_OK; @@ -104,6 +108,7 @@ void ArrowIpcEncoderReset(struct ArrowIpcEncoder* encoder) { private->compressor.release(&private->compressor); } ArrowBufferReset(&private->buffer_views); + ArrowBufferReset(&private->variadic_buffer_counts); for (int64_t i = 0; i < private->n_compressed_buffers; i++) { ArrowBufferReset(&private->compressed_buffers[i]); } @@ -358,6 +363,14 @@ static ArrowErrorCode ArrowIpcEncodeFieldType(flatcc_builder_t* builder, FLATCC_RETURN_UNLESS_0(Field_type_LargeBinary_create(builder), error); return NANOARROW_OK; + case NANOARROW_TYPE_STRING_VIEW: + FLATCC_RETURN_UNLESS_0(Field_type_Utf8View_create(builder), error); + return NANOARROW_OK; + + case NANOARROW_TYPE_BINARY_VIEW: + FLATCC_RETURN_UNLESS_0(Field_type_BinaryView_create(builder), error); + return NANOARROW_OK; + case NANOARROW_TYPE_DATE32: FLATCC_RETURN_UNLESS_0(Field_type_Date_create(builder, ns(DateUnit_DAY)), error); return NANOARROW_OK; @@ -996,7 +1009,14 @@ static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* struct ns(FieldNode) node = {array_view->length, null_count}; NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowBufferAppend(&private->nodes, &node, sizeof(node)), error); - for (int64_t b = 0; b < array_view->array->n_buffers; ++b) { + + // The variadic buffers of a BinaryView or StringView array are collected below. The + // final buffer of such an array (the sizes of the variadic buffers) is not written. + int is_view = array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; + int64_t n_buffers = + is_view ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS : array_view->array->n_buffers; + for (int64_t b = 0; b < n_buffers; ++b) { struct ArrowBufferView buffer_view = array_view->buffer_views[b]; // An offsets buffer holds length + 1 offsets, so even an empty array has one. @@ -1014,6 +1034,40 @@ static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* sizeof(struct ArrowBufferView)), error); } + + if (is_view) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppendInt64(&private->variadic_buffer_counts, + array_view->n_variadic_buffers), + error); + for (int32_t i = 0; i < array_view->n_variadic_buffers; i++) { + struct ArrowBufferView buffer_view; + buffer_view.data.data = array_view->variadic_buffers[i]; + buffer_view.size_bytes = array_view->variadic_buffer_sizes[i]; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppend(&private->buffer_views, &buffer_view, + sizeof(struct ArrowBufferView)), + error); + } + } + + return NANOARROW_OK; +} + +// Add the variadicBufferCounts of the RecordBatch currently being built, if the message +// being encoded has any BinaryView or StringView arrays +static ArrowErrorCode ArrowIpcEncoderEncodeVariadicBufferCounts( + struct ArrowIpcEncoderPrivate* private, struct ArrowError* error) { + if (private->variadic_buffer_counts.size_bytes == 0) { + return NANOARROW_OK; + } + + FLATCC_RETURN_UNLESS_0( + RecordBatch_variadicBufferCounts_create( + &private->builder, (int64_t*)private->variadic_buffer_counts.data, + _ArrowInt64ToSizeT(private->variadic_buffer_counts.size_bytes) / + sizeof(int64_t)), + error); return NANOARROW_OK; } @@ -1074,6 +1128,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffers, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->nodes, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffer_views, 0, 0)); + NANOARROW_ASSERT_OK(ArrowBufferResize(&private->variadic_buffer_counts, 0, 0)); NANOARROW_RETURN_NOT_OK( ArrowIpcEncoderEncodeRecordBatchImpl(private, array_view, error)); NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeBuffers(encoder, buffer_encoder, error)); @@ -1088,6 +1143,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( builder, (struct ns(Buffer)*)private->buffers.data, _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeVariadicBufferCounts(private, error)); FLATCC_RETURN_UNLESS_0(Message_header_RecordBatch_end(builder), error); @@ -1142,6 +1198,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffers, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->nodes, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffer_views, 0, 0)); + NANOARROW_ASSERT_OK(ArrowBufferResize(&private->variadic_buffer_counts, 0, 0)); // The values array is a single top-level column: collect it, then descend into any // nested children. @@ -1160,6 +1217,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( builder, (struct ns(Buffer)*)private->buffers.data, _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeVariadicBufferCounts(private, error)); FLATCC_RETURN_UNLESS_0(DictionaryBatch_data_end(builder), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_isDelta_add(builder, is_delta ? 1 : 0), error); FLATCC_RETURN_UNLESS_0(Message_header_DictionaryBatch_end(builder), error); diff --git a/src/nanoarrow/ipc/files_test.cc b/src/nanoarrow/ipc/files_test.cc index 501b92495..69feeada5 100644 --- a/src/nanoarrow/ipc/files_test.cc +++ b/src/nanoarrow/ipc/files_test.cc @@ -619,7 +619,7 @@ INSTANTIATE_TEST_SUITE_P( TestFile::OK("cpp-21.0.0/generated_extension.stream"), TestFile::OK("cpp-21.0.0/generated_nested_dictionary.stream"), TestFile::NotSupported("cpp-21.0.0/generated_list_view.stream"), - TestFile::NotSupported("cpp-21.0.0/generated_binary_view.stream"), + TestFile::OK("cpp-21.0.0/generated_binary_view.stream"), TestFile::OK("cpp-21.0.0/generated_run_end_encoded.stream") // Comment to keep line from wrapping )); diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc index a67cfb718..e079418ac 100644 --- a/src/nanoarrow/ipc/writer_test.cc +++ b/src/nanoarrow/ipc/writer_test.cc @@ -316,19 +316,19 @@ TEST(NanoarrowIpcWriter, WriteDictionaryBatch) { EXPECT_EQ(p2->bytes_written, bytes_written); } -// Build a struct array with a single dictionary-encoded (int32 -> utf8) child. +// Build a struct array with a single dictionary-encoded (int32 -> utf8 by default) child. static void MakeDictionaryStructArray(struct ArrowArray* array, struct ArrowSchema* schema, - const char* value1 = "bar") { + const char* value1 = "bar", + enum ArrowType value_type = NANOARROW_TYPE_STRING) { ASSERT_EQ(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_STRUCT), NANOARROW_OK); ASSERT_EQ(ArrowSchemaAllocateChildren(schema, 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "dict_col"), NANOARROW_OK); ASSERT_EQ(ArrowSchemaAllocateDictionary(schema->children[0]), NANOARROW_OK); - ASSERT_EQ( - ArrowSchemaInitFromType(schema->children[0]->dictionary, NANOARROW_TYPE_STRING), - NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0]->dictionary, value_type), + NANOARROW_OK); ASSERT_EQ(ArrowArrayInitFromSchema(array, schema, nullptr), NANOARROW_OK); struct ArrowArray* indices = array->children[0]; @@ -556,10 +556,11 @@ static void ReadArrays(struct ArrowBuffer* buffer, } } -static std::vector StringValues(const struct ArrowArray* array) { +static std::vector StringValues( + const struct ArrowArray* array, enum ArrowType type = NANOARROW_TYPE_STRING) { std::vector values; nanoarrow::UniqueArrayView view; - ArrowArrayViewInitFromType(view.get(), NANOARROW_TYPE_STRING); + ArrowArrayViewInitFromType(view.get(), type); EXPECT_EQ(ArrowArrayViewSetArray(view.get(), array, nullptr), NANOARROW_OK); for (int64_t i = 0; i < view->length; i++) { struct ArrowStringView value = ArrowArrayViewGetStringUnsafe(view.get(), i); @@ -676,6 +677,62 @@ TEST(NanoarrowIpcWriter, EmitsChangedDictionary) { (std::vector{"foo", "baz"})); } +TEST(NanoarrowIpcWriter, RoundtripStringViewDictionary) { + // A dictionary of string views, one of which is too long to be inlined + struct ArrowError error; + const char* long_value = "a string view value that is not inlined"; + + for (bool as_file : {false, true}) { + SCOPED_TRACE(as_file ? "file" : "stream"); + nanoarrow::UniqueSchema schema; + std::vector arrays(1); + MakeDictionaryStructArray(arrays[0].get(), schema.get(), long_value, + NANOARROW_TYPE_STRING_VIEW); + + nanoarrow::UniqueBuffer output; + ASSERT_EQ(WriteArrays(schema.get(), &arrays, output.get(), as_file, &error), + NANOARROW_OK) + << error.message; + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) + auto arrow_input = std::make_shared( + arrow::Buffer::Wrap(output->data, output->size_bytes)); + std::shared_ptr arrow_batch; + if (as_file) { + auto maybe_arrow_reader = arrow::ipc::RecordBatchFileReader::Open(arrow_input); + ASSERT_TRUE(maybe_arrow_reader.ok()) << maybe_arrow_reader.status(); + auto maybe_batch = maybe_arrow_reader.ValueUnsafe()->ReadRecordBatch(0); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status(); + arrow_batch = maybe_batch.ValueUnsafe(); + } else { + auto maybe_arrow_reader = arrow::ipc::RecordBatchStreamReader::Open(arrow_input); + ASSERT_TRUE(maybe_arrow_reader.ok()) << maybe_arrow_reader.status(); + ASSERT_TRUE(maybe_arrow_reader.ValueUnsafe()->ReadNext(&arrow_batch).ok()); + } + ASSERT_NE(arrow_batch, nullptr); + ASSERT_TRUE(arrow_batch->ValidateFull().ok()); + auto arrow_dictionary = + std::static_pointer_cast(arrow_batch->column(0)); + auto arrow_values = + std::static_pointer_cast(arrow_dictionary->dictionary()); + ASSERT_EQ(arrow_values->length(), 2); + EXPECT_EQ(arrow_values->GetString(0), "foo"); + EXPECT_EQ(arrow_values->GetString(1), long_value); +#endif + + if (as_file) { + continue; + } + + std::vector roundtrip; + ReadArrays(output.get(), &roundtrip); + ASSERT_EQ(roundtrip.size(), 1); + EXPECT_EQ( + StringValues(roundtrip[0]->children[0]->dictionary, NANOARROW_TYPE_STRING_VIEW), + (std::vector{"foo", long_value})); + } +} + TEST(NanoarrowIpcWriter, RejectsChangedDictionaryInFile) { struct ArrowError error; nanoarrow::UniqueSchema schema;