diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 844ddc72c..e256d613c 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -47,8 +47,12 @@ struct ArrowIpcField { // array is scratch space for any intermediary allocations (i.e., it is never moved // to the user). struct ArrowArray* array; - // The cumulative number of buffers preceding this node. + // The cumulative number of buffers preceding this node, not counting the variadic + // buffers of any BinaryView or StringView nodes (which vary between messages). int64_t buffer_offset; + // The cumulative number of BinaryView or StringView nodes preceding this node (i.e., + // the number of RecordBatch.variadicBufferCounts entries that precede its own). + int64_t variadic_buffer_count_offset; // Dictionary identifier (or NANOARROW_IPC_NO_DICTIONARY_ID if this is not a // dictionary-encoded field) int64_t dictionary_id; @@ -75,10 +79,13 @@ struct ArrowIpcDecoderPrivate { // the ArrowArrayView/ArrowArray where the depth-first buffer/field walk should start. struct ArrowIpcField* fields; // The number of buffers that future RecordBatch messages must have to match the schema - // that has been set. + // that has been set (plus the number of variadic buffers in each message). int64_t n_buffers; // The number of union fields in the Schema. int64_t n_union_fields; + // The number of BinaryView and StringView fields in the Schema (i.e., the number of + // variadicBufferCounts entries that future RecordBatch messages must have). + int64_t n_variadic_buffer_counts; // A pointer to the last flatbuffers message. const void* last_message; // The custom_metadata of the last flatbuffers Message (i.e., Message.custom_metadata, @@ -1246,15 +1253,13 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema* schema, ns(Field_table_t) case ns(Type_FixedSizeBinary): return ArrowIpcDecoderSetTypeFixedSizeBinary(schema, type, error); case ns(Type_BinaryView): - ArrowErrorSet(error, "BinaryView not yet supported in IPC reader"); - return ENOTSUP; + return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_BINARY_VIEW, error); case ns(Type_Utf8): return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_STRING, error); case ns(Type_LargeUtf8): return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_LARGE_STRING, error); case ns(Type_Utf8View): - ArrowErrorSet(error, "Utf8View not yet supported in IPC reader"); - return ENOTSUP; + return ArrowIpcDecoderSetTypeSimple(schema, NANOARROW_TYPE_STRING_VIEW, error); case ns(Type_Date): return ArrowIpcDecoderSetTypeDate(schema, type, error); case ns(Type_Time): @@ -1558,6 +1563,32 @@ static int ArrowIpcDecoderDecodeRecordBatchHeader(struct ArrowIpcDecoder* decode n_expected_buffers += private_data->n_union_fields; } + // Each BinaryView or StringView field has a number of variadic buffers that is given + // by its entry in variadicBufferCounts + flatbuffers_int64_vec_t variadic_buffer_counts = + ns(RecordBatch_variadicBufferCounts(batch)); + int64_t n_variadic_buffer_counts = flatbuffers_int64_vec_len(variadic_buffer_counts); + if (n_variadic_buffer_counts != private_data->n_variadic_buffer_counts) { + ArrowErrorSet(error, + "Expected %" PRId64 + " variadic buffer counts in message but found %" PRId64, + private_data->n_variadic_buffer_counts, n_variadic_buffer_counts); + return EINVAL; + } + + for (int64_t i = 0; i < n_variadic_buffer_counts; i++) { + int64_t count = flatbuffers_int64_vec_at(variadic_buffer_counts, (size_t)i); + if (count < 0 || count > n_buffers) { + ArrowErrorSet(error, + "Expected variadic buffer count between 0 and %" PRId64 + " but found %" PRId64, + n_buffers, count); + return EINVAL; + } + + n_expected_buffers += count; + } + if ((n_buffers + 1) != n_expected_buffers) { ArrowErrorSet(error, "Expected %" PRId64 " buffers in message but found %" PRId64, n_expected_buffers - 1, n_buffers); @@ -2015,11 +2046,13 @@ static ArrowErrorCode ArrowIpcDecoderInitFields( struct ArrowIpcField* fields, const struct ArrowSchema* schema, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowArrayView* array_view, struct ArrowArray* array, int64_t* n_fields, - int64_t* n_buffers, int64_t* n_union_fields, struct ArrowError* error) { + int64_t* n_buffers, int64_t* n_union_fields, int64_t* n_variadic_buffer_counts, + struct ArrowError* error) { struct ArrowIpcField* field = fields + (*n_fields); field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; + field->variadic_buffer_count_offset = *n_variadic_buffer_counts; field->dictionary_id = NANOARROW_IPC_NO_DICTIONARY_ID; if (schema->dictionary != NULL) { @@ -2056,13 +2089,16 @@ static ArrowErrorCode ArrowIpcDecoderInitFields( } *n_union_fields += array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION || array_view->storage_type == NANOARROW_TYPE_DENSE_UNION; + *n_variadic_buffer_counts += array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; *n_fields += 1; for (int64_t i = 0; i < array_view->n_children; i++) { NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( fields, schema->children[i], dictionary_encodings, array_view->children[i], - array->children[i], n_fields, n_buffers, n_union_fields, error)); + array->children[i], n_fields, n_buffers, n_union_fields, n_variadic_buffer_counts, + error)); } return NANOARROW_OK; @@ -2079,6 +2115,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( private_data->n_buffers = 0; private_data->n_fields = 0; private_data->n_union_fields = 0; + private_data->n_variadic_buffer_counts = 0; ArrowArrayViewReset(&private_data->array_view); if (private_data->array.release != NULL) { ArrowArrayRelease(&private_data->array); @@ -2116,7 +2153,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( private_data->fields, schema, dictionary_encodings, &private_data->array_view, &private_data->array, &field_i, &private_data->n_buffers, - &private_data->n_union_fields, error)); + &private_data->n_union_fields, &private_data->n_variadic_buffer_counts, error)); return NANOARROW_OK; } @@ -2348,6 +2385,12 @@ static int ArrowIpcDecoderSwapEndian(struct ArrowIpcBufferSource* src, case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_BINARY: return NANOARROW_OK; + case NANOARROW_TYPE_BINARY_VIEW: + case NANOARROW_TYPE_STRING_VIEW: + ArrowErrorSet(error, + "Endian swapping of binary view and string view arrays is not " + "supported"); + return ENOTSUP; default: break; } @@ -2457,6 +2500,8 @@ struct ArrowIpcArraySetter { int64_t field_i; ns(Buffer_vec_t) buffers; int64_t buffer_i; + flatbuffers_int64_vec_t variadic_buffer_counts; + int64_t variadic_buffer_count_i; struct ArrowIpcBufferSource src; struct ArrowIpcBufferFactory factory; enum ArrowIpcMetadataVersion version; @@ -2497,28 +2542,122 @@ static int ArrowIpcDecoderMakeBuffer(struct ArrowIpcArraySetter* setter, int64_t return NANOARROW_OK; } +// Make the next buffer in the message, using out if any allocation has to occur +static int ArrowIpcDecoderMakeNextBuffer(struct ArrowIpcArraySetter* setter, + struct ArrowBufferView* out_view, + struct ArrowBuffer* out, + struct ArrowError* error) { + ns(Buffer_struct_t) buffer = + ns(Buffer_vec_at(setter->buffers, (size_t)setter->buffer_i)); + int64_t buffer_offset = ns(Buffer_offset(buffer)); + int64_t buffer_length = ns(Buffer_length(buffer)); + setter->buffer_i += 1; + + // Attempt to re-use any previous allocation unless this buffer is + // wrapping a custom allocator. + if (out->allocator.private_data != NULL) { + ArrowBufferReset(out); + } else { + out->size_bytes = 0; + } + + return ArrowIpcDecoderMakeBuffer(setter, buffer_offset, buffer_length, out_view, out, + error); +} + +// The variadic buffers of a BinaryView or StringView array follow its validity and views +// buffers in the message. The scratch array provides an ArrowBuffer for each of them that +// is used if any allocation has to occur, and storage for the buffer pointers and sizes +// that the array view refers to (as they would be for an ArrowArray). +static int ArrowIpcDecoderWalkSetVariadicBuffers(struct ArrowIpcArraySetter* setter, + struct ArrowArrayView* array_view, + struct ArrowArray* array, + struct ArrowError* error) { + // The count was checked against the number of buffers in the message when the + // header was decoded + int64_t n_variadic_buffers = flatbuffers_int64_vec_at( + setter->variadic_buffer_counts, (size_t)setter->variadic_buffer_count_i); + setter->variadic_buffer_count_i += 1; + + int32_t n_scratch_buffers = ArrowArrayVariadicBufferCount(array); + if (n_variadic_buffers > n_scratch_buffers) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAddVariadicBuffers(array, + (int32_t)(n_variadic_buffers - n_scratch_buffers)), + error); + } + + struct ArrowBuffer* sizes = ArrowArrayBuffer(array, array->n_buffers - 1); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferResize(sizes, n_variadic_buffers * (int64_t)sizeof(int64_t), 0), error); + + // Variadic buffers contain bytes, which never need endian swapping + setter->src.data_type = NANOARROW_TYPE_BINARY; + setter->src.element_size_bits = 8; + + for (int64_t i = 0; i < n_variadic_buffers; i++) { + int64_t buffer_i = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i; + struct ArrowBufferView view; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderMakeNextBuffer( + setter, &view, ArrowArrayBuffer(array, buffer_i), error)); + array->buffers[buffer_i] = view.data.data; + ((int64_t*)sizes->data)[i] = view.size_bytes; + } + + array_view->n_variadic_buffers = (int32_t)n_variadic_buffers; + array_view->variadic_buffers = array->buffers + NANOARROW_BINARY_VIEW_FIXED_BUFFERS; + array_view->variadic_buffer_sizes = (int64_t*)sizes->data; + return NANOARROW_OK; +} + +// If the scratch buffer was used, move it to the final array. Otherwise, copy the view. +static int ArrowIpcDecoderGetBuffer(struct ArrowBufferView view, + struct ArrowBuffer* scratch_buffer, + struct ArrowBuffer* buffer_out, + struct ArrowError* error) { + if (scratch_buffer->size_bytes == 0) { + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendBufferView(buffer_out, view)); + } else if (scratch_buffer->data == view.data.as_uint8) { + ArrowBufferMove(scratch_buffer, buffer_out); + } else { + ArrowErrorSet( + error, + "Internal: scratch buffer was used but doesn't point to the same data as view"); + return EINVAL; + } + + return NANOARROW_OK; +} + static int ArrowIpcDecoderWalkGetArray(struct ArrowArrayView* array_view, struct ArrowArray* array, struct ArrowArray* out, struct ArrowError* error) { out->length = array_view->length; out->null_count = array_view->null_count; - for (int64_t i = 0; i < array->n_buffers; i++) { - struct ArrowBufferView view = array_view->buffer_views[i]; - struct ArrowBuffer* scratch_buffer = ArrowArrayBuffer(array, i); - struct ArrowBuffer* buffer_out = ArrowArrayBuffer(out, i); - - // If the scratch buffer was used, move it to the final array. Otherwise, - // copy the view. - if (scratch_buffer->size_bytes == 0) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendBufferView(buffer_out, view)); - } else if (scratch_buffer->data == view.data.as_uint8) { - ArrowBufferMove(scratch_buffer, buffer_out); - } else { - ArrowErrorSet( - error, - "Internal: scratch buffer was used but doesn't point to the same data as view"); - return EINVAL; + int is_view = array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; + + // The variadic buffers of a BinaryView or StringView array are handled below (the + // final buffer with their sizes is populated when the array is finished) + int64_t n_buffers = is_view ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS : array->n_buffers; + for (int64_t i = 0; i < n_buffers; i++) { + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderGetBuffer(array_view->buffer_views[i], + ArrowArrayBuffer(array, i), + ArrowArrayBuffer(out, i), error)); + } + + if (is_view && array_view->n_variadic_buffers > 0) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowArrayAddVariadicBuffers(out, array_view->n_variadic_buffers), error); + for (int32_t i = 0; i < array_view->n_variadic_buffers; i++) { + struct ArrowBufferView view; + view.data.data = array_view->variadic_buffers[i]; + view.size_bytes = array_view->variadic_buffer_sizes[i]; + int64_t buffer_i = NANOARROW_BINARY_VIEW_FIXED_BUFFERS + i; + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderGetBuffer(view, ArrowArrayBuffer(array, buffer_i), + ArrowArrayBuffer(out, buffer_i), error)); } } @@ -2616,29 +2755,17 @@ static int ArrowIpcDecoderWalkSetArrayView(struct ArrowIpcDecoder* decoder, break; } - ns(Buffer_struct_t) buffer = - ns(Buffer_vec_at(setter->buffers, (size_t)setter->buffer_i)); - int64_t buffer_offset = ns(Buffer_offset(buffer)); - int64_t buffer_length = ns(Buffer_length(buffer)); - setter->buffer_i += 1; - - // Provide a buffer that will be used if any allocation has to occur - struct ArrowBuffer* buffer_dst = ArrowArrayBuffer(array, i); - - // Attempt to re-use any previous allocation unless this buffer is - // wrapping a custom allocator. - if (buffer_dst->allocator.private_data != NULL) { - ArrowBufferReset(buffer_dst); - } else { - buffer_dst->size_bytes = 0; - } - setter->src.data_type = array_view->layout.buffer_data_type[i]; setter->src.element_size_bits = array_view->layout.element_size_bits[i]; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderMakeNextBuffer( + setter, &array_view->buffer_views[i], ArrowArrayBuffer(array, i), error)); + } + + if (array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW) { NANOARROW_RETURN_NOT_OK( - ArrowIpcDecoderMakeBuffer(setter, buffer_offset, buffer_length, - &array_view->buffer_views[i], buffer_dst, error)); + ArrowIpcDecoderWalkSetVariadicBuffers(setter, array_view, array, error)); } for (int64_t i = 0; i < array_view->n_children; i++) { @@ -2714,12 +2841,20 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayViewInternal( setter.field_i = field_i; setter.buffers = ns(RecordBatch_buffers(batch)); setter.buffer_i = root->buffer_offset - 1; + setter.variadic_buffer_counts = ns(RecordBatch_variadicBufferCounts(batch)); + setter.variadic_buffer_count_i = root->variadic_buffer_count_offset; setter.factory = factory; setter.src.codec = decoder->codec; setter.src.swap_endian = ArrowIpcDecoderNeedsSwapEndian(decoder); setter.version = decoder->metadata_version; setter.dictionaries = dictionaries; + // The variadic buffers of any BinaryView or StringView fields before this one also + // precede its buffers + for (int64_t i = 0; i < root->variadic_buffer_count_offset; i++) { + setter.buffer_i += flatbuffers_int64_vec_at(setter.variadic_buffer_counts, (size_t)i); + } + // If we are going to need a decompressor here, ensure the default one is // initialized. if (setter.src.codec != NANOARROW_IPC_COMPRESSION_TYPE_NONE) { diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index a2c68c9c5..5605ab3a5 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #endif @@ -55,6 +56,7 @@ struct ArrowIpcField { struct ArrowArrayView* array_view; struct ArrowArray* array; int64_t buffer_offset; + int64_t variadic_buffer_count_offset; int64_t dictionary_id; }; @@ -1641,6 +1643,16 @@ void AssertArrayViewIdentical(const struct ArrowArrayView* actual, } } + ASSERT_EQ(actual->n_variadic_buffers, expected->n_variadic_buffers); + for (int32_t i = 0; i < expected->n_variadic_buffers; i++) { + ASSERT_EQ(actual->variadic_buffer_sizes[i], expected->variadic_buffer_sizes[i]); + if (expected->variadic_buffer_sizes[i] != 0) { + ASSERT_EQ(memcmp(actual->variadic_buffers[i], expected->variadic_buffers[i], + expected->variadic_buffer_sizes[i]), + 0); + } + } + ASSERT_EQ(actual->n_children, expected->n_children); for (int i = 0; i < actual->n_children; i++) { AssertArrayViewIdentical(actual->children[i], expected->children[i]); @@ -2077,6 +2089,11 @@ INSTANTIATE_TEST_SUITE_P( arrow::run_end_encoded(arrow::int16(), arrow::utf8()), arrow::run_end_encoded(arrow::int32(), arrow::int64()), arrow::run_end_encoded(arrow::int64(), arrow::float64()), + // Binary view and string view + arrow::binary_view(), arrow::utf8_view(), + arrow::struct_({arrow::field("col1", arrow::utf8_view()), + arrow::field("col2", arrow::int32()), + arrow::field("col3", arrow::binary_view())}), // Type with nested metadata arrow::list(arrow::field("some_custom_name", arrow::int32(), @@ -2092,6 +2109,8 @@ INSTANTIATE_TEST_SUITE_P( arrow::dictionary(arrow::uint64(), arrow::utf8()), // Ordered dictionary encoding arrow::dictionary(arrow::int32(), arrow::utf8(), true), + // Dictionary-encoded string view + arrow::dictionary(arrow::int32(), arrow::utf8_view()), // Extension type arrow::extension::uuid(), // Extension type with dictionary as the storage type @@ -2821,3 +2840,437 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeEmptyArrayOffsets) { EXPECT_EQ(first_offset, 0); } } + +// A value for the binary view and string view arrays in the tests below. Most are +// longer than the 12 bytes that a view holds inline, so they are stored in variadic +// buffers, and some are empty or inlined. +static std::string ViewTestValue(int64_t i, int64_t max_size) { + std::string value(static_cast((i * 37) % max_size), ' '); + for (size_t j = 0; j < value.size(); j++) { + value[j] = static_cast('a' + (i + static_cast(j)) % 26); + } + return value; +} + +// A struct, i: int32, sv: string_view> array with +// nulls in the binary view and string view columns. Values of up to 400 bytes give +// those arrays several variadic buffers (nanoarrow starts a new one every 32 KB). +static void MakeViewStructArray(struct ArrowSchema* schema, struct ArrowArray* array, + int64_t length) { + ArrowSchemaInit(schema); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema, 4), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_BINARY_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "bv"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_LIST), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[1], "l"), NANOARROW_OK); + ASSERT_EQ( + ArrowSchemaSetType(schema->children[1]->children[0], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[2], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[2], "i"), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[3], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema->children[3], "sv"), NANOARROW_OK); + + ASSERT_EQ(ArrowArrayInitFromSchema(array, schema, nullptr), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(array), NANOARROW_OK); + for (int64_t i = 0; i < length; i++) { + if (i % 7 == 3) { + ASSERT_EQ(ArrowArrayAppendNull(array->children[0], 1), NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendNull(array->children[3], 1), NANOARROW_OK); + } else { + std::string bv = ViewTestValue(i, 400); + std::string sv = ViewTestValue(i + 1, 400); + ASSERT_EQ(ArrowArrayAppendString(array->children[0], ArrowCharView(bv.c_str())), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayAppendString(array->children[3], ArrowCharView(sv.c_str())), + NANOARROW_OK); + } + + for (int64_t j = 0; j < i % 4; j++) { + std::string item = ViewTestValue(i * 3 + j, 400); + ASSERT_EQ(ArrowArrayAppendString(array->children[1]->children[0], + ArrowCharView(item.c_str())), + NANOARROW_OK); + } + ASSERT_EQ(ArrowArrayFinishElement(array->children[1]), NANOARROW_OK); + + ASSERT_EQ(ArrowArrayAppendInt(array->children[2], i), NANOARROW_OK); + ASSERT_EQ(ArrowArrayFinishElement(array), NANOARROW_OK); + } + + ASSERT_EQ(ArrowArrayFinishBuildingDefault(array, nullptr), NANOARROW_OK); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcViewArrayRoundtrip) { + struct ArrowError error; + std::vector codecs = { + NANOARROW_IPC_COMPRESSION_TYPE_NONE}; + if (ArrowIpcGetLZ4CompressionFunction() != nullptr) { + codecs.push_back(NANOARROW_IPC_COMPRESSION_TYPE_LZ4_FRAME); + } + if (ArrowIpcGetZstdCompressionFunction() != nullptr) { + codecs.push_back(NANOARROW_IPC_COMPRESSION_TYPE_ZSTD); + } + + for (auto codec : codecs) { + // One decoder reads batches whose arrays have more, then fewer variadic buffers than + // the batch before. (Lengths are multiples of 8 so that AssertArrayViewIdentical() + // doesn't compare the unset bits at the end of a validity bitmap.) + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + bool decoder_has_schema = false; + + for (int64_t length : {400, 1000, 0, 400}) { + SCOPED_TRACE(::testing::Message() << "codec " << codec << ", length " << length); + nanoarrow::UniqueSchema schema; + nanoarrow::UniqueArray array; + ASSERT_NO_FATAL_FAILURE(MakeViewStructArray(schema.get(), array.get(), length)); + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error), + NANOARROW_OK) + << error.message; + + struct ArrowArrayView* bv = array_view->children[0]; + struct ArrowArrayView* l_sv = array_view->children[1]->children[0]; + struct ArrowArrayView* sv = array_view->children[3]; + if (length > 0) { + ASSERT_GT(bv->n_variadic_buffers, 1); + ASSERT_GT(l_sv->n_variadic_buffers, 1); + ASSERT_GT(sv->n_variadic_buffers, 1); + } + + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + ASSERT_EQ( + ArrowIpcEncoderSetCompression(encoder.get(), codec, + NANOARROW_IPC_COMPRESSION_LEVEL_DEFAULT, &error), + NANOARROW_OK) + << error.message; + nanoarrow::UniqueBuffer message, body; + ASSERT_EQ(ArrowIpcEncoderEncodeSimpleRecordBatch(encoder.get(), array_view.get(), + body.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ(ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, + message.get()), + NANOARROW_OK); + + // The message has a variadicBufferCounts entry for each view array in field order + ns(Message_table_t) fb_message = ns(Message_as_root(message->data + 8)); + ns(RecordBatch_table_t) fb_batch = + (ns(RecordBatch_table_t))ns(Message_header(fb_message)); + flatbuffers_int64_vec_t counts = ns(RecordBatch_variadicBufferCounts(fb_batch)); + ASSERT_EQ(flatbuffers_int64_vec_len(counts), 3); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 0), bv->n_variadic_buffers); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 1), l_sv->n_variadic_buffers); + EXPECT_EQ(flatbuffers_int64_vec_at(counts, 2), sv->n_variadic_buffers); + + if (!decoder_has_schema) { + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), + NANOARROW_OK) + << error.message; + decoder_has_schema = true; + } + ASSERT_EQ(ArrowIpcDecoderDecodeHeader( + decoder.get(), {{message->data}, message->size_bytes}, &error), + NANOARROW_OK) + << error.message; + struct ArrowBufferView body_view = {{body->data}, body->size_bytes}; + + // Decode the whole batch and each field on its own (in the depth-first order of + // field indices), whose buffers follow the variadic buffers of any view arrays + // before it + struct ArrowArrayView* decoded; + ASSERT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, -1, &decoded, &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE(AssertArrayViewIdentical(decoded, array_view.get())); + const struct ArrowArrayView* fields[] = {bv, array_view->children[1], l_sv, + array_view->children[2], sv}; + for (int64_t i = 0; i < 5; i++) { + SCOPED_TRACE(::testing::Message() << "field " << i); + ASSERT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, i, &decoded, &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE(AssertArrayViewIdentical(decoded, fields[i])); + } + + // Decode arrays that own a copy of the body or share it. (The view of an empty + // array has no offsets for the list column, so only compare a non-empty one.) + if (length == 0) { + continue; + } + + nanoarrow::UniqueBuffer body_copy; + ASSERT_EQ(ArrowBufferAppend(body_copy.get(), body->data, body->size_bytes), + NANOARROW_OK); + nanoarrow::UniqueBuffer shared_body; + ASSERT_EQ(ArrowIpcSharedBufferInit(shared_body.get(), body_copy.get()), + NANOARROW_OK); + for (bool shared : {false, true}) { + SCOPED_TRACE(shared ? "shared body" : "copied body"); + nanoarrow::UniqueArray decoded_array; + if (shared) { + ASSERT_EQ(ArrowIpcDecoderDecodeArrayFromShared( + decoder.get(), shared_body.get(), -1, decoded_array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } else { + ASSERT_EQ(ArrowIpcDecoderDecodeArray(decoder.get(), body_view, -1, + decoded_array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } + + nanoarrow::UniqueArrayView decoded_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(decoded_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(decoded_view.get(), decoded_array.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_NO_FATAL_FAILURE( + AssertArrayViewIdentical(decoded_view.get(), array_view.get())); + } + + // As in Arrow C++, view arrays can't be read from a message of the other + // endianness +#if defined(__BIG_ENDIAN__) + ASSERT_EQ( + ArrowIpcDecoderSetEndianness(decoder.get(), NANOARROW_IPC_ENDIANNESS_LITTLE), + NANOARROW_OK); +#else + ASSERT_EQ(ArrowIpcDecoderSetEndianness(decoder.get(), NANOARROW_IPC_ENDIANNESS_BIG), + NANOARROW_OK); +#endif + ASSERT_EQ(ArrowIpcDecoderDecodeHeader( + decoder.get(), {{message->data}, message->size_bytes}, &error), + NANOARROW_OK) + << error.message; + EXPECT_EQ( + ArrowIpcDecoderDecodeArrayView(decoder.get(), body_view, 0, &decoded, &error), + ENOTSUP); + EXPECT_STREQ(error.message, + "Endian swapping of binary view and string view arrays is not " + "supported"); + ASSERT_EQ(ArrowIpcDecoderSetEndianness(decoder.get(), + NANOARROW_IPC_ENDIANNESS_UNINITIALIZED), + NANOARROW_OK); + } + } +} + +// Builds an encapsulated RecordBatch message with one field node, n_buffers empty +// buffers, and the given variadicBufferCounts (omitted if counts is nullptr) +static void MakeRecordBatchWithVariadicBufferCounts(int64_t n_buffers, + const std::vector* counts, + struct ArrowBuffer* out) { + flatcc_builder_t builder; + flatcc_builder_init(&builder); + + ASSERT_EQ(ns(Message_start_as_root(&builder)), 0); + ASSERT_EQ(ns(Message_version_add(&builder, ns(MetadataVersion_V5))), 0); + ASSERT_EQ(ns(Message_header_RecordBatch_start(&builder)), 0); + struct ns(FieldNode) node = {0, 0}; + ASSERT_EQ(ns(RecordBatch_nodes_create(&builder, &node, 1)), 0); + std::vector buffers(n_buffers); + ASSERT_EQ(ns(RecordBatch_buffers_create(&builder, buffers.data(), buffers.size())), 0); + if (counts != nullptr) { + ASSERT_EQ(ns(RecordBatch_variadicBufferCounts_create(&builder, counts->data(), + counts->size())), + 0); + } + ASSERT_EQ(ns(Message_header_RecordBatch_end(&builder)), 0); + ASSERT_NE(ns(Message_end_as_root(&builder)), 0); + + ASSERT_NO_FATAL_FAILURE(FinishEncapsulatedMessage(&builder, out)); + flatcc_builder_clear(&builder); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeVariadicBufferCounts) { + struct ArrowError error; + + // A string view column has a validity buffer, a views buffer, and one buffer for + // each variadic buffer + nanoarrow::UniqueSchema schema; + ArrowSchemaInit(schema.get()); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema.get(), 1), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_STRING_VIEW), + NANOARROW_OK); + + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + + struct Case { + int64_t n_buffers; + bool has_counts; + std::vector counts; + std::string expected_error; + }; + const std::vector cases = { + {2, false, {}, "Expected 1 variadic buffer counts in message but found 0"}, + {2, true, {}, "Expected 1 variadic buffer counts in message but found 0"}, + {2, true, {0, 0}, "Expected 1 variadic buffer counts in message but found 2"}, + {2, true, {-1}, "Expected variadic buffer count between 0 and 2 but found -1"}, + {2, + true, + {INT64_MAX}, + "Expected variadic buffer count between 0 and 2 but found 9223372036854775807"}, + {2, true, {1}, "Expected 3 buffers in message but found 2"}, + {3, true, {0}, "Expected 2 buffers in message but found 3"}, + {2, true, {0}, ""}, + {4, true, {2}, ""}}; + + for (const auto& c : cases) { + SCOPED_TRACE(::testing::Message() << "buffers=" << c.n_buffers << ", counts=" + << ::testing::PrintToString(c.counts)); + nanoarrow::UniqueBuffer message; + ASSERT_NO_FATAL_FAILURE(MakeRecordBatchWithVariadicBufferCounts( + c.n_buffers, c.has_counts ? &c.counts : nullptr, message.get())); + struct ArrowBufferView data = {{message->data}, message->size_bytes}; + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + + if (!c.expected_error.empty()) { + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), EINVAL); + EXPECT_EQ(std::string(error.message), c.expected_error); + continue; + } + + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + struct ArrowArrayView* decoded; + ASSERT_EQ(ArrowIpcDecoderDecodeArrayView(decoder.get(), {{nullptr}, 0}, -1, &decoded, + &error), + NANOARROW_OK) + << error.message; + EXPECT_EQ(decoded->children[0]->n_variadic_buffers, c.counts[0]); + } + + // A schema without view fields does not expect any variadic buffer counts + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + std::vector counts = {0}; + nanoarrow::UniqueBuffer message; + ASSERT_NO_FATAL_FAILURE( + MakeRecordBatchWithVariadicBufferCounts(3, &counts, message.get())); + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), + {{message->data}, message->size_bytes}, &error), + EINVAL); + EXPECT_STREQ(error.message, "Expected 0 variadic buffer counts in message but found 1"); +} + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) +TEST(NanoarrowIpcTest, NanoarrowIpcViewArrayArrowInterop) { + struct ArrowError error; + constexpr int64_t kLength = 200; + + // Values of up to 40 bytes in blocks of 64 bytes give the binary view array a + // variadic buffer for every one or two values that are not inlined. The string view + // array is a slice. + arrow::BinaryViewBuilder bv_builder; + bv_builder.SetBlockSize(64); + arrow::StringViewBuilder sv_builder; + for (int64_t i = 0; i < kLength + 10; i++) { + if (i % 5 == 1) { + ASSERT_TRUE(bv_builder.AppendNull().ok()); + ASSERT_TRUE(sv_builder.AppendNull().ok()); + } else { + ASSERT_TRUE(bv_builder.Append(ViewTestValue(i, 41)).ok()); + ASSERT_TRUE(sv_builder.Append(ViewTestValue(i + 1, 41)).ok()); + } + } + auto bv = bv_builder.Finish().ValueOrDie()->Slice(0, kLength); + auto sv = sv_builder.Finish().ValueOrDie()->Slice(10, kLength); + ASSERT_GT(bv->data()->buffers.size(), static_cast(20)); + + auto arrow_schema = arrow::schema( + {arrow::field("bv", arrow::binary_view()), arrow::field("sv", arrow::utf8_view())}); + auto batch = arrow::RecordBatch::Make(arrow_schema, kLength, {bv, sv}); + nanoarrow::UniqueSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, schema.get()).ok()); + + std::vector codecs = {arrow::Compression::UNCOMPRESSED}; + if (ArrowIpcGetLZ4DecompressionFunction() != nullptr && + arrow::util::Codec::IsAvailable(arrow::Compression::LZ4_FRAME)) { + codecs.push_back(arrow::Compression::LZ4_FRAME); + } + if (ArrowIpcGetZstdDecompressionFunction() != nullptr && + arrow::util::Codec::IsAvailable(arrow::Compression::ZSTD)) { + codecs.push_back(arrow::Compression::ZSTD); + } + + for (auto codec : codecs) { + SCOPED_TRACE(arrow::util::Codec::GetCodecAsString(codec)); + auto options = arrow::ipc::IpcWriteOptions::Defaults(); + if (codec != arrow::Compression::UNCOMPRESSED) { + options.codec = arrow::util::Codec::Create(codec).ValueOrDie(); + } + auto maybe_serialized = arrow::ipc::SerializeRecordBatch(*batch, options); + ASSERT_TRUE(maybe_serialized.ok()) << maybe_serialized.status(); + auto serialized = maybe_serialized.ValueUnsafe(); + + // nanoarrow reads the message that Arrow C++ wrote + nanoarrow::ipc::UniqueDecoder decoder; + ASSERT_EQ(ArrowIpcDecoderInit(decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetSchema(decoder.get(), schema.get(), &error), NANOARROW_OK) + << error.message; + struct ArrowBufferView data = {{serialized->data()}, serialized->size()}; + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), data, &error), NANOARROW_OK) + << error.message; + data.data.as_uint8 += decoder->header_size_bytes; + data.size_bytes -= decoder->header_size_bytes; + nanoarrow::UniqueArray array; + ASSERT_EQ(ArrowIpcDecoderDecodeArray(decoder.get(), data, -1, array.get(), + NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + + // Arrow C++ reads the message that nanoarrow writes from what it read + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), &error), NANOARROW_OK) + << error.message; + nanoarrow::ipc::UniqueEncoder encoder; + ASSERT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + nanoarrow::UniqueBuffer message, body; + ASSERT_EQ(ArrowIpcEncoderEncodeSimpleRecordBatch(encoder.get(), array_view.get(), + body.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, message.get()), + NANOARROW_OK); + ASSERT_EQ(ArrowBufferAppend(message.get(), body->data, body->size_bytes), + NANOARROW_OK); + + arrow::io::BufferReader message_reader( + arrow::Buffer::Wrap(message->data, message->size_bytes)); + auto maybe_message = arrow::ipc::ReadMessage(&message_reader); + ASSERT_TRUE(maybe_message.ok()) << maybe_message.status(); + arrow::ipc::DictionaryMemo memo; + auto maybe_roundtripped = + arrow::ipc::ReadRecordBatch(*maybe_message.ValueUnsafe(), arrow_schema, &memo, + arrow::ipc::IpcReadOptions::Defaults()); + ASSERT_TRUE(maybe_roundtripped.ok()) << maybe_roundtripped.status(); + ASSERT_TRUE(maybe_roundtripped.ValueUnsafe()->ValidateFull().ok()); + EXPECT_TRUE(maybe_roundtripped.ValueUnsafe()->Equals(*batch)); + + auto maybe_batch = arrow::ImportRecordBatch(array.get(), arrow_schema); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status(); + ASSERT_TRUE(maybe_batch.ValueUnsafe()->ValidateFull().ok()); + EXPECT_TRUE(maybe_batch.ValueUnsafe()->Equals(*batch)); + } +} +#endif diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index 5b78f315a..b54799732 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -58,6 +58,9 @@ struct ArrowIpcEncoderPrivate { int has_compressed_body; // Views of the body buffers of the message being encoded, in body order struct ArrowBuffer buffer_views; + // The number of variadic buffers of each BinaryView or StringView array in the + // message being encoded, in the order of their field nodes + struct ArrowBuffer variadic_buffer_counts; // Compressed copies of those buffers (reused between messages) struct ArrowBuffer* compressed_buffers; int64_t n_compressed_buffers; @@ -85,6 +88,7 @@ ArrowErrorCode ArrowIpcEncoderInit(struct ArrowIpcEncoder* encoder) { private->compressor.release = NULL; private->has_compressed_body = 0; ArrowBufferInit(&private->buffer_views); + ArrowBufferInit(&private->variadic_buffer_counts); private->compressed_buffers = NULL; private->n_compressed_buffers = 0; return NANOARROW_OK; @@ -104,6 +108,7 @@ void ArrowIpcEncoderReset(struct ArrowIpcEncoder* encoder) { private->compressor.release(&private->compressor); } ArrowBufferReset(&private->buffer_views); + ArrowBufferReset(&private->variadic_buffer_counts); for (int64_t i = 0; i < private->n_compressed_buffers; i++) { ArrowBufferReset(&private->compressed_buffers[i]); } @@ -358,6 +363,14 @@ static ArrowErrorCode ArrowIpcEncodeFieldType(flatcc_builder_t* builder, FLATCC_RETURN_UNLESS_0(Field_type_LargeBinary_create(builder), error); return NANOARROW_OK; + case NANOARROW_TYPE_STRING_VIEW: + FLATCC_RETURN_UNLESS_0(Field_type_Utf8View_create(builder), error); + return NANOARROW_OK; + + case NANOARROW_TYPE_BINARY_VIEW: + FLATCC_RETURN_UNLESS_0(Field_type_BinaryView_create(builder), error); + return NANOARROW_OK; + case NANOARROW_TYPE_DATE32: FLATCC_RETURN_UNLESS_0(Field_type_Date_create(builder, ns(DateUnit_DAY)), error); return NANOARROW_OK; @@ -996,7 +1009,14 @@ static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* struct ns(FieldNode) node = {array_view->length, null_count}; NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowBufferAppend(&private->nodes, &node, sizeof(node)), error); - for (int64_t b = 0; b < array_view->array->n_buffers; ++b) { + + // The variadic buffers of a BinaryView or StringView array are collected below. The + // final buffer of such an array (the sizes of the variadic buffers) is not written. + int is_view = array_view->storage_type == NANOARROW_TYPE_BINARY_VIEW || + array_view->storage_type == NANOARROW_TYPE_STRING_VIEW; + int64_t n_buffers = + is_view ? NANOARROW_BINARY_VIEW_FIXED_BUFFERS : array_view->array->n_buffers; + for (int64_t b = 0; b < n_buffers; ++b) { struct ArrowBufferView buffer_view = array_view->buffer_views[b]; // An offsets buffer holds length + 1 offsets, so even an empty array has one. @@ -1014,6 +1034,40 @@ static ArrowErrorCode ArrowIpcEncoderCollectArray(struct ArrowIpcEncoderPrivate* sizeof(struct ArrowBufferView)), error); } + + if (is_view) { + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppendInt64(&private->variadic_buffer_counts, + array_view->n_variadic_buffers), + error); + for (int32_t i = 0; i < array_view->n_variadic_buffers; i++) { + struct ArrowBufferView buffer_view; + buffer_view.data.data = array_view->variadic_buffers[i]; + buffer_view.size_bytes = array_view->variadic_buffer_sizes[i]; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowBufferAppend(&private->buffer_views, &buffer_view, + sizeof(struct ArrowBufferView)), + error); + } + } + + return NANOARROW_OK; +} + +// Add the variadicBufferCounts of the RecordBatch currently being built, if the message +// being encoded has any BinaryView or StringView arrays +static ArrowErrorCode ArrowIpcEncoderEncodeVariadicBufferCounts( + struct ArrowIpcEncoderPrivate* private, struct ArrowError* error) { + if (private->variadic_buffer_counts.size_bytes == 0) { + return NANOARROW_OK; + } + + FLATCC_RETURN_UNLESS_0( + RecordBatch_variadicBufferCounts_create( + &private->builder, (int64_t*)private->variadic_buffer_counts.data, + _ArrowInt64ToSizeT(private->variadic_buffer_counts.size_bytes) / + sizeof(int64_t)), + error); return NANOARROW_OK; } @@ -1074,6 +1128,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffers, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->nodes, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffer_views, 0, 0)); + NANOARROW_ASSERT_OK(ArrowBufferResize(&private->variadic_buffer_counts, 0, 0)); NANOARROW_RETURN_NOT_OK( ArrowIpcEncoderEncodeRecordBatchImpl(private, array_view, error)); NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeBuffers(encoder, buffer_encoder, error)); @@ -1088,6 +1143,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeRecordBatch( builder, (struct ns(Buffer)*)private->buffers.data, _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeVariadicBufferCounts(private, error)); FLATCC_RETURN_UNLESS_0(Message_header_RecordBatch_end(builder), error); @@ -1142,6 +1198,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffers, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->nodes, 0, 0)); NANOARROW_ASSERT_OK(ArrowBufferResize(&private->buffer_views, 0, 0)); + NANOARROW_ASSERT_OK(ArrowBufferResize(&private->variadic_buffer_counts, 0, 0)); // The values array is a single top-level column: collect it, then descend into any // nested children. @@ -1160,6 +1217,7 @@ static ArrowErrorCode ArrowIpcEncoderEncodeDictionaryBatch( builder, (struct ns(Buffer)*)private->buffers.data, _ArrowInt64ToSizeT(private->buffers.size_bytes) / sizeof(struct ns(Buffer))), error); + NANOARROW_RETURN_NOT_OK(ArrowIpcEncoderEncodeVariadicBufferCounts(private, error)); FLATCC_RETURN_UNLESS_0(DictionaryBatch_data_end(builder), error); FLATCC_RETURN_UNLESS_0(DictionaryBatch_isDelta_add(builder, is_delta ? 1 : 0), error); FLATCC_RETURN_UNLESS_0(Message_header_DictionaryBatch_end(builder), error); diff --git a/src/nanoarrow/ipc/files_test.cc b/src/nanoarrow/ipc/files_test.cc index 501b92495..69feeada5 100644 --- a/src/nanoarrow/ipc/files_test.cc +++ b/src/nanoarrow/ipc/files_test.cc @@ -619,7 +619,7 @@ INSTANTIATE_TEST_SUITE_P( TestFile::OK("cpp-21.0.0/generated_extension.stream"), TestFile::OK("cpp-21.0.0/generated_nested_dictionary.stream"), TestFile::NotSupported("cpp-21.0.0/generated_list_view.stream"), - TestFile::NotSupported("cpp-21.0.0/generated_binary_view.stream"), + TestFile::OK("cpp-21.0.0/generated_binary_view.stream"), TestFile::OK("cpp-21.0.0/generated_run_end_encoded.stream") // Comment to keep line from wrapping )); diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc index a67cfb718..e079418ac 100644 --- a/src/nanoarrow/ipc/writer_test.cc +++ b/src/nanoarrow/ipc/writer_test.cc @@ -316,19 +316,19 @@ TEST(NanoarrowIpcWriter, WriteDictionaryBatch) { EXPECT_EQ(p2->bytes_written, bytes_written); } -// Build a struct array with a single dictionary-encoded (int32 -> utf8) child. +// Build a struct array with a single dictionary-encoded (int32 -> utf8 by default) child. static void MakeDictionaryStructArray(struct ArrowArray* array, struct ArrowSchema* schema, - const char* value1 = "bar") { + const char* value1 = "bar", + enum ArrowType value_type = NANOARROW_TYPE_STRING) { ASSERT_EQ(ArrowSchemaInitFromType(schema, NANOARROW_TYPE_STRUCT), NANOARROW_OK); ASSERT_EQ(ArrowSchemaAllocateChildren(schema, 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetName(schema->children[0], "dict_col"), NANOARROW_OK); ASSERT_EQ(ArrowSchemaAllocateDictionary(schema->children[0]), NANOARROW_OK); - ASSERT_EQ( - ArrowSchemaInitFromType(schema->children[0]->dictionary, NANOARROW_TYPE_STRING), - NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->children[0]->dictionary, value_type), + NANOARROW_OK); ASSERT_EQ(ArrowArrayInitFromSchema(array, schema, nullptr), NANOARROW_OK); struct ArrowArray* indices = array->children[0]; @@ -556,10 +556,11 @@ static void ReadArrays(struct ArrowBuffer* buffer, } } -static std::vector StringValues(const struct ArrowArray* array) { +static std::vector StringValues( + const struct ArrowArray* array, enum ArrowType type = NANOARROW_TYPE_STRING) { std::vector values; nanoarrow::UniqueArrayView view; - ArrowArrayViewInitFromType(view.get(), NANOARROW_TYPE_STRING); + ArrowArrayViewInitFromType(view.get(), type); EXPECT_EQ(ArrowArrayViewSetArray(view.get(), array, nullptr), NANOARROW_OK); for (int64_t i = 0; i < view->length; i++) { struct ArrowStringView value = ArrowArrayViewGetStringUnsafe(view.get(), i); @@ -676,6 +677,62 @@ TEST(NanoarrowIpcWriter, EmitsChangedDictionary) { (std::vector{"foo", "baz"})); } +TEST(NanoarrowIpcWriter, RoundtripStringViewDictionary) { + // A dictionary of string views, one of which is too long to be inlined + struct ArrowError error; + const char* long_value = "a string view value that is not inlined"; + + for (bool as_file : {false, true}) { + SCOPED_TRACE(as_file ? "file" : "stream"); + nanoarrow::UniqueSchema schema; + std::vector arrays(1); + MakeDictionaryStructArray(arrays[0].get(), schema.get(), long_value, + NANOARROW_TYPE_STRING_VIEW); + + nanoarrow::UniqueBuffer output; + ASSERT_EQ(WriteArrays(schema.get(), &arrays, output.get(), as_file, &error), + NANOARROW_OK) + << error.message; + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) + auto arrow_input = std::make_shared( + arrow::Buffer::Wrap(output->data, output->size_bytes)); + std::shared_ptr arrow_batch; + if (as_file) { + auto maybe_arrow_reader = arrow::ipc::RecordBatchFileReader::Open(arrow_input); + ASSERT_TRUE(maybe_arrow_reader.ok()) << maybe_arrow_reader.status(); + auto maybe_batch = maybe_arrow_reader.ValueUnsafe()->ReadRecordBatch(0); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status(); + arrow_batch = maybe_batch.ValueUnsafe(); + } else { + auto maybe_arrow_reader = arrow::ipc::RecordBatchStreamReader::Open(arrow_input); + ASSERT_TRUE(maybe_arrow_reader.ok()) << maybe_arrow_reader.status(); + ASSERT_TRUE(maybe_arrow_reader.ValueUnsafe()->ReadNext(&arrow_batch).ok()); + } + ASSERT_NE(arrow_batch, nullptr); + ASSERT_TRUE(arrow_batch->ValidateFull().ok()); + auto arrow_dictionary = + std::static_pointer_cast(arrow_batch->column(0)); + auto arrow_values = + std::static_pointer_cast(arrow_dictionary->dictionary()); + ASSERT_EQ(arrow_values->length(), 2); + EXPECT_EQ(arrow_values->GetString(0), "foo"); + EXPECT_EQ(arrow_values->GetString(1), long_value); +#endif + + if (as_file) { + continue; + } + + std::vector roundtrip; + ReadArrays(output.get(), &roundtrip); + ASSERT_EQ(roundtrip.size(), 1); + EXPECT_EQ( + StringValues(roundtrip[0]->children[0]->dictionary, NANOARROW_TYPE_STRING_VIEW), + (std::vector{"foo", long_value})); + } +} + TEST(NanoarrowIpcWriter, RejectsChangedDictionaryInFile) { struct ArrowError error; nanoarrow::UniqueSchema schema;