From efc633a103a79c974a23f2949c06cec0012cc414 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sat, 29 Mar 2025 15:19:06 -0500 Subject: [PATCH 01/22] start adding dictionary --- src/nanoarrow/ipc/decoder.c | 22 +++++++++++++++++++++- src/nanoarrow/nanoarrow_ipc.h | 13 +++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 202715e80..709cf344a 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -95,6 +95,8 @@ struct ArrowIpcDecoderPrivate { int64_t n_union_fields; // A pointer to the last flatbuffers message. const void* last_message; + // Storage for a Dictionary + struct ArrowIpcDictionary dictionary; // Storage for a Footer struct ArrowIpcFooter footer; // Decompressor for compression support @@ -930,6 +932,19 @@ static int ArrowIpcDecoderDecodeSchemaHeader(struct ArrowIpcDecoder* decoder, return NANOARROW_OK; } +static int ArrowIpcDecoderDecodeDictionaryBatchHeader( + struct ArrowIpcDecoder* decoder, flatbuffers_generic_t message_header) { + struct ArrowIpcDecoderPrivate* private_data = + (struct ArrowIpcDecoderPrivate*)decoder->private_data; + + ns(DictionaryBatch_table_t) dictionary = (ns(DictionaryBatch_table_t))message_header; + private_data->dictionary.id = ns(DictionaryBatch_id(dictionary)); + private_data->dictionary.is_delta = ns(DictionaryBatch_isDelta(dictionary)); + + decoder->dictionary = &private_data->dictionary; + return NANOARROW_OK; +} + static int ArrowIpcDecoderDecodeRecordBatchHeader(struct ArrowIpcDecoder* decoder, flatbuffers_generic_t message_header, struct ArrowError* error) { @@ -1000,6 +1015,8 @@ static inline void ArrowIpcDecoderResetHeaderInfo(struct ArrowIpcDecoder* decode decoder->codec = 0; decoder->header_size_bytes = 0; decoder->body_size_bytes = 0; + decoder->dictionary = NULL; + memset(&private_data->dictionary, 0, sizeof(struct ArrowIpcDictionary)); decoder->footer = NULL; ArrowIpcFooterReset(&private_data->footer); private_data->last_message = NULL; @@ -1242,11 +1259,14 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* decoder, NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderDecodeSchemaHeader(decoder, message_header, error)); break; + case ns(MessageHeader_DictionaryBatch): + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderDecodeDictionaryBatchHeader(decoder, message_header)); + break; case ns(MessageHeader_RecordBatch): NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderDecodeRecordBatchHeader(decoder, message_header, error)); break; - case ns(MessageHeader_DictionaryBatch): case ns(MessageHeader_Tensor): case ns(MessageHeader_SparseTensor): ArrowErrorSet(error, "Unsupported message type: '%s'", diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index b9251a6be..ef0ec91d2 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -162,6 +162,15 @@ enum ArrowIpcCompressionType { /// \brief Feature flag for a stream that uses compression #define NANOARROW_IPC_FEATURE_COMPRESSED_BODY 2 +/// \brief Description of an Arrow IPC DictionaryBatch message +struct ArrowIpcDictionary { + /// \brief The identifier for this dictionary + int64_t id; + /// \brief If non-zero, values should be appended to the existing dictionary. + /// Otherwise, values should replace the existing dictionary. + int is_delta; +}; + /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); @@ -307,6 +316,9 @@ struct ArrowIpcDecoder { /// \brief The number of bytes in the forthcoming body message. int64_t body_size_bytes; + /// \brief The last decoded Dictionary + struct ArrowIpcDictionary* dictionary; + /// \brief The last decoded Footer /// /// \warning This API is currently only public for use in integration testing; @@ -670,6 +682,7 @@ ArrowErrorCode ArrowIpcWriterStartFile(struct ArrowIpcWriter* writer, /// Writes the IPC file's footer, footer size, and ending magic. NANOARROW_DLL ArrowErrorCode ArrowIpcWriterFinalizeFile(struct ArrowIpcWriter* writer, struct ArrowError* error); + /// @} // Internal APIs: From 8cb795efd024bb60752482f0492a1fe1572ecd3c Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 30 Mar 2025 15:26:45 -0500 Subject: [PATCH 02/22] add sample dictionary info --- src/nanoarrow/ipc/decoder_test.cc | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 81993bce9..4429600d4 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -159,6 +159,64 @@ alignas(8) static uint8_t kSimpleRecordBatchUncompressible[] = { 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +static uint8_t kDictionarySchema[] = { + 0xff, 0xff, 0xff, 0xff, 0x50, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x0e, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x0c, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x0c, 0x00, 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x41, 0x0a, 0x33, 0x0a, 0x32, 0x36, + 0x33, 0x31, 0x37, 0x30, 0x0a, 0x31, 0x39, 0x37, 0x38, 0x38, 0x38, 0x0a, 0x35, 0x0a, + 0x55, 0x54, 0x46, 0x2d, 0x38, 0x0a, 0x35, 0x33, 0x31, 0x0a, 0x31, 0x0a, 0x35, 0x33, + 0x31, 0x0a, 0x31, 0x0a, 0x32, 0x35, 0x34, 0x0a, 0x31, 0x30, 0x32, 0x36, 0x0a, 0x31, + 0x0a, 0x32, 0x36, 0x32, 0x31, 0x35, 0x33, 0x0a, 0x35, 0x0a, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x0a, 0x31, 0x36, 0x0a, 0x31, 0x0a, 0x32, 0x36, 0x32, 0x31, 0x35, 0x33, 0x0a, + 0x38, 0x0a, 0x73, 0x6f, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x0a, 0x32, 0x35, 0x34, + 0x0a, 0x31, 0x30, 0x32, 0x36, 0x0a, 0x35, 0x31, 0x31, 0x0a, 0x31, 0x36, 0x0a, 0x31, + 0x0a, 0x32, 0x36, 0x32, 0x31, 0x35, 0x33, 0x0a, 0x37, 0x0a, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x73, 0x0a, 0x32, 0x35, 0x34, 0x0a, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x72, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x18, 0x00, 0x08, 0x00, 0x06, 0x00, 0x07, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x05, 0x14, 0x00, 0x00, 0x00, 0x48, 0x00, + 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x73, 0x6f, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +static uint8_t kDictionaryBatch[] = { + 0xff, 0xff, 0xff, 0xff, 0xa8, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x18, 0x00, + 0x0c, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x7a, 0x65, 0x72, 0x6f, + 0x6f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static uint8_t kDictionaryRecordBatch[] = { + 0xff, 0xff, 0xff, 0xff, 0x88, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0c, 0x00, 0x16, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x18, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x18, 0x00, 0x0c, 0x00, + 0x04, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00}; + TEST(NanoarrowIpcTest, NanoarrowIpcCheckHeader) { struct ArrowIpcDecoder decoder; struct ArrowError error; From ad11e4fcda416e28d9373e2258d040b0c0d19a34 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 30 Mar 2025 23:08:49 -0500 Subject: [PATCH 03/22] decode schema maybe --- src/nanoarrow/ipc/decoder.c | 68 ++++++++++++++++++++++++++----- src/nanoarrow/ipc/decoder_test.cc | 27 ++++++++++++ src/nanoarrow/nanoarrow_ipc.h | 6 +-- 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 709cf344a..5eb806cf1 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -95,8 +95,8 @@ struct ArrowIpcDecoderPrivate { int64_t n_union_fields; // A pointer to the last flatbuffers message. const void* last_message; - // Storage for a Dictionary - struct ArrowIpcDictionary dictionary; + // Storage for a DictionaryBatch + struct ArrowIpcDictionaryBatch dictionary; // Storage for a Footer struct ArrowIpcFooter footer; // Decompressor for compression support @@ -825,17 +825,56 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema* schema, ns(Field_table_t) } } +static int ArrowIpcSetDictionaryEncoding( + struct ArrowSchema* schema, ns(DictionaryEncoding_table_t dictionary_encoding), + struct ArrowError* error) { + switch ( + org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind(dictionary_encoding)) { + case ns(DictionaryKind_DenseArray): + break; + default: + ArrowErrorSet(error, "Uexpected value for DictionaryKind"); + return EINVAL; + } + + struct ArrowSchema tmp; + ArrowSchemaMove(schema, &tmp); + + ArrowSchemaInit(schema); + int result = ArrowSchemaAllocateDictionary(schema); + if (result != NANOARROW_OK) { + ArrowSchemaRelease(&tmp); + ArrowErrorSet(error, "ArrowSchemaAllocateDictionary() failed"); + return result; + } + + ArrowSchemaMove(&tmp, schema->dictionary); + + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaSetName(schema, schema->dictionary->name), + error); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowSchemaSetName(schema->dictionary, ""), error); + + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetTypeInt( + schema, ns(DictionaryEncoding_indexType_get(dictionary_encoding)), error)); + + if (ns(DictionaryEncoding_isOrdered_get(dictionary_encoding))) { + schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED; + } + + // TODO: should the non extension-type field metadata get moved back to the + // parent field? + + // TODO: Record the dictionary id in Schema metadata? (Or keep track of it + // some other way?) + + return NANOARROW_OK; +} + static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, struct ArrowError* error); static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) field, struct ArrowError* error) { - // No dictionary support yet - if (ns(Field_dictionary_is_present(field))) { - ArrowErrorSet(error, "Schema message field with DictionaryEncoding not supported"); - return ENOTSUP; - } - int result; if (ns(Field_name_is_present(field))) { result = ArrowSchemaSetName(schema, ns(Field_name_get(field))); @@ -876,7 +915,16 @@ static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) } NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetChildren(schema, children, error)); - return ArrowIpcDecoderSetMetadata(schema, ns(Field_custom_metadata(field)), error); + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetMetadata(schema, ns(Field_custom_metadata(field)), error)); + + // If this is a dictionary encoded field, set the dictionary encoding + if (ns(Field_dictionary_is_present(field))) { + NANOARROW_RETURN_NOT_OK( + ArrowIpcSetDictionaryEncoding(schema, ns(Field_dictionary(field)), error)); + } + + return NANOARROW_OK; } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, @@ -1016,7 +1064,7 @@ static inline void ArrowIpcDecoderResetHeaderInfo(struct ArrowIpcDecoder* decode decoder->header_size_bytes = 0; decoder->body_size_bytes = 0; decoder->dictionary = NULL; - memset(&private_data->dictionary, 0, sizeof(struct ArrowIpcDictionary)); + memset(&private_data->dictionary, 0, sizeof(struct ArrowIpcDictionaryBatch)); decoder->footer = NULL; ArrowIpcFooterReset(&private_data->footer); private_data->last_message = NULL; diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 4429600d4..b55c3dad7 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -596,6 +596,33 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchErrors) { ArrowIpcDecoderReset(&decoder); } +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { + struct ArrowIpcDecoder decoder; + struct ArrowError error; + struct ArrowSchema schema; + + struct ArrowBufferView data; + data.data.as_uint8 = kDictionarySchema; + data.size_bytes = sizeof(kDictionarySchema); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA); + + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error), NANOARROW_OK); + ASSERT_EQ(schema.n_children, 1); + EXPECT_STREQ(schema.children[0]->name, "some_col"); + EXPECT_EQ(schema.children[0]->flags, ARROW_FLAG_NULLABLE); + EXPECT_STREQ(schema.children[0]->format, "c"); + + ASSERT_NE(schema.children[0]->dictionary, nullptr); + EXPECT_STREQ(schema.children[0]->dictionary->format, "u"); + + ArrowSchemaRelease(&schema); + ArrowIpcDecoderReset(&decoder); +} + TEST(NanoarrowIpcTest, NanoarrowIpcSetSchema) { struct ArrowIpcDecoder decoder; struct ArrowSchema schema; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ef0ec91d2..2e772c1b3 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -163,7 +163,7 @@ enum ArrowIpcCompressionType { #define NANOARROW_IPC_FEATURE_COMPRESSED_BODY 2 /// \brief Description of an Arrow IPC DictionaryBatch message -struct ArrowIpcDictionary { +struct ArrowIpcDictionaryBatch { /// \brief The identifier for this dictionary int64_t id; /// \brief If non-zero, values should be appended to the existing dictionary. @@ -316,8 +316,8 @@ struct ArrowIpcDecoder { /// \brief The number of bytes in the forthcoming body message. int64_t body_size_bytes; - /// \brief The last decoded Dictionary - struct ArrowIpcDictionary* dictionary; + /// \brief The last decoded DictionaryBatch + struct ArrowIpcDictionaryBatch* dictionary; /// \brief The last decoded Footer /// From 40820338a8c7d1d9f35bb90e6ce0cb3db748756e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 30 Mar 2025 23:13:13 -0500 Subject: [PATCH 04/22] stub other tests --- src/nanoarrow/ipc/decoder_test.cc | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index b55c3dad7..e23d7b2e9 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -623,6 +623,48 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { + struct ArrowIpcDecoder decoder; + struct ArrowError error; + struct ArrowSchema schema; + + struct ArrowBufferView data; + data.data.as_uint8 = kDictionaryBatch; + data.size_bytes = sizeof(kDictionaryBatch); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH); + + ASSERT_NE(decoder.dictionary, nullptr); + EXPECT_EQ(decoder.dictionary->id, 0); + EXPECT_FALSE(decoder.dictionary->is_delta); + + // TODO: Access RecordBatch content + + ArrowIpcDecoderReset(&decoder); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryRecordBatch) { + struct ArrowIpcDecoder decoder; + struct ArrowError error; + struct ArrowSchema schema; + + struct ArrowBufferView data; + data.data.as_uint8 = kDictionaryRecordBatch; + data.size_bytes = sizeof(kDictionaryRecordBatch); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + EXPECT_EQ(ArrowIpcDecoderVerifyHeader(&decoder, data, &error), NANOARROW_OK); + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH); + + // TODO: Decode RecordBatch content populating dictionary array member + + ArrowIpcDecoderReset(&decoder); +} + TEST(NanoarrowIpcTest, NanoarrowIpcSetSchema) { struct ArrowIpcDecoder decoder; struct ArrowSchema schema; From b2cff7e4f055fe3077eda8afda1dcace5f8015d2 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 31 Mar 2025 09:36:29 -0500 Subject: [PATCH 05/22] fix warning --- src/nanoarrow/ipc/decoder_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index e23d7b2e9..81537350b 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -626,7 +626,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { struct ArrowIpcDecoder decoder; struct ArrowError error; - struct ArrowSchema schema; struct ArrowBufferView data; data.data.as_uint8 = kDictionaryBatch; From 28858114bd3bd1cc5a444ead085d314f6711eab2 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 1 Apr 2025 21:54:22 -0500 Subject: [PATCH 06/22] one more unused variable for now --- src/nanoarrow/ipc/decoder_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 81537350b..38be540bc 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -648,7 +648,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryRecordBatch) { struct ArrowIpcDecoder decoder; struct ArrowError error; - struct ArrowSchema schema; struct ArrowBufferView data; data.data.as_uint8 = kDictionaryRecordBatch; From 6969cc9c51162bded41d42f84519f2faa782f7c2 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 1 Apr 2025 23:09:07 -0500 Subject: [PATCH 07/22] sketch out some dictionaries options --- src/nanoarrow/ipc/decoder.c | 61 +++++++++++++++++++++++++++++++ src/nanoarrow/ipc/decoder_test.cc | 2 + src/nanoarrow/nanoarrow_ipc.h | 24 ++++++++++++ 3 files changed, 87 insertions(+) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 5eb806cf1..4ac39217f 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -66,6 +66,8 @@ struct ArrowIpcField { struct ArrowArray* array; // The cumulative number of buffers preceding this node. int64_t buffer_offset; + // Dictionary information for dictionary-encoded fields + struct ArrowIpcDictionary* dictionary; }; // Internal data specific to the read/decode process @@ -1426,6 +1428,7 @@ static void ArrowIpcDecoderInitFields(struct ArrowIpcField* fields, field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; + field->dictionary = NULL; for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { *n_buffers += array_view->layout.buffer_type[i] != NANOARROW_BUFFER_TYPE_NONE; @@ -2116,3 +2119,61 @@ ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( ArrowArrayMove(&temp, out); return NANOARROW_OK; } + +void ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary) { + NANOARROW_DCHECK(dictionary != NULL); + + memset(dictionary, 0, sizeof(struct ArrowIpcDictionary)); +} + +void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary) { + NANOARROW_DCHECK(dictionary != NULL); + + if (dictionary->schema.release != NULL) { + ArrowSchemaRelease(&dictionary->schema); + } + + if (dictionary->array.release != NULL) { + ArrowArrayRelease(&dictionary->array); + } +} + +void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); +} + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + + for (int64_t i = 0; i < dictionaries->n_dictionaries; i++) { + ArrowIpcDictionaryReset(dictionaries->dictionaries + i); + } + + if (dictionaries->dictionaries != NULL) { + ArrowFree(dictionaries->dictionaries); + } +} + +ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + int64_t id, struct ArrowSchema* schema) { + if ((dictionaries->n_dictionaries + 1) > dictionaries->capacity) { + int64_t new_capacity = + 1 ? dictionaries->n_dictionaries == 0 : dictionaries->n_dictionaries * 2; + struct ArrowIpcDictionaries* new_dictionaries = realloc( + dictionaries->dictionaries, new_capacity * sizeof(struct ArrowIpcDictionary)); + if (new_dictionaries == NULL) { + return ENOMEM; + } + + dictionaries->capacity = new_capacity; + } + + struct ArrowIpcDictionary* dictionary = + dictionaries->dictionaries + dictionaries->n_dictionaries; + ArrowIpcDictionaryInit(dictionary); + dictionary->id = id; + ArrowSchemaMove(schema, &dictionary->schema); + dictionaries->n_dictionaries++; + return NANOARROW_OK; +} diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 38be540bc..1f4479d2d 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -43,6 +43,7 @@ struct ArrowIpcField { struct ArrowArrayView* array_view; struct ArrowArray* array; int64_t buffer_offset; + int64_t dictionary_id; }; struct ArrowIpcDecoderPrivate { @@ -54,6 +55,7 @@ struct ArrowIpcDecoderPrivate { struct ArrowIpcField* fields; int64_t n_buffers; const void* last_message; + struct ArrowIpcDictionaryBatch dictionary; struct ArrowIpcFooter footer; struct ArrowIpcDecompressor decompressor; }; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 2e772c1b3..e66b3640d 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -467,6 +467,30 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( struct ArrowArray* out, enum ArrowValidationLevel validation_level, struct ArrowError* error); +struct ArrowIpcDictionary { + int64_t id; + struct ArrowSchema schema; + struct ArrowArray array; +}; + +void ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary); + +void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary); + +struct ArrowIpcDictionaries { + struct ArrowIpcDictionary* dictionaries; + int64_t n_dictionaries; + int64_t capacity; + void* private_data; +}; + +void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); + +ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + int64_t id, struct ArrowSchema* schema); + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); + /// \brief An user-extensible input data source struct ArrowIpcInputStream { /// \brief Read up to buf_size_bytes from stream into buf From c8f39788adf3fa009fbf3c1e91d74453b2dbea37 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 15:03:26 -0600 Subject: [PATCH 08/22] test lifecycle --- src/nanoarrow/ipc/decoder.c | 8 +++++--- src/nanoarrow/ipc/decoder_test.cc | 33 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 4ac39217f..5694c7628 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2159,13 +2159,15 @@ ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionar int64_t id, struct ArrowSchema* schema) { if ((dictionaries->n_dictionaries + 1) > dictionaries->capacity) { int64_t new_capacity = - 1 ? dictionaries->n_dictionaries == 0 : dictionaries->n_dictionaries * 2; - struct ArrowIpcDictionaries* new_dictionaries = realloc( - dictionaries->dictionaries, new_capacity * sizeof(struct ArrowIpcDictionary)); + dictionaries->n_dictionaries == 0 ? 1 : dictionaries->n_dictionaries * 2; + struct ArrowIpcDictionary* new_dictionaries = + (struct ArrowIpcDictionary*)ArrowRealloc( + dictionaries->dictionaries, new_capacity * sizeof(struct ArrowIpcDictionary)); if (new_dictionaries == NULL) { return ENOMEM; } + dictionaries->dictionaries = new_dictionaries; dictionaries->capacity = new_capacity; } diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 1f4479d2d..835526696 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -1540,3 +1540,36 @@ TEST(NanoarrowIpcTest, NanoarrowIpcFooterDecodingErrors) { << error.message; EXPECT_EQ(decoder->header_size_bytes, 0xFFFF); } + +TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { + struct ArrowIpcDictionaries dictionaries; + + // Test init + ArrowIpcDictionariesInit(&dictionaries); + EXPECT_EQ(dictionaries.dictionaries, nullptr); + EXPECT_EQ(dictionaries.n_dictionaries, 0); + EXPECT_EQ(dictionaries.capacity, 0); + + // Test append first dictionary + nanoarrow::UniqueSchema schema1; + ASSERT_EQ(ArrowSchemaInitFromType(schema1.get(), NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, 42, schema1.get()), NANOARROW_OK); + EXPECT_EQ(dictionaries.n_dictionaries, 1); + EXPECT_GE(dictionaries.capacity, 1); + EXPECT_NE(dictionaries.dictionaries, nullptr); + EXPECT_EQ(dictionaries.dictionaries[0].id, 42); + EXPECT_NE(dictionaries.dictionaries[0].schema.release, nullptr); + + // Test append second dictionary + nanoarrow::UniqueSchema schema2; + ASSERT_EQ(ArrowSchemaInitFromType(schema2.get(), NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, 99, schema2.get()), NANOARROW_OK); + EXPECT_EQ(dictionaries.n_dictionaries, 2); + EXPECT_GE(dictionaries.capacity, 2); + EXPECT_EQ(dictionaries.dictionaries[0].id, 42); + EXPECT_EQ(dictionaries.dictionaries[1].id, 99); + EXPECT_NE(dictionaries.dictionaries[1].schema.release, nullptr); + + // Test reset cleans up all resources + ArrowIpcDictionariesReset(&dictionaries); +} From 55bdf8807479106bfeabf755c0056f80650f37e0 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 16:42:26 -0600 Subject: [PATCH 09/22] update structure of dictionaries --- src/nanoarrow/ipc/decoder.c | 76 +++++++++++++++++++++++------------ src/nanoarrow/nanoarrow_ipc.h | 27 +++++++++---- 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 5694c7628..285f82dda 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1999,12 +1999,6 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayViewInternal( struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; - if (private_data->last_message == NULL || - decoder->message_type != NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH) { - ArrowErrorSet(error, "decoder did not just decode a RecordBatch message"); - return EINVAL; - } - // RecordBatch messages don't count the root node but decoder->fields does // (decoder->fields[0] is the root field) if (field_i + 1 >= private_data->n_fields) { @@ -2067,6 +2061,14 @@ ArrowErrorCode ArrowIpcDecoderDecodeArrayView(struct ArrowIpcDecoder* decoder, struct ArrowBufferView body, int64_t i, struct ArrowArrayView** out, struct ArrowError* error) { + struct ArrowIpcDecoderPrivate* private_data = + (struct ArrowIpcDecoderPrivate*)decoder->private_data; + if (private_data->last_message == NULL || + decoder->message_type != NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH) { + ArrowErrorSet(error, "decoder did not just decode a RecordBatch message"); + return EINVAL; + } + return ArrowIpcDecoderDecodeArrayViewInternal( decoder, ArrowIpcBufferFactoryFromView(&body), i, out, error); } @@ -2076,6 +2078,14 @@ ArrowErrorCode ArrowIpcDecoderDecodeArray(struct ArrowIpcDecoder* decoder, struct ArrowArray* out, enum ArrowValidationLevel validation_level, struct ArrowError* error) { + struct ArrowIpcDecoderPrivate* private_data = + (struct ArrowIpcDecoderPrivate*)decoder->private_data; + if (private_data->last_message == NULL || + decoder->message_type != NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH) { + ArrowErrorSet(error, "decoder did not just decode a RecordBatch message"); + return EINVAL; + } + struct ArrowArrayView* array_view; NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayViewInternal( decoder, ArrowIpcBufferFactoryFromView(&body), i, &array_view, error)); @@ -2120,22 +2130,31 @@ ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( return NANOARROW_OK; } -void ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary) { - NANOARROW_DCHECK(dictionary != NULL); +ArrowErrorCode ArrowIpcDictionaryDecoderInit(struct ArrowIpcDictionaryDecoder* dictionary, + const struct ArrowIpcDecoder* parent, + int64_t id, + struct ArrowSchema* value_schema) { + memset(dictionary, 0, sizeof(struct ArrowIpcDictionaryDecoder)); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInit(&dictionary->decoder)); + ArrowErrorCode result = + ArrowIpcDecoderSetEndianness(&dictionary->decoder, parent->endianness); + if (result != NANOARROW_OK) { + ArrowIpcDecoderReset(&dictionary->decoder); + return result; + } + + result = ArrowIpcDecoderSetSchema(&dictionary->decoder, value_schema, NULL); + if (result != NANOARROW_OK) { + return result; + } - memset(dictionary, 0, sizeof(struct ArrowIpcDictionary)); + dictionary->id = id; + return NANOARROW_OK; } -void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary) { +void ArrowIpcDictionaryDecoderReset(struct ArrowIpcDictionaryDecoder* dictionary) { NANOARROW_DCHECK(dictionary != NULL); - - if (dictionary->schema.release != NULL) { - ArrowSchemaRelease(&dictionary->schema); - } - - if (dictionary->array.release != NULL) { - ArrowArrayRelease(&dictionary->array); - } + ArrowIpcDecoderReset(&dictionary->decoder); } void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries) { @@ -2147,7 +2166,7 @@ void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { NANOARROW_DCHECK(dictionaries != NULL); for (int64_t i = 0; i < dictionaries->n_dictionaries; i++) { - ArrowIpcDictionaryReset(dictionaries->dictionaries + i); + ArrowIpcDictionaryDecoderReset(dictionaries->dictionaries + i); } if (dictionaries->dictionaries != NULL) { @@ -2156,14 +2175,21 @@ void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { } ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDecoder* decoder, int64_t id, struct ArrowSchema* schema) { + struct ArrowIpcDictionaryDecoder decoder_tmp; + NANOARROW_RETURN_NOT_OK( + ArrowIpcDictionaryDecoderInit(&decoder_tmp, decoder, id, schema)); + if ((dictionaries->n_dictionaries + 1) > dictionaries->capacity) { int64_t new_capacity = dictionaries->n_dictionaries == 0 ? 1 : dictionaries->n_dictionaries * 2; - struct ArrowIpcDictionary* new_dictionaries = - (struct ArrowIpcDictionary*)ArrowRealloc( - dictionaries->dictionaries, new_capacity * sizeof(struct ArrowIpcDictionary)); + struct ArrowIpcDictionaryDecoder* new_dictionaries = + (struct ArrowIpcDictionaryDecoder*)ArrowRealloc( + dictionaries->dictionaries, + new_capacity * sizeof(struct ArrowIpcDictionaryDecoder)); if (new_dictionaries == NULL) { + ArrowIpcDictionaryDecoderReset(&decoder_tmp); return ENOMEM; } @@ -2171,11 +2197,9 @@ ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionar dictionaries->capacity = new_capacity; } - struct ArrowIpcDictionary* dictionary = + struct ArrowIpcDictionaryDecoder* dictionary = dictionaries->dictionaries + dictionaries->n_dictionaries; - ArrowIpcDictionaryInit(dictionary); - dictionary->id = id; - ArrowSchemaMove(schema, &dictionary->schema); + memcpy(&decoder_tmp, dictionary, sizeof(struct ArrowIpcDictionaryDecoder)); dictionaries->n_dictionaries++; return NANOARROW_OK; } diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index e66b3640d..fabbf9f75 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -467,29 +467,40 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( struct ArrowArray* out, enum ArrowValidationLevel validation_level, struct ArrowError* error); -struct ArrowIpcDictionary { +struct ArrowIpcDictionaryDecoder { int64_t id; - struct ArrowSchema schema; - struct ArrowArray array; + struct ArrowIpcDecoder decoder; }; -void ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary); +ArrowErrorCode ArrowIpcDictionaryDecoderInit(struct ArrowIpcDictionaryDecoder* dictionary, + const struct ArrowIpcDecoder* parent, + int64_t id, + struct ArrowSchema* value_schema); -void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary); +void ArrowIpcDictionaryDecoderReset(struct ArrowIpcDictionaryDecoder* dictionary); struct ArrowIpcDictionaries { - struct ArrowIpcDictionary* dictionaries; + struct ArrowIpcDictionaryDecoder* dictionaries; int64_t n_dictionaries; int64_t capacity; - void* private_data; }; void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + + const struct ArrowIpcDecoder* decoder, int64_t id, struct ArrowSchema* schema); -void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); +/// \brief Append the current DictionaryBatch message to the encoder +/// +/// After a successful call to ArrowIpcDecoderDecodeHeader(), assemble an ArrowArray given +/// a message body and append (or replace) the corresponding entry in this decoder's +/// ArrowIpcDictionaries. +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderAppendDictionaryFromShared( + struct ArrowIpcDecoder* decoder, struct ArrowIpcSharedBuffer* shared, + enum ArrowValidationLevel validation_level, struct ArrowError* error); /// \brief An user-extensible input data source struct ArrowIpcInputStream { From 238fcba0648f2b8cb197e6d17632b0db4f246e0b Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 16:45:42 -0600 Subject: [PATCH 10/22] test new structure --- src/nanoarrow/ipc/decoder.c | 2 +- src/nanoarrow/ipc/decoder_test.cc | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 285f82dda..d0bdcfa43 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2199,7 +2199,7 @@ ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionar struct ArrowIpcDictionaryDecoder* dictionary = dictionaries->dictionaries + dictionaries->n_dictionaries; - memcpy(&decoder_tmp, dictionary, sizeof(struct ArrowIpcDictionaryDecoder)); + memcpy(dictionary, &decoder_tmp, sizeof(struct ArrowIpcDictionaryDecoder)); dictionaries->n_dictionaries++; return NANOARROW_OK; } diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 835526696..28f0b39b2 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -1550,25 +1550,40 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { EXPECT_EQ(dictionaries.n_dictionaries, 0); EXPECT_EQ(dictionaries.capacity, 0); + // Create a parent decoder for use in append + nanoarrow::ipc::UniqueDecoder parent_decoder; + ASSERT_EQ(ArrowIpcDecoderInit(parent_decoder.get()), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderSetEndianness(parent_decoder.get(), + ArrowIpcSystemEndianness()), + NANOARROW_OK); + // Test append first dictionary nanoarrow::UniqueSchema schema1; - ASSERT_EQ(ArrowSchemaInitFromType(schema1.get(), NANOARROW_TYPE_INT32), NANOARROW_OK); - ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, 42, schema1.get()), NANOARROW_OK); + ArrowSchemaInit(schema1.get()); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema1.get(), 1), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema1->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema1->children[0], "value"), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 42, + schema1.get()), + NANOARROW_OK); EXPECT_EQ(dictionaries.n_dictionaries, 1); EXPECT_GE(dictionaries.capacity, 1); EXPECT_NE(dictionaries.dictionaries, nullptr); EXPECT_EQ(dictionaries.dictionaries[0].id, 42); - EXPECT_NE(dictionaries.dictionaries[0].schema.release, nullptr); // Test append second dictionary nanoarrow::UniqueSchema schema2; - ASSERT_EQ(ArrowSchemaInitFromType(schema2.get(), NANOARROW_TYPE_STRING), NANOARROW_OK); - ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, 99, schema2.get()), NANOARROW_OK); + ArrowSchemaInit(schema2.get()); + ASSERT_EQ(ArrowSchemaSetTypeStruct(schema2.get(), 1), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema2->children[0], NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema2->children[0], "value"), NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 99, + schema2.get()), + NANOARROW_OK); EXPECT_EQ(dictionaries.n_dictionaries, 2); EXPECT_GE(dictionaries.capacity, 2); EXPECT_EQ(dictionaries.dictionaries[0].id, 42); EXPECT_EQ(dictionaries.dictionaries[1].id, 99); - EXPECT_NE(dictionaries.dictionaries[1].schema.release, nullptr); // Test reset cleans up all resources ArrowIpcDictionariesReset(&dictionaries); From de2f8db70076ef28854081c94c2546162b647af5 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 17:24:54 -0600 Subject: [PATCH 11/22] iterate on todos --- src/nanoarrow/ipc/decoder.c | 26 +++++++++++++++++++------- src/nanoarrow/nanoarrow_ipc.h | 1 - 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index d0bdcfa43..6ffe963ea 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -66,8 +66,8 @@ struct ArrowIpcField { struct ArrowArray* array; // The cumulative number of buffers preceding this node. int64_t buffer_offset; - // Dictionary information for dictionary-encoded fields - struct ArrowIpcDictionary* dictionary; + // Dictionary identifier for this node + int64_t dictionary_id; }; // Internal data specific to the read/decode process @@ -103,6 +103,11 @@ struct ArrowIpcDecoderPrivate { struct ArrowIpcFooter footer; // Decompressor for compression support struct ArrowIpcDecompressor decompressor; + // Owned dictionary tracker. Only the root decoder will own its dictionary tracker; + // however, a reference to it will be passed to decode methods such that child + // dictionary decoders can share a single dictionary source to support nested + // dictionaries. + struct ArrowIpcDictionaries owned_dictionaries; }; ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error) { @@ -863,11 +868,18 @@ static int ArrowIpcSetDictionaryEncoding( schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED; } - // TODO: should the non extension-type field metadata get moved back to the - // parent field? + // TODO: The non extension-type field metadata should get moved back to the + // parent field if there was any. - // TODO: Record the dictionary id in Schema metadata? (Or keep track of it - // some other way?) + // TODO: Track the dictionary. This can be as a const struct ArrowSchema* pointer + // paried with an identifier, which must be passed down through all of these calls. + // After the schema has been set, this pairing of ArrowSchema* pointer to id must + // be post-processed to: + // + // - Check that identical identifiers refer to equivalent schemas + // - Set the ArrowIpcField dictionary id member + // - Create the IPC decoders (or maybe this should be done lazily on first dictionary + // batch decode) return NANOARROW_OK; } @@ -1428,7 +1440,7 @@ static void ArrowIpcDecoderInitFields(struct ArrowIpcField* fields, field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; - field->dictionary = NULL; + field->dictionary_id = -1; for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { *n_buffers += array_view->layout.buffer_type[i] != NANOARROW_BUFFER_TYPE_NONE; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index fabbf9f75..356d6ab9d 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -489,7 +489,6 @@ void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, - const struct ArrowIpcDecoder* decoder, int64_t id, struct ArrowSchema* schema); From 3525673333bddca9b2fd037abba5e04470ba8861 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 20:48:55 -0600 Subject: [PATCH 12/22] add tests for schema decode --- src/nanoarrow/ipc/decoder.c | 80 ++++++- src/nanoarrow/ipc/decoder_test.cc | 336 +++++++++++++++++------------- 2 files changed, 265 insertions(+), 151 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 6ffe963ea..afedd548d 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -832,6 +832,80 @@ static int ArrowIpcDecoderSetType(struct ArrowSchema* schema, ns(Field_table_t) } } +// A fun corner case when decoding dictionaries: the extension metadata lives with +// the dictionary (i.e., the non-index type); however, the field metadata still +// needs to exist on the field. +static int ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded( + struct ArrowSchema* schema) { + NANOARROW_DCHECK(schema->dictionary != NULL); + struct ArrowMetadataReader reader; + NANOARROW_RETURN_NOT_OK(ArrowMetadataReaderInit(&reader, schema->dictionary->metadata)); + + // For the most common case (no metadata), nothing needs to be done here + if (reader.remaining_keys == 0) { + return NANOARROW_OK; + } + + struct ArrowBuffer field_metadata; + struct ArrowBuffer extension_metadata; + NANOARROW_RETURN_NOT_OK(ArrowMetadataBuilderInit(&field_metadata, NULL)); + ArrowErrorCode result = ArrowMetadataBuilderInit(&extension_metadata, NULL); + if (result != NANOARROW_OK) { + ArrowBufferReset(&field_metadata); + return result; + } + + const struct ArrowStringView extension_name_key = ArrowCharView("ARROW:extension:name"); + const struct ArrowStringView extension_metadata_key = + ArrowCharView("ARROW:extension:metadata"); + + struct ArrowStringView key; + struct ArrowStringView value; + while (reader.remaining_keys > 0) { + result = ArrowMetadataReaderRead(&reader, &key, &value); + if (result != NANOARROW_OK) { + ArrowBufferReset(&field_metadata); + ArrowBufferReset(&extension_metadata); + return result; + } + + int key_is_extension_name = + key.size_bytes == extension_name_key.size_bytes && + strncmp(key.data, extension_name_key.data, key.size_bytes) == 0; + int key_is_extension_metadata = + key.size_bytes == extension_metadata_key.size_bytes && + strncmp(key.data, extension_metadata_key.data, key.size_bytes) == 0; + if (!key_is_extension_name && !key_is_extension_metadata) { + result = ArrowMetadataBuilderAppend(&field_metadata, key, value); + if (result != NANOARROW_OK) { + ArrowBufferReset(&field_metadata); + ArrowBufferReset(&extension_metadata); + return result; + } + } else { + result = ArrowMetadataBuilderAppend(&extension_metadata, key, value); + if (result != NANOARROW_OK) { + ArrowBufferReset(&field_metadata); + ArrowBufferReset(&extension_metadata); + return result; + } + } + } + + result = ArrowSchemaSetMetadata(schema, (char*)field_metadata.data); + if (result != NANOARROW_OK) { + ArrowBufferReset(&field_metadata); + ArrowBufferReset(&extension_metadata); + return result; + } + + result = ArrowSchemaSetMetadata(schema->dictionary, (char*)extension_metadata.data); + ArrowBufferReset(&field_metadata); + ArrowBufferReset(&extension_metadata); + + return result; +} + static int ArrowIpcSetDictionaryEncoding( struct ArrowSchema* schema, ns(DictionaryEncoding_table_t dictionary_encoding), struct ArrowError* error) { @@ -868,8 +942,10 @@ static int ArrowIpcSetDictionaryEncoding( schema->flags |= ARROW_FLAG_DICTIONARY_ORDERED; } - // TODO: The non extension-type field metadata should get moved back to the - // parent field if there was any. + // Field metadata should stay with the field; however, we need the extension metadata + // to stay with the dictionary. + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded(schema), error); // TODO: Track the dictionary. This can be as a const struct ArrowSchema* pointer // paried with an identifier, which must be passed down through all of these calls. diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 28f0b39b2..91f443394 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -21,6 +21,7 @@ #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) #include #include +#include #include #include #endif @@ -732,139 +733,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSetDecompressor) { ArrowIpcDecoderReset(&decoder); } -#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) -class ArrowTypeParameterizedTestFixture - : public ::testing::TestWithParam> { - protected: - std::shared_ptr data_type; -}; - -TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcArrowTypeRoundtrip) { - const std::shared_ptr& data_type = GetParam(); - std::shared_ptr dummy_schema = - arrow::schema({arrow::field("dummy_name", data_type)}); - auto maybe_serialized = arrow::ipc::SerializeSchema(*dummy_schema); - ASSERT_TRUE(maybe_serialized.ok()); - - struct ArrowBufferView buffer_view; - buffer_view.data.data = maybe_serialized.ValueUnsafe()->data(); - buffer_view.size_bytes = maybe_serialized.ValueOrDie()->size(); - - struct ArrowIpcDecoder decoder; - ArrowIpcDecoderInit(&decoder); - ASSERT_EQ(ArrowIpcDecoderVerifyHeader(&decoder, buffer_view, nullptr), NANOARROW_OK); - EXPECT_EQ(decoder.header_size_bytes, buffer_view.size_bytes); - EXPECT_EQ(decoder.body_size_bytes, 0); - - ASSERT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, buffer_view, nullptr), NANOARROW_OK); - struct ArrowSchema schema; - ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, nullptr), NANOARROW_OK); - auto maybe_schema = arrow::ImportSchema(&schema); - ASSERT_TRUE(maybe_schema.ok()); - - // Better failure message if we first check for string equality - EXPECT_EQ(maybe_schema.ValueUnsafe()->ToString(), dummy_schema->ToString()); - EXPECT_TRUE(maybe_schema.ValueUnsafe()->Equals(dummy_schema, true)); - - ArrowIpcDecoderReset(&decoder); -} -#endif - -std::string ArrowSchemaMetadataToString(const char* metadata) { - struct ArrowMetadataReader reader; - auto st = ArrowMetadataReaderInit(&reader, metadata); - EXPECT_EQ(st, NANOARROW_OK); - - bool comma = false; - std::string out; - while (reader.remaining_keys > 0) { - struct ArrowStringView key, value; - auto st = ArrowMetadataReaderRead(&reader, &key, &value); - EXPECT_EQ(st, NANOARROW_OK); - if (comma) { - out += ", "; - } - comma = true; - - out.append(key.data, key.size_bytes); - out += "="; - out.append(value.data, value.size_bytes); - } - return out; -} - -std::string ArrowSchemaToString(const struct ArrowSchema* schema) { - int64_t n = ArrowSchemaToString(schema, nullptr, 0, /*recursive=*/false); - std::vector out_vec(n, '\0'); - ArrowSchemaToString(schema, out_vec.data(), n, /*recursive=*/false); - std::string out(out_vec.data(), out_vec.size()); - - std::string metadata = ArrowSchemaMetadataToString(schema->metadata); - if (!metadata.empty()) { - out += "{" + metadata + "}"; - } - - bool comma = false; - if (schema->format[0] == '+') { - out += "<"; - for (int64_t i = 0; i < schema->n_children; ++i) { - if (comma) { - out += ", "; - } - comma = true; - - auto* child = schema->children[i]; - if (child && child->name[0] != '\0') { - out += child->name; - out += ": "; - } - out += ArrowSchemaToString(schema->children[i]); - } - out += ">"; - } - - return out; -} - -#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) -TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcNanoarrowTypeRoundtrip) { - nanoarrow::UniqueSchema schema; - ASSERT_TRUE( - arrow::ExportSchema(arrow::Schema({arrow::field("", GetParam())}), schema.get()) - .ok()); - - nanoarrow::ipc::UniqueEncoder encoder; - EXPECT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); - - struct ArrowError error; - EXPECT_EQ(ArrowIpcEncoderEncodeSchema(encoder.get(), schema.get(), &error), - NANOARROW_OK) - << error.message; - - nanoarrow::UniqueBuffer buffer; - EXPECT_EQ( - ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, buffer.get()), - NANOARROW_OK); - - struct ArrowBufferView buffer_view; - buffer_view.data.data = buffer->data; - buffer_view.size_bytes = buffer->size_bytes; - - nanoarrow::ipc::UniqueDecoder decoder; - ArrowIpcDecoderInit(decoder.get()); - ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), buffer_view, nullptr), - NANOARROW_OK); - ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), buffer_view, nullptr), - NANOARROW_OK); - - nanoarrow::UniqueSchema roundtripped; - ASSERT_EQ(ArrowIpcDecoderDecodeSchema(decoder.get(), roundtripped.get(), nullptr), - NANOARROW_OK); - - EXPECT_EQ(ArrowSchemaToString(roundtripped.get()), ArrowSchemaToString(schema.get())); -} -#endif - TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchFromShared) { struct ArrowIpcDecoder decoder; struct ArrowError error; @@ -998,8 +866,149 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) { // We will get a (occasional) memory leak if the atomic counter does not work } +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) +class ArrowTypeParameterizedTestFixture + : public ::testing::TestWithParam> { + protected: + std::shared_ptr data_type; +}; + +TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcArrowTypeRoundtrip) { + const std::shared_ptr& data_type = GetParam(); + std::shared_ptr dummy_schema = + arrow::schema({arrow::field("dummy_name", data_type)}); + auto maybe_serialized = arrow::ipc::SerializeSchema(*dummy_schema); + ASSERT_TRUE(maybe_serialized.ok()); + + struct ArrowBufferView buffer_view; + buffer_view.data.data = maybe_serialized.ValueUnsafe()->data(); + buffer_view.size_bytes = maybe_serialized.ValueOrDie()->size(); + + struct ArrowIpcDecoder decoder; + ArrowIpcDecoderInit(&decoder); + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(&decoder, buffer_view, nullptr), NANOARROW_OK); + EXPECT_EQ(decoder.header_size_bytes, buffer_view.size_bytes); + EXPECT_EQ(decoder.body_size_bytes, 0); + + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, buffer_view, nullptr), NANOARROW_OK); + struct ArrowSchema schema; + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, nullptr), NANOARROW_OK); + auto maybe_schema = arrow::ImportSchema(&schema); + ASSERT_TRUE(maybe_schema.ok()); + + // Better failure message if we first check for string equality + EXPECT_EQ(maybe_schema.ValueUnsafe()->ToString(), dummy_schema->ToString()); + EXPECT_TRUE(maybe_schema.ValueUnsafe()->Equals(dummy_schema, true)); + + ArrowIpcDecoderReset(&decoder); +} +#endif + +std::string ArrowSchemaMetadataToString(const char* metadata) { + struct ArrowMetadataReader reader; + auto st = ArrowMetadataReaderInit(&reader, metadata); + EXPECT_EQ(st, NANOARROW_OK); + + bool comma = false; + std::string out; + while (reader.remaining_keys > 0) { + struct ArrowStringView key, value; + auto st = ArrowMetadataReaderRead(&reader, &key, &value); + EXPECT_EQ(st, NANOARROW_OK); + if (comma) { + out += ", "; + } + comma = true; + + out.append(key.data, key.size_bytes); + out += "="; + out.append(value.data, value.size_bytes); + } + return out; +} + +std::string ArrowSchemaToString(const struct ArrowSchema* schema) { + int64_t n = ArrowSchemaToString(schema, nullptr, 0, /*recursive=*/false); + std::vector out_vec(n, '\0'); + ArrowSchemaToString(schema, out_vec.data(), n, /*recursive=*/false); + std::string out(out_vec.data(), out_vec.size()); + + std::string metadata = ArrowSchemaMetadataToString(schema->metadata); + if (!metadata.empty()) { + out += "{" + metadata + "}"; + } + + bool comma = false; + if (schema->format[0] == '+') { + out += "<"; + for (int64_t i = 0; i < schema->n_children; ++i) { + if (comma) { + out += ", "; + } + comma = true; + + auto* child = schema->children[i]; + if (child && child->name[0] != '\0') { + out += child->name; + out += ": "; + } + out += ArrowSchemaToString(schema->children[i]); + } + out += ">"; + } + + return out; +} + +#if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) +TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcNanoarrowTypeRoundtrip) { + if (GetParam()->id() == arrow::Type::DICTIONARY) { + GTEST_SKIP() << "Dictionary array decode is not yet supported"; + } + + nanoarrow::UniqueSchema schema; + ASSERT_TRUE( + arrow::ExportSchema(arrow::Schema({arrow::field("", GetParam())}), schema.get()) + .ok()); + + nanoarrow::ipc::UniqueEncoder encoder; + EXPECT_EQ(ArrowIpcEncoderInit(encoder.get()), NANOARROW_OK); + + struct ArrowError error; + EXPECT_EQ(ArrowIpcEncoderEncodeSchema(encoder.get(), schema.get(), &error), + NANOARROW_OK) + << error.message; + + nanoarrow::UniqueBuffer buffer; + EXPECT_EQ( + ArrowIpcEncoderFinalizeBuffer(encoder.get(), /*encapsulate=*/true, buffer.get()), + NANOARROW_OK); + + struct ArrowBufferView buffer_view; + buffer_view.data.data = buffer->data; + buffer_view.size_bytes = buffer->size_bytes; + + nanoarrow::ipc::UniqueDecoder decoder; + ArrowIpcDecoderInit(decoder.get()); + ASSERT_EQ(ArrowIpcDecoderVerifyHeader(decoder.get(), buffer_view, nullptr), + NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderDecodeHeader(decoder.get(), buffer_view, nullptr), + NANOARROW_OK); + + nanoarrow::UniqueSchema roundtripped; + ASSERT_EQ(ArrowIpcDecoderDecodeSchema(decoder.get(), roundtripped.get(), nullptr), + NANOARROW_OK); + + EXPECT_EQ(ArrowSchemaToString(roundtripped.get()), ArrowSchemaToString(schema.get())); +} +#endif + #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcArrowArrayRoundtrip) { + if (GetParam()->id() == arrow::Type::DICTIONARY) { + GTEST_SKIP() << "Dictionary array decode is not yet supported"; + } + const std::shared_ptr& data_type = GetParam(); std::shared_ptr dummy_schema = arrow::schema({arrow::field("dummy_name", data_type)}); @@ -1091,6 +1100,10 @@ void AssertArrayViewIdentical(const struct ArrowArrayView* actual, #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) TEST_P(ArrowTypeParameterizedTestFixture, NanoarrowIpcNanoarrowArrayRoundtrip) { + if (GetParam()->id() == arrow::Type::DICTIONARY) { + GTEST_SKIP() << "nanoarrow encoder cannot yet encode dictionaries"; + } + struct ArrowError error; nanoarrow::UniqueSchema schema; ASSERT_TRUE( @@ -1170,8 +1183,7 @@ INSTANTIATE_TEST_SUITE_P( arrow::map(arrow::utf8(), arrow::int64(), true), arrow::struct_({arrow::field("col1", arrow::int32()), arrow::field("col2", arrow::utf8())}), - // Zero-size union doesn't roundtrip through the C Data interface until - // Arrow 11 (which is not yet available on all platforms) + // Zero-size union // arrow::sparse_union(FieldVector()), arrow::dense_union(FieldVector()), // No custom type IDs arrow::sparse_union({arrow::field("col1", arrow::int32()), @@ -1188,9 +1200,14 @@ INSTANTIATE_TEST_SUITE_P( // Type with nested metadata arrow::list(arrow::field("some_custom_name", arrow::int32(), - arrow::KeyValueMetadata::Make({"key1"}, {"value1"}))) - - )); + arrow::KeyValueMetadata::Make({"key1"}, {"value1"}))), + // Dictionary encoding + arrow::dictionary(arrow::int32(), arrow::utf8()), + arrow::dictionary(arrow::int32(), arrow::utf8(), true), + // Extension type + arrow::extension::uuid(), + // Dictionary-encoded extension + arrow::dictionary(arrow::int32(), arrow::extension::uuid()))); class ArrowSchemaParameterizedTestFixture : public ::testing::TestWithParam> { @@ -1228,6 +1245,12 @@ TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcArrowSchemaRoundtrip) { } TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcNanoarrowSchemaRoundtrip) { + for (const auto& field : GetParam()->fields()) { + if (field->type()->id() == arrow::Type::DICTIONARY) { + GTEST_SKIP() << "nanoarrow cannot yet encode arrays with dictionaries"; + } + } + const std::shared_ptr& arrow_schema = GetParam(); nanoarrow::UniqueSchema schema; @@ -1265,6 +1288,12 @@ TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcNanoarrowSchemaRoundtrip } TEST_P(ArrowSchemaParameterizedTestFixture, NanoarrowIpcNanoarrowFooterRoundtrip) { + for (const auto& field : GetParam()->fields()) { + if (field->type()->id() == arrow::Type::DICTIONARY) { + GTEST_SKIP() << "nanoarrow cannot yet encode arrays with dictionaries"; + } + } + using namespace nanoarrow::literals; const std::shared_ptr& arrow_schema = GetParam(); @@ -1335,7 +1364,15 @@ INSTANTIATE_TEST_SUITE_P( // Schema metadata arrow::schema({}, arrow::KeyValueMetadata::Make({"key1"}, {"value1"})), // Non-nullable field - arrow::schema({arrow::field("some_name", arrow::int32(), false)}))); + arrow::schema({arrow::field("some_name", arrow::int32(), false)}), + // Dictionary with field metadata + arrow::schema({arrow::field( + "some_name", arrow::dictionary(arrow::int32(), arrow::utf8()), + arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}), + // Dictionary with field metadata + arrow::schema({arrow::field( + "some_name", arrow::dictionary(arrow::int32(), arrow::extension::uuid()), + arrow::KeyValueMetadata::Make({"key1", "key2"}, {"value1", "value2"}))}))); class ArrowTypeIdParameterizedTestFixture : public ::testing::TestWithParam { @@ -1553,9 +1590,9 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { // Create a parent decoder for use in append nanoarrow::ipc::UniqueDecoder parent_decoder; ASSERT_EQ(ArrowIpcDecoderInit(parent_decoder.get()), NANOARROW_OK); - ASSERT_EQ(ArrowIpcDecoderSetEndianness(parent_decoder.get(), - ArrowIpcSystemEndianness()), - NANOARROW_OK); + ASSERT_EQ( + ArrowIpcDecoderSetEndianness(parent_decoder.get(), ArrowIpcSystemEndianness()), + NANOARROW_OK); // Test append first dictionary nanoarrow::UniqueSchema schema1; @@ -1563,9 +1600,9 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { ASSERT_EQ(ArrowSchemaSetTypeStruct(schema1.get(), 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetType(schema1->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetName(schema1->children[0], "value"), NANOARROW_OK); - ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 42, - schema1.get()), - NANOARROW_OK); + ASSERT_EQ( + ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 42, schema1.get()), + NANOARROW_OK); EXPECT_EQ(dictionaries.n_dictionaries, 1); EXPECT_GE(dictionaries.capacity, 1); EXPECT_NE(dictionaries.dictionaries, nullptr); @@ -1575,11 +1612,12 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { nanoarrow::UniqueSchema schema2; ArrowSchemaInit(schema2.get()); ASSERT_EQ(ArrowSchemaSetTypeStruct(schema2.get(), 1), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetType(schema2->children[0], NANOARROW_TYPE_STRING), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetName(schema2->children[0], "value"), NANOARROW_OK); - ASSERT_EQ(ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 99, - schema2.get()), + ASSERT_EQ(ArrowSchemaSetType(schema2->children[0], NANOARROW_TYPE_STRING), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetName(schema2->children[0], "value"), NANOARROW_OK); + ASSERT_EQ( + ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 99, schema2.get()), + NANOARROW_OK); EXPECT_EQ(dictionaries.n_dictionaries, 2); EXPECT_GE(dictionaries.capacity, 2); EXPECT_EQ(dictionaries.dictionaries[0].id, 42); From 1e5f6f2405a659d88414dfd25579dfd0242ca773 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 20:49:56 -0600 Subject: [PATCH 13/22] move test --- src/nanoarrow/ipc/decoder_test.cc | 82 +++++++++++++++---------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 91f443394..a32acb383 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -866,6 +866,47 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) { // We will get a (occasional) memory leak if the atomic counter does not work } +TEST(NanoarrowIpcTest, NanoarrowIpcFooterDecodingErrors) { + struct ArrowError error; + + nanoarrow::ipc::UniqueDecoder decoder; + ArrowIpcDecoderInit(decoder.get()); + + // not enough data to get the size+magic + EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), {{nullptr}, 3}, &error), ESPIPE) + << error.message; + + // doesn't end with magic + EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), {{"\0\0\0\0blargh"}, 10}, &error), + EINVAL) + << error.message; + + // negative size + EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), + {{"\xFF\xFF\xFF\xFF" + "ARROW1"}, + 10}, + &error), + EINVAL) + << error.message; + + // PeekFooter doesn't check for available data + EXPECT_EQ( + ArrowIpcDecoderPeekFooter(decoder.get(), {{"\xFF\xFF\0\0ARROW1"}, 10}, &error), + NANOARROW_OK) + << error.message; + EXPECT_EQ(decoder->header_size_bytes, 0xFFFF); + + decoder->header_size_bytes = -1; + + // VerifyFooter *does* check for enough available data + EXPECT_EQ( + ArrowIpcDecoderVerifyFooter(decoder.get(), {{"\xFF\xFF\0\0ARROW1"}, 10}, &error), + ESPIPE) + << error.message; + EXPECT_EQ(decoder->header_size_bytes, 0xFFFF); +} + #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) class ArrowTypeParameterizedTestFixture : public ::testing::TestWithParam> { @@ -1537,47 +1578,6 @@ INSTANTIATE_TEST_SUITE_P(NanoarrowIpcTest, ArrowTypeIdParameterizedTestFixture, NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO)); #endif -TEST(NanoarrowIpcTest, NanoarrowIpcFooterDecodingErrors) { - struct ArrowError error; - - nanoarrow::ipc::UniqueDecoder decoder; - ArrowIpcDecoderInit(decoder.get()); - - // not enough data to get the size+magic - EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), {{nullptr}, 3}, &error), ESPIPE) - << error.message; - - // doesn't end with magic - EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), {{"\0\0\0\0blargh"}, 10}, &error), - EINVAL) - << error.message; - - // negative size - EXPECT_EQ(ArrowIpcDecoderPeekFooter(decoder.get(), - {{"\xFF\xFF\xFF\xFF" - "ARROW1"}, - 10}, - &error), - EINVAL) - << error.message; - - // PeekFooter doesn't check for available data - EXPECT_EQ( - ArrowIpcDecoderPeekFooter(decoder.get(), {{"\xFF\xFF\0\0ARROW1"}, 10}, &error), - NANOARROW_OK) - << error.message; - EXPECT_EQ(decoder->header_size_bytes, 0xFFFF); - - decoder->header_size_bytes = -1; - - // VerifyFooter *does* check for enough available data - EXPECT_EQ( - ArrowIpcDecoderVerifyFooter(decoder.get(), {{"\xFF\xFF\0\0ARROW1"}, 10}, &error), - ESPIPE) - << error.message; - EXPECT_EQ(decoder->header_size_bytes, 0xFFFF); -} - TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { struct ArrowIpcDictionaries dictionaries; From 3ff6c98c57eef4791c0473ba8da8d3defd4b67cd Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 20:52:48 -0600 Subject: [PATCH 14/22] remove array decoder bits --- src/nanoarrow/ipc/decoder.c | 79 ------------------------------- src/nanoarrow/ipc/decoder_test.cc | 49 ------------------- src/nanoarrow/nanoarrow_ipc.h | 34 ------------- 3 files changed, 162 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index afedd548d..de812013a 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -103,11 +103,6 @@ struct ArrowIpcDecoderPrivate { struct ArrowIpcFooter footer; // Decompressor for compression support struct ArrowIpcDecompressor decompressor; - // Owned dictionary tracker. Only the root decoder will own its dictionary tracker; - // however, a reference to it will be passed to decode methods such that child - // dictionary decoders can share a single dictionary source to support nested - // dictionaries. - struct ArrowIpcDictionaries owned_dictionaries; }; ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error) { @@ -2217,77 +2212,3 @@ ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( ArrowArrayMove(&temp, out); return NANOARROW_OK; } - -ArrowErrorCode ArrowIpcDictionaryDecoderInit(struct ArrowIpcDictionaryDecoder* dictionary, - const struct ArrowIpcDecoder* parent, - int64_t id, - struct ArrowSchema* value_schema) { - memset(dictionary, 0, sizeof(struct ArrowIpcDictionaryDecoder)); - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInit(&dictionary->decoder)); - ArrowErrorCode result = - ArrowIpcDecoderSetEndianness(&dictionary->decoder, parent->endianness); - if (result != NANOARROW_OK) { - ArrowIpcDecoderReset(&dictionary->decoder); - return result; - } - - result = ArrowIpcDecoderSetSchema(&dictionary->decoder, value_schema, NULL); - if (result != NANOARROW_OK) { - return result; - } - - dictionary->id = id; - return NANOARROW_OK; -} - -void ArrowIpcDictionaryDecoderReset(struct ArrowIpcDictionaryDecoder* dictionary) { - NANOARROW_DCHECK(dictionary != NULL); - ArrowIpcDecoderReset(&dictionary->decoder); -} - -void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries) { - NANOARROW_DCHECK(dictionaries != NULL); - memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); -} - -void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { - NANOARROW_DCHECK(dictionaries != NULL); - - for (int64_t i = 0; i < dictionaries->n_dictionaries; i++) { - ArrowIpcDictionaryDecoderReset(dictionaries->dictionaries + i); - } - - if (dictionaries->dictionaries != NULL) { - ArrowFree(dictionaries->dictionaries); - } -} - -ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, - const struct ArrowIpcDecoder* decoder, - int64_t id, struct ArrowSchema* schema) { - struct ArrowIpcDictionaryDecoder decoder_tmp; - NANOARROW_RETURN_NOT_OK( - ArrowIpcDictionaryDecoderInit(&decoder_tmp, decoder, id, schema)); - - if ((dictionaries->n_dictionaries + 1) > dictionaries->capacity) { - int64_t new_capacity = - dictionaries->n_dictionaries == 0 ? 1 : dictionaries->n_dictionaries * 2; - struct ArrowIpcDictionaryDecoder* new_dictionaries = - (struct ArrowIpcDictionaryDecoder*)ArrowRealloc( - dictionaries->dictionaries, - new_capacity * sizeof(struct ArrowIpcDictionaryDecoder)); - if (new_dictionaries == NULL) { - ArrowIpcDictionaryDecoderReset(&decoder_tmp); - return ENOMEM; - } - - dictionaries->dictionaries = new_dictionaries; - dictionaries->capacity = new_capacity; - } - - struct ArrowIpcDictionaryDecoder* dictionary = - dictionaries->dictionaries + dictionaries->n_dictionaries; - memcpy(dictionary, &decoder_tmp, sizeof(struct ArrowIpcDictionaryDecoder)); - dictionaries->n_dictionaries++; - return NANOARROW_OK; -} diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index a32acb383..aef95fe3d 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -1577,52 +1577,3 @@ INSTANTIATE_TEST_SUITE_P(NanoarrowIpcTest, ArrowTypeIdParameterizedTestFixture, NANOARROW_TYPE_DECIMAL256, NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO)); #endif - -TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesLifecycle) { - struct ArrowIpcDictionaries dictionaries; - - // Test init - ArrowIpcDictionariesInit(&dictionaries); - EXPECT_EQ(dictionaries.dictionaries, nullptr); - EXPECT_EQ(dictionaries.n_dictionaries, 0); - EXPECT_EQ(dictionaries.capacity, 0); - - // Create a parent decoder for use in append - nanoarrow::ipc::UniqueDecoder parent_decoder; - ASSERT_EQ(ArrowIpcDecoderInit(parent_decoder.get()), NANOARROW_OK); - ASSERT_EQ( - ArrowIpcDecoderSetEndianness(parent_decoder.get(), ArrowIpcSystemEndianness()), - NANOARROW_OK); - - // Test append first dictionary - nanoarrow::UniqueSchema schema1; - ArrowSchemaInit(schema1.get()); - ASSERT_EQ(ArrowSchemaSetTypeStruct(schema1.get(), 1), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetType(schema1->children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetName(schema1->children[0], "value"), NANOARROW_OK); - ASSERT_EQ( - ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 42, schema1.get()), - NANOARROW_OK); - EXPECT_EQ(dictionaries.n_dictionaries, 1); - EXPECT_GE(dictionaries.capacity, 1); - EXPECT_NE(dictionaries.dictionaries, nullptr); - EXPECT_EQ(dictionaries.dictionaries[0].id, 42); - - // Test append second dictionary - nanoarrow::UniqueSchema schema2; - ArrowSchemaInit(schema2.get()); - ASSERT_EQ(ArrowSchemaSetTypeStruct(schema2.get(), 1), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetType(schema2->children[0], NANOARROW_TYPE_STRING), - NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetName(schema2->children[0], "value"), NANOARROW_OK); - ASSERT_EQ( - ArrowIpcDictionariesAppend(&dictionaries, parent_decoder.get(), 99, schema2.get()), - NANOARROW_OK); - EXPECT_EQ(dictionaries.n_dictionaries, 2); - EXPECT_GE(dictionaries.capacity, 2); - EXPECT_EQ(dictionaries.dictionaries[0].id, 42); - EXPECT_EQ(dictionaries.dictionaries[1].id, 99); - - // Test reset cleans up all resources - ArrowIpcDictionariesReset(&dictionaries); -} diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 356d6ab9d..2e772c1b3 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -467,40 +467,6 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( struct ArrowArray* out, enum ArrowValidationLevel validation_level, struct ArrowError* error); -struct ArrowIpcDictionaryDecoder { - int64_t id; - struct ArrowIpcDecoder decoder; -}; - -ArrowErrorCode ArrowIpcDictionaryDecoderInit(struct ArrowIpcDictionaryDecoder* dictionary, - const struct ArrowIpcDecoder* parent, - int64_t id, - struct ArrowSchema* value_schema); - -void ArrowIpcDictionaryDecoderReset(struct ArrowIpcDictionaryDecoder* dictionary); - -struct ArrowIpcDictionaries { - struct ArrowIpcDictionaryDecoder* dictionaries; - int64_t n_dictionaries; - int64_t capacity; -}; - -void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); -void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); - -ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, - const struct ArrowIpcDecoder* decoder, - int64_t id, struct ArrowSchema* schema); - -/// \brief Append the current DictionaryBatch message to the encoder -/// -/// After a successful call to ArrowIpcDecoderDecodeHeader(), assemble an ArrowArray given -/// a message body and append (or replace) the corresponding entry in this decoder's -/// ArrowIpcDictionaries. -NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderAppendDictionaryFromShared( - struct ArrowIpcDecoder* decoder, struct ArrowIpcSharedBuffer* shared, - enum ArrowValidationLevel validation_level, struct ArrowError* error); - /// \brief An user-extensible input data source struct ArrowIpcInputStream { /// \brief Read up to buf_size_bytes from stream into buf From 41b599c2fda4a5ce963b0ceebcbc24f143db7de5 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 20:59:46 -0600 Subject: [PATCH 15/22] fix errors --- src/nanoarrow/ipc/files_test.cc | 8 ++++---- src/nanoarrow/ipc/reader.c | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/nanoarrow/ipc/files_test.cc b/src/nanoarrow/ipc/files_test.cc index ee2f384c9..6a1d7c787 100644 --- a/src/nanoarrow/ipc/files_test.cc +++ b/src/nanoarrow/ipc/files_test.cc @@ -481,16 +481,16 @@ INSTANTIATE_TEST_SUITE_P( // Files with features that are not yet supported (Dictionary encoding) TestFile::NotSupported( "generated_dictionary_unsigned.stream", - "Schema message field with DictionaryEncoding not supported"), + "Found valid dictionary batch but dictionary encoding is not yet supported"), TestFile::NotSupported( "generated_dictionary.stream", - "Schema message field with DictionaryEncoding not supported"), + "Found valid dictionary batch but dictionary encoding is not yet supported"), TestFile::NotSupported( "generated_nested_dictionary.stream", - "Schema message field with DictionaryEncoding not supported"), + "Found valid dictionary batch but dictionary encoding is not yet supported"), TestFile::NotSupported( "generated_extension.stream", - "Schema message field with DictionaryEncoding not supported") + "Found valid dictionary batch but dictionary encoding is not yet supported") // Comment to keep last line from wrapping )); diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 53937b802..7ecc4ccdd 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -455,9 +455,18 @@ static int ArrowIpcArrayStreamReaderGetNext(struct ArrowArrayStream* stream, } // Make sure we have a RecordBatch message - if (private_data->decoder.message_type != NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH) { - ArrowErrorSet(&private_data->error, "Unexpected message type (expected RecordBatch)"); - return EINVAL; + switch (private_data->decoder.message_type) { + case NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH: + break; + case NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH: + ArrowErrorSet( + &private_data->error, + "Found valid dictionary batch but dictionary encoding is not yet supported"); + return ENOTSUP; + default: + ArrowErrorSet(&private_data->error, + "Unexpected message type (expected RecordBatch)"); + return EINVAL; } // Read in the body From 4f4e971b35b4224537328001622f18af1bf1a44c Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 21:27:55 -0600 Subject: [PATCH 16/22] remove one more change --- src/nanoarrow/ipc/decoder.c | 3 --- src/nanoarrow/ipc/decoder_test.cc | 1 - 2 files changed, 4 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index de812013a..67576c975 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -66,8 +66,6 @@ struct ArrowIpcField { struct ArrowArray* array; // The cumulative number of buffers preceding this node. int64_t buffer_offset; - // Dictionary identifier for this node - int64_t dictionary_id; }; // Internal data specific to the read/decode process @@ -1511,7 +1509,6 @@ static void ArrowIpcDecoderInitFields(struct ArrowIpcField* fields, field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; - field->dictionary_id = -1; for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { *n_buffers += array_view->layout.buffer_type[i] != NANOARROW_BUFFER_TYPE_NONE; diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index aef95fe3d..aef064286 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -44,7 +44,6 @@ struct ArrowIpcField { struct ArrowArrayView* array_view; struct ArrowArray* array; int64_t buffer_offset; - int64_t dictionary_id; }; struct ArrowIpcDecoderPrivate { From a16dcfa028c5ab6a5bdd35c609326692f053585e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 3 Feb 2026 21:44:38 -0600 Subject: [PATCH 17/22] another check --- src/nanoarrow/ipc/decoder.c | 16 +++++++--------- src/nanoarrow/ipc/decoder_test.cc | 19 +------------------ 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 67576c975..c5a182c09 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -940,15 +940,8 @@ static int ArrowIpcSetDictionaryEncoding( NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded(schema), error); - // TODO: Track the dictionary. This can be as a const struct ArrowSchema* pointer - // paried with an identifier, which must be passed down through all of these calls. - // After the schema has been set, this pairing of ArrowSchema* pointer to id must - // be post-processed to: - // - // - Check that identical identifiers refer to equivalent schemas - // - Set the ArrowIpcField dictionary id member - // - Create the IPC decoders (or maybe this should be done lazily on first dictionary - // batch decode) + // TODO: Track the dictionary + // https://github.com/apache/arrow-nanoarrow/issues/844 return NANOARROW_OK; } @@ -1966,6 +1959,11 @@ static int ArrowIpcDecoderWalkGetArray(struct ArrowArrayView* array_view, array_view->children[i], array->children[i], out->children[i], error)); } + if (array->dictionary != NULL) { + ArrowErrorSet(error, "Decode of dictionary array is not yet supported"); + return ENOTSUP; + } + return NANOARROW_OK; } diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index aef064286..2330190fc 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -643,24 +643,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { EXPECT_FALSE(decoder.dictionary->is_delta); // TODO: Access RecordBatch content - - ArrowIpcDecoderReset(&decoder); -} - -TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryRecordBatch) { - struct ArrowIpcDecoder decoder; - struct ArrowError error; - - struct ArrowBufferView data; - data.data.as_uint8 = kDictionaryRecordBatch; - data.size_bytes = sizeof(kDictionaryRecordBatch); - - ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); - - EXPECT_EQ(ArrowIpcDecoderVerifyHeader(&decoder, data, &error), NANOARROW_OK); - ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH); - - // TODO: Decode RecordBatch content populating dictionary array member + // https://github.com/apache/arrow-nanoarrow/issues/845 ArrowIpcDecoderReset(&decoder); } From 4885b747f66461526254e03805c0c03219191693 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Mar 2026 14:18:04 -0600 Subject: [PATCH 18/22] remove unused variable --- src/nanoarrow/ipc/decoder_test.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 2330190fc..6d9bdb0a3 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -206,19 +206,6 @@ static uint8_t kDictionaryBatch[] = { 0x6f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; -static uint8_t kDictionaryRecordBatch[] = { - 0xff, 0xff, 0xff, 0xff, 0x88, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0c, 0x00, 0x16, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x18, 0x00, 0x00, 0x00, 0x08, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x18, 0x00, 0x0c, 0x00, - 0x04, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x10, 0x00, - 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00}; - TEST(NanoarrowIpcTest, NanoarrowIpcCheckHeader) { struct ArrowIpcDecoder decoder; struct ArrowError error; From 0371d3110b9c81d01e8a2c918287e88f4a96d4d9 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Mar 2026 14:33:18 -0600 Subject: [PATCH 19/22] Update src/nanoarrow/ipc/decoder.c Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/nanoarrow/ipc/decoder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index c5a182c09..86d7d5e88 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -907,7 +907,7 @@ static int ArrowIpcSetDictionaryEncoding( case ns(DictionaryKind_DenseArray): break; default: - ArrowErrorSet(error, "Uexpected value for DictionaryKind"); + ArrowErrorSet(error, "Unexpected value for DictionaryKind"); return EINVAL; } From 8aad721f1b97427dbbbcbe8fcba9dd50a5c35939 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Mar 2026 14:34:33 -0600 Subject: [PATCH 20/22] Update src/nanoarrow/nanoarrow_ipc.h Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/nanoarrow/nanoarrow_ipc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 2e772c1b3..a1a172cb4 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -317,7 +317,7 @@ struct ArrowIpcDecoder { int64_t body_size_bytes; /// \brief The last decoded DictionaryBatch - struct ArrowIpcDictionaryBatch* dictionary; + const struct ArrowIpcDictionaryBatch* dictionary; /// \brief The last decoded Footer /// From 09be5a06a3742e1f4578b5815dfc4e119378105e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Mar 2026 14:41:43 -0600 Subject: [PATCH 21/22] more comments --- src/nanoarrow/ipc/decoder_test.cc | 4 ++-- src/nanoarrow/nanoarrow_ipc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 6d9bdb0a3..403229082 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -161,7 +161,7 @@ alignas(8) static uint8_t kSimpleRecordBatchUncompressible[] = { 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -static uint8_t kDictionarySchema[] = { +alignas(8) static uint8_t kDictionarySchema[] = { 0xff, 0xff, 0xff, 0xff, 0x50, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x0e, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x0c, 0x00, @@ -188,7 +188,7 @@ static uint8_t kDictionarySchema[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -static uint8_t kDictionaryBatch[] = { +alignas(8) static uint8_t kDictionaryBatch[] = { 0xff, 0xff, 0xff, 0xff, 0xa8, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x14, 0x00, 0x00, 0x00, 0x20, 0x00, diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index a1a172cb4..b1058ddfb 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -323,7 +323,7 @@ struct ArrowIpcDecoder { /// /// \warning This API is currently only public for use in integration testing; /// use at your own risk. - struct ArrowIpcFooter* footer; + const struct ArrowIpcFooter* footer; /// \brief Private resources managed by this library void* private_data; From 34b78f227d9d43f5f5f8a405f4728463c1c2ab8f Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Mar 2026 23:32:01 -0600 Subject: [PATCH 22/22] don't make footer const --- src/nanoarrow/nanoarrow_ipc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index b1058ddfb..a1a172cb4 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -323,7 +323,7 @@ struct ArrowIpcDecoder { /// /// \warning This API is currently only public for use in integration testing; /// use at your own risk. - const struct ArrowIpcFooter* footer; + struct ArrowIpcFooter* footer; /// \brief Private resources managed by this library void* private_data;