From 709b3bd299050e7f22d4f3e8e16a69500d9ddd7f Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 19 Mar 2026 00:32:45 -0500 Subject: [PATCH 01/17] start --- src/nanoarrow/ipc/decoder.c | 103 ++++++++++++++++++++++++++++++++++ src/nanoarrow/nanoarrow_ipc.h | 14 +++++ 2 files changed, 117 insertions(+) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 400d483ec..900d168a2 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -318,6 +318,109 @@ void ArrowIpcDictionaryEncodingsReset( ArrowBufferReset(&dictionary_encodings->encodings); } +struct ArrowIpcDictionary { + int64_t id; + struct ArrowIpcDecoder decoder; + struct ArrowIpcSharedBuffer buffer; +}; + +static ArrowErrorCode ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary) { + dictionary->id = NANOARROW_IPC_NO_DICTIONARY_ID; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInit(&dictionary->decoder)); + struct ArrowBuffer empty; + ArrowBufferInit(&empty); + ArrowErrorCode result = ArrowIpcSharedBufferInit(&dictionary->buffer, &empty); + if (result != NANOARROW_OK) { + ArrowIpcDecoderReset(&dictionary->decoder); + return result; + } + + return NANOARROW_OK; +} + +static void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary) { + ArrowIpcDecoderReset(&dictionary->decoder); + ArrowIpcSharedBufferReset(&dictionary->buffer); +} + +struct ArrowIpcDictionariesPrivate { + struct ArrowIpcDictionary* dictionaries; + int64_t num_dictionaries; +}; + +static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( + struct ArrowIpcDictionariesPrivate* private_data, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + int64_t* num_initialized_decoders_out) { + ArrowErrorCode result; + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { + struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; + result = ArrowIpcDictionaryInit(dictionary); + if (result != NANOARROW_OK) { + *num_initialized_decoders_out = i; + return result; + } + + // Set dictionary->decoder schema with encodings, being careful not to + // deep copy the schema because this would invalidate dictionary_encodings + // pointers and would prevent nested dictionaries from working. + } + + *num_initialized_decoders_out = private_data->num_dictionaries; + return NANOARROW_OK; +} + +ArrowErrorCode ArrowIpcDictionariesInit( + struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings) { + NANOARROW_DCHECK(dictionaries != NULL); + NANOARROW_DCHECK(dictionary_encodings != NULL); + + memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); + struct ArrowIpcDictionariesPrivate* private_data = + (struct ArrowIpcDictionariesPrivate*)ArrowMalloc( + sizeof(struct ArrowIpcDictionariesPrivate)); + if (private_data == NULL) { + return ENOMEM; + } + + private_data->num_dictionaries = dictionary_encodings->encodings.size_bytes / + sizeof(struct ArrowIpcDictionaryEncoding); + + private_data->dictionaries = + ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionaries)); + if (private_data->dictionaries == NULL) { + ArrowFree(private_data); + return ENOMEM; + } + + int64_t num_initialized_dictionaries = 0; + ArrowErrorCode result = ArrowIpcDictionariesInitDictionaries( + private_data, dictionary_encodings, &num_initialized_dictionaries); + if (num_initialized_dictionaries != private_data->num_dictionaries) { + for (int64_t i = 0; i < num_initialized_dictionaries; i++) { + ArrowIpcDictionaryReset(private_data->dictionaries + i); + } + + ArrowFree(private_data->dictionaries); + ArrowFree(private_data); + return result; + } + + return NANOARROW_OK; +} + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + struct ArrowIpcDictionariesPrivate* private_data = + (struct ArrowIpcDictionariesPrivate*)dictionaries->private_data; + + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { + struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; + ArrowIpcDictionaryReset(dictionary); + } +} + ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { memset(decoder, 0, sizeof(struct ArrowIpcDecoder)); struct ArrowIpcDecoderPrivate* private_data = diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ba94e5b5d..ba16830a6 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -243,6 +243,20 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodin NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); + +struct ArrowIpcDictionaries { + void* private_data; +}; + +NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesInit( + struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings); + +NANOARROW_DLL void ArrowIpcDictionariesReset( + struct ArrowIpcDictionaries* dictionaries); + + + /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); From eac7880bba8542e9bd3ed3dc5255b7a218a1ffc4 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 29 Mar 2026 20:43:55 -0500 Subject: [PATCH 02/17] fix actual errors --- src/nanoarrow/ipc/decoder.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 900d168a2..bb03b5a44 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -388,7 +388,7 @@ ArrowErrorCode ArrowIpcDictionariesInit( sizeof(struct ArrowIpcDictionaryEncoding); private_data->dictionaries = - ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionaries)); + ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); if (private_data->dictionaries == NULL) { ArrowFree(private_data); return ENOMEM; @@ -407,6 +407,7 @@ ArrowErrorCode ArrowIpcDictionariesInit( return result; } + dictionaries->private_data = private_data; return NANOARROW_OK; } From 0b02eeb0c111a1f9fa6bcf44904f8c3ad397a15b Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 29 Mar 2026 21:28:04 -0500 Subject: [PATCH 03/17] in theory handle the init dictionary --- src/nanoarrow/ipc/decoder.c | 51 ++++++++++++++++++++++++++++++----- src/nanoarrow/nanoarrow_ipc.h | 23 ++++++++++------ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index bb03b5a44..3f30c34b3 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -312,6 +312,24 @@ const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( return NULL; } +const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, int64_t id) { + NANOARROW_DCHECK(dictionary_encodings != NULL); + int64_t length = dictionary_encodings->encodings.size_bytes / + sizeof(struct ArrowIpcDictionaryEncoding); + const struct ArrowIpcDictionaryEncoding* data = + (const struct ArrowIpcDictionaryEncoding*)dictionary_encodings->encodings.data; + + for (int64_t i = 0; i < length; i++) { + const struct ArrowIpcDictionaryEncoding* encoding = data + i; + if (encoding->id == id) { + return encoding; + } + } + + return NULL; +} + void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionary_encodings) { NANOARROW_DCHECK(dictionary_encodings != NULL); @@ -351,19 +369,37 @@ struct ArrowIpcDictionariesPrivate { static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( struct ArrowIpcDictionariesPrivate* private_data, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, - int64_t* num_initialized_decoders_out) { + int64_t* num_initialized_decoders_out, struct ArrowError* error) { ArrowErrorCode result; + + // Before this is called we need to set the dictionary->id member (and + // need to count the number of dictionary IDs in dictionary encodings before + // any of that) + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; result = ArrowIpcDictionaryInit(dictionary); if (result != NANOARROW_OK) { + ArrowErrorSet(error, "Internal error (failed to initialize dictionary)"); *num_initialized_decoders_out = i; return result; } - // Set dictionary->decoder schema with encodings, being careful not to - // deep copy the schema because this would invalidate dictionary_encodings - // pointers and would prevent nested dictionaries from working. + const struct ArrowIpcDictionaryEncoding* encoding = + ArrowIpcDictionaryEncodingsFindById(dictionary_encodings, dictionary->id); + if (encoding == NULL) { + ArrowErrorSet(error, + "Internal error (dictionary with id not present in encodings list)"); + *num_initialized_decoders_out = i; + return EINVAL; + } + + result = ArrowIpcDecoderSetSchemaWithDictionaries( + &dictionary->decoder, encoding->schema, dictionary_encodings, error); + if (result != NANOARROW_OK) { + *num_initialized_decoders_out = i; + return result; + } } *num_initialized_decoders_out = private_data->num_dictionaries; @@ -372,7 +408,8 @@ static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( ArrowErrorCode ArrowIpcDictionariesInit( struct ArrowIpcDictionaries* dictionaries, - const struct ArrowIpcDictionaryEncodings* dictionary_encodings) { + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowError* error) { NANOARROW_DCHECK(dictionaries != NULL); NANOARROW_DCHECK(dictionary_encodings != NULL); @@ -381,6 +418,7 @@ ArrowErrorCode ArrowIpcDictionariesInit( (struct ArrowIpcDictionariesPrivate*)ArrowMalloc( sizeof(struct ArrowIpcDictionariesPrivate)); if (private_data == NULL) { + ArrowErrorSet(error, "Failed to allocate ArrowIpcDictionariesPrivate"); return ENOMEM; } @@ -390,13 +428,14 @@ ArrowErrorCode ArrowIpcDictionariesInit( private_data->dictionaries = ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); if (private_data->dictionaries == NULL) { + ArrowErrorSet(error, "Failed to allocate ArrowIpcDictionary array"); ArrowFree(private_data); return ENOMEM; } int64_t num_initialized_dictionaries = 0; ArrowErrorCode result = ArrowIpcDictionariesInitDictionaries( - private_data, dictionary_encodings, &num_initialized_dictionaries); + private_data, dictionary_encodings, &num_initialized_dictionaries, error); if (num_initialized_dictionaries != private_data->num_dictionaries) { for (int64_t i = 0; i < num_initialized_dictionaries; i++) { ArrowIpcDictionaryReset(private_data->dictionaries + i); diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ba16830a6..ca461acf1 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -239,23 +239,30 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodin const struct ArrowIpcDictionaryEncodings* dictionaries, const struct ArrowSchema* schema); +/// \brief Resolve the first ArrowIpcDictionaryEncoding for a given identifier +/// +/// Note that there may be multiple dictionary-encoded fields with the same +/// identifier in a given schema; however, all of them should have an identical +/// value and index data type. +/// +/// Returns NULL if id does not refer to any of the encodings in this list. +const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, int64_t id); + /// \brief Release an encodings list and associated resources NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); - struct ArrowIpcDictionaries { void* private_data; }; -NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesInit( - struct ArrowIpcDictionaries* dictionaries, - const struct ArrowIpcDictionaryEncodings* dictionary_encodings); - -NANOARROW_DLL void ArrowIpcDictionariesReset( - struct ArrowIpcDictionaries* dictionaries); - +NANOARROW_DLL ArrowErrorCode +ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowError* error); +NANOARROW_DLL void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); From 03a1fa222c32eef36e6bd30b2aaec6beb8359bf0 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 29 Mar 2026 22:11:57 -0500 Subject: [PATCH 04/17] in theory get the init working --- src/nanoarrow/ipc/decoder.c | 99 +++++++++++++++++++++++++++++++---- src/nanoarrow/nanoarrow_ipc.h | 4 ++ 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 3f30c34b3..a90f6e4b8 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -330,6 +330,48 @@ const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( return NULL; } +static int ArrowIpcIdsListContains(const struct ArrowBuffer* ids_buffer, int64_t num_ids, + int64_t id) { + const int64_t* ids = (const int64_t*)ids_buffer->data; + for (int64_t i = 0; i < num_ids; i++) { + if (ids[i] == id) { + return 1; + } + } + + return 0; +} + +ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowBuffer* out) { + int64_t length = dictionary_encodings->encodings.size_bytes / + sizeof(struct ArrowIpcDictionaryEncoding); + const struct ArrowIpcDictionaryEncoding* data = + (const struct ArrowIpcDictionaryEncoding*)dictionary_encodings->encodings.data; + + struct ArrowBuffer tmp; + ArrowBufferInit(&tmp); + + ArrowErrorCode result; + int64_t num_ids = 0; + for (int64_t i = 0; i < length; i++) { + const struct ArrowIpcDictionaryEncoding* encoding = data + i; + if (!ArrowIpcIdsListContains(&tmp, num_ids, encoding->id)) { + result = ArrowBufferAppendInt64(&tmp, encoding->id); + if (result != NANOARROW_OK) { + ArrowBufferReset(&tmp); + return result; + } + + ++num_ids; + } + } + + ArrowBufferMove(&tmp, out); + return NANOARROW_OK; +} + void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionary_encodings) { NANOARROW_DCHECK(dictionary_encodings != NULL); @@ -366,15 +408,33 @@ struct ArrowIpcDictionariesPrivate { int64_t num_dictionaries; }; +static void ArrowIpcDictionariesNoOpSchemaRelease(struct ArrowSchema* schema) { + NANOARROW_UNUSED(schema); +} + static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( struct ArrowIpcDictionariesPrivate* private_data, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, - int64_t* num_initialized_decoders_out, struct ArrowError* error) { + const struct ArrowBuffer* unique_ids_buffer, int64_t* num_initialized_decoders_out, + struct ArrowError* error) { ArrowErrorCode result; - - // Before this is called we need to set the dictionary->id member (and - // need to count the number of dictionary IDs in dictionary encodings before - // any of that) + const int64_t* unique_ids = (const int64_t*)unique_ids_buffer->data; + + // To set the schema of each ArrowIpcDictionary's encoder, we need to construct a + // const ArrowSchema* that points to a struct with exactly one child that is the + // value type. To support nested dictionaries that may exist within + // ArrowIpcDictionaryEncodings, we can't deep copy the value type because it would + // invalidate pointer references within dictionary_encodings (i.e., nested dictionaries + // would fail to resolve when we call ArrowIpcDecoderSetSchemaWithDictionaries()). + // To make this work, we create an ArrowSchema with a noop release to pass to + // ArrowIpcDecoderSetSchemaWithDictionaries(). This works because that function + // accepts a const value (i.e., never releases the schema argument). + struct ArrowSchema decoder_schema; + memset(&decoder_schema, 0, sizeof(struct ArrowSchema)); + decoder_schema.name = ""; + decoder_schema.format = "+s"; + decoder_schema.n_children = 1; + decoder_schema.release = &ArrowIpcDictionariesNoOpSchemaRelease; for (int64_t i = 0; i < private_data->num_dictionaries; i++) { struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; @@ -386,7 +446,7 @@ static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( } const struct ArrowIpcDictionaryEncoding* encoding = - ArrowIpcDictionaryEncodingsFindById(dictionary_encodings, dictionary->id); + ArrowIpcDictionaryEncodingsFindById(dictionary_encodings, unique_ids[i]); if (encoding == NULL) { ArrowErrorSet(error, "Internal error (dictionary with id not present in encodings list)"); @@ -394,12 +454,18 @@ static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( return EINVAL; } + // The decoder's schema will be a struct with one column (the value type of + // the dictionary-encoded field) + decoder_schema.children = (struct ArrowSchema**)&encoding->schema->dictionary; + result = ArrowIpcDecoderSetSchemaWithDictionaries( - &dictionary->decoder, encoding->schema, dictionary_encodings, error); + &dictionary->decoder, &decoder_schema, dictionary_encodings, error); if (result != NANOARROW_OK) { *num_initialized_decoders_out = i; return result; } + + dictionary->id = unique_ids[i]; } *num_initialized_decoders_out = private_data->num_dictionaries; @@ -422,6 +488,18 @@ ArrowErrorCode ArrowIpcDictionariesInit( return ENOMEM; } + // Count the number of unique IDs. This will usually be the number of dictionary + // encodings but may be much smaller if there are a lot of dictionary encoded fields + // that use the same dictionary. + struct ArrowBuffer unique_ids; + ArrowErrorCode result = + ArrowIpcDictionaryEncodingsUniqueIds(dictionary_encodings, &unique_ids); + if (result != NANOARROW_OK) { + ArrowErrorSet(error, "Failed to extract unique identifiers"); + ArrowFree(private_data); + return result; + } + private_data->num_dictionaries = dictionary_encodings->encodings.size_bytes / sizeof(struct ArrowIpcDictionaryEncoding); @@ -429,13 +507,16 @@ ArrowErrorCode ArrowIpcDictionariesInit( ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); if (private_data->dictionaries == NULL) { ArrowErrorSet(error, "Failed to allocate ArrowIpcDictionary array"); + ArrowBufferReset(&unique_ids); ArrowFree(private_data); return ENOMEM; } int64_t num_initialized_dictionaries = 0; - ArrowErrorCode result = ArrowIpcDictionariesInitDictionaries( - private_data, dictionary_encodings, &num_initialized_dictionaries, error); + result = ArrowIpcDictionariesInitDictionaries(private_data, dictionary_encodings, + &unique_ids, + &num_initialized_dictionaries, error); + ArrowBufferReset(&unique_ids); if (num_initialized_dictionaries != private_data->num_dictionaries) { for (int64_t i = 0; i < num_initialized_dictionaries; i++) { ArrowIpcDictionaryReset(private_data->dictionaries + i); diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ca461acf1..347d7c802 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -249,6 +249,10 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodin const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, int64_t id); +ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowBuffer* out); + /// \brief Release an encodings list and associated resources NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); From 31d7830ffc31411db5adf11c8299355b6be06de2 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 29 Mar 2026 22:40:02 -0500 Subject: [PATCH 05/17] basic init --- src/nanoarrow/ipc/decoder.c | 12 +++++++---- src/nanoarrow/ipc/decoder_test.cc | 36 +++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index a90f6e4b8..d89bf8fe9 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -450,7 +450,7 @@ static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( if (encoding == NULL) { ArrowErrorSet(error, "Internal error (dictionary with id not present in encodings list)"); - *num_initialized_decoders_out = i; + *num_initialized_decoders_out = i + 1; return EINVAL; } @@ -461,7 +461,7 @@ static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( result = ArrowIpcDecoderSetSchemaWithDictionaries( &dictionary->decoder, &decoder_schema, dictionary_encodings, error); if (result != NANOARROW_OK) { - *num_initialized_decoders_out = i; + *num_initialized_decoders_out = i + 1; return result; } @@ -500,9 +500,10 @@ ArrowErrorCode ArrowIpcDictionariesInit( return result; } - private_data->num_dictionaries = dictionary_encodings->encodings.size_bytes / - sizeof(struct ArrowIpcDictionaryEncoding); + // unique_ids is a buffer of int64_t + private_data->num_dictionaries = unique_ids.size_bytes / sizeof(int64_t); + // Allocate the array of ArrowIpcDictionary private_data->dictionaries = ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); if (private_data->dictionaries == NULL) { @@ -540,6 +541,9 @@ void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; ArrowIpcDictionaryReset(dictionary); } + + ArrowFree(private_data->dictionaries); + ArrowFree(private_data); } ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 5f958367a..b338c4baa 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -642,7 +642,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } -TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchInfo) { struct ArrowIpcDecoder decoder; struct ArrowError error; @@ -659,12 +659,40 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatch) { EXPECT_EQ(decoder.dictionary->id, 0); EXPECT_FALSE(decoder.dictionary->is_delta); - // TODO: Access RecordBatch content - // https://github.com/apache/arrow-nanoarrow/issues/845 - ArrowIpcDecoderReset(&decoder); } +TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesDecode) { + struct ArrowIpcDictionaryEncodings dictionary_encodings; + struct ArrowIpcDictionaries dictionaries; + struct ArrowError error; + struct ArrowSchema schema; + + // Make a dictionary encoded schema that matches that of the dictionary example batch + ArrowSchemaInit(&schema); + ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema, 1), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(schema.children[0]), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema.children[0]->dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); + + // Initialize the dictionary encodings with a single dictionary + ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend( + &dictionary_encodings, {.id = 0, + .kind = NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY, + .schema = schema.children[0]}), + NANOARROW_OK); + + // Initialize the dictionaires with the encodings + ASSERT_EQ(ArrowIpcDictionariesInit(&dictionaries, &dictionary_encodings, &error), + NANOARROW_OK); + + ArrowIpcDictionariesReset(&dictionaries); + ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); + ArrowSchemaRelease(&schema); +} + TEST(NanoarrowIpcTest, NanoarrowIpcSetSchema) { struct ArrowIpcDecoder decoder; struct ArrowSchema schema; From d133f0283675a8a2104dd2906705a1ac638fd668 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sun, 29 Mar 2026 22:53:01 -0500 Subject: [PATCH 06/17] namespace header --- src/nanoarrow/nanoarrow_ipc.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 347d7c802..eda6edc1a 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -115,8 +115,16 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsAppend) #define ArrowIpcDictionaryEncodingsFind \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsFind) +#define ArrowIpcDictionaryEncodingsFindById \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsFindById) +#define ArrowIpcDictionaryEncodingsUniqueIds \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsUniqueIds) #define ArrowIpcDictionaryEncodingsReset \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsReset) +#define ArrowIpcDictionariesInit \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionariesInit) +#define ArrowIpcDictionariesReset \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionariesReset) #endif @@ -246,10 +254,11 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodin /// value and index data type. /// /// Returns NULL if id does not refer to any of the encodings in this list. -const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( +NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* +ArrowIpcDictionaryEncodingsFindById( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, int64_t id); -ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( +NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowBuffer* out); From 49b421d39c87bb57e6aafc520081591f848e6077 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 31 Mar 2026 11:09:41 -0500 Subject: [PATCH 07/17] batch decoded! --- src/nanoarrow/ipc/decoder.c | 133 +++++++++++++++++++++++++++--- src/nanoarrow/ipc/decoder_test.cc | 43 +++++++--- src/nanoarrow/nanoarrow_ipc.h | 13 +++ 3 files changed, 167 insertions(+), 22 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index d89bf8fe9..5032fbca0 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -57,6 +57,8 @@ #define NANOARROW_IPC_NO_DICTIONARY_ID INT64_MIN +#define ns(x) FLATBUFFERS_WRAP_NAMESPACE(org_apache_arrow_flatbuf, x) + // Internal representation of a parsed "Field" from flatbuffers. This // represents a field in a depth-first walk of column arrays and their // children. @@ -381,26 +383,46 @@ void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionary { int64_t id; struct ArrowIpcDecoder decoder; - struct ArrowIpcSharedBuffer buffer; + struct ArrowArray current_value; }; static ArrowErrorCode ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary) { dictionary->id = NANOARROW_IPC_NO_DICTIONARY_ID; NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInit(&dictionary->decoder)); - struct ArrowBuffer empty; - ArrowBufferInit(&empty); - ArrowErrorCode result = ArrowIpcSharedBufferInit(&dictionary->buffer, &empty); - if (result != NANOARROW_OK) { - ArrowIpcDecoderReset(&dictionary->decoder); - return result; + memset(&dictionary->current_value, 0, sizeof(struct ArrowArray)); + return NANOARROW_OK; +} + +static ArrowErrorCode ArrowIpcDictionaryReplace(struct ArrowIpcDictionary* dictionary, + struct ArrowArray* value, + struct ArrowError* error) { + NANOARROW_UNUSED(error); + + if (dictionary->current_value.release != NULL) { + ArrowArrayRelease(&dictionary->current_value); } + ArrowArrayMove(value, &dictionary->current_value); + return NANOARROW_OK; +} + +static ArrowErrorCode ArrowIpcDictionaryAppend(struct ArrowIpcDictionary* dictionary, + struct ArrowArray* value, + struct ArrowError* error) { + if (dictionary->current_value.release != NULL) { + ArrowErrorSet(error, "Dictionary concatenation is not yet supported"); + return ENOTSUP; + } + + ArrowArrayMove(value, &dictionary->current_value); return NANOARROW_OK; } static void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary) { ArrowIpcDecoderReset(&dictionary->decoder); - ArrowIpcSharedBufferReset(&dictionary->buffer); + if (dictionary->current_value.release) { + ArrowArrayRelease(&dictionary->current_value); + } } struct ArrowIpcDictionariesPrivate { @@ -518,7 +540,7 @@ ArrowErrorCode ArrowIpcDictionariesInit( &unique_ids, &num_initialized_dictionaries, error); ArrowBufferReset(&unique_ids); - if (num_initialized_dictionaries != private_data->num_dictionaries) { + if (result != NANOARROW_OK) { for (int64_t i = 0; i < num_initialized_dictionaries; i++) { ArrowIpcDictionaryReset(private_data->dictionaries + i); } @@ -532,6 +554,38 @@ ArrowErrorCode ArrowIpcDictionariesInit( return NANOARROW_OK; } +static struct ArrowIpcDictionary* ArrowIpcDictionariesFindById( + struct ArrowIpcDictionaries* dictionaries, int64_t id) { + NANOARROW_DCHECK(dictionaries != NULL); + + struct ArrowIpcDictionariesPrivate* private_data = + (struct ArrowIpcDictionariesPrivate*)dictionaries->private_data; + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { + struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; + if (dictionary->id == id) { + return dictionary; + } + } + + return NULL; +} + +ArrowErrorCode ArrowIpcDictionariesFindCurrentValue( + struct ArrowIpcDictionaries* dictionaries, int64_t id, const struct ArrowArray** out, + struct ArrowError* error) { + struct ArrowIpcDictionary* dictionary = ArrowIpcDictionariesFindById(dictionaries, id); + if (dictionary == NULL) { + ArrowErrorSet(error, + "Can't find value for dictionary with ID %" PRId64 + " (dictionary definition not found)", + id); + return EINVAL; + } + + *out = &dictionary->current_value; + return NANOARROW_OK; +} + void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { NANOARROW_DCHECK(dictionaries != NULL); struct ArrowIpcDictionariesPrivate* private_data = @@ -625,8 +679,6 @@ static inline int32_t ArrowIpcReadInt32LE(struct ArrowBufferView* data, int swap return value; } -#define ns(x) FLATBUFFERS_WRAP_NAMESPACE(org_apache_arrow_flatbuf, x) - static int ArrowIpcDecoderSetMetadata(struct ArrowSchema* schema, ns(KeyValue_vec_t) kv_vec, struct ArrowError* error) { @@ -2603,3 +2655,62 @@ ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( ArrowArrayMove(&temp, out); return NANOARROW_OK; } + +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeDictionary( + struct ArrowIpcDecoder* decoder, struct ArrowIpcSharedBuffer* shared, + enum ArrowValidationLevel validation_level, struct ArrowIpcDictionaries* dictionaries, + struct ArrowError* error) { + NANOARROW_DCHECK(decoder != NULL); + NANOARROW_DCHECK(shared != NULL); + NANOARROW_DCHECK(dictionaries != NULL); + + struct ArrowIpcDecoderPrivate* private_data = + (struct ArrowIpcDecoderPrivate*)decoder->private_data; + + if (decoder->message_type != NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH) { + ArrowErrorSet(error, "decoder did not just decode a DictionaryBatch message"); + return EINVAL; + } + + struct ArrowIpcDictionary* dictionary = + ArrowIpcDictionariesFindById(dictionaries, decoder->dictionary->id); + if (dictionary == NULL) { + ArrowErrorSet(error, + "Can't decode DictionaryBatch with ID %" PRId64 + " (dictionary definition not found)", + decoder->dictionary->id); + return EINVAL; + } + + // Set the dictionary->decoder's last message type and last message so that we can + // decode the value. + ns(DictionaryBatch_table_t) dictionary_batch = + (ns(DictionaryBatch_table_t))private_data->last_message; + ns(RecordBatch_table_t) record_batch = ns(DictionaryBatch_data(dictionary_batch)); + + struct ArrowIpcDecoderPrivate* dictionary_decoder_private_data = + (struct ArrowIpcDecoderPrivate*)dictionary->decoder.private_data; + dictionary->decoder.message_type = NANOARROW_IPC_MESSAGE_TYPE_RECORD_BATCH; + dictionary_decoder_private_data->last_message = record_batch; + + struct ArrowArray tmp; + + // TODO: provide ArrowIpcDecoderDecodeArrayInternalWithDictionaries to handle nested + // dictionaries + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayInternal( + &dictionary->decoder, 0, &tmp, validation_level, error)); + + ArrowErrorCode result; + if (decoder->dictionary->is_delta) { + result = ArrowIpcDictionaryAppend(dictionary, &tmp, error); + } else { + result = ArrowIpcDictionaryReplace(dictionary, &tmp, error); + } + + if (result != NANOARROW_OK) { + ArrowArrayRelease(&tmp); + return result; + } + + return NANOARROW_OK; +} diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index b338c4baa..f7ab4dedd 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -642,10 +642,13 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } -TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchInfo) { +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { + struct ArrowIpcDictionaryEncodings dictionary_encodings; + struct ArrowIpcDictionaries dictionaries; struct ArrowIpcDecoder decoder; struct ArrowError error; + // Decode a dictionary batch and inspect metadata struct ArrowBufferView data; data.data.as_uint8 = kDictionaryBatch; data.size_bytes = sizeof(kDictionaryBatch); @@ -659,13 +662,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchInfo) { EXPECT_EQ(decoder.dictionary->id, 0); EXPECT_FALSE(decoder.dictionary->is_delta); - ArrowIpcDecoderReset(&decoder); -} - -TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesDecode) { - struct ArrowIpcDictionaryEncodings dictionary_encodings; - struct ArrowIpcDictionaries dictionaries; - struct ArrowError error; struct ArrowSchema schema; // Make a dictionary encoded schema that matches that of the dictionary example batch @@ -673,8 +669,9 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesDecode) { ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema, 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); ASSERT_EQ(ArrowSchemaAllocateDictionary(schema.children[0]), NANOARROW_OK); - ASSERT_EQ(ArrowSchemaSetType(schema.children[0]->dictionary, NANOARROW_TYPE_STRING), - NANOARROW_OK); + ASSERT_EQ( + ArrowSchemaInitFromType(schema.children[0]->dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); // Initialize the dictionary encodings with a single dictionary ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); @@ -686,11 +683,35 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionariesDecode) { // Initialize the dictionaires with the encodings ASSERT_EQ(ArrowIpcDictionariesInit(&dictionaries, &dictionary_encodings, &error), - NANOARROW_OK); + NANOARROW_OK) + << error.message; + // Decode the dictionary batch + data.data.as_uint8 += decoder.header_size_bytes; + data.size_bytes -= decoder.header_size_bytes; + struct ArrowBuffer body; + ArrowBufferInit(&body); + ASSERT_EQ(ArrowBufferAppendBufferView(&body, data), NANOARROW_OK); + + struct ArrowIpcSharedBuffer shared; + ASSERT_EQ(ArrowIpcSharedBufferInit(&shared, &body), NANOARROW_OK); + ASSERT_EQ( + ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, + &dictionaries, &error), + NANOARROW_OK) + << error.message; + + const struct ArrowArray* dictionary_value; + ASSERT_EQ( + ArrowIpcDictionariesFindCurrentValue(&dictionaries, 0, &dictionary_value, &error), + NANOARROW_OK); + ASSERT_NE(dictionary_value->release, nullptr); + + ArrowIpcSharedBufferReset(&shared); ArrowIpcDictionariesReset(&dictionaries); ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); ArrowSchemaRelease(&schema); + ArrowIpcDecoderReset(&decoder); } TEST(NanoarrowIpcTest, NanoarrowIpcSetSchema) { diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index eda6edc1a..f818294e3 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -123,8 +123,12 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsReset) #define ArrowIpcDictionariesInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionariesInit) +#define ArrowIpcDictionariesFindCurrentValue \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionariesFindCurrentValue) #define ArrowIpcDictionariesReset \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionariesReset) +#define ArrowIpcDecoderDecodeDictionary \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeDictionary) #endif @@ -275,6 +279,10 @@ ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error); +NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesFindCurrentValue( + struct ArrowIpcDictionaries* dictionaries, int64_t id, const struct ArrowArray** out, + struct ArrowError* error); + NANOARROW_DLL void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match @@ -608,6 +616,11 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeArrayFromShared( struct ArrowArray* out, enum ArrowValidationLevel validation_level, struct ArrowError* error); +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeDictionary( + struct ArrowIpcDecoder* decoder, struct ArrowIpcSharedBuffer* shared, + enum ArrowValidationLevel validation_level, struct ArrowIpcDictionaries* out, + struct ArrowError* error); + /// \brief An user-extensible input data source struct ArrowIpcInputStream { /// \brief Read up to buf_size_bytes from stream into buf From 76008ac4bf97c55b0e57c2634b970edf1f360f5d Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 31 Mar 2026 11:20:39 -0500 Subject: [PATCH 08/17] don't bother with c99 inits --- src/nanoarrow/ipc/decoder_test.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index f7ab4dedd..042f1d876 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -675,10 +675,11 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { // Initialize the dictionary encodings with a single dictionary ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); - ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend( - &dictionary_encodings, {.id = 0, - .kind = NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY, - .schema = schema.children[0]}), + struct ArrowIpcDictionaryEncoding encoding; + encoding.id = 0; + encoding.kind = NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY; + encoding.schema = schema.children[0]; + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend(&dictionary_encodings, encoding), NANOARROW_OK); // Initialize the dictionaires with the encodings From 31a36b212383a87fc95ef6bae3d3a5fde0356824 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 20:59:14 -0500 Subject: [PATCH 09/17] array decode! --- src/nanoarrow/ipc/decoder.c | 19 +++++++++++++------ src/nanoarrow/ipc/decoder_test.cc | 4 ++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 5032fbca0..902b9b301 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2068,6 +2068,13 @@ struct ArrowIpcBufferFactory { /// should be made. struct ArrowIpcDecompressor* decompressor; + /// \brief Buffer factory provided buffer length + /// + /// Rather than use the body length declared by the flatbuffer messsage, this is the + /// length that should be used to check the bounds of a buffer source (i.e., this is + /// the actual available length as opposed to the theoretical length). + int64_t buffer_length; + /// \brief Caller-defined private data to be used in the callback. /// /// Usually this would be a description of where the body has been read into memory or @@ -2158,6 +2165,7 @@ static struct ArrowIpcBufferFactory ArrowIpcBufferFactoryFromView( struct ArrowIpcBufferFactory out; out.make_buffer = &ArrowIpcMakeBufferFromView; out.decompressor = NULL; + out.buffer_length = buffer_view->size_bytes; out.private_data = buffer_view; return out; } @@ -2198,6 +2206,7 @@ static struct ArrowIpcBufferFactory ArrowIpcBufferFactoryFromShared( struct ArrowIpcBufferFactory out; out.make_buffer = &ArrowIpcMakeBufferFromShared; out.decompressor = NULL; + out.buffer_length = shared->private_src.size_bytes; out.private_data = shared; return out; } @@ -2332,7 +2341,6 @@ struct ArrowIpcArraySetter { int64_t field_i; ns(Buffer_vec_t) buffers; int64_t buffer_i; - int64_t body_size_bytes; struct ArrowIpcBufferSource src; struct ArrowIpcBufferFactory factory; enum ArrowIpcMetadataVersion version; @@ -2351,11 +2359,11 @@ static int ArrowIpcDecoderMakeBuffer(struct ArrowIpcArraySetter* setter, int64_t // Check that this buffer fits within the body int64_t buffer_start = offset; int64_t buffer_end = buffer_start + length; - if (buffer_start < 0 || buffer_end > setter->body_size_bytes) { + if (buffer_start < 0 || buffer_end > setter->factory.buffer_length) { ArrowErrorSet(error, "Buffer requires body offsets [%" PRId64 "..%" PRId64 ") but body has size %" PRId64, - buffer_start, buffer_end, setter->body_size_bytes); + buffer_start, buffer_end, setter->factory.buffer_length); return EINVAL; } @@ -2541,7 +2549,6 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayViewInternal( setter.field_i = field_i; setter.buffers = ns(RecordBatch_buffers(batch)); setter.buffer_i = root->buffer_offset - 1; - setter.body_size_bytes = decoder->body_size_bytes; setter.factory = factory; setter.src.codec = decoder->codec; setter.src.swap_endian = ArrowIpcDecoderNeedsSwapEndian(decoder); @@ -2697,8 +2704,8 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeDictionary( // TODO: provide ArrowIpcDecoderDecodeArrayInternalWithDictionaries to handle nested // dictionaries - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayInternal( - &dictionary->decoder, 0, &tmp, validation_level, error)); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeArrayFromShared( + &dictionary->decoder, shared, 0, &tmp, validation_level, error)); ArrowErrorCode result; if (decoder->dictionary->is_delta) { diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 042f1d876..5b0a429b6 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -566,7 +566,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchErrors) { ArrowArrayRelease(&array); // Field extract should fail if body is too small - decoder.body_size_bytes = 0; + body.size_bytes = 0; EXPECT_EQ(ArrowIpcDecoderDecodeArray(&decoder, body, 0, &array, NANOARROW_VALIDATION_LEVEL_FULL, &error), EINVAL); @@ -689,7 +689,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { // Decode the dictionary batch data.data.as_uint8 += decoder.header_size_bytes; - data.size_bytes -= decoder.header_size_bytes; + data.size_bytes = decoder.body_size_bytes; struct ArrowBuffer body; ArrowBufferInit(&body); ASSERT_EQ(ArrowBufferAppendBufferView(&body, data), NANOARROW_OK); From 29f700846e5e558691872064ceb0b5f5b7faf593 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 21:34:11 -0500 Subject: [PATCH 10/17] check array value --- src/nanoarrow/ipc/decoder_test.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 5b0a429b6..13895c131 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -32,6 +32,7 @@ // For bswap32() #include "flatcc/portable/pendian.h" +#include "nanoarrow/nanoarrow_gtest_util.hpp" #include "nanoarrow/nanoarrow_ipc.hpp" #if defined(NANOARROW_BUILD_TESTS_WITH_ARROW) @@ -662,9 +663,8 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { EXPECT_EQ(decoder.dictionary->id, 0); EXPECT_FALSE(decoder.dictionary->is_delta); - struct ArrowSchema schema; - // Make a dictionary encoded schema that matches that of the dictionary example batch + struct ArrowSchema schema; ArrowSchemaInit(&schema); ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema, 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); @@ -706,8 +706,19 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { ASSERT_EQ( ArrowIpcDictionariesFindCurrentValue(&dictionaries, 0, &dictionary_value, &error), NANOARROW_OK); - ASSERT_NE(dictionary_value->release, nullptr); + struct ArrowArrayView array_view; + ArrowArrayViewInitFromType(&array_view, NANOARROW_TYPE_STRING); + ASSERT_EQ(ArrowArrayViewSetArray(&array_view, dictionary_value, &error), NANOARROW_OK); + + using namespace nanoarrow::literals; + + ASSERT_EQ(array_view.length, 3); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 0), "zero"_asv); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 1), "one"_asv); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 2), "two"_asv); + + ArrowArrayViewReset(&array_view); ArrowIpcSharedBufferReset(&shared); ArrowIpcDictionariesReset(&dictionaries); ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); From 9b2e640635e169e578658c8ad99747f4e3aab47e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:14:55 -0500 Subject: [PATCH 11/17] test another thing --- src/nanoarrow/ipc/decoder_test.cc | 120 +++++++++++++++++++++++++----- 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 13895c131..dd6316732 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -624,6 +624,10 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ASSERT_EQ(encoding->id, 0); ASSERT_EQ(encoding->kind, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY); + // The dictionary encodings should fail to locate anything except the decoded ID + ASSERT_EQ(ArrowIpcDictionaryEncodingsFindById(&dictionary_encodings, 100), nullptr); + encoding = ArrowIpcDictionaryEncodingsFindById(&dictionary_encodings, 0); + // If we try to set the schema without the dictionaries, we should get an error ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, &error), EINVAL); ASSERT_STREQ(error.message, @@ -643,28 +647,58 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ArrowIpcDecoderReset(&decoder); } -TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { - struct ArrowIpcDictionaryEncodings dictionary_encodings; - struct ArrowIpcDictionaries dictionaries; - struct ArrowIpcDecoder decoder; +TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) { struct ArrowError error; - // Decode a dictionary batch and inspect metadata - struct ArrowBufferView data; - data.data.as_uint8 = kDictionaryBatch; - data.size_bytes = sizeof(kDictionaryBatch); + // Create dictionary encodings there is at least one duplicated identifier + struct ArrowIpcDictionaryEncodings dictionary_encodings; + ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); - ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + struct ArrowIpcDictionaryEncoding encoding; + encoding.id = 42; + encoding.kind = NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY; + encoding.schema = nullptr; - EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); - ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH); + struct ArrowIpcDictionaryEncoding encoding2; + encoding2.id = 142; + encoding2.kind = NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY; + encoding2.schema = nullptr; - ASSERT_NE(decoder.dictionary, nullptr); - EXPECT_EQ(decoder.dictionary->id, 0); - EXPECT_FALSE(decoder.dictionary->is_delta); + // Add the encodings a few times to simulate sharedness + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend(&dictionary_encodings, encoding), + NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend(&dictionary_encodings, encoding2), + NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend(&dictionary_encodings, encoding), + NANOARROW_OK); + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend(&dictionary_encodings, encoding2), + NANOARROW_OK); - // Make a dictionary encoded schema that matches that of the dictionary example batch + struct ArrowBuffer unique_ids; + ArrowBufferInit(&unique_ids); + ASSERT_EQ(ArrowIpcDictionaryEncodingsUniqueIds(&dictionary_encodings, &unique_ids), + NANOARROW_OK); + + // 4 encodings (IDs: 42, 42, 7) should produce 2 unique IDs + ASSERT_EQ(unique_ids.size_bytes, 2 * sizeof(int64_t)); + const int64_t* ids = reinterpret_cast(unique_ids.data); + EXPECT_EQ(ids[0], 42); + EXPECT_EQ(ids[1], 142); + + ArrowBufferReset(&unique_ids); + ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); +} + +TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { + using namespace nanoarrow::literals; + + struct ArrowIpcDictionaryEncodings dictionary_encodings; struct ArrowSchema schema; + struct ArrowIpcDictionaries dictionaries; + struct ArrowIpcDecoder decoder; + struct ArrowError error; + + // Make a dictionary encoded schema that matches that of the dictionary example batch ArrowSchemaInit(&schema); ASSERT_EQ(ArrowSchemaSetTypeStruct(&schema, 1), NANOARROW_OK); ASSERT_EQ(ArrowSchemaSetType(schema.children[0], NANOARROW_TYPE_INT32), NANOARROW_OK); @@ -673,7 +707,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { ArrowSchemaInitFromType(schema.children[0]->dictionary, NANOARROW_TYPE_STRING), NANOARROW_OK); - // Initialize the dictionary encodings with a single dictionary + // Initialize the dictionary encodings with the single dictionary ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); struct ArrowIpcDictionaryEncoding encoding; encoding.id = 0; @@ -687,6 +721,29 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { NANOARROW_OK) << error.message; + // Check that we can't decode a dictionary batch if we haven't read a dictionary batch + // message + struct ArrowIpcSharedBuffer shared; + ASSERT_EQ( + ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, + &dictionaries, &error), + EINVAL); + ASSERT_STREQ(error.message, "decoder did not just decode a DictionaryBatch message"); + + // Decode a dictionary batch and inspect metadata + struct ArrowBufferView data; + data.data.as_uint8 = kDictionaryBatch; + data.size_bytes = sizeof(kDictionaryBatch); + + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); + + EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); + ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH); + + ASSERT_NE(decoder.dictionary, nullptr); + EXPECT_EQ(decoder.dictionary->id, 0); + EXPECT_FALSE(decoder.dictionary->is_delta); + // Decode the dictionary batch data.data.as_uint8 += decoder.header_size_bytes; data.size_bytes = decoder.body_size_bytes; @@ -694,14 +751,13 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { ArrowBufferInit(&body); ASSERT_EQ(ArrowBufferAppendBufferView(&body, data), NANOARROW_OK); - struct ArrowIpcSharedBuffer shared; ASSERT_EQ(ArrowIpcSharedBufferInit(&shared, &body), NANOARROW_OK); ASSERT_EQ( ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, &dictionaries, &error), - NANOARROW_OK) - << error.message; + NANOARROW_OK); + // If we find the current value of the dictionary we should get the correct array const struct ArrowArray* dictionary_value; ASSERT_EQ( ArrowIpcDictionariesFindCurrentValue(&dictionaries, 0, &dictionary_value, &error), @@ -711,13 +767,37 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { ArrowArrayViewInitFromType(&array_view, NANOARROW_TYPE_STRING); ASSERT_EQ(ArrowArrayViewSetArray(&array_view, dictionary_value, &error), NANOARROW_OK); - using namespace nanoarrow::literals; + ASSERT_EQ(array_view.length, 3); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 0), "zero"_asv); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 1), "one"_asv); + ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 2), "two"_asv); + + // If we try to find the current value of a different dictionary we should get an error + ASSERT_EQ(ArrowIpcDictionariesFindCurrentValue(&dictionaries, 9999, &dictionary_value, + &error), + EINVAL); + + // If we try to decode the dictionary again it should succeed (because the dictionary + // is in replacement mode) + ASSERT_EQ( + ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, + &dictionaries, &error), + NANOARROW_OK); + ASSERT_EQ(ArrowArrayViewSetArray(&array_view, dictionary_value, &error), NANOARROW_OK); ASSERT_EQ(array_view.length, 3); ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 0), "zero"_asv); ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 1), "one"_asv); ASSERT_EQ(ArrowArrayViewGetStringUnsafe(&array_view, 2), "two"_asv); + // If we try to decode a delta dictionary, we should fail with a reasonable message + const_cast(decoder.dictionary)->is_delta = 1; + ASSERT_EQ( + ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, + &dictionaries, &error), + ENOTSUP); + ASSERT_STREQ(error.message, "Dictionary concatenation is not yet supported"); + ArrowArrayViewReset(&array_view); ArrowIpcSharedBufferReset(&shared); ArrowIpcDictionariesReset(&dictionaries); From 4defc1b6054b468d1e176a3ceb3cbedcd45dd141 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:16:14 -0500 Subject: [PATCH 12/17] fix a lying comment --- src/nanoarrow/ipc/decoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 902b9b301..940230576 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2070,12 +2070,12 @@ struct ArrowIpcBufferFactory { /// \brief Buffer factory provided buffer length /// - /// Rather than use the body length declared by the flatbuffer messsage, this is the + /// Rather than use the body length declared by the flatbuffer message, this is the /// length that should be used to check the bounds of a buffer source (i.e., this is /// the actual available length as opposed to the theoretical length). int64_t buffer_length; - /// \brief Caller-defined private data to be used in the callback. + /// \brief User-defined private data to be used in the callback. /// /// Usually this would be a description of where the body has been read into memory or /// information required to do so. From 8731083487be7cab741241bd2291134ee6b35b7f Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:32:35 -0500 Subject: [PATCH 13/17] add docs --- src/nanoarrow/ipc/decoder.c | 4 ++++ src/nanoarrow/nanoarrow_ipc.h | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 940230576..5d621fa28 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -380,6 +380,10 @@ void ArrowIpcDictionaryEncodingsReset( ArrowBufferReset(&dictionary_encodings->encodings); } +/// \brief Internal member of an ArrowIpcDictionaries +/// +/// This structure stores a dictionary-specific decoder and the current value +/// of the dictionary (if one exists). struct ArrowIpcDictionary { int64_t id; struct ArrowIpcDecoder decoder; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index f818294e3..ece36a31f 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -262,6 +262,11 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFindById( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, int64_t id); +/// \brief Append a list of unique int64_t identifiers to out +/// +/// Walk all dictionary encodings in this list to find a set of unique identifiers. +/// This is primarily for internal use to initialize the ArrowIpcDictionaries but +/// is exposed for testing purposes. NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowBuffer* out); @@ -270,19 +275,30 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); +/// \brief Dictionaries and their current values +/// +/// This structure is the basis for decoding dictionary batches and decoding +/// record batch messages where some columns are dictionary-encoded. This +/// structure stores a dictionary-specific ArrowIpcDecoder for each dictionary +/// identifier and its last seen value. struct ArrowIpcDictionaries { void* private_data; }; +/// \brief Initialize an ArrowIpcDictionaries +/// +/// Initialize the structure with the dictionaries from an ArrowIpcDictionaryEncodings. NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries, const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error); +/// \brief Find the current value of a dictionary with the given ID as an ArrowArray NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesFindCurrentValue( struct ArrowIpcDictionaries* dictionaries, int64_t id, const struct ArrowArray** out, struct ArrowError* error); +/// \brief Release resources associated with an ArrowIpcDictionaries NANOARROW_DLL void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match From 4fdc7cef275ee241bafaca4f31c5280d97903eaf Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:43:45 -0500 Subject: [PATCH 14/17] add hpp helper --- src/nanoarrow/ipc/ipc_hpp_test.cc | 27 +++++++++++++++++++++++++++ src/nanoarrow/nanoarrow_ipc.hpp | 22 ++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/nanoarrow/ipc/ipc_hpp_test.cc b/src/nanoarrow/ipc/ipc_hpp_test.cc index b0a840906..0ff57dedf 100644 --- a/src/nanoarrow/ipc/ipc_hpp_test.cc +++ b/src/nanoarrow/ipc/ipc_hpp_test.cc @@ -101,3 +101,30 @@ TEST(NanoarrowIpcHppTest, NanoarrowIpcHppTestUniqueDictionaryEncodings) { dictionary_encodings->encodings.data, // NOLINT(clang-analyzer-cplusplus.Move) nullptr); } + +TEST(NanoarrowIpcHppTest, NanoarrowIpcHppTestUniqueDictionaries) { + nanoarrow::UniqueSchema schema; + ArrowSchemaInit(schema.get()); + ASSERT_EQ(ArrowSchemaAllocateDictionary(schema.get()), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->dictionary, NANOARROW_TYPE_STRING), + NANOARROW_OK); + + nanoarrow::ipc::UniqueDictionaryEncodings dictionary_encodings; + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend( + dictionary_encodings.get(), + {schema.get(), 1, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY}), + NANOARROW_OK); + + nanoarrow::ipc::UniqueDictionaries dictionaries; + struct ArrowError error; + ASSERT_EQ( + ArrowIpcDictionariesInit(dictionaries.get(), dictionary_encodings.get(), &error), + NANOARROW_OK) + << error.message; + ASSERT_NE(dictionaries->private_data, nullptr); + + nanoarrow::ipc::UniqueDictionaries dictionaries2 = std::move(dictionaries); + EXPECT_NE(dictionaries2->private_data, nullptr); + EXPECT_EQ(dictionaries->private_data, // NOLINT(clang-analyzer-cplusplus.Move) + nullptr); +} diff --git a/src/nanoarrow/nanoarrow_ipc.hpp b/src/nanoarrow/nanoarrow_ipc.hpp index 84f1d90b7..604a21471 100644 --- a/src/nanoarrow/nanoarrow_ipc.hpp +++ b/src/nanoarrow/nanoarrow_ipc.hpp @@ -74,6 +74,25 @@ inline void release_pointer(struct ArrowIpcDictionaryEncodings* data) { ArrowIpcDictionaryEncodingsReset(data); } +template <> +inline void init_pointer(struct ArrowIpcDictionaries* data) { + data->private_data = nullptr; +} + +template <> +inline void move_pointer(struct ArrowIpcDictionaries* src, + struct ArrowIpcDictionaries* dst) { + memcpy(dst, src, sizeof(struct ArrowIpcDictionaries)); + src->private_data = nullptr; +} + +template <> +inline void release_pointer(struct ArrowIpcDictionaries* data) { + if (data->private_data != nullptr) { + ArrowIpcDictionariesReset(data); + } +} + template <> inline void init_pointer(struct ArrowIpcFooter* data) { ArrowIpcFooterInit(data); @@ -206,6 +225,9 @@ using UniqueFooter = internal::Unique; /// \brief Class wrapping a unique struct ArrowIpcDictionaryEncodings using UniqueDictionaryEncodings = internal::Unique; +/// \brief Class wrapping a unique struct UniqueDictionaries +using UniqueDictionaries = internal::Unique; + /// \brief Class wrapping a unique struct ArrowIpcEncoder using UniqueEncoder = internal::Unique; From 549cd2f0c22f19fffa0eddee0ceb3c817f93f0ca Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:44:30 -0500 Subject: [PATCH 15/17] unused --- src/nanoarrow/ipc/decoder_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index dd6316732..aaeafc20f 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -648,8 +648,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { } TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) { - struct ArrowError error; - // Create dictionary encodings there is at least one duplicated identifier struct ArrowIpcDictionaryEncodings dictionary_encodings; ArrowIpcDictionaryEncodingsInit(&dictionary_encodings); From 8f8cfbc5e83042fb41281dc7bada6072463ed53e Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 3 Apr 2026 22:45:27 -0500 Subject: [PATCH 16/17] unused code --- src/nanoarrow/ipc/decoder_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index aaeafc20f..a713dd9c4 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -626,7 +626,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { // The dictionary encodings should fail to locate anything except the decoded ID ASSERT_EQ(ArrowIpcDictionaryEncodingsFindById(&dictionary_encodings, 100), nullptr); - encoding = ArrowIpcDictionaryEncodingsFindById(&dictionary_encodings, 0); // If we try to set the schema without the dictionaries, we should get an error ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, &error), EINVAL); From 3fce3f9343a87aa7798d701dbeef724fb6bc868f Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Sat, 4 Apr 2026 20:16:10 -0500 Subject: [PATCH 17/17] fix a few errors --- src/nanoarrow/ipc/decoder.c | 26 ++++++++++++++++---------- src/nanoarrow/ipc/decoder_test.cc | 5 ++--- src/nanoarrow/nanoarrow_ipc.h | 12 ++++++------ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 5d621fa28..520219ed9 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -287,10 +287,10 @@ void ArrowIpcDictionaryEncodingsInit( } ArrowErrorCode ArrowIpcDictionaryEncodingsAppend( - struct ArrowIpcDictionaryEncodings* dictionaries, + struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowIpcDictionaryEncoding encoding) { - NANOARROW_DCHECK(dictionaries != NULL); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(&dictionaries->encodings, &encoding, + NANOARROW_DCHECK(dictionary_encodings != NULL); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(&dictionary_encodings->encodings, &encoding, sizeof(struct ArrowIpcDictionaryEncoding))); return NANOARROW_OK; } @@ -347,6 +347,7 @@ static int ArrowIpcIdsListContains(const struct ArrowBuffer* ids_buffer, int64_t ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( const struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowBuffer* out) { + NANOARROW_DCHECK(dictionary_encodings != NULL); int64_t length = dictionary_encodings->encodings.size_bytes / sizeof(struct ArrowIpcDictionaryEncoding); const struct ArrowIpcDictionaryEncoding* data = @@ -530,13 +531,17 @@ ArrowErrorCode ArrowIpcDictionariesInit( private_data->num_dictionaries = unique_ids.size_bytes / sizeof(int64_t); // Allocate the array of ArrowIpcDictionary - private_data->dictionaries = - ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); - if (private_data->dictionaries == NULL) { - ArrowErrorSet(error, "Failed to allocate ArrowIpcDictionary array"); - ArrowBufferReset(&unique_ids); - ArrowFree(private_data); - return ENOMEM; + if (private_data->num_dictionaries > 0) { + private_data->dictionaries = + ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionary)); + if (private_data->dictionaries == NULL) { + ArrowErrorSet(error, "Failed to allocate ArrowIpcDictionary array"); + ArrowBufferReset(&unique_ids); + ArrowFree(private_data); + return ENOMEM; + } + } else { + private_data->dictionaries = NULL; } int64_t num_initialized_dictionaries = 0; @@ -602,6 +607,7 @@ void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { ArrowFree(private_data->dictionaries); ArrowFree(private_data); + dictionaries->private_data = NULL; } ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index a713dd9c4..b6aeddb77 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -676,7 +676,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDictionaryEncodingsUniqueIds) { ASSERT_EQ(ArrowIpcDictionaryEncodingsUniqueIds(&dictionary_encodings, &unique_ids), NANOARROW_OK); - // 4 encodings (IDs: 42, 42, 7) should produce 2 unique IDs + // 4 encodings (IDs: 42, 142, 42, 142) should produce 2 unique IDs ASSERT_EQ(unique_ids.size_bytes, 2 * sizeof(int64_t)); const int64_t* ids = reinterpret_cast(unique_ids.data); EXPECT_EQ(ids[0], 42); @@ -720,6 +720,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { // Check that we can't decode a dictionary batch if we haven't read a dictionary batch // message + ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); struct ArrowIpcSharedBuffer shared; ASSERT_EQ( ArrowIpcDecoderDecodeDictionary(&decoder, &shared, NANOARROW_VALIDATION_LEVEL_FULL, @@ -732,8 +733,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionaryBatchDecode) { data.data.as_uint8 = kDictionaryBatch; data.size_bytes = sizeof(kDictionaryBatch); - ASSERT_EQ(ArrowIpcDecoderInit(&decoder), NANOARROW_OK); - EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_DICTIONARY_BATCH); diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ece36a31f..e2b114cd0 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -236,19 +236,19 @@ struct ArrowIpcDictionaryEncodings { /// \brief Initialize an ArrowIpcDictionaryEncodings list NANOARROW_DLL void ArrowIpcDictionaryEncodingsInit( - struct ArrowIpcDictionaryEncodings* dictionaries); + struct ArrowIpcDictionaryEncodings* dictionary_encodings); /// \brief Append a given ArrowIpcDictionaryEncoding to this list -NANOARROW_DLL ArrowErrorCode -ArrowIpcDictionaryEncodingsAppend(struct ArrowIpcDictionaryEncodings* dictionaries, - struct ArrowIpcDictionaryEncoding encoding); +NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsAppend( + struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowIpcDictionaryEncoding encoding); /// \brief Resolve a ArrowIpcDictionaryEncoding for a given dictionary encoded field /// /// Returns NULL if the pointed to schema does not match any of the pointed to /// schemas contained in this list. NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( - const struct ArrowIpcDictionaryEncodings* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, const struct ArrowSchema* schema); /// \brief Resolve the first ArrowIpcDictionaryEncoding for a given identifier @@ -273,7 +273,7 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsUniqueIds( /// \brief Release an encodings list and associated resources NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( - struct ArrowIpcDictionaryEncodings* dictionaries); + struct ArrowIpcDictionaryEncodings* dictionary_encodings); /// \brief Dictionaries and their current values ///