From db6b33bdb137ca7680edcb2bb71adf5ad9c9a1d9 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 16 Mar 2026 22:03:59 -0500 Subject: [PATCH 01/21] sketch lifecycle --- src/nanoarrow/ipc/decoder.c | 46 +++++++++++++++++++++++++++++++++++ src/nanoarrow/nanoarrow_ipc.h | 14 +++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index f3b4b4435..6d026c3e2 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -272,6 +272,52 @@ static int ArrowIpcDecoderNeedsSwapEndian(struct ArrowIpcDecoder* decoder) { } } +void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); +} + +ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + int64_t id, const struct ArrowSchema* schema) { + NANOARROW_DCHECK(dictionaries != NULL); + if ((dictionaries->length + 1) >= dictionaries->capacity) { + int64_t min_capacity = dictionaries->length == 0 ? 1 : dictionaries->length * 2; + + dictionaries->dictionary_id = (int64_t*)ArrowRealloc(dictionaries->dictionary_id, + min_capacity * sizeof(int64_t*)); + if (dictionaries->dictionary_id == NULL) { + return ENOMEM; + } + + dictionaries->dictionary_schema = (const struct ArrowSchema**)ArrowRealloc( + dictionaries->dictionary_id, min_capacity * sizeof(struct ArrowSchema**)); + if (dictionaries->dictionary_id == NULL) { + return ENOMEM; + } + + dictionaries->capacity = min_capacity; + } + + NANOARROW_DCHECK(dictionaries->capacity > dictionaries->length); + dictionaries->dictionary_id[dictionaries->length] = id; + dictionaries->dictionary_schema[dictionaries->length] = schema; + dictionaries->length++; + return NANOARROW_OK; +} + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + if (dictionaries->dictionary_id != NULL) { + ArrowFree(dictionaries->dictionary_id); + } + + if (dictionaries->dictionary_schema != NULL) { + ArrowFree(dictionaries->dictionary_schema); + } + + ArrowIpcDictionariesInit(dictionaries); +} + ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { memset(decoder, 0, sizeof(struct ArrowIpcDecoder)); struct ArrowIpcDecoderPrivate* private_data = diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index a1a172cb4..3112a80f0 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -171,6 +171,20 @@ struct ArrowIpcDictionaryBatch { int is_delta; }; +struct ArrowIpcDictionaries { + int64_t* dictionary_id; + const struct ArrowSchema** dictionary_schema; + int64_t length; + int64_t capacity; +}; + +void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); + +ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, + int64_t id, const struct ArrowSchema* schema); + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); + /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); From e74dc2d083b72ee1f223916a71aae3e23265065d Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 16 Mar 2026 22:25:22 -0500 Subject: [PATCH 02/21] thread dictionaries through field setters --- src/nanoarrow/ipc/decoder.c | 58 +++++++++++++++++++++++++---------- src/nanoarrow/nanoarrow_ipc.h | 5 +++ 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 6d026c3e2..b8e3ecd31 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -989,7 +989,7 @@ static int ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded( static int ArrowIpcSetDictionaryEncoding( struct ArrowSchema* schema, ns(DictionaryEncoding_table_t dictionary_encoding), - struct ArrowError* error) { + struct ArrowIpcDictionaries* dictionaries, struct ArrowError* error) { switch ( org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind(dictionary_encoding)) { case ns(DictionaryKind_DenseArray): @@ -1028,16 +1028,22 @@ static int ArrowIpcSetDictionaryEncoding( NANOARROW_RETURN_NOT_OK_WITH_ERROR( ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded(schema), error); - // TODO: Track the dictionary - // https://github.com/apache/arrow-nanoarrow/issues/844 + // Track the identifier if we have a dictionaries object in which to track it + if (dictionaries != NULL) { + int64_t id = ns(DictionaryEncoding_id(dictionary_encoding)); + NANOARROW_RETURN_NOT_OK_WITH_ERROR( + ArrowIpcDictionariesAppend(dictionaries, id, schema), error); + } return NANOARROW_OK; } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, + struct ArrowIpcDictionaries* dictionaries, struct ArrowError* error); static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) field, + struct ArrowIpcDictionaries* dictionaries, struct ArrowError* error) { int result; if (ns(Field_name_is_present(field))) { @@ -1078,26 +1084,29 @@ static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) ArrowSchemaInit(schema->children[i]); } - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetChildren(schema, children, error)); + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetChildren(schema, children, dictionaries, error)); NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSetMetadata(schema, ns(Field_custom_metadata(field)), error)); // If this is a dictionary encoded field, set the dictionary encoding if (ns(Field_dictionary_is_present(field))) { - NANOARROW_RETURN_NOT_OK( - ArrowIpcSetDictionaryEncoding(schema, ns(Field_dictionary(field)), error)); + NANOARROW_RETURN_NOT_OK(ArrowIpcSetDictionaryEncoding( + schema, ns(Field_dictionary(field)), dictionaries, error)); } return NANOARROW_OK; } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, + struct ArrowIpcDictionaries* dictionaries, struct ArrowError* error) { int64_t n_fields = ns(Schema_vec_len(fields)); for (int64_t i = 0; i < n_fields; i++) { ns(Field_table_t) field = ns(Field_vec_at(fields, i)); - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetField(schema->children[i], field, error)); + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetField(schema->children[i], field, dictionaries, error)); } return NANOARROW_OK; @@ -1493,9 +1502,9 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* decoder, return NANOARROW_OK; } -static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl(ns(Schema_table_t) schema, - struct ArrowSchema* out, - struct ArrowError* error) { +static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( + ns(Schema_table_t) schema, struct ArrowSchema* out, + struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error) { ArrowSchemaInit(out); // Top-level batch schema is typically non-nullable out->flags = 0; @@ -1510,15 +1519,16 @@ static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl(ns(Schema_table_t) schema, return result; } - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderSetChildren(out, fields, error)); + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderSetChildren(out, fields, dictionaries_out, error)); NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSetMetadata(out, ns(Schema_custom_metadata(schema)), error)); return NANOARROW_OK; } -ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* decoder, - struct ArrowSchema* out, - struct ArrowError* error) { +ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( + struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, + struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1528,18 +1538,33 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* decoder, return EINVAL; } + if (dictionaries_out != NULL) { + ArrowIpcDictionariesInit(dictionaries_out); + } + struct ArrowSchema tmp; ArrowErrorCode result = ArrowIpcDecoderDecodeSchemaImpl( - (ns(Schema_table_t))private_data->last_message, &tmp, error); + (ns(Schema_table_t))private_data->last_message, &tmp, dictionaries_out, error); if (result != NANOARROW_OK) { ArrowSchemaRelease(&tmp); + + if (dictionaries_out != NULL) { + ArrowIpcDictionariesReset(dictionaries_out); + } + return result; } ArrowSchemaMove(&tmp, out); return NANOARROW_OK; } +ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* decoder, + struct ArrowSchema* out, + struct ArrowError* error) { + return ArrowIpcDecoderDecodeSchemaWithDictionaries(decoder, out, NULL, error); +} + ArrowErrorCode ArrowIpcDecoderDecodeFooter(struct ArrowIpcDecoder* decoder, struct ArrowBufferView data, struct ArrowError* error) { @@ -1555,8 +1580,9 @@ ArrowErrorCode ArrowIpcDecoderDecodeFooter(struct ArrowIpcDecoder* decoder, NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderDecodeSchemaHeader(decoder, ns(Footer_schema(footer)), error)); + // TODO: include dictionaries in footer NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeSchemaImpl( - ns(Footer_schema(footer)), &private_data->footer.schema, error)); + ns(Footer_schema(footer)), &private_data->footer.schema, NULL, error)); ns(Block_vec_t) blocks = ns(Footer_recordBatches(footer)); int64_t n = ns(Block_vec_len(blocks)); diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 3112a80f0..d67fcfee5 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -413,6 +413,11 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* struct ArrowSchema* out, struct ArrowError* error); +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries(struct ArrowIpcDecoder* decoder, + struct ArrowSchema* out, + struct ArrowIpcDictionaries* dictionaries_out, + struct ArrowError* error); + /// \brief Set the ArrowSchema used to decode future record batch messages /// /// Prepares the decoder for future record batch messages From 5f5ecf5b515afc73078a153863018380503cb001 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 16 Mar 2026 22:31:40 -0500 Subject: [PATCH 03/21] add dll suffix --- src/nanoarrow/ipc/decoder.c | 4 ++-- src/nanoarrow/ipc/encoder.c | 2 ++ src/nanoarrow/nanoarrow_ipc.h | 20 +++++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index b8e3ecd31..a92bd5973 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1580,9 +1580,9 @@ ArrowErrorCode ArrowIpcDecoderDecodeFooter(struct ArrowIpcDecoder* decoder, NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderDecodeSchemaHeader(decoder, ns(Footer_schema(footer)), error)); - // TODO: include dictionaries in footer NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeSchemaImpl( - ns(Footer_schema(footer)), &private_data->footer.schema, NULL, error)); + ns(Footer_schema(footer)), &private_data->footer.schema, + &private_data->footer.dictionaries, error)); ns(Block_vec_t) blocks = ns(Footer_recordBatches(footer)); int64_t n = ns(Block_vec_len(blocks)); diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index d58761432..4040d5e1d 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -629,6 +629,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeSimpleRecordBatch( void ArrowIpcFooterInit(struct ArrowIpcFooter* footer) { footer->schema.release = NULL; ArrowBufferInit(&footer->record_batch_blocks); + ArrowIpcDictionariesInit(&footer->dictionaries); } void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { @@ -636,6 +637,7 @@ void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { ArrowSchemaRelease(&footer->schema); } ArrowBufferReset(&footer->record_batch_blocks); + ArrowIpcDictionariesReset(&footer->dictionaries); } ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index d67fcfee5..7d202cd50 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -178,12 +178,13 @@ struct ArrowIpcDictionaries { int64_t capacity; }; -void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); +NANOARROW_DLL void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); -ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, - int64_t id, const struct ArrowSchema* schema); +NANOARROW_DLL ArrowErrorCode +ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, int64_t id, + const struct ArrowSchema* schema); -void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); +NANOARROW_DLL void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); @@ -413,10 +414,9 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* struct ArrowSchema* out, struct ArrowError* error); -NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries(struct ArrowIpcDecoder* decoder, - struct ArrowSchema* out, - struct ArrowIpcDictionaries* dictionaries_out, - struct ArrowError* error); +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( + struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, + struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error); /// \brief Set the ArrowSchema used to decode future record batch messages /// @@ -727,8 +727,10 @@ struct ArrowIpcFileBlock { /// This structure is intended to be allocated by the caller, initialized using /// ArrowIpcFooterInit(), and released with ArrowIpcFooterReset(). struct ArrowIpcFooter { - /// \brief the Footer's embedded Schema + /// \brief The Footer's embedded Schema struct ArrowSchema schema; + /// \brief Dictionaries present in the footer Schema + struct ArrowIpcDictionaries dictionaries; /// \brief all blocks containing RecordBatch Messages struct ArrowBuffer record_batch_blocks; }; From ce7bb95167df19267304872554ddd688a524b693 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 16 Mar 2026 22:34:42 -0500 Subject: [PATCH 04/21] rename --- src/nanoarrow/ipc/decoder.c | 31 ++++++++++++++++--------------- src/nanoarrow/ipc/encoder.c | 4 ++-- src/nanoarrow/nanoarrow_ipc.h | 16 +++++++++------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index a92bd5973..9dfe1da47 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -272,13 +272,14 @@ static int ArrowIpcDecoderNeedsSwapEndian(struct ArrowIpcDecoder* decoder) { } } -void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries) { +void ArrowIpcDictionaryMappingInit(struct ArrowIpcDictionaryMapping* dictionaries) { NANOARROW_DCHECK(dictionaries != NULL); - memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); + memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaryMapping)); } -ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, - int64_t id, const struct ArrowSchema* schema) { +ArrowErrorCode ArrowIpcDictionaryMappingAppend( + struct ArrowIpcDictionaryMapping* dictionaries, int64_t id, + const struct ArrowSchema* schema) { NANOARROW_DCHECK(dictionaries != NULL); if ((dictionaries->length + 1) >= dictionaries->capacity) { int64_t min_capacity = dictionaries->length == 0 ? 1 : dictionaries->length * 2; @@ -305,7 +306,7 @@ ArrowErrorCode ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionar return NANOARROW_OK; } -void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { +void ArrowIpcDictionaryMappingReset(struct ArrowIpcDictionaryMapping* dictionaries) { NANOARROW_DCHECK(dictionaries != NULL); if (dictionaries->dictionary_id != NULL) { ArrowFree(dictionaries->dictionary_id); @@ -315,7 +316,7 @@ void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { ArrowFree(dictionaries->dictionary_schema); } - ArrowIpcDictionariesInit(dictionaries); + ArrowIpcDictionaryMappingInit(dictionaries); } ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { @@ -989,7 +990,7 @@ static int ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded( static int ArrowIpcSetDictionaryEncoding( struct ArrowSchema* schema, ns(DictionaryEncoding_table_t dictionary_encoding), - struct ArrowIpcDictionaries* dictionaries, struct ArrowError* error) { + struct ArrowIpcDictionaryMapping* dictionaries, struct ArrowError* error) { switch ( org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind(dictionary_encoding)) { case ns(DictionaryKind_DenseArray): @@ -1032,18 +1033,18 @@ static int ArrowIpcSetDictionaryEncoding( if (dictionaries != NULL) { int64_t id = ns(DictionaryEncoding_id(dictionary_encoding)); NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowIpcDictionariesAppend(dictionaries, id, schema), error); + ArrowIpcDictionaryMappingAppend(dictionaries, id, schema), error); } return NANOARROW_OK; } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, - struct ArrowIpcDictionaries* dictionaries, + struct ArrowIpcDictionaryMapping* dictionaries, struct ArrowError* error); static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) field, - struct ArrowIpcDictionaries* dictionaries, + struct ArrowIpcDictionaryMapping* dictionaries, struct ArrowError* error) { int result; if (ns(Field_name_is_present(field))) { @@ -1099,7 +1100,7 @@ static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, - struct ArrowIpcDictionaries* dictionaries, + struct ArrowIpcDictionaryMapping* dictionaries, struct ArrowError* error) { int64_t n_fields = ns(Schema_vec_len(fields)); @@ -1504,7 +1505,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* decoder, static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ns(Schema_table_t) schema, struct ArrowSchema* out, - struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error) { ArrowSchemaInit(out); // Top-level batch schema is typically non-nullable out->flags = 0; @@ -1528,7 +1529,7 @@ static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, - struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1539,7 +1540,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( } if (dictionaries_out != NULL) { - ArrowIpcDictionariesInit(dictionaries_out); + ArrowIpcDictionaryMappingInit(dictionaries_out); } struct ArrowSchema tmp; @@ -1550,7 +1551,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( ArrowSchemaRelease(&tmp); if (dictionaries_out != NULL) { - ArrowIpcDictionariesReset(dictionaries_out); + ArrowIpcDictionaryMappingReset(dictionaries_out); } return result; diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index 4040d5e1d..9f0ce2d96 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -629,7 +629,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeSimpleRecordBatch( void ArrowIpcFooterInit(struct ArrowIpcFooter* footer) { footer->schema.release = NULL; ArrowBufferInit(&footer->record_batch_blocks); - ArrowIpcDictionariesInit(&footer->dictionaries); + ArrowIpcDictionaryMappingInit(&footer->dictionaries); } void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { @@ -637,7 +637,7 @@ void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { ArrowSchemaRelease(&footer->schema); } ArrowBufferReset(&footer->record_batch_blocks); - ArrowIpcDictionariesReset(&footer->dictionaries); + ArrowIpcDictionaryMappingReset(&footer->dictionaries); } ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 7d202cd50..0e62ca373 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -171,20 +171,22 @@ struct ArrowIpcDictionaryBatch { int is_delta; }; -struct ArrowIpcDictionaries { +struct ArrowIpcDictionaryMapping { int64_t* dictionary_id; const struct ArrowSchema** dictionary_schema; int64_t length; int64_t capacity; }; -NANOARROW_DLL void ArrowIpcDictionariesInit(struct ArrowIpcDictionaries* dictionaries); +NANOARROW_DLL void ArrowIpcDictionaryMappingInit( + struct ArrowIpcDictionaryMapping* dictionaries); NANOARROW_DLL ArrowErrorCode -ArrowIpcDictionariesAppend(struct ArrowIpcDictionaries* dictionaries, int64_t id, - const struct ArrowSchema* schema); +ArrowIpcDictionaryMappingAppend(struct ArrowIpcDictionaryMapping* dictionaries, + int64_t id, const struct ArrowSchema* schema); -NANOARROW_DLL void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries); +NANOARROW_DLL void ArrowIpcDictionaryMappingReset( + struct ArrowIpcDictionaryMapping* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); @@ -416,7 +418,7 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, - struct ArrowIpcDictionaries* dictionaries_out, struct ArrowError* error); + struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error); /// \brief Set the ArrowSchema used to decode future record batch messages /// @@ -730,7 +732,7 @@ struct ArrowIpcFooter { /// \brief The Footer's embedded Schema struct ArrowSchema schema; /// \brief Dictionaries present in the footer Schema - struct ArrowIpcDictionaries dictionaries; + struct ArrowIpcDictionaryMapping dictionaries; /// \brief all blocks containing RecordBatch Messages struct ArrowBuffer record_batch_blocks; }; From 241ef434a092ac7b97e9d968c5d8a2babef4cf74 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Mon, 16 Mar 2026 22:37:15 -0500 Subject: [PATCH 05/21] add namespacing --- src/nanoarrow/nanoarrow_ipc.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 0e62ca373..78bf1182e 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -49,6 +49,8 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeHeader) #define ArrowIpcDecoderDecodeSchema \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeSchema) +#define ArrowIpcDecoderDecodeSchemaWithDictionaries \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeSchemaWithDictionaries) #define ArrowIpcDecoderDecodeArrayView \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeArrayView) #define ArrowIpcDecoderDecodeArray \ @@ -105,6 +107,12 @@ #define ArrowIpcFooterReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcFooterReset) #define ArrowIpcEncoderEncodeFooter \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcEncoderEncodeFooter) +#define ArrowIpcDictionaryMappingInit \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingInit) +#define ArrowIpcDictionaryMappingAppend \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingAppend) +#define ArrowIpcDictionaryMappingReset \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingReset) #endif From c90284bd063424e6cc7dbbf198f327bac3cd9666 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 10:12:52 -0500 Subject: [PATCH 06/21] one more --- src/nanoarrow/ipc/decoder.c | 80 ++++++++++++++++------------------- src/nanoarrow/ipc/encoder.c | 4 +- src/nanoarrow/nanoarrow_ipc.h | 70 +++++++++++++++++++++--------- 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 9dfe1da47..26732c882 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -272,51 +272,42 @@ static int ArrowIpcDecoderNeedsSwapEndian(struct ArrowIpcDecoder* decoder) { } } -void ArrowIpcDictionaryMappingInit(struct ArrowIpcDictionaryMapping* dictionaries) { +void ArrowIpcDictionaryEncodingsInit(struct ArrowIpcDictionaryEncodings* dictionaries) { NANOARROW_DCHECK(dictionaries != NULL); - memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaryMapping)); + ArrowBufferInit(&dictionaries->encodings); } -ArrowErrorCode ArrowIpcDictionaryMappingAppend( - struct ArrowIpcDictionaryMapping* dictionaries, int64_t id, - const struct ArrowSchema* schema) { +ArrowErrorCode ArrowIpcDictionaryEncodingsAppend( + struct ArrowIpcDictionaryEncodings* dictionaries, + struct ArrowIpcDictionaryEncoding encoding) { NANOARROW_DCHECK(dictionaries != NULL); - if ((dictionaries->length + 1) >= dictionaries->capacity) { - int64_t min_capacity = dictionaries->length == 0 ? 1 : dictionaries->length * 2; - - dictionaries->dictionary_id = (int64_t*)ArrowRealloc(dictionaries->dictionary_id, - min_capacity * sizeof(int64_t*)); - if (dictionaries->dictionary_id == NULL) { - return ENOMEM; - } - - dictionaries->dictionary_schema = (const struct ArrowSchema**)ArrowRealloc( - dictionaries->dictionary_id, min_capacity * sizeof(struct ArrowSchema**)); - if (dictionaries->dictionary_id == NULL) { - return ENOMEM; - } - - dictionaries->capacity = min_capacity; - } - - NANOARROW_DCHECK(dictionaries->capacity > dictionaries->length); - dictionaries->dictionary_id[dictionaries->length] = id; - dictionaries->dictionary_schema[dictionaries->length] = schema; - dictionaries->length++; + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(&dictionaries->encodings, &encoding, + sizeof(struct ArrowIpcDictionaryEncoding))); return NANOARROW_OK; } -void ArrowIpcDictionaryMappingReset(struct ArrowIpcDictionaryMapping* dictionaries) { +const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( + const struct ArrowIpcDictionaryEncodings* dictionaries, + const struct ArrowSchema* schema) { NANOARROW_DCHECK(dictionaries != NULL); - if (dictionaries->dictionary_id != NULL) { - ArrowFree(dictionaries->dictionary_id); + int64_t length = + dictionaries->encodings.size_bytes / sizeof(struct ArrowIpcDictionaryEncoding); + const struct ArrowIpcDictionaryEncoding* data = + (const struct ArrowIpcDictionaryEncoding*)dictionaries->encodings.data; + + for (int64_t i = 0; i < length; i++) { + const struct ArrowIpcDictionaryEncoding* encoding = data + i; + if (encoding->schema == schema) { + return encoding; + } } - if (dictionaries->dictionary_schema != NULL) { - ArrowFree(dictionaries->dictionary_schema); - } + return NULL; +} - ArrowIpcDictionaryMappingInit(dictionaries); +void ArrowIpcDictionaryEncodingsReset(struct ArrowIpcDictionaryEncodings* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + ArrowBufferReset(&dictionaries->encodings); } ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { @@ -990,7 +981,7 @@ static int ArrowIpcMoveNonExtensionFieldMetadataBackToFieldIfNeeded( static int ArrowIpcSetDictionaryEncoding( struct ArrowSchema* schema, ns(DictionaryEncoding_table_t dictionary_encoding), - struct ArrowIpcDictionaryMapping* dictionaries, struct ArrowError* error) { + struct ArrowIpcDictionaryEncodings* dictionaries, struct ArrowError* error) { switch ( org_apache_arrow_flatbuf_DictionaryEncoding_dictionaryKind(dictionary_encoding)) { case ns(DictionaryKind_DenseArray): @@ -1032,19 +1023,22 @@ static int ArrowIpcSetDictionaryEncoding( // Track the identifier if we have a dictionaries object in which to track it if (dictionaries != NULL) { int64_t id = ns(DictionaryEncoding_id(dictionary_encoding)); + struct ArrowIpcDictionaryEncoding encoding = { + schema, id, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY}; + NANOARROW_RETURN_NOT_OK_WITH_ERROR( - ArrowIpcDictionaryMappingAppend(dictionaries, id, schema), error); + ArrowIpcDictionaryEncodingsAppend(dictionaries, encoding), error); } return NANOARROW_OK; } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, - struct ArrowIpcDictionaryMapping* dictionaries, + struct ArrowIpcDictionaryEncodings* dictionaries, struct ArrowError* error); static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) field, - struct ArrowIpcDictionaryMapping* dictionaries, + struct ArrowIpcDictionaryEncodings* dictionaries, struct ArrowError* error) { int result; if (ns(Field_name_is_present(field))) { @@ -1100,7 +1094,7 @@ static int ArrowIpcDecoderSetField(struct ArrowSchema* schema, ns(Field_table_t) } static int ArrowIpcDecoderSetChildren(struct ArrowSchema* schema, ns(Field_vec_t) fields, - struct ArrowIpcDictionaryMapping* dictionaries, + struct ArrowIpcDictionaryEncodings* dictionaries, struct ArrowError* error) { int64_t n_fields = ns(Schema_vec_len(fields)); @@ -1505,7 +1499,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* decoder, static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ns(Schema_table_t) schema, struct ArrowSchema* out, - struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error) { ArrowSchemaInit(out); // Top-level batch schema is typically non-nullable out->flags = 0; @@ -1529,7 +1523,7 @@ static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, - struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1540,7 +1534,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( } if (dictionaries_out != NULL) { - ArrowIpcDictionaryMappingInit(dictionaries_out); + ArrowIpcDictionaryEncodingsInit(dictionaries_out); } struct ArrowSchema tmp; @@ -1551,7 +1545,7 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( ArrowSchemaRelease(&tmp); if (dictionaries_out != NULL) { - ArrowIpcDictionaryMappingReset(dictionaries_out); + ArrowIpcDictionaryEncodingsReset(dictionaries_out); } return result; diff --git a/src/nanoarrow/ipc/encoder.c b/src/nanoarrow/ipc/encoder.c index 9f0ce2d96..8b3fc4896 100644 --- a/src/nanoarrow/ipc/encoder.c +++ b/src/nanoarrow/ipc/encoder.c @@ -629,7 +629,7 @@ ArrowErrorCode ArrowIpcEncoderEncodeSimpleRecordBatch( void ArrowIpcFooterInit(struct ArrowIpcFooter* footer) { footer->schema.release = NULL; ArrowBufferInit(&footer->record_batch_blocks); - ArrowIpcDictionaryMappingInit(&footer->dictionaries); + ArrowIpcDictionaryEncodingsInit(&footer->dictionaries); } void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { @@ -637,7 +637,7 @@ void ArrowIpcFooterReset(struct ArrowIpcFooter* footer) { ArrowSchemaRelease(&footer->schema); } ArrowBufferReset(&footer->record_batch_blocks); - ArrowIpcDictionaryMappingReset(&footer->dictionaries); + ArrowIpcDictionaryEncodingsReset(&footer->dictionaries); } ArrowErrorCode ArrowIpcEncoderEncodeFooter(struct ArrowIpcEncoder* encoder, diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 78bf1182e..298f8a3f1 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -107,12 +107,14 @@ #define ArrowIpcFooterReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcFooterReset) #define ArrowIpcEncoderEncodeFooter \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcEncoderEncodeFooter) -#define ArrowIpcDictionaryMappingInit \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingInit) -#define ArrowIpcDictionaryMappingAppend \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingAppend) -#define ArrowIpcDictionaryMappingReset \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryMappingReset) +#define ArrowIpcDictionaryEncodingsInit \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsInit) +#define ArrowIpcDictionaryEncodingsAppend \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsAppend) +#define ArrowIpcDictionaryEncodingsFind \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsFind) +#define ArrowIpcDictionaryEncodingsReset \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDictionaryEncodingsReset) #endif @@ -164,6 +166,12 @@ enum ArrowIpcCompressionType { NANOARROW_IPC_COMPRESSION_TYPE_ZSTD }; +/// \brief Dictionary kind enumerator +enum ArrowIpcDictionaryKind { + NANOARROW_IPC_DICTIONARY_KIND_UNINITIALIZED, + NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY +}; + /// \brief Feature flag for a stream that uses dictionary replacement #define NANOARROW_IPC_FEATURE_DICTIONARY_REPLACEMENT 1 @@ -179,22 +187,46 @@ struct ArrowIpcDictionaryBatch { int is_delta; }; -struct ArrowIpcDictionaryMapping { - int64_t* dictionary_id; - const struct ArrowSchema** dictionary_schema; - int64_t length; - int64_t capacity; +/// \brief Description of a dictionary-encoded field +/// +/// This struct is intended to be passed by value; however, its data is invalidated +/// if the underlying ArrowSchema that contains the dictionary-encoded field is +/// released. +struct ArrowIpcDictionaryEncoding { + /// \brief A pointer to the ArrowSchema node of the dictionary-encoded field + /// + /// This is a reference into another object and care must be taken to ensure + /// that if that object is copied that the schema pointers are updated + /// appropriately. + const struct ArrowSchema* schema; + + /// \brief The identifier used that will appear in dictionary batch messages + int64_t id; + + /// \brief The dictionary kind + /// + /// Currently only one dictionary kind is permitted by the Arrow specification + /// (DenseArray). + enum ArrowIpcDictionaryKind kind; }; -NANOARROW_DLL void ArrowIpcDictionaryMappingInit( - struct ArrowIpcDictionaryMapping* dictionaries); +struct ArrowIpcDictionaryEncodings { + struct ArrowBuffer encodings; +}; + +NANOARROW_DLL void ArrowIpcDictionaryEncodingsInit( + struct ArrowIpcDictionaryEncodings* dictionaries); NANOARROW_DLL ArrowErrorCode -ArrowIpcDictionaryMappingAppend(struct ArrowIpcDictionaryMapping* dictionaries, - int64_t id, const struct ArrowSchema* schema); +ArrowIpcDictionaryEncodingsAppend(struct ArrowIpcDictionaryEncodings* dictionaries, + struct ArrowIpcDictionaryEncoding encoding); + +NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( + const struct ArrowIpcDictionaryEncodings* dictionaries, + const struct ArrowSchema* schema); -NANOARROW_DLL void ArrowIpcDictionaryMappingReset( - struct ArrowIpcDictionaryMapping* dictionaries); +NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( + struct ArrowIpcDictionaryEncodings* dictionaries); /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error); @@ -426,7 +458,7 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, - struct ArrowIpcDictionaryMapping* dictionaries_out, struct ArrowError* error); + struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error); /// \brief Set the ArrowSchema used to decode future record batch messages /// @@ -740,7 +772,7 @@ struct ArrowIpcFooter { /// \brief The Footer's embedded Schema struct ArrowSchema schema; /// \brief Dictionaries present in the footer Schema - struct ArrowIpcDictionaryMapping dictionaries; + struct ArrowIpcDictionaryEncodings dictionaries; /// \brief all blocks containing RecordBatch Messages struct ArrowBuffer record_batch_blocks; }; From 116a7a8bb804118f16dffc23766930a31b2e7823 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 10:22:31 -0500 Subject: [PATCH 07/21] test --- src/nanoarrow/ipc/decoder_test.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 403229082..eff0ad855 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -589,6 +589,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { struct ArrowIpcDecoder decoder; struct ArrowError error; struct ArrowSchema schema; + struct ArrowIpcDictionaryEncodings encodings; struct ArrowBufferView data; data.data.as_uint8 = kDictionarySchema; @@ -599,7 +600,9 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA); - ASSERT_EQ(ArrowIpcDecoderDecodeSchema(&decoder, &schema, &error), NANOARROW_OK); + ASSERT_EQ( + ArrowIpcDecoderDecodeSchemaWithDictionaries(&decoder, &schema, &encodings, &error), + NANOARROW_OK); ASSERT_EQ(schema.n_children, 1); EXPECT_STREQ(schema.children[0]->name, "some_col"); EXPECT_EQ(schema.children[0]->flags, ARROW_FLAG_NULLABLE); @@ -608,7 +611,19 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ASSERT_NE(schema.children[0]->dictionary, nullptr); EXPECT_STREQ(schema.children[0]->dictionary->format, "u"); + // The dictionary encodings should fail to locate anything except the dictionary-encoded + // field + ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&encodings, nullptr), nullptr); + ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&encodings, &schema), nullptr); + const struct ArrowIpcDictionaryEncoding* encoding = + ArrowIpcDictionaryEncodingsFind(&encodings, schema.children[0]); + ASSERT_NE(encoding, nullptr); + ASSERT_EQ(encoding->schema, schema.children[0]); + ASSERT_EQ(encoding->id, 0); + ASSERT_EQ(encoding->kind, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY); + ArrowSchemaRelease(&schema); + ArrowIpcDictionaryEncodingsReset(&encodings); ArrowIpcDecoderReset(&decoder); } From 7a8a66c3023978ea420db5133f2558964472cf7a Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 11:14:37 -0500 Subject: [PATCH 08/21] verbose but more accurate name --- src/nanoarrow/ipc/decoder.c | 45 +++++++++++++++++-------------- src/nanoarrow/ipc/decoder_test.cc | 16 +++++------ 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 26732c882..ecf2d62f6 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -272,9 +272,10 @@ static int ArrowIpcDecoderNeedsSwapEndian(struct ArrowIpcDecoder* decoder) { } } -void ArrowIpcDictionaryEncodingsInit(struct ArrowIpcDictionaryEncodings* dictionaries) { - NANOARROW_DCHECK(dictionaries != NULL); - ArrowBufferInit(&dictionaries->encodings); +void ArrowIpcDictionaryEncodingsInit( + struct ArrowIpcDictionaryEncodings* dictionary_encodings) { + NANOARROW_DCHECK(dictionary_encodings != NULL); + ArrowBufferInit(&dictionary_encodings->encodings); } ArrowErrorCode ArrowIpcDictionaryEncodingsAppend( @@ -287,13 +288,13 @@ ArrowErrorCode ArrowIpcDictionaryEncodingsAppend( } const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( - const struct ArrowIpcDictionaryEncodings* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, const struct ArrowSchema* schema) { - NANOARROW_DCHECK(dictionaries != NULL); - int64_t length = - dictionaries->encodings.size_bytes / sizeof(struct ArrowIpcDictionaryEncoding); + NANOARROW_DCHECK(dictionary_encodings != NULL); + int64_t length = dictionary_encodings->encodings.size_bytes / + sizeof(struct ArrowIpcDictionaryEncoding); const struct ArrowIpcDictionaryEncoding* data = - (const struct ArrowIpcDictionaryEncoding*)dictionaries->encodings.data; + (const struct ArrowIpcDictionaryEncoding*)dictionary_encodings->encodings.data; for (int64_t i = 0; i < length; i++) { const struct ArrowIpcDictionaryEncoding* encoding = data + i; @@ -305,9 +306,10 @@ const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( return NULL; } -void ArrowIpcDictionaryEncodingsReset(struct ArrowIpcDictionaryEncodings* dictionaries) { - NANOARROW_DCHECK(dictionaries != NULL); - ArrowBufferReset(&dictionaries->encodings); +void ArrowIpcDictionaryEncodingsReset( + struct ArrowIpcDictionaryEncodings* dictionary_encodings) { + NANOARROW_DCHECK(dictionary_encodings != NULL); + ArrowBufferReset(&dictionary_encodings->encodings); } ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { @@ -1499,7 +1501,8 @@ ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* decoder, static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ns(Schema_table_t) schema, struct ArrowSchema* out, - struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryEncodings* dictionary_encodings_out, + struct ArrowError* error) { ArrowSchemaInit(out); // Top-level batch schema is typically non-nullable out->flags = 0; @@ -1515,7 +1518,7 @@ static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( } NANOARROW_RETURN_NOT_OK( - ArrowIpcDecoderSetChildren(out, fields, dictionaries_out, error)); + ArrowIpcDecoderSetChildren(out, fields, dictionary_encodings_out, error)); NANOARROW_RETURN_NOT_OK( ArrowIpcDecoderSetMetadata(out, ns(Schema_custom_metadata(schema)), error)); return NANOARROW_OK; @@ -1523,7 +1526,8 @@ static ArrowErrorCode ArrowIpcDecoderDecodeSchemaImpl( ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, - struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error) { + struct ArrowIpcDictionaryEncodings* dictionary_encodings_out, + struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1533,19 +1537,20 @@ ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( return EINVAL; } - if (dictionaries_out != NULL) { - ArrowIpcDictionaryEncodingsInit(dictionaries_out); + if (dictionary_encodings_out != NULL) { + ArrowIpcDictionaryEncodingsInit(dictionary_encodings_out); } struct ArrowSchema tmp; - ArrowErrorCode result = ArrowIpcDecoderDecodeSchemaImpl( - (ns(Schema_table_t))private_data->last_message, &tmp, dictionaries_out, error); + ArrowErrorCode result = + ArrowIpcDecoderDecodeSchemaImpl((ns(Schema_table_t))private_data->last_message, + &tmp, dictionary_encodings_out, error); if (result != NANOARROW_OK) { ArrowSchemaRelease(&tmp); - if (dictionaries_out != NULL) { - ArrowIpcDictionaryEncodingsReset(dictionaries_out); + if (dictionary_encodings_out != NULL) { + ArrowIpcDictionaryEncodingsReset(dictionary_encodings_out); } return result; diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index eff0ad855..b7b24f20a 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -589,7 +589,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { struct ArrowIpcDecoder decoder; struct ArrowError error; struct ArrowSchema schema; - struct ArrowIpcDictionaryEncodings encodings; + struct ArrowIpcDictionaryEncodings dictionary_encodings; struct ArrowBufferView data; data.data.as_uint8 = kDictionarySchema; @@ -600,9 +600,9 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); ASSERT_EQ(decoder.message_type, NANOARROW_IPC_MESSAGE_TYPE_SCHEMA); - ASSERT_EQ( - ArrowIpcDecoderDecodeSchemaWithDictionaries(&decoder, &schema, &encodings, &error), - NANOARROW_OK); + ASSERT_EQ(ArrowIpcDecoderDecodeSchemaWithDictionaries(&decoder, &schema, + &dictionary_encodings, &error), + NANOARROW_OK); ASSERT_EQ(schema.n_children, 1); EXPECT_STREQ(schema.children[0]->name, "some_col"); EXPECT_EQ(schema.children[0]->flags, ARROW_FLAG_NULLABLE); @@ -613,17 +613,17 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { // The dictionary encodings should fail to locate anything except the dictionary-encoded // field - ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&encodings, nullptr), nullptr); - ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&encodings, &schema), nullptr); + ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&dictionary_encodings, nullptr), nullptr); + ASSERT_EQ(ArrowIpcDictionaryEncodingsFind(&dictionary_encodings, &schema), nullptr); const struct ArrowIpcDictionaryEncoding* encoding = - ArrowIpcDictionaryEncodingsFind(&encodings, schema.children[0]); + ArrowIpcDictionaryEncodingsFind(&dictionary_encodings, schema.children[0]); ASSERT_NE(encoding, nullptr); ASSERT_EQ(encoding->schema, schema.children[0]); ASSERT_EQ(encoding->id, 0); ASSERT_EQ(encoding->kind, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY); ArrowSchemaRelease(&schema); - ArrowIpcDictionaryEncodingsReset(&encodings); + ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); ArrowIpcDecoderReset(&decoder); } From 043bfef59423c2f9841873740c7ac24c0ccbaf99 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 12:17:58 -0500 Subject: [PATCH 09/21] maybe add dictionary id to ArrowIpcField --- src/nanoarrow/ipc/decoder.c | 71 +++++++++++++++++++++++++------ src/nanoarrow/ipc/decoder_test.cc | 1 + src/nanoarrow/nanoarrow_ipc.h | 4 ++ 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index ecf2d62f6..ec57c416d 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -54,6 +55,8 @@ #define NANOARROW_IPC_MAGIC "ARROW1" +#define NANOARROW_IPC_NO_DICTIONARY_ID INT64_MIN + // Internal representation of a parsed "Field" from flatbuffers. This // represents a field in a depth-first walk of column arrays and their // children. @@ -66,6 +69,9 @@ struct ArrowIpcField { struct ArrowArray* array; // The cumulative number of buffers preceding this node. int64_t buffer_offset; + // Dictionary identifier (or NANOARROW_IPC_NO_DICTIONARY_ID if this is not a + // dictionary-encoded field) + int64_t dictionary_id; }; // Internal data specific to the read/decode process @@ -1608,14 +1614,45 @@ static void ArrowIpcDecoderCountFields(struct ArrowSchema* schema, int64_t* n_fi } } -static void ArrowIpcDecoderInitFields(struct ArrowIpcField* fields, - struct ArrowArrayView* array_view, - struct ArrowArray* array, int64_t* n_fields, - int64_t* n_buffers, int64_t* n_union_fields) { +static ArrowErrorCode ArrowIpcDecoderInitFields( + struct ArrowIpcField* fields, const struct ArrowSchema* schema, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowArrayView* array_view, struct ArrowArray* array, int64_t* n_fields, + int64_t* n_buffers, int64_t* n_union_fields, struct ArrowError* error) { struct ArrowIpcField* field = fields + (*n_fields); field->array_view = array_view; field->array = array; field->buffer_offset = *n_buffers; + field->dictionary_id = NANOARROW_IPC_NO_DICTIONARY_ID; + + if (schema->dictionary != NULL) { + if (dictionary_encodings == NULL) { + const char* name = schema->name; + if (name == NULL) { + name = ""; + } + + ArrowErrorSet(error, + "Can't resolve dictionary ID for field '%s' (dictionary encodings " + "not provided)", + name); + return EINVAL; + } + + const struct ArrowIpcDictionaryEncoding* dictionary_encoding = + ArrowIpcDictionaryEncodingsFind(dictionary_encodings, schema); + if (dictionary_encoding == NULL) { + const char* name = schema->name; + if (name == NULL) { + name = ""; + } + + ArrowErrorSet(error, "Can't resolve dictionary ID for field '%s'", name); + return EINVAL; + } + + field->dictionary_id = dictionary_encoding->id; + } for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { *n_buffers += array_view->layout.buffer_type[i] != NANOARROW_BUFFER_TYPE_NONE; @@ -1626,14 +1663,17 @@ static void ArrowIpcDecoderInitFields(struct ArrowIpcField* fields, *n_fields += 1; for (int64_t i = 0; i < array_view->n_children; i++) { - ArrowIpcDecoderInitFields(fields, array_view->children[i], array->children[i], - n_fields, n_buffers, n_union_fields); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( + fields, schema->children[i], dictionary_encodings, array_view->children[i], + array->children[i], n_fields, n_buffers, n_union_fields, error)); } + + return NANOARROW_OK; } -ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* decoder, - struct ArrowSchema* schema, - struct ArrowError* error) { +ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( + struct ArrowIpcDecoder* decoder, struct ArrowSchema* schema, + struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1674,13 +1714,20 @@ ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* decoder, // Init field information and calculate starting buffer offset for each int64_t field_i = 0; - ArrowIpcDecoderInitFields(private_data->fields, &private_data->array_view, - &private_data->array, &field_i, &private_data->n_buffers, - &private_data->n_union_fields); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInitFields( + private_data->fields, schema, dictionary_encodings, &private_data->array_view, + &private_data->array, &field_i, &private_data->n_buffers, + &private_data->n_union_fields, error)); return NANOARROW_OK; } +ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* decoder, + struct ArrowSchema* schema, + struct ArrowError* error) { + return ArrowIpcDecoderSetSchemaWithDictionaries(decoder, schema, NULL, error); +} + ArrowErrorCode ArrowIpcDecoderSetEndianness(struct ArrowIpcDecoder* decoder, enum ArrowIpcEndianness endianness) { struct ArrowIpcDecoderPrivate* private_data = diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index b7b24f20a..464ffab44 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -44,6 +44,7 @@ struct ArrowIpcField { struct ArrowArrayView* array_view; struct ArrowArray* array; int64_t buffer_offset; + int64_t dictionary_id; }; struct ArrowIpcDecoderPrivate { diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 298f8a3f1..4ed907b4a 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -473,6 +473,10 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* de struct ArrowSchema* schema, struct ArrowError* error); +NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( + struct ArrowIpcDecoder* decoder, struct ArrowSchema* schema, + struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error); + /// \brief Set the endianness used to decode future record batch messages /// /// Prepares the decoder for future record batch messages with the specified From 4b3e0b2d92bf7e3a103891a150c2246faa408a8d Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 12:35:20 -0500 Subject: [PATCH 10/21] update error and test --- src/nanoarrow/ipc/decoder.c | 10 +++++----- src/nanoarrow/ipc/decoder_test.cc | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index ec57c416d..29bd46093 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2120,11 +2120,6 @@ static int ArrowIpcDecoderWalkGetArray(struct ArrowArrayView* array_view, array_view->children[i], array->children[i], out->children[i], error)); } - if (array->dictionary != NULL) { - ArrowErrorSet(error, "Decode of dictionary array is not yet supported"); - return ENOTSUP; - } - return NANOARROW_OK; } @@ -2199,6 +2194,11 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayInternal( struct ArrowIpcField* root = private_data->fields + field_i + 1; + if (root->dictionary_id != NANOARROW_IPC_NO_DICTIONARY_ID) { + ArrowErrorSet(error, "Decoding a dictionary-encoded field is not yet supported"); + return ENOTSUP; + } + if (field_i == -1) { NANOARROW_RETURN_NOT_OK( ArrowArrayInitFromArrayView(out, &private_data->array_view, error)); diff --git a/src/nanoarrow/ipc/decoder_test.cc b/src/nanoarrow/ipc/decoder_test.cc index 464ffab44..5f958367a 100644 --- a/src/nanoarrow/ipc/decoder_test.cc +++ b/src/nanoarrow/ipc/decoder_test.cc @@ -623,6 +623,20 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeDictionarySchema) { ASSERT_EQ(encoding->id, 0); ASSERT_EQ(encoding->kind, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY); + // If we try to set the schema without the dictionaries, we should get an error + ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, &error), EINVAL); + ASSERT_STREQ(error.message, + "Can't resolve dictionary ID for field 'some_col' (dictionary encodings " + "not provided)"); + + // When we do set the schema, the ID should propagate into the fields + ASSERT_EQ(ArrowIpcDecoderSetSchemaWithDictionaries(&decoder, &schema, + &dictionary_encodings, &error), + NANOARROW_OK); + auto decoder_private = + reinterpret_cast(decoder.private_data); + ASSERT_EQ(decoder_private->fields[1].dictionary_id, 0); + ArrowSchemaRelease(&schema); ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); ArrowIpcDecoderReset(&decoder); From a3ff050c7ae1b138a21deee06d97a8533cd8e6fb Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 12:39:11 -0500 Subject: [PATCH 11/21] set schema with dictionaries in reader --- src/nanoarrow/ipc/reader.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 7ecc4ccdd..64d873d50 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -191,6 +191,7 @@ struct ArrowIpcArrayStreamReaderPrivate { struct ArrowBuffer header; struct ArrowBuffer body; int32_t expected_header_prefix_size; + struct ArrowIpcDictionaryEncodings dictionary_encodings; struct ArrowError error; }; @@ -210,6 +211,7 @@ static void ArrowIpcArrayStreamReaderRelease(struct ArrowArrayStream* stream) { ArrowBufferReset(&private_data->header); ArrowBufferReset(&private_data->body); + ArrowIpcDictionaryEncodingsReset(&private_data->dictionary_encodings); ArrowFree(private_data); stream->release = NULL; @@ -403,8 +405,9 @@ static int ArrowIpcArrayStreamReaderReadSchemaIfNeeded( &private_data->error); struct ArrowSchema tmp; - NANOARROW_RETURN_NOT_OK( - ArrowIpcDecoderDecodeSchema(&private_data->decoder, &tmp, &private_data->error)); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeSchemaWithDictionaries( + &private_data->decoder, &tmp, &private_data->dictionary_encodings, + &private_data->error)); // Only support "read the whole thing" for now if (private_data->field_index != -1) { @@ -414,8 +417,9 @@ static int ArrowIpcArrayStreamReaderReadSchemaIfNeeded( } // Notify the decoder of the schema for forthcoming messages - int result = - ArrowIpcDecoderSetSchema(&private_data->decoder, &tmp, &private_data->error); + int result = ArrowIpcDecoderSetSchemaWithDictionaries( + &private_data->decoder, &tmp, &private_data->dictionary_encodings, + &private_data->error); if (result != NANOARROW_OK) { ArrowSchemaRelease(&tmp); return result; From 392b7dbf79110d0d8bef37d662eabe6bb9a94547 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 12:39:46 -0500 Subject: [PATCH 12/21] namespace --- src/nanoarrow/nanoarrow_ipc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 4ed907b4a..9cceacdf0 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -59,6 +59,8 @@ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderDecodeArrayFromShared) #define ArrowIpcDecoderSetSchema \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderSetSchema) +#define ArrowIpcDecoderSetSchemaWithDictionaries \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderSetSchemaWithDictionaries) #define ArrowIpcDecoderSetEndianness \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowIpcDecoderSetEndianness) #define ArrowIpcDecoderPeekFooter \ From 16172bcac2851369500860b4935fb26dea2176ad Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 12:53:41 -0500 Subject: [PATCH 13/21] docs --- src/nanoarrow/ipc/decoder.c | 10 ++++---- src/nanoarrow/nanoarrow_ipc.h | 44 ++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 29bd46093..48470f047 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -1607,7 +1607,8 @@ ArrowErrorCode ArrowIpcDecoderDecodeFooter(struct ArrowIpcDecoder* decoder, return NANOARROW_OK; } -static void ArrowIpcDecoderCountFields(struct ArrowSchema* schema, int64_t* n_fields) { +static void ArrowIpcDecoderCountFields(const struct ArrowSchema* schema, + int64_t* n_fields) { *n_fields += 1; for (int64_t i = 0; i < schema->n_children; i++) { ArrowIpcDecoderCountFields(schema->children[i], n_fields); @@ -1672,8 +1673,9 @@ static ArrowErrorCode ArrowIpcDecoderInitFields( } ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( - struct ArrowIpcDecoder* decoder, struct ArrowSchema* schema, - struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error) { + struct ArrowIpcDecoder* decoder, const struct ArrowSchema* schema, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowError* error) { struct ArrowIpcDecoderPrivate* private_data = (struct ArrowIpcDecoderPrivate*)decoder->private_data; @@ -1723,7 +1725,7 @@ ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( } ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* decoder, - struct ArrowSchema* schema, + const struct ArrowSchema* schema, struct ArrowError* error) { return ArrowIpcDecoderSetSchemaWithDictionaries(decoder, schema, NULL, error); } diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 9cceacdf0..a05e05084 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -212,21 +212,34 @@ struct ArrowIpcDictionaryEncoding { enum ArrowIpcDictionaryKind kind; }; +/// \brief List of ArrowIpcDictionaryEncoding structs +/// +/// This structure provides a list of dictionary encoded fields extracted +/// from an ArrowSchema during decoding. Its members refer to pointers +/// within a specific schema, so care must be taken to keep the schema +/// containing the pointed-to ArrowSchema fields valid. struct ArrowIpcDictionaryEncodings { struct ArrowBuffer encodings; }; +/// \brief Initialize an ArrowIpcDictionaryEncodings list NANOARROW_DLL void ArrowIpcDictionaryEncodingsInit( struct ArrowIpcDictionaryEncodings* dictionaries); +/// \brief Append a given ArrowIpcDictionaryEncoding to this list NANOARROW_DLL ArrowErrorCode ArrowIpcDictionaryEncodingsAppend(struct ArrowIpcDictionaryEncodings* dictionaries, struct ArrowIpcDictionaryEncoding encoding); +/// \brief Resolve a ArrowIpcDictionaryEncoding for a given dictionary encoded field +/// +/// Returns NULL if the pointed to schema does not match any of the pointed to +/// schemas contained in this list. NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodingsFind( const struct ArrowIpcDictionaryEncodings* dictionaries, const struct ArrowSchema* schema); +/// \brief Release an encodings list and associated resources NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); @@ -451,6 +464,8 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeHeader(struct ArrowIpcDecoder* /// /// After a successful call to ArrowIpcDecoderDecodeHeader(), retrieve an ArrowSchema. /// The caller is responsible for releasing the schema if NANOARROW_OK is returned. +/// This is equivalent to calling ArrowIpcDecoderDecodeSchemaWithDictionaries() with +/// dictionaries_out = NULL. /// /// Returns EINVAL if the decoder did not just decode a schema message or /// NANOARROW_OK otherwise. @@ -458,6 +473,13 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* struct ArrowSchema* out, struct ArrowError* error); +/// \brief Decode an ArrowSchema with dictionary encoding information +/// +/// After a successful call to ArrowIpcDecoderDecodeHeader(), retrieve an ArrowSchema. +/// The caller is responsible for releasing the schema if NANOARROW_OK is returned. +/// +/// Returns EINVAL if the decoder did not just decode a schema message or +/// NANOARROW_OK otherwise. NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( struct ArrowIpcDecoder* decoder, struct ArrowSchema* out, struct ArrowIpcDictionaryEncodings* dictionaries_out, struct ArrowError* error); @@ -470,14 +492,30 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( /// Schema message (i.e., the decoder does not assume that the last-decoded /// schema message applies to future record batch messages). /// +/// This is equivalent to calling ArrowIpcDecoderSetSchemaWithDictionaries() with +/// dictionary_encodings = NULL. +/// /// Returns EINVAL if schema validation fails or NANOARROW_OK otherwise. NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* decoder, - struct ArrowSchema* schema, + const struct ArrowSchema* schema, struct ArrowError* error); +/// \brief Set the ArrowSchema and dictionary encodings used to decode future record batch +/// messages +/// +/// Prepares the decoder for future record batch messages +/// of this type. The decoder takes ownership of schema if NANOARROW_OK is returned. +/// Note that you must call this explicitly after decoding a +/// Schema message (i.e., the decoder does not assume that the last-decoded +/// schema message applies to future record batch messages). +/// +/// Returns EINVAL if schema validation fails or if the schema contains +/// dictionary encodings that could not be resolved in the provided +/// ArrowIpcDictionaryEncodings object, or NANOARROW_OK otherwise. NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderSetSchemaWithDictionaries( - struct ArrowIpcDecoder* decoder, struct ArrowSchema* schema, - struct ArrowIpcDictionaryEncodings* dictionary_encodings, struct ArrowError* error); + struct ArrowIpcDecoder* decoder, const struct ArrowSchema* schema, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + struct ArrowError* error); /// \brief Set the endianness used to decode future record batch messages /// From f958b15c43a3907e3677b872f28004ace3c7c491 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 17:15:36 -0500 Subject: [PATCH 14/21] initialize --- src/nanoarrow/ipc/reader.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index 64d873d50..e9d1d0441 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -538,6 +538,8 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit( private_data->use_shared_buffers = ArrowIpcSharedBufferIsThreadSafe(); } + ArrowIpcDictionaryEncodingsInit(&private_data->dictionary_encodings); + out->private_data = private_data; out->get_schema = &ArrowIpcArrayStreamReaderGetSchema; out->get_next = &ArrowIpcArrayStreamReaderGetNext; From ce8edb642c55aa98955ad44920d4081695c4c0f7 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 20:04:06 -0500 Subject: [PATCH 15/21] add hpp --- src/nanoarrow/ipc/ipc_hpp_test.cc | 15 +++++++++++++++ src/nanoarrow/nanoarrow_ipc.hpp | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/nanoarrow/ipc/ipc_hpp_test.cc b/src/nanoarrow/ipc/ipc_hpp_test.cc index e5867395c..b0a840906 100644 --- a/src/nanoarrow/ipc/ipc_hpp_test.cc +++ b/src/nanoarrow/ipc/ipc_hpp_test.cc @@ -86,3 +86,18 @@ TEST(NanoarrowIpcHppTest, NanoarrowIpcHppTestUniqueOutputStream) { EXPECT_NE(output2->release, nullptr); EXPECT_EQ(output->release, nullptr); // NOLINT(clang-analyzer-cplusplus.Move) } + +TEST(NanoarrowIpcHppTest, NanoarrowIpcHppTestUniqueDictionaryEncodings) { + nanoarrow::ipc::UniqueDictionaryEncodings dictionary_encodings; + ASSERT_EQ(ArrowIpcDictionaryEncodingsAppend( + dictionary_encodings.get(), + {nullptr, 1, NANOARROW_IPC_DICTIONARY_KIND_DENSE_ARRAY}), + NANOARROW_OK); + + nanoarrow::ipc::UniqueDictionaryEncodings dictionary_encodings2 = + std::move(dictionary_encodings); + EXPECT_NE(dictionary_encodings2->encodings.data, nullptr); + EXPECT_EQ( + dictionary_encodings->encodings.data, // NOLINT(clang-analyzer-cplusplus.Move) + nullptr); +} diff --git a/src/nanoarrow/nanoarrow_ipc.hpp b/src/nanoarrow/nanoarrow_ipc.hpp index a9fadf1f1..2524777d3 100644 --- a/src/nanoarrow/nanoarrow_ipc.hpp +++ b/src/nanoarrow/nanoarrow_ipc.hpp @@ -73,6 +73,23 @@ inline void release_pointer(struct ArrowIpcFooter* data) { ArrowIpcFooterReset(data); } +template <> +inline void init_pointer(struct ArrowIpcDictionaryEncodings* data) { + ArrowIpcDictionaryEncodingsInit(data); +} + +template <> +inline void move_pointer(struct ArrowIpcDictionaryEncodings* src, + struct ArrowIpcDictionaryEncodings* dst) { + memcpy(dst, src, sizeof(struct ArrowIpcDictionaryEncodings)); + ArrowIpcDictionaryEncodingsInit(src); +} + +template <> +inline void release_pointer(struct ArrowIpcDictionaryEncodings* data) { + ArrowIpcDictionaryEncodingsReset(data); +} + template <> inline void init_pointer(struct ArrowIpcEncoder* data) { data->private_data = nullptr; @@ -185,6 +202,9 @@ using UniqueDecoder = internal::Unique; /// \brief Class wrapping a unique struct ArrowIpcFooter using UniqueFooter = internal::Unique; +/// \brief Class wrapping a unique struct ArrowIpcFooter +using UniqueDictionaryEncodings = internal::Unique; + /// \brief Class wrapping a unique struct ArrowIpcEncoder using UniqueEncoder = internal::Unique; From f67b3b0204b63022cf5c74abd1ba0adac6469f84 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Tue, 17 Mar 2026 20:55:03 -0500 Subject: [PATCH 16/21] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/nanoarrow/nanoarrow_ipc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/nanoarrow_ipc.hpp b/src/nanoarrow/nanoarrow_ipc.hpp index 2524777d3..62e8ebd3c 100644 --- a/src/nanoarrow/nanoarrow_ipc.hpp +++ b/src/nanoarrow/nanoarrow_ipc.hpp @@ -202,7 +202,7 @@ using UniqueDecoder = internal::Unique; /// \brief Class wrapping a unique struct ArrowIpcFooter using UniqueFooter = internal::Unique; -/// \brief Class wrapping a unique struct ArrowIpcFooter +/// \brief Class wrapping a unique struct ArrowIpcDictionaryEncodings using UniqueDictionaryEncodings = internal::Unique; /// \brief Class wrapping a unique struct ArrowIpcEncoder From 81fca3b12b9eef998cd0ed583650c7ac86fb42b1 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 18 Mar 2026 09:00:36 -0500 Subject: [PATCH 17/21] local scope and docs for initializing the dictionaries --- src/nanoarrow/ipc/reader.c | 13 +++++-------- src/nanoarrow/nanoarrow_ipc.h | 2 ++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/nanoarrow/ipc/reader.c b/src/nanoarrow/ipc/reader.c index e9d1d0441..c70b14483 100644 --- a/src/nanoarrow/ipc/reader.c +++ b/src/nanoarrow/ipc/reader.c @@ -191,7 +191,6 @@ struct ArrowIpcArrayStreamReaderPrivate { struct ArrowBuffer header; struct ArrowBuffer body; int32_t expected_header_prefix_size; - struct ArrowIpcDictionaryEncodings dictionary_encodings; struct ArrowError error; }; @@ -211,7 +210,6 @@ static void ArrowIpcArrayStreamReaderRelease(struct ArrowArrayStream* stream) { ArrowBufferReset(&private_data->header); ArrowBufferReset(&private_data->body); - ArrowIpcDictionaryEncodingsReset(&private_data->dictionary_encodings); ArrowFree(private_data); stream->release = NULL; @@ -405,21 +403,22 @@ static int ArrowIpcArrayStreamReaderReadSchemaIfNeeded( &private_data->error); struct ArrowSchema tmp; + struct ArrowIpcDictionaryEncodings dictionary_encodings; NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeSchemaWithDictionaries( - &private_data->decoder, &tmp, &private_data->dictionary_encodings, - &private_data->error)); + &private_data->decoder, &tmp, &dictionary_encodings, &private_data->error)); // Only support "read the whole thing" for now if (private_data->field_index != -1) { ArrowSchemaRelease(&tmp); + ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); ArrowErrorSet(&private_data->error, "Field index != -1 is not yet supported"); return ENOTSUP; } // Notify the decoder of the schema for forthcoming messages int result = ArrowIpcDecoderSetSchemaWithDictionaries( - &private_data->decoder, &tmp, &private_data->dictionary_encodings, - &private_data->error); + &private_data->decoder, &tmp, &dictionary_encodings, &private_data->error); + ArrowIpcDictionaryEncodingsReset(&dictionary_encodings); if (result != NANOARROW_OK) { ArrowSchemaRelease(&tmp); return result; @@ -538,8 +537,6 @@ ArrowErrorCode ArrowIpcArrayStreamReaderInit( private_data->use_shared_buffers = ArrowIpcSharedBufferIsThreadSafe(); } - ArrowIpcDictionaryEncodingsInit(&private_data->dictionary_encodings); - out->private_data = private_data; out->get_schema = &ArrowIpcArrayStreamReaderGetSchema; out->get_next = &ArrowIpcArrayStreamReaderGetNext; diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index a05e05084..604d4671a 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -477,6 +477,8 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchema(struct ArrowIpcDecoder* /// /// After a successful call to ArrowIpcDecoderDecodeHeader(), retrieve an ArrowSchema. /// The caller is responsible for releasing the schema if NANOARROW_OK is returned. +/// Neither out nor dictionaries_out should be initialized; dictionaries_out may be +/// null to omit exporting dictionary identifiers. /// /// Returns EINVAL if the decoder did not just decode a schema message or /// NANOARROW_OK otherwise. From 86b5f985d0ac069057d613b74ad9a23f403053ba Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 18 Mar 2026 09:10:42 -0500 Subject: [PATCH 18/21] pass through error to the correct place --- src/nanoarrow/ipc/decoder.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 48470f047..8ce1f31dd 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2125,10 +2125,19 @@ static int ArrowIpcDecoderWalkGetArray(struct ArrowArrayView* array_view, return NANOARROW_OK; } -static int ArrowIpcDecoderWalkSetArrayView(struct ArrowIpcArraySetter* setter, +static int ArrowIpcDecoderWalkSetArrayView(struct ArrowIpcDecoder* decoder, + struct ArrowIpcArraySetter* setter, struct ArrowArrayView* array_view, struct ArrowArray* array, struct ArrowError* error) { + struct ArrowIpcDecoderPrivate* private_data = + (struct ArrowIpcDecoderPrivate*)decoder->private_data; + struct ArrowIpcField* ipc_field = private_data->fields + setter->field_i; + if (ipc_field->dictionary_id != NANOARROW_IPC_NO_DICTIONARY_ID) { + ArrowErrorSet(error, "Decoding a dictionary-encoding field is not supported"); + return ENOTSUP; + } + ns(FieldNode_struct_t) field = ns(FieldNode_vec_at(setter->fields, (size_t)setter->field_i)); array_view->length = ns(FieldNode_length(field)); @@ -2182,7 +2191,7 @@ static int ArrowIpcDecoderWalkSetArrayView(struct ArrowIpcArraySetter* setter, for (int64_t i = 0; i < array_view->n_children; i++) { NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderWalkSetArrayView( - setter, array_view->children[i], array->children[i], error)); + decoder, setter, array_view->children[i], array->children[i], error)); } return NANOARROW_OK; @@ -2279,12 +2288,13 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayViewInternal( setter.buffer_i++; for (int64_t i = 0; i < root->array_view->n_children; i++) { - NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderWalkSetArrayView( - &setter, root->array_view->children[i], root->array->children[i], error)); + NANOARROW_RETURN_NOT_OK( + ArrowIpcDecoderWalkSetArrayView(decoder, &setter, root->array_view->children[i], + root->array->children[i], error)); } } else { - NANOARROW_RETURN_NOT_OK( - ArrowIpcDecoderWalkSetArrayView(&setter, root->array_view, root->array, error)); + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderWalkSetArrayView( + decoder, &setter, root->array_view, root->array, error)); } // If we decoded a compressed message, wait for any pending decompression tasks to From ce3a46a891d9a6a15f3662b0b1a8db7787987eb5 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 18 Mar 2026 16:14:24 -0500 Subject: [PATCH 19/21] footer move --- src/nanoarrow/ipc/decoder.c | 5 ----- src/nanoarrow/nanoarrow_ipc.hpp | 31 ++++++++++++++++--------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 8ce1f31dd..400d483ec 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -2205,11 +2205,6 @@ static ArrowErrorCode ArrowIpcDecoderDecodeArrayInternal( struct ArrowIpcField* root = private_data->fields + field_i + 1; - if (root->dictionary_id != NANOARROW_IPC_NO_DICTIONARY_ID) { - ArrowErrorSet(error, "Decoding a dictionary-encoded field is not yet supported"); - return ENOTSUP; - } - if (field_i == -1) { NANOARROW_RETURN_NOT_OK( ArrowArrayInitFromArrayView(out, &private_data->array_view, error)); diff --git a/src/nanoarrow/nanoarrow_ipc.hpp b/src/nanoarrow/nanoarrow_ipc.hpp index 62e8ebd3c..84f1d90b7 100644 --- a/src/nanoarrow/nanoarrow_ipc.hpp +++ b/src/nanoarrow/nanoarrow_ipc.hpp @@ -58,36 +58,37 @@ inline void release_pointer(struct ArrowIpcDecoder* data) { } template <> -inline void init_pointer(struct ArrowIpcFooter* data) { - ArrowIpcFooterInit(data); +inline void init_pointer(struct ArrowIpcDictionaryEncodings* data) { + ArrowIpcDictionaryEncodingsInit(data); } template <> -inline void move_pointer(struct ArrowIpcFooter* src, struct ArrowIpcFooter* dst) { - ArrowSchemaMove(&src->schema, &dst->schema); - ArrowBufferMove(&src->record_batch_blocks, &dst->record_batch_blocks); +inline void move_pointer(struct ArrowIpcDictionaryEncodings* src, + struct ArrowIpcDictionaryEncodings* dst) { + memcpy(dst, src, sizeof(struct ArrowIpcDictionaryEncodings)); + ArrowIpcDictionaryEncodingsInit(src); } template <> -inline void release_pointer(struct ArrowIpcFooter* data) { - ArrowIpcFooterReset(data); +inline void release_pointer(struct ArrowIpcDictionaryEncodings* data) { + ArrowIpcDictionaryEncodingsReset(data); } template <> -inline void init_pointer(struct ArrowIpcDictionaryEncodings* data) { - ArrowIpcDictionaryEncodingsInit(data); +inline void init_pointer(struct ArrowIpcFooter* data) { + ArrowIpcFooterInit(data); } template <> -inline void move_pointer(struct ArrowIpcDictionaryEncodings* src, - struct ArrowIpcDictionaryEncodings* dst) { - memcpy(dst, src, sizeof(struct ArrowIpcDictionaryEncodings)); - ArrowIpcDictionaryEncodingsInit(src); +inline void move_pointer(struct ArrowIpcFooter* src, struct ArrowIpcFooter* dst) { + ArrowSchemaMove(&src->schema, &dst->schema); + ArrowBufferMove(&src->record_batch_blocks, &dst->record_batch_blocks); + move_pointer(&src->dictionaries, &dst->dictionaries); } template <> -inline void release_pointer(struct ArrowIpcDictionaryEncodings* data) { - ArrowIpcDictionaryEncodingsReset(data); +inline void release_pointer(struct ArrowIpcFooter* data) { + ArrowIpcFooterReset(data); } template <> From a202cc6accb835ea2f9bcd88dd64d1e20847682c Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 18 Mar 2026 16:16:00 -0500 Subject: [PATCH 20/21] fix comment --- src/nanoarrow/nanoarrow_ipc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index 604d4671a..ba94e5b5d 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -489,7 +489,7 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderDecodeSchemaWithDictionaries( /// \brief Set the ArrowSchema used to decode future record batch messages /// /// Prepares the decoder for future record batch messages -/// of this type. The decoder takes ownership of schema if NANOARROW_OK is returned. +/// of this type. The decoder does not take ownership of schema. /// Note that you must call this explicitly after decoding a /// Schema message (i.e., the decoder does not assume that the last-decoded /// schema message applies to future record batch messages). @@ -506,7 +506,7 @@ NANOARROW_DLL ArrowErrorCode ArrowIpcDecoderSetSchema(struct ArrowIpcDecoder* de /// messages /// /// Prepares the decoder for future record batch messages -/// of this type. The decoder takes ownership of schema if NANOARROW_OK is returned. +/// of this type. The decoder does not take ownership of schema. /// Note that you must call this explicitly after decoding a /// Schema message (i.e., the decoder does not assume that the last-decoded /// schema message applies to future record batch messages). From 0ffd5f63053adb56210822a134795bb048dab873 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 19 Mar 2026 00:32:45 -0500 Subject: [PATCH 21/21] start --- src/nanoarrow/ipc/decoder.c | 103 ++++++++++++++++++++++++++++++++++ src/nanoarrow/nanoarrow_ipc.h | 14 +++++ 2 files changed, 117 insertions(+) diff --git a/src/nanoarrow/ipc/decoder.c b/src/nanoarrow/ipc/decoder.c index 400d483ec..900d168a2 100644 --- a/src/nanoarrow/ipc/decoder.c +++ b/src/nanoarrow/ipc/decoder.c @@ -318,6 +318,109 @@ void ArrowIpcDictionaryEncodingsReset( ArrowBufferReset(&dictionary_encodings->encodings); } +struct ArrowIpcDictionary { + int64_t id; + struct ArrowIpcDecoder decoder; + struct ArrowIpcSharedBuffer buffer; +}; + +static ArrowErrorCode ArrowIpcDictionaryInit(struct ArrowIpcDictionary* dictionary) { + dictionary->id = NANOARROW_IPC_NO_DICTIONARY_ID; + NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderInit(&dictionary->decoder)); + struct ArrowBuffer empty; + ArrowBufferInit(&empty); + ArrowErrorCode result = ArrowIpcSharedBufferInit(&dictionary->buffer, &empty); + if (result != NANOARROW_OK) { + ArrowIpcDecoderReset(&dictionary->decoder); + return result; + } + + return NANOARROW_OK; +} + +static void ArrowIpcDictionaryReset(struct ArrowIpcDictionary* dictionary) { + ArrowIpcDecoderReset(&dictionary->decoder); + ArrowIpcSharedBufferReset(&dictionary->buffer); +} + +struct ArrowIpcDictionariesPrivate { + struct ArrowIpcDictionary* dictionaries; + int64_t num_dictionaries; +}; + +static ArrowErrorCode ArrowIpcDictionariesInitDictionaries( + struct ArrowIpcDictionariesPrivate* private_data, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings, + int64_t* num_initialized_decoders_out) { + ArrowErrorCode result; + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { + struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; + result = ArrowIpcDictionaryInit(dictionary); + if (result != NANOARROW_OK) { + *num_initialized_decoders_out = i; + return result; + } + + // Set dictionary->decoder schema with encodings, being careful not to + // deep copy the schema because this would invalidate dictionary_encodings + // pointers and would prevent nested dictionaries from working. + } + + *num_initialized_decoders_out = private_data->num_dictionaries; + return NANOARROW_OK; +} + +ArrowErrorCode ArrowIpcDictionariesInit( + struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings) { + NANOARROW_DCHECK(dictionaries != NULL); + NANOARROW_DCHECK(dictionary_encodings != NULL); + + memset(dictionaries, 0, sizeof(struct ArrowIpcDictionaries)); + struct ArrowIpcDictionariesPrivate* private_data = + (struct ArrowIpcDictionariesPrivate*)ArrowMalloc( + sizeof(struct ArrowIpcDictionariesPrivate)); + if (private_data == NULL) { + return ENOMEM; + } + + private_data->num_dictionaries = dictionary_encodings->encodings.size_bytes / + sizeof(struct ArrowIpcDictionaryEncoding); + + private_data->dictionaries = + ArrowMalloc(private_data->num_dictionaries * sizeof(struct ArrowIpcDictionaries)); + if (private_data->dictionaries == NULL) { + ArrowFree(private_data); + return ENOMEM; + } + + int64_t num_initialized_dictionaries = 0; + ArrowErrorCode result = ArrowIpcDictionariesInitDictionaries( + private_data, dictionary_encodings, &num_initialized_dictionaries); + if (num_initialized_dictionaries != private_data->num_dictionaries) { + for (int64_t i = 0; i < num_initialized_dictionaries; i++) { + ArrowIpcDictionaryReset(private_data->dictionaries + i); + } + + ArrowFree(private_data->dictionaries); + ArrowFree(private_data); + return result; + } + + return NANOARROW_OK; +} + +void ArrowIpcDictionariesReset(struct ArrowIpcDictionaries* dictionaries) { + NANOARROW_DCHECK(dictionaries != NULL); + struct ArrowIpcDictionariesPrivate* private_data = + (struct ArrowIpcDictionariesPrivate*)dictionaries->private_data; + + for (int64_t i = 0; i < private_data->num_dictionaries; i++) { + struct ArrowIpcDictionary* dictionary = private_data->dictionaries + i; + ArrowIpcDictionaryReset(dictionary); + } +} + ArrowErrorCode ArrowIpcDecoderInit(struct ArrowIpcDecoder* decoder) { memset(decoder, 0, sizeof(struct ArrowIpcDecoder)); struct ArrowIpcDecoderPrivate* private_data = diff --git a/src/nanoarrow/nanoarrow_ipc.h b/src/nanoarrow/nanoarrow_ipc.h index ba94e5b5d..ba16830a6 100644 --- a/src/nanoarrow/nanoarrow_ipc.h +++ b/src/nanoarrow/nanoarrow_ipc.h @@ -243,6 +243,20 @@ NANOARROW_DLL const struct ArrowIpcDictionaryEncoding* ArrowIpcDictionaryEncodin NANOARROW_DLL void ArrowIpcDictionaryEncodingsReset( struct ArrowIpcDictionaryEncodings* dictionaries); + +struct ArrowIpcDictionaries { + void* private_data; +}; + +NANOARROW_DLL ArrowErrorCode ArrowIpcDictionariesInit( + struct ArrowIpcDictionaries* dictionaries, + const struct ArrowIpcDictionaryEncodings* dictionary_encodings); + +NANOARROW_DLL void ArrowIpcDictionariesReset( + struct ArrowIpcDictionaries* dictionaries); + + + /// \brief Checks the nanoarrow runtime to make sure the run/build versions match NANOARROW_DLL ArrowErrorCode ArrowIpcCheckRuntime(struct ArrowError* error);