From 54be19de82bafe3e4bf4847da1802b7404b1cda5 Mon Sep 17 00:00:00 2001 From: SabyaXSachi Date: Wed, 18 Nov 2026 01:20:32 +0530 Subject: [PATCH] GH-55650: [C++][ORC] Preserve offsets when normalizing structs NormalizeArray()'s STRUCT branch rebuilt each child of a struct with struct-level nulls without propagating offsets: the child and parent ArrayData were reconstructed at offset 0 and the validity bitmaps were combined starting at bit 0. Writing a sliced struct column therefore serialized the wrong rows, misaligning the column with the rest of the file. Fix by reusing StructArray::GetFlattenedField() (already used by the SPARSE_UNION branch of the same function) to push the parent slice down to each child and merge parent and child validity at the correct bit positions, then rebasing the struct's own validity bitmap to offset 0 before rebuilding the struct. The null_count() == 0 early return is unchanged. Adds deterministic regression tests covering a sliced struct column with struct-level nulls, a struct whose child has an independent offset, and both combined. Closes #55650. --- cpp/src/arrow/adapters/orc/adapter_test.cc | 41 +++++++++++++++++++ cpp/src/arrow/adapters/orc/util.cc | 46 ++++++++++------------ 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/cpp/src/arrow/adapters/orc/adapter_test.cc b/cpp/src/arrow/adapters/orc/adapter_test.cc index 3cbb6d7828f1..eec7aea10ade 100644 --- a/cpp/src/arrow/adapters/orc/adapter_test.cc +++ b/cpp/src/arrow/adapters/orc/adapter_test.cc @@ -746,6 +746,47 @@ TEST_F(TestORCWriterTrivialNoConversion, writeSlicedBatch) { AssertTableWriteReadEqual(table, table, kDefaultSmallMemStreamSize / 16); } +TEST_F(TestORCWriterTrivialNoConversion, writeSlicedBatchWithStructNulls) { + std::shared_ptr local_schema = schema( + {field("id", int32()), field("s", struct_({field("a", int64())}))}); + std::shared_ptr table = TableFromJSON(local_schema, {R"([ + {"id": 0, "s": {"a": 0}}, {"id": 1, "s": null}, {"id": 2, "s": {"a": 2}}, + {"id": 3, "s": {"a": 3}}, {"id": 4, "s": null}, {"id": 5, "s": {"a": 5}}])"}); + table = table->Slice(1, 4); + + AssertTableWriteReadEqual(table, table, kDefaultSmallMemStreamSize / 16); +} + +TEST_F(TestORCWriterTrivialNoConversion, writeStructWithOffsetChild) { + // The child array carries its own offset (logical values are [13, 14, 15]), + // while the struct itself starts at offset 0 and has a struct-level null at + // row 1. + std::shared_ptr values = + ArrayFromJSON(int64(), "[10, 11, 12, 13, 14, 15]")->Slice(3, 3); + ASSERT_OK_AND_ASSIGN(auto null_bitmap, AllocateBuffer(1, default_memory_pool())); + // Bit i is set when row i is valid: rows 0 and 2 are valid, row 1 is null. + null_bitmap->mutable_data()[0] = 0b101; + ASSERT_OK_AND_ASSIGN( + auto struct_array, + StructArray::Make({values}, {"a"}, std::move(null_bitmap), /*null_count=*/1)); + + AssertArrayWriteReadEqual(struct_array, struct_array, kDefaultSmallMemStreamSize / 16); +} + +TEST_F(TestORCWriterTrivialNoConversion, writeSlicedStructWithOffsetChild) { + // Combine a non-zero struct offset with an independently offset child. + std::shared_ptr values = + ArrayFromJSON(int64(), "[10, 11, 12, 13, 14, 15]")->Slice(3, 3); + ASSERT_OK_AND_ASSIGN(auto null_bitmap, AllocateBuffer(1, default_memory_pool())); + null_bitmap->mutable_data()[0] = 0b101; + ASSERT_OK_AND_ASSIGN( + auto struct_array, + StructArray::Make({values}, {"a"}, std::move(null_bitmap), /*null_count=*/1)); + std::shared_ptr sliced = struct_array->Slice(1, 2); + + AssertArrayWriteReadEqual(sliced, sliced, kDefaultSmallMemStreamSize / 16); +} + class TestORCWriterTrivialWithConversion : public ::testing::Test { public: TestORCWriterTrivialWithConversion() { diff --git a/cpp/src/arrow/adapters/orc/util.cc b/cpp/src/arrow/adapters/orc/util.cc index 91260302abd3..56786d0d1a58 100644 --- a/cpp/src/arrow/adapters/orc/util.cc +++ b/cpp/src/arrow/adapters/orc/util.cc @@ -24,6 +24,7 @@ #include #include "arrow/array/builder_base.h" +#include "arrow/buffer.h" #include "arrow/builder.h" #include "arrow/chunked_array.h" #include "arrow/scalar.h" @@ -397,33 +398,26 @@ Result> NormalizeArray(const std::shared_ptr& arra return array; } else { auto struct_array = checked_pointer_cast(array); - const std::shared_ptr bitmap = struct_array->null_bitmap(); - std::shared_ptr struct_type = struct_array->type(); - std::size_t size = struct_type->fields().size(); - std::vector> new_children(size, nullptr); - for (std::size_t i = 0; i < size; i++) { - std::shared_ptr child = struct_array->field(static_cast(i)); - const std::shared_ptr child_bitmap = child->null_bitmap(); - std::shared_ptr final_child_bitmap; - if (child_bitmap == nullptr) { - final_child_bitmap = bitmap; - } else { - ARROW_ASSIGN_OR_RAISE( - final_child_bitmap, - internal::BitmapAnd(default_memory_pool(), bitmap->data(), 0, - child_bitmap->data(), 0, struct_array->length(), 0)); - } - std::shared_ptr child_array_data = child->data(); - std::vector> child_buffers = child_array_data->buffers; - child_buffers[0] = final_child_bitmap; - std::shared_ptr new_child_array_data = - ArrayData::Make(child->type(), child->length(), child_buffers, - child_array_data->child_data, child_array_data->dictionary); - ARROW_ASSIGN_OR_RAISE(new_children[i], - NormalizeArray(MakeArray(new_child_array_data))); + std::vector> new_children( + struct_array->type()->num_fields()); + for (int i = 0; i < static_cast(new_children.size()); ++i) { + // Push the struct's offset down and merge the struct's validity into + // the child's, so each child is valid only on rows valid in both. + ARROW_ASSIGN_OR_RAISE(auto child, struct_array->GetFlattenedField(i)); + ARROW_ASSIGN_OR_RAISE(new_children[i], NormalizeArray(child)); } - return std::make_shared(struct_type, struct_array->length(), - new_children, bitmap); + // The children have been rebased to offset 0, so rebase the struct's + // own validity bitmap to offset 0 as well. + std::shared_ptr new_bitmap; + ARROW_ASSIGN_OR_RAISE( + new_bitmap, + AllocateEmptyBitmap(struct_array->length(), default_memory_pool())); + internal::CopyBitmap(struct_array->null_bitmap_data(), struct_array->offset(), + struct_array->length(), new_bitmap->mutable_data(), 0); + return std::make_shared(struct_array->type(), + struct_array->length(), new_children, + std::move(new_bitmap), + struct_array->null_count()); } } case Type::type::LIST: {