From be3b42f58ab10619c3769105468b9ce880809bf2 Mon Sep 17 00:00:00 2001 From: JonSnow1807 Date: Mon, 9 Nov 2026 23:06:14 -0600 Subject: [PATCH] fix: Read unaligned buffers safely in ArrowArrayView getters The ArrowArrayView getters read values, offsets and binary views through typed pointers (e.g., data.as_int32[i]). Buffers that arrive through the C Data interface do not have to be aligned, and for those each of these reads is a misaligned load that UBSan reports. This adds memcpy-based ArrowBufferViewGet*Unsafe() helpers for the remaining fixed-width types next to the 32- and 64-bit ones from #946 and uses them in ArrowArrayViewGetIntUnsafe(), ArrowArrayViewGetUIntUnsafe(), ArrowArrayViewGetDoubleUnsafe(), ArrowArrayViewGetStringUnsafe(), ArrowArrayViewGetBytesUnsafe(), ArrowArrayViewListChildOffset() and ArrowArrayViewUnionChildOffset(). Binary views are copied before their fields are read. The helpers index the buffer as an array of byte arrays instead of adding i * sizeof(T) to a byte pointer. With the multiplication, clang stopped inlining ArrowArrayViewGetIntUnsafe() and ArrowArrayViewGetDoubleUnsafe() in a file with more than one call site. Part of #950. --- src/nanoarrow/common/array_test.cc | 146 ++++++++++++++++++++++++++++ src/nanoarrow/common/inline_array.h | 103 +++++++++++--------- src/nanoarrow/common/inline_types.h | 50 +++++++++- 3 files changed, 249 insertions(+), 50 deletions(-) diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index 90b561397..3a29f6597 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -5141,6 +5141,152 @@ TEST(ArrayViewTest, ArrayViewTestGetDecimal256) { } #endif +// Copies the content of view to an address that is misaligned by misalignment +// bytes (1-7) relative to an 8-byte boundary. Getters must be able to read from +// such buffers without misaligned loads (checked by -fsanitize=alignment). +static const uint8_t* CopyToMisalignedStorage(std::vector* storage, + struct ArrowBufferView view, + int misalignment) { + storage->resize(view.size_bytes / sizeof(uint64_t) + 2); + uint8_t* out = reinterpret_cast(storage->data()) + misalignment; + memcpy(out, view.data.data, view.size_bytes); + return out; +} + +TEST(ArrayViewTest, ArrayViewTestGetNumericUnaligned) { + for (enum ArrowType type : + {NANOARROW_TYPE_INT8, NANOARROW_TYPE_UINT8, NANOARROW_TYPE_INT16, + NANOARROW_TYPE_UINT16, NANOARROW_TYPE_INT32, NANOARROW_TYPE_UINT32, + NANOARROW_TYPE_INT64, NANOARROW_TYPE_UINT64, NANOARROW_TYPE_HALF_FLOAT, + NANOARROW_TYPE_FLOAT, NANOARROW_TYPE_DOUBLE}) { + SCOPED_TRACE(ArrowTypeString(type)); + nanoarrow::UniqueArray array; + ASSERT_EQ(ArrowArrayInitFromType(array.get(), type), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK); + for (int64_t value = 0; value < 5; value++) { + ASSERT_EQ(ArrowArrayAppendInt(array.get(), value), NANOARROW_OK); + } + ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK); + array->offset = 1; + array->length = 4; + + nanoarrow::UniqueArrayView array_view; + ArrowArrayViewInitFromType(array_view.get(), type); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr), + NANOARROW_OK); + struct ArrowBufferView data = array_view->buffer_views[1]; + + for (int misalignment = 1; misalignment < 8; misalignment++) { + SCOPED_TRACE(misalignment); + std::vector storage; + array_view->buffer_views[1].data.as_uint8 = + CopyToMisalignedStorage(&storage, data, misalignment); + for (int64_t i = 0; i < array_view->length; i++) { + EXPECT_EQ(ArrowArrayViewGetIntUnsafe(array_view.get(), i), i + 1); + EXPECT_EQ(ArrowArrayViewGetUIntUnsafe(array_view.get(), i), + static_cast(i + 1)); + EXPECT_EQ(ArrowArrayViewGetDoubleUnsafe(array_view.get(), i), + static_cast(i + 1)); + } + } + } +} + +TEST(ArrayViewTest, ArrayViewTestGetStringUnaligned) { + // Includes the largest inlined and the smallest non-inlined binary view + const std::vector values = {"a", "", "twelve bytes", "thirteen byte", + "fourteen bytes"}; + + for (enum ArrowType type : {NANOARROW_TYPE_STRING, NANOARROW_TYPE_LARGE_STRING, + NANOARROW_TYPE_BINARY, NANOARROW_TYPE_LARGE_BINARY, + NANOARROW_TYPE_STRING_VIEW, NANOARROW_TYPE_BINARY_VIEW}) { + SCOPED_TRACE(ArrowTypeString(type)); + nanoarrow::UniqueArray array; + ASSERT_EQ(ArrowArrayInitFromType(array.get(), type), NANOARROW_OK); + ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK); + for (const auto& value : values) { + ASSERT_EQ(ArrowArrayAppendString(array.get(), ArrowCharView(value.c_str())), + NANOARROW_OK); + } + ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK); + array->offset = 1; + array->length = 4; + + nanoarrow::UniqueArrayView array_view; + ArrowArrayViewInitFromType(array_view.get(), type); + ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr), + NANOARROW_OK); + // Offsets, or views for the binary view types + struct ArrowBufferView offsets = array_view->buffer_views[1]; + + for (int misalignment = 1; misalignment < 8; misalignment++) { + SCOPED_TRACE(misalignment); + std::vector storage; + array_view->buffer_views[1].data.as_uint8 = + CopyToMisalignedStorage(&storage, offsets, misalignment); + for (int64_t i = 0; i < array_view->length; i++) { + struct ArrowStringView string_view = + ArrowArrayViewGetStringUnsafe(array_view.get(), i); + EXPECT_EQ(std::string(string_view.data, string_view.size_bytes), values[i + 1]); + struct ArrowBufferView bytes_view = + ArrowArrayViewGetBytesUnsafe(array_view.get(), i); + EXPECT_EQ(std::string(bytes_view.data.as_char, bytes_view.size_bytes), + values[i + 1]); + + if ((type == NANOARROW_TYPE_STRING_VIEW || type == NANOARROW_TYPE_BINARY_VIEW) && + values[i + 1].size() <= NANOARROW_BINARY_VIEW_INLINE_SIZE) { + // Inlined bytes must point into the views buffer, after the 4-byte size + const uint8_t* view = array_view->buffer_views[1].data.as_uint8 + + (array_view->offset + i) * sizeof(union ArrowBinaryView); + EXPECT_EQ(bytes_view.data.as_uint8, view + sizeof(int32_t)); + } + } + } + } +} + +TEST(ArrayViewTest, ArrayViewTestGetChildOffsetUnaligned) { + const int32_t offsets32[] = {0, 1, 3, 6}; + const int64_t offsets64[] = {0, 1, 3, 6}; + + for (enum ArrowType type : + {NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, NANOARROW_TYPE_MAP, + NANOARROW_TYPE_LIST_VIEW, NANOARROW_TYPE_LARGE_LIST_VIEW, + NANOARROW_TYPE_DENSE_UNION}) { + SCOPED_TRACE(ArrowTypeString(type)); + nanoarrow::UniqueArrayView array_view; + ArrowArrayViewInitFromType(array_view.get(), type); + array_view->offset = 1; + array_view->length = 3; + + struct ArrowBufferView offsets; + if (array_view->layout.element_size_bits[1] == 64) { + offsets.data.as_int64 = offsets64; + offsets.size_bytes = sizeof(offsets64); + } else { + offsets.data.as_int32 = offsets32; + offsets.size_bytes = sizeof(offsets32); + } + + for (int misalignment = 1; misalignment < 8; misalignment++) { + SCOPED_TRACE(misalignment); + std::vector storage; + array_view->buffer_views[1].data.as_uint8 = + CopyToMisalignedStorage(&storage, offsets, misalignment); + for (int64_t i = 0; i < array_view->length; i++) { + if (type == NANOARROW_TYPE_DENSE_UNION) { + EXPECT_EQ(ArrowArrayViewUnionChildOffset(array_view.get(), i), + offsets32[i + 1]); + } else { + // ArrowArrayViewListChildOffset() does not apply the array offset + EXPECT_EQ(ArrowArrayViewListChildOffset(array_view.get(), i + 1), + offsets32[i + 1]); + } + } + } + } +} + // Helper to build a simple int32 array with known data for shared array tests static void MakeInt32ArrayForSharedTest(struct ArrowArray* array, const int32_t* values, int64_t n) { diff --git a/src/nanoarrow/common/inline_array.h b/src/nanoarrow/common/inline_array.h index 7153a3355..bfd821ffe 100644 --- a/src/nanoarrow/common/inline_array.h +++ b/src/nanoarrow/common/inline_array.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1110,7 +1111,8 @@ static inline int64_t ArrowArrayViewUnionChildOffset( const struct ArrowArrayView* array_view, int64_t i) { switch (array_view->storage_type) { case NANOARROW_TYPE_DENSE_UNION: - return array_view->buffer_views[1].data.as_int32[array_view->offset + i]; + return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1], + array_view->offset + i); case NANOARROW_TYPE_SPARSE_UNION: return array_view->offset + i; default: @@ -1124,10 +1126,10 @@ static inline int64_t ArrowArrayViewListChildOffset( case NANOARROW_TYPE_LIST: case NANOARROW_TYPE_MAP: case NANOARROW_TYPE_LIST_VIEW: - return array_view->buffer_views[1].data.as_int32[i]; + return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1], i); case NANOARROW_TYPE_LARGE_LIST: case NANOARROW_TYPE_LARGE_LIST_VIEW: - return array_view->buffer_views[1].data.as_int64[i]; + return ArrowBufferViewGetInt64Unsafe(&array_view->buffer_views[1], i); default: return -1; } @@ -1135,15 +1137,19 @@ static inline int64_t ArrowArrayViewListChildOffset( static struct ArrowBufferView ArrowArrayViewGetBytesFromViewArrayUnsafe( const struct ArrowArrayView* array_view, int64_t i) { - const union ArrowBinaryView* bv = &array_view->buffer_views[1].data.as_binary_view[i]; - struct ArrowBufferView out = {{NULL}, bv->inlined.size}; - if (bv->inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) { - out.data.as_uint8 = bv->inlined.data; + const uint8_t* bv_data = + array_view->buffer_views[1].data.as_uint8 + i * sizeof(union ArrowBinaryView); + union ArrowBinaryView bv; + memcpy(&bv, bv_data, sizeof(bv)); + struct ArrowBufferView out = {{NULL}, bv.inlined.size}; + if (bv.inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) { + // Point to the inlined bytes in the buffer, not in the local copy + out.data.as_uint8 = bv_data + offsetof(struct ArrowBinaryViewInlined, data); return out; } - out.data.data = array_view->variadic_buffers[bv->ref.buffer_index]; - out.data.as_uint8 += bv->ref.offset; + out.data.data = array_view->variadic_buffers[bv.ref.buffer_index]; + out.data.as_uint8 += bv.ref.offset; return out; } @@ -1153,28 +1159,28 @@ static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* ar i += array_view->offset; switch (array_view->storage_type) { case NANOARROW_TYPE_INT64: - return data_view->data.as_int64[i]; + return ArrowBufferViewGetInt64Unsafe(data_view, i); case NANOARROW_TYPE_UINT64: - return data_view->data.as_uint64[i]; + return ArrowBufferViewGetUInt64Unsafe(data_view, i); case NANOARROW_TYPE_INTERVAL_MONTHS: case NANOARROW_TYPE_INT32: - return data_view->data.as_int32[i]; + return ArrowBufferViewGetInt32Unsafe(data_view, i); case NANOARROW_TYPE_UINT32: - return data_view->data.as_uint32[i]; + return ArrowBufferViewGetUInt32Unsafe(data_view, i); case NANOARROW_TYPE_INT16: - return data_view->data.as_int16[i]; + return ArrowBufferViewGetInt16Unsafe(data_view, i); case NANOARROW_TYPE_UINT16: - return data_view->data.as_uint16[i]; + return ArrowBufferViewGetUInt16Unsafe(data_view, i); case NANOARROW_TYPE_INT8: return data_view->data.as_int8[i]; case NANOARROW_TYPE_UINT8: return data_view->data.as_uint8[i]; case NANOARROW_TYPE_DOUBLE: - return (int64_t)data_view->data.as_double[i]; + return (int64_t)ArrowBufferViewGetDoubleUnsafe(data_view, i); case NANOARROW_TYPE_FLOAT: - return (int64_t)data_view->data.as_float[i]; + return (int64_t)ArrowBufferViewGetFloatUnsafe(data_view, i); case NANOARROW_TYPE_HALF_FLOAT: - return (int64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]); + return (int64_t)ArrowHalfFloatToFloat(ArrowBufferViewGetUInt16Unsafe(data_view, i)); case NANOARROW_TYPE_BOOL: return ArrowBitGet(data_view->data.as_uint8, i); default: @@ -1188,28 +1194,29 @@ static inline uint64_t ArrowArrayViewGetUIntUnsafe( const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; switch (array_view->storage_type) { case NANOARROW_TYPE_INT64: - return data_view->data.as_int64[i]; + return ArrowBufferViewGetInt64Unsafe(data_view, i); case NANOARROW_TYPE_UINT64: - return data_view->data.as_uint64[i]; + return ArrowBufferViewGetUInt64Unsafe(data_view, i); case NANOARROW_TYPE_INTERVAL_MONTHS: case NANOARROW_TYPE_INT32: - return data_view->data.as_int32[i]; + return ArrowBufferViewGetInt32Unsafe(data_view, i); case NANOARROW_TYPE_UINT32: - return data_view->data.as_uint32[i]; + return ArrowBufferViewGetUInt32Unsafe(data_view, i); case NANOARROW_TYPE_INT16: - return data_view->data.as_int16[i]; + return ArrowBufferViewGetInt16Unsafe(data_view, i); case NANOARROW_TYPE_UINT16: - return data_view->data.as_uint16[i]; + return ArrowBufferViewGetUInt16Unsafe(data_view, i); case NANOARROW_TYPE_INT8: return data_view->data.as_int8[i]; case NANOARROW_TYPE_UINT8: return data_view->data.as_uint8[i]; case NANOARROW_TYPE_DOUBLE: - return (uint64_t)data_view->data.as_double[i]; + return (uint64_t)ArrowBufferViewGetDoubleUnsafe(data_view, i); case NANOARROW_TYPE_FLOAT: - return (uint64_t)data_view->data.as_float[i]; + return (uint64_t)ArrowBufferViewGetFloatUnsafe(data_view, i); case NANOARROW_TYPE_HALF_FLOAT: - return (uint64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]); + return (uint64_t)ArrowHalfFloatToFloat( + ArrowBufferViewGetUInt16Unsafe(data_view, i)); case NANOARROW_TYPE_BOOL: return ArrowBitGet(data_view->data.as_uint8, i); default: @@ -1223,27 +1230,27 @@ static inline double ArrowArrayViewGetDoubleUnsafe( const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; switch (array_view->storage_type) { case NANOARROW_TYPE_INT64: - return (double)data_view->data.as_int64[i]; + return (double)ArrowBufferViewGetInt64Unsafe(data_view, i); case NANOARROW_TYPE_UINT64: - return (double)data_view->data.as_uint64[i]; + return (double)ArrowBufferViewGetUInt64Unsafe(data_view, i); case NANOARROW_TYPE_INT32: - return data_view->data.as_int32[i]; + return ArrowBufferViewGetInt32Unsafe(data_view, i); case NANOARROW_TYPE_UINT32: - return data_view->data.as_uint32[i]; + return ArrowBufferViewGetUInt32Unsafe(data_view, i); case NANOARROW_TYPE_INT16: - return data_view->data.as_int16[i]; + return ArrowBufferViewGetInt16Unsafe(data_view, i); case NANOARROW_TYPE_UINT16: - return data_view->data.as_uint16[i]; + return ArrowBufferViewGetUInt16Unsafe(data_view, i); case NANOARROW_TYPE_INT8: return data_view->data.as_int8[i]; case NANOARROW_TYPE_UINT8: return data_view->data.as_uint8[i]; case NANOARROW_TYPE_DOUBLE: - return data_view->data.as_double[i]; + return ArrowBufferViewGetDoubleUnsafe(data_view, i); case NANOARROW_TYPE_FLOAT: - return data_view->data.as_float[i]; + return ArrowBufferViewGetFloatUnsafe(data_view, i); case NANOARROW_TYPE_HALF_FLOAT: - return ArrowHalfFloatToFloat(data_view->data.as_uint16[i]); + return ArrowHalfFloatToFloat(ArrowBufferViewGetUInt16Unsafe(data_view, i)); case NANOARROW_TYPE_BOOL: return ArrowBitGet(data_view->data.as_uint8, i); default: @@ -1261,15 +1268,15 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe( switch (array_view->storage_type) { case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_BINARY: - view.data = data_view + offsets_view->data.as_int32[i]; - view.size_bytes = - (int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i]; + view.data = data_view + ArrowBufferViewGetInt32Unsafe(offsets_view, i); + view.size_bytes = (int64_t)ArrowBufferViewGetInt32Unsafe(offsets_view, i + 1) - + ArrowBufferViewGetInt32Unsafe(offsets_view, i); break; case NANOARROW_TYPE_LARGE_STRING: case NANOARROW_TYPE_LARGE_BINARY: - view.data = data_view + offsets_view->data.as_int64[i]; - view.size_bytes = - offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i]; + view.data = data_view + ArrowBufferViewGetInt64Unsafe(offsets_view, i); + view.size_bytes = ArrowBufferViewGetInt64Unsafe(offsets_view, i + 1) - + ArrowBufferViewGetInt64Unsafe(offsets_view, i); break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; @@ -1302,15 +1309,15 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( switch (array_view->storage_type) { case NANOARROW_TYPE_STRING: case NANOARROW_TYPE_BINARY: - view.size_bytes = - (int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i]; - view.data.as_uint8 = data_view + offsets_view->data.as_int32[i]; + view.size_bytes = (int64_t)ArrowBufferViewGetInt32Unsafe(offsets_view, i + 1) - + ArrowBufferViewGetInt32Unsafe(offsets_view, i); + view.data.as_uint8 = data_view + ArrowBufferViewGetInt32Unsafe(offsets_view, i); break; case NANOARROW_TYPE_LARGE_STRING: case NANOARROW_TYPE_LARGE_BINARY: - view.size_bytes = - offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i]; - view.data.as_uint8 = data_view + offsets_view->data.as_int64[i]; + view.size_bytes = ArrowBufferViewGetInt64Unsafe(offsets_view, i + 1) - + ArrowBufferViewGetInt64Unsafe(offsets_view, i); + view.data.as_uint8 = data_view + ArrowBufferViewGetInt64Unsafe(offsets_view, i); break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; diff --git a/src/nanoarrow/common/inline_types.h b/src/nanoarrow/common/inline_types.h index 5737d6012..35e0f9e35 100644 --- a/src/nanoarrow/common/inline_types.h +++ b/src/nanoarrow/common/inline_types.h @@ -730,17 +730,63 @@ struct ArrowBufferView { int64_t size_bytes; }; +// Read element i of a buffer that is not necessarily aligned to the element size. +// Indexing the buffer as an array of byte arrays (instead of computing +// i * sizeof(value)) avoids an extra multiplication in clang's inlining cost model, +// so callers like ArrowArrayViewGetIntUnsafe() are still inlined. static inline int32_t ArrowBufferViewGetInt32Unsafe(const struct ArrowBufferView* view, int64_t i) { int32_t value; - memcpy(&value, view->data.as_uint8 + i * sizeof(int32_t), sizeof(value)); + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); return value; } static inline int64_t ArrowBufferViewGetInt64Unsafe(const struct ArrowBufferView* view, int64_t i) { int64_t value; - memcpy(&value, view->data.as_uint8 + i * sizeof(int64_t), sizeof(value)); + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline int16_t ArrowBufferViewGetInt16Unsafe(const struct ArrowBufferView* view, + int64_t i) { + int16_t value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline uint16_t ArrowBufferViewGetUInt16Unsafe(const struct ArrowBufferView* view, + int64_t i) { + uint16_t value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline uint32_t ArrowBufferViewGetUInt32Unsafe(const struct ArrowBufferView* view, + int64_t i) { + uint32_t value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline uint64_t ArrowBufferViewGetUInt64Unsafe(const struct ArrowBufferView* view, + int64_t i) { + uint64_t value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline float ArrowBufferViewGetFloatUnsafe(const struct ArrowBufferView* view, + int64_t i) { + float value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); + return value; +} + +static inline double ArrowBufferViewGetDoubleUnsafe(const struct ArrowBufferView* view, + int64_t i) { + double value; + memcpy(&value, ((const uint8_t(*)[sizeof(value)])view->data.data)[i], sizeof(value)); return value; }