Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 146 additions & 0 deletions src/nanoarrow/common/array_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5141,6 +5141,152 @@ TEST(ArrayViewTest, ArrayViewTestGetDecimal256) {
}
#endif

// Copies the content of view to an address that is misaligned by misalignment
// bytes (1-7) relative to an 8-byte boundary. Getters must be able to read from
// such buffers without misaligned loads (checked by -fsanitize=alignment).
static const uint8_t* CopyToMisalignedStorage(std::vector<uint64_t>* storage,
struct ArrowBufferView view,
int misalignment) {
storage->resize(view.size_bytes / sizeof(uint64_t) + 2);
uint8_t* out = reinterpret_cast<uint8_t*>(storage->data()) + misalignment;
memcpy(out, view.data.data, view.size_bytes);
return out;
}

TEST(ArrayViewTest, ArrayViewTestGetNumericUnaligned) {
for (enum ArrowType type :
{NANOARROW_TYPE_INT8, NANOARROW_TYPE_UINT8, NANOARROW_TYPE_INT16,
NANOARROW_TYPE_UINT16, NANOARROW_TYPE_INT32, NANOARROW_TYPE_UINT32,
NANOARROW_TYPE_INT64, NANOARROW_TYPE_UINT64, NANOARROW_TYPE_HALF_FLOAT,
NANOARROW_TYPE_FLOAT, NANOARROW_TYPE_DOUBLE}) {
SCOPED_TRACE(ArrowTypeString(type));
nanoarrow::UniqueArray array;
ASSERT_EQ(ArrowArrayInitFromType(array.get(), type), NANOARROW_OK);
ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
for (int64_t value = 0; value < 5; value++) {
ASSERT_EQ(ArrowArrayAppendInt(array.get(), value), NANOARROW_OK);
}
ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
array->offset = 1;
array->length = 4;

nanoarrow::UniqueArrayView array_view;
ArrowArrayViewInitFromType(array_view.get(), type);
ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr),
NANOARROW_OK);
struct ArrowBufferView data = array_view->buffer_views[1];

for (int misalignment = 1; misalignment < 8; misalignment++) {
SCOPED_TRACE(misalignment);
std::vector<uint64_t> storage;
array_view->buffer_views[1].data.as_uint8 =
CopyToMisalignedStorage(&storage, data, misalignment);
for (int64_t i = 0; i < array_view->length; i++) {
EXPECT_EQ(ArrowArrayViewGetIntUnsafe(array_view.get(), i), i + 1);
EXPECT_EQ(ArrowArrayViewGetUIntUnsafe(array_view.get(), i),
static_cast<uint64_t>(i + 1));
EXPECT_EQ(ArrowArrayViewGetDoubleUnsafe(array_view.get(), i),
static_cast<double>(i + 1));
}
}
}
}

TEST(ArrayViewTest, ArrayViewTestGetStringUnaligned) {
// Includes the largest inlined and the smallest non-inlined binary view
const std::vector<std::string> values = {"a", "", "twelve bytes", "thirteen byte",
"fourteen bytes"};

for (enum ArrowType type : {NANOARROW_TYPE_STRING, NANOARROW_TYPE_LARGE_STRING,
NANOARROW_TYPE_BINARY, NANOARROW_TYPE_LARGE_BINARY,
NANOARROW_TYPE_STRING_VIEW, NANOARROW_TYPE_BINARY_VIEW}) {
SCOPED_TRACE(ArrowTypeString(type));
nanoarrow::UniqueArray array;
ASSERT_EQ(ArrowArrayInitFromType(array.get(), type), NANOARROW_OK);
ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
for (const auto& value : values) {
ASSERT_EQ(ArrowArrayAppendString(array.get(), ArrowCharView(value.c_str())),
NANOARROW_OK);
}
ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
array->offset = 1;
array->length = 4;

nanoarrow::UniqueArrayView array_view;
ArrowArrayViewInitFromType(array_view.get(), type);
ASSERT_EQ(ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr),
NANOARROW_OK);
// Offsets, or views for the binary view types
struct ArrowBufferView offsets = array_view->buffer_views[1];

for (int misalignment = 1; misalignment < 8; misalignment++) {
SCOPED_TRACE(misalignment);
std::vector<uint64_t> storage;
array_view->buffer_views[1].data.as_uint8 =
CopyToMisalignedStorage(&storage, offsets, misalignment);
for (int64_t i = 0; i < array_view->length; i++) {
struct ArrowStringView string_view =
ArrowArrayViewGetStringUnsafe(array_view.get(), i);
EXPECT_EQ(std::string(string_view.data, string_view.size_bytes), values[i + 1]);
struct ArrowBufferView bytes_view =
ArrowArrayViewGetBytesUnsafe(array_view.get(), i);
EXPECT_EQ(std::string(bytes_view.data.as_char, bytes_view.size_bytes),
values[i + 1]);

if ((type == NANOARROW_TYPE_STRING_VIEW || type == NANOARROW_TYPE_BINARY_VIEW) &&
values[i + 1].size() <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {
// Inlined bytes must point into the views buffer, after the 4-byte size
const uint8_t* view = array_view->buffer_views[1].data.as_uint8 +
(array_view->offset + i) * sizeof(union ArrowBinaryView);
EXPECT_EQ(bytes_view.data.as_uint8, view + sizeof(int32_t));
}
}
}
}
}

TEST(ArrayViewTest, ArrayViewTestGetChildOffsetUnaligned) {
const int32_t offsets32[] = {0, 1, 3, 6};
const int64_t offsets64[] = {0, 1, 3, 6};

for (enum ArrowType type :
{NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, NANOARROW_TYPE_MAP,
NANOARROW_TYPE_LIST_VIEW, NANOARROW_TYPE_LARGE_LIST_VIEW,
NANOARROW_TYPE_DENSE_UNION}) {
SCOPED_TRACE(ArrowTypeString(type));
nanoarrow::UniqueArrayView array_view;
ArrowArrayViewInitFromType(array_view.get(), type);
array_view->offset = 1;
array_view->length = 3;

struct ArrowBufferView offsets;
if (array_view->layout.element_size_bits[1] == 64) {
offsets.data.as_int64 = offsets64;
offsets.size_bytes = sizeof(offsets64);
} else {
offsets.data.as_int32 = offsets32;
offsets.size_bytes = sizeof(offsets32);
}

for (int misalignment = 1; misalignment < 8; misalignment++) {
SCOPED_TRACE(misalignment);
std::vector<uint64_t> storage;
array_view->buffer_views[1].data.as_uint8 =
CopyToMisalignedStorage(&storage, offsets, misalignment);
for (int64_t i = 0; i < array_view->length; i++) {
if (type == NANOARROW_TYPE_DENSE_UNION) {
EXPECT_EQ(ArrowArrayViewUnionChildOffset(array_view.get(), i),
offsets32[i + 1]);
} else {
// ArrowArrayViewListChildOffset() does not apply the array offset
EXPECT_EQ(ArrowArrayViewListChildOffset(array_view.get(), i + 1),
offsets32[i + 1]);
}
}
}
}
}

// Helper to build a simple int32 array with known data for shared array tests
static void MakeInt32ArrayForSharedTest(struct ArrowArray* array, const int32_t* values,
int64_t n) {
Expand Down
103 changes: 55 additions & 48 deletions src/nanoarrow/common/inline_array.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <errno.h>
#include <float.h>
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

Expand Down Expand Up @@ -1110,7 +1111,8 @@ static inline int64_t ArrowArrayViewUnionChildOffset(
const struct ArrowArrayView* array_view, int64_t i) {
switch (array_view->storage_type) {
case NANOARROW_TYPE_DENSE_UNION:
return array_view->buffer_views[1].data.as_int32[array_view->offset + i];
return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1],
array_view->offset + i);
case NANOARROW_TYPE_SPARSE_UNION:
return array_view->offset + i;
default:
Expand All @@ -1124,26 +1126,30 @@ static inline int64_t ArrowArrayViewListChildOffset(
case NANOARROW_TYPE_LIST:
case NANOARROW_TYPE_MAP:
case NANOARROW_TYPE_LIST_VIEW:
return array_view->buffer_views[1].data.as_int32[i];
return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1], i);
case NANOARROW_TYPE_LARGE_LIST:
case NANOARROW_TYPE_LARGE_LIST_VIEW:
return array_view->buffer_views[1].data.as_int64[i];
return ArrowBufferViewGetInt64Unsafe(&array_view->buffer_views[1], i);
default:
return -1;
}
}

static struct ArrowBufferView ArrowArrayViewGetBytesFromViewArrayUnsafe(
const struct ArrowArrayView* array_view, int64_t i) {
const union ArrowBinaryView* bv = &array_view->buffer_views[1].data.as_binary_view[i];
struct ArrowBufferView out = {{NULL}, bv->inlined.size};
if (bv->inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {
out.data.as_uint8 = bv->inlined.data;
const uint8_t* bv_data =
array_view->buffer_views[1].data.as_uint8 + i * sizeof(union ArrowBinaryView);
union ArrowBinaryView bv;
memcpy(&bv, bv_data, sizeof(bv));
struct ArrowBufferView out = {{NULL}, bv.inlined.size};
if (bv.inlined.size <= NANOARROW_BINARY_VIEW_INLINE_SIZE) {
// Point to the inlined bytes in the buffer, not in the local copy
out.data.as_uint8 = bv_data + offsetof(struct ArrowBinaryViewInlined, data);
return out;
}

out.data.data = array_view->variadic_buffers[bv->ref.buffer_index];
out.data.as_uint8 += bv->ref.offset;
out.data.data = array_view->variadic_buffers[bv.ref.buffer_index];
out.data.as_uint8 += bv.ref.offset;
return out;
}

Expand All @@ -1153,28 +1159,28 @@ static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* ar
i += array_view->offset;
switch (array_view->storage_type) {
case NANOARROW_TYPE_INT64:
return data_view->data.as_int64[i];
return ArrowBufferViewGetInt64Unsafe(data_view, i);
case NANOARROW_TYPE_UINT64:
return data_view->data.as_uint64[i];
return ArrowBufferViewGetUInt64Unsafe(data_view, i);
case NANOARROW_TYPE_INTERVAL_MONTHS:
case NANOARROW_TYPE_INT32:
return data_view->data.as_int32[i];
return ArrowBufferViewGetInt32Unsafe(data_view, i);
case NANOARROW_TYPE_UINT32:
return data_view->data.as_uint32[i];
return ArrowBufferViewGetUInt32Unsafe(data_view, i);
case NANOARROW_TYPE_INT16:
return data_view->data.as_int16[i];
return ArrowBufferViewGetInt16Unsafe(data_view, i);
case NANOARROW_TYPE_UINT16:
return data_view->data.as_uint16[i];
return ArrowBufferViewGetUInt16Unsafe(data_view, i);
case NANOARROW_TYPE_INT8:
return data_view->data.as_int8[i];
case NANOARROW_TYPE_UINT8:
return data_view->data.as_uint8[i];
case NANOARROW_TYPE_DOUBLE:
return (int64_t)data_view->data.as_double[i];
return (int64_t)ArrowBufferViewGetDoubleUnsafe(data_view, i);
case NANOARROW_TYPE_FLOAT:
return (int64_t)data_view->data.as_float[i];
return (int64_t)ArrowBufferViewGetFloatUnsafe(data_view, i);
case NANOARROW_TYPE_HALF_FLOAT:
return (int64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
return (int64_t)ArrowHalfFloatToFloat(ArrowBufferViewGetUInt16Unsafe(data_view, i));
case NANOARROW_TYPE_BOOL:
return ArrowBitGet(data_view->data.as_uint8, i);
default:
Expand All @@ -1188,28 +1194,29 @@ static inline uint64_t ArrowArrayViewGetUIntUnsafe(
const struct ArrowBufferView* data_view = &array_view->buffer_views[1];
switch (array_view->storage_type) {
case NANOARROW_TYPE_INT64:
return data_view->data.as_int64[i];
return ArrowBufferViewGetInt64Unsafe(data_view, i);
case NANOARROW_TYPE_UINT64:
return data_view->data.as_uint64[i];
return ArrowBufferViewGetUInt64Unsafe(data_view, i);
case NANOARROW_TYPE_INTERVAL_MONTHS:
case NANOARROW_TYPE_INT32:
return data_view->data.as_int32[i];
return ArrowBufferViewGetInt32Unsafe(data_view, i);
case NANOARROW_TYPE_UINT32:
return data_view->data.as_uint32[i];
return ArrowBufferViewGetUInt32Unsafe(data_view, i);
case NANOARROW_TYPE_INT16:
return data_view->data.as_int16[i];
return ArrowBufferViewGetInt16Unsafe(data_view, i);
case NANOARROW_TYPE_UINT16:
return data_view->data.as_uint16[i];
return ArrowBufferViewGetUInt16Unsafe(data_view, i);
case NANOARROW_TYPE_INT8:
return data_view->data.as_int8[i];
case NANOARROW_TYPE_UINT8:
return data_view->data.as_uint8[i];
case NANOARROW_TYPE_DOUBLE:
return (uint64_t)data_view->data.as_double[i];
return (uint64_t)ArrowBufferViewGetDoubleUnsafe(data_view, i);
case NANOARROW_TYPE_FLOAT:
return (uint64_t)data_view->data.as_float[i];
return (uint64_t)ArrowBufferViewGetFloatUnsafe(data_view, i);
case NANOARROW_TYPE_HALF_FLOAT:
return (uint64_t)ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
return (uint64_t)ArrowHalfFloatToFloat(
ArrowBufferViewGetUInt16Unsafe(data_view, i));
case NANOARROW_TYPE_BOOL:
return ArrowBitGet(data_view->data.as_uint8, i);
default:
Expand All @@ -1223,27 +1230,27 @@ static inline double ArrowArrayViewGetDoubleUnsafe(
const struct ArrowBufferView* data_view = &array_view->buffer_views[1];
switch (array_view->storage_type) {
case NANOARROW_TYPE_INT64:
return (double)data_view->data.as_int64[i];
return (double)ArrowBufferViewGetInt64Unsafe(data_view, i);
case NANOARROW_TYPE_UINT64:
return (double)data_view->data.as_uint64[i];
return (double)ArrowBufferViewGetUInt64Unsafe(data_view, i);
case NANOARROW_TYPE_INT32:
return data_view->data.as_int32[i];
return ArrowBufferViewGetInt32Unsafe(data_view, i);
case NANOARROW_TYPE_UINT32:
return data_view->data.as_uint32[i];
return ArrowBufferViewGetUInt32Unsafe(data_view, i);
case NANOARROW_TYPE_INT16:
return data_view->data.as_int16[i];
return ArrowBufferViewGetInt16Unsafe(data_view, i);
case NANOARROW_TYPE_UINT16:
return data_view->data.as_uint16[i];
return ArrowBufferViewGetUInt16Unsafe(data_view, i);
case NANOARROW_TYPE_INT8:
return data_view->data.as_int8[i];
case NANOARROW_TYPE_UINT8:
return data_view->data.as_uint8[i];
case NANOARROW_TYPE_DOUBLE:
return data_view->data.as_double[i];
return ArrowBufferViewGetDoubleUnsafe(data_view, i);
case NANOARROW_TYPE_FLOAT:
return data_view->data.as_float[i];
return ArrowBufferViewGetFloatUnsafe(data_view, i);
case NANOARROW_TYPE_HALF_FLOAT:
return ArrowHalfFloatToFloat(data_view->data.as_uint16[i]);
return ArrowHalfFloatToFloat(ArrowBufferViewGetUInt16Unsafe(data_view, i));
case NANOARROW_TYPE_BOOL:
return ArrowBitGet(data_view->data.as_uint8, i);
default:
Expand All @@ -1261,15 +1268,15 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe(
switch (array_view->storage_type) {
case NANOARROW_TYPE_STRING:
case NANOARROW_TYPE_BINARY:
view.data = data_view + offsets_view->data.as_int32[i];
view.size_bytes =
(int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];
view.data = data_view + ArrowBufferViewGetInt32Unsafe(offsets_view, i);
view.size_bytes = (int64_t)ArrowBufferViewGetInt32Unsafe(offsets_view, i + 1) -
ArrowBufferViewGetInt32Unsafe(offsets_view, i);
break;
case NANOARROW_TYPE_LARGE_STRING:
case NANOARROW_TYPE_LARGE_BINARY:
view.data = data_view + offsets_view->data.as_int64[i];
view.size_bytes =
offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];
view.data = data_view + ArrowBufferViewGetInt64Unsafe(offsets_view, i);
view.size_bytes = ArrowBufferViewGetInt64Unsafe(offsets_view, i + 1) -
ArrowBufferViewGetInt64Unsafe(offsets_view, i);
break;
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
view.size_bytes = array_view->layout.element_size_bits[1] / 8;
Expand Down Expand Up @@ -1302,15 +1309,15 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe(
switch (array_view->storage_type) {
case NANOARROW_TYPE_STRING:
case NANOARROW_TYPE_BINARY:
view.size_bytes =
(int64_t)offsets_view->data.as_int32[i + 1] - offsets_view->data.as_int32[i];
view.data.as_uint8 = data_view + offsets_view->data.as_int32[i];
view.size_bytes = (int64_t)ArrowBufferViewGetInt32Unsafe(offsets_view, i + 1) -
ArrowBufferViewGetInt32Unsafe(offsets_view, i);
view.data.as_uint8 = data_view + ArrowBufferViewGetInt32Unsafe(offsets_view, i);
break;
case NANOARROW_TYPE_LARGE_STRING:
case NANOARROW_TYPE_LARGE_BINARY:
view.size_bytes =
offsets_view->data.as_int64[i + 1] - offsets_view->data.as_int64[i];
view.data.as_uint8 = data_view + offsets_view->data.as_int64[i];
view.size_bytes = ArrowBufferViewGetInt64Unsafe(offsets_view, i + 1) -
ArrowBufferViewGetInt64Unsafe(offsets_view, i);
view.data.as_uint8 = data_view + ArrowBufferViewGetInt64Unsafe(offsets_view, i);
break;
case NANOARROW_TYPE_FIXED_SIZE_BINARY:
view.size_bytes = array_view->layout.element_size_bits[1] / 8;
Expand Down
Loading
Loading