Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion python/src/nanoarrow/_buffer.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,9 @@ cdef class CBufferView:
return iter(memoryview(self)[offset:(offset + length)])

def _iter_struct(self, int64_t offset, int64_t length):
for value in iter_unpack(self.format, self):
view = memoryview(self).cast("B")
view = view[offset * self.itemsize:(offset + length) * self.itemsize]
Comment on lines +354 to +355

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can just do this?

Suggested change
view = memoryview(self).cast("B")
view = view[offset * self.itemsize:(offset + length) * self.itemsize]
view = memoryview(self)[offset:(offset + length)]

for value in iter_unpack(self.format, view):
if len(value) == 1:
yield value[0]
else:
Expand Down
35 changes: 35 additions & 0 deletions python/tests/test_c_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,41 @@ def test_c_buffer_fixed_size_binary():
assert list(buffer) == items


@pytest.mark.parametrize(
"schema,items",
[
(na.float16(), [1.0, 2.0, 3.0]),
(na.fixed_size_binary(2), [b"aa", b"bb", b"cc"]),
(na.interval_day_time(), [(1, 2), (3, 4), (5, 6)]),
(na.interval_month_day_nano(), [(1, 2, 3), (4, 5, 6), (7, 8, 9)]),
(na.decimal128(10, 0), [bytes([i]) * 16 for i in range(3)]),
(na.decimal256(10, 0), [bytes([i]) * 32 for i in range(3)]),
],
)
def test_c_buffer_struct_elements_slice(schema, items):
view = na.c_buffer(items, schema).view()
assert memoryview(view).readonly
assert list(view) == items
assert list(view.elements()) == items
assert list(na.c_buffer([], schema).view().elements()) == []

for offset in range(len(items) + 1):
for length in range(len(items) - offset + 1):
assert (
list(view.elements(offset, length)) == items[offset : offset + length]
)

for offset, length in [(-1, 1), (0, -1), (len(items), 1)]:
with pytest.raises(IndexError, match="do not describe a valid slice"):
view.elements(offset, length)

# The iterator must keep the underlying buffer alive after the view is deleted.
iterator = view.elements(1, 2)
assert next(iterator) == items[1]
del view
assert list(iterator) == items[2:]


def test_c_buffer_builder():
builder = CBufferBuilder()
assert builder.size_bytes == 0
Expand Down
28 changes: 28 additions & 0 deletions python/tests/test_iterator.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,34 @@ def test_iterator_nullable_binary(arrow_type):
assert list(iter_py(sliced)) == [b"cde", None]


@pytest.mark.parametrize(
"type_name,type_args,items",
[
("binary", (2,), [b"aa", b"bb", b"cc", b"dd"]),
("float16", (), [1.0, 2.0, 3.0, 4.0]),
("decimal128", (10, 2), [decimal.Decimal(i) for i in range(1, 5)]),
("decimal256", (10, 2), [decimal.Decimal(i) for i in range(1, 5)]),
("month_day_nano_interval", (), [(i, i + 1, i + 2) for i in range(4)]),
],
)
@pytest.mark.parametrize("nullable", [False, True])
def test_iterator_struct_buffer_slice(type_name, type_args, items, nullable):
pa = pytest.importorskip("pyarrow")
items = items.copy()
if nullable:
items[2] = None
array = pa.array(items, getattr(pa, type_name)(*type_args))
batch = pa.record_batch([array], names=["value"])

for offset, length in [(0, 1), (1, 1), (1, 3), (4, 0)]:
sliced = array.slice(offset, length)
assert list(iter_py(sliced)) == sliced.to_pylist()
assert na.Array(sliced).to_pylist() == sliced.to_pylist()

sliced_batch = batch.slice(offset, length)
assert na.Array(sliced_batch).to_pylist() == sliced_batch.to_pylist()


def test_iter_tuples():
array = na.c_array_from_buffers(
na.struct({"col1": na.int32(), "col2": na.bool_()}),
Expand Down
Loading