diff --git a/python/src/nanoarrow/_buffer.pyx b/python/src/nanoarrow/_buffer.pyx index 4f427e4eb..ca501dc81 100644 --- a/python/src/nanoarrow/_buffer.pyx +++ b/python/src/nanoarrow/_buffer.pyx @@ -351,7 +351,9 @@ cdef class CBufferView: return iter(memoryview(self)[offset:(offset + length)]) def _iter_struct(self, int64_t offset, int64_t length): - for value in iter_unpack(self.format, self): + view = memoryview(self).cast("B") + view = view[offset * self.itemsize:(offset + length) * self.itemsize] + for value in iter_unpack(self.format, view): if len(value) == 1: yield value[0] else: diff --git a/python/tests/test_c_buffer.py b/python/tests/test_c_buffer.py index 41db69b8f..c12c3161c 100644 --- a/python/tests/test_c_buffer.py +++ b/python/tests/test_c_buffer.py @@ -197,6 +197,41 @@ def test_c_buffer_fixed_size_binary(): assert list(buffer) == items +@pytest.mark.parametrize( + "schema,items", + [ + (na.float16(), [1.0, 2.0, 3.0]), + (na.fixed_size_binary(2), [b"aa", b"bb", b"cc"]), + (na.interval_day_time(), [(1, 2), (3, 4), (5, 6)]), + (na.interval_month_day_nano(), [(1, 2, 3), (4, 5, 6), (7, 8, 9)]), + (na.decimal128(10, 0), [bytes([i]) * 16 for i in range(3)]), + (na.decimal256(10, 0), [bytes([i]) * 32 for i in range(3)]), + ], +) +def test_c_buffer_struct_elements_slice(schema, items): + view = na.c_buffer(items, schema).view() + assert memoryview(view).readonly + assert list(view) == items + assert list(view.elements()) == items + assert list(na.c_buffer([], schema).view().elements()) == [] + + for offset in range(len(items) + 1): + for length in range(len(items) - offset + 1): + assert ( + list(view.elements(offset, length)) == items[offset : offset + length] + ) + + for offset, length in [(-1, 1), (0, -1), (len(items), 1)]: + with pytest.raises(IndexError, match="do not describe a valid slice"): + view.elements(offset, length) + + # The iterator must keep the underlying buffer alive after the view is deleted. + iterator = view.elements(1, 2) + assert next(iterator) == items[1] + del view + assert list(iterator) == items[2:] + + def test_c_buffer_builder(): builder = CBufferBuilder() assert builder.size_bytes == 0 diff --git a/python/tests/test_iterator.py b/python/tests/test_iterator.py index b07a585d2..11ee7904f 100644 --- a/python/tests/test_iterator.py +++ b/python/tests/test_iterator.py @@ -116,6 +116,34 @@ def test_iterator_nullable_binary(arrow_type): assert list(iter_py(sliced)) == [b"cde", None] +@pytest.mark.parametrize( + "type_name,type_args,items", + [ + ("binary", (2,), [b"aa", b"bb", b"cc", b"dd"]), + ("float16", (), [1.0, 2.0, 3.0, 4.0]), + ("decimal128", (10, 2), [decimal.Decimal(i) for i in range(1, 5)]), + ("decimal256", (10, 2), [decimal.Decimal(i) for i in range(1, 5)]), + ("month_day_nano_interval", (), [(i, i + 1, i + 2) for i in range(4)]), + ], +) +@pytest.mark.parametrize("nullable", [False, True]) +def test_iterator_struct_buffer_slice(type_name, type_args, items, nullable): + pa = pytest.importorskip("pyarrow") + items = items.copy() + if nullable: + items[2] = None + array = pa.array(items, getattr(pa, type_name)(*type_args)) + batch = pa.record_batch([array], names=["value"]) + + for offset, length in [(0, 1), (1, 1), (1, 3), (4, 0)]: + sliced = array.slice(offset, length) + assert list(iter_py(sliced)) == sliced.to_pylist() + assert na.Array(sliced).to_pylist() == sliced.to_pylist() + + sliced_batch = batch.slice(offset, length) + assert na.Array(sliced_batch).to_pylist() == sliced_batch.to_pylist() + + def test_iter_tuples(): array = na.c_array_from_buffers( na.struct({"col1": na.int32(), "col2": na.bool_()}),