diff --git a/pyiceberg/table/deletion_vector.py b/pyiceberg/table/deletion_vector.py index 18e1578a2c..8b0ef0602f 100644 --- a/pyiceberg/table/deletion_vector.py +++ b/pyiceberg/table/deletion_vector.py @@ -87,7 +87,10 @@ def _deserialize_bitmap(payload: bytes) -> list[BitMap]: def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": import pyarrow as pa - return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)) + # Use int64 even when the first bitmap is empty or there are no bitmaps. + return pa.chunked_array( + ([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)), type=pa.int64() + ) def to_vector(self) -> "pa.ChunkedArray": return self._bitmaps_to_chunked_array(self._bitmaps) diff --git a/tests/table/test_deletion_vector.py b/tests/table/test_deletion_vector.py index ac6ae98256..8bc294b50c 100644 --- a/tests/table/test_deletion_vector.py +++ b/tests/table/test_deletion_vector.py @@ -19,6 +19,7 @@ from os import path from pathlib import Path +import pyarrow as pa import pytest from pyroaring import BitMap @@ -133,6 +134,23 @@ def test_map_empty() -> None: actual = DeletionVector._deserialize_bitmap(puffin) assert expected == actual + vector = DeletionVector("data.parquet", actual).to_vector() + assert vector.type == pa.int64() + assert vector.to_pylist() == [] + + +def test_to_vector_with_empty_bitmaps_for_low_keys() -> None: + # Keys without positions have empty bitmaps, such as key 0 when every position is at or above 2**32 + for bitmaps, expected in [ + ([BitMap(), BitMap([5])], [2**32 + 5]), + ([BitMap(), BitMap(), BitMap([0, 7])], [2 * 2**32, 2 * 2**32 + 7]), + ([BitMap([2]), BitMap(), BitMap([1])], [2, 2 * 2**32 + 1]), + ([BitMap()], []), + ([BitMap(), BitMap()], []), + ]: + vector = DeletionVector("data.parquet", bitmaps).to_vector() + assert vector.type == pa.int64() + assert vector.to_pylist() == expected def test_map_bitvals() -> None: