Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyiceberg/table/deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@ def _deserialize_bitmap(payload: bytes) -> list[BitMap]:
def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray":
import pyarrow as pa

return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps))
# Use int64 even when the first bitmap is empty or there are no bitmaps.
return pa.chunked_array(
([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)), type=pa.int64()
)

def to_vector(self) -> "pa.ChunkedArray":
return self._bitmaps_to_chunked_array(self._bitmaps)
Expand Down
18 changes: 18 additions & 0 deletions tests/table/test_deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from os import path
from pathlib import Path

import pyarrow as pa
import pytest
from pyroaring import BitMap

Expand Down Expand Up @@ -133,6 +134,23 @@ def test_map_empty() -> None:
actual = DeletionVector._deserialize_bitmap(puffin)

assert expected == actual
vector = DeletionVector("data.parquet", actual).to_vector()
assert vector.type == pa.int64()
assert vector.to_pylist() == []


def test_to_vector_with_empty_bitmaps_for_low_keys() -> None:
# Keys without positions have empty bitmaps, such as key 0 when every position is at or above 2**32
for bitmaps, expected in [
([BitMap(), BitMap([5])], [2**32 + 5]),
([BitMap(), BitMap(), BitMap([0, 7])], [2 * 2**32, 2 * 2**32 + 7]),
([BitMap([2]), BitMap(), BitMap([1])], [2, 2 * 2**32 + 1]),
([BitMap()], []),
([BitMap(), BitMap()], []),
Comment on lines +145 to +149

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If one case fails, the output shows only the assert on raw BitMaps and large integers, so it's hard to tell which scenario broke. Not blocking, since the project prefers a for loop over parametrize for many simple cases.

]:
vector = DeletionVector("data.parquet", bitmaps).to_vector()
assert vector.type == pa.int64()
assert vector.to_pylist() == expected


def test_map_bitvals() -> None:
Expand Down
Loading