Skip to content

Commit aa32cc0

Browse files
rahulsmahadevIsaac
andcommitted
Read deletion vectors whose first bitmaps are empty
DeletionVector.to_vector() built its chunked array without a type, so an empty first bitmap (positions only at or above 2**32) or a vector with no bitmaps raised ArrowInvalid, and a vector of only empty bitmaps came back null-typed. Build it as int64. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 9955502 commit aa32cc0

2 files changed

Lines changed: 22 additions & 1 deletion

File tree

‎pyiceberg/table/deletion_vector.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,10 @@ def _deserialize_bitmap(payload: bytes) -> list[BitMap]:
8787
def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray":
8888
import pyarrow as pa
8989

90-
return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps))
90+
# Use int64 even when the first bitmap is empty or there are no bitmaps.
91+
return pa.chunked_array(
92+
([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)), type=pa.int64()
93+
)
9194

9295
def to_vector(self) -> "pa.ChunkedArray":
9396
return self._bitmaps_to_chunked_array(self._bitmaps)

‎tests/table/test_deletion_vector.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from os import path
2020
from pathlib import Path
2121

22+
import pyarrow as pa
2223
import pytest
2324
from pyroaring import BitMap
2425

@@ -133,6 +134,23 @@ def test_map_empty() -> None:
133134
actual = DeletionVector._deserialize_bitmap(puffin)
134135

135136
assert expected == actual
137+
vector = DeletionVector("data.parquet", actual).to_vector()
138+
assert vector.type == pa.int64()
139+
assert vector.to_pylist() == []
140+
141+
142+
def test_to_vector_with_empty_bitmaps_for_low_keys() -> None:
143+
# Keys without positions have empty bitmaps, such as key 0 when every position is at or above 2**32
144+
for bitmaps, expected in [
145+
([BitMap(), BitMap([5])], [2**32 + 5]),
146+
([BitMap(), BitMap(), BitMap([0, 7])], [2 * 2**32, 2 * 2**32 + 7]),
147+
([BitMap([2]), BitMap(), BitMap([1])], [2, 2 * 2**32 + 1]),
148+
([BitMap()], []),
149+
([BitMap(), BitMap()], []),
150+
]:
151+
vector = DeletionVector("data.parquet", bitmaps).to_vector()
152+
assert vector.type == pa.int64()
153+
assert vector.to_pylist() == expected
136154

137155

138156
def test_map_bitvals() -> None:

0 commit comments

Comments
 (0)