Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions pyiceberg/table/deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
_DV_BLOB_CRC = struct.Struct(">I")
_DV_BLOB_MAGIC_NUMBER = 1681511377
_ROARING_BITMAP_COUNT_SIZE_BYTES = 8
_ROARING_BITMAP_KEY_SIZE_BYTES = 4
_DV_BLOB_MIN_SIZE_BYTES = _DV_BLOB_LENGTH.size + _DV_BLOB_MAGIC.size + _ROARING_BITMAP_COUNT_SIZE_BYTES + _DV_BLOB_CRC.size


Expand All @@ -49,30 +50,32 @@ def __init__(self, referenced_data_file: str, bitmaps: list[BitMap]) -> None:
self._bitmaps = bitmaps

@staticmethod
def _deserialize_bitmap(pl: bytes) -> list[BitMap]:
number_of_bitmaps = int.from_bytes(pl[0:8], byteorder="little")
pl = pl[8:]
def _deserialize_bitmap(payload: bytes) -> list[BitMap]:
payload_view = memoryview(payload)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using memoryview is the main optimization here: it lets us advance through the payload without repeatedly copying the remaining bytes.

offset = _ROARING_BITMAP_COUNT_SIZE_BYTES
number_of_bitmaps = int.from_bytes(payload_view[:offset], byteorder="little")

bitmaps = []
last_key = -1

for _ in range(number_of_bitmaps):
key = int.from_bytes(pl[0:4], byteorder="little")
key = int.from_bytes(payload_view[offset : offset + _ROARING_BITMAP_KEY_SIZE_BYTES], byteorder="little")
if key < 0:
raise ValueError(f"Invalid unsigned key: {key}")
if key <= last_key:
raise ValueError("Keys must be sorted in ascending order")
if key > MAX_JAVA_SIGNED:
raise ValueError(f"Key {key} is too large, max {MAX_JAVA_SIGNED} to maintain compatibility with Java impl")
pl = pl[4:]
offset += _ROARING_BITMAP_KEY_SIZE_BYTES

while last_key < key - 1:
bitmaps.append(EMPTY_BITMAP)
last_key += 1

bm = BitMap().deserialize(pl)
# TODO: Optimize this
pl = pl[len(bm.serialize()) :]
bitmaps.append(bm)
bitmap = BitMap.deserialize(payload_view[offset:])
# PyRoaring's __sizeof__ returns the portable serialized size.
offset += bitmap.__sizeof__()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

another optimization: __sizeof__() uses the same size calculation as serialize(), without allocating or serializing bytes.

bitmaps.append(bitmap)

last_key = key

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ dependencies = [
"fsspec>=2023.1.0",
"pyparsing>=3.1.0",
"tenacity>=8.2.3",
"pyroaring>=1.0.0",
"pyroaring>=1.1.0",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Requires PyRoaring ≥1.1.0 for memoryview support; see the updated deserialize() signature.

"cachetools>=5.5",
"zstandard>=0.13.0"
]
Expand Down
37 changes: 37 additions & 0 deletions tests/table/test_deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,43 @@ def test_map_spread_vals() -> None:
assert expected == actual


def test_map_mixed_containers_with_key_gaps() -> None:
array_bitmap = BitMap([1, 3, 5])
bitset_bitmap = BitMap(range(0, 20_000, 2))
run_bitmap = BitMap(range(100_000))
multi_container_bitmap = BitMap([1, 65_537])
bitmaps_by_key = {
0: BitMap(),
2: array_bitmap,
3: bitset_bitmap,
5: run_bitmap,
6: multi_container_bitmap,
}
payload = len(bitmaps_by_key).to_bytes(8, byteorder="little") + b"".join(
key.to_bytes(4, byteorder="little") + bitmap.serialize() for key, bitmap in bitmaps_by_key.items()
)

assert DeletionVector._deserialize_bitmap(payload) == [
BitMap(),
BitMap(),
array_bitmap,
bitset_bitmap,
BitMap(),
run_bitmap,
multi_container_bitmap,
]


def test_map_deserialization_does_not_reserialize(monkeypatch: pytest.MonkeyPatch) -> None:
class NonSerializingBitMap(BitMap):
def serialize(self) -> bytes:
raise AssertionError("Deletion vector deserialization must not serialize bitmaps")

monkeypatch.setattr("pyiceberg.table.deletion_vector.BitMap", NonSerializingBitMap)

assert DeletionVector._deserialize_bitmap(_bitmap_payload()) == [BitMap([1, 3, 5])]


def test_map_high_vals() -> None:
puffin = _open_file("64maphighvals.bin")

Expand Down
10 changes: 5 additions & 5 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading