Skip to content

Commit 0a2c3d2

Browse files
ArslanAnjumkevinjqliuCopilot
authored
perf(deletion-vector): avoid payload copies and bitmap reserialization (#4181)
* perf(deletion-vector): use zero-copy memoryview for deserialization Replace repeated byte string slicing with pointer offset tracking via Python's built-in memoryview. This avoids O(N^2) memory buffer allocations during parsing, improving throughput by up to 39x on large payload sizes. * Avoid reserializing deletion vectors while parsing Use PyRoaring's portable serialized size and require memoryview-compatible versions. Cover mixed bitmap encodings and key gaps without reserialization. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Clarify PyRoaring serialized-size comment Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Kevin Liu <kevin.jq.liu@gmail.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 581e716 commit 0a2c3d2

4 files changed

Lines changed: 55 additions & 15 deletions

File tree

‎pyiceberg/table/deletion_vector.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
_DV_BLOB_CRC = struct.Struct(">I")
4040
_DV_BLOB_MAGIC_NUMBER = 1681511377
4141
_ROARING_BITMAP_COUNT_SIZE_BYTES = 8
42+
_ROARING_BITMAP_KEY_SIZE_BYTES = 4
4243
_DV_BLOB_MIN_SIZE_BYTES = _DV_BLOB_LENGTH.size + _DV_BLOB_MAGIC.size + _ROARING_BITMAP_COUNT_SIZE_BYTES + _DV_BLOB_CRC.size
4344

4445

@@ -51,30 +52,32 @@ def __init__(self, referenced_data_file: str, bitmaps: list[BitMap]) -> None:
5152
self._bitmaps = bitmaps
5253

5354
@staticmethod
54-
def _deserialize_bitmap(pl: bytes) -> list[BitMap]:
55-
number_of_bitmaps = int.from_bytes(pl[0:8], byteorder="little")
56-
pl = pl[8:]
55+
def _deserialize_bitmap(payload: bytes) -> list[BitMap]:
56+
payload_view = memoryview(payload)
57+
offset = _ROARING_BITMAP_COUNT_SIZE_BYTES
58+
number_of_bitmaps = int.from_bytes(payload_view[:offset], byteorder="little")
5759

5860
bitmaps = []
5961
last_key = -1
62+
6063
for _ in range(number_of_bitmaps):
61-
key = int.from_bytes(pl[0:4], byteorder="little")
64+
key = int.from_bytes(payload_view[offset : offset + _ROARING_BITMAP_KEY_SIZE_BYTES], byteorder="little")
6265
if key < 0:
6366
raise ValueError(f"Invalid unsigned key: {key}")
6467
if key <= last_key:
6568
raise ValueError("Keys must be sorted in ascending order")
6669
if key > MAX_JAVA_SIGNED:
6770
raise ValueError(f"Key {key} is too large, max {MAX_JAVA_SIGNED} to maintain compatibility with Java impl")
68-
pl = pl[4:]
71+
offset += _ROARING_BITMAP_KEY_SIZE_BYTES
6972

7073
while last_key < key - 1:
7174
bitmaps.append(EMPTY_BITMAP)
7275
last_key += 1
7376

74-
bm = BitMap().deserialize(pl)
75-
# TODO: Optimize this
76-
pl = pl[len(bm.serialize()) :]
77-
bitmaps.append(bm)
77+
bitmap = BitMap.deserialize(payload_view[offset:])
78+
# PyRoaring's __sizeof__ returns the portable serialized size.
79+
offset += bitmap.__sizeof__()
80+
bitmaps.append(bitmap)
7881

7982
last_key = key
8083

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ dependencies = [
4040
"fsspec>=2023.1.0",
4141
"pyparsing>=3.1.0",
4242
"tenacity>=8.2.3",
43-
"pyroaring>=1.0.0",
43+
"pyroaring>=1.1.0",
4444
"cachetools>=5.5",
4545
"zstandard>=0.13.0"
4646
]

‎tests/table/test_deletion_vector.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,43 @@ def test_map_spread_vals() -> None:
164164
assert expected == actual
165165

166166

167+
def test_map_mixed_containers_with_key_gaps() -> None:
168+
array_bitmap = BitMap([1, 3, 5])
169+
bitset_bitmap = BitMap(range(0, 20_000, 2))
170+
run_bitmap = BitMap(range(100_000))
171+
multi_container_bitmap = BitMap([1, 65_537])
172+
bitmaps_by_key = {
173+
0: BitMap(),
174+
2: array_bitmap,
175+
3: bitset_bitmap,
176+
5: run_bitmap,
177+
6: multi_container_bitmap,
178+
}
179+
payload = len(bitmaps_by_key).to_bytes(8, byteorder="little") + b"".join(
180+
key.to_bytes(4, byteorder="little") + bitmap.serialize() for key, bitmap in bitmaps_by_key.items()
181+
)
182+
183+
assert DeletionVector._deserialize_bitmap(payload) == [
184+
BitMap(),
185+
BitMap(),
186+
array_bitmap,
187+
bitset_bitmap,
188+
BitMap(),
189+
run_bitmap,
190+
multi_container_bitmap,
191+
]
192+
193+
194+
def test_map_deserialization_does_not_reserialize(monkeypatch: pytest.MonkeyPatch) -> None:
195+
class NonSerializingBitMap(BitMap):
196+
def serialize(self) -> bytes:
197+
raise AssertionError("Deletion vector deserialization must not serialize bitmaps")
198+
199+
monkeypatch.setattr("pyiceberg.table.deletion_vector.BitMap", NonSerializingBitMap)
200+
201+
assert DeletionVector._deserialize_bitmap(_bitmap_payload()) == [BitMap([1, 3, 5])]
202+
203+
167204
def test_map_high_vals() -> None:
168205
puffin = _open_file("64maphighvals.bin")
169206

‎uv.lock‎

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)