From aa32cc0cc64a78a283db35231618ea20dc753329 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 18 Nov 2026 19:39:01 +0000 Subject: [PATCH] Read deletion vectors whose first bitmaps are empty DeletionVector.to_vector() built its chunked array without a type, so an empty first bitmap (positions only at or above 2**32) or a vector with no bitmaps raised ArrowInvalid, and a vector of only empty bitmaps came back null-typed. Build it as int64. Co-authored-by: Isaac --- pyiceberg/table/deletion_vector.py | 5 ++++- tests/table/test_deletion_vector.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pyiceberg/table/deletion_vector.py b/pyiceberg/table/deletion_vector.py index 18e1578a2c..8b0ef0602f 100644 --- a/pyiceberg/table/deletion_vector.py +++ b/pyiceberg/table/deletion_vector.py @@ -87,7 +87,10 @@ def _deserialize_bitmap(payload: bytes) -> list[BitMap]: def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray": import pyarrow as pa - return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)) + # Use int64 even when the first bitmap is empty or there are no bitmaps. + return pa.chunked_array( + ([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)), type=pa.int64() + ) def to_vector(self) -> "pa.ChunkedArray": return self._bitmaps_to_chunked_array(self._bitmaps) diff --git a/tests/table/test_deletion_vector.py b/tests/table/test_deletion_vector.py index ac6ae98256..8bc294b50c 100644 --- a/tests/table/test_deletion_vector.py +++ b/tests/table/test_deletion_vector.py @@ -19,6 +19,7 @@ from os import path from pathlib import Path +import pyarrow as pa import pytest from pyroaring import BitMap @@ -133,6 +134,23 @@ def test_map_empty() -> None: actual = DeletionVector._deserialize_bitmap(puffin) assert expected == actual + vector = DeletionVector("data.parquet", actual).to_vector() + assert vector.type == pa.int64() + assert vector.to_pylist() == [] + + +def test_to_vector_with_empty_bitmaps_for_low_keys() -> None: + # Keys without positions have empty bitmaps, such as key 0 when every position is at or above 2**32 + for bitmaps, expected in [ + ([BitMap(), BitMap([5])], [2**32 + 5]), + ([BitMap(), BitMap(), BitMap([0, 7])], [2 * 2**32, 2 * 2**32 + 7]), + ([BitMap([2]), BitMap(), BitMap([1])], [2, 2 * 2**32 + 1]), + ([BitMap()], []), + ([BitMap(), BitMap()], []), + ]: + vector = DeletionVector("data.parquet", bitmaps).to_vector() + assert vector.type == pa.int64() + assert vector.to_pylist() == expected def test_map_bitvals() -> None: