Skip to content

Commit 17db412

Browse files
committed
Preserve validated DV field types and keep helper internal
1 parent 7c78db8 commit 17db412

2 files changed

Lines changed: 13 additions & 16 deletions

File tree

‎pyiceberg/table/deletion_vector.py‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import math
1818
import struct
1919
import zlib
20-
from typing import TYPE_CHECKING, cast
20+
from typing import TYPE_CHECKING
2121

2222
from pyroaring import BitMap, FrozenBitMap
2323

@@ -32,7 +32,6 @@
3232
EMPTY_BITMAP = FrozenBitMap()
3333
MAX_JAVA_SIGNED = int(math.pow(2, 31)) - 1
3434
PROPERTY_REFERENCED_DATA_FILE = "referenced-data-file"
35-
_MAX_DELETION_VECTOR_CONTENT_SIZE = 2**31 - 1
3635
_DV_BLOB_LENGTH = struct.Struct(">I")
3736
_DV_BLOB_MAGIC = struct.Struct("<I")
3837
_DV_BLOB_CRC = struct.Struct(">I")
@@ -123,7 +122,7 @@ def _deserialize_dv_blob(blob: bytes, record_count: int | None = None) -> list[B
123122
return bitmaps
124123

125124

126-
def _validate_deletion_vector_content(dv: "DataFile") -> None:
125+
def _validate_deletion_vector_content(dv: "DataFile") -> tuple[int, int, str]:
127126
content_offset = dv.content_offset
128127
content_size_in_bytes = dv.content_size_in_bytes
129128
referenced_data_file = dv.referenced_data_file
@@ -136,13 +135,15 @@ def _validate_deletion_vector_content(dv: "DataFile") -> None:
136135
raise ValueError(f"Invalid deletion vector, content offset cannot be negative: {content_offset}")
137136
if content_size_in_bytes < 0:
138137
raise ValueError(f"Invalid deletion vector, content size cannot be negative: {content_size_in_bytes}")
139-
if content_size_in_bytes > _MAX_DELETION_VECTOR_CONTENT_SIZE:
138+
if content_size_in_bytes > MAX_JAVA_SIGNED:
140139
raise ValueError(f"Cannot read deletion vector larger than 2GB: {content_size_in_bytes}")
141140
if referenced_data_file is None:
142141
raise ValueError(f"Invalid deletion vector, referenced data file is missing: {dv.file_path}")
143142

143+
return content_offset, content_size_in_bytes, referenced_data_file
144144

145-
def has_deletion_vector_content_reference(dv: "DataFile") -> bool:
145+
146+
def _has_deletion_vector_content_reference(dv: "DataFile") -> bool:
146147
"""Return whether any manifest content-reference field is present.
147148
148149
Partial references must be validated as range reads. Whole-Puffin reads
@@ -152,11 +153,7 @@ def has_deletion_vector_content_reference(dv: "DataFile") -> bool:
152153

153154

154155
def _read_deletion_vector(io: "FileIO", dv: "DataFile") -> DeletionVector:
155-
_validate_deletion_vector_content(dv)
156-
157-
content_offset = cast(int, dv.content_offset)
158-
content_size_in_bytes = cast(int, dv.content_size_in_bytes)
159-
referenced_data_file = cast(str, dv.referenced_data_file)
156+
content_offset, content_size_in_bytes, referenced_data_file = _validate_deletion_vector_content(dv)
160157

161158
with io.new_input(dv.file_path).open() as fi:
162159
fi.seek(content_offset)
@@ -173,7 +170,7 @@ def _read_deletion_vector(io: "FileIO", dv: "DataFile") -> DeletionVector:
173170

174171
def read_deletion_vectors(io: "FileIO", dv: "DataFile") -> list[DeletionVector]:
175172
"""Read deletion vectors from a delete file or its manifest content range."""
176-
if has_deletion_vector_content_reference(dv):
173+
if _has_deletion_vector_content_reference(dv):
177174
return [_read_deletion_vector(io, dv)]
178175

179176
with io.new_input(dv.file_path).open() as fi:

‎tests/table/test_deletion_vector.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
_DV_BLOB_MAGIC_NUMBER,
2929
DeletionVector,
3030
_deserialize_dv_blob,
31-
has_deletion_vector_content_reference,
31+
_has_deletion_vector_content_reference,
3232
read_deletion_vectors,
3333
)
3434

@@ -98,10 +98,10 @@ def _data_file(
9898

9999

100100
def test_has_deletion_vector_content_reference() -> None:
101-
assert not has_deletion_vector_content_reference(_data_file())
102-
assert has_deletion_vector_content_reference(_data_file(content_offset=0))
103-
assert has_deletion_vector_content_reference(_data_file(content_size_in_bytes=1))
104-
assert has_deletion_vector_content_reference(_data_file(referenced_data_file="data.parquet"))
101+
assert not _has_deletion_vector_content_reference(_data_file())
102+
assert _has_deletion_vector_content_reference(_data_file(content_offset=0))
103+
assert _has_deletion_vector_content_reference(_data_file(content_size_in_bytes=1))
104+
assert _has_deletion_vector_content_reference(_data_file(referenced_data_file="data.parquet"))
105105

106106

107107
def test_read_external_deletion_vector_content_range() -> None:

0 commit comments

Comments
 (0)