1717import math
1818import struct
1919import zlib
20- from typing import TYPE_CHECKING , cast
20+ from typing import TYPE_CHECKING
2121
2222from pyroaring import BitMap , FrozenBitMap
2323
3232EMPTY_BITMAP = FrozenBitMap ()
3333MAX_JAVA_SIGNED = int (math .pow (2 , 31 )) - 1
3434PROPERTY_REFERENCED_DATA_FILE = "referenced-data-file"
35- _MAX_DELETION_VECTOR_CONTENT_SIZE = 2 ** 31 - 1
3635_DV_BLOB_LENGTH = struct .Struct (">I" )
3736_DV_BLOB_MAGIC = struct .Struct ("<I" )
3837_DV_BLOB_CRC = struct .Struct (">I" )
@@ -123,7 +122,7 @@ def _deserialize_dv_blob(blob: bytes, record_count: int | None = None) -> list[B
123122 return bitmaps
124123
125124
126- def _validate_deletion_vector_content (dv : "DataFile" ) -> None :
125+ def _validate_deletion_vector_content (dv : "DataFile" ) -> tuple [ int , int , str ] :
127126 content_offset = dv .content_offset
128127 content_size_in_bytes = dv .content_size_in_bytes
129128 referenced_data_file = dv .referenced_data_file
@@ -136,13 +135,15 @@ def _validate_deletion_vector_content(dv: "DataFile") -> None:
136135 raise ValueError (f"Invalid deletion vector, content offset cannot be negative: { content_offset } " )
137136 if content_size_in_bytes < 0 :
138137 raise ValueError (f"Invalid deletion vector, content size cannot be negative: { content_size_in_bytes } " )
139- if content_size_in_bytes > _MAX_DELETION_VECTOR_CONTENT_SIZE :
138+ if content_size_in_bytes > MAX_JAVA_SIGNED :
140139 raise ValueError (f"Cannot read deletion vector larger than 2GB: { content_size_in_bytes } " )
141140 if referenced_data_file is None :
142141 raise ValueError (f"Invalid deletion vector, referenced data file is missing: { dv .file_path } " )
143142
143+ return content_offset , content_size_in_bytes , referenced_data_file
144144
145- def has_deletion_vector_content_reference (dv : "DataFile" ) -> bool :
145+
146+ def _has_deletion_vector_content_reference (dv : "DataFile" ) -> bool :
146147 """Return whether any manifest content-reference field is present.
147148
148149 Partial references must be validated as range reads. Whole-Puffin reads
@@ -152,11 +153,7 @@ def has_deletion_vector_content_reference(dv: "DataFile") -> bool:
152153
153154
154155def _read_deletion_vector (io : "FileIO" , dv : "DataFile" ) -> DeletionVector :
155- _validate_deletion_vector_content (dv )
156-
157- content_offset = cast (int , dv .content_offset )
158- content_size_in_bytes = cast (int , dv .content_size_in_bytes )
159- referenced_data_file = cast (str , dv .referenced_data_file )
156+ content_offset , content_size_in_bytes , referenced_data_file = _validate_deletion_vector_content (dv )
160157
161158 with io .new_input (dv .file_path ).open () as fi :
162159 fi .seek (content_offset )
@@ -173,7 +170,7 @@ def _read_deletion_vector(io: "FileIO", dv: "DataFile") -> DeletionVector:
173170
174171def read_deletion_vectors (io : "FileIO" , dv : "DataFile" ) -> list [DeletionVector ]:
175172 """Read deletion vectors from a delete file or its manifest content range."""
176- if has_deletion_vector_content_reference (dv ):
173+ if _has_deletion_vector_content_reference (dv ):
177174 return [_read_deletion_vector (io , dv )]
178175
179176 with io .new_input (dv .file_path ).open () as fi :
0 commit comments