157157from pyiceberg .table .deletion_vector import read_deletion_vectors
158158from pyiceberg .table .locations import load_location_provider
159159from pyiceberg .table .metadata import TableMetadata
160+ from pyiceberg .table .metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER , ROW_ID
160161from pyiceberg .table .name_mapping import NameMapping , apply_name_mapping
161162from pyiceberg .transforms import IdentityTransform , TruncateTransform
162163from pyiceberg .typedef import EMPTY_DICT , Properties , Record , TableVersion
214215LIST_ELEMENT_NAME = "element"
215216MAP_KEY_NAME = "key"
216217MAP_VALUE_NAME = "value"
218+ # Temporary column that tracks the position of each row in its file while rows are filtered
219+ _POSITION_COLUMN = "__iceberg_file_position"
217220DOC = "doc"
218221UTC_ALIASES = {"UTC" , "+00:00" , "Etc/UTC" , "Z" }
219222ADLS_SCHEMES = frozenset ({"abfs" , "abfss" , "wasb" , "wasbs" })
@@ -1793,12 +1796,18 @@ def _task_to_record_batches(
17931796
17941797 file_project_schema = prune_columns (file_schema , projected_field_ids , select_full_types = False )
17951798
1799+ # Row IDs are derived from the position of each row in the file, so rows must not be filtered before they are counted
1800+ row_id_index = next (
1801+ (index for index , field in enumerate (projected_schema .fields ) if field .field_id == ROW_ID .field_id ), None
1802+ )
1803+ filter_after_read = bool (positional_deletes ) or row_id_index is not None
1804+
17961805 fragment_scanner = ds .Scanner .from_fragment (
17971806 fragment = fragment ,
17981807 schema = physical_schema ,
17991808 # This will push down the query to Arrow.
1800- # But in case there are positional deletes, we have to apply them first
1801- filter = pyarrow_filter if not positional_deletes else None ,
1809+ # But in case there are positional deletes or row IDs, rows must be counted before filtering
1810+ filter = pyarrow_filter if not filter_after_read else None ,
18021811 columns = [col .name for col in file_project_schema .columns ],
18031812 )
18041813
@@ -1808,34 +1817,58 @@ def _task_to_record_batches(
18081817 next_index = next_index + len (batch )
18091818 current_index = next_index - len (batch )
18101819 current_batch = batch
1820+ positions = pa .array (range (current_index , next_index ), type = pa .int64 ()) if row_id_index is not None else None
18111821
18121822 if positional_deletes :
18131823 # Create the mask of indices that we're interested in
18141824 indices = _combine_positional_deletes (positional_deletes , current_index , current_index + len (batch ))
18151825 current_batch = current_batch .take (indices )
1816- if pyarrow_filter is not None :
1817- # Temporary fix until PyArrow 21 is the minimum supported version
1818- # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
1819- # IndexError on PyArrow <21 when the result is empty; Table.filter does not.
1820- table = pa .Table .from_batches ([current_batch ])
1821- table = table .filter (pyarrow_filter )
1822- if table .num_rows == 0 :
1823- current_batch = current_batch .slice (0 , 0 )
1824- else :
1825- current_batch = table .combine_chunks ().to_batches ()[0 ]
1826+ if positions is not None :
1827+ positions = positions .take (indices )
1828+
1829+ if filter_after_read and pyarrow_filter is not None :
1830+ # Temporary fix until PyArrow 21 is the minimum supported version
1831+ # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
1832+ # IndexError on PyArrow <21 when the result is empty; Table.filter does not.
1833+ table = pa .Table .from_batches ([current_batch ])
1834+ if positions is not None :
1835+ table = table .append_column (_POSITION_COLUMN , positions )
1836+ table = table .filter (pyarrow_filter )
1837+ if positions is not None :
1838+ positions = table .column (_POSITION_COLUMN ).combine_chunks ()
1839+ table = table .drop_columns ([_POSITION_COLUMN ])
1840+ if table .num_rows == 0 :
1841+ current_batch = current_batch .slice (0 , 0 )
1842+ else :
1843+ current_batch = table .combine_chunks ().to_batches ()[0 ]
18261844
18271845 # skip empty batches
18281846 if current_batch .num_rows == 0 :
18291847 continue
18301848
1831- yield _to_requested_schema (
1849+ result_batch = _to_requested_schema (
18321850 projected_schema ,
18331851 file_project_schema ,
18341852 current_batch ,
18351853 downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us ,
18361854 projected_missing_fields = projected_missing_fields ,
18371855 allow_timestamp_tz_mismatch = True ,
18381856 )
1857+ if row_id_index is not None and positions is not None :
1858+ result_batch = _assign_row_ids (result_batch , row_id_index , positions , task .file .first_row_id )
1859+ yield result_batch
1860+
1861+
1862+ def _assign_row_ids (batch : pa .RecordBatch , row_id_index : int , positions : pa .Array , first_row_id : int | None ) -> pa .RecordBatch :
1863+ """Replace null row IDs with the file's first row ID plus the row position.
1864+
1865+ When the data file has no first row ID, every row ID is null.
1866+ """
1867+ if first_row_id is None :
1868+ row_ids = pa .nulls (len (batch ), type = pa .int64 ())
1869+ else :
1870+ row_ids = pc .coalesce (batch .column (row_id_index ), pc .add (positions , pa .scalar (first_row_id , type = pa .int64 ())))
1871+ return batch .set_column (row_id_index , batch .schema .field (row_id_index ), row_ids )
18391872
18401873
18411874@dataclass (frozen = True , slots = True )
@@ -1904,6 +1937,9 @@ def __init__(
19041937 * ,
19051938 dictionary_columns : tuple [str , ...] = (),
19061939 ) -> None :
1940+ if any (field .field_id == LAST_UPDATED_SEQUENCE_NUMBER .field_id for field in projected_schema .fields ):
1941+ # The data sequence number is not carried on the scan task yet
1942+ raise NotImplementedError (f"Reading { LAST_UPDATED_SEQUENCE_NUMBER .name } is not yet supported" )
19071943 self ._table_metadata = table_metadata
19081944 self ._io = io
19091945 self ._projected_schema = projected_schema
0 commit comments