Skip to content

Commit fce8185

Browse files
authored
Row Lineage: Add metadata column support for _row_id (#4375)
Add metadata column support
1 parent 5436520 commit fce8185

6 files changed

Lines changed: 316 additions & 22 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
from pyiceberg.table.deletion_vector import read_deletion_vectors
158158
from pyiceberg.table.locations import load_location_provider
159159
from pyiceberg.table.metadata import TableMetadata
160+
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
160161
from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping
161162
from pyiceberg.transforms import IdentityTransform, TruncateTransform
162163
from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion
@@ -214,6 +215,8 @@
214215
LIST_ELEMENT_NAME = "element"
215216
MAP_KEY_NAME = "key"
216217
MAP_VALUE_NAME = "value"
218+
# Temporary column that tracks the position of each row in its file while rows are filtered
219+
_POSITION_COLUMN = "__iceberg_file_position"
217220
DOC = "doc"
218221
UTC_ALIASES = {"UTC", "+00:00", "Etc/UTC", "Z"}
219222
ADLS_SCHEMES = frozenset({"abfs", "abfss", "wasb", "wasbs"})
@@ -1793,12 +1796,18 @@ def _task_to_record_batches(
17931796

17941797
file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
17951798

1799+
# Row IDs are derived from the position of each row in the file, so rows must not be filtered before they are counted
1800+
row_id_index = next(
1801+
(index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None
1802+
)
1803+
filter_after_read = bool(positional_deletes) or row_id_index is not None
1804+
17961805
fragment_scanner = ds.Scanner.from_fragment(
17971806
fragment=fragment,
17981807
schema=physical_schema,
17991808
# This will push down the query to Arrow.
1800-
# But in case there are positional deletes, we have to apply them first
1801-
filter=pyarrow_filter if not positional_deletes else None,
1809+
# But in case there are positional deletes or row IDs, rows must be counted before filtering
1810+
filter=pyarrow_filter if not filter_after_read else None,
18021811
columns=[col.name for col in file_project_schema.columns],
18031812
)
18041813

@@ -1808,34 +1817,58 @@ def _task_to_record_batches(
18081817
next_index = next_index + len(batch)
18091818
current_index = next_index - len(batch)
18101819
current_batch = batch
1820+
positions = pa.array(range(current_index, next_index), type=pa.int64()) if row_id_index is not None else None
18111821

18121822
if positional_deletes:
18131823
# Create the mask of indices that we're interested in
18141824
indices = _combine_positional_deletes(positional_deletes, current_index, current_index + len(batch))
18151825
current_batch = current_batch.take(indices)
1816-
if pyarrow_filter is not None:
1817-
# Temporary fix until PyArrow 21 is the minimum supported version
1818-
# (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
1819-
# IndexError on PyArrow <21 when the result is empty; Table.filter does not.
1820-
table = pa.Table.from_batches([current_batch])
1821-
table = table.filter(pyarrow_filter)
1822-
if table.num_rows == 0:
1823-
current_batch = current_batch.slice(0, 0)
1824-
else:
1825-
current_batch = table.combine_chunks().to_batches()[0]
1826+
if positions is not None:
1827+
positions = positions.take(indices)
1828+
1829+
if filter_after_read and pyarrow_filter is not None:
1830+
# Temporary fix until PyArrow 21 is the minimum supported version
1831+
# (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
1832+
# IndexError on PyArrow <21 when the result is empty; Table.filter does not.
1833+
table = pa.Table.from_batches([current_batch])
1834+
if positions is not None:
1835+
table = table.append_column(_POSITION_COLUMN, positions)
1836+
table = table.filter(pyarrow_filter)
1837+
if positions is not None:
1838+
positions = table.column(_POSITION_COLUMN).combine_chunks()
1839+
table = table.drop_columns([_POSITION_COLUMN])
1840+
if table.num_rows == 0:
1841+
current_batch = current_batch.slice(0, 0)
1842+
else:
1843+
current_batch = table.combine_chunks().to_batches()[0]
18261844

18271845
# skip empty batches
18281846
if current_batch.num_rows == 0:
18291847
continue
18301848

1831-
yield _to_requested_schema(
1849+
result_batch = _to_requested_schema(
18321850
projected_schema,
18331851
file_project_schema,
18341852
current_batch,
18351853
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
18361854
projected_missing_fields=projected_missing_fields,
18371855
allow_timestamp_tz_mismatch=True,
18381856
)
1857+
if row_id_index is not None and positions is not None:
1858+
result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id)
1859+
yield result_batch
1860+
1861+
1862+
def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Array, first_row_id: int | None) -> pa.RecordBatch:
1863+
"""Replace null row IDs with the file's first row ID plus the row position.
1864+
1865+
When the data file has no first row ID, every row ID is null.
1866+
"""
1867+
if first_row_id is None:
1868+
row_ids = pa.nulls(len(batch), type=pa.int64())
1869+
else:
1870+
row_ids = pc.coalesce(batch.column(row_id_index), pc.add(positions, pa.scalar(first_row_id, type=pa.int64())))
1871+
return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids)
18391872

18401873

18411874
@dataclass(frozen=True, slots=True)
@@ -1904,6 +1937,9 @@ def __init__(
19041937
*,
19051938
dictionary_columns: tuple[str, ...] = (),
19061939
) -> None:
1940+
if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields):
1941+
# The data sequence number is not carried on the scan task yet
1942+
raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported")
19071943
self._table_metadata = table_metadata
19081944
self._io = io
19091945
self._projected_schema = projected_schema

‎pyiceberg/table/__init__.py‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
from pyiceberg.table.locations import LocationProvider, load_location_provider
6565
from pyiceberg.table.maintenance import MaintenanceTable
6666
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
67+
from pyiceberg.table.metadata_columns import project_schema, split_selected_fields
6768
from pyiceberg.table.name_mapping import NameMapping
6869
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
6970
from pyiceberg.table.snapshots import (
@@ -2302,10 +2303,7 @@ def projection(self) -> Schema:
23022303
else:
23032304
raise ValueError(f"Snapshot not found: {self.snapshot_id}")
23042305

2305-
if "*" in self.selected_fields:
2306-
return current_schema
2307-
2308-
return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)
2306+
return project_schema(current_schema, self.selected_fields, self.case_sensitive, self.table_metadata.format_version)
23092307

23102308
def use_ref(self: S, name: str) -> S:
23112309
if self.snapshot_id is not None:
@@ -2550,9 +2548,11 @@ def _plan_files_server_side(self) -> Iterable[FileScanTask]:
25502548
if self.table_identifier is None:
25512549
raise ValueError("REST scan planning requires a table identifier")
25522550

2551+
# Metadata columns are not in the table schema, so the server cannot resolve them
2552+
schema_fields, _ = split_selected_fields(self.table_metadata.schema(), self.selected_fields, self.case_sensitive)
25532553
request = PlanTableScanRequest(
25542554
snapshot_id=self.snapshot_id,
2555-
select=list(self.selected_fields) if self.selected_fields != ("*",) else None,
2555+
select=list(schema_fields) if schema_fields != ("*",) else None,
25562556
filter=self.row_filter if self.row_filter != ALWAYS_TRUE else None,
25572557
case_sensitive=self.case_sensitive,
25582558
)
@@ -2736,10 +2736,9 @@ def to_snapshot_id_inclusive(self: IAS, to_snapshot_id: int) -> IAS:
27362736
return self.update(to_snapshot_id=to_snapshot_id)
27372737

27382738
def projection(self) -> Schema:
2739-
current_schema = self.table_metadata.schema()
2740-
if "*" in self.selected_fields:
2741-
return current_schema
2742-
return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)
2739+
return project_schema(
2740+
self.table_metadata.schema(), self.selected_fields, self.case_sensitive, self.table_metadata.format_version
2741+
)
27432742

27442743
def plan_files(self) -> Iterable[FileScanTask]:
27452744
"""Plans the relevant files added between the specified snapshots."""
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""Metadata columns that are not part of a table schema, but can be selected in a scan."""
18+
19+
from __future__ import annotations
20+
21+
from collections.abc import Mapping
22+
from types import MappingProxyType
23+
24+
from pyiceberg.schema import Schema
25+
from pyiceberg.types import LongType, NestedField
26+
27+
ROW_ID = NestedField(
28+
field_id=2147483540,
29+
name="_row_id",
30+
field_type=LongType(),
31+
required=False,
32+
doc="Implicit row ID that is automatically assigned",
33+
)
34+
LAST_UPDATED_SEQUENCE_NUMBER = NestedField(
35+
field_id=2147483539,
36+
name="_last_updated_sequence_number",
37+
field_type=LongType(),
38+
required=False,
39+
doc="Sequence number when the row was last updated",
40+
)
41+
42+
METADATA_COLUMNS: Mapping[str, NestedField] = MappingProxyType(
43+
{field.name: field for field in (ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER)}
44+
)
45+
46+
47+
def _find_metadata_column(name: str, case_sensitive: bool) -> NestedField | None:
48+
if case_sensitive:
49+
return METADATA_COLUMNS.get(name)
50+
return next((field for field in METADATA_COLUMNS.values() if field.name.lower() == name.lower()), None)
51+
52+
53+
def _has_field(schema: Schema, name: str, case_sensitive: bool) -> bool:
54+
try:
55+
schema.find_field(name, case_sensitive=case_sensitive)
56+
return True
57+
except ValueError:
58+
return False
59+
60+
61+
def split_selected_fields(
62+
schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool
63+
) -> tuple[tuple[str, ...], tuple[NestedField, ...]]:
64+
"""Split selected names into schema columns and metadata columns.
65+
66+
A name that resolves in the schema is a schema column, even when it matches a metadata column name.
67+
"""
68+
schema_names: list[str] = []
69+
metadata_fields: list[NestedField] = []
70+
for name in selected_fields:
71+
if name != "*" and not _has_field(schema, name, case_sensitive):
72+
if (field := _find_metadata_column(name, case_sensitive)) is not None:
73+
if field not in metadata_fields:
74+
metadata_fields.append(field)
75+
continue
76+
schema_names.append(name)
77+
return tuple(schema_names), tuple(metadata_fields)
78+
79+
80+
def project_schema(schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool, format_version: int) -> Schema:
81+
"""Prune the schema to the selected names, and append any selected metadata columns."""
82+
schema_names, metadata_fields = split_selected_fields(schema, selected_fields, case_sensitive)
83+
84+
if metadata_fields and format_version < 3:
85+
names = ", ".join(field.name for field in metadata_fields)
86+
raise ValueError(
87+
f"Cannot select {names}: row lineage requires format version 3, but the table has format version {format_version}"
88+
)
89+
90+
projected = schema if "*" in schema_names else schema.select(*schema_names, case_sensitive=case_sensitive)
91+
if not metadata_fields:
92+
return projected
93+
94+
return Schema(
95+
*projected.fields,
96+
*metadata_fields,
97+
schema_id=projected.schema_id,
98+
identifier_field_ids=projected.identifier_field_ids,
99+
)

‎tests/integration/test_reads.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1299,6 +1299,55 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi
12991299
assert actual == expected
13001300

13011301

1302+
@pytest.mark.integration
1303+
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
1304+
def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
1305+
identifier = "default.test_read_row_ids_written_by_spark"
1306+
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
1307+
spark.sql(
1308+
f"""
1309+
CREATE TABLE {identifier} (id int, data string)
1310+
USING ICEBERG
1311+
TBLPROPERTIES (
1312+
'format-version'='3',
1313+
'write.update.mode'='copy-on-write',
1314+
'write.delete.mode'='merge-on-read',
1315+
'write.merge.mode'='merge-on-read'
1316+
)
1317+
"""
1318+
)
1319+
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')")
1320+
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (5, 'e'), (6, 'f')")
1321+
# Copy-on-write rewrites the data file and stores the existing row IDs in it
1322+
spark.sql(f"UPDATE {identifier} SET data = 'x' WHERE id = 2")
1323+
# Merge-on-read adds deletion vectors, and new rows inherit their row IDs
1324+
spark.sql(f"DELETE FROM {identifier} WHERE id = 3")
1325+
spark.sql(
1326+
f"""
1327+
MERGE INTO {identifier} t
1328+
USING (SELECT * FROM VALUES (5, 'y'), (7, 'g') AS s(id, data)) s
1329+
ON t.id = s.id
1330+
WHEN MATCHED THEN UPDATE SET t.data = s.data
1331+
WHEN NOT MATCHED THEN INSERT *
1332+
"""
1333+
)
1334+
spark.sql(f"CALL rest.system.rewrite_data_files(table => '{identifier}', options => map('rewrite-all', 'true'))")
1335+
# Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted
1336+
spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')")
1337+
1338+
expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect())
1339+
assert all(row_id is not None for _, row_id in expected)
1340+
1341+
table = catalog.load_table(identifier)
1342+
result = table.scan(selected_fields=("id", "_row_id")).to_arrow()
1343+
assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected
1344+
1345+
filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow()
1346+
assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [
1347+
(id_, row_id) for id_, row_id in expected if id_ >= 9
1348+
]
1349+
1350+
13021351
@pytest.mark.integration
13031352
@pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")])
13041353
def test_filter_after_arrow_scan(catalog: Catalog) -> None:

0 commit comments

Comments
 (0)