Skip to content

Commit f823088

Browse files
committed
Row lineage: Show first_row_id in inspect.files()
Add the first_row_id column to the files metadata tables, matching the Java files table schema. Part of #4080.
1 parent 6138315 commit f823088

3 files changed

Lines changed: 9 additions & 1 deletion

File tree

‎mkdocs/docs/api.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,7 @@ data_file: struct<content: int8 not null, file_path: string not null, file_forma
713713
child 14, equality_ids: list<item: int32>
714714
child 0, item: int32
715715
child 15, sort_order_id: int32
716+
first_row_id: int64
716717
readable_metrics: struct<city: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: string, upper_bound: string> not null, lat: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: double, upper_bound: double> not null, long: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: double, upper_bound: double> not null>
717718
child 0, city: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: string, upper_bound: string> not null
718719
child 0, column_size: int64
@@ -1024,6 +1025,7 @@ key_metadata: [[0100,0100]]
10241025
split_offsets:[[[],[]]]
10251026
equality_ids:[[[],[]]]
10261027
sort_order_id:[[[],[]]]
1028+
first_row_id:[[null,null]]
10271029
readable_metrics: [
10281030
-- is_valid: all not null
10291031
-- child 0 type: struct<column_size: int64, value_count: int64, null_value_count: int64, nan_value_count: int64, lower_bound: large_string, upper_bound: large_string>

‎pyiceberg/table/inspect.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,7 @@ def _get_files_from_manifest(
784784
"split_offsets": data_file.split_offsets,
785785
"equality_ids": data_file.equality_ids,
786786
"sort_order_id": data_file.sort_order_id,
787+
"first_row_id": data_file.first_row_id,
787788
"readable_metrics": readable_metrics,
788789
}
789790
)
@@ -804,7 +805,7 @@ def _get_files_schema(self) -> pa.Schema:
804805
``file_size_in_bytes``, ``column_sizes``, ``value_counts``,
805806
``null_value_counts``, ``nan_value_counts``, ``lower_bounds``,
806807
``upper_bounds``, ``key_metadata``, ``split_offsets``,
807-
``equality_ids``, ``sort_order_id``, and ``readable_metrics``.
808+
``equality_ids``, ``sort_order_id``, ``first_row_id``, and ``readable_metrics``.
808809
"""
809810
import pyarrow as pa
810811

@@ -853,6 +854,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
853854
pa.field("split_offsets", pa.list_(pa.int64()), nullable=True),
854855
pa.field("equality_ids", pa.list_(pa.int32()), nullable=True),
855856
pa.field("sort_order_id", pa.int32(), nullable=True),
857+
pa.field("first_row_id", pa.int64(), nullable=True),
856858
pa.field("readable_metrics", pa.struct(readable_metrics_struct), nullable=True),
857859
]
858860
)

‎tests/integration/test_inspect_table.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
101101
"split_offsets",
102102
"equality_ids",
103103
"sort_order_id",
104+
"first_row_id",
104105
"readable_metrics",
105106
]
106107

@@ -132,6 +133,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
132133
"split_offsets",
133134
"equality_ids",
134135
"sort_order_id",
136+
"first_row_id",
135137
]
136138
]
137139
rhs_subset = rhs[
@@ -145,6 +147,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None:
145147
"split_offsets",
146148
"equality_ids",
147149
"sort_order_id",
150+
"first_row_id",
148151
]
149152
]
150153

@@ -943,6 +946,7 @@ def inspect_files_asserts(df: pa.Table) -> None:
943946
"split_offsets",
944947
"equality_ids",
945948
"sort_order_id",
949+
"first_row_id",
946950
"readable_metrics",
947951
]
948952

0 commit comments

Comments
 (0)