Skip to content

Commit 2532c5f

Browse files
committed
Core: Honor identity sort orders for Arrow table writes
Sort materialized Arrow writes that use only identity sort fields with a consistent null order, and record the table's sort order id on the written data files. Each partition is sorted independently. Sort orders that cannot be applied, such as non-identity transforms, nested or missing sort columns, mixed null ordering, and streaming readers, leave the files marked unsorted and emit a warning.
1 parent 07216ed commit 2532c5f

2 files changed

Lines changed: 191 additions & 15 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 68 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@
159159
from pyiceberg.table.metadata import TableMetadata
160160
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
161161
from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping
162+
from pyiceberg.table.sorting import NullOrder, SortDirection
162163
from pyiceberg.transforms import IdentityTransform, TruncateTransform
163164
from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion
164165
from pyiceberg.types import (
@@ -3056,10 +3057,7 @@ def write_data_file(task: WriteTask) -> DataFile:
30563057
file_format=file_format,
30573058
partition=task.partition_key.partition if task.partition_key else Record(),
30583059
file_size_in_bytes=len(fo),
3059-
# After this has been fixed:
3060-
# https://github.com/apache/iceberg-python/issues/271
3061-
# sort_order_id=task.sort_order_id,
3062-
sort_order_id=None,
3060+
sort_order_id=task.sort_order_id,
30633061
# Just copy these from the table for now
30643062
spec_id=table_metadata.default_spec_id,
30653063
equality_ids=None,
@@ -3304,6 +3302,11 @@ def _dataframe_to_data_files(
33043302
"Materialise the reader as a pa.Table first, or follow "
33053303
"https://github.com/apache/iceberg-python/issues/2152 for partitioned streaming support."
33063304
)
3305+
if not table_metadata.sort_order().is_unsorted:
3306+
warnings.warn(
3307+
"Sort order is not applied to streaming RecordBatchReader writes; data files are marked unsorted",
3308+
stacklevel=2,
3309+
)
33073310
yield from write_file(
33083311
io=io,
33093312
table_metadata=table_metadata,
@@ -3315,31 +3318,82 @@ def _dataframe_to_data_files(
33153318
return
33163319

33173320
if table_metadata.spec().is_unpartitioned():
3321+
sorted_df, sort_order_id = _sort_table_for_write(table_metadata, df)
33183322
yield from write_file(
33193323
io=io,
33203324
table_metadata=table_metadata,
33213325
tasks=(
3322-
WriteTask(write_uuid=write_uuid, task_id=next(counter), record_batches=batches, schema=task_schema)
3323-
for batches in bin_pack_arrow_table(df, target_file_size)
3326+
WriteTask(
3327+
write_uuid=write_uuid,
3328+
task_id=next(counter),
3329+
record_batches=batches,
3330+
schema=task_schema,
3331+
sort_order_id=sort_order_id,
3332+
)
3333+
for batches in bin_pack_arrow_table(sorted_df, target_file_size)
33243334
),
33253335
)
33263336
else:
33273337
partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df)
3328-
yield from write_file(
3329-
io=io,
3330-
table_metadata=table_metadata,
3331-
tasks=(
3338+
tasks: list[WriteTask] = []
3339+
for partition in partitions:
3340+
sorted_partition, sort_order_id = _sort_table_for_write(table_metadata, partition.arrow_table_partition)
3341+
tasks.extend(
33323342
WriteTask(
33333343
write_uuid=write_uuid,
33343344
task_id=next(counter),
33353345
record_batches=batches,
33363346
partition_key=partition.partition_key,
33373347
schema=task_schema,
3348+
sort_order_id=sort_order_id,
33383349
)
3339-
for partition in partitions
3340-
for batches in bin_pack_arrow_table(partition.arrow_table_partition, target_file_size)
3341-
),
3342-
)
3350+
for batches in bin_pack_arrow_table(sorted_partition, target_file_size)
3351+
)
3352+
yield from write_file(io=io, table_metadata=table_metadata, tasks=tasks)
3353+
3354+
3355+
def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tuple[pa.Table, int | None]:
3356+
"""Sort the table for the table's sort order and return its file sort-order id.
3357+
3358+
Returns the table unchanged and ``None`` when the sort order cannot be applied, so the
3359+
written files are marked unsorted.
3360+
"""
3361+
sort_order = table_metadata.sort_order()
3362+
if sort_order.is_unsorted:
3363+
return table, None
3364+
3365+
schema = table_metadata.schema()
3366+
sort_keys: list[tuple[str, str]] = []
3367+
null_orders: set[NullOrder] = set()
3368+
for sort_field in sort_order.fields:
3369+
if not isinstance(sort_field.transform, IdentityTransform):
3370+
warnings.warn(
3371+
f"Sort transform {sort_field.transform} is not supported for writes; data files are marked unsorted",
3372+
stacklevel=2,
3373+
)
3374+
return table, None
3375+
column_name = schema.find_column_name(sort_field.source_id)
3376+
if column_name is None or column_name not in table.column_names:
3377+
warnings.warn(
3378+
f"Sort field with source id {sort_field.source_id} refers to a nested or missing column; "
3379+
"data files are marked unsorted",
3380+
stacklevel=2,
3381+
)
3382+
return table, None
3383+
sort_keys.append((column_name, "descending" if sort_field.direction == SortDirection.DESC else "ascending"))
3384+
null_orders.add(sort_field.null_order)
3385+
3386+
if len(null_orders) != 1:
3387+
warnings.warn("Mixed null ordering is not supported for writes; data files are marked unsorted", stacklevel=2)
3388+
return table, None
3389+
3390+
null_placement = "at_start" if NullOrder.NULLS_FIRST in null_orders else "at_end"
3391+
if int(pa.__version__.split(".")[0]) >= 25:
3392+
# Per-key null placement replaced the SortOptions-level argument in Arrow 25.
3393+
indices = pc.sort_indices(table, sort_keys=cast(Any, [(*key, null_placement) for key in sort_keys]))
3394+
else:
3395+
indices = pc.sort_indices(table, sort_keys=cast(Any, sort_keys), null_placement=cast(Any, null_placement))
3396+
return table.take(indices), sort_order.order_id
33433397

33443398

33453399
@dataclass(frozen=True)

‎tests/io/test_pyarrow.py‎

Lines changed: 123 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,12 @@
8686
_check_pyarrow_schema_compatible,
8787
_ConvertToArrowSchema,
8888
_create_arrow_read_format,
89+
_dataframe_to_data_files,
8990
_determine_partitions,
9091
_primitive_to_physical,
9192
_read_all_delete_files,
9293
_read_deletes,
94+
_sort_table_for_write,
9395
_task_to_record_batches,
9496
_to_requested_schema,
9597
bin_pack_arrow_table,
@@ -111,10 +113,11 @@
111113
PROPERTY_REFERENCED_DATA_FILE,
112114
deletion_vectors_from_puffin_file,
113115
)
114-
from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3
116+
from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3, new_table_metadata
115117
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
116118
from pyiceberg.table.name_mapping import create_mapping_from_schema
117119
from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile
120+
from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder
118121
from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform
119122
from pyiceberg.typedef import UTF8, Properties, Record, TableVersion
120123
from pyiceberg.types import (
@@ -3111,6 +3114,125 @@ def test_bin_pack_arrow_table_target_size_smaller_than_row(arrow_table_with_null
31113114
assert sum(batch.num_rows for bin_ in bin_packed for batch in bin_) == arrow_table_with_null.num_rows
31123115

31133116

3117+
def test_sort_table_for_write_uses_identity_sort_order() -> None:
3118+
schema = Schema(
3119+
NestedField(1, "id", LongType(), required=False),
3120+
NestedField(2, "value", StringType(), required=False),
3121+
)
3122+
cases = [
3123+
(SortDirection.ASC, NullOrder.NULLS_LAST, [1, 2, None]),
3124+
(SortDirection.ASC, NullOrder.NULLS_FIRST, [None, 1, 2]),
3125+
(SortDirection.DESC, NullOrder.NULLS_LAST, [2, 1, None]),
3126+
(SortDirection.DESC, NullOrder.NULLS_FIRST, [None, 2, 1]),
3127+
]
3128+
3129+
for direction, null_order, expected in cases:
3130+
metadata = new_table_metadata(
3131+
schema=schema,
3132+
partition_spec=PartitionSpec(),
3133+
sort_order=SortOrder(SortField(1, IdentityTransform(), direction, null_order)),
3134+
location="file:///tmp/sorted",
3135+
)
3136+
table = pa.table({"id": [2, None, 1], "value": ["b", "null", "a"]})
3137+
3138+
sorted_table, sort_order_id = _sort_table_for_write(metadata, table)
3139+
3140+
assert sorted_table["id"].to_pylist() == expected
3141+
assert sort_order_id == metadata.default_sort_order_id
3142+
3143+
3144+
def test_sort_table_for_write_leaves_unsupported_sort_orders_unsorted() -> None:
3145+
schema = Schema(
3146+
NestedField(1, "id", LongType(), required=False),
3147+
NestedField(2, "value", StringType(), required=False),
3148+
)
3149+
table = pa.table({"id": [2, 1], "value": ["b", "a"]})
3150+
cases = [
3151+
(
3152+
SortOrder(SortField(1, TruncateTransform(10))),
3153+
"Sort transform truncate\\[10\\] is not supported",
3154+
),
3155+
(
3156+
SortOrder(
3157+
SortField(1, IdentityTransform(), null_order=NullOrder.NULLS_FIRST),
3158+
SortField(2, IdentityTransform(), null_order=NullOrder.NULLS_LAST),
3159+
),
3160+
"Mixed null ordering",
3161+
),
3162+
]
3163+
3164+
for sort_order, message in cases:
3165+
metadata = new_table_metadata(
3166+
schema=schema,
3167+
partition_spec=PartitionSpec(),
3168+
sort_order=sort_order,
3169+
location="file:///tmp/unsorted",
3170+
)
3171+
3172+
with pytest.warns(UserWarning, match=message):
3173+
sorted_table, sort_order_id = _sort_table_for_write(metadata, table)
3174+
3175+
assert sorted_table is table
3176+
assert sort_order_id is None
3177+
3178+
metadata = new_table_metadata(
3179+
schema=schema,
3180+
partition_spec=PartitionSpec(),
3181+
sort_order=SortOrder(SortField(2, IdentityTransform())),
3182+
location="file:///tmp/unsorted-missing-column",
3183+
)
3184+
table_without_sort_column = pa.table({"id": [2, 1]})
3185+
3186+
with pytest.warns(UserWarning, match="nested or missing column"):
3187+
sorted_table, sort_order_id = _sort_table_for_write(metadata, table_without_sort_column)
3188+
3189+
assert sorted_table is table_without_sort_column
3190+
assert sort_order_id is None
3191+
3192+
3193+
def test_write_sorted_data_files_per_partition(tmp_path: Path) -> None:
3194+
schema = Schema(
3195+
NestedField(1, "id", LongType(), required=False),
3196+
NestedField(2, "category", StringType(), required=False),
3197+
)
3198+
metadata = new_table_metadata(
3199+
schema=schema,
3200+
partition_spec=PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="category")),
3201+
sort_order=SortOrder(SortField(1, IdentityTransform(), SortDirection.ASC, NullOrder.NULLS_LAST)),
3202+
location=f"file://{tmp_path}/sorted-partitioned",
3203+
)
3204+
table = pa.table({"id": [3, None, 1, 2, None, 4], "category": ["a", "b", "b", "a", "a", "b"]})
3205+
io = PyArrowFileIO()
3206+
3207+
data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table, io=io))
3208+
3209+
assert len(data_files) == 2
3210+
for data_file in data_files:
3211+
assert data_file.sort_order_id == metadata.default_sort_order_id
3212+
with io.new_input(data_file.file_path).open() as f:
3213+
read_table = pq.read_table(f)
3214+
assert len(read_table["category"].unique()) == 1
3215+
ids = read_table["id"].to_pylist()
3216+
assert ids == sorted(ids, key=lambda value: (value is None, value))
3217+
3218+
3219+
def test_streaming_write_ignores_sort_order(tmp_path: Path) -> None:
3220+
schema = Schema(NestedField(1, "id", LongType(), required=False))
3221+
metadata = new_table_metadata(
3222+
schema=schema,
3223+
partition_spec=PartitionSpec(),
3224+
sort_order=SortOrder(SortField(1, IdentityTransform())),
3225+
location=f"file://{tmp_path}/sorted-streaming",
3226+
)
3227+
table = pa.table({"id": [2, 1]})
3228+
3229+
with pytest.warns(UserWarning, match="streaming RecordBatchReader writes"):
3230+
data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table.to_reader(), io=PyArrowFileIO()))
3231+
3232+
assert data_files
3233+
assert all(data_file.sort_order_id is None for data_file in data_files)
3234+
3235+
31143236
def test_bin_pack_record_batches_single_bin(arrow_table_with_null: pa.Table) -> None:
31153237
batches = arrow_table_with_null.to_batches()
31163238
bins = list(bin_pack_record_batches(iter(batches), target_file_size=arrow_table_with_null.nbytes * 10))

0 commit comments

Comments
 (0)