5050 TypeVar ,
5151 cast ,
5252)
53+ from typing import (
54+ Literal as LiteralType ,
55+ )
5356from urllib .parse import urlparse
5457
5558import pyarrow as pa
@@ -3059,11 +3062,13 @@ def _dataframe_to_data_files(
30593062
30603063def _sort_table_for_write (table_metadata : TableMetadata , table : pa .Table ) -> tuple [pa .Table , int | None ]:
30613064 """Apply a supported Iceberg sort order and return its file sort-order id."""
3065+ from packaging import version
3066+
30623067 sort_order = table_metadata .sort_order ()
30633068 if sort_order .is_unsorted :
30643069 return table , None
30653070
3066- sort_keys : list [tuple [str , str ]] = []
3071+ sort_keys : list [tuple [str , LiteralType [ "ascending" , "descending" ] ]] = []
30673072 null_orders = set ()
30683073 for field in sort_order .fields :
30693074 if not isinstance (field .transform , IdentityTransform ):
@@ -3079,15 +3084,22 @@ def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tup
30793084 stacklevel = 2 ,
30803085 )
30813086 return table , None
3082- direction = "descending" if field .direction == SortDirection .DESC else "ascending"
3087+ direction : LiteralType [ "ascending" , "descending" ] = "descending" if field .direction == SortDirection .DESC else "ascending"
30833088 sort_keys .append ((name , direction ))
30843089 null_orders .add (field .null_order )
30853090
30863091 if len (null_orders ) != 1 :
30873092 warnings .warn ("Mixed null ordering is not supported; data files are marked unsorted" , stacklevel = 2 )
30883093 return table , None
3089- null_placement = "at_start" if null_orders == {NullOrder .NULLS_FIRST } else "at_end"
3090- indices = pc .sort_indices (table , sort_keys = sort_keys , null_placement = null_placement )
3094+ null_placement : LiteralType ["at_start" , "at_end" ] = "at_start" if null_orders == {NullOrder .NULLS_FIRST } else "at_end"
3095+ if version .parse (pyarrow .__version__ ) < version .parse ("25.0.0" ):
3096+ indices = pc .sort_indices (table , sort_keys = sort_keys , null_placement = null_placement )
3097+ else :
3098+ # Per-key null placement replaced the SortOptions-level argument in Arrow 25
3099+ indices = pc .sort_indices (
3100+ table ,
3101+ sort_keys = cast (Any , [(name , order , null_placement ) for name , order in sort_keys ]),
3102+ )
30913103 return table .take (indices ), sort_order .order_id
30923104
30933105
0 commit comments