|
86 | 86 | _check_pyarrow_schema_compatible, |
87 | 87 | _ConvertToArrowSchema, |
88 | 88 | _create_arrow_read_format, |
| 89 | + _dataframe_to_data_files, |
89 | 90 | _determine_partitions, |
90 | 91 | _primitive_to_physical, |
91 | 92 | _read_all_delete_files, |
92 | 93 | _read_deletes, |
| 94 | + _sort_table_for_write, |
93 | 95 | _task_to_record_batches, |
94 | 96 | _to_requested_schema, |
95 | 97 | bin_pack_arrow_table, |
|
111 | 113 | PROPERTY_REFERENCED_DATA_FILE, |
112 | 114 | deletion_vectors_from_puffin_file, |
113 | 115 | ) |
114 | | -from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3 |
| 116 | +from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3, new_table_metadata |
115 | 117 | from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID |
116 | 118 | from pyiceberg.table.name_mapping import create_mapping_from_schema |
117 | 119 | from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile |
| 120 | +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder |
118 | 121 | from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform |
119 | 122 | from pyiceberg.typedef import UTF8, Properties, Record, TableVersion |
120 | 123 | from pyiceberg.types import ( |
@@ -3111,6 +3114,125 @@ def test_bin_pack_arrow_table_target_size_smaller_than_row(arrow_table_with_null |
3111 | 3114 | assert sum(batch.num_rows for bin_ in bin_packed for batch in bin_) == arrow_table_with_null.num_rows |
3112 | 3115 |
|
3113 | 3116 |
|
| 3117 | +def test_sort_table_for_write_uses_identity_sort_order() -> None: |
| 3118 | + schema = Schema( |
| 3119 | + NestedField(1, "id", LongType(), required=False), |
| 3120 | + NestedField(2, "value", StringType(), required=False), |
| 3121 | + ) |
| 3122 | + cases = [ |
| 3123 | + (SortDirection.ASC, NullOrder.NULLS_LAST, [1, 2, None]), |
| 3124 | + (SortDirection.ASC, NullOrder.NULLS_FIRST, [None, 1, 2]), |
| 3125 | + (SortDirection.DESC, NullOrder.NULLS_LAST, [2, 1, None]), |
| 3126 | + (SortDirection.DESC, NullOrder.NULLS_FIRST, [None, 2, 1]), |
| 3127 | + ] |
| 3128 | + |
| 3129 | + for direction, null_order, expected in cases: |
| 3130 | + metadata = new_table_metadata( |
| 3131 | + schema=schema, |
| 3132 | + partition_spec=PartitionSpec(), |
| 3133 | + sort_order=SortOrder(SortField(1, IdentityTransform(), direction, null_order)), |
| 3134 | + location="file:///tmp/sorted", |
| 3135 | + ) |
| 3136 | + table = pa.table({"id": [2, None, 1], "value": ["b", "null", "a"]}) |
| 3137 | + |
| 3138 | + sorted_table, sort_order_id = _sort_table_for_write(metadata, table) |
| 3139 | + |
| 3140 | + assert sorted_table["id"].to_pylist() == expected |
| 3141 | + assert sort_order_id == metadata.default_sort_order_id |
| 3142 | + |
| 3143 | + |
| 3144 | +def test_sort_table_for_write_leaves_unsupported_sort_orders_unsorted() -> None: |
| 3145 | + schema = Schema( |
| 3146 | + NestedField(1, "id", LongType(), required=False), |
| 3147 | + NestedField(2, "value", StringType(), required=False), |
| 3148 | + ) |
| 3149 | + table = pa.table({"id": [2, 1], "value": ["b", "a"]}) |
| 3150 | + cases = [ |
| 3151 | + ( |
| 3152 | + SortOrder(SortField(1, TruncateTransform(10))), |
| 3153 | + "Sort transform truncate\\[10\\] is not supported", |
| 3154 | + ), |
| 3155 | + ( |
| 3156 | + SortOrder( |
| 3157 | + SortField(1, IdentityTransform(), null_order=NullOrder.NULLS_FIRST), |
| 3158 | + SortField(2, IdentityTransform(), null_order=NullOrder.NULLS_LAST), |
| 3159 | + ), |
| 3160 | + "Mixed null ordering", |
| 3161 | + ), |
| 3162 | + ] |
| 3163 | + |
| 3164 | + for sort_order, message in cases: |
| 3165 | + metadata = new_table_metadata( |
| 3166 | + schema=schema, |
| 3167 | + partition_spec=PartitionSpec(), |
| 3168 | + sort_order=sort_order, |
| 3169 | + location="file:///tmp/unsorted", |
| 3170 | + ) |
| 3171 | + |
| 3172 | + with pytest.warns(UserWarning, match=message): |
| 3173 | + sorted_table, sort_order_id = _sort_table_for_write(metadata, table) |
| 3174 | + |
| 3175 | + assert sorted_table is table |
| 3176 | + assert sort_order_id is None |
| 3177 | + |
| 3178 | + metadata = new_table_metadata( |
| 3179 | + schema=schema, |
| 3180 | + partition_spec=PartitionSpec(), |
| 3181 | + sort_order=SortOrder(SortField(2, IdentityTransform())), |
| 3182 | + location="file:///tmp/unsorted-missing-column", |
| 3183 | + ) |
| 3184 | + table_without_sort_column = pa.table({"id": [2, 1]}) |
| 3185 | + |
| 3186 | + with pytest.warns(UserWarning, match="nested or missing column"): |
| 3187 | + sorted_table, sort_order_id = _sort_table_for_write(metadata, table_without_sort_column) |
| 3188 | + |
| 3189 | + assert sorted_table is table_without_sort_column |
| 3190 | + assert sort_order_id is None |
| 3191 | + |
| 3192 | + |
| 3193 | +def test_write_sorted_data_files_per_partition(tmp_path: Path) -> None: |
| 3194 | + schema = Schema( |
| 3195 | + NestedField(1, "id", LongType(), required=False), |
| 3196 | + NestedField(2, "category", StringType(), required=False), |
| 3197 | + ) |
| 3198 | + metadata = new_table_metadata( |
| 3199 | + schema=schema, |
| 3200 | + partition_spec=PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="category")), |
| 3201 | + sort_order=SortOrder(SortField(1, IdentityTransform(), SortDirection.ASC, NullOrder.NULLS_LAST)), |
| 3202 | + location=f"file://{tmp_path}/sorted-partitioned", |
| 3203 | + ) |
| 3204 | + table = pa.table({"id": [3, None, 1, 2, None, 4], "category": ["a", "b", "b", "a", "a", "b"]}) |
| 3205 | + io = PyArrowFileIO() |
| 3206 | + |
| 3207 | + data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table, io=io)) |
| 3208 | + |
| 3209 | + assert len(data_files) == 2 |
| 3210 | + for data_file in data_files: |
| 3211 | + assert data_file.sort_order_id == metadata.default_sort_order_id |
| 3212 | + with io.new_input(data_file.file_path).open() as f: |
| 3213 | + read_table = pq.read_table(f) |
| 3214 | + assert len(read_table["category"].unique()) == 1 |
| 3215 | + ids = read_table["id"].to_pylist() |
| 3216 | + assert ids == sorted(ids, key=lambda value: (value is None, value)) |
| 3217 | + |
| 3218 | + |
| 3219 | +def test_streaming_write_ignores_sort_order(tmp_path: Path) -> None: |
| 3220 | + schema = Schema(NestedField(1, "id", LongType(), required=False)) |
| 3221 | + metadata = new_table_metadata( |
| 3222 | + schema=schema, |
| 3223 | + partition_spec=PartitionSpec(), |
| 3224 | + sort_order=SortOrder(SortField(1, IdentityTransform())), |
| 3225 | + location=f"file://{tmp_path}/sorted-streaming", |
| 3226 | + ) |
| 3227 | + table = pa.table({"id": [2, 1]}) |
| 3228 | + |
| 3229 | + with pytest.warns(UserWarning, match="streaming RecordBatchReader writes"): |
| 3230 | + data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table.to_reader(), io=PyArrowFileIO())) |
| 3231 | + |
| 3232 | + assert data_files |
| 3233 | + assert all(data_file.sort_order_id is None for data_file in data_files) |
| 3234 | + |
| 3235 | + |
3114 | 3236 | def test_bin_pack_record_batches_single_bin(arrow_table_with_null: pa.Table) -> None: |
3115 | 3237 | batches = arrow_table_with_null.to_batches() |
3116 | 3238 | bins = list(bin_pack_record_batches(iter(batches), target_file_size=arrow_table_with_null.nbytes * 10)) |
|
0 commit comments