|
26 | 26 | import zlib |
27 | 27 | from collections.abc import Iterator |
28 | 28 | from datetime import UTC, date, datetime |
| 29 | +from decimal import Decimal |
29 | 30 | from pathlib import Path |
30 | 31 | from typing import Any, Literal |
31 | 32 | from unittest.mock import MagicMock, patch |
|
66 | 67 | EqualTo, |
67 | 68 | GreaterThan, |
68 | 69 | IsNull, |
| 70 | + LessThan, |
69 | 71 | Not, |
| 72 | + NotEqualTo, |
70 | 73 | Or, |
71 | 74 | ) |
72 | 75 | from pyiceberg.expressions.literals import literal |
| 76 | +from pyiceberg.expressions.visitors import bind |
73 | 77 | from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io |
74 | 78 | from pyiceberg.io.pyarrow import ( |
75 | 79 | ICEBERG_SCHEMA, |
|
98 | 102 | from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestEntry, ManifestEntryStatus |
99 | 103 | from pyiceberg.partitioning import PartitionField, PartitionSpec |
100 | 104 | from pyiceberg.schema import Schema, make_compatible_name, visit |
101 | | -from pyiceberg.table import FileScanTask, TableProperties, WriteTask |
| 105 | +from pyiceberg.table import FileScanTask, Table, TableProperties, WriteTask |
102 | 106 | from pyiceberg.table.delete_file_index import DeleteFileIndex |
103 | 107 | from pyiceberg.table.deletion_vector import ( |
104 | 108 | _DV_BLOB_MAGIC_NUMBER, |
@@ -6102,3 +6106,260 @@ def test_dictionary_columns_produces_dict_encoded_output(tmpdir: str) -> None: |
6102 | 6106 |
|
6103 | 6107 | # Values must be identical |
6104 | 6108 | assert result_plain.column("label").to_pylist() == result_dict.column("label").to_pylist() |
| 6109 | + |
| 6110 | + |
| 6111 | +def test_row_filter_on_files_written_before_type_promotion(catalog: InMemoryCatalog) -> None: |
| 6112 | + """Filters compare in the table's type, not the type of older data files (https://github.com/apache/iceberg-python/issues/4203).""" |
| 6113 | + catalog.create_namespace("default") |
| 6114 | + table = catalog.create_table( |
| 6115 | + "default.test_filter_type_promotion", |
| 6116 | + schema=Schema( |
| 6117 | + NestedField(1, "id", IntegerType(), required=False), |
| 6118 | + NestedField(2, "f", FloatType(), required=False), |
| 6119 | + NestedField(3, "d", DecimalType(5, 2), required=False), |
| 6120 | + ), |
| 6121 | + ) |
| 6122 | + table.append( |
| 6123 | + pa.table( |
| 6124 | + { |
| 6125 | + "id": pa.array([1, 2], pa.int32()), |
| 6126 | + "f": pa.array([0.1, 2.5], pa.float32()), |
| 6127 | + "d": pa.array([Decimal("1.00"), Decimal("999.99")], pa.decimal128(5, 2)), |
| 6128 | + } |
| 6129 | + ) |
| 6130 | + ) |
| 6131 | + |
| 6132 | + with table.update_schema() as update: |
| 6133 | + update.update_column("f", DoubleType()) |
| 6134 | + update.update_column("d", DecimalType(12, 2)) |
| 6135 | + update.update_column("id", LongType()) |
| 6136 | + table = catalog.load_table("default.test_filter_type_promotion") |
| 6137 | + |
| 6138 | + # The stored float32 0.1 is promoted to 0.10000000149011612, which is greater than and not equal to the double 0.1 |
| 6139 | + for row_filter, expected in [ |
| 6140 | + ("f > 0.1", [1, 2]), |
| 6141 | + ("f != 0.1", [1, 2]), |
| 6142 | + ("f = 0.1", []), |
| 6143 | + ("f < 0.1", []), |
| 6144 | + ("d < '1000.00'", [1, 2]), |
| 6145 | + ("d > '999.99'", []), |
| 6146 | + ("id < 3000000000", [1, 2]), |
| 6147 | + ]: |
| 6148 | + assert table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist() == expected, row_filter |
| 6149 | + |
| 6150 | + |
| 6151 | +def test_row_filter_after_type_promotion_with_evolved_schemas(catalog: InMemoryCatalog) -> None: |
| 6152 | + """Promoted-column filters hold for nested, renamed and added columns, mixed files, and nulls (issue 4203).""" |
| 6153 | + catalog.create_namespace("default") |
| 6154 | + |
| 6155 | + def check(table: Table, cases: list[tuple[str, list[int]]]) -> None: |
| 6156 | + for row_filter, expected in cases: |
| 6157 | + ids = sorted(table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist()) |
| 6158 | + assert ids == expected, f"{table.name()}: {row_filter}" |
| 6159 | + |
| 6160 | + # Nested field |
| 6161 | + struct_type = pa.struct([("x", pa.float32())]) |
| 6162 | + table = catalog.create_table( |
| 6163 | + "default.promotion_nested", |
| 6164 | + schema=Schema( |
| 6165 | + NestedField(1, "id", LongType(), required=False), |
| 6166 | + NestedField(2, "s", StructType(NestedField(3, "x", FloatType(), required=False)), required=False), |
| 6167 | + ), |
| 6168 | + ) |
| 6169 | + table.append( |
| 6170 | + pa.table( |
| 6171 | + {"id": pa.array([1, 2], pa.int64()), "s": pa.array([{"x": 0.1}, {"x": 2.5}], struct_type)}, |
| 6172 | + ) |
| 6173 | + ) |
| 6174 | + with table.update_schema() as update: |
| 6175 | + update.update_column("s.x", DoubleType()) |
| 6176 | + check( |
| 6177 | + catalog.load_table("default.promotion_nested"), |
| 6178 | + [("s.x > 0.1", [1, 2]), ("s.x != 0.1", [1, 2]), ("s.x = 0.1", []), ("s.x < 1.0", [1])], |
| 6179 | + ) |
| 6180 | + |
| 6181 | + float_data = pa.table({"id": pa.array([1, 2], pa.int64()), "f": pa.array([0.1, 2.5], pa.float32())}) |
| 6182 | + float_schema = Schema( |
| 6183 | + NestedField(1, "id", LongType(), required=False), |
| 6184 | + NestedField(2, "f", FloatType(), required=False), |
| 6185 | + ) |
| 6186 | + |
| 6187 | + # Renamed column: the file keeps the old name |
| 6188 | + table = catalog.create_table("default.promotion_renamed", schema=float_schema) |
| 6189 | + table.append(float_data) |
| 6190 | + with table.update_schema() as update: |
| 6191 | + update.update_column("f", DoubleType()) |
| 6192 | + with table.update_schema() as update: |
| 6193 | + update.rename_column("f", "g") |
| 6194 | + check( |
| 6195 | + catalog.load_table("default.promotion_renamed"), |
| 6196 | + [("g > 0.1", [1, 2]), ("g != 0.1", [1, 2]), ("g = 0.1", [])], |
| 6197 | + ) |
| 6198 | + |
| 6199 | + # Column added after the file was written, combined with the promoted column |
| 6200 | + table = catalog.create_table("default.promotion_added", schema=float_schema) |
| 6201 | + table.append(float_data) |
| 6202 | + with table.update_schema() as update: |
| 6203 | + update.update_column("f", DoubleType()) |
| 6204 | + update.add_column("n", StringType()) |
| 6205 | + check( |
| 6206 | + catalog.load_table("default.promotion_added"), |
| 6207 | + [ |
| 6208 | + ("f > 0.1 AND n IS NULL", [1, 2]), |
| 6209 | + ("f > 0.1 AND n IS NOT NULL", []), |
| 6210 | + ("f > 0.1 OR n = 'a'", [1, 2]), |
| 6211 | + ("NOT (f > 0.1 AND n = 'a')", [1, 2]), |
| 6212 | + ("f = 0.1 OR n IS NULL", [1, 2]), |
| 6213 | + ("f < 1.0 AND n IS NULL", [1]), |
| 6214 | + ], |
| 6215 | + ) |
| 6216 | + |
| 6217 | + # Files written before and after the promotion in the same scan |
| 6218 | + table = catalog.create_table("default.promotion_mixed", schema=float_schema) |
| 6219 | + table.append(float_data) |
| 6220 | + with table.update_schema() as update: |
| 6221 | + update.update_column("f", DoubleType()) |
| 6222 | + table = catalog.load_table("default.promotion_mixed") |
| 6223 | + table.append(pa.table({"id": pa.array([3, 4], pa.int64()), "f": pa.array([0.1, 2.5], pa.float64())})) |
| 6224 | + check( |
| 6225 | + table, |
| 6226 | + [("f > 0.1", [1, 2, 4]), ("f != 0.1", [1, 2, 4]), ("f = 0.1", [3]), ("f <= 0.1", [3])], |
| 6227 | + ) |
| 6228 | + |
| 6229 | + # Nulls in the promoted column |
| 6230 | + table = catalog.create_table("default.promotion_nulls", schema=float_schema) |
| 6231 | + table.append(pa.table({"id": pa.array([1, 2, 3], pa.int64()), "f": pa.array([0.1, None, 2.5], pa.float32())})) |
| 6232 | + with table.update_schema() as update: |
| 6233 | + update.update_column("f", DoubleType()) |
| 6234 | + check( |
| 6235 | + catalog.load_table("default.promotion_nulls"), |
| 6236 | + [ |
| 6237 | + ("f != 0.1", [1, 3]), |
| 6238 | + ("f > 0.1", [1, 3]), |
| 6239 | + ("f IS NULL", [2]), |
| 6240 | + ("f IS NOT NULL", [1, 3]), |
| 6241 | + ("NOT (f > 0.1)", []), |
| 6242 | + ("f NOT IN (2.5)", [1]), |
| 6243 | + ("f IS NULL OR f > 0.1", [1, 2, 3]), |
| 6244 | + ], |
| 6245 | + ) |
| 6246 | + |
| 6247 | + |
| 6248 | +def _scan_ids_with_promoted_float( |
| 6249 | + tmpdir: str, |
| 6250 | + table_schema: Schema, |
| 6251 | + row_filter: BooleanExpression, |
| 6252 | + partition_spec: PartitionSpec | None = None, |
| 6253 | + partition: Record | None = None, |
| 6254 | + positional_deletes: list[pa.ChunkedArray] | None = None, |
| 6255 | +) -> list[int]: |
| 6256 | + """Scan a file written with a float32 `f` column through a table schema where `f` is a double.""" |
| 6257 | + arrow_schema = pa.schema( |
| 6258 | + [ |
| 6259 | + pa.field("id", pa.int64(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}), |
| 6260 | + pa.field("f", pa.float32(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}), |
| 6261 | + ] |
| 6262 | + ) |
| 6263 | + file_path = _write_table_to_file( |
| 6264 | + f"{tmpdir}/promoted_float.parquet", |
| 6265 | + arrow_schema, |
| 6266 | + pa.table({"id": [10, 20, 30, 40], "f": pa.array([0.1, 0.1, 2.5, 0.1], pa.float32())}, schema=arrow_schema), |
| 6267 | + ) |
| 6268 | + data_file = DataFile.from_args( |
| 6269 | + content=DataFileContent.DATA, |
| 6270 | + file_path=file_path, |
| 6271 | + file_format=FileFormat.PARQUET, |
| 6272 | + partition=partition if partition is not None else Record(), |
| 6273 | + record_count=4, |
| 6274 | + file_size_in_bytes=os.path.getsize(file_path), |
| 6275 | + ) |
| 6276 | + batches = _task_to_record_batches( |
| 6277 | + PyArrowFileIO(), |
| 6278 | + FileScanTask(data_file), |
| 6279 | + bound_row_filter=bind(table_schema, row_filter, case_sensitive=True), |
| 6280 | + projected_schema=table_schema, |
| 6281 | + table_schema=table_schema, |
| 6282 | + projected_field_ids={field.field_id for field in table_schema.fields}, |
| 6283 | + positional_deletes=positional_deletes, |
| 6284 | + case_sensitive=True, |
| 6285 | + partition_spec=partition_spec, |
| 6286 | + ) |
| 6287 | + return sorted(id_ for batch in batches for id_ in batch.column("id").to_pylist()) |
| 6288 | + |
| 6289 | + |
| 6290 | +def test_row_filter_on_promoted_column_with_positional_deletes(tmpdir: str) -> None: |
| 6291 | + """Positional deletes are applied before the table-typed filter on a promoted column (issue 4203).""" |
| 6292 | + table_schema = Schema( |
| 6293 | + NestedField(1, "id", LongType(), required=False), |
| 6294 | + NestedField(2, "f", DoubleType(), required=False), |
| 6295 | + ) |
| 6296 | + # File positions 0-3 hold ids 10, 20, 30, 40; position 1 (id 20) is deleted |
| 6297 | + positional_deletes = [pa.chunked_array([pa.array([1], type=pa.int64())])] |
| 6298 | + |
| 6299 | + for row_filter, expected in [ |
| 6300 | + (GreaterThan("f", 0.1), [10, 30, 40]), |
| 6301 | + (NotEqualTo("f", 0.1), [10, 30, 40]), |
| 6302 | + (EqualTo("f", 0.1), []), |
| 6303 | + (LessThan("f", 1.0), [10, 40]), |
| 6304 | + ]: |
| 6305 | + assert ( |
| 6306 | + _scan_ids_with_promoted_float(tmpdir, table_schema, row_filter, positional_deletes=positional_deletes) == expected |
| 6307 | + ), row_filter |
| 6308 | + |
| 6309 | + |
| 6310 | +def test_row_filter_on_promoted_column_with_projected_missing_column(tmpdir: str) -> None: |
| 6311 | + """Predicates on columns missing from the file resolve to constants next to table-typed predicates (issue 4203).""" |
| 6312 | + # Value of an identity-partitioned column that is not stored in the file |
| 6313 | + partitioned_schema = Schema( |
| 6314 | + NestedField(1, "id", LongType(), required=False), |
| 6315 | + NestedField(2, "f", DoubleType(), required=False), |
| 6316 | + NestedField(3, "p", IntegerType(), required=False), |
| 6317 | + ) |
| 6318 | + spec = PartitionSpec(PartitionField(3, 1000, IdentityTransform(), "p")) |
| 6319 | + for row_filter, expected in [ |
| 6320 | + (And(GreaterThan("f", 0.1), EqualTo("p", 5)), [10, 20, 30, 40]), |
| 6321 | + (And(GreaterThan("f", 1.0), EqualTo("p", 5)), [30]), |
| 6322 | + (And(GreaterThan("f", 0.1), EqualTo("p", 6)), []), |
| 6323 | + (Or(EqualTo("f", 0.1), EqualTo("p", 6)), []), |
| 6324 | + (Or(EqualTo("f", 0.1), EqualTo("p", 5)), [10, 20, 30, 40]), |
| 6325 | + (Or(GreaterThan("f", 1.0), EqualTo("p", 6)), [30]), |
| 6326 | + ]: |
| 6327 | + assert _scan_ids_with_promoted_float(tmpdir, partitioned_schema, row_filter, spec, Record(5)) == expected, row_filter |
| 6328 | + |
| 6329 | + # initial-default of a column added after the file was written |
| 6330 | + default_schema = Schema( |
| 6331 | + NestedField(1, "id", LongType(), required=False), |
| 6332 | + NestedField(2, "f", DoubleType(), required=False), |
| 6333 | + NestedField(3, "d", IntegerType(), required=False, initial_default=7), |
| 6334 | + ) |
| 6335 | + for row_filter, expected in [ |
| 6336 | + (And(GreaterThan("f", 0.1), EqualTo("d", 7)), [10, 20, 30, 40]), |
| 6337 | + (And(GreaterThan("f", 1.0), EqualTo("d", 7)), [30]), |
| 6338 | + (And(GreaterThan("f", 0.1), EqualTo("d", 8)), []), |
| 6339 | + (Or(EqualTo("f", 0.1), EqualTo("d", 8)), []), |
| 6340 | + (Or(EqualTo("f", 0.1), EqualTo("d", 7)), [10, 20, 30, 40]), |
| 6341 | + (Or(GreaterThan("f", 1.0), EqualTo("d", 8)), [30]), |
| 6342 | + ]: |
| 6343 | + assert _scan_ids_with_promoted_float(tmpdir, default_schema, row_filter) == expected, row_filter |
| 6344 | + |
| 6345 | + |
| 6346 | +def test_delete_matches_scan_after_type_promotion(catalog: InMemoryCatalog) -> None: |
| 6347 | + """`delete` removes exactly the rows a scan with the same filter returns after a type promotion (issue 4203).""" |
| 6348 | + catalog.create_namespace("default") |
| 6349 | + table = catalog.create_table( |
| 6350 | + "default.promotion_delete", |
| 6351 | + schema=Schema( |
| 6352 | + NestedField(1, "id", LongType(), required=False), |
| 6353 | + NestedField(2, "f", FloatType(), required=False), |
| 6354 | + ), |
| 6355 | + ) |
| 6356 | + table.append(pa.table({"id": pa.array([1, 2, 3], pa.int64()), "f": pa.array([0.1, 2.5, 0.0], pa.float32())})) |
| 6357 | + with table.update_schema() as update: |
| 6358 | + update.update_column("f", DoubleType()) |
| 6359 | + table = catalog.load_table("default.promotion_delete") |
| 6360 | + |
| 6361 | + matched = sorted(table.scan(row_filter="f > 0.1").to_arrow()["id"].to_pylist()) |
| 6362 | + assert matched == [1, 2] |
| 6363 | + |
| 6364 | + table.delete("f > 0.1") |
| 6365 | + assert table.scan().to_arrow()["id"].to_pylist() == [3] |
0 commit comments