|
25 | 25 | from pyiceberg.catalog.rest import RestCatalog |
26 | 26 | from pyiceberg.exceptions import NoSuchTableError |
27 | 27 | from pyiceberg.expressions import AlwaysTrue, EqualTo, LessThanOrEqual |
28 | | -from pyiceberg.manifest import ManifestEntryStatus |
| 28 | +from pyiceberg.manifest import ManifestContent, ManifestEntryStatus |
29 | 29 | from pyiceberg.partitioning import PartitionField, PartitionSpec |
30 | 30 | from pyiceberg.schema import Schema |
31 | 31 | from pyiceberg.table import Table |
| 32 | +from pyiceberg.table.puffin import PuffinFile |
32 | 33 | from pyiceberg.table.snapshots import Operation, Summary |
33 | 34 | from pyiceberg.transforms import IdentityTransform |
34 | 35 | from pyiceberg.types import FloatType, IntegerType, LongType, NestedField, StringType, TimestampType |
@@ -1024,3 +1025,66 @@ def test_manifest_entry_snapshot_id_after_partial_deletes(session_catalog: RestC |
1024 | 1025 | f"DELETED entry snapshot_id should be {after_delete_snapshot.snapshot_id} " |
1025 | 1026 | f"(the deleting snapshot), but was {entry.snapshot_id}" |
1026 | 1027 | ) |
| 1028 | + |
| 1029 | + |
| 1030 | +@pytest.mark.integration |
| 1031 | +def test_read_spark_written_puffin_dv(spark: SparkSession, session_catalog: RestCatalog) -> None: |
| 1032 | + """Verify pyiceberg can read Puffin DVs written by Spark.""" |
| 1033 | + identifier = "default.spark_puffin_format_test" |
| 1034 | + |
| 1035 | + spark.sql(f"DROP TABLE IF EXISTS {identifier}") |
| 1036 | + spark.sql( |
| 1037 | + f""" |
| 1038 | + CREATE TABLE {identifier} (id BIGINT) |
| 1039 | + USING iceberg |
| 1040 | + TBLPROPERTIES ( |
| 1041 | + 'format-version' = '3', |
| 1042 | + 'write.delete.mode' = 'merge-on-read' |
| 1043 | + ) |
| 1044 | + """ |
| 1045 | + ) |
| 1046 | + |
| 1047 | + df = spark.range(1, 51) |
| 1048 | + df.coalesce(1).writeTo(identifier).append() |
| 1049 | + |
| 1050 | + files_before = spark.sql(f"SELECT * FROM {identifier}.files").collect() |
| 1051 | + assert len(files_before) == 1, f"Expected 1 file, got {len(files_before)}" |
| 1052 | + |
| 1053 | + spark.sql(f"DELETE FROM {identifier} WHERE id IN (10, 20, 30, 40)") |
| 1054 | + |
| 1055 | + table = session_catalog.load_table(identifier) |
| 1056 | + current_snapshot = table.current_snapshot() |
| 1057 | + assert current_snapshot is not None |
| 1058 | + |
| 1059 | + manifests = current_snapshot.manifests(table.io) |
| 1060 | + delete_manifests = [m for m in manifests if m.content == ManifestContent.DELETES] |
| 1061 | + assert len(delete_manifests) == 1, "Expected exactly one delete manifest with DVs" |
| 1062 | + |
| 1063 | + delete_manifest = delete_manifests[0] |
| 1064 | + entries = list(delete_manifest.fetch_manifest_entry(table.io)) |
| 1065 | + assert len(entries) == 1, "Expected exactly one delete file entry" |
| 1066 | + |
| 1067 | + delete_entry = entries[0] |
| 1068 | + puffin_path = delete_entry.data_file.file_path |
| 1069 | + assert puffin_path.endswith(".puffin"), f"Expected Puffin file, got: {puffin_path}" |
| 1070 | + |
| 1071 | + input_file = table.io.new_input(puffin_path) |
| 1072 | + with input_file.open() as f: |
| 1073 | + puffin_bytes = f.read() |
| 1074 | + |
| 1075 | + puffin = PuffinFile(puffin_bytes) |
| 1076 | + |
| 1077 | + assert len(puffin.footer.blobs) == 1, "Expected exactly one blob" |
| 1078 | + |
| 1079 | + blob = puffin.footer.blobs[0] |
| 1080 | + assert blob.type == "deletion-vector-v1" |
| 1081 | + assert "referenced-data-file" in blob.properties |
| 1082 | + assert blob.properties["cardinality"] == "4" |
| 1083 | + |
| 1084 | + dv_dict = puffin.to_vector() |
| 1085 | + assert len(dv_dict) == 1, "Expected one data file's deletions" |
| 1086 | + |
| 1087 | + for _data_file_path, chunked_array in dv_dict.items(): |
| 1088 | + positions = chunked_array.to_pylist() |
| 1089 | + assert len(positions) == 4, f"Expected 4 deleted positions, got {len(positions)}" |
| 1090 | + assert sorted(positions) == [9, 19, 29, 39], f"Unexpected positions: {positions}" |
0 commit comments