Skip to content

Commit 3cce3d3

Browse files
authored
Add Spark interop test for reading Puffin deletion vectors (#3476)
* Add Spark interop test for reading Puffin deletion vectors * Address review comments: move Puffin DV interop test to test_deletes.py - Move test_read_spark_written_puffin_dv from test_puffin_spark_interop.py to test_deletes.py; delete the old file - Replace run_spark_commands helper calls with direct spark.sql() calls - Tighten entries assertion from > 0 to == 1 * Tighten delete manifest assertion to == 1 Co-authored-by: Isaac
1 parent 30cf177 commit 3cce3d3

1 file changed

Lines changed: 65 additions & 1 deletion

File tree

‎tests/integration/test_deletes.py‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@
2525
from pyiceberg.catalog.rest import RestCatalog
2626
from pyiceberg.exceptions import NoSuchTableError
2727
from pyiceberg.expressions import AlwaysTrue, EqualTo, LessThanOrEqual
28-
from pyiceberg.manifest import ManifestEntryStatus
28+
from pyiceberg.manifest import ManifestContent, ManifestEntryStatus
2929
from pyiceberg.partitioning import PartitionField, PartitionSpec
3030
from pyiceberg.schema import Schema
3131
from pyiceberg.table import Table
32+
from pyiceberg.table.puffin import PuffinFile
3233
from pyiceberg.table.snapshots import Operation, Summary
3334
from pyiceberg.transforms import IdentityTransform
3435
from pyiceberg.types import FloatType, IntegerType, LongType, NestedField, StringType, TimestampType
@@ -1024,3 +1025,66 @@ def test_manifest_entry_snapshot_id_after_partial_deletes(session_catalog: RestC
10241025
f"DELETED entry snapshot_id should be {after_delete_snapshot.snapshot_id} "
10251026
f"(the deleting snapshot), but was {entry.snapshot_id}"
10261027
)
1028+
1029+
1030+
@pytest.mark.integration
1031+
def test_read_spark_written_puffin_dv(spark: SparkSession, session_catalog: RestCatalog) -> None:
1032+
"""Verify pyiceberg can read Puffin DVs written by Spark."""
1033+
identifier = "default.spark_puffin_format_test"
1034+
1035+
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
1036+
spark.sql(
1037+
f"""
1038+
CREATE TABLE {identifier} (id BIGINT)
1039+
USING iceberg
1040+
TBLPROPERTIES (
1041+
'format-version' = '3',
1042+
'write.delete.mode' = 'merge-on-read'
1043+
)
1044+
"""
1045+
)
1046+
1047+
df = spark.range(1, 51)
1048+
df.coalesce(1).writeTo(identifier).append()
1049+
1050+
files_before = spark.sql(f"SELECT * FROM {identifier}.files").collect()
1051+
assert len(files_before) == 1, f"Expected 1 file, got {len(files_before)}"
1052+
1053+
spark.sql(f"DELETE FROM {identifier} WHERE id IN (10, 20, 30, 40)")
1054+
1055+
table = session_catalog.load_table(identifier)
1056+
current_snapshot = table.current_snapshot()
1057+
assert current_snapshot is not None
1058+
1059+
manifests = current_snapshot.manifests(table.io)
1060+
delete_manifests = [m for m in manifests if m.content == ManifestContent.DELETES]
1061+
assert len(delete_manifests) == 1, "Expected exactly one delete manifest with DVs"
1062+
1063+
delete_manifest = delete_manifests[0]
1064+
entries = list(delete_manifest.fetch_manifest_entry(table.io))
1065+
assert len(entries) == 1, "Expected exactly one delete file entry"
1066+
1067+
delete_entry = entries[0]
1068+
puffin_path = delete_entry.data_file.file_path
1069+
assert puffin_path.endswith(".puffin"), f"Expected Puffin file, got: {puffin_path}"
1070+
1071+
input_file = table.io.new_input(puffin_path)
1072+
with input_file.open() as f:
1073+
puffin_bytes = f.read()
1074+
1075+
puffin = PuffinFile(puffin_bytes)
1076+
1077+
assert len(puffin.footer.blobs) == 1, "Expected exactly one blob"
1078+
1079+
blob = puffin.footer.blobs[0]
1080+
assert blob.type == "deletion-vector-v1"
1081+
assert "referenced-data-file" in blob.properties
1082+
assert blob.properties["cardinality"] == "4"
1083+
1084+
dv_dict = puffin.to_vector()
1085+
assert len(dv_dict) == 1, "Expected one data file's deletions"
1086+
1087+
for _data_file_path, chunked_array in dv_dict.items():
1088+
positions = chunked_array.to_pylist()
1089+
assert len(positions) == 4, f"Expected 4 deleted positions, got {len(positions)}"
1090+
assert sorted(positions) == [9, 19, 29, 39], f"Unexpected positions: {positions}"

0 commit comments

Comments
 (0)