|
17 | 17 | from unittest.mock import MagicMock |
18 | 18 | from uuid import uuid4 |
19 | 19 |
|
| 20 | +import pyarrow as pa |
20 | 21 | import pytest |
21 | 22 |
|
| 23 | +from pyiceberg.catalog import Catalog |
22 | 24 | from pyiceberg.table import CommitTableResponse, Table |
23 | 25 | from pyiceberg.table.update import SetSnapshotRefUpdate, TableUpdate |
24 | 26 |
|
@@ -177,3 +179,172 @@ def test_set_current_snapshot_chained_with_create_tag(table_v2: Table) -> None: |
177 | 179 | # The main branch should point to the same snapshot as the tag |
178 | 180 | main_update = next(u for u in set_ref_updates if u.ref_name == "main") |
179 | 181 | assert main_update.snapshot_id == snapshot_one |
| 182 | + |
| 183 | + |
| 184 | +def _staged_wap_table( |
| 185 | + catalog_with_warehouse: Catalog, |
| 186 | + namespace: str, |
| 187 | + advance_main: bool = True, |
| 188 | + wap_id: str | None = None, |
| 189 | +) -> tuple[Table, int]: |
| 190 | + """Build the WAP shape: a staged branch write, with main optionally moved on since. |
| 191 | +
|
| 192 | + Returns the reloaded table and the staged snapshot id. |
| 193 | + """ |
| 194 | + catalog_with_warehouse.create_namespace(namespace) |
| 195 | + schema = pa.schema([pa.field("id", pa.int64())]) |
| 196 | + tbl = catalog_with_warehouse.create_table(f"{namespace}.tbl", schema=schema) |
| 197 | + |
| 198 | + tbl.append(pa.table({"id": [1]}, schema=schema)) |
| 199 | + tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl") |
| 200 | + |
| 201 | + tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit() |
| 202 | + tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl") |
| 203 | + |
| 204 | + tbl.append( |
| 205 | + pa.table({"id": [2, 3]}, schema=schema), |
| 206 | + branch="audit", |
| 207 | + snapshot_properties={"wap.id": wap_id} if wap_id else {}, |
| 208 | + ) |
| 209 | + tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl") |
| 210 | + staged = tbl.metadata.refs["audit"].snapshot_id |
| 211 | + |
| 212 | + if advance_main: |
| 213 | + tbl.append(pa.table({"id": [9]}, schema=schema)) |
| 214 | + tbl = catalog_with_warehouse.load_table(f"{namespace}.tbl") |
| 215 | + |
| 216 | + return tbl, staged |
| 217 | + |
| 218 | + |
| 219 | +def _ids(tbl: Table) -> list[int]: |
| 220 | + return sorted(tbl.scan().to_arrow().column("id").to_pylist()) |
| 221 | + |
| 222 | + |
| 223 | +def _current_summary_props(tbl: Table) -> dict[str, str]: |
| 224 | + snapshot = tbl.current_snapshot() |
| 225 | + assert snapshot is not None and snapshot.summary is not None |
| 226 | + return snapshot.summary.additional_properties |
| 227 | + |
| 228 | + |
| 229 | +def _current_snapshot_id(tbl: Table) -> int: |
| 230 | + snapshot_id = tbl.metadata.current_snapshot_id |
| 231 | + assert snapshot_id is not None |
| 232 | + return snapshot_id |
| 233 | + |
| 234 | + |
| 235 | +def test_cherry_pick_snapshot_replays_staged_append(catalog_with_warehouse: Catalog) -> None: |
| 236 | + """The staged rows land on main even though main moved on after the branch was cut.""" |
| 237 | + tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_replay") |
| 238 | + assert _ids(tbl) == [1, 9] |
| 239 | + |
| 240 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 241 | + |
| 242 | + tbl = catalog_with_warehouse.load_table("cp_replay.tbl") |
| 243 | + assert _ids(tbl) == [1, 2, 3, 9] |
| 244 | + assert _current_summary_props(tbl)["source-snapshot-id"] == str(staged) |
| 245 | + |
| 246 | + |
| 247 | +def test_cherry_pick_snapshot_records_published_wap_id(catalog_with_warehouse: Catalog) -> None: |
| 248 | + tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_wap", wap_id="etl-001") |
| 249 | + |
| 250 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 251 | + |
| 252 | + tbl = catalog_with_warehouse.load_table("cp_wap.tbl") |
| 253 | + assert _current_summary_props(tbl)["published-wap-id"] == "etl-001" |
| 254 | + |
| 255 | + |
| 256 | +def test_cherry_pick_snapshot_rejects_duplicate_wap_publish(catalog_with_warehouse: Catalog) -> None: |
| 257 | + """A wap id may only be published once.""" |
| 258 | + tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_dup", wap_id="etl-001") |
| 259 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 260 | + |
| 261 | + tbl = catalog_with_warehouse.load_table("cp_dup.tbl") |
| 262 | + with pytest.raises(ValueError, match="Duplicate request to cherry pick wap id"): |
| 263 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 264 | + |
| 265 | + |
| 266 | +def test_cherry_pick_snapshot_replays_append_even_when_parent_is_current( |
| 267 | + catalog_with_warehouse: Catalog, |
| 268 | +) -> None: |
| 269 | + """Appends are always replayed, so the wap trail is recorded even on an unmoved table.""" |
| 270 | + tbl, staged = _staged_wap_table(catalog_with_warehouse, "cp_ff", advance_main=False, wap_id="etl-001") |
| 271 | + assert _ids(tbl) == [1] |
| 272 | + |
| 273 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 274 | + |
| 275 | + tbl = catalog_with_warehouse.load_table("cp_ff.tbl") |
| 276 | + assert _ids(tbl) == [1, 2, 3] |
| 277 | + assert tbl.metadata.current_snapshot_id != staged |
| 278 | + assert _current_summary_props(tbl)["published-wap-id"] == "etl-001" |
| 279 | + |
| 280 | + |
| 281 | +def test_cherry_pick_snapshot_fast_forwards_non_append(catalog_with_warehouse: Catalog) -> None: |
| 282 | + """A non-append whose parent is current cannot be replayed, so the ref advances to it.""" |
| 283 | + catalog_with_warehouse.create_namespace("cp_ffna") |
| 284 | + schema = pa.schema([pa.field("id", pa.int64())]) |
| 285 | + tbl = catalog_with_warehouse.create_table("cp_ffna.tbl", schema=schema) |
| 286 | + tbl.append(pa.table({"id": [1, 2]}, schema=schema)) |
| 287 | + tbl = catalog_with_warehouse.load_table("cp_ffna.tbl") |
| 288 | + |
| 289 | + tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit() |
| 290 | + tbl = catalog_with_warehouse.load_table("cp_ffna.tbl") |
| 291 | + tbl.delete("id = 1", branch="audit") |
| 292 | + tbl = catalog_with_warehouse.load_table("cp_ffna.tbl") |
| 293 | + staged = tbl.metadata.refs["audit"].snapshot_id |
| 294 | + |
| 295 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
| 296 | + |
| 297 | + tbl = catalog_with_warehouse.load_table("cp_ffna.tbl") |
| 298 | + assert tbl.metadata.current_snapshot_id == staged |
| 299 | + assert _ids(tbl) == [2] |
| 300 | + |
| 301 | + |
| 302 | +def test_cherry_pick_snapshot_is_noop_for_ancestor(catalog_with_warehouse: Catalog) -> None: |
| 303 | + """Picking a snapshot already in main's history changes nothing.""" |
| 304 | + catalog_with_warehouse.create_namespace("cp_anc") |
| 305 | + schema = pa.schema([pa.field("id", pa.int64())]) |
| 306 | + tbl = catalog_with_warehouse.create_table("cp_anc.tbl", schema=schema) |
| 307 | + tbl.append(pa.table({"id": [1]}, schema=schema)) |
| 308 | + tbl = catalog_with_warehouse.load_table("cp_anc.tbl") |
| 309 | + first = _current_snapshot_id(tbl) |
| 310 | + tbl.append(pa.table({"id": [2]}, schema=schema)) |
| 311 | + tbl = catalog_with_warehouse.load_table("cp_anc.tbl") |
| 312 | + before = _current_snapshot_id(tbl) |
| 313 | + |
| 314 | + tbl.manage_snapshots().cherry_pick_snapshot(first).commit() |
| 315 | + |
| 316 | + tbl = catalog_with_warehouse.load_table("cp_anc.tbl") |
| 317 | + assert tbl.metadata.current_snapshot_id == before |
| 318 | + assert _ids(tbl) == [1, 2] |
| 319 | + |
| 320 | + |
| 321 | +def test_cherry_pick_snapshot_rejects_unknown_snapshot(catalog_with_warehouse: Catalog) -> None: |
| 322 | + catalog_with_warehouse.create_namespace("cp_unknown") |
| 323 | + schema = pa.schema([pa.field("id", pa.int64())]) |
| 324 | + tbl = catalog_with_warehouse.create_table("cp_unknown.tbl", schema=schema) |
| 325 | + tbl.append(pa.table({"id": [1]}, schema=schema)) |
| 326 | + tbl = catalog_with_warehouse.load_table("cp_unknown.tbl") |
| 327 | + |
| 328 | + with pytest.raises(ValueError, match="Cannot cherry-pick unknown snapshot id"): |
| 329 | + tbl.manage_snapshots().cherry_pick_snapshot(1234567890).commit() |
| 330 | + |
| 331 | + |
| 332 | +def test_cherry_pick_snapshot_rejects_non_append(catalog_with_warehouse: Catalog) -> None: |
| 333 | + """Only append snapshots can be replayed; anything else must say so rather than silently skip.""" |
| 334 | + catalog_with_warehouse.create_namespace("cp_nonappend") |
| 335 | + schema = pa.schema([pa.field("id", pa.int64())]) |
| 336 | + tbl = catalog_with_warehouse.create_table("cp_nonappend.tbl", schema=schema) |
| 337 | + tbl.append(pa.table({"id": [1, 2]}, schema=schema)) |
| 338 | + tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl") |
| 339 | + |
| 340 | + tbl.manage_snapshots().create_branch(snapshot_id=_current_snapshot_id(tbl), branch_name="audit").commit() |
| 341 | + tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl") |
| 342 | + tbl.delete("id = 1", branch="audit") |
| 343 | + tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl") |
| 344 | + staged = tbl.metadata.refs["audit"].snapshot_id |
| 345 | + |
| 346 | + tbl.append(pa.table({"id": [9]}, schema=schema)) |
| 347 | + tbl = catalog_with_warehouse.load_table("cp_nonappend.tbl") |
| 348 | + |
| 349 | + with pytest.raises(ValueError, match="not append, dynamic overwrite, or fast-forward"): |
| 350 | + tbl.manage_snapshots().cherry_pick_snapshot(staged).commit() |
0 commit comments