Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyiceberg/table/update/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -746,11 +746,11 @@ def update_table_metadata(
for update in updates:
new_metadata = _apply_table_update(update, new_metadata, context)

# Update last_updated_ms if it was not updated by update operations
# last_updated_ms must not precede the base's, which becomes the latest metadata-log entry
if context.has_changes():
if metadata_location:
new_metadata = _update_table_metadata_log(new_metadata, metadata_location, base_metadata.last_updated_ms)
if base_metadata.last_updated_ms == new_metadata.last_updated_ms:
if new_metadata.last_updated_ms <= base_metadata.last_updated_ms:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I compared this against Java's TableMetadata.Builder (core/src/main/java/org/apache/iceberg/TableMetadata.java). Java never inherits the snapshot timestamp into last-updated-ms at all: addSnapshot() does not touch it, and build() stamps System.currentTimeMillis() (lines 1559-1561). PyIceberg inherits the snapshot timestamp in AddSnapshotUpdate because snapshots are staged eagerly (in Java the snapshot is produced at commit time), which is the root cause of #4189. This change restores the invariant Java enforces at lines 394-403 (last-updated-ms within one minute after the latest metadata-log entry), so from Java's perspective the output is accepted again.

new_metadata = new_metadata.model_copy(update={"last_updated_ms": datetime_to_millis(datetime.now().astimezone())})

# Check correctness of partition spec, and sort order
Expand Down
21 changes: 19 additions & 2 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -1252,7 +1252,7 @@ def test_update_metadata_with_multiple_updates(table_v1: Table) -> None:
snapshot_id=25,
parent_snapshot_id=19,
sequence_number=200,
timestamp_ms=1602638573590,
timestamp_ms=1602638593590,
manifest_list="s3:/a/b/c.avro",
summary=Summary(Operation.APPEND),
schema_id=3,
Expand Down Expand Up @@ -1295,7 +1295,7 @@ def test_update_metadata_with_multiple_updates(table_v1: Table) -> None:
assert len(new_metadata.snapshot_log) == 1
assert new_metadata.snapshot_log[0].snapshot_id == 25
assert new_metadata.current_snapshot_id == 25
assert new_metadata.last_updated_ms == 1602638573590
assert new_metadata.last_updated_ms == 1602638593590
assert new_metadata.refs["main"] == SnapshotRef(
snapshot_id=25,
snapshot_ref_type="branch",
Expand Down Expand Up @@ -1683,6 +1683,23 @@ def test_update_metadata_log(table_v2: Table) -> None:
assert len(new_metadata.metadata_log) == 2


def test_update_metadata_last_updated_ms_not_before_metadata_log(table_v2: Table) -> None:
stale_snapshot = Snapshot(
snapshot_id=25,
parent_snapshot_id=19,
sequence_number=200,
timestamp_ms=table_v2.metadata.last_updated_ms - 61_000,
manifest_list="s3:/a/b/c.avro",
summary=Summary(Operation.APPEND),
schema_id=3,
)

new_metadata = update_table_metadata(
table_v2.metadata, (AddSnapshotUpdate(snapshot=stale_snapshot),), False, table_v2.metadata_location
)
assert new_metadata.last_updated_ms > new_metadata.metadata_log[-1].timestamp_ms

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: nice regression test. The issue's repro also covers the SetSnapshotRefUpdate path (the snapshot log gets stamped with the stale snapshot timestamp). I probed that separately: Java tolerates it (snapshot-log entries are validated with the same one-minute tolerance, and the stamping actually matches Java's timeOfChange behavior for added snapshots), so this is fine as is. Just mentioning in case you want a variant of this test with a SetSnapshotRefUpdate to main for completeness.



def test_update_metadata_log_overflow(table_v2: Table) -> None:
metadata_log = [
MetadataLogEntry(
Expand Down
Loading