From a1a0ac11017161e3d1e820031ad4535eeb518186 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 15 Nov 2026 15:16:47 +0100 Subject: [PATCH 01/25] fix(table): reject snapshot expiry with refs-only history --- table/transaction.go | 3 +++ table/transaction_internal_test.go | 13 +++++++++++++ website/src/releases.md | 2 ++ 3 files changed, 18 insertions(+) diff --git a/table/transaction.go b/table/transaction.go index c8a6a037d8..3794241d01 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -827,6 +827,9 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { if err := validateGCEnabledForSnapshotExpiration(meta.props); err != nil { return err } + if t.tbl.snapshotHistoryRefsOnly { + return fmt.Errorf("cannot expire snapshots with incomplete history under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all") + } updates, walked, err := planExpiration(meta, cfg, nowMs) if err != nil { diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go index 91d8aeb695..54d0b1fb87 100644 --- a/table/transaction_internal_test.go +++ b/table/transaction_internal_test.go @@ -579,6 +579,19 @@ func TestExpireSnapshotsWithOlderThanDoesNotExpireSnapshotRefs(t *testing.T) { require.True(t, tagExists, "WithOlderThan must not expire a tag without max-ref-age-ms") } +func TestExpireSnapshotsRejectsRefsOnlySnapshotHistory(t *testing.T) { + txn := newTransactionWithSnapshotRefs(t) + txn.tbl.snapshotHistoryRefsOnly = true + + err := txn.ExpireSnapshots( + WithOlderThan(0), + WithRetainLast(1), + WithPostCommit(false), + ) + require.ErrorContains(t, err, "snapshot-loading-mode=refs") + require.ErrorContains(t, err, "snapshot-loading-mode=all") +} + func TestExpireSnapshotsMaterializesDeferredHistory(t *testing.T) { const historicalID = int64(3051729675574597004) diff --git a/website/src/releases.md b/website/src/releases.md index b65d22df15..73fa693925 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -139,6 +139,8 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Commit rejections now match `table.ErrCommitRejected`.** SQL, Glue, Hive and Hadoop failures while preparing metadata, plus REST commit responses 400, 401, 403 and 404, leave transactions committable (after a REST 404, the next `Commit` fails when it reloads the table). Existing `errors.Is` matches keep working, but SQL, Glue, Hive and Hadoop rejection messages now start with `commit rejected: `, so code that matches error text may need updating. - **Float comparisons against literals beyond the float32 range follow IEEE 754.** On a float32 column, `x < 1e100` no longer returns NaN, `+Inf` or NULL rows, and `NOT(x > 1e100)` no longer returns NULL rows. An infinite literal now binds to a float32 infinity, so `x = +Inf` matches `+Inf` rows. Java still folds these comparisons to always true or false, see [apache/iceberg#18700](https://github.com/apache/iceberg/issues/18700). +- **Snapshot expiration requires complete snapshot history.** `Transaction.ExpireSnapshots` now returns an error when REST loaded the table with `snapshot-loading-mode=refs`; reload with `snapshot-loading-mode=all`. + ## Verifying and producing releases If you are validating an RC or cutting a new release, see: From b3ccff0a64ff2721d691c9602cdd60203112f375 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 15 Nov 2026 16:43:47 +0100 Subject: [PATCH 02/25] fix(table): reject expiry replay with partial history --- table/expire_replay.go | 5 ++ table/expire_replay_test.go | 93 +++++++++++++++++++++++++++++++++++++ table/staged_replay.go | 4 ++ table/table.go | 3 ++ table/transaction.go | 2 +- 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/table/expire_replay.go b/table/expire_replay.go index 7a0a589461..cdb3f38007 100644 --- a/table/expire_replay.go +++ b/table/expire_replay.go @@ -18,10 +18,15 @@ package table import ( + "errors" "fmt" "slices" ) +var errSnapshotHistoryIncompleteForExpiration = errors.New( + "cannot expire snapshots with incomplete history under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", +) + func (r *stagedReplay) replan(b *MetadataBuilder, rec expirationRecord) ([]Update, []string, error) { updates, walked, err := planExpiration(b, rec.cfg, rec.nowMs) if err != nil { diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index a418d46f99..1d05985de5 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -51,6 +51,40 @@ func newExpireReplayTable(t *testing.T, props iceberg.Properties) (*memIO, *reqV return fs, cat, newOCCTable(t, meta, fs, cat) } +type refsOnlyReloadCatalog struct { + *reqValidatingCatalog + partial Metadata + loads int +} + +func (c *refsOnlyReloadCatalog) LoadTable(_ context.Context, ident Identifier) (*Table, error) { + c.loads++ + + return New(ident, c.partial, "", nil, c, WithSnapshotHistoryLimitedToRefs(true)), nil +} + +func refsOnlySnapshotMetadata(t *testing.T, meta Metadata, keepIDs ...int64) Metadata { + t.Helper() + keep := make(map[int64]struct{}, len(keepIDs)) + for _, id := range keepIDs { + keep[id] = struct{}{} + } + var removeIDs []int64 + for _, snap := range meta.Snapshots() { + if _, ok := keep[snap.SnapshotID]; !ok { + removeIDs = append(removeIDs, snap.SnapshotID) + } + } + + builder, err := MetadataBuilderFromBase(meta, "") + require.NoError(t, err) + require.NoError(t, builder.RemoveSnapshots(removeIDs, false)) + partial, err := builder.Build() + require.NoError(t, err) + + return partial +} + func stageFiles(t *testing.T, tx *Transaction, names ...string) { t.Helper() files := make([]iceberg.DataFile, 0, len(names)) @@ -167,6 +201,65 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t *testing.T) { }) } +func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + var ids [4]int64 + for i, name := range []string{"s1", "s2", "s3", "s4"} { + tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name) + } + peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag", ids[0], TagRef, time.Hour.Milliseconds(), 0, 0)) + time.Sleep(2 * time.Hour) + + cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + require.NoError(t, tx.ExpireSnapshots( + WithOlderThan(0), WithRetainLast(5), WithPostCommit(false), + )) + staged, err := tx.StagedTable() + require.NoError(t, err) + assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) + assert.Equal(t, sortedIDs(ids[:]...), survivors(staged.Metadata())) + + peer := peerAppend(t, fs, inner, MainBranch, "s5") + partial := refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) + assert.Equal(t, sortedIDs(ids[0], peer), survivors(partial)) + cat.partial = partial + attemptsBefore := inner.attempts.Load() + + _, err = tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrTransactionUnusable) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + assert.Equal(t, 1, cat.loads) + assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "retry must stop before resubmitting expiration") + assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) + assert.NotNil(t, inner.metadata.SnapshotByID(ids[0]), "retained history must remain in catalog metadata") + assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag"), "failed expiry must not remove the tag") + assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + }) +} + +func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, first := appendOnBranch(t, tbl, MainBranch, "first") + tbl, _ = appendOnBranch(t, tbl, MainBranch, "head") + cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + stageFiles(t, tx, "ours") + + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + assert.Equal(t, []int64{peer}, survivors(cat.partial)) + assert.NotContains(t, survivors(cat.partial), first) + + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, cat.loads) + assert.Equal(t, peer, parentOf(t, committed.Metadata(), refHead(t, committed.Metadata(), MainBranch))) + }) +} + func TestExpireReplayAfterRollback(t *testing.T) { stage := func(t *testing.T) (*memIO, *reqValidatingCatalog, *Transaction, [3]int64, int64) { t.Helper() diff --git a/table/staged_replay.go b/table/staged_replay.go index b28f3dc16d..b6742b2761 100644 --- a/table/staged_replay.go +++ b/table/staged_replay.go @@ -52,6 +52,10 @@ func newStagedReplay(branch string, updates []Update, records []expirationRecord } } +func (r *stagedReplay) hasExpirationRecords() bool { + return r != nil && len(r.records) > 0 +} + // On error, replay returns partial updates and replaced lists for cleanup. // The partial updates include a rebuilt snapshot that failed to apply. func (r *stagedReplay) replay(ctx context.Context, fresh Metadata, fs io.WriteFileIO, attempt int, force bool) ([]Update, []string, error) { diff --git a/table/table.go b/table/table.go index 34fb2d8862..bec0f24f3a 100644 --- a/table/table.go +++ b/table/table.go @@ -824,6 +824,9 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen return nil, fmt.Errorf("%w: table was replaced: %w", ErrTransactionUnusable, err) } } + if fresh.snapshotHistoryRefsOnly && co.replay.hasExpirationRecords() { + return nil, fmt.Errorf("%w: %w", ErrTransactionUnusable, errSnapshotHistoryIncompleteForExpiration) + } current, currentLoc = fresh.metadata, fresh.metadataLocation reqs = rewriteRefSnapshotRequirements(reqs, co.branch, current, pinned) diff --git a/table/transaction.go b/table/transaction.go index 3794241d01..7b73c7a8d7 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -828,7 +828,7 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { return err } if t.tbl.snapshotHistoryRefsOnly { - return fmt.Errorf("cannot expire snapshots with incomplete history under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all") + return errSnapshotHistoryIncompleteForExpiration } updates, walked, err := planExpiration(meta, cfg, nowMs) From 91642ed2c3df80f1bb2a8b24686a5d922f728f39 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 15 Nov 2026 17:52:29 +0100 Subject: [PATCH 03/25] docs(table): clarify expiration replay history errors --- table/table.go | 12 +++++++----- website/src/releases.md | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/table/table.go b/table/table.go index bec0f24f3a..ef8eb41aad 100644 --- a/table/table.go +++ b/table/table.go @@ -94,11 +94,13 @@ var ErrCommitRejected = errors.New("commit rejected") // concurrent commit removed, a commit branch that became a tag or was // deleted, a replaced table, or a staged update that cannot apply after // refresh. With snapshot expiry, they also include disabled GC and replanning -// that changes whether a commit branch other than main is removed. Only the -// commit branch's head assertion is rebased. A staged update that cannot -// apply keeps the builder's error as its cause. Transaction.Commit also -// returns it, on any attempt, when a delete references data files that are -// missing (ErrDataFilesMissing). +// that changes whether a commit branch other than main is removed. An expiry +// replay also becomes unusable if a commit retry refreshes to refs-only +// snapshot history; load with `snapshot-loading-mode=all` and start a new +// transaction. Only the commit branch's head assertion is rebased. A staged +// update that cannot apply keeps the builder's error as its cause. +// Transaction.Commit also returns it, on any attempt, when a delete references +// data files that are missing (ErrDataFilesMissing). var ErrTransactionUnusable = errors.New("transaction cannot be committed again") // ErrWriteIORequired is returned by write paths when the table's file system diff --git a/website/src/releases.md b/website/src/releases.md index 73fa693925..359cc0ec04 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -139,7 +139,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Commit rejections now match `table.ErrCommitRejected`.** SQL, Glue, Hive and Hadoop failures while preparing metadata, plus REST commit responses 400, 401, 403 and 404, leave transactions committable (after a REST 404, the next `Commit` fails when it reloads the table). Existing `errors.Is` matches keep working, but SQL, Glue, Hive and Hadoop rejection messages now start with `commit rejected: `, so code that matches error text may need updating. - **Float comparisons against literals beyond the float32 range follow IEEE 754.** On a float32 column, `x < 1e100` no longer returns NaN, `+Inf` or NULL rows, and `NOT(x > 1e100)` no longer returns NULL rows. An infinite literal now binds to a float32 infinity, so `x = +Inf` matches `+Inf` rows. Java still folds these comparisons to always true or false, see [apache/iceberg#18700](https://github.com/apache/iceberg/issues/18700). -- **Snapshot expiration requires complete snapshot history.** `Transaction.ExpireSnapshots` now returns an error when REST loaded the table with `snapshot-loading-mode=refs`; reload with `snapshot-loading-mode=all`. +- **Snapshot expiration requires complete snapshot history.** Staging `Transaction.ExpireSnapshots` fails when REST loaded the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. ## Verifying and producing releases From 91d4a99ef734b20777d7d0dbd6f73e218a6f5402 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 08:21:45 +0100 Subject: [PATCH 04/25] fix(table): address refs-only expiration review (expire_replay_test.go) --- table/expire_replay_test.go | 43 ++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index 1d05985de5..ad756bb8be 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -239,11 +239,52 @@ func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) { }) } +func TestExpireReplayRejectsRefsOnlyHistoryOnResume(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: "0"}) + var ids [4]int64 + for i, name := range []string{"s1", "s2", "s3", "s4"} { + tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name) + } + peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag", ids[0], TagRef, time.Hour.Milliseconds(), 0, 0)) + time.Sleep(2 * time.Hour) + + cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + require.NoError(t, tx.ExpireSnapshots( + WithOlderThan(0), WithRetainLast(5), WithPostCommit(false), + )) + staged, err := tx.StagedTable() + require.NoError(t, err) + assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) + + peer := peerAppend(t, fs, inner, MainBranch, "s5") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) + attemptsBefore := inner.attempts.Load() + + _, err = tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrCommitFailed) + require.NotErrorIs(t, err, ErrTransactionUnusable) + assert.Equal(t, 0, cat.loads, "first attempt must not refresh") + assert.Equal(t, attemptsBefore+1, inner.attempts.Load()) + + _, err = tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrTransactionUnusable) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + assert.Equal(t, 1, cat.loads) + assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "resume must stop before resubmitting expiration") + assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) + assert.NotNil(t, inner.metadata.SnapshotByID(ids[0])) + assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag")) + assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + }) +} + func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) { synctest.Test(t, func(t *testing.T) { fs, inner, tbl := newExpireReplayTable(t, nil) tbl, first := appendOnBranch(t, tbl, MainBranch, "first") - tbl, _ = appendOnBranch(t, tbl, MainBranch, "head") + appendOnBranch(t, tbl, MainBranch, "head") cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() stageFiles(t, tx, "ours") From 6f9091dc265d0732be859129620ef8fdafcf9626 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 08:21:47 +0100 Subject: [PATCH 05/25] fix(table): address refs-only expiration review (transaction.go) --- table/transaction.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/table/transaction.go b/table/transaction.go index 7b73c7a8d7..8976fcc86f 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -804,6 +804,11 @@ func getRetentionInt64(props iceberg.Properties, standardKey, legacyKey string, // committing. A transaction containing only expiries with no updates makes no // commit attempt and never refreshes or replans. // +// Snapshot expiration requires complete history. REST catalogs using +// snapshot-loading-mode=refs cannot stage it; load the table with +// snapshot-loading-mode=all. A retry or resumed commit refreshing to refs-only +// history makes the transaction unusable. +// // The replay failures that make the transaction unusable are listed on // ErrTransactionUnusable. // @@ -828,7 +833,7 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { return err } if t.tbl.snapshotHistoryRefsOnly { - return errSnapshotHistoryIncompleteForExpiration + return fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) } updates, walked, err := planExpiration(meta, cfg, nowMs) From 045a9593d06434b57d3bf1e71842c06ca07a5da3 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 08:21:49 +0100 Subject: [PATCH 06/25] fix(table): address refs-only expiration review (transaction_internal_test.go) --- table/transaction_internal_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go index 54d0b1fb87..1e650e9227 100644 --- a/table/transaction_internal_test.go +++ b/table/transaction_internal_test.go @@ -588,6 +588,8 @@ func TestExpireSnapshotsRejectsRefsOnlySnapshotHistory(t *testing.T) { WithRetainLast(1), WithPostCommit(false), ) + require.ErrorIs(t, err, ErrInvalidOperation) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) require.ErrorContains(t, err, "snapshot-loading-mode=refs") require.ErrorContains(t, err, "snapshot-loading-mode=all") } From fd05064e5b9bbcde37510dccfef7a32a5a3d364f Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 08:21:53 +0100 Subject: [PATCH 07/25] fix(table): address refs-only expiration review (api.md) --- website/src/api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/src/api.md b/website/src/api.md index 78fd30b34f..2505da5ccf 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -853,6 +853,8 @@ if err != nil { /* ... */ } newTbl, err := txn.Commit(ctx) ``` +Snapshot expiration requires complete snapshot history. REST catalogs using `snapshot-loading-mode=refs` cannot stage `Transaction.ExpireSnapshots`; use `snapshot-loading-mode=all` before expiring snapshots. If a retry or resumed commit reloads refs-only history, it returns `table.ErrTransactionUnusable`. Reload with all snapshots and start a new transaction. + Pass `table.WithPostCommit(true)` to delete the unreferenced data and metadata files after the commit lands. The `iceberg expire-snapshots` CLI command wraps the same operation - see [CLI](./cli.md). ## Views From 31e38da6a4eb961273733461ad59b4616e2849e4 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 08:21:55 +0100 Subject: [PATCH 08/25] fix(table): address refs-only expiration review (cli.md) --- website/src/cli.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/src/cli.md b/website/src/cli.md index ad1b32bad6..543ba19adf 100644 --- a/website/src/cli.md +++ b/website/src/cli.md @@ -217,6 +217,8 @@ iceberg expire-snapshots --older-than 7d --dry-run my_db.events iceberg expire-snapshots --older-than 7d --retain-last 5 my_db.events ``` +For REST catalogs, the `iceberg expire-snapshots` command requires complete snapshot history. Configure `snapshot-loading-mode=all` before running it; `snapshot-loading-mode=refs` omits history needed to enforce retention and causes expiration to fail. + ### `clean-orphan-files` Remove data files in the table location that are not referenced by any snapshot's manifests (e.g. left behind by failed writes). From 9c5329ca7a7cc138d59515fc9746a1b8866bd2d3 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 11:21:49 +0100 Subject: [PATCH 09/25] docs(cli): clarify REST server controls snapshot-loading-mode --- website/src/cli.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/src/cli.md b/website/src/cli.md index 543ba19adf..db6d1cd51f 100644 --- a/website/src/cli.md +++ b/website/src/cli.md @@ -217,7 +217,7 @@ iceberg expire-snapshots --older-than 7d --dry-run my_db.events iceberg expire-snapshots --older-than 7d --retain-last 5 my_db.events ``` -For REST catalogs, the `iceberg expire-snapshots` command requires complete snapshot history. Configure `snapshot-loading-mode=all` before running it; `snapshot-loading-mode=refs` omits history needed to enforce retention and causes expiration to fail. +`iceberg expire-snapshots` fails when the REST server sets `snapshot-loading-mode=refs`; the CLI cannot override it. Configure the REST server to use `snapshot-loading-mode=all` before running the command. ### `clean-orphan-files` @@ -311,9 +311,9 @@ This can discard the branch's current history from the branch head. The prompt shows both the previous and next snapshot IDs. Retention settings are preserved. Use `--yes` to skip confirmation. -For both commands, REST catalogs using `snapshot-loading-mode=refs` may omit -historical snapshots needed for ancestry checks or replacement. Use -`snapshot-loading-mode=all` for these operations. +For both commands, the REST server's `snapshot-loading-mode=refs` setting may omit +historical snapshots needed for ancestry checks or replacement. The CLI cannot +override this setting; configure the REST server with `snapshot-loading-mode=all`. ### `tag create` From 1ea601688ef94cddeb533a27155756632d864bc2 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 14:36:07 +0100 Subject: [PATCH 10/25] test(table): keep transaction usable after refs-only expiration rejection --- table/expire_replay_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index ad756bb8be..1197a63605 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -201,6 +201,30 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t *testing.T) { }) } +// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a +// history-dependent operation does not poison an otherwise valid transaction. +// Even a no-op expiration is unsafe to plan from refs-only metadata. +func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) { + fs, cat, tbl := newExpireReplayTable(t, nil) + tbl.snapshotHistoryRefsOnly = true + tx := tbl.NewTransaction() + + err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false)) + require.ErrorIs(t, err, ErrInvalidOperation) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + staged, err := tx.StagedTable() + require.NoError(t, err) + require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration must not stage updates") + + stageFiles(t, tx, "after-rejected-expiration") + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Len(t, committed.Metadata().Snapshots(), 1) + require.Equal(t, []string{"after-rejected-expiration"}, + branchFiles(t, fs, committed.Metadata(), MainBranch)) + require.Equal(t, int32(1), cat.attempts.Load(), "append should commit once") +} + func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) { synctest.Test(t, func(t *testing.T) { fs, inner, tbl := newExpireReplayTable(t, nil) From 7122a4c53b50abe960bb577399711aa32616d25b Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 14:47:53 +0100 Subject: [PATCH 11/25] docs(table): spell out expiration errors and recovery under REST refs mode --- website/src/api.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/website/src/api.md b/website/src/api.md index 2505da5ccf..01411e7150 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -853,7 +853,20 @@ if err != nil { /* ... */ } newTbl, err := txn.Commit(ctx) ``` -Snapshot expiration requires complete snapshot history. REST catalogs using `snapshot-loading-mode=refs` cannot stage `Transaction.ExpireSnapshots`; use `snapshot-loading-mode=all` before expiring snapshots. If a retry or resumed commit reloads refs-only history, it returns `table.ErrTransactionUnusable`. Reload with all snapshots and start a new transaction. +Snapshot expiration needs complete ancestry to honor branch retention and snapshot +references. A table loaded by REST with `snapshot-loading-mode=refs` may contain +only directly referenced snapshots. Calling `Transaction.ExpireSnapshots` on +such a table fails before staging any updates; `errors.Is(err, +table.ErrInvalidOperation)` returns true. The rejected operation does not +prevent other operations, such as appends, from using the transaction. + +If a concurrent writer forces a commit retry or a second `Commit` attempt +to reload refs-only history, the expiration cannot be replayed safely. +`Transaction.Commit` then returns `table.ErrTransactionUnusable` and +does not resubmit the expiration; it leaves the catalog's existing snapshots +and tags untouched. Configure the REST server to use +`snapshot-loading-mode=all`, reload the table, and start a **new** +transaction to retry expiration. The CLI cannot override the server mode. Pass `table.WithPostCommit(true)` to delete the unreferenced data and metadata files after the commit lands. The `iceberg expire-snapshots` CLI command wraps the same operation - see [CLI](./cli.md). From a352d5daa5f0295f7f1a8e22c0a7b6365a7e5f9a Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 14:47:56 +0100 Subject: [PATCH 12/25] docs(table): explain expiration refs-only error contracts --- table/transaction.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/table/transaction.go b/table/transaction.go index 8976fcc86f..2f4e63d02a 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -804,10 +804,13 @@ func getRetentionInt64(props iceberg.Properties, standardKey, legacyKey string, // committing. A transaction containing only expiries with no updates makes no // commit attempt and never refreshes or replans. // -// Snapshot expiration requires complete history. REST catalogs using -// snapshot-loading-mode=refs cannot stage it; load the table with -// snapshot-loading-mode=all. A retry or resumed commit refreshing to refs-only -// history makes the transaction unusable. +// Snapshot expiration requires complete history, even if the proposed +// expiration would be a no-op. REST snapshot-loading-mode=refs can omit +// ancestors needed for retention; staging on such a table fails with +// ErrInvalidOperation before recording updates. Other operations may still +// be staged in that transaction. A retry or resumed commit that refreshes +// to refs-only history fails with ErrTransactionUnusable without replaying +// the expiration. Reload with snapshot-loading-mode=all and begin anew. // // The replay failures that make the transaction unusable are listed on // ErrTransactionUnusable. From 069c7f86d2d5981a6955d2cd19c73729decd128a Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 14:47:58 +0100 Subject: [PATCH 13/25] docs(table): explain expiration replay history detection --- table/staged_replay.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/table/staged_replay.go b/table/staged_replay.go index b6742b2761..5938ffacb6 100644 --- a/table/staged_replay.go +++ b/table/staged_replay.go @@ -52,6 +52,9 @@ func newStagedReplay(branch string, updates []Update, records []expirationRecord } } +// A staged expiration may have yielded no metadata updates but can produce +// removals after a concurrent commit. Check the recorded operations, not the +// length of staged updates, before replaying against incomplete history. func (r *stagedReplay) hasExpirationRecords() bool { return r != nil && len(r.records) > 0 } From 75702f54810f32720e15868b6201a789e8cb9958 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 14:48:01 +0100 Subject: [PATCH 14/25] docs(table): explain incomplete-history error sentinel --- table/expire_replay.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/table/expire_replay.go b/table/expire_replay.go index cdb3f38007..08f9d6f6b9 100644 --- a/table/expire_replay.go +++ b/table/expire_replay.go @@ -23,6 +23,9 @@ import ( "slices" ) +// Preserve a distinct cause for both staging and retry failures so callers +// can match the public ErrInvalidOperation or ErrTransactionUnusable wrapper +// without relying on the human-readable REST configuration hint. var errSnapshotHistoryIncompleteForExpiration = errors.New( "cannot expire snapshots with incomplete history under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", ) From 46f97e3b4c38dd1ef018753a8ef9e781ce9d47a4 Mon Sep 17 00:00:00 2001 From: Vu Date: Tue, 17 Nov 2026 15:54:03 +0100 Subject: [PATCH 15/25] docs: preserve upstream release notes alongside PR changes Signed-off-by: Vu --- website/src/releases.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/src/releases.md b/website/src/releases.md index 359cc0ec04..3a26ffd70c 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -42,6 +42,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on ### Unreleased compatibility notes +- **`RowDelta.Commit` can be called once per builder.** A second `Commit` on the same `RowDelta` now returns an error wrapping `table.ErrInvalidOperation` and stages nothing, even if the first call failed, as `RewriteFiles.Commit` does. Previously, calling `Commit` again after a failed first call could succeed, and calling it again after a successful one staged the builder's files a second time. Create a new builder with `Transaction.NewRowDelta` for each commit. - **Data-file replacements now reject concurrent deletes.** Commits staged through `ReplaceDataFiles`, `ReplaceDataFilesByPath`, `ReplaceFiles`, or rewrite operations can return `table.ErrConflictingDeleteFiles` when a concurrent position delete targets a replaced file or any concurrent equality delete is added. Compaction on tables with streaming equality deletes may see additional conflicts. - **Orphan cleanup now rejects `gc.enabled=false`.** `DeleteOrphanFiles`, `PlanOrphanFiles`, `ExecuteOrphanCleanup`, and dry runs now return an error before filesystem access. The `iceberg clean-orphan-files` command is also affected. - **Partition specs reject source types a transform cannot handle.** Creating or updating a spec with an incompatible source, such as `day(string)`, now fails. Writes to existing incompatible specs return `ErrInvalidTransform` instead of writing null partition values. @@ -72,11 +73,13 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **`LocationProvider.NewDataLocation` takes the partition path.** The method is now `NewDataLocation(dataFileName, partitionPath string)`, matching Iceberg Java and PyIceberg, so custom `LocationProvider` implementations must add the argument. Callers pass the partition path there, or `""` for an unpartitioned file, instead of prefixing the file name. - **Partitioned data files follow the table's location provider.** With `write.object-storage.enabled=true` and the default `write.object-storage.partitioned-paths=true`, new partitioned files land under `data///` instead of `data///`, which can affect prefix-based lifecycle rules and bucket policies. `write.object-storage.partitioned-paths=false` and `write.data.path` now apply to partitioned writes. Partitioned paths also no longer percent-encode the table location: `s3://bucket/my table` stays literal instead of becoming `my%20table`, as for unpartitioned writes. Existing files stay where they are. - **External object-store data paths include table context.** For object-storage tables whose `write.data.path` is outside the table location, new data files now use `///...` after the shared storage prefix. This applies whether `write.object-storage.partitioned-paths` is `true` or `false`. The new prefixes may affect prefix-based lifecycle rules and bucket policies. Existing data files do not move. +- **SQL catalog nested namespaces match the JDBC catalog.** `sql.Catalog.ListNamespaces(ctx, parent)` returns only the direct children of `parent`, no longer `parent` itself or deeper descendants, and `ListNamespaces(ctx, nil)` returns the distinct first levels. A namespace that has no row of its own but has child namespaces, such as `a` after `CreateNamespace(a.b)`, now exists for listing, `CheckNamespaceExists`, `LoadNamespaceProperties`, `CreateTable`, `CreateView`, `CommitTable` creating a table with `AssertCreate`, and as the target of `RenameTable` and `RenameView`; only `CreateNamespace` and `UpdateNamespaceProperties` still need its own row. `DropNamespace` returns `catalog.ErrNamespaceNotEmpty` while child namespaces exist. Walk the hierarchy one level at a time, and drop children before their parent. - **Catalog capabilities split into narrower interfaces.** `catalog.Catalog` now embeds `TableCatalog` and `NamespaceCatalog` with unchanged table and namespace methods. `Name() string` is now required, breaking third-party catalogs and test fakes. `ViewCatalog` is optional (REST, Hive, and SQL), while REST, SQL, and Hive implement `UpdatableViewCatalog` for `UpdateView`. REST and SQL implement `RenamableViewCatalog` for `RenameView`. Use `catalog.As[T](cat)` and the `Unwrap() Catalog` convention to discover optional capabilities through decorators. Future capabilities should use optional interfaces rather than extending `Catalog`. - **Rename IO methods take a context.** `RenameIO.Rename` and `RenameNoReplaceIO.RenameNoReplace` now take `context.Context` first, so custom implementations and callers must update. - **`ReplaceFilesWithDeleteFiles` removed.** For delete-file replacements, use `tx.NewRewrite(snapshotProps)`, call `DeleteFile` for each source file and `AddDeleteFileWithDataSequenceNumber` for each replacement, then commit the builder. Rewrites record a `replace` snapshot instead of `overwrite`. The five-argument `ReplaceFiles` API is unchanged and remains available for compaction and delete-file removals. - **Manifest and deletion-vector reads take a context.** `Snapshot.Manifests`, `ManifestFile.Entries`, `iceberg.EntriesWithProjection`, `dv.ReadDV` and `dv.ReadDVs` now take a `context.Context` first argument and pass it to `IO.Open`. Custom `ManifestFile` implementations must add it to `Entries`. - **Path-based data-file addition renamed.** Replace `Transaction.AddFiles` with `AddDataFilesByPath`, `AddFilesOption` with `AddDataFilesByPathOption`, and `WithAddFilesConcurrency` with `WithAddDataFilesByPathConcurrency`. Behavior is unchanged. +- **Directory IO methods take a context.** `RemoveAllIO.RemoveAll` and `MkdirAllIO.MkdirAll` now take `context.Context` first, so custom implementations and callers must update. - **Manifest builder constructor renamed.** Replace `iceberg.NewManifestFile(...)` with `iceberg.NewManifestBuilder(...)`. It still returns a `ManifestBuilder`. - **Manifest sequence APIs use explicit names.** `ManifestBuilder.SequenceNum` and `ManifestFile.SequenceNum` become `SequenceNumber`, and `ManifestFile.MinSequenceNum` becomes `MinSequenceNumber`. `ManifestEntryBuilder.SequenceNum` and `ManifestEntry.SequenceNum` become `DataSequenceNumber`. `FileSequenceNum` becomes `FileSequenceNumber` for entries and their builder. Avro and JSON field names are unchanged. - **`io.ReadFileIO.ReadFile` accepts a context.** The method now takes a `context.Context` before the path, so custom `ReadFileIO` implementations and callers must pass the operation context. @@ -92,6 +95,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. - **Retries renew snapshot timestamps.** A retry or resumed `Commit` stamps each snapshot it replays from a staged write with the current time, at least one millisecond after its parent, as Java's `SnapshotProducer` does. A snapshot staged more than a minute before a peer's commit no longer fails the retry. The committed timestamp is the time of the attempt that lands, so an `ExpireSnapshots` staged after the write in the same transaction keeps that snapshot when the commit retries. - **Snapshot expiration now tolerates concurrent ref changes.** If a concurrent commit moves a ref that the expiry read, a retry or later `Commit` replans instead of failing with `table.ErrCommitFailed`. The committed expiry can differ from `Transaction.StagedTable`. See [Snapshot expiration](./concurrent-writes.md#snapshot-expiration) for cases that now fail with `table.ErrTransactionUnusable`. +- **RowDelta reports a concurrent deletion vector as a commit conflict.** If a concurrent commit adds a deletion vector for a data file that a `RowDelta` also adds one for, `Transaction.Commit` now fails with an error matching `table.ErrConflictingDeleteFiles`, `table.ErrCommitFailed` and `table.ErrTransactionUnusable`, and the transaction cannot be committed again. Previously, the error matched none of them and the transaction stayed usable. To resolve the conflict, reload the table and commit a `RowDelta` that adds one deletion vector merging the positions of both vectors and removes the concurrent one with `RemoveDeletes`. - **Schema JSON constructor uses the Go initialism.** Replace `iceberg.NewSchemaFromJsonFields` with `iceberg.NewSchemaFromJSONFields`. - **AWS identifiers use the Go initialism.** `glue.AwsProperties`, `glue.WithAwsConfig` and `glue.WithAwsProperties` become `glue.AWSProperties`, `glue.WithAWSConfig` and `glue.WithAWSProperties`; `sigv4.WithAwsConfig` becomes `sigv4.WithAWSConfig`; `utils.WithAwsConfig` and `utils.GetAwsConfig` become `utils.WithAWSConfig` and `utils.GetAWSConfig`; and `config.CatalogConfig.AwsProfile` becomes `AWSProfile`. The `aws-profile` YAML key is unchanged. - **KMS factory registration now takes a context.** Change `encryption.KMSFactory` implementations to accept `(context.Context, map[string]string)` instead of only properties, and call `encryption.LoadKeyManagementClient(ctx, props)` instead of `LoadKeyManagementClient(props)`. The existing global `RegisterKMS`, `UnregisterKMS`, and `GetRegisteredKMSNames` functions still use the default registry; `RegisterKMS` still panics for invalid registrations. Use `encryption.MustRegisterKMS` for provider initialization. For isolated registrations, create `registry := encryption.NewKMSRegistry()` after importing any KMS providers and call `registry.Register`, `registry.Load(ctx, props)`, `registry.Names`, or `registry.Unregister`. New registries snapshot the global registrations; subsequent changes do not propagate. From 36f2e225571e5fbce70ea23c3795261838a3899f Mon Sep 17 00:00:00 2001 From: Vu Date: Wed, 18 Nov 2026 05:07:55 +0100 Subject: [PATCH 16/25] test(table): cover no-op expiration retries and consolidate refs-only replay tests --- table/expire_replay_test.go | 123 ++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index 1197a63605..2f31c1cf13 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -208,26 +208,32 @@ func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) { fs, cat, tbl := newExpireReplayTable(t, nil) tbl.snapshotHistoryRefsOnly = true tx := tbl.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{"before-expiration": "kept"})) + updatesBefore := slices.Clone(tx.meta.updates) + reqsBefore := slices.Clone(tx.reqs) + expirationsBefore := len(tx.expirations) err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false)) require.ErrorIs(t, err, ErrInvalidOperation) require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) - staged, err := tx.StagedTable() - require.NoError(t, err) - require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration must not stage updates") + require.Equal(t, updatesBefore, tx.meta.updates, "rejected expiration must not add updates") + require.Equal(t, reqsBefore, tx.reqs, "rejected expiration must not add requirements") + require.Len(t, tx.expirations, expirationsBefore, "rejected expiration must not add replay records") stageFiles(t, tx, "after-rejected-expiration") committed, err := tx.Commit(t.Context()) require.NoError(t, err) require.Len(t, committed.Metadata().Snapshots(), 1) + require.Equal(t, "kept", committed.Properties()["before-expiration"]) require.Equal(t, []string{"after-rejected-expiration"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) require.Equal(t, int32(1), cat.attempts.Load(), "append should commit once") } -func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - fs, inner, tbl := newExpireReplayTable(t, nil) +func TestExpireReplayRejectsRefsOnlyHistoryOnRetryOrResume(t *testing.T) { + stage := func(t *testing.T, numRetries string) (*reqValidatingCatalog, *refsOnlyReloadCatalog, *Transaction, [4]int64, int64) { + t.Helper() + fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: numRetries}) var ids [4]int64 for i, name := range []string{"s1", "s2", "s3", "s4"} { tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name) @@ -242,65 +248,86 @@ func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) { )) staged, err := tx.StagedTable() require.NoError(t, err) - assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) - assert.Equal(t, sortedIDs(ids[:]...), survivors(staged.Metadata())) + require.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) + require.Equal(t, sortedIDs(ids[:]...), survivors(staged.Metadata())) peer := peerAppend(t, fs, inner, MainBranch, "s5") - partial := refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) - assert.Equal(t, sortedIDs(ids[0], peer), survivors(partial)) - cat.partial = partial - attemptsBefore := inner.attempts.Load() + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) + require.Equal(t, sortedIDs(ids[0], peer), survivors(cat.partial)) - _, err = tx.Commit(t.Context()) - require.ErrorIs(t, err, ErrTransactionUnusable) - require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) - assert.Equal(t, 1, cat.loads) - assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "retry must stop before resubmitting expiration") - assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) - assert.NotNil(t, inner.metadata.SnapshotByID(ids[0]), "retained history must remain in catalog metadata") - assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag"), "failed expiry must not remove the tag") - assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) - }) + return inner, cat, tx, ids, peer + } + + for _, tt := range []struct { + name string + numRetries string + resume bool + }{ + {name: "retry", numRetries: "3"}, + {name: "resume", numRetries: "0", resume: true}, + } { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + inner, cat, tx, ids, peer := stage(t, tt.numRetries) + attemptsBefore := inner.attempts.Load() + + if tt.resume { + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrCommitFailed) + require.NotErrorIs(t, err, ErrTransactionUnusable) + require.Zero(t, cat.loads, "the first failed attempt must not refresh") + require.Equal(t, attemptsBefore+1, inner.attempts.Load()) + } + + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrTransactionUnusable) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + require.Equal(t, 1, cat.loads) + require.Equal(t, attemptsBefore+1, inner.attempts.Load(), + "refresh must stop before resubmitting expiration") + require.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) + require.NotNil(t, inner.metadata.SnapshotByID(ids[0]), + "retained history must remain in catalog metadata") + require.NotNil(t, inner.metadata.SnapshotByName("expired-tag"), + "failed expiration must not remove the tag") + require.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + }) + }) + } } -func TestExpireReplayRejectsRefsOnlyHistoryOnResume(t *testing.T) { +// A no-op expiration still needs complete history when a retry replans it. +// The absence of planned removals must not bypass the refs-only guard. +func TestExpireReplayRejectsRefsOnlyHistoryWithNoOpExpiration(t *testing.T) { synctest.Test(t, func(t *testing.T) { - fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: "0"}) - var ids [4]int64 - for i, name := range []string{"s1", "s2", "s3", "s4"} { - tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name) - } - peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag", ids[0], TagRef, time.Hour.Milliseconds(), 0, 0)) - time.Sleep(2 * time.Hour) + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, s1 := appendOnBranch(t, tbl, MainBranch, "s1") + tbl, s2 := appendOnBranch(t, tbl, MainBranch, "s2") cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() require.NoError(t, tx.ExpireSnapshots( WithOlderThan(0), WithRetainLast(5), WithPostCommit(false), )) - staged, err := tx.StagedTable() - require.NoError(t, err) - assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) + require.Len(t, tx.expirations, 1, "no-op expiration must still be recorded") + require.Zero(t, tx.expirations[0].count, "expiration must stage no updates") + stageFiles(t, tx, "ours") - peer := peerAppend(t, fs, inner, MainBranch, "s5") - cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + require.Equal(t, []int64{peer}, survivors(cat.partial)) attemptsBefore := inner.attempts.Load() - _, err = tx.Commit(t.Context()) - require.ErrorIs(t, err, ErrCommitFailed) - require.NotErrorIs(t, err, ErrTransactionUnusable) - assert.Equal(t, 0, cat.loads, "first attempt must not refresh") - assert.Equal(t, attemptsBefore+1, inner.attempts.Load()) - - _, err = tx.Commit(t.Context()) + _, err := tx.Commit(t.Context()) require.ErrorIs(t, err, ErrTransactionUnusable) require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) - assert.Equal(t, 1, cat.loads) - assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "resume must stop before resubmitting expiration") - assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) - assert.NotNil(t, inner.metadata.SnapshotByID(ids[0])) - assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag")) - assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + require.Equal(t, 1, cat.loads) + require.Equal(t, attemptsBefore+1, inner.attempts.Load(), + "failed retry must not resubmit the append or expiration") + require.Equal(t, sortedIDs(s1, s2, peer), survivors(inner.metadata)) + require.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + require.Equal(t, []string{"peer", "s1", "s2"}, + branchFiles(t, fs, inner.metadata, MainBranch)) }) } From 7ce9c444b3b453a47ab78ccada3ab82bb38617bc Mon Sep 17 00:00:00 2001 From: Vu Date: Wed, 18 Nov 2026 05:07:58 +0100 Subject: [PATCH 17/25] docs(table): clarify client REST snapshot history recovery --- website/src/api.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/website/src/api.md b/website/src/api.md index 01411e7150..453a47c93e 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -856,17 +856,19 @@ newTbl, err := txn.Commit(ctx) Snapshot expiration needs complete ancestry to honor branch retention and snapshot references. A table loaded by REST with `snapshot-loading-mode=refs` may contain only directly referenced snapshots. Calling `Transaction.ExpireSnapshots` on -such a table fails before staging any updates; `errors.Is(err, -table.ErrInvalidOperation)` returns true. The rejected operation does not +such a table fails before staging any updates and matches +`table.ErrInvalidOperation` through `errors.Is`. The rejected operation does not prevent other operations, such as appends, from using the transaction. If a concurrent writer forces a commit retry or a second `Commit` attempt to reload refs-only history, the expiration cannot be replayed safely. `Transaction.Commit` then returns `table.ErrTransactionUnusable` and does not resubmit the expiration; it leaves the catalog's existing snapshots -and tags untouched. Configure the REST server to use -`snapshot-loading-mode=all`, reload the table, and start a **new** -transaction to retry expiration. The CLI cannot override the server mode. +and tags untouched. For library users, create the REST catalog with +`snapshot-loading-mode=all` (or omit that property), reload the table, +and start a **new transaction** to retry expiration. If the server forces +`refs` through configuration overrides, expiration is not available +through that catalog. The CLI cannot override the server mode. Pass `table.WithPostCommit(true)` to delete the unreferenced data and metadata files after the commit lands. The `iceberg expire-snapshots` CLI command wraps the same operation - see [CLI](./cli.md). From cfe43d6af09bda13efd5f3ecbd07987aa2a043f5 Mon Sep 17 00:00:00 2001 From: Vu Date: Wed, 18 Nov 2026 05:08:02 +0100 Subject: [PATCH 18/25] docs(releases): group snapshot expiration compatibility notes --- website/src/releases.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/src/releases.md b/website/src/releases.md index 3a26ffd70c..3dc984e712 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -95,6 +95,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. - **Retries renew snapshot timestamps.** A retry or resumed `Commit` stamps each snapshot it replays from a staged write with the current time, at least one millisecond after its parent, as Java's `SnapshotProducer` does. A snapshot staged more than a minute before a peer's commit no longer fails the retry. The committed timestamp is the time of the attempt that lands, so an `ExpireSnapshots` staged after the write in the same transaction keeps that snapshot when the commit retries. - **Snapshot expiration now tolerates concurrent ref changes.** If a concurrent commit moves a ref that the expiry read, a retry or later `Commit` replans instead of failing with `table.ErrCommitFailed`. The committed expiry can differ from `Transaction.StagedTable`. See [Snapshot expiration](./concurrent-writes.md#snapshot-expiration) for cases that now fail with `table.ErrTransactionUnusable`. +- **Snapshot expiration requires complete snapshot history.** Staging `Transaction.ExpireSnapshots` fails when REST loaded the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. - **RowDelta reports a concurrent deletion vector as a commit conflict.** If a concurrent commit adds a deletion vector for a data file that a `RowDelta` also adds one for, `Transaction.Commit` now fails with an error matching `table.ErrConflictingDeleteFiles`, `table.ErrCommitFailed` and `table.ErrTransactionUnusable`, and the transaction cannot be committed again. Previously, the error matched none of them and the transaction stayed usable. To resolve the conflict, reload the table and commit a `RowDelta` that adds one deletion vector merging the positions of both vectors and removes the concurrent one with `RemoveDeletes`. - **Schema JSON constructor uses the Go initialism.** Replace `iceberg.NewSchemaFromJsonFields` with `iceberg.NewSchemaFromJSONFields`. - **AWS identifiers use the Go initialism.** `glue.AwsProperties`, `glue.WithAwsConfig` and `glue.WithAwsProperties` become `glue.AWSProperties`, `glue.WithAWSConfig` and `glue.WithAWSProperties`; `sigv4.WithAwsConfig` becomes `sigv4.WithAWSConfig`; `utils.WithAwsConfig` and `utils.GetAwsConfig` become `utils.WithAWSConfig` and `utils.GetAWSConfig`; and `config.CatalogConfig.AwsProfile` becomes `AWSProfile`. The `aws-profile` YAML key is unchanged. @@ -143,8 +144,6 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Commit rejections now match `table.ErrCommitRejected`.** SQL, Glue, Hive and Hadoop failures while preparing metadata, plus REST commit responses 400, 401, 403 and 404, leave transactions committable (after a REST 404, the next `Commit` fails when it reloads the table). Existing `errors.Is` matches keep working, but SQL, Glue, Hive and Hadoop rejection messages now start with `commit rejected: `, so code that matches error text may need updating. - **Float comparisons against literals beyond the float32 range follow IEEE 754.** On a float32 column, `x < 1e100` no longer returns NaN, `+Inf` or NULL rows, and `NOT(x > 1e100)` no longer returns NULL rows. An infinite literal now binds to a float32 infinity, so `x = +Inf` matches `+Inf` rows. Java still folds these comparisons to always true or false, see [apache/iceberg#18700](https://github.com/apache/iceberg/issues/18700). -- **Snapshot expiration requires complete snapshot history.** Staging `Transaction.ExpireSnapshots` fails when REST loaded the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. - ## Verifying and producing releases If you are validating an RC or cutting a new release, see: From 7e956966ca6282f12da6554a7c2928a9d64f9f66 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 05:39:24 +0100 Subject: [PATCH 19/25] test(table): drop unused expiration result Signed-off-by: Minh Vu --- table/expire_replay_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index 2f31c1cf13..e62444e40a 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -302,7 +302,7 @@ func TestExpireReplayRejectsRefsOnlyHistoryWithNoOpExpiration(t *testing.T) { synctest.Test(t, func(t *testing.T) { fs, inner, tbl := newExpireReplayTable(t, nil) tbl, s1 := appendOnBranch(t, tbl, MainBranch, "s1") - tbl, s2 := appendOnBranch(t, tbl, MainBranch, "s2") + _, s2 := appendOnBranch(t, tbl, MainBranch, "s2") cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() From bc58b1cf3d5468355ada0d39817170140d36410e Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 06:03:34 +0100 Subject: [PATCH 20/25] docs: preserve upstream API examples and release notes Signed-off-by: Minh Vu --- website/src/api.md | 4 ++-- website/src/releases.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/website/src/api.md b/website/src/api.md index 453a47c93e..3c1d339fa2 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -55,7 +55,7 @@ import ( db, err := sql.Open(sqliteshim.ShimName, "file:catalog.db") // handle err -cat, err := sqlcat.NewCatalog("default", db, sqlcat.SQLite, iceberg.Properties{ +cat, err := sqlcat.NewCatalog(context.Background(), "default", db, sqlcat.SQLite, iceberg.Properties{ "warehouse": "file:///tmp/warehouse", }) ``` @@ -81,7 +81,7 @@ import ( "github.com/apache/iceberg-go/catalog/hive" ) -cat, err := hive.NewCatalog(iceberg.Properties{}, +cat, err := hive.NewCatalog(context.Background(), iceberg.Properties{}, hive.WithURI("thrift://localhost:9083"), hive.WithWarehouse("s3://my-bucket/warehouse")) ``` diff --git a/website/src/releases.md b/website/src/releases.md index 3dc984e712..215dadc88f 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -67,6 +67,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Schemas store binary and fixed defaults as hex strings.** `NewSchema`, `NewSchemaWithIdentifiers` and `UpdateSchema.AddColumn` (through `Apply()`) convert `[]byte`, `BinaryLiteral` and `FixedLiteral` defaults of binary and fixed fields, including fields nested in structs, lists and maps, to the lowercase hex string iceberg-go writes to metadata JSON, so a field read back from a schema holds a `string` such as `"0aff"`. Code that type-asserts these defaults to `[]byte` must decode the string instead, for example with `hex.DecodeString`. A schema built in process now equals the same schema read from metadata with lowercase hex. `Equals` compares the strings case-sensitively, and Java writes uppercase hex (see [apache/iceberg#18352](https://github.com/apache/iceberg/issues/18352)). The fields and types passed in are not modified. - **Predicates reject NaN literals when bound.** `EqualTo`, `NotEqualTo`, `LessThan`, `LessThanEqual`, `GreaterThan`, `GreaterThanEqual`, `IsIn`, `NotIn` and `LiteralPredicate`/`SetPredicate` with a NaN float literal now fail in `Bind` with `ErrInvalidArgument`, as Java does. That includes the string `"NaN"` on a float column and a NaN on a decimal column, which used to bind as zero. Scans, `ExpressionEvaluator` and the metrics evaluators all bind first, so they return the error too. Use `IsNaN` and `NotNaN` to match NaN values. - **Out-of-range nanosecond timestamp strings are rejected.** String literals for `timestamp_ns` and `timestamptz_ns` outside the signed 64-bit nanosecond range now fail expression binding with `ErrBadCast`. Scans and `ExpressionEvaluator` return the error for these filters instead of comparing a wrapped value. +- **`VisitBoundPredicateRef` is removed.** It was meant only for evaluators inside iceberg-go. Visitors in other modules should call `VisitBoundPredicate`, which passes `VisitIn` and `VisitNotIn` a detached copy of the predicate's literal set, as before. - **Empty string partition bounds are written as non-null bytes.** New manifests preserve empty string lower and upper bounds, fixing filtered scans on identity partitions. Manifests written before this fix may retain null bounds for an empty string and can continue to skip matching rows until rewritten. - **Name mapping rejects ambiguous aliases and duplicate field IDs.** Scans of files without field IDs now return `ErrInvalidSchema` when the same name belongs to multiple mapped fields at one level, instead of silently choosing the first. Repeated names on one mapping entry and same names under separate structs remain valid. Reusing a field ID returns `ErrInvalidSchema` instead of panicking. - **SQL view APIs align with `catalog.ViewCatalog`.** `CreateView` now takes a `*view.Version`, schema and options, and returns `*view.View`. `LoadView` now returns `*view.View`; use `Metadata()` to access its metadata. If the caller leaves `DefaultCatalog` empty, SQL still sets it to the catalog name. @@ -90,6 +91,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Snapshot timestamp property key corrected.** `PropertyCurrentSnapshotTimestamp` now uses `current-snapshot-timestamp-ms`. The old `current-snapshot-timestamp` spelling is no longer reserved and remains usable as a custom property. - **FileIO open and remove take a context.** `io.IO.Open` and `io.IO.Remove` now take a `context.Context` first argument, so custom `io.IO` implementations must add it. `maintenance.WithCleanupDeleteFunc` now takes `func(context.Context, string) error`. - **Hadoop catalog construction takes a context.** `hadoop.NewCatalog(name, warehouse, props)` becomes `hadoop.NewCatalog(ctx, name, warehouse, props)`. The context bounds filesystem setup; the catalog's filesystem keeps its values but not its cancellation. +- **SQL and Hive catalog construction takes a context.** `sql.NewCatalog(name, db, dialect, props)` becomes `sql.NewCatalog(ctx, name, db, dialect, props)`, and `hive.NewCatalog(props, opts...)` becomes `hive.NewCatalog(ctx, props, opts...)`. The context bounds the SQL table setup queries and is checked before the Hive metastore connect. - **Config loading reports errors.** `config.LoadConfig` now returns `([]byte, error)`, and `config.LoadConfigFile` is removed; call `LoadConfig` instead. A missing default `~/.iceberg-go.yaml` still returns no data and no error. - **Branch transactions return initialization errors.** `Table.NewTransactionOnBranch(branch)` now returns `(*Transaction, error)`, and `Table.NewTransactionOnBranchWithError` is removed. Use `NewTransactionOnBranch` and handle its error. - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. From d7e258d06dba8cfa3ad7a8e097853320434a0cbd Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 06:05:58 +0100 Subject: [PATCH 21/25] docs(table): separate refs-only expiration retry guidance Signed-off-by: Minh Vu --- table/table.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/table/table.go b/table/table.go index ef8eb41aad..c0c0174b54 100644 --- a/table/table.go +++ b/table/table.go @@ -94,13 +94,15 @@ var ErrCommitRejected = errors.New("commit rejected") // concurrent commit removed, a commit branch that became a tag or was // deleted, a replaced table, or a staged update that cannot apply after // refresh. With snapshot expiry, they also include disabled GC and replanning -// that changes whether a commit branch other than main is removed. An expiry -// replay also becomes unusable if a commit retry refreshes to refs-only -// snapshot history; load with `snapshot-loading-mode=all` and start a new -// transaction. Only the commit branch's head assertion is rebased. A staged -// update that cannot apply keeps the builder's error as its cause. -// Transaction.Commit also returns it, on any attempt, when a delete references -// data files that are missing (ErrDataFilesMissing). +// that changes whether a commit branch other than main is removed. Only the +// commit branch's head assertion is rebased. A staged update that cannot +// apply keeps the builder's error as its cause. Transaction.Commit also +// returns it, on any attempt, when a delete references data files that are +// missing (ErrDataFilesMissing). +// +// An expiry replay also becomes unusable if a commit retry refreshes to +// refs-only snapshot history. Load with snapshot-loading-mode=all and start +// a new transaction. var ErrTransactionUnusable = errors.New("transaction cannot be committed again") // ErrWriteIORequired is returned by write paths when the table's file system From 7ba98d89ea4d63b4ad0252a9e2f2d80e52b314a8 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 06:06:28 +0100 Subject: [PATCH 22/25] docs(table): keep expiration retry details with ExpireSnapshots Signed-off-by: Minh Vu --- table/table.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/table/table.go b/table/table.go index c0c0174b54..bec0f24f3a 100644 --- a/table/table.go +++ b/table/table.go @@ -99,10 +99,6 @@ var ErrCommitRejected = errors.New("commit rejected") // apply keeps the builder's error as its cause. Transaction.Commit also // returns it, on any attempt, when a delete references data files that are // missing (ErrDataFilesMissing). -// -// An expiry replay also becomes unusable if a commit retry refreshes to -// refs-only snapshot history. Load with snapshot-loading-mode=all and start -// a new transaction. var ErrTransactionUnusable = errors.New("transaction cannot be committed again") // ErrWriteIORequired is returned by write paths when the table's file system From 68248f00591bfca68193f3b4eb63a8c1a20a00fc Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 06:29:47 +0100 Subject: [PATCH 23/25] docs(release): mention CLI snapshot expiration guard Signed-off-by: Minh Vu --- website/src/releases.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/releases.md b/website/src/releases.md index 215dadc88f..757c40eee3 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -97,7 +97,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. - **Retries renew snapshot timestamps.** A retry or resumed `Commit` stamps each snapshot it replays from a staged write with the current time, at least one millisecond after its parent, as Java's `SnapshotProducer` does. A snapshot staged more than a minute before a peer's commit no longer fails the retry. The committed timestamp is the time of the attempt that lands, so an `ExpireSnapshots` staged after the write in the same transaction keeps that snapshot when the commit retries. - **Snapshot expiration now tolerates concurrent ref changes.** If a concurrent commit moves a ref that the expiry read, a retry or later `Commit` replans instead of failing with `table.ErrCommitFailed`. The committed expiry can differ from `Transaction.StagedTable`. See [Snapshot expiration](./concurrent-writes.md#snapshot-expiration) for cases that now fail with `table.ErrTransactionUnusable`. -- **Snapshot expiration requires complete snapshot history.** Staging `Transaction.ExpireSnapshots` fails when REST loaded the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. +- **Snapshot expiration requires complete snapshot history.** `Transaction.ExpireSnapshots` and `iceberg expire-snapshots` fail when REST loads the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. - **RowDelta reports a concurrent deletion vector as a commit conflict.** If a concurrent commit adds a deletion vector for a data file that a `RowDelta` also adds one for, `Transaction.Commit` now fails with an error matching `table.ErrConflictingDeleteFiles`, `table.ErrCommitFailed` and `table.ErrTransactionUnusable`, and the transaction cannot be committed again. Previously, the error matched none of them and the transaction stayed usable. To resolve the conflict, reload the table and commit a `RowDelta` that adds one deletion vector merging the positions of both vectors and removes the concurrent one with `RemoveDeletes`. - **Schema JSON constructor uses the Go initialism.** Replace `iceberg.NewSchemaFromJsonFields` with `iceberg.NewSchemaFromJSONFields`. - **AWS identifiers use the Go initialism.** `glue.AwsProperties`, `glue.WithAwsConfig` and `glue.WithAwsProperties` become `glue.AWSProperties`, `glue.WithAWSConfig` and `glue.WithAWSProperties`; `sigv4.WithAwsConfig` becomes `sigv4.WithAWSConfig`; `utils.WithAwsConfig` and `utils.GetAwsConfig` become `utils.WithAWSConfig` and `utils.GetAWSConfig`; and `config.CatalogConfig.AwsProfile` becomes `AWSProfile`. The `aws-profile` YAML key is unchanged. From 90fa7097be0696fddd41947a02cf908b382011f1 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 19:54:12 +0100 Subject: [PATCH 24/25] fix(table): load full REST history for snapshot expiration Signed-off-by: Minh Vu --- catalog/rest/rest.go | 10 ++ catalog/rest/rest_test.go | 74 +++++++++++++ cmd/iceberg/expire_snapshots.go | 12 ++- cmd/iceberg/expire_snapshots_test.go | 53 ++++++++++ cmd/iceberg/main.go | 6 +- table/expire_replay.go | 45 +++++++- table/expire_replay_test.go | 149 ++++++++++++++++++++++++++- table/table.go | 12 ++- table/transaction.go | 57 +++++----- website/src/api.md | 26 ++--- website/src/cli.md | 5 +- website/src/releases.md | 2 +- 12 files changed, 402 insertions(+), 49 deletions(-) diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go index e29e1ffc65..b8dca69b00 100644 --- a/catalog/rest/rest.go +++ b/catalog/rest/rest.go @@ -2050,6 +2050,16 @@ func (r *Catalog) LoadTable(ctx context.Context, identifier table.Identifier) (* return r.loadTableWithMode(ctx, identifier, r.snapshotMode) } +// LoadTableWithFullSnapshotHistory requests all valid snapshots for this load. +// The catalog's configured mode still applies to regular LoadTable calls. +func (r *Catalog) LoadTableWithFullSnapshotHistory(ctx context.Context, identifier table.Identifier) (*table.Table, error) { + if r.snapshotMode != SnapshotModeRefs { + return r.LoadTable(ctx, identifier) + } + + return r.loadTableWithMode(ctx, identifier, SnapshotModeAll) +} + func (r *Catalog) loadTableWithMode(ctx context.Context, identifier table.Identifier, mode SnapshotMode) (*table.Table, error) { if err := r.endpoints.check(endpointLoadTable); err != nil { return nil, err diff --git a/catalog/rest/rest_test.go b/catalog/rest/rest_test.go index a5820ccc79..bfb2499a18 100644 --- a/catalog/rest/rest_test.go +++ b/catalog/rest/rest_test.go @@ -29,7 +29,9 @@ import ( "runtime" "strconv" "strings" + "sync/atomic" "testing" + "time" "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/catalog" @@ -2258,6 +2260,78 @@ func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeAll() { r.EqualValues(3497810964824022504, tbl.CurrentSnapshot().SnapshotID) } +func (r *RestCatalogSuite) TestExpireSnapshotsRequestsFullHistoryOnDemand() { + ctx := r.T().Context() + const location = "s3://warehouse/db/table" + const metadataLocation = location + "/metadata/v2.metadata.json" + meta, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) + r.Require().NoError(err) + builder, err := table.MetadataBuilderFromBase(meta, "") + r.Require().NoError(err) + now := time.Now() + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 1, SequenceNumber: 1, TimestampMs: now.Add(-10 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: now.Add(-5 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) + full, err := builder.Build() + r.Require().NoError(err) + partialBuilder, err := table.MetadataBuilderFromBase(full, "") + r.Require().NoError(err) + r.Require().NoError(partialBuilder.RemoveSnapshots([]int64{1}, false)) + partial, err := partialBuilder.Build() + r.Require().NoError(err) + + var refsLoads, fullLoads atomic.Int32 + r.mux.HandleFunc("GET /v1/namespaces/db/tables/table", func(w http.ResponseWriter, req *http.Request) { + var metadata table.Metadata + switch req.URL.Query().Get("snapshots") { + case "refs": + refsLoads.Add(1) + metadata = partial + case "all": + fullLoads.Add(1) + metadata = full + default: + http.Error(w, "expected snapshots=refs or snapshots=all", http.StatusBadRequest) + + return + } + r.NoError(json.NewEncoder(w).Encode(map[string]any{ + "metadata-location": metadataLocation, + "metadata": metadata, + })) + }) + cat, err := rest.NewCatalog(ctx, "rest", r.srv.URL, rest.WithOAuthToken(TestToken), + rest.WithAdditionalProps(iceberg.Properties{"snapshot-loading-mode": "refs"})) + r.Require().NoError(err) + + ident := catalog.ToIdentifier("db", "table") + tbl, err := cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.Zero(fullLoads.Load()) + + tx := tbl.NewTransaction() + r.Require().NoError(tx.ExpireSnapshotsWithContext(ctx, table.WithOlderThan(0), table.WithRetainLast(1), table.WithPostCommit(false))) + commit, err := tx.TableCommit() + r.Require().NoError(err) + r.Len(commit.Updates, 1) + update, err := json.Marshal(commit.Updates[0]) + r.Require().NoError(err) + r.JSONEq(`{"action":"remove-snapshots","snapshot-ids":[1]}`, string(update)) + r.EqualValues(1, fullLoads.Load()) + + tbl, err = cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.EqualValues(2, refsLoads.Load()) +} + func (r *RestCatalogSuite) TestRenameTable409() { r.mux.HandleFunc("/v1/tables/rename", func(w http.ResponseWriter, req *http.Request) { if !r.Equal(http.MethodPost, req.Method) { diff --git a/cmd/iceberg/expire_snapshots.go b/cmd/iceberg/expire_snapshots.go index 197aa49cd6..aadbe44148 100644 --- a/cmd/iceberg/expire_snapshots.go +++ b/cmd/iceberg/expire_snapshots.go @@ -31,7 +31,13 @@ import ( ) func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, cmd *ExpireSnapshotsCmd) { - tbl := loadTable(ctx, output, cat, cmd.TableID) + // The preview compares the base and staged snapshot lists, so both + // must include history that an ordinary refs-only load can omit. + load := cat.LoadTable + if loader, ok := cat.(table.FullSnapshotHistoryLoader); ok { + load = loader.LoadTableWithFullSnapshotHistory + } + tbl := loadTableUsing(ctx, output, load, cmd.TableID) var opts []table.ExpireSnapshotsOpt @@ -52,7 +58,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, opts = append(opts, table.WithPostCommit(false)) tx := tbl.NewTransaction() - if err := tx.ExpireSnapshots(opts...); err != nil { + if err := tx.ExpireSnapshotsWithContext(ctx, opts...); err != nil { output.Error(fmt.Errorf("expire snapshots failed: %w", err)) osExit(1) } @@ -100,7 +106,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, } realTx := tbl.NewTransaction() - if err := realTx.ExpireSnapshots(realOpts...); err != nil { + if err := realTx.ExpireSnapshotsWithContext(ctx, realOpts...); err != nil { output.Error(fmt.Errorf("expire snapshots failed: %w", err)) osExit(1) } diff --git a/cmd/iceberg/expire_snapshots_test.go b/cmd/iceberg/expire_snapshots_test.go index 70f3d5f377..10490eaf8b 100644 --- a/cmd/iceberg/expire_snapshots_test.go +++ b/cmd/iceberg/expire_snapshots_test.go @@ -19,15 +19,68 @@ package main import ( "bytes" + "context" "os" "testing" + "time" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" "github.com/apache/iceberg-go/table" "github.com/pterm/pterm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type expirationPreviewCatalog struct { + catalog.Catalog + full *table.Table +} + +func (c expirationPreviewCatalog) LoadTableWithFullSnapshotHistory(context.Context, table.Identifier) (*table.Table, error) { + return c.full, nil +} + +type expirationPreviewOutput struct { + Output + result ExpireSnapshotsResult +} + +func (o *expirationPreviewOutput) ExpireSnapshotsResult(result ExpireSnapshotsResult) { + o.result = result +} + +func TestExpireSnapshotsDryRunPreviewsFullHistory(t *testing.T) { + location := t.TempDir() + meta, err := table.NewMetadata(iceberg.NewSchema(0), iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) + require.NoError(t, err) + builder, err := table.MetadataBuilderFromBase(meta, "") + require.NoError(t, err) + require.NoError(t, builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 1, SequenceNumber: 1, TimestampMs: time.Now().Add(-10 * time.Second).UnixMilli(), + ManifestList: location + "/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + require.NoError(t, builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: time.Now().Add(-5 * time.Second).UnixMilli(), + ManifestList: location + "/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + require.NoError(t, builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) + meta, err = builder.Build() + require.NoError(t, err) + + cat := expirationPreviewCatalog{ + full: table.New(table.Identifier{"db", "tbl"}, meta, location+"/metadata.json", nil, nil), + } + out := &expirationPreviewOutput{} + runExpireSnapshots(t.Context(), out, cat, &ExpireSnapshotsCmd{ + TableID: "db.tbl", DryRun: true, OlderThan: "0s", RetainLast: new(1), + }) + require.True(t, out.result.DryRun) + require.Equal(t, 1, out.result.ExpiredSnapshotCount) + require.Len(t, out.result.ExpiredSnapshots, 1) + require.Equal(t, int64(1), out.result.ExpiredSnapshots[0].SnapshotID) +} + func TestDiffSnapshots(t *testing.T) { before := []table.Snapshot{ {SnapshotID: 1, TimestampMs: 1000000000000, Summary: &table.Summary{Operation: table.OpAppend}}, diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index d27bc51eba..333b6782fc 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -875,7 +875,11 @@ func describe(ctx context.Context, output Output, cat catalog.Catalog, id string } func loadTable(ctx context.Context, output Output, cat catalog.Catalog, id string) *table.Table { - tbl, err := cat.LoadTable(ctx, catalog.ToIdentifier(id)) + return loadTableUsing(ctx, output, cat.LoadTable, id) +} + +func loadTableUsing(ctx context.Context, output Output, load func(context.Context, table.Identifier) (*table.Table, error), id string) *table.Table { + tbl, err := load(ctx, catalog.ToIdentifier(id)) if err != nil { output.Error(err) osExit(1) diff --git a/table/expire_replay.go b/table/expire_replay.go index 08f9d6f6b9..26f6cab434 100644 --- a/table/expire_replay.go +++ b/table/expire_replay.go @@ -18,6 +18,7 @@ package table import ( + "context" "errors" "fmt" "slices" @@ -27,9 +28,51 @@ import ( // can match the public ErrInvalidOperation or ErrTransactionUnusable wrapper // without relying on the human-readable REST configuration hint. var errSnapshotHistoryIncompleteForExpiration = errors.New( - "cannot expire snapshots with incomplete history under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", + "cannot expire snapshots: history may be incomplete under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", ) +// Load the same metadata revision with complete history, then rebuild the +// staged state over that base. Keep the original Table and builder intact +// until the expiration has been staged successfully. +func loadExpirationHistory(ctx context.Context, base *Table, staged *MetadataBuilder) (*Table, *MetadataBuilder, error) { + loader, ok := base.cat.(FullSnapshotHistoryLoader) + if !ok { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) + } + full, err := loader.LoadTableWithFullSnapshotHistory(ctx, slices.Clone(base.identifier)) + if err != nil { + return nil, nil, fmt.Errorf("%w: load full snapshot history: %w", ErrInvalidOperation, err) + } + if full.snapshotHistoryRefsOnly { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) + } + if err := validateTableUUID(base.identifier, base.metadata.TableUUID(), full.metadata.TableUUID()); err != nil { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, err) + } + if base.metadataLocation == "" || base.metadataLocation != full.metadataLocation { + return nil, nil, fmt.Errorf("%w: could not load full snapshot history for the transaction's metadata revision; reload the table and start a new transaction", + ErrInvalidOperation) + } + + meta, err := MetadataBuilderFromBase(full.metadata, base.metadataLocation) + if err != nil { + return nil, nil, err + } + for _, update := range staged.updates { + if err := update.Apply(meta); err != nil { + return nil, nil, fmt.Errorf("%w: apply staged %s with full snapshot history: %w", ErrInvalidOperation, update.Action(), err) + } + } + // Keep the same staged operations and their timestamp. Rebuilding can + // make an update redundant now that more of the base history is visible. + meta.updates = slices.Clone(staged.updates) + meta.lastUpdatedMS = staged.lastUpdatedMS + base = base.withMetadata(full.metadata, base.metadataLocation) + base.snapshotHistoryRefsOnly = false + + return base, meta, nil +} + func (r *stagedReplay) replan(b *MetadataBuilder, rec expirationRecord) ([]Update, []string, error) { updates, walked, err := planExpiration(b, rec.cfg, rec.nowMs) if err != nil { diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index e62444e40a..51fc6e8d7e 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -63,6 +63,21 @@ func (c *refsOnlyReloadCatalog) LoadTable(_ context.Context, ident Identifier) ( return New(ident, c.partial, "", nil, c, WithSnapshotHistoryLimitedToRefs(true)), nil } +type fullHistoryReloadCatalog struct { + *refsOnlyReloadCatalog + location string + fullLoads int +} + +func (c *fullHistoryReloadCatalog) LoadTableWithFullSnapshotHistory(ctx context.Context, ident Identifier) (*Table, error) { + c.fullLoads++ + if err := ctx.Err(); err != nil { + return nil, err + } + + return New(ident, c.metadata, c.location, nil, c), nil +} + func refsOnlySnapshotMetadata(t *testing.T, meta Metadata, keepIDs ...int64) Metadata { t.Helper() keep := make(map[int64]struct{}, len(keepIDs)) @@ -230,6 +245,135 @@ func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) { require.Equal(t, int32(1), cat.attempts.Load(), "append should commit once") } +func TestExpireSnapshotsLoadsFullHistoryWithStagedAppend(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, old := appendOnBranch(t, tbl, MainBranch, "old") + tbl, head := appendOnBranch(t, tbl, MainBranch, "head") + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{ + reqValidatingCatalog: inner, + partial: refsOnlySnapshotMetadata(t, inner.metadata, head), + }, + location: "mem://default/table-location/metadata/partial.json", + } + partial := New(tbl.Identifier(), cat.partial, cat.location, tbl.fsF, cat, + WithSnapshotHistoryLimitedToRefs(true)) + tx := partial.NewTransaction() + stageFiles(t, tx, "staged") + stagedID := stagedHead(t, tx, MainBranch) + time.Sleep(time.Second) + + opts := []ExpireSnapshotsOpt{WithOlderThan(0), WithRetainLast(2), WithPostCommit(false)} + require.NoError(t, tx.ExpireSnapshots(opts...)) + require.NoError(t, tx.ExpireSnapshotsWithContext(t.Context(), opts...)) + staged, err := tx.StagedTable() + require.NoError(t, err) + require.Equal(t, sortedIDs(head, stagedID), survivors(staged.Metadata())) + require.Equal(t, sortedIDs(old, head), survivors(tx.tbl.Metadata())) + require.Equal(t, []int64{head}, survivors(partial.Metadata())) + require.True(t, partial.snapshotHistoryRefsOnly) + require.Equal(t, 1, cat.fullLoads) + require.Zero(t, cat.loads) + + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, sortedIDs(head, stagedID), survivors(committed.Metadata())) + require.Equal(t, []string{"head", "old", "staged"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) +} + +func TestExpireSnapshotsFullHistoryLoadFailurePreservesStagedState(t *testing.T) { + for _, name := range []string{"metadata revision changed", "context canceled"} { + t.Run(name, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, head := appendOnBranch(t, tbl, MainBranch, "head") + partial := refsOnlySnapshotMetadata(t, inner.metadata, head) + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner, partial: partial}, + location: "mem://default/table-location/metadata/base.json", + } + original := New(tbl.Identifier(), partial, cat.location, tbl.fsF, cat, + WithSnapshotHistoryLimitedToRefs(true)) + tx := original.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{"staged": "kept"})) + before := tx.meta + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if name == "context canceled" { + cancel() + } else { + cat.location = "mem://default/table-location/metadata/replaced.json" + } + + err := tx.ExpireSnapshotsWithContext(ctx, WithPostCommit(false)) + require.ErrorIs(t, err, ErrInvalidOperation) + if name == "context canceled" { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorContains(t, err, "metadata revision") + } + require.Same(t, before, tx.meta) + require.True(t, tx.tbl.snapshotHistoryRefsOnly) + require.Empty(t, tx.expirations) + require.Equal(t, 1, cat.fullLoads) + + stageFiles(t, tx, "append") + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, "kept", committed.Properties()["staged"]) + require.Equal(t, []string{"append", "head"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) + } +} + +func TestExpireReplayLoadsFullHistoryForNoOpExpiration(t *testing.T) { + for _, tc := range []struct { + name, retries string + resume bool + }{ + {name: "retry", retries: "3"}, + {name: "resume", retries: "0", resume: true}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: tc.retries}) + tbl, first := appendOnBranch(t, tbl, MainBranch, "first") + _, head := appendOnBranch(t, tbl, MainBranch, "head") + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner}, + } + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + time.Sleep(time.Second) + require.NoError(t, tx.ExpireSnapshots(WithOlderThan(0), WithRetainLast(2), WithPostCommit(false))) + require.Len(t, tx.expirations, 1) + require.Zero(t, tx.expirations[0].count) + stageFiles(t, tx, "ours") + ours := stagedHead(t, tx, MainBranch) + + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + attemptsBefore := inner.attempts.Load() + if tc.resume { + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrCommitFailed) + require.Zero(t, cat.fullLoads) + } + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, attemptsBefore+2, inner.attempts.Load()) + require.Equal(t, 1, cat.fullLoads) + require.Zero(t, cat.loads) + require.Equal(t, sortedIDs(head, peer, ours), survivors(committed.Metadata())) + require.Nil(t, committed.Metadata().SnapshotByID(first)) + require.Equal(t, peer, parentOf(t, committed.Metadata(), ours)) + require.Equal(t, []string{"first", "head", "ours", "peer"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) + }) + } +} + func TestExpireReplayRejectsRefsOnlyHistoryOnRetryOrResume(t *testing.T) { stage := func(t *testing.T, numRetries string) (*reqValidatingCatalog, *refsOnlyReloadCatalog, *Transaction, [4]int64, int64) { t.Helper() @@ -336,7 +480,9 @@ func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) { fs, inner, tbl := newExpireReplayTable(t, nil) tbl, first := appendOnBranch(t, tbl, MainBranch, "first") appendOnBranch(t, tbl, MainBranch, "head") - cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner}, + } tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() stageFiles(t, tx, "ours") @@ -348,6 +494,7 @@ func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) { committed, err := tx.Commit(t.Context()) require.NoError(t, err) assert.Equal(t, 1, cat.loads) + assert.Zero(t, cat.fullLoads) assert.Equal(t, peer, parentOf(t, committed.Metadata(), refHead(t, committed.Metadata(), MainBranch))) }) } diff --git a/table/table.go b/table/table.go index 1108bc3496..6000809d11 100644 --- a/table/table.go +++ b/table/table.go @@ -142,6 +142,12 @@ type CatalogIO interface { CommitTable(context.Context, Identifier, []Requirement, []Update) (Metadata, string, error) } +// FullSnapshotHistoryLoader is an optional catalog capability for operations +// that need the complete snapshot history of a table. +type FullSnapshotHistoryLoader interface { + LoadTableWithFullSnapshotHistory(context.Context, Identifier) (*Table, error) +} + type Table struct { identifier Identifier metadata Metadata @@ -820,7 +826,11 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen // the new branch head so re-submission is not rejected // just because a peer advanced the head with a // non-conflicting commit. - fresh, refreshErr := t.cat.LoadTable(retryCtx, slices.Clone(t.identifier)) + load := t.cat.LoadTable + if loader, ok := t.cat.(FullSnapshotHistoryLoader); ok && co.replay.hasExpirationRecords() { + load = loader.LoadTableWithFullSnapshotHistory + } + fresh, refreshErr := load(retryCtx, slices.Clone(t.identifier)) if refreshErr != nil { return nil, fmt.Errorf("refresh table for retry: %w", refreshErr) } diff --git a/table/transaction.go b/table/transaction.go index 2f4e63d02a..f1c731a5c8 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -804,19 +804,24 @@ func getRetentionInt64(props iceberg.Properties, standardKey, legacyKey string, // committing. A transaction containing only expiries with no updates makes no // commit attempt and never refreshes or replans. // -// Snapshot expiration requires complete history, even if the proposed -// expiration would be a no-op. REST snapshot-loading-mode=refs can omit -// ancestors needed for retention; staging on such a table fails with -// ErrInvalidOperation before recording updates. Other operations may still -// be staged in that transaction. A retry or resumed commit that refreshes -// to refs-only history fails with ErrTransactionUnusable without replaying -// the expiration. Reload with snapshot-loading-mode=all and begin anew. +// Snapshot expiration requires complete history even when the plan is empty. +// On a refs-only REST table, expiration loads full history on demand for the +// transaction's metadata revision. Staging fails with ErrInvalidOperation +// if the catalog cannot supply that history or the table has advanced. +// Retries also request full history before replaying the expiration. +// Use [Transaction.ExpireSnapshotsWithContext] to control the staging request. // // The replay failures that make the transaction unusable are listed on // ErrTransactionUnusable. // // The "iceberg expire-snapshots" CLI command wraps the same operation. func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { + return t.ExpireSnapshotsWithContext(context.Background(), opts...) +} + +// ExpireSnapshotsWithContext stages expiration like [Transaction.ExpireSnapshots], +// using ctx to load complete snapshot history when the table was loaded refs-only. +func (t *Transaction) ExpireSnapshotsWithContext(ctx context.Context, opts ...ExpireSnapshotsOpt) error { var ( cfg = expireSnapshotsCfg{postCommit: true} nowMs = time.Now().UnixMilli() @@ -828,15 +833,28 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { if cfg.validationErr != nil { return cfg.validationErr } + if err := t.checkNotNil(); err != nil { + return err + } + t.mx.Lock() + defer t.mx.Unlock() + meta, err := t.txnMeta() if err != nil { return err } + if err := t.checkUsable(); err != nil { + return err + } if err := validateGCEnabledForSnapshotExpiration(meta.props); err != nil { return err } - if t.tbl.snapshotHistoryRefsOnly { - return fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) + base := t.tbl + if base.snapshotHistoryRefsOnly { + base, meta, err = loadExpirationHistory(ctx, base, meta) + if err != nil { + return err + } } updates, walked, err := planExpiration(meta, cfg, nowMs) @@ -844,30 +862,21 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { return err } - return t.stageExpiration(expirationRecord{ + rec := expirationRecord{ cfg: cfg, nowMs: nowMs, walked: walked, removedBranch: removesRef(updates, cmp.Or(t.branch, MainBranch)), - }, updates) -} - -// Record positions under the same lock as the updates they locate. -func (t *Transaction) stageExpiration(rec expirationRecord, updates []Update) error { - if err := t.checkNotNil(); err != nil { - return err - } - t.mx.Lock() - defer t.mx.Unlock() - - meta, err := t.txnMeta() - if err != nil { - return err } rec.pos = len(meta.updates) + original := t.meta + t.meta = meta if err := t.applyLocked(updates, nil); err != nil { + t.meta = original + return err } + t.tbl = base rec.count = len(t.meta.updates) - rec.pos t.expirations = append(t.expirations, rec) diff --git a/website/src/api.md b/website/src/api.md index 3c1d339fa2..d63d32d0db 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -854,21 +854,17 @@ newTbl, err := txn.Commit(ctx) ``` Snapshot expiration needs complete ancestry to honor branch retention and snapshot -references. A table loaded by REST with `snapshot-loading-mode=refs` may contain -only directly referenced snapshots. Calling `Transaction.ExpireSnapshots` on -such a table fails before staging any updates and matches -`table.ErrInvalidOperation` through `errors.Is`. The rejected operation does not -prevent other operations, such as appends, from using the transaction. - -If a concurrent writer forces a commit retry or a second `Commit` attempt -to reload refs-only history, the expiration cannot be replayed safely. -`Transaction.Commit` then returns `table.ErrTransactionUnusable` and -does not resubmit the expiration; it leaves the catalog's existing snapshots -and tags untouched. For library users, create the REST catalog with -`snapshot-loading-mode=all` (or omit that property), reload the table, -and start a **new transaction** to retry expiration. If the server forces -`refs` through configuration overrides, expiration is not available -through that catalog. The CLI cannot override the server mode. +references. When a REST table was loaded with `snapshot-loading-mode=refs`, +expiration requests `snapshots=all` for the transaction's metadata revision +and keeps any updates already staged in the transaction. Use +`Transaction.ExpireSnapshotsWithContext(ctx, ...)` to control that request. + +If the catalog cannot supply complete history or the metadata revision has +changed, staging returns an error matching `table.ErrInvalidOperation`. +Reload the table and start a new transaction if the revision changed. +Commit retries and resumed commits also request full history before replanning +expiration. If the catalog still returns refs-only history, the commit stops +before resubmitting and returns an error matching `table.ErrTransactionUnusable`. Pass `table.WithPostCommit(true)` to delete the unreferenced data and metadata files after the commit lands. The `iceberg expire-snapshots` CLI command wraps the same operation - see [CLI](./cli.md). diff --git a/website/src/cli.md b/website/src/cli.md index db6d1cd51f..637125c6e7 100644 --- a/website/src/cli.md +++ b/website/src/cli.md @@ -217,7 +217,8 @@ iceberg expire-snapshots --older-than 7d --dry-run my_db.events iceberg expire-snapshots --older-than 7d --retain-last 5 my_db.events ``` -`iceberg expire-snapshots` fails when the REST server sets `snapshot-loading-mode=refs`; the CLI cannot override it. Configure the REST server to use `snapshot-loading-mode=all` before running the command. +For REST catalogs, expiration and `--dry-run` load full snapshot history. +The catalog's snapshot loading mode still applies to other commands. ### `clean-orphan-files` @@ -373,4 +374,4 @@ Every write command has multiple layers of protection: - **`--yes`** — required to skip the prompt; without it, non-interactive shells get an explicit error rather than hanging. - **TTY detection** — interactive prompts are only shown when stdout is a terminal. - **Ancestor validation** — `rollback` rejects target snapshots that are not in the current branch's history. -- **Version check** — `upgrade` refuses same-version or downgrade requests. \ No newline at end of file +- **Version check** — `upgrade` refuses same-version or downgrade requests. diff --git a/website/src/releases.md b/website/src/releases.md index 0485fbf5af..b0593abb57 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -100,7 +100,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. - **Retries renew snapshot timestamps.** A retry or resumed `Commit` stamps each snapshot it replays from a staged write with the current time, at least one millisecond after its parent, as Java's `SnapshotProducer` does. A snapshot staged more than a minute before a peer's commit no longer fails the retry. The committed timestamp is the time of the attempt that lands, so an `ExpireSnapshots` staged after the write in the same transaction keeps that snapshot when the commit retries. - **Snapshot expiration now tolerates concurrent ref changes.** If a concurrent commit moves a ref that the expiry read, a retry or later `Commit` replans instead of failing with `table.ErrCommitFailed`. The committed expiry can differ from `Transaction.StagedTable`. See [Snapshot expiration](./concurrent-writes.md#snapshot-expiration) for cases that now fail with `table.ErrTransactionUnusable`. -- **Snapshot expiration requires complete snapshot history.** `Transaction.ExpireSnapshots` and `iceberg expire-snapshots` fail when REST loads the table with `snapshot-loading-mode=refs`. A commit retry can also return `table.ErrTransactionUnusable` if its refresh omits history before expiration replay. Reload with `snapshot-loading-mode=all` and start a new transaction. +- **Snapshot expiration loads full REST history on demand.** Expiration requests `snapshots=all` when a REST table was loaded in refs mode, including on commit retries. Use `Transaction.ExpireSnapshotsWithContext` to control the staging request. Catalogs that only supply refs-only history cannot safely expire snapshots. - **RowDelta reports a concurrent deletion vector as a commit conflict.** If a concurrent commit adds a deletion vector for a data file that a `RowDelta` also adds one for, `Transaction.Commit` now fails with an error matching `table.ErrConflictingDeleteFiles`, `table.ErrCommitFailed` and `table.ErrTransactionUnusable`, and the transaction cannot be committed again. Previously, the error matched none of them and the transaction stayed usable. To resolve the conflict, reload the table and commit a `RowDelta` that adds one deletion vector merging the positions of both vectors and removes the concurrent one with `RemoveDeletes`. - **Schema JSON constructor uses the Go initialism.** Replace `iceberg.NewSchemaFromJsonFields` with `iceberg.NewSchemaFromJSONFields`. - **AWS identifiers use the Go initialism.** `glue.AwsProperties`, `glue.WithAwsConfig` and `glue.WithAwsProperties` become `glue.AWSProperties`, `glue.WithAWSConfig` and `glue.WithAWSProperties`; `sigv4.WithAwsConfig` becomes `sigv4.WithAWSConfig`; `utils.WithAwsConfig` and `utils.GetAwsConfig` become `utils.WithAWSConfig` and `utils.GetAWSConfig`; and `config.CatalogConfig.AwsProfile` becomes `AWSProfile`. The `aws-profile` YAML key is unchanged. From 84417eb827a44a9c45c09d8a82951d1386cfb26d Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 18 Nov 2026 19:54:42 +0100 Subject: [PATCH 25/25] test(rest): keep expiration and orphan cleanup tests mergeable Signed-off-by: Minh Vu --- catalog/rest/rest_test.go | 144 +++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/catalog/rest/rest_test.go b/catalog/rest/rest_test.go index bfb2499a18..61f6e10c17 100644 --- a/catalog/rest/rest_test.go +++ b/catalog/rest/rest_test.go @@ -2133,6 +2133,78 @@ func (r *RestCatalogSuite) TestCreateTableLabels() { r.Equal(iceberg.Properties{"classification": "pii"}, labels.Field(2)) } +func (r *RestCatalogSuite) TestExpireSnapshotsRequestsFullHistoryOnDemand() { + ctx := r.T().Context() + const location = "s3://warehouse/db/table" + const metadataLocation = location + "/metadata/v2.metadata.json" + meta, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) + r.Require().NoError(err) + builder, err := table.MetadataBuilderFromBase(meta, "") + r.Require().NoError(err) + now := time.Now() + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 1, SequenceNumber: 1, TimestampMs: now.Add(-10 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: now.Add(-5 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) + full, err := builder.Build() + r.Require().NoError(err) + partialBuilder, err := table.MetadataBuilderFromBase(full, "") + r.Require().NoError(err) + r.Require().NoError(partialBuilder.RemoveSnapshots([]int64{1}, false)) + partial, err := partialBuilder.Build() + r.Require().NoError(err) + + var refsLoads, fullLoads atomic.Int32 + r.mux.HandleFunc("GET /v1/namespaces/db/tables/table", func(w http.ResponseWriter, req *http.Request) { + var metadata table.Metadata + switch req.URL.Query().Get("snapshots") { + case "refs": + refsLoads.Add(1) + metadata = partial + case "all": + fullLoads.Add(1) + metadata = full + default: + http.Error(w, "expected snapshots=refs or snapshots=all", http.StatusBadRequest) + + return + } + r.NoError(json.NewEncoder(w).Encode(map[string]any{ + "metadata-location": metadataLocation, + "metadata": metadata, + })) + }) + cat, err := rest.NewCatalog(ctx, "rest", r.srv.URL, rest.WithOAuthToken(TestToken), + rest.WithAdditionalProps(iceberg.Properties{"snapshot-loading-mode": "refs"})) + r.Require().NoError(err) + + ident := catalog.ToIdentifier("db", "table") + tbl, err := cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.Zero(fullLoads.Load()) + + tx := tbl.NewTransaction() + r.Require().NoError(tx.ExpireSnapshotsWithContext(ctx, table.WithOlderThan(0), table.WithRetainLast(1), table.WithPostCommit(false))) + commit, err := tx.TableCommit() + r.Require().NoError(err) + r.Len(commit.Updates, 1) + update, err := json.Marshal(commit.Updates[0]) + r.Require().NoError(err) + r.JSONEq(`{"action":"remove-snapshots","snapshot-ids":[1]}`, string(update)) + r.EqualValues(1, fullLoads.Load()) + + tbl, err = cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.EqualValues(2, refsLoads.Load()) +} + func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeRefs() { // Server returns only the snapshot referenced by the main branch (refs behaviour). r.mux.HandleFunc("/v1/namespaces/fokko/tables/table", func(w http.ResponseWriter, req *http.Request) { @@ -2260,78 +2332,6 @@ func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeAll() { r.EqualValues(3497810964824022504, tbl.CurrentSnapshot().SnapshotID) } -func (r *RestCatalogSuite) TestExpireSnapshotsRequestsFullHistoryOnDemand() { - ctx := r.T().Context() - const location = "s3://warehouse/db/table" - const metadataLocation = location + "/metadata/v2.metadata.json" - meta, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) - r.Require().NoError(err) - builder, err := table.MetadataBuilderFromBase(meta, "") - r.Require().NoError(err) - now := time.Now() - r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ - SnapshotID: 1, SequenceNumber: 1, TimestampMs: now.Add(-10 * time.Second).UnixMilli(), - ManifestList: location + "/metadata/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, - })) - r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ - SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: now.Add(-5 * time.Second).UnixMilli(), - ManifestList: location + "/metadata/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, - })) - r.Require().NoError(builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) - full, err := builder.Build() - r.Require().NoError(err) - partialBuilder, err := table.MetadataBuilderFromBase(full, "") - r.Require().NoError(err) - r.Require().NoError(partialBuilder.RemoveSnapshots([]int64{1}, false)) - partial, err := partialBuilder.Build() - r.Require().NoError(err) - - var refsLoads, fullLoads atomic.Int32 - r.mux.HandleFunc("GET /v1/namespaces/db/tables/table", func(w http.ResponseWriter, req *http.Request) { - var metadata table.Metadata - switch req.URL.Query().Get("snapshots") { - case "refs": - refsLoads.Add(1) - metadata = partial - case "all": - fullLoads.Add(1) - metadata = full - default: - http.Error(w, "expected snapshots=refs or snapshots=all", http.StatusBadRequest) - - return - } - r.NoError(json.NewEncoder(w).Encode(map[string]any{ - "metadata-location": metadataLocation, - "metadata": metadata, - })) - }) - cat, err := rest.NewCatalog(ctx, "rest", r.srv.URL, rest.WithOAuthToken(TestToken), - rest.WithAdditionalProps(iceberg.Properties{"snapshot-loading-mode": "refs"})) - r.Require().NoError(err) - - ident := catalog.ToIdentifier("db", "table") - tbl, err := cat.LoadTable(ctx, ident) - r.Require().NoError(err) - r.Len(tbl.Metadata().Snapshots(), 1) - r.Zero(fullLoads.Load()) - - tx := tbl.NewTransaction() - r.Require().NoError(tx.ExpireSnapshotsWithContext(ctx, table.WithOlderThan(0), table.WithRetainLast(1), table.WithPostCommit(false))) - commit, err := tx.TableCommit() - r.Require().NoError(err) - r.Len(commit.Updates, 1) - update, err := json.Marshal(commit.Updates[0]) - r.Require().NoError(err) - r.JSONEq(`{"action":"remove-snapshots","snapshot-ids":[1]}`, string(update)) - r.EqualValues(1, fullLoads.Load()) - - tbl, err = cat.LoadTable(ctx, ident) - r.Require().NoError(err) - r.Len(tbl.Metadata().Snapshots(), 1) - r.EqualValues(2, refsLoads.Load()) -} - func (r *RestCatalogSuite) TestRenameTable409() { r.mux.HandleFunc("/v1/tables/rename", func(w http.ResponseWriter, req *http.Request) { if !r.Equal(http.MethodPost, req.Method) {