diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go index e29e1ffc6..b8dca69b0 100644 --- a/catalog/rest/rest.go +++ b/catalog/rest/rest.go @@ -2050,6 +2050,16 @@ func (r *Catalog) LoadTable(ctx context.Context, identifier table.Identifier) (* return r.loadTableWithMode(ctx, identifier, r.snapshotMode) } +// LoadTableWithFullSnapshotHistory requests all valid snapshots for this load. +// The catalog's configured mode still applies to regular LoadTable calls. +func (r *Catalog) LoadTableWithFullSnapshotHistory(ctx context.Context, identifier table.Identifier) (*table.Table, error) { + if r.snapshotMode != SnapshotModeRefs { + return r.LoadTable(ctx, identifier) + } + + return r.loadTableWithMode(ctx, identifier, SnapshotModeAll) +} + func (r *Catalog) loadTableWithMode(ctx context.Context, identifier table.Identifier, mode SnapshotMode) (*table.Table, error) { if err := r.endpoints.check(endpointLoadTable); err != nil { return nil, err diff --git a/catalog/rest/rest_test.go b/catalog/rest/rest_test.go index a5820ccc7..61f6e10c1 100644 --- a/catalog/rest/rest_test.go +++ b/catalog/rest/rest_test.go @@ -29,7 +29,9 @@ import ( "runtime" "strconv" "strings" + "sync/atomic" "testing" + "time" "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/catalog" @@ -2131,6 +2133,78 @@ func (r *RestCatalogSuite) TestCreateTableLabels() { r.Equal(iceberg.Properties{"classification": "pii"}, labels.Field(2)) } +func (r *RestCatalogSuite) TestExpireSnapshotsRequestsFullHistoryOnDemand() { + ctx := r.T().Context() + const location = "s3://warehouse/db/table" + const metadataLocation = location + "/metadata/v2.metadata.json" + meta, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) + r.Require().NoError(err) + builder, err := table.MetadataBuilderFromBase(meta, "") + r.Require().NoError(err) + now := time.Now() + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 1, SequenceNumber: 1, TimestampMs: now.Add(-10 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: now.Add(-5 * time.Second).UnixMilli(), + ManifestList: location + "/metadata/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + r.Require().NoError(builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) + full, err := builder.Build() + r.Require().NoError(err) + partialBuilder, err := table.MetadataBuilderFromBase(full, "") + r.Require().NoError(err) + r.Require().NoError(partialBuilder.RemoveSnapshots([]int64{1}, false)) + partial, err := partialBuilder.Build() + r.Require().NoError(err) + + var refsLoads, fullLoads atomic.Int32 + r.mux.HandleFunc("GET /v1/namespaces/db/tables/table", func(w http.ResponseWriter, req *http.Request) { + var metadata table.Metadata + switch req.URL.Query().Get("snapshots") { + case "refs": + refsLoads.Add(1) + metadata = partial + case "all": + fullLoads.Add(1) + metadata = full + default: + http.Error(w, "expected snapshots=refs or snapshots=all", http.StatusBadRequest) + + return + } + r.NoError(json.NewEncoder(w).Encode(map[string]any{ + "metadata-location": metadataLocation, + "metadata": metadata, + })) + }) + cat, err := rest.NewCatalog(ctx, "rest", r.srv.URL, rest.WithOAuthToken(TestToken), + rest.WithAdditionalProps(iceberg.Properties{"snapshot-loading-mode": "refs"})) + r.Require().NoError(err) + + ident := catalog.ToIdentifier("db", "table") + tbl, err := cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.Zero(fullLoads.Load()) + + tx := tbl.NewTransaction() + r.Require().NoError(tx.ExpireSnapshotsWithContext(ctx, table.WithOlderThan(0), table.WithRetainLast(1), table.WithPostCommit(false))) + commit, err := tx.TableCommit() + r.Require().NoError(err) + r.Len(commit.Updates, 1) + update, err := json.Marshal(commit.Updates[0]) + r.Require().NoError(err) + r.JSONEq(`{"action":"remove-snapshots","snapshot-ids":[1]}`, string(update)) + r.EqualValues(1, fullLoads.Load()) + + tbl, err = cat.LoadTable(ctx, ident) + r.Require().NoError(err) + r.Len(tbl.Metadata().Snapshots(), 1) + r.EqualValues(2, refsLoads.Load()) +} + func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeRefs() { // Server returns only the snapshot referenced by the main branch (refs behaviour). r.mux.HandleFunc("/v1/namespaces/fokko/tables/table", func(w http.ResponseWriter, req *http.Request) { diff --git a/cmd/iceberg/expire_snapshots.go b/cmd/iceberg/expire_snapshots.go index 197aa49cd..aadbe4414 100644 --- a/cmd/iceberg/expire_snapshots.go +++ b/cmd/iceberg/expire_snapshots.go @@ -31,7 +31,13 @@ import ( ) func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, cmd *ExpireSnapshotsCmd) { - tbl := loadTable(ctx, output, cat, cmd.TableID) + // The preview compares the base and staged snapshot lists, so both + // must include history that an ordinary refs-only load can omit. + load := cat.LoadTable + if loader, ok := cat.(table.FullSnapshotHistoryLoader); ok { + load = loader.LoadTableWithFullSnapshotHistory + } + tbl := loadTableUsing(ctx, output, load, cmd.TableID) var opts []table.ExpireSnapshotsOpt @@ -52,7 +58,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, opts = append(opts, table.WithPostCommit(false)) tx := tbl.NewTransaction() - if err := tx.ExpireSnapshots(opts...); err != nil { + if err := tx.ExpireSnapshotsWithContext(ctx, opts...); err != nil { output.Error(fmt.Errorf("expire snapshots failed: %w", err)) osExit(1) } @@ -100,7 +106,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, } realTx := tbl.NewTransaction() - if err := realTx.ExpireSnapshots(realOpts...); err != nil { + if err := realTx.ExpireSnapshotsWithContext(ctx, realOpts...); err != nil { output.Error(fmt.Errorf("expire snapshots failed: %w", err)) osExit(1) } diff --git a/cmd/iceberg/expire_snapshots_test.go b/cmd/iceberg/expire_snapshots_test.go index 70f3d5f37..10490eaf8 100644 --- a/cmd/iceberg/expire_snapshots_test.go +++ b/cmd/iceberg/expire_snapshots_test.go @@ -19,15 +19,68 @@ package main import ( "bytes" + "context" "os" "testing" + "time" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" "github.com/apache/iceberg-go/table" "github.com/pterm/pterm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type expirationPreviewCatalog struct { + catalog.Catalog + full *table.Table +} + +func (c expirationPreviewCatalog) LoadTableWithFullSnapshotHistory(context.Context, table.Identifier) (*table.Table, error) { + return c.full, nil +} + +type expirationPreviewOutput struct { + Output + result ExpireSnapshotsResult +} + +func (o *expirationPreviewOutput) ExpireSnapshotsResult(result ExpireSnapshotsResult) { + o.result = result +} + +func TestExpireSnapshotsDryRunPreviewsFullHistory(t *testing.T) { + location := t.TempDir() + meta, err := table.NewMetadata(iceberg.NewSchema(0), iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil) + require.NoError(t, err) + builder, err := table.MetadataBuilderFromBase(meta, "") + require.NoError(t, err) + require.NoError(t, builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 1, SequenceNumber: 1, TimestampMs: time.Now().Add(-10 * time.Second).UnixMilli(), + ManifestList: location + "/old.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + require.NoError(t, builder.AddSnapshot(&table.Snapshot{ + SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: time.Now().Add(-5 * time.Second).UnixMilli(), + ManifestList: location + "/head.avro", Summary: &table.Summary{Operation: table.OpAppend}, + })) + require.NoError(t, builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef)) + meta, err = builder.Build() + require.NoError(t, err) + + cat := expirationPreviewCatalog{ + full: table.New(table.Identifier{"db", "tbl"}, meta, location+"/metadata.json", nil, nil), + } + out := &expirationPreviewOutput{} + runExpireSnapshots(t.Context(), out, cat, &ExpireSnapshotsCmd{ + TableID: "db.tbl", DryRun: true, OlderThan: "0s", RetainLast: new(1), + }) + require.True(t, out.result.DryRun) + require.Equal(t, 1, out.result.ExpiredSnapshotCount) + require.Len(t, out.result.ExpiredSnapshots, 1) + require.Equal(t, int64(1), out.result.ExpiredSnapshots[0].SnapshotID) +} + func TestDiffSnapshots(t *testing.T) { before := []table.Snapshot{ {SnapshotID: 1, TimestampMs: 1000000000000, Summary: &table.Summary{Operation: table.OpAppend}}, diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index d27bc51eb..333b6782f 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -875,7 +875,11 @@ func describe(ctx context.Context, output Output, cat catalog.Catalog, id string } func loadTable(ctx context.Context, output Output, cat catalog.Catalog, id string) *table.Table { - tbl, err := cat.LoadTable(ctx, catalog.ToIdentifier(id)) + return loadTableUsing(ctx, output, cat.LoadTable, id) +} + +func loadTableUsing(ctx context.Context, output Output, load func(context.Context, table.Identifier) (*table.Table, error), id string) *table.Table { + tbl, err := load(ctx, catalog.ToIdentifier(id)) if err != nil { output.Error(err) osExit(1) diff --git a/table/expire_replay.go b/table/expire_replay.go index 7a0a58946..26f6cab43 100644 --- a/table/expire_replay.go +++ b/table/expire_replay.go @@ -18,10 +18,61 @@ package table import ( + "context" + "errors" "fmt" "slices" ) +// Preserve a distinct cause for both staging and retry failures so callers +// can match the public ErrInvalidOperation or ErrTransactionUnusable wrapper +// without relying on the human-readable REST configuration hint. +var errSnapshotHistoryIncompleteForExpiration = errors.New( + "cannot expire snapshots: history may be incomplete under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", +) + +// Load the same metadata revision with complete history, then rebuild the +// staged state over that base. Keep the original Table and builder intact +// until the expiration has been staged successfully. +func loadExpirationHistory(ctx context.Context, base *Table, staged *MetadataBuilder) (*Table, *MetadataBuilder, error) { + loader, ok := base.cat.(FullSnapshotHistoryLoader) + if !ok { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) + } + full, err := loader.LoadTableWithFullSnapshotHistory(ctx, slices.Clone(base.identifier)) + if err != nil { + return nil, nil, fmt.Errorf("%w: load full snapshot history: %w", ErrInvalidOperation, err) + } + if full.snapshotHistoryRefsOnly { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration) + } + if err := validateTableUUID(base.identifier, base.metadata.TableUUID(), full.metadata.TableUUID()); err != nil { + return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, err) + } + if base.metadataLocation == "" || base.metadataLocation != full.metadataLocation { + return nil, nil, fmt.Errorf("%w: could not load full snapshot history for the transaction's metadata revision; reload the table and start a new transaction", + ErrInvalidOperation) + } + + meta, err := MetadataBuilderFromBase(full.metadata, base.metadataLocation) + if err != nil { + return nil, nil, err + } + for _, update := range staged.updates { + if err := update.Apply(meta); err != nil { + return nil, nil, fmt.Errorf("%w: apply staged %s with full snapshot history: %w", ErrInvalidOperation, update.Action(), err) + } + } + // Keep the same staged operations and their timestamp. Rebuilding can + // make an update redundant now that more of the base history is visible. + meta.updates = slices.Clone(staged.updates) + meta.lastUpdatedMS = staged.lastUpdatedMS + base = base.withMetadata(full.metadata, base.metadataLocation) + base.snapshotHistoryRefsOnly = false + + return base, meta, nil +} + func (r *stagedReplay) replan(b *MetadataBuilder, rec expirationRecord) ([]Update, []string, error) { updates, walked, err := planExpiration(b, rec.cfg, rec.nowMs) if err != nil { diff --git a/table/expire_replay_test.go b/table/expire_replay_test.go index a418d46f9..51fc6e8d7 100644 --- a/table/expire_replay_test.go +++ b/table/expire_replay_test.go @@ -51,6 +51,55 @@ func newExpireReplayTable(t *testing.T, props iceberg.Properties) (*memIO, *reqV return fs, cat, newOCCTable(t, meta, fs, cat) } +type refsOnlyReloadCatalog struct { + *reqValidatingCatalog + partial Metadata + loads int +} + +func (c *refsOnlyReloadCatalog) LoadTable(_ context.Context, ident Identifier) (*Table, error) { + c.loads++ + + return New(ident, c.partial, "", nil, c, WithSnapshotHistoryLimitedToRefs(true)), nil +} + +type fullHistoryReloadCatalog struct { + *refsOnlyReloadCatalog + location string + fullLoads int +} + +func (c *fullHistoryReloadCatalog) LoadTableWithFullSnapshotHistory(ctx context.Context, ident Identifier) (*Table, error) { + c.fullLoads++ + if err := ctx.Err(); err != nil { + return nil, err + } + + return New(ident, c.metadata, c.location, nil, c), nil +} + +func refsOnlySnapshotMetadata(t *testing.T, meta Metadata, keepIDs ...int64) Metadata { + t.Helper() + keep := make(map[int64]struct{}, len(keepIDs)) + for _, id := range keepIDs { + keep[id] = struct{}{} + } + var removeIDs []int64 + for _, snap := range meta.Snapshots() { + if _, ok := keep[snap.SnapshotID]; !ok { + removeIDs = append(removeIDs, snap.SnapshotID) + } + } + + builder, err := MetadataBuilderFromBase(meta, "") + require.NoError(t, err) + require.NoError(t, builder.RemoveSnapshots(removeIDs, false)) + partial, err := builder.Build() + require.NoError(t, err) + + return partial +} + func stageFiles(t *testing.T, tx *Transaction, names ...string) { t.Helper() files := make([]iceberg.DataFile, 0, len(names)) @@ -167,6 +216,289 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t *testing.T) { }) } +// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a +// history-dependent operation does not poison an otherwise valid transaction. +// Even a no-op expiration is unsafe to plan from refs-only metadata. +func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) { + fs, cat, tbl := newExpireReplayTable(t, nil) + tbl.snapshotHistoryRefsOnly = true + tx := tbl.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{"before-expiration": "kept"})) + updatesBefore := slices.Clone(tx.meta.updates) + reqsBefore := slices.Clone(tx.reqs) + expirationsBefore := len(tx.expirations) + + err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false)) + require.ErrorIs(t, err, ErrInvalidOperation) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + require.Equal(t, updatesBefore, tx.meta.updates, "rejected expiration must not add updates") + require.Equal(t, reqsBefore, tx.reqs, "rejected expiration must not add requirements") + require.Len(t, tx.expirations, expirationsBefore, "rejected expiration must not add replay records") + + stageFiles(t, tx, "after-rejected-expiration") + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Len(t, committed.Metadata().Snapshots(), 1) + require.Equal(t, "kept", committed.Properties()["before-expiration"]) + require.Equal(t, []string{"after-rejected-expiration"}, + branchFiles(t, fs, committed.Metadata(), MainBranch)) + require.Equal(t, int32(1), cat.attempts.Load(), "append should commit once") +} + +func TestExpireSnapshotsLoadsFullHistoryWithStagedAppend(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, old := appendOnBranch(t, tbl, MainBranch, "old") + tbl, head := appendOnBranch(t, tbl, MainBranch, "head") + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{ + reqValidatingCatalog: inner, + partial: refsOnlySnapshotMetadata(t, inner.metadata, head), + }, + location: "mem://default/table-location/metadata/partial.json", + } + partial := New(tbl.Identifier(), cat.partial, cat.location, tbl.fsF, cat, + WithSnapshotHistoryLimitedToRefs(true)) + tx := partial.NewTransaction() + stageFiles(t, tx, "staged") + stagedID := stagedHead(t, tx, MainBranch) + time.Sleep(time.Second) + + opts := []ExpireSnapshotsOpt{WithOlderThan(0), WithRetainLast(2), WithPostCommit(false)} + require.NoError(t, tx.ExpireSnapshots(opts...)) + require.NoError(t, tx.ExpireSnapshotsWithContext(t.Context(), opts...)) + staged, err := tx.StagedTable() + require.NoError(t, err) + require.Equal(t, sortedIDs(head, stagedID), survivors(staged.Metadata())) + require.Equal(t, sortedIDs(old, head), survivors(tx.tbl.Metadata())) + require.Equal(t, []int64{head}, survivors(partial.Metadata())) + require.True(t, partial.snapshotHistoryRefsOnly) + require.Equal(t, 1, cat.fullLoads) + require.Zero(t, cat.loads) + + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, sortedIDs(head, stagedID), survivors(committed.Metadata())) + require.Equal(t, []string{"head", "old", "staged"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) +} + +func TestExpireSnapshotsFullHistoryLoadFailurePreservesStagedState(t *testing.T) { + for _, name := range []string{"metadata revision changed", "context canceled"} { + t.Run(name, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, head := appendOnBranch(t, tbl, MainBranch, "head") + partial := refsOnlySnapshotMetadata(t, inner.metadata, head) + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner, partial: partial}, + location: "mem://default/table-location/metadata/base.json", + } + original := New(tbl.Identifier(), partial, cat.location, tbl.fsF, cat, + WithSnapshotHistoryLimitedToRefs(true)) + tx := original.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{"staged": "kept"})) + before := tx.meta + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if name == "context canceled" { + cancel() + } else { + cat.location = "mem://default/table-location/metadata/replaced.json" + } + + err := tx.ExpireSnapshotsWithContext(ctx, WithPostCommit(false)) + require.ErrorIs(t, err, ErrInvalidOperation) + if name == "context canceled" { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorContains(t, err, "metadata revision") + } + require.Same(t, before, tx.meta) + require.True(t, tx.tbl.snapshotHistoryRefsOnly) + require.Empty(t, tx.expirations) + require.Equal(t, 1, cat.fullLoads) + + stageFiles(t, tx, "append") + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, "kept", committed.Properties()["staged"]) + require.Equal(t, []string{"append", "head"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) + } +} + +func TestExpireReplayLoadsFullHistoryForNoOpExpiration(t *testing.T) { + for _, tc := range []struct { + name, retries string + resume bool + }{ + {name: "retry", retries: "3"}, + {name: "resume", retries: "0", resume: true}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: tc.retries}) + tbl, first := appendOnBranch(t, tbl, MainBranch, "first") + _, head := appendOnBranch(t, tbl, MainBranch, "head") + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner}, + } + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + time.Sleep(time.Second) + require.NoError(t, tx.ExpireSnapshots(WithOlderThan(0), WithRetainLast(2), WithPostCommit(false))) + require.Len(t, tx.expirations, 1) + require.Zero(t, tx.expirations[0].count) + stageFiles(t, tx, "ours") + ours := stagedHead(t, tx, MainBranch) + + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + attemptsBefore := inner.attempts.Load() + if tc.resume { + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrCommitFailed) + require.Zero(t, cat.fullLoads) + } + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, attemptsBefore+2, inner.attempts.Load()) + require.Equal(t, 1, cat.fullLoads) + require.Zero(t, cat.loads) + require.Equal(t, sortedIDs(head, peer, ours), survivors(committed.Metadata())) + require.Nil(t, committed.Metadata().SnapshotByID(first)) + require.Equal(t, peer, parentOf(t, committed.Metadata(), ours)) + require.Equal(t, []string{"first", "head", "ours", "peer"}, branchFiles(t, fs, committed.Metadata(), MainBranch)) + }) + }) + } +} + +func TestExpireReplayRejectsRefsOnlyHistoryOnRetryOrResume(t *testing.T) { + stage := func(t *testing.T, numRetries string) (*reqValidatingCatalog, *refsOnlyReloadCatalog, *Transaction, [4]int64, int64) { + t.Helper() + fs, inner, tbl := newExpireReplayTable(t, iceberg.Properties{CommitNumRetriesKey: numRetries}) + var ids [4]int64 + for i, name := range []string{"s1", "s2", "s3", "s4"} { + tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name) + } + peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag", ids[0], TagRef, time.Hour.Milliseconds(), 0, 0)) + time.Sleep(2 * time.Hour) + + cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + require.NoError(t, tx.ExpireSnapshots( + WithOlderThan(0), WithRetainLast(5), WithPostCommit(false), + )) + staged, err := tx.StagedTable() + require.NoError(t, err) + require.Nil(t, staged.Metadata().SnapshotByName("expired-tag")) + require.Equal(t, sortedIDs(ids[:]...), survivors(staged.Metadata())) + + peer := peerAppend(t, fs, inner, MainBranch, "s5") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, ids[0], peer) + require.Equal(t, sortedIDs(ids[0], peer), survivors(cat.partial)) + + return inner, cat, tx, ids, peer + } + + for _, tt := range []struct { + name string + numRetries string + resume bool + }{ + {name: "retry", numRetries: "3"}, + {name: "resume", numRetries: "0", resume: true}, + } { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + inner, cat, tx, ids, peer := stage(t, tt.numRetries) + attemptsBefore := inner.attempts.Load() + + if tt.resume { + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrCommitFailed) + require.NotErrorIs(t, err, ErrTransactionUnusable) + require.Zero(t, cat.loads, "the first failed attempt must not refresh") + require.Equal(t, attemptsBefore+1, inner.attempts.Load()) + } + + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrTransactionUnusable) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + require.Equal(t, 1, cat.loads) + require.Equal(t, attemptsBefore+1, inner.attempts.Load(), + "refresh must stop before resubmitting expiration") + require.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], peer), survivors(inner.metadata)) + require.NotNil(t, inner.metadata.SnapshotByID(ids[0]), + "retained history must remain in catalog metadata") + require.NotNil(t, inner.metadata.SnapshotByName("expired-tag"), + "failed expiration must not remove the tag") + require.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + }) + }) + } +} + +// A no-op expiration still needs complete history when a retry replans it. +// The absence of planned removals must not bypass the refs-only guard. +func TestExpireReplayRejectsRefsOnlyHistoryWithNoOpExpiration(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, s1 := appendOnBranch(t, tbl, MainBranch, "s1") + _, s2 := appendOnBranch(t, tbl, MainBranch, "s2") + + cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner} + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + require.NoError(t, tx.ExpireSnapshots( + WithOlderThan(0), WithRetainLast(5), WithPostCommit(false), + )) + require.Len(t, tx.expirations, 1, "no-op expiration must still be recorded") + require.Zero(t, tx.expirations[0].count, "expiration must stage no updates") + stageFiles(t, tx, "ours") + + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + require.Equal(t, []int64{peer}, survivors(cat.partial)) + attemptsBefore := inner.attempts.Load() + + _, err := tx.Commit(t.Context()) + require.ErrorIs(t, err, ErrTransactionUnusable) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + require.Equal(t, 1, cat.loads) + require.Equal(t, attemptsBefore+1, inner.attempts.Load(), + "failed retry must not resubmit the append or expiration") + require.Equal(t, sortedIDs(s1, s2, peer), survivors(inner.metadata)) + require.Equal(t, peer, refHead(t, inner.metadata, MainBranch)) + require.Equal(t, []string{"peer", "s1", "s2"}, + branchFiles(t, fs, inner.metadata, MainBranch)) + }) +} + +func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fs, inner, tbl := newExpireReplayTable(t, nil) + tbl, first := appendOnBranch(t, tbl, MainBranch, "first") + appendOnBranch(t, tbl, MainBranch, "head") + cat := &fullHistoryReloadCatalog{ + refsOnlyReloadCatalog: &refsOnlyReloadCatalog{reqValidatingCatalog: inner}, + } + tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction() + stageFiles(t, tx, "ours") + + peer := peerAppend(t, fs, inner, MainBranch, "peer") + cat.partial = refsOnlySnapshotMetadata(t, inner.metadata, peer) + assert.Equal(t, []int64{peer}, survivors(cat.partial)) + assert.NotContains(t, survivors(cat.partial), first) + + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + assert.Equal(t, 1, cat.loads) + assert.Zero(t, cat.fullLoads) + assert.Equal(t, peer, parentOf(t, committed.Metadata(), refHead(t, committed.Metadata(), MainBranch))) + }) +} + func TestExpireReplayAfterRollback(t *testing.T) { stage := func(t *testing.T) (*memIO, *reqValidatingCatalog, *Transaction, [3]int64, int64) { t.Helper() diff --git a/table/staged_replay.go b/table/staged_replay.go index b28f3dc16..5938ffacb 100644 --- a/table/staged_replay.go +++ b/table/staged_replay.go @@ -52,6 +52,13 @@ func newStagedReplay(branch string, updates []Update, records []expirationRecord } } +// A staged expiration may have yielded no metadata updates but can produce +// removals after a concurrent commit. Check the recorded operations, not the +// length of staged updates, before replaying against incomplete history. +func (r *stagedReplay) hasExpirationRecords() bool { + return r != nil && len(r.records) > 0 +} + // On error, replay returns partial updates and replaced lists for cleanup. // The partial updates include a rebuilt snapshot that failed to apply. func (r *stagedReplay) replay(ctx context.Context, fresh Metadata, fs io.WriteFileIO, attempt int, force bool) ([]Update, []string, error) { diff --git a/table/table.go b/table/table.go index 85e29173a..66bbce6d8 100644 --- a/table/table.go +++ b/table/table.go @@ -142,6 +142,12 @@ type CatalogIO interface { CommitTable(context.Context, Identifier, []Requirement, []Update) (Metadata, string, error) } +// FullSnapshotHistoryLoader is an optional catalog capability for operations +// that need the complete snapshot history of a table. +type FullSnapshotHistoryLoader interface { + LoadTableWithFullSnapshotHistory(context.Context, Identifier) (*Table, error) +} + type Table struct { identifier Identifier metadata Metadata @@ -816,7 +822,11 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen // the new branch head so re-submission is not rejected // just because a peer advanced the head with a // non-conflicting commit. - fresh, refreshErr := t.cat.LoadTable(retryCtx, slices.Clone(t.identifier)) + load := t.cat.LoadTable + if loader, ok := t.cat.(FullSnapshotHistoryLoader); ok && co.replay.hasExpirationRecords() { + load = loader.LoadTableWithFullSnapshotHistory + } + fresh, refreshErr := load(retryCtx, slices.Clone(t.identifier)) if refreshErr != nil { return nil, fmt.Errorf("refresh table for retry: %w", refreshErr) } @@ -825,6 +835,9 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen return nil, fmt.Errorf("%w: table was replaced: %w", ErrTransactionUnusable, err) } } + if fresh.snapshotHistoryRefsOnly && co.replay.hasExpirationRecords() { + return nil, fmt.Errorf("%w: %w", ErrTransactionUnusable, errSnapshotHistoryIncompleteForExpiration) + } current, currentLoc = fresh.metadata, fresh.metadataLocation reqs = rewriteRefSnapshotRequirements(reqs, co.branch, current, pinned) diff --git a/table/transaction.go b/table/transaction.go index c8a6a037d..f1c731a5c 100644 --- a/table/transaction.go +++ b/table/transaction.go @@ -804,11 +804,24 @@ func getRetentionInt64(props iceberg.Properties, standardKey, legacyKey string, // committing. A transaction containing only expiries with no updates makes no // commit attempt and never refreshes or replans. // +// Snapshot expiration requires complete history even when the plan is empty. +// On a refs-only REST table, expiration loads full history on demand for the +// transaction's metadata revision. Staging fails with ErrInvalidOperation +// if the catalog cannot supply that history or the table has advanced. +// Retries also request full history before replaying the expiration. +// Use [Transaction.ExpireSnapshotsWithContext] to control the staging request. +// // The replay failures that make the transaction unusable are listed on // ErrTransactionUnusable. // // The "iceberg expire-snapshots" CLI command wraps the same operation. func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { + return t.ExpireSnapshotsWithContext(context.Background(), opts...) +} + +// ExpireSnapshotsWithContext stages expiration like [Transaction.ExpireSnapshots], +// using ctx to load complete snapshot history when the table was loaded refs-only. +func (t *Transaction) ExpireSnapshotsWithContext(ctx context.Context, opts ...ExpireSnapshotsOpt) error { var ( cfg = expireSnapshotsCfg{postCommit: true} nowMs = time.Now().UnixMilli() @@ -820,43 +833,50 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error { if cfg.validationErr != nil { return cfg.validationErr } + if err := t.checkNotNil(); err != nil { + return err + } + t.mx.Lock() + defer t.mx.Unlock() + meta, err := t.txnMeta() if err != nil { return err } + if err := t.checkUsable(); err != nil { + return err + } if err := validateGCEnabledForSnapshotExpiration(meta.props); err != nil { return err } + base := t.tbl + if base.snapshotHistoryRefsOnly { + base, meta, err = loadExpirationHistory(ctx, base, meta) + if err != nil { + return err + } + } updates, walked, err := planExpiration(meta, cfg, nowMs) if err != nil { return err } - return t.stageExpiration(expirationRecord{ + rec := expirationRecord{ cfg: cfg, nowMs: nowMs, walked: walked, removedBranch: removesRef(updates, cmp.Or(t.branch, MainBranch)), - }, updates) -} - -// Record positions under the same lock as the updates they locate. -func (t *Transaction) stageExpiration(rec expirationRecord, updates []Update) error { - if err := t.checkNotNil(); err != nil { - return err - } - t.mx.Lock() - defer t.mx.Unlock() - - meta, err := t.txnMeta() - if err != nil { - return err } rec.pos = len(meta.updates) + original := t.meta + t.meta = meta if err := t.applyLocked(updates, nil); err != nil { + t.meta = original + return err } + t.tbl = base rec.count = len(t.meta.updates) - rec.pos t.expirations = append(t.expirations, rec) diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go index 91d8aeb69..1e650e922 100644 --- a/table/transaction_internal_test.go +++ b/table/transaction_internal_test.go @@ -579,6 +579,21 @@ func TestExpireSnapshotsWithOlderThanDoesNotExpireSnapshotRefs(t *testing.T) { require.True(t, tagExists, "WithOlderThan must not expire a tag without max-ref-age-ms") } +func TestExpireSnapshotsRejectsRefsOnlySnapshotHistory(t *testing.T) { + txn := newTransactionWithSnapshotRefs(t) + txn.tbl.snapshotHistoryRefsOnly = true + + err := txn.ExpireSnapshots( + WithOlderThan(0), + WithRetainLast(1), + WithPostCommit(false), + ) + require.ErrorIs(t, err, ErrInvalidOperation) + require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration) + require.ErrorContains(t, err, "snapshot-loading-mode=refs") + require.ErrorContains(t, err, "snapshot-loading-mode=all") +} + func TestExpireSnapshotsMaterializesDeferredHistory(t *testing.T) { const historicalID = int64(3051729675574597004) diff --git a/website/src/api.md b/website/src/api.md index 72c39ae95..f525cf3da 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -853,6 +853,19 @@ if err != nil { /* ... */ } newTbl, err := txn.Commit(ctx) ``` +Snapshot expiration needs complete ancestry to honor branch retention and snapshot +references. When a REST table was loaded with `snapshot-loading-mode=refs`, +expiration requests `snapshots=all` for the transaction's metadata revision +and keeps any updates already staged in the transaction. Use +`Transaction.ExpireSnapshotsWithContext(ctx, ...)` to control that request. + +If the catalog cannot supply complete history or the metadata revision has +changed, staging returns an error matching `table.ErrInvalidOperation`. +Reload the table and start a new transaction if the revision changed. +Commit retries and resumed commits also request full history before replanning +expiration. If the catalog still returns refs-only history, the commit stops +before resubmitting and returns an error matching `table.ErrTransactionUnusable`. + Pass `table.WithPostCommit(true)` to delete the unreferenced data and metadata files after the commit lands. The `iceberg expire-snapshots` CLI command wraps the same operation - see [CLI](./cli.md). ## Views diff --git a/website/src/cli.md b/website/src/cli.md index ad1b32bad..637125c6e 100644 --- a/website/src/cli.md +++ b/website/src/cli.md @@ -217,6 +217,9 @@ iceberg expire-snapshots --older-than 7d --dry-run my_db.events iceberg expire-snapshots --older-than 7d --retain-last 5 my_db.events ``` +For REST catalogs, expiration and `--dry-run` load full snapshot history. +The catalog's snapshot loading mode still applies to other commands. + ### `clean-orphan-files` Remove data files in the table location that are not referenced by any snapshot's manifests (e.g. left behind by failed writes). @@ -309,9 +312,9 @@ This can discard the branch's current history from the branch head. The prompt shows both the previous and next snapshot IDs. Retention settings are preserved. Use `--yes` to skip confirmation. -For both commands, REST catalogs using `snapshot-loading-mode=refs` may omit -historical snapshots needed for ancestry checks or replacement. Use -`snapshot-loading-mode=all` for these operations. +For both commands, the REST server's `snapshot-loading-mode=refs` setting may omit +historical snapshots needed for ancestry checks or replacement. The CLI cannot +override this setting; configure the REST server with `snapshot-loading-mode=all`. ### `tag create` @@ -371,4 +374,4 @@ Every write command has multiple layers of protection: - **`--yes`** — required to skip the prompt; without it, non-interactive shells get an explicit error rather than hanging. - **TTY detection** — interactive prompts are only shown when stdout is a terminal. - **Ancestor validation** — `rollback` rejects target snapshots that are not in the current branch's history. -- **Version check** — `upgrade` refuses same-version or downgrade requests. \ No newline at end of file +- **Version check** — `upgrade` refuses same-version or downgrade requests. diff --git a/website/src/releases.md b/website/src/releases.md index 26c1f867e..a217c7153 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -101,6 +101,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Retries preserve staged update order.** An append staged after `RollbackToSnapshot` stays on the rollback target after a retry or resumed `Commit`. If a staged update cannot apply after refresh, the transaction ends with `table.ErrTransactionUnusable`. For example, `UpgradeFormatVersion` cannot apply if a peer has committed a higher format version. The error preserves its cause and also matches `table.ErrCommitFailed` if that cause is a conflict. Previously, for transactions without a staged expiry, these failures returned `table.ErrCommitRejected` and left the transaction resumable. A catalog rejection on the first attempt still returns `table.ErrCommitRejected` and leaves the transaction resumable. - **Retries renew snapshot timestamps.** A retry or resumed `Commit` stamps each snapshot it replays from a staged write with the current time, at least one millisecond after its parent, as Java's `SnapshotProducer` does. A snapshot staged more than a minute before a peer's commit no longer fails the retry. The committed timestamp is the time of the attempt that lands, so an `ExpireSnapshots` staged after the write in the same transaction keeps that snapshot when the commit retries. - **Snapshot expiration now tolerates concurrent ref changes.** If a concurrent commit moves a ref that the expiry read, a retry or later `Commit` replans instead of failing with `table.ErrCommitFailed`. The committed expiry can differ from `Transaction.StagedTable`. See [Snapshot expiration](./concurrent-writes.md#snapshot-expiration) for cases that now fail with `table.ErrTransactionUnusable`. +- **Snapshot expiration loads full REST history on demand.** Expiration requests `snapshots=all` when a REST table was loaded in refs mode, including on commit retries. Use `Transaction.ExpireSnapshotsWithContext` to control the staging request. Catalogs that only supply refs-only history cannot safely expire snapshots. - **RowDelta reports a concurrent deletion vector as a commit conflict.** If a concurrent commit adds a deletion vector for a data file that a `RowDelta` also adds one for, `Transaction.Commit` now fails with an error matching `table.ErrConflictingDeleteFiles`, `table.ErrCommitFailed` and `table.ErrTransactionUnusable`, and the transaction cannot be committed again. Previously, the error matched none of them and the transaction stayed usable. To resolve the conflict, reload the table and commit a `RowDelta` that adds one deletion vector merging the positions of both vectors and removes the concurrent one with `RemoveDeletes`. - **Schema JSON constructor uses the Go initialism.** Replace `iceberg.NewSchemaFromJsonFields` with `iceberg.NewSchemaFromJSONFields`. - **AWS identifiers use the Go initialism.** `glue.AwsProperties`, `glue.WithAwsConfig` and `glue.WithAwsProperties` become `glue.AWSProperties`, `glue.WithAWSConfig` and `glue.WithAWSProperties`; `sigv4.WithAwsConfig` becomes `sigv4.WithAWSConfig`; `utils.WithAwsConfig` and `utils.GetAwsConfig` become `utils.WithAWSConfig` and `utils.GetAWSConfig`; and `config.CatalogConfig.AwsProfile` becomes `AWSProfile`. The `aws-profile` YAML key is unchanged.