Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
a1a0ac1
fix(table): reject snapshot expiry with refs-only history
fallintoplace Nov 15, 2026
b3ccff0
fix(table): reject expiry replay with partial history
fallintoplace Nov 15, 2026
91642ed
docs(table): clarify expiration replay history errors
fallintoplace Nov 15, 2026
91d4a99
fix(table): address refs-only expiration review (expire_replay_test.go)
iceberg-kitten Nov 17, 2026
6f9091d
fix(table): address refs-only expiration review (transaction.go)
iceberg-kitten Nov 17, 2026
045a959
fix(table): address refs-only expiration review (transaction_internal…
iceberg-kitten Nov 17, 2026
fd05064
fix(table): address refs-only expiration review (api.md)
iceberg-kitten Nov 17, 2026
31e38da
fix(table): address refs-only expiration review (cli.md)
iceberg-kitten Nov 17, 2026
9c5329c
docs(cli): clarify REST server controls snapshot-loading-mode
iceberg-kitten Nov 17, 2026
1ea6016
test(table): keep transaction usable after refs-only expiration rejec…
iceberg-kitten Nov 17, 2026
7122a4c
docs(table): spell out expiration errors and recovery under REST refs…
iceberg-kitten Nov 17, 2026
a352d5d
docs(table): explain expiration refs-only error contracts
iceberg-kitten Nov 17, 2026
069c7f8
docs(table): explain expiration replay history detection
iceberg-kitten Nov 17, 2026
75702f5
docs(table): explain incomplete-history error sentinel
iceberg-kitten Nov 17, 2026
46f97e3
docs: preserve upstream release notes alongside PR changes
iceberg-kitten Nov 17, 2026
36f2e22
test(table): cover no-op expiration retries and consolidate refs-only…
iceberg-kitten Nov 18, 2026
7ce9c44
docs(table): clarify client REST snapshot history recovery
iceberg-kitten Nov 18, 2026
cfe43d6
docs(releases): group snapshot expiration compatibility notes
iceberg-kitten Nov 18, 2026
8bce00f
Merge apache/iceberg-go main into refs-only expiration fix
iceberg-kitten Nov 18, 2026
bb2634d
Merge latest apache/iceberg-go main into expiration history fix
iceberg-kitten Nov 18, 2026
7e95696
test(table): drop unused expiration result
fallintoplace Nov 18, 2026
bc58b1c
docs: preserve upstream API examples and release notes
fallintoplace Nov 18, 2026
d7e258d
docs(table): separate refs-only expiration retry guidance
fallintoplace Nov 18, 2026
7ba98d8
docs(table): keep expiration retry details with ExpireSnapshots
fallintoplace Nov 18, 2026
68248f0
docs(release): mention CLI snapshot expiration guard
fallintoplace Nov 18, 2026
9dfbe92
Merge remote-tracking branch 'upstream/main' into fix/expire-refs-onl…
fallintoplace Nov 18, 2026
90fa709
fix(table): load full REST history for snapshot expiration
fallintoplace Nov 18, 2026
84417eb
test(rest): keep expiration and orphan cleanup tests mergeable
fallintoplace Nov 18, 2026
0568fe7
Merge remote-tracking branch 'upstream/main' into fix/expire-refs-onl…
fallintoplace Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions catalog/rest/rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -2050,6 +2050,16 @@ func (r *Catalog) LoadTable(ctx context.Context, identifier table.Identifier) (*
return r.loadTableWithMode(ctx, identifier, r.snapshotMode)
}

// LoadTableWithFullSnapshotHistory requests all valid snapshots for this load.
// The catalog's configured mode still applies to regular LoadTable calls.
func (r *Catalog) LoadTableWithFullSnapshotHistory(ctx context.Context, identifier table.Identifier) (*table.Table, error) {
if r.snapshotMode != SnapshotModeRefs {
return r.LoadTable(ctx, identifier)
}

return r.loadTableWithMode(ctx, identifier, SnapshotModeAll)
}

func (r *Catalog) loadTableWithMode(ctx context.Context, identifier table.Identifier, mode SnapshotMode) (*table.Table, error) {
if err := r.endpoints.check(endpointLoadTable); err != nil {
return nil, err
Expand Down
74 changes: 74 additions & 0 deletions catalog/rest/rest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ import (
"runtime"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/catalog"
Expand Down Expand Up @@ -2131,6 +2133,78 @@ func (r *RestCatalogSuite) TestCreateTableLabels() {
r.Equal(iceberg.Properties{"classification": "pii"}, labels.Field(2))
}

func (r *RestCatalogSuite) TestExpireSnapshotsRequestsFullHistoryOnDemand() {
ctx := r.T().Context()
const location = "s3://warehouse/db/table"
const metadataLocation = location + "/metadata/v2.metadata.json"
meta, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil)
r.Require().NoError(err)
builder, err := table.MetadataBuilderFromBase(meta, "")
r.Require().NoError(err)
now := time.Now()
r.Require().NoError(builder.AddSnapshot(&table.Snapshot{
SnapshotID: 1, SequenceNumber: 1, TimestampMs: now.Add(-10 * time.Second).UnixMilli(),
ManifestList: location + "/metadata/old.avro", Summary: &table.Summary{Operation: table.OpAppend},
}))
r.Require().NoError(builder.AddSnapshot(&table.Snapshot{
SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: now.Add(-5 * time.Second).UnixMilli(),
ManifestList: location + "/metadata/head.avro", Summary: &table.Summary{Operation: table.OpAppend},
}))
r.Require().NoError(builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef))
full, err := builder.Build()
r.Require().NoError(err)
partialBuilder, err := table.MetadataBuilderFromBase(full, "")
r.Require().NoError(err)
r.Require().NoError(partialBuilder.RemoveSnapshots([]int64{1}, false))
partial, err := partialBuilder.Build()
r.Require().NoError(err)

var refsLoads, fullLoads atomic.Int32
r.mux.HandleFunc("GET /v1/namespaces/db/tables/table", func(w http.ResponseWriter, req *http.Request) {
var metadata table.Metadata
switch req.URL.Query().Get("snapshots") {
case "refs":
refsLoads.Add(1)
metadata = partial
case "all":
fullLoads.Add(1)
metadata = full
default:
http.Error(w, "expected snapshots=refs or snapshots=all", http.StatusBadRequest)

return
}
r.NoError(json.NewEncoder(w).Encode(map[string]any{
"metadata-location": metadataLocation,
"metadata": metadata,
}))
})
cat, err := rest.NewCatalog(ctx, "rest", r.srv.URL, rest.WithOAuthToken(TestToken),
rest.WithAdditionalProps(iceberg.Properties{"snapshot-loading-mode": "refs"}))
r.Require().NoError(err)

ident := catalog.ToIdentifier("db", "table")
tbl, err := cat.LoadTable(ctx, ident)
r.Require().NoError(err)
r.Len(tbl.Metadata().Snapshots(), 1)
r.Zero(fullLoads.Load())

tx := tbl.NewTransaction()
r.Require().NoError(tx.ExpireSnapshotsWithContext(ctx, table.WithOlderThan(0), table.WithRetainLast(1), table.WithPostCommit(false)))
commit, err := tx.TableCommit()
r.Require().NoError(err)
r.Len(commit.Updates, 1)
update, err := json.Marshal(commit.Updates[0])
r.Require().NoError(err)
r.JSONEq(`{"action":"remove-snapshots","snapshot-ids":[1]}`, string(update))
r.EqualValues(1, fullLoads.Load())

tbl, err = cat.LoadTable(ctx, ident)
r.Require().NoError(err)
r.Len(tbl.Metadata().Snapshots(), 1)
r.EqualValues(2, refsLoads.Load())
}

func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeRefs() {
// Server returns only the snapshot referenced by the main branch (refs behaviour).
r.mux.HandleFunc("/v1/namespaces/fokko/tables/table", func(w http.ResponseWriter, req *http.Request) {
Expand Down
12 changes: 9 additions & 3 deletions cmd/iceberg/expire_snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,13 @@ import (
)

func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog, cmd *ExpireSnapshotsCmd) {
tbl := loadTable(ctx, output, cat, cmd.TableID)
// The preview compares the base and staged snapshot lists, so both
// must include history that an ordinary refs-only load can omit.
load := cat.LoadTable
if loader, ok := cat.(table.FullSnapshotHistoryLoader); ok {
load = loader.LoadTableWithFullSnapshotHistory
}
tbl := loadTableUsing(ctx, output, load, cmd.TableID)

var opts []table.ExpireSnapshotsOpt

Expand All @@ -52,7 +58,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog,
opts = append(opts, table.WithPostCommit(false))

tx := tbl.NewTransaction()
if err := tx.ExpireSnapshots(opts...); err != nil {
if err := tx.ExpireSnapshotsWithContext(ctx, opts...); err != nil {
output.Error(fmt.Errorf("expire snapshots failed: %w", err))
osExit(1)
}
Expand Down Expand Up @@ -100,7 +106,7 @@ func runExpireSnapshots(ctx context.Context, output Output, cat catalog.Catalog,
}

realTx := tbl.NewTransaction()
if err := realTx.ExpireSnapshots(realOpts...); err != nil {
if err := realTx.ExpireSnapshotsWithContext(ctx, realOpts...); err != nil {
output.Error(fmt.Errorf("expire snapshots failed: %w", err))
osExit(1)
}
Expand Down
53 changes: 53 additions & 0 deletions cmd/iceberg/expire_snapshots_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,68 @@ package main

import (
"bytes"
"context"
"os"
"testing"
"time"

"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/catalog"
"github.com/apache/iceberg-go/table"
"github.com/pterm/pterm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

type expirationPreviewCatalog struct {
catalog.Catalog
full *table.Table
}

func (c expirationPreviewCatalog) LoadTableWithFullSnapshotHistory(context.Context, table.Identifier) (*table.Table, error) {
return c.full, nil
}

type expirationPreviewOutput struct {
Output
result ExpireSnapshotsResult
}

func (o *expirationPreviewOutput) ExpireSnapshotsResult(result ExpireSnapshotsResult) {
o.result = result
}

func TestExpireSnapshotsDryRunPreviewsFullHistory(t *testing.T) {
location := t.TempDir()
meta, err := table.NewMetadata(iceberg.NewSchema(0), iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, nil)
require.NoError(t, err)
builder, err := table.MetadataBuilderFromBase(meta, "")
require.NoError(t, err)
require.NoError(t, builder.AddSnapshot(&table.Snapshot{
SnapshotID: 1, SequenceNumber: 1, TimestampMs: time.Now().Add(-10 * time.Second).UnixMilli(),
ManifestList: location + "/old.avro", Summary: &table.Summary{Operation: table.OpAppend},
}))
require.NoError(t, builder.AddSnapshot(&table.Snapshot{
SnapshotID: 2, ParentSnapshotID: new(int64(1)), SequenceNumber: 2, TimestampMs: time.Now().Add(-5 * time.Second).UnixMilli(),
ManifestList: location + "/head.avro", Summary: &table.Summary{Operation: table.OpAppend},
}))
require.NoError(t, builder.SetSnapshotRef(table.MainBranch, 2, table.BranchRef))
meta, err = builder.Build()
require.NoError(t, err)

cat := expirationPreviewCatalog{
full: table.New(table.Identifier{"db", "tbl"}, meta, location+"/metadata.json", nil, nil),
}
out := &expirationPreviewOutput{}
runExpireSnapshots(t.Context(), out, cat, &ExpireSnapshotsCmd{
TableID: "db.tbl", DryRun: true, OlderThan: "0s", RetainLast: new(1),
})
require.True(t, out.result.DryRun)
require.Equal(t, 1, out.result.ExpiredSnapshotCount)
require.Len(t, out.result.ExpiredSnapshots, 1)
require.Equal(t, int64(1), out.result.ExpiredSnapshots[0].SnapshotID)
}

func TestDiffSnapshots(t *testing.T) {
before := []table.Snapshot{
{SnapshotID: 1, TimestampMs: 1000000000000, Summary: &table.Summary{Operation: table.OpAppend}},
Expand Down
6 changes: 5 additions & 1 deletion cmd/iceberg/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -875,7 +875,11 @@ func describe(ctx context.Context, output Output, cat catalog.Catalog, id string
}

func loadTable(ctx context.Context, output Output, cat catalog.Catalog, id string) *table.Table {
tbl, err := cat.LoadTable(ctx, catalog.ToIdentifier(id))
return loadTableUsing(ctx, output, cat.LoadTable, id)
}

func loadTableUsing(ctx context.Context, output Output, load func(context.Context, table.Identifier) (*table.Table, error), id string) *table.Table {
tbl, err := load(ctx, catalog.ToIdentifier(id))
if err != nil {
output.Error(err)
osExit(1)
Expand Down
51 changes: 51 additions & 0 deletions table/expire_replay.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,61 @@
package table

import (
"context"
"errors"
"fmt"
"slices"
)

// Preserve a distinct cause for both staging and retry failures so callers
// can match the public ErrInvalidOperation or ErrTransactionUnusable wrapper
// without relying on the human-readable REST configuration hint.
var errSnapshotHistoryIncompleteForExpiration = errors.New(
"cannot expire snapshots: history may be incomplete under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all",
)

// Load the same metadata revision with complete history, then rebuild the
// staged state over that base. Keep the original Table and builder intact
// until the expiration has been staged successfully.
func loadExpirationHistory(ctx context.Context, base *Table, staged *MetadataBuilder) (*Table, *MetadataBuilder, error) {
loader, ok := base.cat.(FullSnapshotHistoryLoader)
if !ok {
return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration)
}
full, err := loader.LoadTableWithFullSnapshotHistory(ctx, slices.Clone(base.identifier))
if err != nil {
return nil, nil, fmt.Errorf("%w: load full snapshot history: %w", ErrInvalidOperation, err)
}
if full.snapshotHistoryRefsOnly {
return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, errSnapshotHistoryIncompleteForExpiration)
}
if err := validateTableUUID(base.identifier, base.metadata.TableUUID(), full.metadata.TableUUID()); err != nil {
return nil, nil, fmt.Errorf("%w: %w", ErrInvalidOperation, err)
}
if base.metadataLocation == "" || base.metadataLocation != full.metadataLocation {
return nil, nil, fmt.Errorf("%w: could not load full snapshot history for the transaction's metadata revision; reload the table and start a new transaction",
ErrInvalidOperation)
}

meta, err := MetadataBuilderFromBase(full.metadata, base.metadataLocation)
if err != nil {
return nil, nil, err
}
for _, update := range staged.updates {
if err := update.Apply(meta); err != nil {
return nil, nil, fmt.Errorf("%w: apply staged %s with full snapshot history: %w", ErrInvalidOperation, update.Action(), err)
}
}
// Keep the same staged operations and their timestamp. Rebuilding can
// make an update redundant now that more of the base history is visible.
meta.updates = slices.Clone(staged.updates)
meta.lastUpdatedMS = staged.lastUpdatedMS
base = base.withMetadata(full.metadata, base.metadataLocation)
base.snapshotHistoryRefsOnly = false

return base, meta, nil
}

func (r *stagedReplay) replan(b *MetadataBuilder, rec expirationRecord) ([]Update, []string, error) {
updates, walked, err := planExpiration(b, rec.cfg, rec.nowMs)
if err != nil {
Expand Down
Loading
Loading