Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions catalog/rest/rest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@ import (
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/catalog"
"github.com/apache/iceberg-go/catalog/rest"
iceio "github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/metrics"
"github.com/apache/iceberg-go/table"
"github.com/apache/iceberg-go/table/maintenance"
"github.com/apache/iceberg-go/view"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -2184,6 +2186,8 @@ func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeRefs() {
tbl, err := cat.LoadTable(context.Background(), catalog.ToIdentifier("fokko", "table"))
r.Require().NoError(err)

r.True(tbl.SnapshotHistoryLimitedToRefs())

// Only the snapshot referenced by main is returned.
r.Len(tbl.Metadata().Snapshots(), 1)
r.EqualValues(3497810964824022504, tbl.Metadata().Snapshots()[0].SnapshotID)
Expand Down Expand Up @@ -2251,13 +2255,89 @@ func (r *RestCatalogSuite) TestLoadTableWithSnapshotModeAll() {
tbl, err := cat.LoadTable(context.Background(), catalog.ToIdentifier("fokko", "table"))
r.Require().NoError(err)

r.False(tbl.SnapshotHistoryLimitedToRefs())

// Both the current and the unreferenced historical snapshot are returned.
r.Len(tbl.Metadata().Snapshots(), 2)
r.EqualValues(1000000000000000001, tbl.Metadata().Snapshots()[0].SnapshotID)
r.EqualValues(3497810964824022504, tbl.Metadata().Snapshots()[1].SnapshotID)
r.EqualValues(3497810964824022504, tbl.CurrentSnapshot().SnapshotID)
}

func TestOrphanCleanupHonorsEffectiveRESTSnapshotMode(t *testing.T) {
const scheme = "orphan-cleanup-test"
const location = scheme + "://warehouse/table"
const metadataLocation = location + "/metadata/v1.metadata.json"
const orphan = location + "/data/orphan.parquet"
const modeKey = "snapshot-loading-mode"

metadata, err := table.NewMetadata(tableSchemaSimple, iceberg.UnpartitionedSpec,
table.UnsortedSortOrder, location, nil)
require.NoError(t, err)
response, err := json.Marshal(map[string]any{
"metadata-location": metadataLocation,
"metadata": metadata,
})
require.NoError(t, err)

for _, tc := range []struct {
name string
overrides iceberg.Properties
wantMode string
}{
{name: "client all overrides refs default", wantMode: "all"},
{name: "server refs override wins", overrides: iceberg.Properties{modeKey: "refs"}, wantMode: "refs"},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := t.Context()
fs := iceio.NewMemFS()
require.NoError(t, fs.WriteFile(ctx, orphan, []byte("orphan")))
registry := &iceio.Registry{}
fsCalls := 0
require.NoError(t, registry.Register(scheme, func(context.Context, *url.URL, map[string]string) (iceio.IO, error) {
fsCalls++

return fs, nil
}))

mux := http.NewServeMux()
mux.HandleFunc("GET /v1/config", func(w http.ResponseWriter, _ *http.Request) {
assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{
"defaults": iceberg.Properties{modeKey: "refs"},
"overrides": tc.overrides,
"endpoints": rest.AllEndpointStrings,
}))
})
mux.HandleFunc("GET /v1/namespaces/db/tables/tbl", func(w http.ResponseWriter, req *http.Request) {
assert.Equal(t, tc.wantMode, req.URL.Query().Get("snapshots"))
_, err := w.Write(response)
assert.NoError(t, err)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)

cat, err := rest.NewCatalog(ctx, "rest", srv.URL,
rest.WithAdditionalProps(iceberg.Properties{modeKey: "all"}),
rest.WithIORegistry(registry))
require.NoError(t, err)
tbl, err := cat.LoadTable(ctx, catalog.ToIdentifier("db", "tbl"))
require.NoError(t, err)

plan, err := maintenance.PlanOrphanFiles(ctx, tbl, maintenance.WithCleanupFilesOlderThan(0))
if tc.wantMode == "refs" {
require.True(t, tbl.SnapshotHistoryLimitedToRefs())
require.ErrorIs(t, err, table.ErrInvalidOperation)
require.Zero(t, fsCalls, "refs-only planning should stop before getting FileIO")

return
}
require.False(t, tbl.SnapshotHistoryLimitedToRefs())
require.NoError(t, err)
require.Equal(t, []string{orphan}, plan.Files())
})
}
}

func (r *RestCatalogSuite) TestRenameTable409() {
r.mux.HandleFunc("/v1/tables/rename", func(w http.ResponseWriter, req *http.Request) {
if !r.Equal(http.MethodPost, req.Method) {
Expand Down
38 changes: 31 additions & 7 deletions table/maintenance/orphan_cleanup.go
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,15 @@ func rejectPlanOption(cfg *orphanCleanupConfig, name string) {
// after the second refresh.
// Dry runs plan from the supplied table metadata.
// Orphan cleanup, including dry runs, is rejected when garbage collection is disabled via gc.enabled.
// It also rejects REST refs-only snapshot history, which can omit still-live files.
// Configure the REST catalog with snapshot-loading-mode=all and reload the
// table before cleanup. A server override can force refs mode.
func DeleteOrphanFiles(ctx context.Context, tbl *table.Table, opts ...OrphanCleanupOption) (OrphanCleanupResult, error) {
cfg := newOrphanCleanupConfig(opts...)
if cfg.validationErr != nil {
return OrphanCleanupResult{}, cfg.validationErr
}
if err := validateOrphanCleanupGC(tbl); err != nil {
if err := validateOrphanCleanupTable(tbl); err != nil {
return OrphanCleanupResult{}, err
}

Expand All @@ -368,15 +371,13 @@ func DeleteOrphanFiles(ctx context.Context, tbl *table.Table, opts ...OrphanClea

// PlanOrphanFiles identifies orphan files without deleting anything. The
// returned plan can be shown to a user and passed to ExecuteOrphanCleanup to
// delete exactly that set.
// delete exactly that set. Like DeleteOrphanFiles, it requires GC enabled and
// complete snapshot history; REST refs-only loads fail with table.ErrInvalidOperation.
func PlanOrphanFiles(ctx context.Context, tbl *table.Table, opts ...OrphanCleanupOption) (OrphanCleanupPlan, error) {
cfg := newOrphanCleanupConfig(opts...)
if cfg.validationErr != nil {
return OrphanCleanupPlan{}, cfg.validationErr
}
if err := validateOrphanCleanupGC(tbl); err != nil {
return OrphanCleanupPlan{}, err
}

return planOrphanFiles(ctx, tbl, cfg)
}
Expand All @@ -389,6 +390,7 @@ func PlanOrphanFiles(ctx context.Context, tbl *table.Table, opts ...OrphanCleanu
// If this returns ErrStaleOrphanCleanupPlan, tbl is unchanged; refresh or
// reload it before planning again.
// Orphan cleanup is rejected when garbage collection is disabled via gc.enabled.
// When REST loaded only snapshot refs, it returns table.ErrInvalidOperation.
func ExecuteOrphanCleanup(ctx context.Context, tbl *table.Table, plan OrphanCleanupPlan, opts ...OrphanCleanupOption) (OrphanCleanupResult, error) {
cfg := newExecutionOrphanCleanupConfig(opts...)
if cfg.validationErr != nil {
Expand All @@ -397,7 +399,7 @@ func ExecuteOrphanCleanup(ctx context.Context, tbl *table.Table, plan OrphanClea
if cfg.invalidPlanOption != nil {
return OrphanCleanupResult{}, cfg.invalidPlanOption
}
if err := validateOrphanCleanupGC(tbl); err != nil {
if err := validateOrphanCleanupTable(tbl); err != nil {
return OrphanCleanupResult{}, err
}
if cfg.dryRun {
Expand Down Expand Up @@ -443,7 +445,7 @@ func refreshOrphanCleanupTable(ctx context.Context, tbl *table.Table) (*table.Ta
if err := refreshed.Refresh(ctx); err != nil {
return nil, fmt.Errorf("failed to refresh table before orphan cleanup: %w", err)
}
if err := validateOrphanCleanupGC(&refreshed); err != nil {
if err := validateOrphanCleanupTable(&refreshed); err != nil {
return nil, err
}

Expand All @@ -458,6 +460,19 @@ func validateOrphanCleanupGC(tbl *table.Table) error {
return nil
}

func validateOrphanCleanupTable(tbl *table.Table) error {
if err := validateOrphanCleanupGC(tbl); err != nil {
return err
}
// Refs-only responses can omit still-live historical snapshots, making
// their data appear unreferenced.
if tbl.SnapshotHistoryLimitedToRefs() {
return fmt.Errorf("%w: table snapshot history is incomplete under REST snapshot-loading-mode=refs; reload with snapshot-loading-mode=all", table.ErrInvalidOperation)
}

return nil
}

type scannedFile struct {
path string
size int64
Expand All @@ -469,6 +484,10 @@ type referencedFileIndex struct {
}

func planOrphanFiles(ctx context.Context, tbl *table.Table, cfg *orphanCleanupConfig) (OrphanCleanupPlan, error) {
if err := validateOrphanCleanupTable(tbl); err != nil {
return OrphanCleanupPlan{}, err
}

fs, err := tbl.FS(ctx)
if err != nil {
return OrphanCleanupPlan{}, fmt.Errorf("failed to get filesystem: %w", err)
Expand Down Expand Up @@ -1467,6 +1486,11 @@ func pathPrefix(path string) (scheme, authority string, ok bool) {
// does not get out of sync with storage. Non-bulk deletion invokes Remove
// concurrently and is bounded to defaultPurgeMaxConcurrency operations because
// it is typically I/O-bound.
//
// Unlike orphan cleanup, purge is allowed with REST refs-only snapshot history:
// the catalog entry may already be gone, so a full reload is impossible. A
// location walk removes files under the table root when GC is enabled; paths
// outside that root omitted by refs-only metadata may remain for later cleanup.
func PurgeFiles(ctx context.Context, tbl *table.Table) error {
gcEnabled := table.IsGCEnabled(tbl.Properties())

Expand Down
30 changes: 29 additions & 1 deletion table/maintenance/orphan_cleanup_stale_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,33 @@ func TestExecuteOrphanCleanupRejectsStalePlanWhenFileBecomesLive(t *testing.T) {
require.Empty(t, result.DeletedFiles)
}

func TestExecuteOrphanCleanupRejectsRefsOnlyHistoryAfterRefresh(t *testing.T) {
ctx := t.Context()
fs := iceio.NewMemFS()
const location = "mem://stale-plan/refs-only"
const metadataLocation = location + "/metadata/v1.metadata.json"
const orphanPath = location + "/data/orphan.parquet"
meta, err := table.NewMetadata(iceberg.NewSchema(0), iceberg.UnpartitionedSpec,
table.UnsortedSortOrder, location, nil)
require.NoError(t, err)
fsF := testFSF(fs)
catalog := &orphanCleanupTestCatalog{metadata: meta, location: metadataLocation, fsF: fsF}
tbl := table.New(table.Identifier{"db", "refs_only_refresh"}, meta, metadataLocation, fsF, catalog)
require.NoError(t, fs.WriteFile(ctx, orphanPath, []byte("orphan")))
plan, err := PlanOrphanFiles(ctx, tbl, WithCleanupFilesOlderThan(0))
require.NoError(t, err)
require.Equal(t, []string{orphanPath}, plan.Files())

// A load can return refs-only metadata at the same metadata location.
catalog.refsOnly = true
_, err = ExecuteOrphanCleanup(ctx, tbl, plan)
require.ErrorIs(t, err, table.ErrInvalidOperation)
require.ErrorContains(t, err, "snapshot-loading-mode=refs")
file, err := fs.Open(ctx, orphanPath)
require.NoError(t, err)
require.NoError(t, file.Close())
}

func TestDeleteOrphanFilesRefreshesAroundPlanning(t *testing.T) {
for _, afterPlanning := range []bool{false, true} {
name := "peer commits before cleanup"
Expand Down Expand Up @@ -335,6 +362,7 @@ type orphanCleanupTestCatalog struct {
fsF table.FSysF
nextFile int
beforeLoad func() error
refsOnly bool
}

func (c *orphanCleanupTestCatalog) LoadTable(_ context.Context, ident table.Identifier) (*table.Table, error) {
Expand All @@ -346,7 +374,7 @@ func (c *orphanCleanupTestCatalog) LoadTable(_ context.Context, ident table.Iden
}
}

return table.New(ident, c.metadata, c.location, c.fsF, c), nil
return table.New(ident, c.metadata, c.location, c.fsF, c, table.WithSnapshotHistoryLimitedToRefs(c.refsOnly)), nil
}

func (c *orphanCleanupTestCatalog) CommitTable(
Expand Down
Loading
Loading